From 83b0e2094de5a2c7e825ca1c662c1b04247b6974 Mon Sep 17 00:00:00 2001 From: Ali Clark Date: Thu, 20 Aug 2020 16:21:37 +0100 Subject: [PATCH 01/12] Display a context print out and mark out error tokens with ASCII --- moo.js | 45 +++++++++++++++++++++++++++++++++++++-------- test/test.js | 23 +++++++++++++++-------- 2 files changed, 52 insertions(+), 16 deletions(-) diff --git a/moo.js b/moo.js index cf3a27e..c06805d 100644 --- a/moo.js +++ b/moo.js @@ -15,6 +15,10 @@ /***************************************************************************/ + function pad(text, width) { + return Array(width - text.length + 1).join(" ") + text + } + function isRegExp(o) { return o && toString.call(o) === '[object RegExp]' } function isObject(o) { return o && typeof o === 'object' && !isRegExp(o) && !Array.isArray(o) } @@ -562,21 +566,46 @@ Lexer.prototype.formatError = function(token, message) { if (token == null) { // An undefined token indicates EOF - var text = this.buffer.slice(this.index) var token = { - text: text, + text: "", offset: this.index, - lineBreaks: text.indexOf('\n') === -1 ? 0 : 1, + lineBreaks: 0, line: this.line, col: this.col, } } - var start = Math.max(0, token.offset - token.col + 1) - var eol = token.lineBreaks ? token.text.indexOf('\n') : token.text.length - var firstLine = this.buffer.substring(start, token.offset + eol) + + var lines = this.buffer + .split("\n") + .slice(Math.max(0, token.line + token.lineBreaks - 5), token.line + token.lineBreaks) + message += " at line " + token.line + " col " + token.col + ":\n\n" - message += " " + firstLine + "\n" - message += " " + Array(token.col).join(" ") + "^" + + message += lines + .map(line => line.replace(/\t/g, " ")) + .map(function(curLine, i) { + return pad(String(token.line + token.lineBreaks - (lines.length - i) + 1), 6) + "\t" + curLine + "\n" + }, this) + .join("") + + var lastLine = lines[lines.length - 1] + var lastLineExpanded = lastLine.replace(/\t/g, " ") + + var tokenValueLines = token.text.split("\n") + var tokenValueLastLine = tokenValueLines[tokenValueLines.length - 1] + var tokenValueLastLineExpanded = tokenValueLastLine.replace(/\t/g, " ") + + var tokenLastLinePrepend = tokenValueLines.length === 1 ? lastLine.slice(0, token.col - 1) : "" + var tokenLastLinePrependExpanded = tokenLastLinePrepend.replace(/\t/g, " ") + + var highlightIndentation = lastLineExpanded.replace(/[^ ]/g, " ") + var highlightLength = !tokenValueLastLine || tokenLastLinePrependExpanded.length >= lastLineExpanded.length ? + 0 : + tokenValueLastLineExpanded.length + var highlight = highlightLength ? Array(highlightLength + 1).join("~") : "^" + + message += " \t" + highlightIndentation.slice(0, tokenLastLinePrependExpanded.length) + highlight + "\n" + return message } diff --git a/test/test.js b/test/test.js index 2e20d18..3acefdd 100644 --- a/test/test.js +++ b/test/test.js @@ -905,8 +905,9 @@ describe('errors', () => { expect(lexer.next()).toMatchObject({value: '456'}) expect(() => lexer.next()).toThrow( "invalid syntax at line 2 col 4:\n\n" + - " 456baa\n" + - " ^" + " 1\t123\n" + + " 2\t456baa\n" + + " \t ~~~\n" ) }) @@ -921,8 +922,12 @@ describe('errors', () => { expect(tok).toMatchObject({type: 'error', value: ' 12\n345\n6', lineBreaks: 2}) expect(lexer.formatError(tok, "numbers!")).toBe( "numbers! at line 3 col 2:\n\n" + - " g 12\n" + - " ^" + " 1\tabc\n" + + " 2\tdef\n" + + " 3\tg 12\n" + + " 4\t345\n" + + " 5\t6\n" + + " \t~\n" ) }) @@ -937,8 +942,9 @@ describe('errors', () => { expect(lexer.col).toBe(9) expect(lexer.formatError(undefined, "EOF!")).toBe( "EOF! at line 2 col 9:\n\n" + - " def quxx\n" + - " ^" + " 1\tabc\n" + + " 2\tdef quxx\n" + + " \t ^\n" ) }) @@ -954,8 +960,9 @@ describe('errors', () => { expect(lexer.col).toBe(1) expect(lexer.formatError(undefined, "oh no!")).toBe( "oh no! at line 2 col 1:\n\n" + - " def quxx\n" + - " ^" + " 1\tabc\n" + + " 2\tdef quxx\n" + + " \t^\n" ) }) From a44c30d420b837b1f63027678d7fbf984ba21a57 Mon Sep 17 00:00:00 2001 From: Ali Clark Date: Thu, 20 Aug 2020 18:02:11 +0100 Subject: [PATCH 02/12] Only take into account inner-lines of a multi-line token --- moo.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/moo.js b/moo.js index c06805d..5e45b8a 100644 --- a/moo.js +++ b/moo.js @@ -575,23 +575,25 @@ } } + const innerLineBreaks = token.lineBreaks - Number(token.text.substr(-1) === "\n") + var lines = this.buffer .split("\n") - .slice(Math.max(0, token.line + token.lineBreaks - 5), token.line + token.lineBreaks) + .slice(Math.max(0, token.line + innerLineBreaks - 5), token.line + innerLineBreaks) message += " at line " + token.line + " col " + token.col + ":\n\n" message += lines .map(line => line.replace(/\t/g, " ")) .map(function(curLine, i) { - return pad(String(token.line + token.lineBreaks - (lines.length - i) + 1), 6) + "\t" + curLine + "\n" + return pad(String(token.line + innerLineBreaks - (lines.length - i) + 1), 6) + "\t" + curLine + "\n" }, this) .join("") var lastLine = lines[lines.length - 1] var lastLineExpanded = lastLine.replace(/\t/g, " ") - var tokenValueLines = token.text.split("\n") + var tokenValueLines = token.text.split("\n").slice(0, innerLineBreaks + 1) var tokenValueLastLine = tokenValueLines[tokenValueLines.length - 1] var tokenValueLastLineExpanded = tokenValueLastLine.replace(/\t/g, " ") From 015365bf29440afb1840f6bc3968d63538979dd5 Mon Sep 17 00:00:00 2001 From: Ali Clark Date: Thu, 20 Aug 2020 18:02:47 +0100 Subject: [PATCH 03/12] Add EOF and EOL markers to understand why a line is terminated --- moo.js | 7 ++++++- test/test.js | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/moo.js b/moo.js index 5e45b8a..aad501a 100644 --- a/moo.js +++ b/moo.js @@ -604,7 +604,12 @@ var highlightLength = !tokenValueLastLine || tokenLastLinePrependExpanded.length >= lastLineExpanded.length ? 0 : tokenValueLastLineExpanded.length - var highlight = highlightLength ? Array(highlightLength + 1).join("~") : "^" + + var highlight = highlightLength ? + Array(highlightLength + 1).join("~") : + (token.offset >= this.buffer.length ? + "^EOF" : + (tokenLastLinePrependExpanded.length >= lastLineExpanded.length ? "^EOL" : "^")) message += " \t" + highlightIndentation.slice(0, tokenLastLinePrependExpanded.length) + highlight + "\n" diff --git a/test/test.js b/test/test.js index 3acefdd..e0a1ac0 100644 --- a/test/test.js +++ b/test/test.js @@ -944,7 +944,7 @@ describe('errors', () => { "EOF! at line 2 col 9:\n\n" + " 1\tabc\n" + " 2\tdef quxx\n" + - " \t ^\n" + " \t ^EOF\n" ) }) From bbd8661d70601e824b9fd26540e54b128df61ddd Mon Sep 17 00:00:00 2001 From: Ali Clark Date: Thu, 20 Aug 2020 18:04:28 +0100 Subject: [PATCH 04/12] Rename variable Value -> Text to more accurately match its meaning --- moo.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/moo.js b/moo.js index aad501a..66ce931 100644 --- a/moo.js +++ b/moo.js @@ -593,17 +593,17 @@ var lastLine = lines[lines.length - 1] var lastLineExpanded = lastLine.replace(/\t/g, " ") - var tokenValueLines = token.text.split("\n").slice(0, innerLineBreaks + 1) - var tokenValueLastLine = tokenValueLines[tokenValueLines.length - 1] - var tokenValueLastLineExpanded = tokenValueLastLine.replace(/\t/g, " ") + var tokenTextLines = token.text.split("\n").slice(0, innerLineBreaks + 1) + var tokenTextLastLine = tokenTextLines[tokenTextLines.length - 1] + var tokenTextLastLineExpanded = tokenTextLastLine.replace(/\t/g, " ") - var tokenLastLinePrepend = tokenValueLines.length === 1 ? lastLine.slice(0, token.col - 1) : "" + var tokenLastLinePrepend = tokenTextLines.length === 1 ? lastLine.slice(0, token.col - 1) : "" var tokenLastLinePrependExpanded = tokenLastLinePrepend.replace(/\t/g, " ") var highlightIndentation = lastLineExpanded.replace(/[^ ]/g, " ") - var highlightLength = !tokenValueLastLine || tokenLastLinePrependExpanded.length >= lastLineExpanded.length ? + var highlightLength = !tokenTextLastLine || tokenLastLinePrependExpanded.length >= lastLineExpanded.length ? 0 : - tokenValueLastLineExpanded.length + tokenTextLastLineExpanded.length var highlight = highlightLength ? Array(highlightLength + 1).join("~") : From 8d96354a3da48c5e3bb5707b9df7fb6b19b5f27c Mon Sep 17 00:00:00 2001 From: Ali Clark Date: Thu, 20 Aug 2020 18:07:49 +0100 Subject: [PATCH 05/12] Only mark out EOF not EOL, as it can be confusing with NL tokens --- moo.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/moo.js b/moo.js index 66ce931..83f6ad9 100644 --- a/moo.js +++ b/moo.js @@ -607,9 +607,7 @@ var highlight = highlightLength ? Array(highlightLength + 1).join("~") : - (token.offset >= this.buffer.length ? - "^EOF" : - (tokenLastLinePrependExpanded.length >= lastLineExpanded.length ? "^EOL" : "^")) + (token.offset >= this.buffer.length ? "^EOF" : "^") message += " \t" + highlightIndentation.slice(0, tokenLastLinePrependExpanded.length) + highlight + "\n" From 7018fa7bcd10590fa08104a021b8f833d5b03efe Mon Sep 17 00:00:00 2001 From: Ali Clark Date: Thu, 20 Aug 2020 18:33:01 +0100 Subject: [PATCH 06/12] Not using arrow functions --- moo.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moo.js b/moo.js index 83f6ad9..c1439d4 100644 --- a/moo.js +++ b/moo.js @@ -584,7 +584,7 @@ message += " at line " + token.line + " col " + token.col + ":\n\n" message += lines - .map(line => line.replace(/\t/g, " ")) + .map(function (line) { return line.replace(/\t/g, " ") }) .map(function(curLine, i) { return pad(String(token.line + innerLineBreaks - (lines.length - i) + 1), 6) + "\t" + curLine + "\n" }, this) From 4e733def225ab4b66123260d6e2e943720c14f89 Mon Sep 17 00:00:00 2001 From: Ali Clark Date: Thu, 20 Aug 2020 18:33:23 +0100 Subject: [PATCH 07/12] Remove unnecessary 'this' arg --- moo.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moo.js b/moo.js index c1439d4..719da32 100644 --- a/moo.js +++ b/moo.js @@ -587,7 +587,7 @@ .map(function (line) { return line.replace(/\t/g, " ") }) .map(function(curLine, i) { return pad(String(token.line + innerLineBreaks - (lines.length - i) + 1), 6) + "\t" + curLine + "\n" - }, this) + }) .join("") var lastLine = lines[lines.length - 1] From af77f000633f531ffd72897fafe9171caaa9d672 Mon Sep 17 00:00:00 2001 From: Ali Clark Date: Thu, 20 Aug 2020 18:34:02 +0100 Subject: [PATCH 08/12] If the token is multi-line, then highlight all of the above --- moo.js | 10 ++++++---- test/test.js | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/moo.js b/moo.js index 719da32..e33d992 100644 --- a/moo.js +++ b/moo.js @@ -601,12 +601,14 @@ var tokenLastLinePrependExpanded = tokenLastLinePrepend.replace(/\t/g, " ") var highlightIndentation = lastLineExpanded.replace(/[^ ]/g, " ") - var highlightLength = !tokenTextLastLine || tokenLastLinePrependExpanded.length >= lastLineExpanded.length ? - 0 : - tokenTextLastLineExpanded.length + var highlightLength = innerLineBreaks ? + Math.max.apply(null, lines.map(function (line) { return line.length })) : + (!tokenTextLastLine || tokenLastLinePrependExpanded.length >= lastLineExpanded.length ? + 0 : + tokenTextLastLineExpanded.length) var highlight = highlightLength ? - Array(highlightLength + 1).join("~") : + Array(highlightLength + 1).join(innerLineBreaks ? "^" : "~") : (token.offset >= this.buffer.length ? "^EOF" : "^") message += " \t" + highlightIndentation.slice(0, tokenLastLinePrependExpanded.length) + highlight + "\n" diff --git a/test/test.js b/test/test.js index e0a1ac0..ce6f239 100644 --- a/test/test.js +++ b/test/test.js @@ -927,7 +927,7 @@ describe('errors', () => { " 3\tg 12\n" + " 4\t345\n" + " 5\t6\n" + - " \t~\n" + " \t^^^^\n" ) }) From e52309558248a9d727b1c1fa82131946ee17af2f Mon Sep 17 00:00:00 2001 From: Ali Clark Date: Thu, 20 Aug 2020 20:44:50 +0100 Subject: [PATCH 09/12] Point to the start of errors in case of multi-line --- moo.js | 23 ++++++++++------------- test/test.js | 4 +--- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/moo.js b/moo.js index e33d992..a38a674 100644 --- a/moo.js +++ b/moo.js @@ -579,39 +579,36 @@ var lines = this.buffer .split("\n") - .slice(Math.max(0, token.line + innerLineBreaks - 5), token.line + innerLineBreaks) + .slice(Math.max(0, token.line - 5), token.line) message += " at line " + token.line + " col " + token.col + ":\n\n" message += lines .map(function (line) { return line.replace(/\t/g, " ") }) .map(function(curLine, i) { - return pad(String(token.line + innerLineBreaks - (lines.length - i) + 1), 6) + "\t" + curLine + "\n" + return pad(String(token.line - (lines.length - i) + 1), 6) + "\t" + curLine + "\n" }) .join("") var lastLine = lines[lines.length - 1] var lastLineExpanded = lastLine.replace(/\t/g, " ") - var tokenTextLines = token.text.split("\n").slice(0, innerLineBreaks + 1) - var tokenTextLastLine = tokenTextLines[tokenTextLines.length - 1] - var tokenTextLastLineExpanded = tokenTextLastLine.replace(/\t/g, " ") + var lastLinePrepend = lastLine.slice(0, token.col - 1) + var lastLinePrependExpanded = lastLinePrepend.replace(/\t/g, " ") - var tokenLastLinePrepend = tokenTextLines.length === 1 ? lastLine.slice(0, token.col - 1) : "" - var tokenLastLinePrependExpanded = tokenLastLinePrepend.replace(/\t/g, " ") + var tokenTextFirstLine = token.text.split("\n")[0] + var tokenTextFirstLineExpanded = tokenTextFirstLine.replace(/\t/g, " ") var highlightIndentation = lastLineExpanded.replace(/[^ ]/g, " ") - var highlightLength = innerLineBreaks ? - Math.max.apply(null, lines.map(function (line) { return line.length })) : - (!tokenTextLastLine || tokenLastLinePrependExpanded.length >= lastLineExpanded.length ? - 0 : - tokenTextLastLineExpanded.length) + var highlightLength = !tokenTextFirstLine || lastLinePrependExpanded.length >= lastLineExpanded.length ? + 0 : + tokenTextFirstLineExpanded.length var highlight = highlightLength ? Array(highlightLength + 1).join(innerLineBreaks ? "^" : "~") : (token.offset >= this.buffer.length ? "^EOF" : "^") - message += " \t" + highlightIndentation.slice(0, tokenLastLinePrependExpanded.length) + highlight + "\n" + message += " \t" + highlightIndentation.slice(0, lastLinePrependExpanded.length) + highlight + "\n" return message } diff --git a/test/test.js b/test/test.js index ce6f239..dabafad 100644 --- a/test/test.js +++ b/test/test.js @@ -925,9 +925,7 @@ describe('errors', () => { " 1\tabc\n" + " 2\tdef\n" + " 3\tg 12\n" + - " 4\t345\n" + - " 5\t6\n" + - " \t^^^^\n" + " \t ^^^\n" ) }) From a66662507e4f1d59599359e890dc1bc8bfd264b2 Mon Sep 17 00:00:00 2001 From: Ali Clark Date: Fri, 21 Aug 2020 06:32:25 +0100 Subject: [PATCH 10/12] Mark out tabs graphically --- moo.js | 11 ++++++----- test/test.js | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/moo.js b/moo.js index a38a674..41feaf0 100644 --- a/moo.js +++ b/moo.js @@ -591,16 +591,14 @@ .join("") var lastLine = lines[lines.length - 1] - var lastLineExpanded = lastLine.replace(/\t/g, " ") - var lastLinePrepend = lastLine.slice(0, token.col - 1) var lastLinePrependExpanded = lastLinePrepend.replace(/\t/g, " ") var tokenTextFirstLine = token.text.split("\n")[0] var tokenTextFirstLineExpanded = tokenTextFirstLine.replace(/\t/g, " ") - var highlightIndentation = lastLineExpanded.replace(/[^ ]/g, " ") - var highlightLength = !tokenTextFirstLine || lastLinePrependExpanded.length >= lastLineExpanded.length ? + var highlightIndentation = lastLine.replace(/[^ \t]/g, " ").replace(/\t/g, ".___") + var highlightLength = !tokenTextFirstLine || lastLinePrependExpanded.length >= highlightIndentation.length ? 0 : tokenTextFirstLineExpanded.length @@ -608,7 +606,10 @@ Array(highlightLength + 1).join(innerLineBreaks ? "^" : "~") : (token.offset >= this.buffer.length ? "^EOF" : "^") - message += " \t" + highlightIndentation.slice(0, lastLinePrependExpanded.length) + highlight + "\n" + message += " \t" + + highlightIndentation.slice(0, lastLinePrependExpanded.length) + + highlight + + highlightIndentation.slice(lastLinePrependExpanded.length + highlight.length) + "\n" return message } diff --git a/test/test.js b/test/test.js index dabafad..c9d4d0e 100644 --- a/test/test.js +++ b/test/test.js @@ -960,7 +960,7 @@ describe('errors', () => { "oh no! at line 2 col 1:\n\n" + " 1\tabc\n" + " 2\tdef quxx\n" + - " \t^\n" + " \t^ \n" ) }) From 78c655a06402ddce28c586563ffc14803d3dfa80 Mon Sep 17 00:00:00 2001 From: Ali Clark Date: Fri, 21 Aug 2020 08:12:09 +0100 Subject: [PATCH 11/12] Remove unneeded innerLineBreaks concept --- moo.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/moo.js b/moo.js index 41feaf0..c2e6546 100644 --- a/moo.js +++ b/moo.js @@ -575,8 +575,6 @@ } } - const innerLineBreaks = token.lineBreaks - Number(token.text.substr(-1) === "\n") - var lines = this.buffer .split("\n") .slice(Math.max(0, token.line - 5), token.line) @@ -603,7 +601,7 @@ tokenTextFirstLineExpanded.length var highlight = highlightLength ? - Array(highlightLength + 1).join(innerLineBreaks ? "^" : "~") : + Array(highlightLength + 1).join(token.lineBreaks ? "^" : "~") : (token.offset >= this.buffer.length ? "^EOF" : "^") message += " \t" + From 5c7d2192ae251a1963121f2a1fd98aac12aadebd Mon Sep 17 00:00:00 2001 From: Ali Clark Date: Sun, 23 Aug 2020 15:38:54 +0100 Subject: [PATCH 12/12] Use \tab instead of .___ to indicate tab character --- moo.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moo.js b/moo.js index c2e6546..79e9b12 100644 --- a/moo.js +++ b/moo.js @@ -595,7 +595,7 @@ var tokenTextFirstLine = token.text.split("\n")[0] var tokenTextFirstLineExpanded = tokenTextFirstLine.replace(/\t/g, " ") - var highlightIndentation = lastLine.replace(/[^ \t]/g, " ").replace(/\t/g, ".___") + var highlightIndentation = lastLine.replace(/[^ \t]/g, " ").replace(/\t/g, "\\tab") var highlightLength = !tokenTextFirstLine || lastLinePrependExpanded.length >= highlightIndentation.length ? 0 : tokenTextFirstLineExpanded.length