fix(lexer): fix bug with ignored tokens not being ignored

In inlineable states. Adds some (more!) technical debt to `build.ts` (see `!` non-null assertions); could really use a refactor at this point.
wincent · Aug 28, 2023 · a1edfb3 · a1edfb3
1 parent 405db0e
commit a1edfb3
Show file tree

Hide file tree

Showing 2 changed files with 36 additions and 24 deletions.
diff --git a/packages/lexer/src/__tests__/union-test.ts b/packages/lexer/src/__tests__/union-test.ts
@@ -200,13 +200,9 @@ describe('union()', () => {
 
     const input = '= == ===';
 
-    // BUG: why isn't IGNORED stuff being ignored?
-
     expect([...getLexer(table)(input)]).toEqual([
       new Token('ASSIGN', 0, 1, input),
-      new Token('IGNORED', 1, 2, input),
       new Token('EQUALS', 2, 4, input),
-      new Token('IGNORED', 4, 5, input),
       new Token('STRICT_EQUALS', 5, 8, input),
     ]);
 
@@ -256,7 +252,10 @@ describe('union()', () => {
             let ch = this.index < length ? input.charCodeAt(this.index) : -1;
             if (state === START) {
               if (ch === 0x20) {
-                return this.emit('IGNORED', this.index + 1, input);
+                // IGNORED token.
+                this.index = this.index + 1;
+                this.tokenStart = this.index;
+                continue;
               } else if (ch === 0x3d) {
                 this.state = 2;
               } else {

diff --git a/packages/lexer/src/build.ts b/packages/lexer/src/build.ts
@@ -126,15 +126,16 @@ export default function build(
       ),
       block: [],
     };
-    const isAccept = table.acceptStates.has(i)
-      ? Array.from(table.labels?.[i] ?? [])[0]
-      : undefined;
+    const isAccept = (state: number) =>
+      table.acceptStates.has(state)
+        ? Array.from(table.labels?.[state] ?? [])[0]
+        : undefined;
 
-    const isIgnored = isAccept === 'IGNORED';
+    const isIgnored = (state: number) => isAccept(state) === 'IGNORED';
 
     if (!conditions.length) {
       // Should only get here for an ignored state.
-      invariant(isIgnored);
+      invariant(isIgnored(i));
       consequent.block.push(
         ...filterEmpty(
           ast.comment('IGNORED token.'),
@@ -160,7 +161,7 @@ export default function build(
         consequent.block.push(loop);
       }
 
-      const ignoreToken = isIgnored
+      const ignoreToken = isIgnored(i)
         ? filterEmpty(
             ast.comment('IGNORED token.'),
             ast.statement('this.tokenStart = this.index'),
@@ -169,13 +170,13 @@ export default function build(
             ast.continue(),
           )
         : undefined;
-      const acceptToken = isAccept
+      const acceptToken = isAccept(i)
         ? filterEmpty(
             i === START ? ast.empty : ast.statement('this.state = START'),
             ast.return(
               ast.call(
                 'this.emit',
-                ast.string(isAccept),
+                ast.string(isAccept(i)!),
                 'this.index',
                 'input',
               ),
@@ -199,19 +200,31 @@ export default function build(
           const condition = expressionForTransitions(transitions);
           const block: Array<Statement> = [];
           if (inlineableStates.has(j)) {
-            block.push(
-              ...filterEmpty(
-                i === START ? ast.empty : ast.statement('this.state = START'),
-                ast.return(
-                  ast.call(
-                    'this.emit',
-                    ast.string(Array.from(table.labels?.[j] ?? [])[0]),
-                    'this.index + 1',
-                    'input',
+            if (isIgnored(j)) {
+              block.push(
+                ...filterEmpty(
+                  ast.comment('IGNORED token.'),
+                  ast.statement('this.index = this.index + 1'),
+                  ast.statement('this.tokenStart = this.index'),
+                  i === START ? ast.empty : ast.statement('this.state = START'),
+                  ast.continue(),
+                ),
+              );
+            } else {
+              block.push(
+                ...filterEmpty(
+                  i === START ? ast.empty : ast.statement('this.state = START'),
+                  ast.return(
+                    ast.call(
+                      'this.emit',
+                      ast.string(isAccept(j)!),
+                      'this.index + 1',
+                      'input',
+                    ),
                   ),
                 ),
-              ),
-            );
+              );
+            }
             stats['inlinedAcceptStates']++;
           } else {
             invariant(