Correct position reporting and test that it works.

- Changes via rspivak/slimit@8f9a39c776 (which got pulled in via import
  of rspivak/slimit#65 did not have tests, so it naturally never worked.

src/calmjs/parse/lexers/es5.py

@@ -147,7 +147,11 @@ def token(self):
             if char != '/' or (char == '/' and next_char in ('/', '*')):
                 tok = self._get_update_token()
                 if tok.type in DIVISION_SYNTAX_MARKERS:
-                    lexer.lineno += len(tok.value.splitlines())
+                    if tok.type == 'LINE_TERMINATOR':
+                        lexer.lineno += len(tok.value.splitlines())
+                    elif tok.type == 'BLOCK_COMMENT':
+                        # only grab the middle bits.
+                        lexer.lineno += len(tok.value.splitlines()) - 1
                     continue
                 else:
                     return tok

src/calmjs/parse/tests/test_es5_lexer.py

@@ -26,6 +26,7 @@
 __author__ = 'Ruslan Spivak <[email protected]>'
 
 import unittest
+import textwrap
 
 from calmjs.parse.lexers.es5 import Lexer
 from calmjs.parse.exceptions import ECMASyntaxError
@@ -407,3 +408,64 @@ def run_lex(value):
         '())))',
     )], ECMASyntaxError,
 )
+
+
+def run_lex_pos(value):
+    lexer = Lexer()
+    lexer.input(textwrap.dedent(value).strip())
+    return [
+        '%s %d:%d' % (token.value, token.lineno, token.lexpos)
+        for token in lexer
+    ]
+
+
+LexerPosTestCase = build_equality_testcase(
+    'LexerPosTestCase', run_lex_pos, [(
+        'single_line',
+        """
+        var foo = bar;  // line 1
+        """, [
+            'var 1:0', 'foo 1:4', '= 1:8', 'bar 1:10', '; 1:13'
+        ]
+    ), (
+        'multi_line',
+        """
+        var foo = bar;  // line 1
+
+
+        var bar = baz;  // line 4
+        """, [
+            'var 1:0', 'foo 1:4', '= 1:8', 'bar 1:10', '; 1:13',
+            'var 4:28', 'bar 4:32', '= 4:36', 'baz 4:38', '; 4:41',
+        ]
+    ), (
+        'inline_comment',
+        """
+        // this is a comment  // line 1
+        var foo = bar;  // line 2
+
+        // another one  // line 4
+        var bar = baz;  // line 5
+        """, [
+            'var 2:32', 'foo 2:36', '= 2:40', 'bar 2:42', '; 2:45',
+            'var 5:85', 'bar 5:89', '= 5:93', 'baz 5:95', '; 5:98',
+        ]
+    ), (
+        'block_comment',
+        """
+        /*
+        This is a block comment
+        */
+        var foo = bar;  // line 4
+
+        /* block single line */ // line 6
+        var bar = baz;  // line 7
+
+        /* oops */bar();  // line 9
+        """, [
+            'var 4:30', 'foo 4:34', '= 4:38', 'bar 4:40', '; 4:43',
+            'var 7:91', 'bar 7:95', '= 7:99', 'baz 7:101', '; 7:104',
+            'bar 9:128', '( 9:131', ') 9:132', '; 9:133',
+        ]
+    )]
+)