Merge pull request #242 from tomhea/feature/faster-ptr

fix python warnings, now fully supports python 3.6+

flipjump/assembler/fj_parser.py

@@ -3,7 +3,9 @@
 from typing import Set, List, Tuple, Dict, Union
 
 import sly
+# noinspection PyProtectedMember
 from sly.lex import Token
+# noinspection PyProtectedMember
 from sly.yacc import YaccProduction as ParsedRule
 
 from flipjump.utils.exceptions import FlipJumpExprException, FlipJumpParsingException
@@ -74,7 +76,7 @@ def get_char_value_and_length(s: str) -> Tuple[int, int]:
     return int(s[2:4], 16), 4
 
 
-# noinspection PyUnboundLocalVariable,PyRedeclaration
+# noinspection PyUnboundLocalVariable,PyRedeclaration,PyPep8Naming,PyMethodMayBeStatic
 class FJLexer(sly.Lexer):
     # noinspection PyUnresolvedReferences
     tokens = {NS, DEF, REP,
@@ -183,7 +185,12 @@ def next_address() -> Expr:
     return Expr('$')
 
 
-# noinspection PyUnusedLocal,PyUnresolvedReferences
+def _get_main_macro_code_position(first_file: Tuple[str, Path]) -> CodePosition:
+    (short_file_name, fj_file_path) = first_file
+    return CodePosition(str(fj_file_path.absolute()), short_file_name, 1)
+
+
+# noinspection PyUnusedLocal,PyUnresolvedReferences,PyPep8Naming
 class FJParser(sly.Parser):
     tokens = FJLexer.tokens
     # TODO add Unary Minus (-), Unary Not (~). Maybe add logical or (||) and logical and (&&). Maybe handle power (**).
@@ -201,10 +208,12 @@ class FJParser(sly.Parser):
     )
     # debugfile = 'src/parser.out'
 
-    def __init__(self, memory_width: int, warning_as_errors: bool):
+    def __init__(self, memory_width: int, warning_as_errors: bool, first_file: Tuple[str, Path]):
         self.consts: Dict[str, Expr] = {'w': Expr(memory_width)}
         self.warning_as_errors: bool = warning_as_errors
-        self.macros: Dict[MacroName, Macro] = {initial_macro_name: Macro([], [], [], '', None)}
+        self.macros: Dict[MacroName, Macro] = {
+            initial_macro_name: Macro([], [], [], '', _get_main_macro_code_position(first_file))
+        }
 
     def validate_free_macro_name(self, name: MacroName, lineno: int) -> None:
         if name in self.macros:
@@ -672,7 +681,8 @@ def expr_(self, p: ParsedRule) -> Tuple[Expr, int]:
 
 def exit_if_errors() -> None:
     if error_occurred:
-        raise FlipJumpParsingException(f'Errors found in file {curr_file}. Assembly stopped.\n\nThe Errors:\n{all_errors}')
+        raise FlipJumpParsingException(f'Errors found in file {curr_file}. '
+                                       f'Assembly stopped.\n\nThe Errors:\n{all_errors}')
 
 
 def validate_current_file(files_seen: Set[Union[str, Path]]) -> None:
@@ -718,8 +728,11 @@ def parse_macro_tree(input_files: List[Tuple[str, Path]], memory_width: int, war
 
     files_seen: Set[Union[str, Path]] = set()
 
+    if not input_files:
+        raise FlipJumpParsingException(f"The FlipJump parser got an empty files list.")
+
     lexer = FJLexer()
-    parser = FJParser(memory_width, warning_as_errors)
+    parser = FJParser(memory_width, warning_as_errors, input_files[0])
 
     for curr_file_short_name, curr_file in input_files:
         validate_current_file(files_seen)

flipjump/fjm/fjm_reader.py

@@ -35,16 +35,16 @@ def __init__(self, input_file: Path, *, garbage_handling: GarbageHandling = Garb
         """
         self.garbage_handling = garbage_handling
 
-        with open(input_file, 'rb') as fjm_file:
-            try:
+        try:
+            with open(input_file, 'rb') as fjm_file:
                 self._init_header_fields(fjm_file)
                 self._validate_header()
                 segments = self._init_segments(fjm_file)
                 data = self._read_decompressed_data(fjm_file)
                 self._init_memory(segments, data)
-            except struct.error as se:
-                exception_message = f"Bad file {input_file}, can't unpack. Maybe it's not a .fjm file?"
-                raise FlipJumpReadFjmException(exception_message) from se
+        except struct.error as se:
+            exception_message = f"Bad file {input_file}, can't unpack. Maybe it's not a .fjm file?"
+            raise FlipJumpReadFjmException(exception_message) from se
 
     def _init_header_fields(self, fjm_file: BinaryIO) -> None:
         self.magic, self.memory_width, self.version, self.segment_num = \
@@ -99,7 +99,8 @@ def _init_memory(self, segments: List[Tuple], data: List[int]) -> None:
                 word = ((1 << self.memory_width) - 1)
                 for i in range(0, data_length, 2):
                     self.memory[segment_start + i] = data[data_start + i]
-                    self.memory[segment_start + i+1] = (data[data_start + i+1] + (segment_start + i+1) * self.memory_width) & word
+                    self.memory[segment_start + i+1] =\
+                        (data[data_start + i+1] + (segment_start + i+1) * self.memory_width) & word
             else:
                 for i in range(data_length):
                     self.memory[segment_start + i] = data[data_start + i]
@@ -119,7 +120,8 @@ def _get_memory_word(self, word_address: int) -> int:
                     return 0
 
             garbage_val = _new_garbage_val()
-            garbage_message = f'Reading garbage word at mem[{hex(word_address << self.memory_width)[2:]}] = {hex(garbage_val)[2:]}'
+            garbage_message = (f'Reading garbage word at mem[{hex(word_address << self.memory_width)[2:]}]'
+                               f' = {hex(garbage_val)[2:]}')
 
             if GarbageHandling.Stop == self.garbage_handling:
                 raise FlipJumpRuntimeMemoryException(garbage_message)

flipjump/interpretter/debugging/breakpoints.py

@@ -200,7 +200,8 @@ def update_breakpoints_from_addresses_set(breakpoint_addresses: Optional[Set[int
 def load_labels_dictionary(debugging_file: Optional[Path], labels_file_needed: bool) -> Dict[str, int]:
     """
     load the labels_dictionary from debugging_file, if possible.
-    @param labels_file_needed: if True, prints a warning if debugging-file is None
+    @param debugging_file: The debugging file. If None: return an empty dictionary.
+    @param labels_file_needed: If True, prints a warning if debugging-file is None
     @return: the label-to-address dictionary
     """
     if debugging_file is None:

flipjump/interpretter/fjm_run.py

@@ -127,18 +127,18 @@ def run(fjm_path: Path,
     @param last_ops_debugging_list_length: The length of the last-ops list
     @return: the run's termination-statistics
     """
-    try:
-        with PrintTimer('  loading memory:  ', print_time=print_time):
-            mem = fjm_reader.Reader(fjm_path)
+    with PrintTimer('  loading memory:  ', print_time=print_time):
+        mem = fjm_reader.Reader(fjm_path)
 
-        if io_device is None:
-            io_device = BrokenIO()
+    if io_device is None:
+        io_device = BrokenIO()
 
-        ip = 0
-        w = mem.memory_width
+    ip = 0
+    w = mem.memory_width
 
-        statistics = RunStatistics(w, last_ops_debugging_list_length)
+    statistics = RunStatistics(w, last_ops_debugging_list_length)
 
+    try:
         while True:
             statistics.register_op_address(ip)
 

flipjump/utils/constants.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import lzma
+from pathlib import Path
 from typing import List, Dict
 
 
@@ -16,3 +17,6 @@
 DEBUG_JSON_ENCODING = 'utf-8'
 DEBUG_JSON_LZMA_FORMAT = lzma.FORMAT_RAW
 DEBUG_JSON_LZMA_FILTERS: List[Dict[str, int]] = [{"id": lzma.FILTER_LZMA2}]
+
+PACKAGE_ROOT_PATH = Path(__file__).parent.parent
+STL_PATH = PACKAGE_ROOT_PATH / 'stl'

flipjump/utils/functions.py

@@ -6,18 +6,16 @@
 from pathlib import Path
 from typing import List, Dict, Tuple, Union
 
-from flipjump.utils.constants import DEBUG_JSON_ENCODING, DEBUG_JSON_LZMA_FORMAT, DEBUG_JSON_LZMA_FILTERS
-from importlib.resources import path
+from flipjump.utils.constants import DEBUG_JSON_ENCODING, DEBUG_JSON_LZMA_FORMAT, DEBUG_JSON_LZMA_FILTERS, STL_PATH
 
 
 def get_stl_paths() -> List[Path]:
     """
     @return: list of the ordered standard-library paths
     """
-    with path('flipjump', 'stl') as stl_path:
-        with open(stl_path / 'conf.json', 'r') as stl_json:
-            stl_options = json.load(stl_json)
-        return [stl_path / f'{lib}.fj' for lib in stl_options['all']]
+    with open(STL_PATH / 'conf.json') as stl_conf:
+        stl_options = json.load(stl_conf)
+    return [STL_PATH / f'{lib}.fj' for lib in stl_options['all']]
 
 
 def save_debugging_labels(debugging_file_path: Path, labels: Dict[str, int]) -> None:

tests/conftest.py

@@ -80,19 +80,19 @@ def argument_line_iterator(csv_file_path: Path, num_of_args: int) -> Iterable[Li
                 yield list(map(str.strip, line))
 
 
-def get_compile_tests_params_from_csv(csv_file_path: Path, xfail_list: List[str], save_debug_info: bool) -> List:
+def get_compile_tests_params_from_csv(csv_file_path: Path, xfail_list: List[str], save_debug_file: bool) -> List:
     """
     read the compile-tests from the csv
     @param csv_file_path: read tests from this csv
     @param xfail_list: list of tests names to mark with xfail (expected to fail)
-    @param save_debug_info: should save the debugging info file
+    @param save_debug_file: should save the debugging info file
 
     @return: the list of pytest.params(CompileTestArgs, marks=...)
     """
     params = []
 
     for line in argument_line_iterator(csv_file_path, CompileTestArgs.num_of_csv_line_args):
-        args = CompileTestArgs(save_debug_info, *line)
+        args = CompileTestArgs(save_debug_file, *line)
         test_marks = [pytest.mark.run(order=COMPILE_ORDER_INDEX)]
         if args.test_name in xfail_list:
             test_marks.append(pytest.mark.xfail())

tests/inout/prime_sieve_tests/primes50.in

@@ -0,0 +1 @@
+50

tests/inout/prime_sieve_tests/primes50.out

@@ -0,0 +1,16 @@
+Search primes up to: 2
+3
+5
+7
+11
+13
+17
+19
+23
+29
+31
+37
+41
+43
+47
+Number of prime numbers: 15

tests/test_fj.py

@@ -1,5 +1,3 @@
-from queue import Queue
-from threading import Lock
 from pathlib import Path
 
 from flipjump import run_test_output
@@ -28,7 +26,7 @@ class CompileTestArgs:
 
     num_of_csv_line_args = 8
 
-    def __init__(self, save_debug_info: bool, test_name: str, fj_paths: str, fjm_out_path: str,
+    def __init__(self, save_debug_file: bool, test_name: str, fj_paths: str, fjm_out_path: str,
                  word_size__str: str, version__str: str, flags__str: str,
                  use_stl__str: str, warning_as_errors__str: str):
         """
@@ -39,8 +37,6 @@ def __init__(self, save_debug_info: bool, test_name: str, fj_paths: str, fjm_out
         self.use_stl = use_stl__str == CSV_TRUE
         self.warning_as_errors = warning_as_errors__str == CSV_TRUE
 
-        self.save_debug_info = save_debug_info
-
         self.test_name = test_name
 
         included_files = get_stl_paths() if self.use_stl else []
@@ -52,6 +48,9 @@ def __init__(self, save_debug_info: bool, test_name: str, fj_paths: str, fjm_out
         self.fj_files_tuples = included_files_tuples + fj_paths_tuples
 
         self.fjm_out_path = ROOT_PATH / fjm_out_path
+        self.debugging_file_path = None
+        if save_debug_file:
+            self.debugging_file_path = Path(f'{self.fjm_out_path.absolute()}{DEBUGGING_FILE_SUFFIX}')
 
         self.word_size = int(word_size__str)
         self.version = int(version__str)
@@ -81,13 +80,9 @@ def test_compile(compile_args: CompileTestArgs) -> None:
     fjm_writer = Writer(compile_args.fjm_out_path, compile_args.word_size, FJMVersion(compile_args.version),
                         flags=compile_args.flags)
 
-    debugging_file_path = None
-    if compile_args.save_debug_info:
-        debugging_file_path = Path(f'{compile_args.fjm_out_path}{DEBUGGING_FILE_SUFFIX}')
-
     assembler.assemble(compile_args.fj_files_tuples, compile_args.word_size, fjm_writer,
                        warning_as_errors=compile_args.warning_as_errors,
-                       debugging_file_path=debugging_file_path)
+                       debugging_file_path=compile_args.debugging_file_path)
 
 
 class RunTestArgs:
@@ -109,11 +104,13 @@ def __init__(self, save_debug_file: bool, debug_info_length: int, test_name: str
         self.read_in_as_binary = read_in_as_binary__str == CSV_TRUE
         self.read_out_as_binary = read_out_as_binary__str == CSV_TRUE
 
-        self.save_debug_file = save_debug_file
         self.debug_info_length = debug_info_length
 
         self.test_name = test_name
         self.fjm_path = ROOT_PATH / fjm_path
+        self.debugging_file_path = None
+        if save_debug_file:
+            self.debugging_file_path = Path(f'{self.fjm_path.absolute()}{DEBUGGING_FILE_SUFFIX}')
 
         if in_file_path:
             self.in_file_path = ROOT_PATH / in_file_path
@@ -171,7 +168,7 @@ def test_run(run_args: RunTestArgs) -> None:
         run_args.get_defined_input(),
         run_args.get_expected_output(),
         should_raise_assertion_error=True,
-        debugging_file=Path(f'{run_args.fjm_path}{DEBUGGING_FILE_SUFFIX}'),
+        debugging_file=run_args.debugging_file_path,
         print_time=True,
         last_ops_debugging_list_length=run_args.debug_info_length,
     )

tests/test_run_slow.csv

@@ -30,6 +30,7 @@ prime_sieve_121, tests/compiled/prime_sieve.fjm, tests/inout/prime_sieve_tests/p
 prime_sieve_108, tests/compiled/prime_sieve.fjm, tests/inout/prime_sieve_tests/primes108.in,tests/inout/prime_sieve_tests/primes108.out, False,False
 prime_sieve_107, tests/compiled/prime_sieve.fjm, tests/inout/prime_sieve_tests/primes107.in,tests/inout/prime_sieve_tests/primes107.out, False,False
 prime_sieve_100, tests/compiled/prime_sieve.fjm, tests/inout/prime_sieve_tests/primes100.in,tests/inout/prime_sieve_tests/primes100.out, False,False
+prime_sieve_50, tests/compiled/prime_sieve.fjm, tests/inout/prime_sieve_tests/primes50.in,tests/inout/prime_sieve_tests/primes50.out, False,False
 prime_sieve_10, tests/compiled/prime_sieve.fjm, tests/inout/prime_sieve_tests/primes10.in,tests/inout/prime_sieve_tests/primes10.out, False,False
 prime_sieve_7, tests/compiled/prime_sieve.fjm, tests/inout/prime_sieve_tests/primes7.in,tests/inout/prime_sieve_tests/primes7.out, False,False
 prime_sieve_6, tests/compiled/prime_sieve.fjm, tests/inout/prime_sieve_tests/primes6.in,tests/inout/prime_sieve_tests/primes6.out, False,False