Merge pull request #267 from stevehalliwell/219_scanner

Merge 219_scanner into main: Improve scanner speed
stevehalliwell · Sep 19, 2024 · ca6baec · ca6baec
2 parents ed13f7c + e936fee
commit ca6baec
Show file tree

Hide file tree

Showing 69 changed files with 782 additions and 712 deletions.
diff --git a/ulox/ulox.core.bench/BenchmarkScripts.cs b/ulox/ulox.core.bench/BenchmarkScripts.cs
@@ -2,7 +2,7 @@
 {
     public static class BenchmarkScripts
     {
-        public static readonly Script Loop = new Script(nameof(Loop), @"
+        public static readonly Script Loop = new(nameof(Loop), @"
 var arr = [];
 arr.Resize(100,0);
 
@@ -11,7 +11,7 @@ loop arr
     item = i;
 }");
 
-        public static readonly Script If = new Script(nameof(If), @"
+        public static readonly Script If = new(nameof(If), @"
 var i = 0;
 
 if(i == 1)
@@ -31,7 +31,7 @@ loop arr
 
     public static class Vec2Variants
     {
-        public static readonly Script Type = new Script(nameof(Vec2Variants), @"
+        public static readonly Script Type = new(nameof(Vec2Variants), @"
 class Vec2
 {
 	static Create(x,y)
@@ -261,7 +261,7 @@ fun Bench()
 	var n = Vec2.Normalise(d);
 }
 ");
-        public static readonly Script Tuple = new Script(nameof(Vec2Variants), @"
+        public static readonly Script Tuple = new(nameof(Vec2Variants), @"
 
 class Vec2
 {

diff --git a/ulox/ulox.core.bench/CompileVsExecute.cs b/ulox/ulox.core.bench/CompileVsExecute.cs
@@ -2,14 +2,14 @@
 {
     public static class CompileVsExecute
     {
-        public static readonly Script Script = new Script(nameof(Script), @"
+        public static readonly Script Script = new(nameof(Script), @"
 var a = 1;
 
 class Foo { var b = 2, c, d = ""Hello""; }
 
 class Bar
 {
-    var e = 3, f, g = ""World"";
+    var e = 3, f, g = ""World"", superNull;
     Meth(){retval = this.e;}
 }
 
@@ -18,18 +18,21 @@ class FooBar
     mixin
         Foo,
         Bar;
+
+    init(c,f){}
 }
 
-var fb = FooBar();
+var fb = FooBar(7,8);
 
 expect
     fb.b == 2,
-    fb.c == null,
+    fb.c == 7,
     fb.d == ""Hello"",
     fb.e == 3,
-    fb.f == null,
+    fb.f == 8,
     fb.g == ""World"",
-    fb.Meth() == fb.e;
+    fb.Meth() == fb.e,
+    fb.superNull == null;
 ");
     }
 }
diff --git a/ulox/ulox.core.bench/ObjectVsSoa.cs b/ulox/ulox.core.bench/ObjectVsSoa.cs
@@ -2,7 +2,7 @@
 {
     public class ObjectVsSoa
     {
-        public const string ObjectBasedScript = @"
+        public static readonly Script ObjectBasedScript = new(nameof(ObjectBasedScript),@"
 class Foo
 {
 var
@@ -36,10 +36,10 @@ fun TickAllFoos(foos, dt)
 {
     TickAllFoos(foos, 0.01);
 }
-";
+");
 
 
-        public const string SoaBasedScript = @"
+        public static readonly Script SoaBasedScript = new(nameof(SoaBasedScript),@"
 class Foo
 {
 var
@@ -81,6 +81,6 @@ fun TickAllFoos(foos, dt)
 {
     TickAllFoos(foos, 0.01);
 }
-";
+");
     }
 }
diff --git a/ulox/ulox.core.bench/Program.cs b/ulox/ulox.core.bench/Program.cs
@@ -1,5 +1,8 @@
-using BenchmarkDotNet.Attributes;
+using System.Collections.Generic;
+using System.Linq;
+using BenchmarkDotNet.Attributes;
 using BenchmarkDotNet.Running;
+using Newtonsoft.Json.Linq;
 
 namespace ULox.Core.Bench
 {
@@ -8,15 +11,25 @@ public class Program
     {
         private CompiledScript _loopCompiled;
         private CompiledScript _ifCompiled;
-        private CompiledScript _scriptCompiled;
+        private CompiledScript _scriptCompiledAndOpt;
+        private CompiledScript _scriptCompiledNotOpt;
+        private List<Token> _compiledTokens;
+        private int[] _lineLengths;
+        private Engine _engine;
 
         [GlobalSetup]
         public void Setup()
         {
             var engine = Engine.CreateDefault();
             _loopCompiled = engine.Context.CompileScript(BenchmarkScripts.Loop);
             _ifCompiled = engine.Context.CompileScript(BenchmarkScripts.If);
-            _scriptCompiled = engine.Context.CompileScript(CompileVsExecute.Script);
+            _scriptCompiledAndOpt = engine.Context.CompileScript(CompileVsExecute.Script);
+
+            engine = Engine.CreateDefault();
+            var scanner = engine.Context.Program.Scanner;
+            _compiledTokens = scanner.Scan(CompileVsExecute.Script);
+            _lineLengths = scanner.GetLineLengths();
+            _scriptCompiledNotOpt = engine.Context.Program.Compiler.Compile(_compiledTokens.Select(x=>x).ToList(), _lineLengths, CompileVsExecute.Script);
         }
 
         static void Main(string[] args)
@@ -39,54 +52,94 @@ static void Main(string[] args)
         //    engine.RunScript(new Script("", ScriptVsNativeFunctional.FunctionalNative));
         //}
 
+        //[Benchmark]
+        //public void Object_PosVelUpdate()
+        //{
+        //    var engine = Engine.CreateDefault();
+        //    engine.RunScript(new Script("", ObjectVsSoa.ObjectBasedScript));
+        //}
+
+        //[Benchmark]
+        //public void Soa_PosVelUpdate()
+        //{
+        //    var engine = Engine.CreateDefault();
+        //    engine.RunScript(new Script("", ObjectVsSoa.SoaBasedScript));
+        //}
+
         [Benchmark]
-        public void Object_PosVelUpdate()
+        public Engine CompileVsExecute_NewEngineOnly()
         {
-            var engine = Engine.CreateDefault();
-            engine.RunScript(new Script("", ObjectVsSoa.ObjectBasedScript));
+            _engine = Engine.CreateDefault();
+            return _engine;
         }
 
         [Benchmark]
-        public void Soa_PosVelUpdate()
+        public List<Token> CompileVsExecute_TokeniseOnly()
         {
-            var engine = Engine.CreateDefault();
-            engine.RunScript(new Script("", ObjectVsSoa.SoaBasedScript));
+            _engine = Engine.CreateDefault();
+            return _engine.Context.Program.Scanner.Scan(CompileVsExecute.Script);
         }
 
         [Benchmark]
-        public void CompileVsExecute_All()
+        public List<Token> CompileVsExecute_DupTokensOnly()
         {
-            var engine = Engine.CreateDefault();
-            engine.RunScript(CompileVsExecute.Script);
+            _engine = Engine.CreateDefault();
+            var tokens = _compiledTokens.Select(x => x).ToList();
+            return tokens;
         }
 
         [Benchmark]
-        public void CompileVsExecute_CompileOnly()
+        public CompiledScript CompileVsExecute_CompileOnly()
         {
-            var engine = Engine.CreateDefault();
-            engine.Context.CompileScript(CompileVsExecute.Script);
+            _engine = Engine.CreateDefault();
+            var tokens = _compiledTokens.Select(x => x).ToList();
+            return _engine.Context.Program.Compiler.Compile(tokens, _lineLengths, CompileVsExecute.Script);
         }
 
         [Benchmark]
-        public void Looping_Loop()
+        public CompiledScript CompileVsExecute_DeepCloneOnly()
         {
-            var engine = Engine.CreateDefault();
-            engine.Context.Vm.Interpret(_loopCompiled.TopLevelChunk);
+            _engine = Engine.CreateDefault();
+            var compiled = _scriptCompiledNotOpt.DeepClone();
+            return compiled;
         }
 
         [Benchmark]
-        public void Conditional_If()
+        public CompiledScript CompileVsExecute_OptimiseOnly()
         {
-            var engine = Engine.CreateDefault();
-            engine.Context.Vm.Interpret(_ifCompiled.TopLevelChunk);
+            _engine = Engine.CreateDefault();
+            var compiled = _scriptCompiledNotOpt.DeepClone();
+            _engine.Context.Program.Optimiser.Optimise(compiled);
+            return compiled;
         }
 
         [Benchmark]
-        public string Dissasm_Script()
+        public void CompileVsExecute_All()
         {
-            var dis = new Disassembler();
-            dis.Iterate(_scriptCompiled);
-            return dis.GetString();
+            _engine = Engine.CreateDefault();
+            _engine.RunScript(CompileVsExecute.Script);
         }
+
+        //[Benchmark]
+        //public void Looping_Loop()
+        //{
+        //    var engine = Engine.CreateDefault();
+        //    engine.Context.Vm.Interpret(_loopCompiled.TopLevelChunk);
+        //}
+
+        //[Benchmark]
+        //public void Conditional_If()
+        //{
+        //    var engine = Engine.CreateDefault();
+        //    engine.Context.Vm.Interpret(_ifCompiled.TopLevelChunk);
+        //}
+
+        //[Benchmark]
+        //public string Dissasm_Script()
+        //{
+        //    var dis = new Disassembler();
+        //    dis.Iterate(_scriptCompiled);
+        //    return dis.GetString();
+        //}
     }
 }
diff --git a/ulox/ulox.core.bench/ScriptVsNativeFunctional.cs b/ulox/ulox.core.bench/ScriptVsNativeFunctional.cs
@@ -34,7 +34,7 @@ fun isthree(x)
 fun ByThree(x){retval = x*3;}
 ";
 
-        public const string FunctionalNative = CommonScript + @"
+        public static readonly Script FunctionalNative = new(nameof(FunctionalNative),CommonScript + @"
 var arr = MakeTestArray();
 var reducedResult = arr.Reduce(accum);
 Assert.AreEqual(10, reducedResult);
@@ -65,9 +65,9 @@ fun isthree(x)
 var forkRes = forkMethods.Fork(forkOn);
 var forkReduceRes = forkRes.Reduce(accum);
 Assert.AreEqual(3+6+9, forkReduceRes);
-";
+");
 
-        public const string FunctionalUlox = CommonScript + @"
+        public static readonly Script FunctionalUlox = new(nameof(FunctionalUlox),CommonScript + @"
 fun reduce(arr, fn)
 {
     var res = arr[0];
@@ -179,7 +179,6 @@ loop arr
 var forkRes = fork(forkMethods, forkOn);
 var forkReduceRes = reduce(forkRes, accum);
 Assert.AreEqual(3+6+9, forkReduceRes);
-";
-
+");
     }
 }
diff --git a/ulox/ulox.core.tests/ProfileSourceTests.cs → ulox/ulox.core.tests/BenchSourceTests.cs b/ulox/ulox.core.tests/ProfileSourceTests.cs → ulox/ulox.core.tests/BenchSourceTests.cs
@@ -4,23 +4,27 @@
 namespace ULox.Core.Tests
 {
     [TestFixture]
-    public class ProfileSourceTests : EngineTestBase
+    public class BenchSourceTests : EngineTestBase
     {
         [Test]
-        [TestCaseSource(nameof(DivideCases))]
-        public void Run_BouncingBallProfileScript(Script script)
+        [TestCaseSource(nameof(BenchScripts))]
+        public void Run_BenchScripts_Clean(Script script)
         {
             testEngine.Run(script);
             Assert.AreEqual("", testEngine.InterpreterResult);
         }
 
-        public static Script[] DivideCases = new Script[]
+        public static Script[] BenchScripts = new Script[]
         {
             BenchmarkScripts.Loop,
             BenchmarkScripts.If,
             CompileVsExecute.Script,
             Vec2Variants.Type,
             Vec2Variants.Tuple,
+            ObjectVsSoa.ObjectBasedScript,
+            ObjectVsSoa.SoaBasedScript,
+            ScriptVsNativeFunctional.FunctionalNative,
+            ScriptVsNativeFunctional.FunctionalUlox,
         };
     }
 }
diff --git a/ulox/ulox.core.tests/ByteCodeEngineTests.cs b/ulox/ulox.core.tests/ByteCodeEngineTests.cs
@@ -1,5 +1,4 @@
-using System.Linq;
-using System.Runtime.InteropServices;
+using System.Runtime.InteropServices;
 using NUnit.Framework;
 
 namespace ULox.Core.Tests

diff --git a/ulox/ulox.core.tests/ByteCodeIteratorTests.cs b/ulox/ulox.core.tests/ByteCodeIteratorTests.cs
@@ -3,7 +3,6 @@
 namespace ULox.Core.Tests
 {
     [TestFixture]
-
     public class ByteCodeIteratorTests : EngineTestBase
     {
         private CompiledScript CompileByteCode(string str)