Merge staging into main

.gitignore

@@ -9,4 +9,6 @@ workspace/zigem
 workspace/.gen
 workspace/.out
 workspace/.main-*.zig
-workspace/.zigem-main.zig
+workspace/.zigem-*.zig
+utils/*.log
+utils/*.log~

INSTALL.md

@@ -1,23 +1,39 @@
 # Installation
 
-## Pre-requisites
+These installation instructions contain example commands used in an Ubuntu Linux environment.  Similar operations will work for Windows / MacOS, and we will add them here over time.
 
-### Required PC Hardware / Operating System
+ZigEm is focused on low-power, small-code operation.  As such, we run it on a tiny wireless chip (the Texas Instruments CC2340) to demonstrate its effectiveness.  Programs written are cross-compiled for the CC2340, and loaded onto the CC2340 chip for operation.  We use a TI LaunchPad for the CC2340 chip as the evaluation hardware.  The LaunchPad includes the CC2340 along with a Bluetooth 5.0 radio, a serial UART, two LEDs (red and green), and a push-button.  ZigEm uses these devices in addition to the processor itself.
+
+Please visit the ZigEm blog: https://blog.zigem.tech/ for much more information and insight.
+
+## Pre-requisite Requirements
+
+### PC Hardware / Operating System
 
 ZigEm requires one of three different PC / Operating Systems for development, compilation, etc.  Only 64-bit x86 architectures are supported.  And only Ubuntu, MacOS and Windows are supported.
 
-### Required Embedded Hardware
+### Make
+
+ZigEm requires make to be installed and in the path.  On Windows, you'll need to download and install make (https://gnuwin32.sourceforge.net/packages/make.htm).
+
+### Git
+
+If using a Windows PC, you'll need to download and install `Git for Windows` (https://git-scm.com/download/win) and use a `git bash` window to do the below.
+
+For MacOS or Linux, bash is included, but you'll need to be sure that `git` is installed.
+
+### Embedded Hardware
 
 ZigEm requires that you purchase the following two items from Texas Instruments (or one of their distributors e.g. Arrow, DigiKey, Mouser).
 
 In the future we may support additional hardware.  But for now, the only way to run ZigEm programs is on the hardware specified below.
 
-Note:  If you already own an XDS110 Debugger, it may be configured to work with the LaunchPad.  We are suggesting the XDS110ET because it is the simplest connection and best suited to this job.
+Note:  The XDS110-style debugger called out below is available in other forms; sometimes as part of a different LaunchPad board, and sometimes stand-alone.  If you already own an XDS110 Debugger, it may be configured to work with the CC2340R5 LaunchPad.  We are suggesting the XDS110ET model because it is the simplest connection.  Using another XDS110 debugger can be done but will likely require jumper wires rather than the simple connector used herein.
 
 | Item                           | TI part number | Price (approx) | Links to purchase from TI              |
 | ------------------------------ | -------------- | -------------- | -------------------------------------- |
-| TI CC2340R5 LauncPad board     | LP-EM-CC2340R5 | $30            | https://www.ti.com/tool/LP-EM-CC2340R5 |
-| TI XDS110ET LaunchPad debugger | LP-XDS110ET    | $50            | https://www.ti.com/tool/LP-XDS110ET    |
+| TI CC2340R5 LaunchPad board    | LP-EM-CC2340R5 | $30            | https://www.ti.com/tool/LP-EM-CC2340R5 |
+| TI XDS110ET debugger           | LP-XDS110ET    | $50            | https://www.ti.com/tool/LP-XDS110ET    |
 
 These two pieces of hardware are the only two things that need to be purchased.  Everything else needed for ZigEm is available at no cost.
 
@@ -42,8 +58,6 @@ source $HOME/.bashrc
 
 - Clone the ZigEm repo into a folder
 
-Note:  Under Windows OS, you'll need to install `Git for Windows` (https://git-scm.com/download/win) and use a `git bash` window to do the below.
-
 An example on Ubuntu; yours may vary:
 
 ``` bash
@@ -78,6 +92,8 @@ The ZigEm VScode Extension is not yet available in the VScode marketplace.  As a
 - In lower left:  Settings => Command Pallette... => Extensions: Install from vsix...
 - In file explorer pop-up, navigate to the `zig-out/tools` folder and select the supplied `vscode-zigem-<version>.vsix` file
 
+Note:  Because the extension is supplied as a .vsix file (rather than from the marketplace), it will not automatically prompt when an update comes out.  We don't expect there to be frequent updates.  However, every now and then you may want to check the `zig-out/tools` folder and repeat the above procedure if a newer version comes available.
+
 ## Install the Texas Instruments LaunchPad hardware
 
 - Connect the CC2340R5 LaunchPad to the XDS110ET Debugger (via 2x10 connector)
@@ -128,3 +144,15 @@ zigem compile --load --file em.core/em.examples.basic/BlinkerP.em.zig
 There are about a dozen additional example programs that can be run now as well.
 
 Please consult additional ZigEm documentation and see https://blog.zigem.tech/ for more information.
+
+## Just in case you want to clean out the installation
+
+ZigEm and Zig maintain caches of previously downloaded / installed / built items.  This allows for speedy operation.
+
+If however for whatever reason you would like to "clean out" the caches, we have provided a bash script that will either do a _shallow_ or a _deep_ clean.
+
+Usage:  `clean.sh [-d, --deep]`
+
+Without the `-d` or `--deep` option, the clean script will clean the workspace folder only; removing remnants of prior `zigem compile` commands.
+
+With the deep cleaning option, the clean script will also clean out the remants of the prior `zig build` command.

build.zig

@@ -19,6 +19,8 @@ pub fn build(b: *std.Build) void {
         exe.root_module.addImport(name, dep.module(name));
     }
 
+    exe.root_module.addImport("zls", b.dependency("zigem-zls", .{}).module("zls"));
+
     const chmod_step = b.addSystemCommand(&.{ "chmod", "-R", "777", "zig-out/tools" });
     exe.step.dependOn(&chmod_step.step);
 
@@ -34,17 +36,12 @@ pub fn build(b: *std.Build) void {
         }
     }
 
-    b.installDirectory(.{
-        .source_dir = b.dependency("vscode-zigem", .{}).path("."),
-        .install_dir = std.Build.InstallDir{ .custom = "tools" },
-        .install_subdir = "",
-    });
-    b.installArtifact(b.dependency("zls-em", .{}).artifact("zls-em"));
+    b.installArtifact(b.dependency("zigem-zls", .{}).artifact("zigem-zls"));
     b.installArtifact(exe);
 
     const verify_exe = b.addRunArtifact(exe);
     verify_exe.setCwd(std.Build.LazyPath{ .src_path = .{ .owner = b, .sub_path = "workspace" } });
-    verify_exe.addArgs(&.{ "compile", "-f", "em.core/em.examples.basic/BlinkerP.em.zig" });
+    verify_exe.addArgs(&.{ "compile", "-f", "em.core/em.examples.basic/Ex02_BlinkerP.em.zig" });
     const verify_step = b.step("verify", "Verify ZigEM");
     verify_step.dependOn(&verify_exe.step);
 

build.zig.zon

@@ -31,23 +31,19 @@
             .hash = "12205de1eef9492a1cbee4461db6d9db803f14b32e5309ee4a036ac5a3d2a7640902",
             .lazy = true,
         },
-        .@"ti-uniflash-macos-x86_64" = .{
-            .url = "https://github.com/em-foundation/tools-pkg/releases/download/ti-uniflash-8.7.0/ti-uniflash-macos-x86_64.tar.gz",
-            .hash = "12203b87852ad433493c47cb209df32a3090d1b5605526c7d85bb3d980db079d5f31",
+        .@"ti-uniflash-macos-x86_64" = .{ // dummy
+            .url = "https://github.com/em-foundation/tools-pkg/releases/download/ti-uniflash-8.7.0/ti-uniflash-linux-x86_64.zip",
+            .hash = "12205de1eef9492a1cbee4461db6d9db803f14b32e5309ee4a036ac5a3d2a7640902",
             .lazy = true,
         },
         .@"arm-binutils-macos-x86_64" = .{
             .url = "https://github.com/em-foundation/tools-pkg/releases/download/arm-binutils-2.37/arm-binutils-macos-x86_64.tar.gz",
             .hash = "1220e083de87e1aadeb0c0446c301d4214b2eebb2632389eb650556462c80ec9b71d",
             .lazy = true,
         },
-        .@"zls-em" = .{
-            .url = "https://github.com/em-foundation/zls-em/archive/refs/tags/v0.0.1.tar.gz",
-            .hash = "12201b80d7156dfba486ba255f282e67ee3e84f782b6816d93fcd68466f9193be267",
-        },
-        .@"vscode-zigem" = .{
-            .url = "https://github.com/em-foundation/vscode-zigem/releases/download/v0.0.2.202409131250/vscode-zigem.zip",
-            .hash = "1220291e6f88b53eb17dbd6c47ef8f45eb8bae0a7181e51b1a3158bbe414862c110b",
+        .@"zigem-zls" = .{
+            .url = "https://github.com/em-foundation/zigem-zls/archive/refs/tags/v0.0.3.tar.gz",
+            .hash = "122050bf95425972f281a2018b48fa4ff04f4e0b2fece8e1ea1be33a75317244f264",
         },
     },
 }

src/AstUtils.zig

@@ -0,0 +1,162 @@
+const std = @import("std");
+
+const Fs = @import("Fs.zig");
+const Heap = @import("Heap.zig");
+
+const Ast = std.zig.Ast;
+
+pub const NodeIter = struct {
+    ast: *const Ast,
+    root_decls: []const Ast.Node.Index,
+    root_idx: usize,
+    loc: std.ArrayList(NodeLoc),
+
+    const NodeLoc = struct {
+        idx: Ast.Node.Index,
+        // 0 self, 1..n, n
+        // node.children.len + 1
+        last_step: u32,
+    };
+
+    const WalkDir = enum {
+        down,
+        up,
+    };
+
+    const Output = struct {
+        idx: Ast.Node.Index,
+        direction: WalkDir,
+    };
+
+    pub fn init(ast: *const Ast) NodeIter {
+        return .{
+            .ast = ast,
+            .root_decls = ast.rootDecls(),
+            .root_idx = 0,
+            .loc = std.ArrayList(NodeLoc).init(Heap.get()),
+        };
+    }
+
+    pub fn deinit(self: *NodeIter) void {
+        self.loc.deinit();
+    }
+
+    fn append(self: *NodeIter, idx: Ast.Node.Index) !Output {
+        try self.loc.append(.{
+            .idx = idx,
+            .last_step = 0,
+        });
+
+        return .{
+            .idx = idx,
+            .direction = WalkDir.down,
+        };
+    }
+
+    // NOTE: When calling this externally on WalkDir.up, this will pop the
+    // parent of the node that was returned. This should probably only be
+    // called on WalkDir.down
+    pub fn pop(self: *NodeIter) Output {
+        const loc = self.loc.pop();
+        return .{
+            .idx = loc.idx,
+            .direction = WalkDir.up,
+        };
+    }
+
+    fn resolveChildNodeId(self: *NodeIter, node_idx: Ast.Node.Index, num_children_walked: u32) ?Ast.Node.Index {
+        const node = self.ast.nodes.get(node_idx);
+        switch (node.tag) {
+            .simple_var_decl,
+            .fn_decl,
+            .fn_proto_multi,
+            .fn_proto_simple,
+            .builtin_call_two,
+            .number_literal,
+            .string_literal,
+            .container_decl_two,
+            .container_field_init,
+            .block_two_semicolon,
+            .identifier,
+            => {
+                switch (num_children_walked) {
+                    0 => return node.data.lhs,
+                    1 => return node.data.rhs,
+                    else => return null,
+                }
+            },
+            .container_decl => {
+                const idx = node.data.lhs + num_children_walked;
+                if (idx >= node.data.rhs) {
+                    return null;
+                }
+                return self.ast.extra_data[idx];
+            },
+            else => {
+                std.log.err("Unhandled tag: {any}", .{node.tag});
+                return null;
+            },
+        }
+    }
+
+    pub fn next(self: *NodeIter) !?Output {
+        while (true) {
+            if (self.loc.items.len == 0) {
+                if (self.root_idx >= self.root_decls.len) {
+                    return null;
+                }
+
+                defer self.root_idx += 1;
+                return try self.append(self.root_decls[self.root_idx]);
+            }
+
+            const loc = &self.loc.items[self.loc.items.len - 1];
+
+            while (self.resolveChildNodeId(loc.idx, loc.last_step)) |node_idx| {
+                defer loc.last_step += 1;
+                if (node_idx == 0) {
+                    continue;
+                }
+                return try self.append(node_idx);
+            }
+
+            return self.pop();
+        }
+    }
+};
+
+pub fn parse(path: []const u8) !Ast {
+    const norm = try Fs.normalize(path);
+    const source = Fs.readFileZ(norm);
+    const ast = try Ast.parse(Heap.get(), source, .zig);
+    return ast;
+}
+
+pub fn printTree(ast: Ast) void {
+    std.debug.print(
+        \\printTree:
+        \\nodes   tag                            lhs         rhs         ln   col   tok
+        \\-----------------------------------------------------------------------------
+        \\
+    , .{});
+    for (ast.nodes.items(.tag), ast.nodes.items(.data), ast.nodes.items(.main_token), 0..) |tag, data, main_token, i| {
+        const loc = ast.tokenLocation(0, main_token);
+        std.debug.print(
+            "    {d:<3} {s:<30} {d:<11} {d:<11} {d:<4} {d:<5} {d:<5} {s}\n",
+            .{ i, @tagName(tag), data.lhs, data.rhs, loc.line + 1, loc.column + 1, main_token, ast.tokenSlice(main_token) },
+        );
+    }
+
+    std.debug.print(
+        \\
+        \\tokens  tag                  start
+        \\----------------------------------
+        \\
+    , .{});
+    for (ast.tokens.items(.tag), ast.tokens.items(.start), 0..) |tag, start, i| {
+        std.debug.print(
+            "    {d:<3} {s:<20} {d:<}\n",
+            .{ i, @tagName(tag), start },
+        );
+    }
+}

src/Fs.zig

@@ -5,6 +5,8 @@ const fs = std.fs;
 
 const Heap = @import("./Heap.zig");
 
+const MAX_BYTES = 100_000;
+
 pub fn basename(path: []const u8) []const u8 {
     return fs.path.basename(path);
 }
@@ -46,31 +48,31 @@ pub fn normalize(path: []const u8) ![]const u8 {
 }
 
 pub fn openDir(path: []const u8) fs.Dir {
-    const dir = fs.openDirAbsolute(path, .{ .iterate = true }) catch fatal("Path.openDir: {s}", .{path});
+    const dir = fs.openDirAbsolute(path, .{ .iterate = true }) catch fatal("Fs.openDir: {s}", .{path});
     return dir;
 }
 
 pub fn openFile(path: []const u8) fs.File {
-    const file = fs.openFileAbsolute(path, .{ .mode = .read_only }) catch fatal("Path.openFile: {s}", .{path});
+    const file = fs.openFileAbsolute(path, .{ .mode = .read_only }) catch fatal("Fs.openFile: {s}", .{path});
     return file;
 }
 
 pub fn readFile(path: []const u8) []const u8 {
     const file = openFile(path);
     defer file.close();
-    const buf = file.readToEndAlloc(Heap.get(), 1000) catch fatal("Path.readFile: {s}", .{path});
+    const buf = file.readToEndAlloc(Heap.get(), MAX_BYTES) catch fatal("Fs.readFile: {s}", .{path});
     return buf;
 }
 
 pub fn readFileZ(path: []const u8) [:0]const u8 {
     const file = openFile(path);
     defer file.close();
-    const buf = file.readToEndAllocOptions(Heap.get(), 1000, null, @alignOf(u8), 0) catch fatal("Path.readFile", .{});
+    const buf = file.readToEndAllocOptions(Heap.get(), MAX_BYTES, null, @alignOf(u8), 0) catch fatal("Fs.readFile", .{});
     return buf;
 }
 
 pub fn slashify(path: []const u8) []const u8 {
-    const dupe = Heap.get().dupe(u8, path) catch fatal("Path.slashify: {s}", .{path});
+    const dupe = Heap.get().dupe(u8, path) catch fatal("Fs.slashify: {s}", .{path});
     std.mem.replaceScalar(u8, dupe, '\\', '/');
     return dupe;
 }