dev: split bootloader into two files with executables embedded in ker…

…nel binary.

This was supposed to make booting faster but it only eliminates the double loading
time of the service worker on first boot (or after clearing local data). For me
it went from taking 30s on first boot to 17s, almost half. It doesn't change the
regular reload boot time at all, which I was hoping it would.

In this setup, there are now two files you need. The wanix-bootloader.js and then
a gzipped kernel binary. We still pack files into the bootloader, but leaving the
executables out. Instead, they're embedded in the kernel binary. I'm hoping eventually
all initfs files can be embedded in the kernel.

This also removes hugo from being included because its too large. It pushed the
file size over 100M which GitHub starts to prevent without LFS which interacts with
GitHub pages strangely. I think keeping it under 100M is a good idea, ideally much
smaller. Hugo can be loaded separately by another mechanism for now.

Makefile

@@ -8,30 +8,29 @@ all: kernel shell build micro hugo
 dev: all
 	go run ./dev
 
-bundle: local/bin
+bundle: local/bin/kernel
+	cp local/bin/kernel local/wanix-kernel
+	gzip -f -9 local/wanix-kernel
 	go run -tags bundle ./dev
 
-kernel: kernel/main.go local/bin
+kernel: local/bin/kernel
+local/bin/kernel: kernel/**/*.go kernel/bin/shell kernel/bin/micro kernel/bin/build
 	cd kernel && GOOS=js GOARCH=wasm go build -ldflags="-X 'main.Version=${VERSION}' -X 'tractor.dev/wanix/kernel/fs.DebugLog=${DEBUG}'" -o ../local/bin/kernel .
 
-shell: shell/main.go local/bin
-	cd shell && GOOS=js GOARCH=wasm go build -o ../local/bin/shell .
+shell: kernel/bin/shell
+kernel/bin/shell: shell/main.go
+	cd shell && GOOS=js GOARCH=wasm go build -o ../kernel/bin/shell .
 
-micro: local/bin/micro
-
-local/bin/micro: external/micro/ local/bin
+micro: kernel/bin/micro
+kernel/bin/micro: external/micro/
 	make -C external/micro build
 
-hugo: local/bin/hugo
-
-local/bin/hugo: external/hugo/ local/bin
+hugo: external/hugo/
 	make -C external/hugo build
 
 build/pkg.zip: build/build-pkgs/imports/imports.go build/build-pkgs/main.go
 	cd build && go run ./build-pkgs/main.go ./build-pkgs/imports ./pkg.zip
 
-build: build/main.go build/pkg.zip local/bin
-	cd build && GOOS=js GOARCH=wasm go build -o ../local/bin/build .
-
-local/bin:
-	mkdir -p local/bin
+build: kernel/bin/build
+kernel/bin/build: build/main.go build/pkg.zip
+	cd build && GOOS=js GOARCH=wasm go build -o ../kernel/bin/build .

dev/bootloader.js

@@ -4,6 +4,7 @@ if (!globalThis["ServiceWorkerGlobalScope"]) {
   // registers Service Worker using this file (see bottom) if none is registered,
   // and sets up a mechanism to fullfill requests from initfs or kernel
   async function setupServiceWorker() {
+    const timeout = (ms) => new Promise((resolve, reject) => setTimeout(() => reject(new Error("Timeout")), ms));
     const unzip = async (b64data) => {
       const gzipData = atob(b64data);
       const gzipBuf = new Uint8Array(gzipData.length);
@@ -20,13 +21,17 @@ if (!globalThis["ServiceWorkerGlobalScope"]) {
 
     let registration = await navigator.serviceWorker.getRegistration();
     if (!registration) {
+      console.log("Registering service worker...");
       await navigator.serviceWorker.register("./wanix-bootloader.js?sw", {type: "module"});
       registration = await navigator.serviceWorker.ready;
-      await new Promise((resolve) => {
-        navigator.serviceWorker.addEventListener("controllerchange", async (event) => {
-          resolve();
-        });
-      });
+      await Promise.race([
+        new Promise((resolve) => {
+          navigator.serviceWorker.addEventListener("controllerchange", async (event) => {
+            resolve();
+          });
+        }),
+        timeout(3000)
+      ]).catch(err => console.warn(err));
     }
     if (registration.active && !navigator.serviceWorker.controller) {
       // Perform a soft reload to load everything from the SW and get
@@ -76,16 +81,28 @@ if (!globalThis["ServiceWorkerGlobalScope"]) {
       registration.active.postMessage({response: { reqId: req.id, body: buf.bytes(), headers }});
     });
 
+    console.log("Initializing service worker...");
     registration.active.postMessage({init: true, basePath});
     await ready;
   }
 
+  async function fetchKernel() {
+    const resp = await fetch("./wanix-kernel.gz");
+    const gzipBlob = await resp.blob();
+    const ds = new DecompressionStream('gzip');
+    const out = gzipBlob.stream().pipeThrough(ds);
+    const response = new Response(out);
+    globalThis.initfs["kernel"] = { mtimeMs: Date.now(), blob: await response.blob() };
+  }
+
   // bootloader starts here
   (async function() {
     console.log("Wanix booting...")
+
+    globalThis.initfs = {};
+    const kernelReady = fetchKernel();
     await setupServiceWorker();
 
-    globalThis.initfs = {};
     const load = async (name, file) => {
       // Determine if file contains a path to fetch or embedded file contents to load.
       if(file.type === "text/plain") {
@@ -105,6 +122,7 @@ if (!globalThis["ServiceWorkerGlobalScope"]) {
     globalThis.duplex = await import(URL.createObjectURL(initfs["duplex.js"].blob));
     globalThis.task = await import(URL.createObjectURL(initfs["task.js"].blob));
 
+    await kernelReady;
     globalThis.sys = new task.Task(initfs);
 
     // start kernel
@@ -141,13 +159,14 @@ if (globalThis["ServiceWorkerGlobalScope"] && self instanceof ServiceWorkerGloba
     const req = event.request;
     const url = new URL(req.url);
     if (url.pathname === "/favicon.ico" || 
-      url.hostname !== "localhost" || // TODO: something else to allow cross-domain requests
       url.pathname === basePath ||
       url.pathname.startsWith(`${basePath}wanix-bootloader.js`) ||
       url.pathname.startsWith(`${basePath}sys/dev`) || 
       url.pathname.startsWith(`${basePath}bootloader`) || 
       url.pathname.startsWith(`${basePath}index.html`) ||
-      url.hostname !== "localhost" || // TEMPORARY WORKAROUND
+      url.pathname.startsWith(`${basePath}wanix-kernel.gz`) ||
+      url.pathname.startsWith("/auth") ||
+      !url.hostname.endsWith(location.hostname) ||
       !host) return;
 
     reqId++;

dev/initdata.go

@@ -20,11 +20,6 @@ var files = []File{
 	{Name: "wasm.js", Path: "./kernel/web/lib/wasm.js"},
 	{Name: "host.js", Path: "./kernel/web/lib/host.js"},
 	{Name: "indexedfs.js", Path: "./internal/indexedfs/indexedfs.js"},
-	{Name: "kernel", Path: "./local/bin/kernel"},
-	{Name: "build", Path: "./local/bin/build"},
-	{Name: "micro", Path: "./local/bin/micro"},
-	{Name: "shell", Path: "./local/bin/shell"},
-	{Name: "hugo", Path: "./local/bin/hugo"},
 
 	// Shell source files
 	{Name: "shell/main.go", Path: "shell/main.go"},

dev/server.go

@@ -3,10 +3,13 @@
 package main
 
 import (
+	"compress/gzip"
 	"fmt"
+	"io"
 	"log"
 	"net/http"
 	"os"
+	"path/filepath"
 	"time"
 
 	"tractor.dev/wanix/internal/httpfs"
@@ -35,6 +38,28 @@ func main() {
 	mux.Handle(fmt.Sprintf("%s/sys/dev/", basePath), http.StripPrefix(fmt.Sprintf("%s/sys/dev/", basePath), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		gwutil.FileTransformer(os.DirFS(dir), httpfs.FileServer).ServeHTTP(w, r)
 	})))
+	mux.Handle(fmt.Sprintf("%s/wanix-kernel.gz", basePath), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("content-type", "application/gzip")
+
+		file, err := os.Open(filepath.Join(dir, "local/bin/kernel"))
+		if err != nil {
+			http.Error(w, "File not found.", http.StatusNotFound)
+			return
+		}
+		defer file.Close()
+
+		gzipWriter := gzip.NewWriter(w)
+		defer gzipWriter.Close()
+
+		if _, err := io.Copy(gzipWriter, file); err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+		if err := gzipWriter.Flush(); err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+	}))
 	mux.Handle(fmt.Sprintf("%s/wanix-bootloader.js", basePath), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("content-type", "application/javascript")
 

external/micro/Makefile

@@ -1,6 +1,6 @@
 build: update
 	make -C repo wasm
-	mv ./repo/micro.wasm ./../../local/bin/micro
+	mv ./repo/micro.wasm ./../../kernel/bin/micro
 
 repo: 
 	git clone -q https://github.com/zyedidia/micro ./repo

kernel/.gitignore

@@ -0,0 +1 @@
+bin

kernel/fs/fs.go

@@ -18,7 +18,6 @@ import (
 	"tractor.dev/toolkit-go/engine/fs/watchfs"
 
 	"tractor.dev/wanix/internal"
-	"tractor.dev/wanix/internal/githubfs"
 	"tractor.dev/wanix/internal/httpfs"
 	"tractor.dev/wanix/internal/indexedfs"
 	"tractor.dev/wanix/internal/jsutil"
@@ -34,6 +33,12 @@ func log(args ...any) {
 	}
 }
 
+func must(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
 type Service struct {
 	// don't love passing this here but kernel
 	// package is a main package so cant reference it
@@ -79,54 +84,42 @@ func (s *Service) Initialize() {
 	fs.MkdirAll(s.fsys, "sys/tmp", 0755)
 
 	// copy some apps include terminal
-	s.copyAllFS(s.fsys, "sys/app/terminal", internal.Dir, "app/terminal")
-	s.copyAllFS(s.fsys, "sys/app/todo", internal.Dir, "app/todo")
-
-	// copy of kernel source into filesystem.
-	s.copyAllFS(s.fsys, "sys/cmd/kernel", s.KernelSource, ".")
+	must(s.copyAllFS(s.fsys, "sys/app/terminal", internal.Dir, "app/terminal"))
+	must(s.copyAllFS(s.fsys, "sys/app/todo", internal.Dir, "app/todo"))
 
 	// copy shell source into filesystem
 	fs.MkdirAll(s.fsys, "sys/cmd/shell", 0755)
 	shellFiles := getPrefixedInitFiles("shell/")
 	for _, path := range shellFiles {
-		if err = s.copyFromInitFS(filepath.Join("sys/cmd", path), path); err != nil {
-			panic(err)
-		}
+		must(s.copyFromInitFS(filepath.Join("sys/cmd", path), path))
 	}
 
-	// copy builtin exe's into filesystem
-	s.copyFromInitFS("sys/cmd/build.wasm", "build")
-	s.copyFromInitFS("sys/bin/shell.wasm", "shell")
-	s.copyFromInitFS("cmd/micro.wasm", "micro")
-	s.copyFromInitFS("cmd/hugo.wasm", "hugo")
+	// copy of kernel source into filesystem.
+	must(s.copyAllFS(s.fsys, "sys/cmd/kernel", s.KernelSource, "."))
+
+	// move builtin kernel exe's into filesystem
+	must(s.fsys.Rename("sys/cmd/kernel/bin/build", "sys/cmd/build.wasm"))
+	must(s.fsys.Rename("sys/cmd/kernel/bin/shell", "sys/bin/shell.wasm"))
+	must(s.fsys.Rename("sys/cmd/kernel/bin/micro", "sys/cmd/micro.wasm"))
 
 	devURL := fmt.Sprintf("%ssys/dev", js.Global().Get("hostURL").String())
 	resp, err := http.DefaultClient.Get(devURL)
-	if err != nil {
-		panic(err)
-	}
+	must(err)
 	if resp.StatusCode == 200 {
-		if err := s.fsys.(*mountablefs.FS).Mount(httpfs.New(devURL), "/sys/dev"); err != nil {
-			panic(err)
-		}
-	}
-
-	if err := s.fsys.(*mountablefs.FS).Mount(memfs.New(), "/sys/tmp"); err != nil {
-		panic(err)
+		must(s.fsys.(*mountablefs.FS).Mount(httpfs.New(devURL), "/sys/dev"))
 	}
 
-	fs.MkdirAll(s.fsys, "sys/git", 0755)
-	err = s.fsys.(*mountablefs.FS).Mount(
-		githubfs.New(
-			"tractordev",
-			"wanix",
-			"INSERT_TOKEN_HERE",
-		),
-		"/sys/git",
-	)
-	if err != nil {
-		panic(err)
-	}
+	must(s.fsys.(*mountablefs.FS).Mount(memfs.New(), "/sys/tmp"))
+
+	// fs.MkdirAll(s.fsys, "sys/git", 0755)
+	// must(s.fsys.(*mountablefs.FS).Mount(
+	// 	githubfs.New(
+	// 		"tractordev",
+	// 		"wanix",
+	// 		"INSERT_TOKEN_HERE",
+	// 	),
+	// 	"/sys/git",
+	// ))
 
 }