Scylla Server in Rust

.github/workflows/build-check.yml

@@ -21,14 +21,9 @@ jobs:
         run: |
           sudo apt-get -y update
           sudo apt-get -y install docker-compose
-      - name: Generate Angular Dist
-        run: |
-          cd ./angular-client
-          npm install
-          npm run build
       - name: Build Docker Compose
         run: |
           if ! docker-compose build; then
             echo "Docker Compose build failed."
             exit 1  # This will cause the workflow to fail
-          fi
+          fi

.github/workflows/build-image.yml → .github/workflows/client-build.yml

@@ -1,5 +1,5 @@
 #
-name: Create and publish a Docker image
+name: Create and publish client docker image
 
 on:
   workflow_dispatch:
@@ -37,7 +37,7 @@ jobs:
         id: meta
         uses: docker/[email protected]
         with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-client
       # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages.
       # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository.
       # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step.
@@ -48,7 +48,7 @@ jobs:
           push: true
           target: production
           platforms: linux/arm64,linux/amd64
-          tags: ${{ steps.meta.outputs.tags }}-angular-client # argos-angular-client
+          tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           # for caching
           # cache-from: type=gha,scope=global
@@ -59,18 +59,4 @@ jobs:
             PROD=true
             BACKEND_URL=http://192.168.100.1:8000
             MAP_ACCESS_TOKEN=pk.eyJ1IjoibWNrZWVwIiwiYSI6ImNscXBrcmU1ZTBscWIya284cDFyYjR3Nm8ifQ.6TQHlxhAJzptZyV-W28dnw
-      - name: Build and push Docker image for scylla server
-        uses: docker/[email protected]
-        with:
-          context: ./scylla-server
-          push: true
-          target: production
-          platforms: linux/arm64,linux/amd64
-          tags: ${{ steps.meta.outputs.tags }}-scylla-server # argos-scylla-server
-          labels: ${{ steps.meta.outputs.labels }}
-          # for caching
-          # cache-from: type=gha
-          # cache-to: type=gha,mode=max
-          # https://github.com/docker/build-push-action/issues/820
-          provenance: false
 

.github/workflows/lint-check.yml

@@ -9,6 +9,7 @@ on:
       - main
       - develop
       - 'feature/**'
+
 jobs:
   run-linting-check:
     runs-on: ubuntu-latest
@@ -24,6 +25,6 @@ jobs:
           node-version: 16.17.1
           cache: 'npm'
       - name: Install modules
-        run: npm install && npm run prisma:generate
+        run: npm install
       - name: Run linting check
         run: npm run lint --max-warnings=0

.github/workflows/prettier-check.yml

@@ -9,6 +9,8 @@ on:
       - main
       - develop
       - 'feature/**'
+
+
 jobs:
   run-prettier-check:
     runs-on: ubuntu-latest

.github/workflows/scylla-build.yml

@@ -0,0 +1,57 @@
+#
+name: Create and publish scylla docker image
+
+on:
+  workflow_dispatch:
+
+
+
+# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds.
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu.
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
+    permissions:
+      contents: read
+      packages: write
+      #
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/[email protected]
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here.
+      - name: Log in to the Container registry
+        uses: docker/[email protected]
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels.
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/[email protected]
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-scylla
+      # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages.
+      # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository.
+      # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step.
+      - name: Build and push Docker image for scylla server rust
+        uses: docker/[email protected]
+        with:
+          context: ./scylla-server
+          push: true
+          platforms: linux/arm64,linux/amd64
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          # for caching
+          # cache-from: type=gha
+          # cache-to: type=gha,mode=max
+          # https://github.com/docker/build-push-action/issues/820
+          provenance: false
+

.github/workflows/scylla-ci.yml

@@ -0,0 +1,34 @@
+name: Scylla CI
+
+on:
+   push:
+      branches:
+         - main
+         - develop
+   pull_request:
+      branches:
+         - main
+         - develop
+         - 'feature/**'
+
+
+defaults:
+    run:
+        working-directory: scylla-server
+
+
+jobs:
+   build:
+      runs-on: ubuntu-latest
+
+      steps:
+         - name: Setup Rust
+           uses: actions/checkout@v3
+           with:
+            submodules: recursive
+         - name: Generate prisma
+           run: cargo prisma generate
+         - name: Build
+           run: cargo build --verbose
+         - name: Clippy
+           run: cargo clippy --verbose -- -D warnings

.github/workflows/scylla-tests.yml

@@ -0,0 +1,36 @@
+name: Scylla Rust Tests
+
+on:
+  push:
+    branches:
+      - main
+      - develop
+  pull_request:
+    branches:
+      - main
+      - develop
+      - 'feature/**'
+
+env:
+    SOURCE_DATABASE_URL: postgresql://postgres:[email protected]:5432/timescaledb cargo run
+
+jobs:
+  run-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+      - name: Run db
+        run: docker compose run -P -d odyssey-timescale
+      - name: Generate prisma
+        run: cargo prisma generate
+        working-directory: scylla-server
+      - name: Deploy prisma
+        run: cargo prisma migrate deploy
+        working-directory: scylla-server
+      - name: Run tests
+        working-directory: scylla-server
+        run: cargo test -- --test-threads=1

.gitignore

@@ -3,8 +3,8 @@
 
 # environment config
 **/.env
-scylla-server/src/prisma/mydatabase.db
-scylla-server/src/prisma/mydatabase.db-journal
+scylla-server-typescript/src/prisma/mydatabase.db
+scylla-server-typescript/src/prisma/mydatabase.db-journal
 
 # Xcode
 #
@@ -95,4 +95,7 @@ fastlane/test_output
 # After new code Injection tools there's a generated folder /iOSInjectionProject
 # https://github.com/johnno1962/injectionforxcode
 
-iOSInjectionProject/
+iOSInjectionProject/
+
+# user compose override
+compose.override.yml

.gitmodules

@@ -1,3 +1,3 @@
-[submodule "scylla-server/src/odyssey-base"]
-	path = scylla-server/src/odyssey-base
+[submodule "scylla-server-typescript/src/odyssey-base"]
+	path = scylla-server-typescript/src/odyssey-base
 	url = https://github.com/Northeastern-Electric-Racing/Odyssey-Base.git

.prettierignore

@@ -1,8 +1,9 @@
 .github
 .vscode
 
-docker-compose.yml
-docker-compose-dev.yml
+# ignore docker, etc
+*.yml
+
 .eslintrc.js
 tsconfig.json
 

README.md

@@ -11,40 +11,61 @@ Setup angular-client and scylla-server:
 [Angular Client](./angular-client/README.md)\
 [Scylla Server](./scylla-server/README.md)
 
-Once you've sucessfully setup Scylla and the Client, you can either run them separately, or run the following command to run them concurrently (the outputs from both will be output to the same terminal):
-
-`npm run start`
+Once you've sucessfully setup Scylla and the Client, you can either run them separately, or follow the docker commands below to run them together.
 
 ## Production
 
-### Running the Project in Prod Mode
+### Quick start shortcuts
+
+The `argos.sh` can be used on POSIX compliant OSes to quickly get up and running without worrying about profiles.  Its syntax is `./argos.sh <profile> <cmd>` where profile could be router, tpu, scylla-dev, or client-dev and the cmd is the normal argument passed into docker compose, such as `up -d` to start the process and fork to background.  **Make sure to `down` with the same profile that you used `up` with!**
+
+### Customizing and more info
 
-There is a `docker-compose-dev.yml` file for a dev which varies from the router deployment:
+The base docker compose (`compose.yml`) contains some important features to note.  However, it is useless standalone.  Please read the profile customization selection below before using the base compose.
 - It matches the number of CPUs as the router to roughly simulate router CPU (your CPU is still faster)
-- You must build it locally first!
-- It does not persist the database between `down` commands
+- It persists the database between `down` commands via a volume called `argos_db-data`.  Delete it with `docker volume rm argos_db-data` to start with a new database next `up`.
+- It weighs the CPU usage of siren higher, so it is prioritized in CPU starvation scenarios.
+- It limits memory according to the capacity of the router.
+
+
+#### Customizing runtime profiles of the project via docker compose
+
+This project uses docker compose overrides to secify configurations.  Therefore there are multiple "profiles" to choose from when running in production, and there are some profiles for development testing.  Also, there are fragment files for siren and client in `siren-base` and `angular-client` respectively, as they are services only used in certain cases.  These profiles are specified via the command line on top of the base `compose.yml` file as follows.
+
+```
+docker compose -f compose.yml -f <override_file_name> <your command here>
+```
+
+Additionally if you need to create your own options, you can create a `compose.override.yml` file in this directory and specify what settings should be changed, which is ignored by git.  If you think the override would become useful, document it here and name it `compose.<name>.yml`. Here is the current list of overrides, designed so only one is used at any given time:
+
+- `scylla-dev`*: Testing the client and interactions of scylla (scylla prod, siren local, client pt local)
+- `client-dev`*: Testing the client development using the scylla mock data mode (scylla mock, client pt local)
+- `router`: For production deployment to the base gateway node (scylla prod, siren local, client pt 192.168.100.1)
+- `tpu`: Production deployment to the TPU on the car (no client, no siren, scylla pt siren external)
 
-Note that both compose files limit memory to the same amount.  However, the disk I/O of the router is **much** slower than yours.
+***Note that since client settings are changed via rebuild, overrides with a * must be rebuilt via `docker compose -f compose.yml -f compose.<override>.yml build client`.  Further, a build should be done when reverting to profiles without stars. **
 
-This will build the docker images that will be run:
+Examples:
 
-`docker-compose -f ./docker-compose-dev.yml build`
+Router deploy and send to background: `docker compose -f compose.yml -f compose.router.yml up -d`
 
-This will start the containers and output all the outputs from both of them to the terminal. If the container is not already an image through docker-compose build, it will attempt to pull the images from docker hub. 
+### Build and deploy
 
-`docker-compose up`
+Using the above profiles, one can `build` the app.  Then, with correct permissions, you can `push` the app then `pull` it elsewhere for usage.  Note that you must `push` and `pull` on the same architecture, so you cannot use for example a dell laptop to build for the router!  To get `push` permissions, create a PAT [here](https://github.com/settings/tokens/new?scopes=write:packages) and copy the token into this command:
 
-If changes are made to either the client or scylla you will need to rebuild and push to docker hub then pull on the router. Contact the current Argos lead or Chief Software Engineer to get access to the docker hub.
+```
+sudo docker login ghcr.io -u <gh username> -p <token>
+```
 
-### Running on the Openwrt router
+Now you can update the image on a remote server.  Note to save time you can just specify which service to upload, like `scylla-server` or `client`.
+```
+sudo docker compose -f compose.yml -f compose.router.yml build && sudo docker compose -f compose.yml -f compose.router.yml push
+```
 
-The `docker-compose.yml` file is made for the router.  When you push a commit it automatically gets built for the router in 20-30 minutes.
-To use a non-standard branch edit the docker-compose.yml file to the name of the tag specified by the name [here](https://github.com/Northeastern-Electric-Racing/Argos/pkgs/container/argos).
-Then do `docker compose down` and `docker compose pull` and `docker compose up -d`.
 
-**The database is stored in a volume called `argos_db-data`, delete the volume to start the database fresh!**
+## Codegen Protobuf Types (client only)
 
-## Codegen Protobuf Types
+Server protobuf generation is automatic.  See below for client protobuf generation.
 
 ##### Mac
 
@@ -54,6 +75,6 @@ Then do `docker compose down` and `docker compose pull` and `docker compose up -
 #### Codegen
 `npm run build:proto`
 
-### Siren
+## Siren
 The configuration for the Mosquitto MQTT server on the router is in the siren-base folder.
 Note that the configuration is used in the docker compose file, but the configuration on the TPU is stored in [Odysseus.](https://github.com/Northeastern-Electric-Racing/Odysseus/tree/cb12fb3240d5fd58adfeae26262e158ad6dd889b/odysseus_tree/overlays/rootfs_overlay_tpu/etc/mosquitto)

angular-client/compose.client.yml

@@ -0,0 +1,16 @@
+services:
+  client:
+    container_name: client
+    restart: unless-stopped
+    image: ghcr.io/northeastern-electric-racing/argos-client:develop
+    build:
+      context: .
+      args:
+        PROD: "true"
+        BACKEND_URL: http://192.168.100.1:8000
+        MAP_ACCESS_TOKEN: pk.eyJ1IjoibWNrZWVwIiwiYSI6ImNscXBrcmU1ZTBscWIya284cDFyYjR3Nm8ifQ.6TQHlxhAJzptZyV-W28dnw
+      target: production
+      dockerfile: Dockerfile
+    ports:
+      - 80:80
+    cpu_shares: 512