From 8cbf82826c42d118efb0449cc8ad3ec61e85f241 Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Thu, 4 Jul 2024 23:51:18 -0400 Subject: [PATCH 01/68] initial rust rewrite, basic single endpoint fetch working --- .github/workflows/build-check.yml | 7 +- .github/workflows/lint-check.yml | 2 +- .github/workflows/scylla-rust-ci.yml | 31 + .github/workflows/tests-check.yml | 37 - scylla-server-rust/.cargo/config.toml | 2 + scylla-server-rust/Cargo.lock | 3771 +++++++++++++++++ scylla-server-rust/Cargo.toml | 23 + scylla-server-rust/Dockerfile | 4 + scylla-server-rust/README.md | 20 + scylla-server-rust/build.rs | 14 + scylla-server-rust/prisma-cli/Cargo.toml | 9 + scylla-server-rust/prisma-cli/src/main.rs | 3 + .../20240212220042_init/migration.sql | 109 + .../20240212220355_init/migration.sql | 2 + .../prisma/migrations/migration_lock.toml | 3 + scylla-server-rust/prisma/schema.prisma | 64 + scylla-server-rust/shell.nix | 30 + .../src/controller/data_controller.rs | 25 + scylla-server-rust/src/controller/mod.rs | 1 + scylla-server-rust/src/lib.rs | 8 + scylla-server-rust/src/main.rs | 17 + scylla-server-rust/src/proto/serverdata.proto | 8 + .../src/services/data_service.rs | 16 + scylla-server-rust/src/services/mod.rs | 1 + 24 files changed, 4163 insertions(+), 44 deletions(-) create mode 100644 .github/workflows/scylla-rust-ci.yml delete mode 100644 .github/workflows/tests-check.yml create mode 100644 scylla-server-rust/.cargo/config.toml create mode 100755 scylla-server-rust/Cargo.lock create mode 100644 scylla-server-rust/Cargo.toml create mode 100755 scylla-server-rust/Dockerfile create mode 100644 scylla-server-rust/README.md create mode 100644 scylla-server-rust/build.rs create mode 100755 scylla-server-rust/prisma-cli/Cargo.toml create mode 100755 scylla-server-rust/prisma-cli/src/main.rs create mode 100644 scylla-server-rust/prisma/migrations/20240212220042_init/migration.sql create mode 100644 scylla-server-rust/prisma/migrations/20240212220355_init/migration.sql create mode 100644 scylla-server-rust/prisma/migrations/migration_lock.toml create mode 100755 scylla-server-rust/prisma/schema.prisma create mode 100755 scylla-server-rust/shell.nix create mode 100644 scylla-server-rust/src/controller/data_controller.rs create mode 100644 scylla-server-rust/src/controller/mod.rs create mode 100644 scylla-server-rust/src/lib.rs create mode 100755 scylla-server-rust/src/main.rs create mode 100644 scylla-server-rust/src/proto/serverdata.proto create mode 100644 scylla-server-rust/src/services/data_service.rs create mode 100644 scylla-server-rust/src/services/mod.rs diff --git a/.github/workflows/build-check.yml b/.github/workflows/build-check.yml index 966ba71b..ac6fdac4 100644 --- a/.github/workflows/build-check.yml +++ b/.github/workflows/build-check.yml @@ -21,14 +21,9 @@ jobs: run: | sudo apt-get -y update sudo apt-get -y install docker-compose - - name: Generate Angular Dist - run: | - cd ./angular-client - npm install - npm run build - name: Build Docker Compose run: | if ! docker-compose build; then echo "Docker Compose build failed." exit 1 # This will cause the workflow to fail - fi \ No newline at end of file + fi diff --git a/.github/workflows/lint-check.yml b/.github/workflows/lint-check.yml index 8e45f04a..de4a638c 100644 --- a/.github/workflows/lint-check.yml +++ b/.github/workflows/lint-check.yml @@ -24,6 +24,6 @@ jobs: node-version: 16.17.1 cache: 'npm' - name: Install modules - run: npm install && npm run prisma:generate + run: npm install - name: Run linting check run: npm run lint --max-warnings=0 diff --git a/.github/workflows/scylla-rust-ci.yml b/.github/workflows/scylla-rust-ci.yml new file mode 100644 index 00000000..dddf7a73 --- /dev/null +++ b/.github/workflows/scylla-rust-ci.yml @@ -0,0 +1,31 @@ +name: Scylla Rust CI + +on: + push: + branches: + - main + - Develop + pull_request: + branches: + - main + - Develop + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Setup Rust + uses: actions/checkout@v3 + with: + submodules: recursive + - name: Install cargo-audit + run: cargo install cargo-audit + - name: Build + run: cargo build --verbose + - name: Test + run: cargo test --verbose + - name: Clippy + run: cargo clippy --verbose -- -D warnings + - name: Audit + run: cargo audit diff --git a/.github/workflows/tests-check.yml b/.github/workflows/tests-check.yml deleted file mode 100644 index f9874bc5..00000000 --- a/.github/workflows/tests-check.yml +++ /dev/null @@ -1,37 +0,0 @@ -# This workflow will do a clean installation of node dependencies, cache/restore them, build the source code and run tests across different versions of node -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions - -name: Tests - -on: - push: - branches: - - main - - develop - pull_request: - branches: - - main - - develop - - 'feature/**' -jobs: - run-tests: - runs-on: ubuntu-latest - timeout-minutes: 15 - steps: - - name: Checkout repository - uses: actions/checkout@v3 - with: - submodules: true - - name: Set up Node.js - uses: actions/setup-node@v3 - with: - node-version: 16.17.1 - cache: npm - - name: Set Env Variable - run: cd scylla-server && echo "SOURCE_DATABASE_URL=postgresql://postgres:password@localhost:5432/timescaledb" > .env - - name: Start Database - run: docker run -e POSTGRES_HOST_AUTH_METHOD=trust -d -p 5432:5432 --platform=linux/amd64 timescale/timescaledb:2.13.1-pg15 - - name: Install modules - run: npm install && npm run prisma:generate && npm run prisma:migrate:prod - - name: Run tests - run: npm run test diff --git a/scylla-server-rust/.cargo/config.toml b/scylla-server-rust/.cargo/config.toml new file mode 100644 index 00000000..eea34c5d --- /dev/null +++ b/scylla-server-rust/.cargo/config.toml @@ -0,0 +1,2 @@ +[alias] +prisma = "run --package prisma-cli --" \ No newline at end of file diff --git a/scylla-server-rust/Cargo.lock b/scylla-server-rust/Cargo.lock new file mode 100755 index 00000000..8cda45d5 --- /dev/null +++ b/scylla-server-rust/Cargo.lock @@ -0,0 +1,3771 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "addr2line" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "ahash" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" +dependencies = [ + "getrandom 0.2.15", + "once_cell", + "version_check", +] + +[[package]] +name = "aho-corasick" +version = "0.7.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" +dependencies = [ + "memchr", +] + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" + +[[package]] +name = "ascii" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eab1c04a571841102f5345a8fc0f6bb3d31c315dec879b5c6e42e40ce7ffa34e" + +[[package]] +name = "async-trait" +version = "0.1.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.67", +] + +[[package]] +name = "atomic-shim" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67cd4b51d303cf3501c301e8125df442128d3c6d7c69f71b27833d253de47e77" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "autocfg" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" + +[[package]] +name = "axum" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf" +dependencies = [ + "async-trait", + "axum-core", + "bytes", + "futures-util", + "http 1.1.0", + "http-body 1.0.0", + "http-body-util", + "hyper 1.4.0", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper 1.0.1", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a15c63fd72d41492dc4f497196f5da1fb04fb7529e631d73630d1b491e47a2e3" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http 1.1.0", + "http-body 1.0.0", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper 0.1.2", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "backtrace" +version = "0.3.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "base-x" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" + +[[package]] +name = "base36" +version = "0.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9c26bddc1271f7112e5ec797e8eeba6de2de211c1488e506b9500196dbf77c5" +dependencies = [ + "base-x", + "failure", +] + +[[package]] +name = "base64" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff" + +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + +[[package]] +name = "bigdecimal" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6773ddc0eafc0e509fb60e48dff7f450f8e674a0686ae8605e8d9901bd5eefa" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", + "serde", +] + +[[package]] +name = "bit-vec" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" + +[[package]] +name = "block-buffer" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "builtin-psl-connectors" +version = "0.1.0" +source = "git+https://github.com/Brendonovich/prisma-engines?tag=pcr-0.6.10#c4aeef82dbae310e974d6122160c7e3b5fb6df53" +dependencies = [ + "connection-string", + "either", + "enumflags2", + "indoc", + "lsp-types", + "once_cell", + "psl-core", + "regex", +] + +[[package]] +name = "bumpalo" +version = "3.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" + +[[package]] +name = "cc" +version = "1.0.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96c51067fd44124faa7f870b4b1c969379ad32b2ba805aa959430ceaa384f695" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-targets 0.52.5", +] + +[[package]] +name = "colored" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbf2150cce219b664a8a70df7a1f933836724b503f8a413af9365b4dcc4d90b8" +dependencies = [ + "lazy_static", + "windows-sys 0.48.0", +] + +[[package]] +name = "combine" +version = "3.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3da6baa321ec19e1cc41d31bf599f00c783d0517095cdaf0332e3fe8d20680" +dependencies = [ + "ascii", + "byteorder", + "either", + "memchr", + "unreachable", +] + +[[package]] +name = "connection-string" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c4ecb0dc8c35d2c626e45ae70bbfcb1050b302f42bcdf025d913cc0c5a0b443" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "convert_case" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb4a24b1aaf0fd0ce8b45161144d6f42cd91677fd5940fd431183eb023b3a2b8" + +[[package]] +name = "convert_case" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" + +[[package]] +name = "cpufeatures" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "cuid" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51294db11d38eb763c92936c5c88425d0090e27dce21dd15748134af9e53e739" +dependencies = [ + "base36", + "cuid-util", + "cuid2", + "hostname", + "num", + "once_cell", + "rand 0.8.5", +] + +[[package]] +name = "cuid-util" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ea2bfe0336ff1b7ca74819b2df8dfae9afea358aff6b1688baa5c181d8c3713" + +[[package]] +name = "cuid2" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d99cacd52fd67db7490ad051c8c1973fb75520174d69aabbae08c534c9d0e8" +dependencies = [ + "cuid-util", + "num", + "rand 0.8.5", + "sha3", +] + +[[package]] +name = "datamodel-renderer" +version = "0.1.0" +source = "git+https://github.com/Brendonovich/prisma-engines?tag=pcr-0.6.10#c4aeef82dbae310e974d6122160c7e3b5fb6df53" +dependencies = [ + "base64 0.13.1", + "once_cell", + "psl", + "regex", +] + +[[package]] +name = "diagnostics" +version = "0.1.0" +source = "git+https://github.com/Brendonovich/prisma-engines?tag=pcr-0.6.10#c4aeef82dbae310e974d6122160c7e3b5fb6df53" +dependencies = [ + "colored", + "indoc", + "pest", +] + +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "crypto-common", + "subtle", +] + +[[package]] +name = "directories" +version = "4.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f51c5d4ddabd36886dd3e1438cb358cdcb0d7c499cb99cb4ac2e38e18b5cb210" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + +[[package]] +name = "dml" +version = "0.1.0" +source = "git+https://github.com/Brendonovich/prisma-engines?tag=pcr-0.6.10#c4aeef82dbae310e974d6122160c7e3b5fb6df53" +dependencies = [ + "chrono", + "cuid", + "either", + "enumflags2", + "indoc", + "prisma-value", + "psl-core", + "schema-ast", + "serde", + "serde_json", + "uuid", +] + +[[package]] +name = "dmmf" +version = "0.1.0" +source = "git+https://github.com/Brendonovich/prisma-engines?tag=pcr-0.6.10#c4aeef82dbae310e974d6122160c7e3b5fb6df53" +dependencies = [ + "bigdecimal", + "indexmap 1.9.3", + "prisma-models", + "psl", + "schema", + "schema-builder", + "serde", + "serde_json", +] + +[[package]] +name = "dotenv" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" + +[[package]] +name = "either" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dca9240753cf90908d7e4aac30f630662b02aebaa1b58a3cadabdb23385b58b" + +[[package]] +name = "encoding_rs" +version = "0.8.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "endian-type" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" + +[[package]] +name = "enumflags2" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d232db7f5956f3f14313dc2f87985c58bd2c695ce124c8cdd984e08e15ac133d" +dependencies = [ + "enumflags2_derive", +] + +[[package]] +name = "enumflags2_derive" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de0d48a183585823424a4ce1aa132d174a6a81bd540895822eb4c8373a8e49e8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.67", +] + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "errno" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "failure" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86" +dependencies = [ + "backtrace", + "failure_derive", +] + +[[package]] +name = "failure_derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "synstructure", +] + +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + +[[package]] +name = "fastrand" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" + +[[package]] +name = "fixedbitset" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86d4de0081402f5e88cdac65c8dcdcc73118c1a7a465e2a05f0da05843a8ea33" + +[[package]] +name = "flate2" +version = "1.0.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f54427cfd1c7829e2a139fcefea601bf088ebca651d2bf53ebc600eac295dae" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" + +[[package]] +name = "futures-executor" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" + +[[package]] +name = "futures-macro" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.67", +] + +[[package]] +name = "futures-sink" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" + +[[package]] +name = "futures-task" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" + +[[package]] +name = "futures-timer" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" + +[[package]] +name = "futures-util" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.9.0+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", +] + +[[package]] +name = "gimli" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" + +[[package]] +name = "graphql-parser" +version = "0.3.0" +source = "git+https://github.com/prisma/graphql-parser#6a3f58bd879065588e710cb02b5bd30c1ce182c3" +dependencies = [ + "combine", + "indexmap 1.9.3", + "thiserror", +] + +[[package]] +name = "h2" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 0.2.12", + "indexmap 2.2.6", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" +dependencies = [ + "ahash", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +dependencies = [ + "ahash", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "heck" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "hermit-abi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", +] + +[[package]] +name = "home" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "hostname" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867" +dependencies = [ + "libc", + "match_cfg", + "winapi", +] + +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http 0.2.12", + "pin-project-lite", +] + +[[package]] +name = "http-body" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" +dependencies = [ + "bytes", + "http 1.1.0", +] + +[[package]] +name = "http-body-util" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" +dependencies = [ + "bytes", + "futures-util", + "http 1.1.0", + "http-body 1.0.0", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "0.14.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f361cde2f109281a220d4307746cdfd5ee3f410da58a70377762396775634b33" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http 0.2.12", + "http-body 0.4.6", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2 0.5.7", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4fe55fb7a772d59a5ff1dfbff4fe0258d19b89fec4b233e75d35d5d2316badc" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http 1.1.0", + "http-body 1.0.0", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper 0.14.29", + "native-tls", + "tokio", + "tokio-native-tls 0.3.1", +] + +[[package]] +name = "hyper-util" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ab92f4f49ee4fb4f997c784b7a2e0fa70050211e0b6a287f898c3c9785ca956" +dependencies = [ + "bytes", + "futures-util", + "http 1.1.0", + "http-body 1.0.0", + "hyper 1.4.0", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "idna" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + +[[package]] +name = "indexmap" +version = "2.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" +dependencies = [ + "equivalent", + "hashbrown 0.14.5", +] + +[[package]] +name = "indoc" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa799dd5ed20a7e349f3b4639aa80d74549c81716d9ec4f994c9b5815598306" + +[[package]] +name = "instant" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "introspection-connector" +version = "0.1.0" +source = "git+https://github.com/Brendonovich/prisma-engines?tag=pcr-0.6.10#c4aeef82dbae310e974d6122160c7e3b5fb6df53" +dependencies = [ + "anyhow", + "async-trait", + "enumflags2", + "psl", + "serde", + "serde_json", + "thiserror", + "user-facing-errors", +] + +[[package]] +name = "ipnet" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" + +[[package]] +name = "js-sys" +version = "0.3.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a38fc24e30fd564ce974c02bf1d337caddff65be6cc4735a1f7eab22a7440f04" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "json-rpc-api-build" +version = "0.1.0" +source = "git+https://github.com/Brendonovich/prisma-engines?tag=pcr-0.6.10#c4aeef82dbae310e974d6122160c7e3b5fb6df53" +dependencies = [ + "backtrace", + "heck", + "serde", + "toml", +] + +[[package]] +name = "jsonrpc-core" +version = "17.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4467ab6dfa369b69e52bd0692e480c4d117410538526a57a304a0f2250fd95e" +dependencies = [ + "futures", + "futures-executor", + "futures-util", + "log", + "serde", + "serde_derive", + "serde_json", +] + +[[package]] +name = "keccak" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc2af9a1119c51f12a14607e783cb977bde58bc069ff0c3da1095e635d70654" +dependencies = [ + "cpufeatures", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.155" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" + +[[package]] +name = "libredox" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +dependencies = [ + "bitflags 2.5.0", + "libc", +] + +[[package]] +name = "linked-hash-map" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" + +[[package]] +name = "linux-raw-sys" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" + +[[package]] +name = "lock_api" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" + +[[package]] +name = "lru" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999beba7b6e8345721bd280141ed958096a2e4abdf74f67ff4ce49b4b54e47a" +dependencies = [ + "hashbrown 0.12.3", +] + +[[package]] +name = "lru-cache" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c" +dependencies = [ + "linked-hash-map", +] + +[[package]] +name = "lsp-types" +version = "0.91.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2368312c59425dd133cb9a327afee65be0a633a8ce471d248e2202a48f8f68ae" +dependencies = [ + "bitflags 1.3.2", + "serde", + "serde_json", + "serde_repr", + "url", +] + +[[package]] +name = "mach" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa" +dependencies = [ + "libc", +] + +[[package]] +name = "match_cfg" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" + +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest 0.10.7", +] + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "metrics" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e52eb6380b6d2a10eb3434aec0885374490f5b82c8aaf5cd487a183c98be834" +dependencies = [ + "ahash", + "metrics-macros", +] + +[[package]] +name = "metrics" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "142c53885123b68d94108295a09d4afe1a1388ed95b54d5dacd9a454753030f2" +dependencies = [ + "ahash", + "metrics-macros", +] + +[[package]] +name = "metrics-exporter-prometheus" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "953cbbb6f9ba4b9304f4df79b98cdc9d14071ed93065a9fca11c00c5d9181b66" +dependencies = [ + "hyper 0.14.29", + "indexmap 1.9.3", + "ipnet", + "metrics 0.19.0", + "metrics-util 0.13.0", + "parking_lot 0.11.2", + "quanta", + "thiserror", + "tokio", + "tracing", +] + +[[package]] +name = "metrics-macros" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49e30813093f757be5cf21e50389a24dc7dbb22c49f23b7e8f51d69b508a5ffa" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "metrics-util" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65a9e83b833e1d2e07010a386b197c13aa199bbd0fca5cf69bfa147972db890a" +dependencies = [ + "aho-corasick 0.7.20", + "atomic-shim", + "crossbeam-epoch", + "crossbeam-utils", + "hashbrown 0.11.2", + "indexmap 1.9.3", + "metrics 0.18.1", + "num_cpus", + "ordered-float", + "parking_lot 0.11.2", + "quanta", + "radix_trie", + "sketches-ddsketch", +] + +[[package]] +name = "metrics-util" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd1f4b69bef1e2b392b2d4a12902f2af90bb438ba4a66aa222d1023fa6561b50" +dependencies = [ + "atomic-shim", + "crossbeam-epoch", + "crossbeam-utils", + "hashbrown 0.11.2", + "metrics 0.19.0", + "num_cpus", + "parking_lot 0.11.2", + "quanta", + "sketches-ddsketch", +] + +[[package]] +name = "migration-connector" +version = "0.1.0" +source = "git+https://github.com/Brendonovich/prisma-engines?tag=pcr-0.6.10#c4aeef82dbae310e974d6122160c7e3b5fb6df53" +dependencies = [ + "chrono", + "enumflags2", + "introspection-connector", + "psl", + "sha2 0.9.9", + "tracing", + "tracing-error", + "user-facing-errors", +] + +[[package]] +name = "migration-core" +version = "0.1.0" +source = "git+https://github.com/Brendonovich/prisma-engines?tag=pcr-0.6.10#c4aeef82dbae310e974d6122160c7e3b5fb6df53" +dependencies = [ + "async-trait", + "chrono", + "enumflags2", + "json-rpc-api-build", + "jsonrpc-core", + "migration-connector", + "psl", + "serde", + "serde_json", + "sql-migration-connector", + "tokio", + "tracing", + "tracing-futures", + "tracing-subscriber", + "url", + "user-facing-errors", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" +dependencies = [ + "adler", +] + +[[package]] +name = "mio" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" +dependencies = [ + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", + "windows-sys 0.48.0", +] + +[[package]] +name = "mobc" +version = "0.7.3" +source = "git+https://github.com/prisma/mobc?tag=1.0.6#80462c4870a2bf6aab49da15c88c021bae531da8" +dependencies = [ + "async-trait", + "futures-channel", + "futures-core", + "futures-timer", + "futures-util", + "log", + "metrics 0.18.1", + "thiserror", + "tokio", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "native-tls" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8614eb2c83d59d1c8cc974dd3f920198647674a0a035e1af1fa58707e317466" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "nibble_vec" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43" +dependencies = [ + "smallvec", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c165a9ab64cf766f73521c0dd2cfdff64f488b8f0b3e621face3462d3db536d7" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "object" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "576dfe1fc8f9df304abb159d767a29d0476f7750fbf8aa7ad07816004a207434" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + +[[package]] +name = "openssl" +version = "0.10.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" +dependencies = [ + "bitflags 2.5.0", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.67", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c597637d56fbc83893a35eb0dd04b2b8e7a50c91e64e9493e398b5df4fb45fa2" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "opentelemetry" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6105e89802af13fdf48c49d7646d3b533a70e536d818aae7e78ba0433d01acb8" +dependencies = [ + "async-trait", + "crossbeam-channel", + "futures-channel", + "futures-executor", + "futures-util", + "js-sys", + "lazy_static", + "percent-encoding", + "pin-project", + "rand 0.8.5", + "thiserror", + "tokio", +] + +[[package]] +name = "ordered-float" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" +dependencies = [ + "num-traits", +] + +[[package]] +name = "ordermap" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a86ed3f5f244b372d6b1a00b72ef7f8876d0bc6a78a4c9985c53614041512063" + +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core 0.8.6", +] + +[[package]] +name = "parking_lot" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" +dependencies = [ + "lock_api", + "parking_lot_core 0.9.10", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" +dependencies = [ + "cfg-if", + "instant", + "libc", + "redox_syscall 0.2.16", + "smallvec", + "winapi", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.5.2", + "smallvec", + "windows-targets 0.52.5", +] + +[[package]] +name = "parser-database" +version = "0.1.0" +source = "git+https://github.com/Brendonovich/prisma-engines?tag=pcr-0.6.10#c4aeef82dbae310e974d6122160c7e3b5fb6df53" +dependencies = [ + "diagnostics", + "either", + "enumflags2", + "indexmap 1.9.3", + "schema-ast", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "pest" +version = "2.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "560131c633294438da9f7c4b08189194b20946c8274c6b9e38881a7874dc8ee8" +dependencies = [ + "memchr", + "thiserror", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26293c9193fbca7b1a3bf9b79dc1e388e927e6cacaa78b4a3ab705a1d3d41459" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ec22af7d3fb470a85dd2ca96b7c577a1eb4ef6f1683a9fe9a8c16e136c04687" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn 2.0.67", +] + +[[package]] +name = "pest_meta" +version = "2.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7a240022f37c361ec1878d646fc5b7d7c4d28d5946e1a80ad5a7a4f4ca0bdcd" +dependencies = [ + "once_cell", + "pest", + "sha2 0.10.8", +] + +[[package]] +name = "petgraph" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3659d1ee90221741f65dd128d9998311b0e40c5d3c23a62445938214abce4f" +dependencies = [ + "fixedbitset", + "ordermap", +] + +[[package]] +name = "phf" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_shared" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.67", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" + +[[package]] +name = "postgres-native-tls" +version = "0.5.0" +source = "git+https://github.com/pimeys/rust-postgres?rev=064a6a50427542e2c166e870027735aab3b52e77#064a6a50427542e2c166e870027735aab3b52e77" +dependencies = [ + "native-tls", + "tokio", + "tokio-native-tls 0.3.0", + "tokio-postgres", +] + +[[package]] +name = "postgres-protocol" +version = "0.6.4" +source = "git+https://github.com/pimeys/rust-postgres?rev=064a6a50427542e2c166e870027735aab3b52e77#064a6a50427542e2c166e870027735aab3b52e77" +dependencies = [ + "base64 0.13.1", + "byteorder", + "bytes", + "fallible-iterator", + "hmac", + "md-5", + "memchr", + "rand 0.8.5", + "sha2 0.10.8", + "stringprep", +] + +[[package]] +name = "postgres-types" +version = "0.2.4" +source = "git+https://github.com/pimeys/rust-postgres?rev=064a6a50427542e2c166e870027735aab3b52e77#064a6a50427542e2c166e870027735aab3b52e77" +dependencies = [ + "bit-vec", + "bytes", + "chrono", + "fallible-iterator", + "postgres-protocol", + "serde", + "serde_json", + "uuid", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "prisma-cli" +version = "0.1.0" +dependencies = [ + "prisma-client-rust-cli", +] + +[[package]] +name = "prisma-client-rust" +version = "0.6.11" +source = "git+https://github.com/Brendonovich/prisma-client-rust?tag=0.6.11#3ac68d0052533d3ae0332d93a56a8ad169c2ee18" +dependencies = [ + "base64 0.13.1", + "bigdecimal", + "chrono", + "diagnostics", + "dml", + "dmmf", + "dotenv", + "futures", + "indexmap 1.9.3", + "migration-core", + "paste", + "prisma-client-rust-macros", + "prisma-models", + "psl", + "query-connector", + "query-core", + "schema", + "serde", + "serde-value", + "serde_json", + "thiserror", + "user-facing-errors", + "uuid", +] + +[[package]] +name = "prisma-client-rust-cli" +version = "0.6.11" +source = "git+https://github.com/Brendonovich/prisma-client-rust?tag=0.6.11#3ac68d0052533d3ae0332d93a56a8ad169c2ee18" +dependencies = [ + "directories", + "flate2", + "http 0.2.12", + "prisma-client-rust-sdk", + "proc-macro2", + "quote", + "regex", + "reqwest", + "serde", + "serde_json", + "serde_path_to_error", + "syn 1.0.109", + "thiserror", +] + +[[package]] +name = "prisma-client-rust-macros" +version = "0.6.11" +source = "git+https://github.com/Brendonovich/prisma-client-rust?tag=0.6.11#3ac68d0052533d3ae0332d93a56a8ad169c2ee18" +dependencies = [ + "convert_case 0.6.0", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "prisma-client-rust-sdk" +version = "0.6.11" +source = "git+https://github.com/Brendonovich/prisma-client-rust?tag=0.6.11#3ac68d0052533d3ae0332d93a56a8ad169c2ee18" +dependencies = [ + "convert_case 0.5.0", + "dml", + "dmmf", + "prisma-models", + "proc-macro2", + "psl", + "query-core", + "quote", + "request-handlers", + "serde", + "serde_json", + "serde_path_to_error", + "syn 1.0.109", + "thiserror", +] + +[[package]] +name = "prisma-models" +version = "0.0.0" +source = "git+https://github.com/Brendonovich/prisma-engines?tag=pcr-0.6.10#c4aeef82dbae310e974d6122160c7e3b5fb6df53" +dependencies = [ + "bigdecimal", + "chrono", + "itertools", + "once_cell", + "prisma-value", + "psl", + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "prisma-value" +version = "0.1.0" +source = "git+https://github.com/Brendonovich/prisma-engines?tag=pcr-0.6.10#c4aeef82dbae310e974d6122160c7e3b5fb6df53" +dependencies = [ + "base64 0.12.3", + "bigdecimal", + "chrono", + "once_cell", + "regex", + "serde", + "serde_json", + "uuid", +] + +[[package]] +name = "proc-macro2" +version = "1.0.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "protobuf" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df67496db1a89596beaced1579212e9b7c53c22dca1d9745de00ead76573d514" +dependencies = [ + "once_cell", + "protobuf-support", + "thiserror", +] + +[[package]] +name = "protobuf-codegen" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eab09155fad2d39333d3796f67845d43e29b266eea74f7bc93f153f707f126dc" +dependencies = [ + "anyhow", + "once_cell", + "protobuf", + "protobuf-parse", + "regex", + "tempfile", + "thiserror", +] + +[[package]] +name = "protobuf-parse" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a16027030d4ec33e423385f73bb559821827e9ec18c50e7874e4d6de5a4e96f" +dependencies = [ + "anyhow", + "indexmap 2.2.6", + "log", + "protobuf", + "protobuf-support", + "tempfile", + "thiserror", + "which", +] + +[[package]] +name = "protobuf-support" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e2d30ab1878b2e72d1e2fc23ff5517799c9929e2cf81a8516f9f4dcf2b9cf3" +dependencies = [ + "thiserror", +] + +[[package]] +name = "psl" +version = "0.1.0" +source = "git+https://github.com/Brendonovich/prisma-engines?tag=pcr-0.6.10#c4aeef82dbae310e974d6122160c7e3b5fb6df53" +dependencies = [ + "builtin-psl-connectors", + "dml", + "psl-core", +] + +[[package]] +name = "psl-core" +version = "0.1.0" +source = "git+https://github.com/Brendonovich/prisma-engines?tag=pcr-0.6.10#c4aeef82dbae310e974d6122160c7e3b5fb6df53" +dependencies = [ + "bigdecimal", + "chrono", + "diagnostics", + "enumflags2", + "indoc", + "itertools", + "lsp-types", + "once_cell", + "parser-database", + "prisma-value", + "regex", + "schema-ast", + "serde", + "serde_json", + "url", +] + +[[package]] +name = "quaint" +version = "0.2.0-alpha.13" +source = "git+https://github.com/Brendonovich/quaint?tag=0.6.5#c502995f5e9dd4a86d80fa27f63e641bd6bbc6f2" +dependencies = [ + "async-trait", + "base64 0.12.3", + "bigdecimal", + "bit-vec", + "byteorder", + "bytes", + "chrono", + "connection-string", + "futures", + "hex", + "lru-cache", + "metrics 0.18.1", + "mobc", + "native-tls", + "num_cpus", + "percent-encoding", + "postgres-native-tls", + "postgres-types", + "serde_json", + "sqlformat", + "thiserror", + "tokio", + "tokio-postgres", + "tracing", + "tracing-core", + "url", + "uuid", +] + +[[package]] +name = "quanta" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20afe714292d5e879d8b12740aa223c6a88f118af41870e8b6196e39a02238a8" +dependencies = [ + "crossbeam-utils", + "libc", + "mach", + "once_cell", + "raw-cpuid", + "wasi 0.10.2+wasi-snapshot-preview1", + "web-sys", + "winapi", +] + +[[package]] +name = "query-connector" +version = "0.1.0" +source = "git+https://github.com/Brendonovich/prisma-engines?tag=pcr-0.6.10#c4aeef82dbae310e974d6122160c7e3b5fb6df53" +dependencies = [ + "anyhow", + "async-trait", + "chrono", + "futures", + "indexmap 1.9.3", + "itertools", + "prisma-models", + "prisma-value", + "serde", + "serde_json", + "thiserror", + "user-facing-errors", + "uuid", +] + +[[package]] +name = "query-core" +version = "0.1.0" +source = "git+https://github.com/Brendonovich/prisma-engines?tag=pcr-0.6.10#c4aeef82dbae310e974d6122160c7e3b5fb6df53" +dependencies = [ + "async-trait", + "base64 0.12.3", + "bigdecimal", + "chrono", + "connection-string", + "crossbeam-queue", + "cuid", + "enumflags2", + "futures", + "indexmap 1.9.3", + "itertools", + "lazy_static", + "lru", + "once_cell", + "opentelemetry", + "parking_lot 0.12.3", + "petgraph", + "pin-utils", + "prisma-models", + "prisma-value", + "psl", + "query-connector", + "query-engine-metrics", + "schema", + "schema-builder", + "serde", + "serde_json", + "sql-query-connector", + "thiserror", + "tokio", + "tracing", + "tracing-futures", + "tracing-opentelemetry", + "tracing-subscriber", + "url", + "user-facing-errors", + "uuid", +] + +[[package]] +name = "query-engine-metrics" +version = "0.1.0" +source = "git+https://github.com/Brendonovich/prisma-engines?tag=pcr-0.6.10#c4aeef82dbae310e974d6122160c7e3b5fb6df53" +dependencies = [ + "metrics 0.18.1", + "metrics-exporter-prometheus", + "metrics-util 0.12.1", + "once_cell", + "parking_lot 0.12.3", + "serde", + "serde_json", + "tracing", + "tracing-futures", + "tracing-subscriber", +] + +[[package]] +name = "quote" +version = "1.0.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "radix_trie" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd" +dependencies = [ + "endian-type", + "nibble_vec", +] + +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom 0.1.16", + "libc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom 0.1.16", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.15", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", +] + +[[package]] +name = "raw-cpuid" +version = "10.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "redox_syscall" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c82cf8cff14456045f55ec4241383baeff27af886adb72ffb2162f99911de0fd" +dependencies = [ + "bitflags 2.5.0", +] + +[[package]] +name = "redox_users" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd283d9651eeda4b2a83a43c1c91b266c40fd76ecd39a50a8c630ae69dc72891" +dependencies = [ + "getrandom 0.2.15", + "libredox", + "thiserror", +] + +[[package]] +name = "regex" +version = "1.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" +dependencies = [ + "aho-corasick 1.1.3", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" +dependencies = [ + "aho-corasick 1.1.3", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" + +[[package]] +name = "request-handlers" +version = "0.1.0" +source = "git+https://github.com/Brendonovich/prisma-engines?tag=pcr-0.6.10#c4aeef82dbae310e974d6122160c7e3b5fb6df53" +dependencies = [ + "bigdecimal", + "connection-string", + "dmmf", + "futures", + "graphql-parser", + "indexmap 1.9.3", + "itertools", + "psl", + "query-core", + "serde", + "serde_json", + "thiserror", + "tracing", + "url", + "user-facing-errors", +] + +[[package]] +name = "reqwest" +version = "0.11.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" +dependencies = [ + "base64 0.21.7", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.29", + "hyper-tls", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls-pemfile", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper 0.1.2", + "system-configuration", + "tokio", + "tokio-native-tls 0.3.1", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" + +[[package]] +name = "rustix" +version = "0.38.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" +dependencies = [ + "bitflags 2.5.0", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64 0.21.7", +] + +[[package]] +name = "rustversion" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" + +[[package]] +name = "ryu" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" + +[[package]] +name = "schannel" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "schema" +version = "0.1.0" +source = "git+https://github.com/Brendonovich/prisma-engines?tag=pcr-0.6.10#c4aeef82dbae310e974d6122160c7e3b5fb6df53" +dependencies = [ + "once_cell", + "prisma-models", + "psl", +] + +[[package]] +name = "schema-ast" +version = "0.1.0" +source = "git+https://github.com/Brendonovich/prisma-engines?tag=pcr-0.6.10#c4aeef82dbae310e974d6122160c7e3b5fb6df53" +dependencies = [ + "diagnostics", + "pest", + "pest_derive", +] + +[[package]] +name = "schema-builder" +version = "0.1.0" +source = "git+https://github.com/Brendonovich/prisma-engines?tag=pcr-0.6.10#c4aeef82dbae310e974d6122160c7e3b5fb6df53" +dependencies = [ + "itertools", + "lazy_static", + "once_cell", + "prisma-models", + "psl", + "schema", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "scylla-server-rust" +version = "0.0.1" +dependencies = [ + "axum", + "prisma-client-rust", + "protobuf", + "protobuf-codegen", + "serde", + "tokio", +] + +[[package]] +name = "security-framework" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c627723fd09706bacdb5cf41499e95098555af3c3c29d014dc3c458ef6be11c0" +dependencies = [ + "bitflags 2.5.0", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "317936bbbd05227752583946b9e66d7ce3b489f84e11a94a510b4437fef407d7" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.203" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde-value" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a1a3341211875ef120e117ea7fd5228530ae7e7036a779fdc9117be6b3282c" +dependencies = [ + "ordered-float", + "serde", +] + +[[package]] +name = "serde_derive" +version = "1.0.203" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.67", +] + +[[package]] +name = "serde_json" +version = "1.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3" +dependencies = [ + "indexmap 2.2.6", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af99884400da37c88f5e9146b7f1fd0fbcae8f6eec4e9da38b67d05486f814a6" +dependencies = [ + "itoa", + "serde", +] + +[[package]] +name = "serde_repr" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.67", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha2" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" +dependencies = [ + "block-buffer 0.9.0", + "cfg-if", + "cpufeatures", + "digest 0.9.0", + "opaque-debug", +] + +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest 0.10.7", +] + +[[package]] +name = "sha3" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75872d278a8f37ef87fa0ddbda7802605cb18344497949862c0d4dcb291eba60" +dependencies = [ + "digest 0.10.7", + "keccak", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" +dependencies = [ + "libc", +] + +[[package]] +name = "siphasher" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" + +[[package]] +name = "sketches-ddsketch" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04d2ecae5fcf33b122e2e6bd520a57ccf152d2dde3b38c71039df1a6867264ee" + +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + +[[package]] +name = "socket2" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "socket2" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "sql-ddl" +version = "0.1.0" +source = "git+https://github.com/Brendonovich/prisma-engines?tag=pcr-0.6.10#c4aeef82dbae310e974d6122160c7e3b5fb6df53" + +[[package]] +name = "sql-introspection-connector" +version = "0.1.0" +source = "git+https://github.com/Brendonovich/prisma-engines?tag=pcr-0.6.10#c4aeef82dbae310e974d6122160c7e3b5fb6df53" +dependencies = [ + "anyhow", + "async-trait", + "bigdecimal", + "datamodel-renderer", + "enumflags2", + "introspection-connector", + "once_cell", + "psl", + "quaint", + "regex", + "serde", + "serde_json", + "sql-schema-describer", + "thiserror", + "tracing", + "tracing-futures", + "user-facing-errors", +] + +[[package]] +name = "sql-migration-connector" +version = "0.1.0" +source = "git+https://github.com/Brendonovich/prisma-engines?tag=pcr-0.6.10#c4aeef82dbae310e974d6122160c7e3b5fb6df53" +dependencies = [ + "chrono", + "connection-string", + "either", + "enumflags2", + "indoc", + "migration-connector", + "once_cell", + "psl", + "quaint", + "regex", + "serde_json", + "sql-ddl", + "sql-introspection-connector", + "sql-schema-describer", + "tokio", + "tracing", + "tracing-futures", + "url", + "user-facing-errors", + "uuid", +] + +[[package]] +name = "sql-query-connector" +version = "0.1.0" +source = "git+https://github.com/Brendonovich/prisma-engines?tag=pcr-0.6.10#c4aeef82dbae310e974d6122160c7e3b5fb6df53" +dependencies = [ + "anyhow", + "async-trait", + "bigdecimal", + "chrono", + "cuid", + "futures", + "itertools", + "once_cell", + "opentelemetry", + "prisma-models", + "prisma-value", + "psl", + "quaint", + "query-connector", + "rand 0.7.3", + "serde", + "serde_json", + "socket2 0.4.10", + "thiserror", + "tokio", + "tracing", + "tracing-futures", + "tracing-opentelemetry", + "user-facing-errors", + "uuid", +] + +[[package]] +name = "sql-schema-describer" +version = "0.1.0" +source = "git+https://github.com/Brendonovich/prisma-engines?tag=pcr-0.6.10#c4aeef82dbae310e974d6122160c7e3b5fb6df53" +dependencies = [ + "async-trait", + "bigdecimal", + "enumflags2", + "indexmap 1.9.3", + "indoc", + "once_cell", + "psl", + "quaint", + "regex", + "serde", + "tracing", + "tracing-error", + "tracing-futures", +] + +[[package]] +name = "sqlformat" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f895e3734318cc55f1fe66258926c9b910c124d47520339efecbb6c59cec7c1f" +dependencies = [ + "nom", + "unicode_categories", +] + +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + +[[package]] +name = "subtle" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d0208408ba0c3df17ed26eb06992cb1a1268d41b2c0e12e65203fbe3972cee5" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff8655ed1d86f3af4ee3fd3263786bc14245ad17c4c7e85ba7187fb3ae028c90" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + +[[package]] +name = "sync_wrapper" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" + +[[package]] +name = "synstructure" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "unicode-xid", +] + +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tempfile" +version = "3.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" +dependencies = [ + "cfg-if", + "fastrand", + "rustix", + "windows-sys 0.52.0", +] + +[[package]] +name = "thiserror" +version = "1.0.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.67", +] + +[[package]] +name = "thread_local" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" +dependencies = [ + "cfg-if", + "once_cell", +] + +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba4f4a02a7a80d6f274636f0aa95c7e383b912d41fe721a31f29e29698585a4a" +dependencies = [ + "backtrace", + "bytes", + "libc", + "mio", + "num_cpus", + "parking_lot 0.12.3", + "pin-project-lite", + "signal-hook-registry", + "socket2 0.5.7", + "tokio-macros", + "windows-sys 0.48.0", +] + +[[package]] +name = "tokio-macros" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.67", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.0" +source = "git+https://github.com/pimeys/tls?branch=vendored-openssl#6d0e6fc7a4bf6f290b1033764b47cb3f26d7fceb" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-postgres" +version = "0.7.7" +source = "git+https://github.com/pimeys/rust-postgres?rev=064a6a50427542e2c166e870027735aab3b52e77#064a6a50427542e2c166e870027735aab3b52e77" +dependencies = [ + "async-trait", + "byteorder", + "bytes", + "fallible-iterator", + "futures-channel", + "futures-util", + "log", + "parking_lot 0.12.3", + "percent-encoding", + "phf", + "pin-project-lite", + "postgres-protocol", + "postgres-types", + "socket2 0.4.10", + "tokio", + "tokio-util", +] + +[[package]] +name = "tokio-util" +version = "0.7.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "toml" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" +dependencies = [ + "serde", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "pin-project", + "pin-project-lite", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + +[[package]] +name = "tracing" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.67", +] + +[[package]] +name = "tracing-core" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-error" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d686ec1c0f384b1277f097b2f279a2ecc11afe8c133c1aabf036a27cb4cd206e" +dependencies = [ + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "tracing-futures" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" +dependencies = [ + "pin-project", + "tracing", +] + +[[package]] +name = "tracing-log" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f751112709b4e791d8ce53e32c4ed2d353565a795ce84da2285393f41557bdf2" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-opentelemetry" +version = "0.17.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbbe89715c1dbbb790059e2565353978564924ee85017b5fff365c872ff6721f" +dependencies = [ + "once_cell", + "opentelemetry", + "tracing", + "tracing-core", + "tracing-log 0.1.4", + "tracing-subscriber", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" +dependencies = [ + "nu-ansi-term", + "sharded-slab", + "smallvec", + "thread_local", + "tracing-core", + "tracing-log 0.2.0", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + +[[package]] +name = "ucd-trie" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" + +[[package]] +name = "unicode-bidi" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "unicode-normalization" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4259d9d4425d9f0661581b804cb85fe66a4c631cadd8f490d1c13a35d5d9291" + +[[package]] +name = "unicode-segmentation" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" + +[[package]] +name = "unicode-xid" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" + +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + +[[package]] +name = "unreachable" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56" +dependencies = [ + "void", +] + +[[package]] +name = "url" +version = "2.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "user-facing-error-macros" +version = "0.1.0" +source = "git+https://github.com/Brendonovich/prisma-engines?tag=pcr-0.6.10#c4aeef82dbae310e974d6122160c7e3b5fb6df53" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "user-facing-errors" +version = "0.1.0" +source = "git+https://github.com/Brendonovich/prisma-engines?tag=pcr-0.6.10#c4aeef82dbae310e974d6122160c7e3b5fb6df53" +dependencies = [ + "backtrace", + "indoc", + "quaint", + "serde", + "serde_json", + "tracing", + "user-facing-error-macros", +] + +[[package]] +name = "uuid" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a183cf7feeba97b4dd1c0d46788634f6221d87fa961b305bed08c851829efcc0" +dependencies = [ + "getrandom 0.2.15", + "serde", +] + +[[package]] +name = "valuable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "void" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + +[[package]] +name = "wasi" +version = "0.10.2+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25f1af7423d8588a3d840681122e72e6a24ddbcb3f0ec385cac0d12d24256c06" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b21c0df030f5a177f3cba22e9bc4322695ec43e7257d865302900290bcdedca" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2", + "quote", + "syn 1.0.109", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2eb6ec270a31b1d3c7e266b999739109abce8b6c87e4b31fcfcd788b65267395" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4203d69e40a52ee523b2529a773d5ffc1dc0071801c87b3d270b471b80ed01" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa8a30d46208db204854cadbb5d4baf5fcf8071ba5bf48190c3e59937962ebc" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d958d035c4438e28c70e4321a2911302f10135ce78a9c7834c0cab4123d06a2" + +[[package]] +name = "web-sys" +version = "0.3.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c060b319f29dd25724f09a2ba1418f142f539b2be99fbf4d2d5a8f7330afb8eb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets 0.52.5", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.5", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" +dependencies = [ + "windows_aarch64_gnullvm 0.52.5", + "windows_aarch64_msvc 0.52.5", + "windows_i686_gnu 0.52.5", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.5", + "windows_x86_64_gnu 0.52.5", + "windows_x86_64_gnullvm 0.52.5", + "windows_x86_64_msvc 0.52.5", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" + +[[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] diff --git a/scylla-server-rust/Cargo.toml b/scylla-server-rust/Cargo.toml new file mode 100644 index 00000000..869de2ba --- /dev/null +++ b/scylla-server-rust/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "scylla-server-rust" +version = "0.0.1" +edition = "2021" + +[dependencies] +prisma-client-rust = {git = "https://github.com/Brendonovich/prisma-client-rust", tag = "0.6.11", default-features = false,features = ["postgresql"] } +serde = "1.0.203" +protobuf-codegen = "3.3.0" +protobuf = "3.3.0" +tokio = { version = "1.38.0", features = ["full"] } +axum = "0.7.5" + +[workspace] +members = ["prisma-cli"] + +[build-dependencies] +protobuf-codegen = "3.3.0" + +[profile.release] +lto = true +codegen-units = 1 +panic = "abort" diff --git a/scylla-server-rust/Dockerfile b/scylla-server-rust/Dockerfile new file mode 100755 index 00000000..2f03ccd9 --- /dev/null +++ b/scylla-server-rust/Dockerfile @@ -0,0 +1,4 @@ +FROM rust:latest + +WORKDIR /usr/src/myapp + diff --git a/scylla-server-rust/README.md b/scylla-server-rust/README.md new file mode 100644 index 00000000..fe1d523f --- /dev/null +++ b/scylla-server-rust/README.md @@ -0,0 +1,20 @@ +# Scylla Server (Rust) + + +### Generate prisma + +``` +cargo prisma generate +``` + +### Run the app + +``` +# in argos proper +docker compose run -P odyssey-timescale +``` + +``` +# in this directory +SOURCE_DATABASE_URL=postgresql://postgres:password@127.0.0.1:5432/timescaledb cargo run +``` \ No newline at end of file diff --git a/scylla-server-rust/build.rs b/scylla-server-rust/build.rs new file mode 100644 index 00000000..a57e70ec --- /dev/null +++ b/scylla-server-rust/build.rs @@ -0,0 +1,14 @@ +/* Prebuild script */ +fn main() { + println!("cargo:rerun-if-changed=src/proto"); + + protobuf_codegen::Codegen::new() + .pure() + // All inputs and imports from the inputs must reside in `includes` directories. + .includes(["src/proto"]) + // Inputs must reside in some of include paths. + .input("src/proto/serverdata.proto") + // Specify output directory relative to Cargo output directory. + .out_dir("src") + .run_from_script(); +} diff --git a/scylla-server-rust/prisma-cli/Cargo.toml b/scylla-server-rust/prisma-cli/Cargo.toml new file mode 100755 index 00000000..a81d9964 --- /dev/null +++ b/scylla-server-rust/prisma-cli/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "prisma-cli" +version = "0.1.0" +edition = "2021" + +[dependencies] +prisma-client-rust-cli = { git = "https://github.com/Brendonovich/prisma-client-rust", tag = "0.6.11", features = [ + "postgresql" +], default-features = false } \ No newline at end of file diff --git a/scylla-server-rust/prisma-cli/src/main.rs b/scylla-server-rust/prisma-cli/src/main.rs new file mode 100755 index 00000000..f7580155 --- /dev/null +++ b/scylla-server-rust/prisma-cli/src/main.rs @@ -0,0 +1,3 @@ +fn main() { + prisma_client_rust_cli::run(); +} diff --git a/scylla-server-rust/prisma/migrations/20240212220042_init/migration.sql b/scylla-server-rust/prisma/migrations/20240212220042_init/migration.sql new file mode 100644 index 00000000..a351b84f --- /dev/null +++ b/scylla-server-rust/prisma/migrations/20240212220042_init/migration.sql @@ -0,0 +1,109 @@ +-- CreateTable +CREATE TABLE "run" ( + "id" SERIAL NOT NULL, + "locationName" TEXT, + "driverName" TEXT, + "systemName" TEXT, + "time" TIMESTAMPTZ NOT NULL, + + CONSTRAINT "run_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "location" ( + "name" TEXT NOT NULL, + "latitude" DOUBLE PRECISION NOT NULL, + "longitude" DOUBLE PRECISION NOT NULL, + "radius" DOUBLE PRECISION NOT NULL, + + CONSTRAINT "location_pkey" PRIMARY KEY ("name") +); + +-- CreateTable +CREATE TABLE "driver" ( + "username" TEXT NOT NULL, + + CONSTRAINT "driver_pkey" PRIMARY KEY ("username") +); + +-- CreateTable +CREATE TABLE "system" ( + "name" TEXT NOT NULL, + + CONSTRAINT "system_pkey" PRIMARY KEY ("name") +); + +-- CreateTable +CREATE TABLE "data" ( + "id" TEXT NOT NULL, + "values" DOUBLE PRECISION[], + "dataTypeName" TEXT NOT NULL, + "time" TIMESTAMPTZ NOT NULL, + "runId" INTEGER NOT NULL +); + +-- CreateTable +CREATE TABLE "dataType" ( + "name" TEXT NOT NULL, + "unit" TEXT NOT NULL, + "nodeName" TEXT NOT NULL, + + CONSTRAINT "dataType_pkey" PRIMARY KEY ("name") +); + +-- CreateTable +CREATE TABLE "node" ( + "name" TEXT NOT NULL, + + CONSTRAINT "node_pkey" PRIMARY KEY ("name") +); + +-- CreateIndex +CREATE UNIQUE INDEX "run_id_key" ON "run"("id"); + +-- CreateIndex +CREATE UNIQUE INDEX "location_name_key" ON "location"("name"); + +-- CreateIndex +CREATE UNIQUE INDEX "driver_username_key" ON "driver"("username"); + +-- CreateIndex +CREATE UNIQUE INDEX "system_name_key" ON "system"("name"); + +-- CreateIndex +CREATE UNIQUE INDEX "data_id_time_key" ON "data"("id", "time"); + +SELECT * FROM create_hypertable('data', by_range('time')); +SELECT * FROM add_dimension('data', by_hash('id', 4)); + +-- CreateIndex +CREATE UNIQUE INDEX "dataType_name_key" ON "dataType"("name"); + +-- CreateIndex +CREATE UNIQUE INDEX "node_name_key" ON "node"("name"); + +-- AddForeignKey +ALTER TABLE "run" ADD CONSTRAINT "run_locationName_fkey" FOREIGN KEY ("locationName") REFERENCES "location"("name") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "run" ADD CONSTRAINT "run_driverName_fkey" FOREIGN KEY ("driverName") REFERENCES "driver"("username") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "run" ADD CONSTRAINT "run_systemName_fkey" FOREIGN KEY ("systemName") REFERENCES "system"("name") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "data" ADD CONSTRAINT "data_dataTypeName_fkey" FOREIGN KEY ("dataTypeName") REFERENCES "dataType"("name") ON DELETE RESTRICT ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "data" ADD CONSTRAINT "data_runId_fkey" FOREIGN KEY ("runId") REFERENCES "run"("id") ON DELETE RESTRICT ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "dataType" ADD CONSTRAINT "dataType_nodeName_fkey" FOREIGN KEY ("nodeName") REFERENCES "node"("name") ON DELETE RESTRICT ON UPDATE CASCADE; + +ALTER TABLE "data" SET (timescaledb.compress, + timescaledb.compress_orderby = 'time DESC', + timescaledb.compress_segmentby = '"runId", "dataTypeName"', + timescaledb.compress_chunk_time_interval='24 hours' +); + +SELECT add_compression_policy('data', compress_after => INTERVAL '1d'); \ No newline at end of file diff --git a/scylla-server-rust/prisma/migrations/20240212220355_init/migration.sql b/scylla-server-rust/prisma/migrations/20240212220355_init/migration.sql new file mode 100644 index 00000000..404aedb6 --- /dev/null +++ b/scylla-server-rust/prisma/migrations/20240212220355_init/migration.sql @@ -0,0 +1,2 @@ +-- DropIndex +DROP INDEX "data_time_idx"; diff --git a/scylla-server-rust/prisma/migrations/migration_lock.toml b/scylla-server-rust/prisma/migrations/migration_lock.toml new file mode 100644 index 00000000..fbffa92c --- /dev/null +++ b/scylla-server-rust/prisma/migrations/migration_lock.toml @@ -0,0 +1,3 @@ +# Please do not edit this file manually +# It should be added in your version-control system (i.e. Git) +provider = "postgresql" \ No newline at end of file diff --git a/scylla-server-rust/prisma/schema.prisma b/scylla-server-rust/prisma/schema.prisma new file mode 100755 index 00000000..29a4aaa5 --- /dev/null +++ b/scylla-server-rust/prisma/schema.prisma @@ -0,0 +1,64 @@ +generator client { + provider = "cargo prisma" + +} + +datasource db { + provider = "postgresql" + url = env("SOURCE_DATABASE_URL") +} + +model run { + id Int @id @unique @default(autoincrement()) + locationName String? + location location? @relation(fields: [locationName], references: [name]) + driverName String? + driver driver? @relation(fields: [driverName], references: [username]) + systemName String? + system system? @relation(fields: [systemName], references: [name]) + data data[] + time DateTime @db.Timestamptz() +} + +model location { + name String @id @unique + latitude Float + longitude Float + radius Float + runs run[] +} + +model driver { + username String @id @unique + runs run[] +} + +model system { + name String @id @unique + runs run[] +} + +model data { + id String @default(uuid()) + values Float[] + dataTypeName String + dataType dataType @relation(fields: [dataTypeName], references: [name]) + time DateTime @db.Timestamptz() + runId Int + run run @relation(fields: [runId], references: [id]) + + @@unique([id, time]) +} + +model dataType { + name String @id @unique + unit String + data data[] + node node @relation(fields: [nodeName], references: [name]) + nodeName String +} + +model node { + name String @id @unique + dataTypes dataType[] +} \ No newline at end of file diff --git a/scylla-server-rust/shell.nix b/scylla-server-rust/shell.nix new file mode 100755 index 00000000..0db6465f --- /dev/null +++ b/scylla-server-rust/shell.nix @@ -0,0 +1,30 @@ +# ln -s /home/jack/Documents/nixos/shell-rust.nix /home/jack/Projects/NER/Calypso/shell.nix + +{ + pkgs ? import { }, +}: +pkgs.mkShell { + nativeBuildInputs = with pkgs; [ + rustc + cargo + gcc + rustfmt + clippy + + # for paho-mqtt + openssl + pkg-config + cmake + python3 + python311Packages.ruamel-yaml-clib +python311Packages.ruamel-base +python311Packages.ruamel-yaml + # rust + rust-analyzer + ]; + + # Certain Rust tools won't work without this + # This can also be fixed by using oxalica/rust-overlay and specifying the rust-src extension + # See https://discourse.nixos.org/t/rust-src-not-found-and-other-misadventures-of-developing-rust-on-nixos/11570/3?u=samuela. for more details. + RUST_SRC_PATH = "${pkgs.rust.packages.stable.rustPlatform.rustLibSrc}"; +} diff --git a/scylla-server-rust/src/controller/data_controller.rs b/scylla-server-rust/src/controller/data_controller.rs new file mode 100644 index 00000000..8268f50e --- /dev/null +++ b/scylla-server-rust/src/controller/data_controller.rs @@ -0,0 +1,25 @@ +use axum::{extract::Path, http::StatusCode, Json}; +use serde::Serialize; + +use crate::{services::data_service, Database}; + +#[derive(Serialize)] +pub struct DataSend { + values: Vec, + time: i64, +} + +pub async fn get_data( + db: Database, + Path((data_type_name, run_id)): Path<(String, String)>, +) -> Result>, StatusCode> { + let data = data_service::get_data(db, data_type_name, run_id.parse::().unwrap()).await; + let transformed_data: Vec = data + .iter() + .map(|x| DataSend { + values: x.values.iter().map(|f| f.to_string()).collect(), + time: x.time.timestamp_millis(), + }) + .collect(); + Ok(Json::from(transformed_data)) +} diff --git a/scylla-server-rust/src/controller/mod.rs b/scylla-server-rust/src/controller/mod.rs new file mode 100644 index 00000000..c9cd4b5a --- /dev/null +++ b/scylla-server-rust/src/controller/mod.rs @@ -0,0 +1 @@ +pub mod data_controller; diff --git a/scylla-server-rust/src/lib.rs b/scylla-server-rust/src/lib.rs new file mode 100644 index 00000000..837b75e1 --- /dev/null +++ b/scylla-server-rust/src/lib.rs @@ -0,0 +1,8 @@ +pub mod controller; +pub mod services; + +#[allow(clippy::all)] +#[allow(warnings)] +pub mod prisma; + +pub type Database = axum::Extension>; diff --git a/scylla-server-rust/src/main.rs b/scylla-server-rust/src/main.rs new file mode 100755 index 00000000..edff0410 --- /dev/null +++ b/scylla-server-rust/src/main.rs @@ -0,0 +1,17 @@ +use std::sync::Arc; + +use axum::{routing::get, Extension, Router}; +use scylla_server_rust::{controller, prisma::PrismaClient}; + +#[tokio::main] +async fn main() { + let client = Arc::new(PrismaClient::_builder().build().await.unwrap()); + + let app = Router::new().route( + "/data/:dataTypeName/:runId", + get(controller::data_controller::get_data).layer(Extension(client)), + ); + + let listener = tokio::net::TcpListener::bind("0.0.0.0:8000").await.unwrap(); + axum::serve(listener, app).await.unwrap(); +} diff --git a/scylla-server-rust/src/proto/serverdata.proto b/scylla-server-rust/src/proto/serverdata.proto new file mode 100644 index 00000000..98f812f0 --- /dev/null +++ b/scylla-server-rust/src/proto/serverdata.proto @@ -0,0 +1,8 @@ +syntax = "proto3"; + +package serverdata.v1; + +message ServerData { + repeated string value = 1; + string unit = 2; +} diff --git a/scylla-server-rust/src/services/data_service.rs b/scylla-server-rust/src/services/data_service.rs new file mode 100644 index 00000000..7abc06c5 --- /dev/null +++ b/scylla-server-rust/src/services/data_service.rs @@ -0,0 +1,16 @@ +use crate::{prisma, Database}; + +pub async fn get_data( + db: Database, + data_type_name: String, + run_id: i32, +) -> Vec { + db.data() + .find_many(vec![ + prisma::data::data_type_name::equals(data_type_name), + prisma::data::run_id::equals(run_id), + ]) + .exec() + .await + .unwrap() +} diff --git a/scylla-server-rust/src/services/mod.rs b/scylla-server-rust/src/services/mod.rs new file mode 100644 index 00000000..23e6bc6a --- /dev/null +++ b/scylla-server-rust/src/services/mod.rs @@ -0,0 +1 @@ +pub mod data_service; From 0e73a092996537533ee0a2944212bbd4c6511763 Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Fri, 5 Jul 2024 22:59:00 -0400 Subject: [PATCH 02/68] finish services, controllers --- scylla-server-rust/Cargo.lock | 1 + scylla-server-rust/Cargo.toml | 1 + .../src/controller/data_controller.rs | 45 +++++++++----- .../src/controller/data_type_controller.rs | 29 +++++++++ .../src/controller/driver_controller.rs | 37 +++++++++++ .../src/controller/location_controller.rs | 43 +++++++++++++ scylla-server-rust/src/controller/mod.rs | 6 ++ .../src/controller/node_controller.rs | 36 +++++++++++ .../src/controller/run_controller.rs | 62 +++++++++++++++++++ .../src/controller/system_controller.rs | 37 +++++++++++ scylla-server-rust/src/error.rs | 37 +++++++++++ scylla-server-rust/src/lib.rs | 5 +- scylla-server-rust/src/main.rs | 27 ++++++-- scylla-server-rust/src/mod.rs | 3 + .../src/services/data_service.rs | 39 +++++++++++- .../src/services/data_type_service.rs | 31 ++++++++++ .../src/services/driver_service.rs | 38 ++++++++++++ .../src/services/location_service.rs | 45 ++++++++++++++ scylla-server-rust/src/services/mod.rs | 6 ++ .../src/services/node_service.rs | 21 +++++++ .../src/services/run_service.rs | 50 +++++++++++++++ .../src/services/system_service.rs | 38 ++++++++++++ 22 files changed, 612 insertions(+), 25 deletions(-) create mode 100644 scylla-server-rust/src/controller/data_type_controller.rs create mode 100644 scylla-server-rust/src/controller/driver_controller.rs create mode 100644 scylla-server-rust/src/controller/location_controller.rs create mode 100644 scylla-server-rust/src/controller/node_controller.rs create mode 100644 scylla-server-rust/src/controller/run_controller.rs create mode 100644 scylla-server-rust/src/controller/system_controller.rs create mode 100644 scylla-server-rust/src/error.rs create mode 100644 scylla-server-rust/src/mod.rs create mode 100644 scylla-server-rust/src/services/data_type_service.rs create mode 100644 scylla-server-rust/src/services/driver_service.rs create mode 100644 scylla-server-rust/src/services/location_service.rs create mode 100644 scylla-server-rust/src/services/node_service.rs create mode 100644 scylla-server-rust/src/services/run_service.rs create mode 100644 scylla-server-rust/src/services/system_service.rs diff --git a/scylla-server-rust/Cargo.lock b/scylla-server-rust/Cargo.lock index 8cda45d5..fbf47381 100755 --- a/scylla-server-rust/Cargo.lock +++ b/scylla-server-rust/Cargo.lock @@ -2650,6 +2650,7 @@ dependencies = [ "protobuf-codegen", "serde", "tokio", + "tower", ] [[package]] diff --git a/scylla-server-rust/Cargo.toml b/scylla-server-rust/Cargo.toml index 869de2ba..7a8656d2 100644 --- a/scylla-server-rust/Cargo.toml +++ b/scylla-server-rust/Cargo.toml @@ -10,6 +10,7 @@ protobuf-codegen = "3.3.0" protobuf = "3.3.0" tokio = { version = "1.38.0", features = ["full"] } axum = "0.7.5" +tower = { version = "0.4.13", features = ["timeout"] } [workspace] members = ["prisma-cli"] diff --git a/scylla-server-rust/src/controller/data_controller.rs b/scylla-server-rust/src/controller/data_controller.rs index 8268f50e..02abf258 100644 --- a/scylla-server-rust/src/controller/data_controller.rs +++ b/scylla-server-rust/src/controller/data_controller.rs @@ -1,25 +1,40 @@ -use axum::{extract::Path, http::StatusCode, Json}; +use axum::{ + extract::{Path, State}, + Json, +}; use serde::Serialize; -use crate::{services::data_service, Database}; +use crate::{ + error::ScyllaError, + prisma::{self}, + services::data_service, + Database, +}; -#[derive(Serialize)] +#[derive(Serialize, PartialEq, Eq, PartialOrd, Ord)] pub struct DataSend { - values: Vec, time: i64, + values: Vec, +} + +impl From<&prisma::data::Data> for DataSend { + fn from(value: &prisma::data::Data) -> Self { + DataSend { + values: value.values.iter().map(|f| f.to_string()).collect(), + time: value.time.timestamp_millis(), + } + } } pub async fn get_data( - db: Database, - Path((data_type_name, run_id)): Path<(String, String)>, -) -> Result>, StatusCode> { - let data = data_service::get_data(db, data_type_name, run_id.parse::().unwrap()).await; - let transformed_data: Vec = data - .iter() - .map(|x| DataSend { - values: x.values.iter().map(|f| f.to_string()).collect(), - time: x.time.timestamp_millis(), - }) - .collect(); + State(db): State, + Path((data_type_name, run_id)): Path<(String, i32)>, +) -> Result>, ScyllaError> { + let data = data_service::get_data(db, data_type_name, run_id).await?; + + // map data to correct types + let mut transformed_data: Vec = data.iter().map(DataSend::from).collect(); + transformed_data.sort(); + Ok(Json::from(transformed_data)) } diff --git a/scylla-server-rust/src/controller/data_type_controller.rs b/scylla-server-rust/src/controller/data_type_controller.rs new file mode 100644 index 00000000..157f729a --- /dev/null +++ b/scylla-server-rust/src/controller/data_type_controller.rs @@ -0,0 +1,29 @@ +use axum::{extract::State, Json}; +use serde::Serialize; + +use crate::{error::ScyllaError, prisma, services::data_type_service, Database}; + +#[derive(Serialize)] +pub struct DataTypeSend { + pub name: String, + pub unit: String, +} + +impl From<&prisma::data_type::Data> for DataTypeSend { + fn from(value: &prisma::data_type::Data) -> Self { + DataTypeSend { + name: value.name.clone(), + unit: value.unit.clone(), + } + } +} + +pub async fn get_all_data_types( + State(db): State, +) -> Result>, ScyllaError> { + let data = data_type_service::get_all_data_types(db).await?; + + let transformed_data: Vec = data.iter().map(DataTypeSend::from).collect(); + + Ok(Json::from(transformed_data)) +} diff --git a/scylla-server-rust/src/controller/driver_controller.rs b/scylla-server-rust/src/controller/driver_controller.rs new file mode 100644 index 00000000..699f2519 --- /dev/null +++ b/scylla-server-rust/src/controller/driver_controller.rs @@ -0,0 +1,37 @@ +use axum::{extract::State, Json}; +use serde::Serialize; + +use crate::{error::ScyllaError, prisma, services::driver_service, Database}; + +use super::run_controller::{self, RunSend}; + +#[derive(Serialize)] +pub struct DriverSend { + username: String, + runs: Vec, +} + +impl From<&prisma::driver::Data> for DriverSend { + fn from(value: &prisma::driver::Data) -> Self { + DriverSend { + username: value.username.clone(), + runs: value + .runs + .clone() + .unwrap_or_default() + .iter() + .map(RunSend::from) + .collect(), + } + } +} + +pub async fn get_all_drivers( + State(db): State, +) -> Result>, ScyllaError> { + let data = driver_service::get_all_drivers(db).await?; + + let transformed_data: Vec = data.iter().map(DriverSend::from).collect(); + + Ok(Json::from(transformed_data)) +} diff --git a/scylla-server-rust/src/controller/location_controller.rs b/scylla-server-rust/src/controller/location_controller.rs new file mode 100644 index 00000000..6a68d67e --- /dev/null +++ b/scylla-server-rust/src/controller/location_controller.rs @@ -0,0 +1,43 @@ +use axum::{extract::State, Json}; +use serde::Serialize; + +use crate::{error::ScyllaError, prisma, services::location_service, Database}; + +use super::run_controller::{self, RunSend}; + +#[derive(Serialize)] +pub struct LocationSend { + pub name: String, + pub latitude: f64, + pub longitude: f64, + pub radius: f64, + pub runs: Vec, +} + +impl From<&prisma::location::Data> for LocationSend { + fn from(value: &prisma::location::Data) -> Self { + LocationSend { + name: value.name.clone(), + latitude: value.latitude, + longitude: value.longitude, + radius: value.radius, + runs: value + .runs + .clone() + .unwrap_or_default() + .iter() + .map(RunSend::from) + .collect(), + } + } +} + +pub async fn get_all_locations( + State(db): State, +) -> Result>, ScyllaError> { + let data = location_service::get_all_locations(db).await?; + + let transformed_data: Vec = data.iter().map(LocationSend::from).collect(); + + Ok(Json::from(transformed_data)) +} diff --git a/scylla-server-rust/src/controller/mod.rs b/scylla-server-rust/src/controller/mod.rs index c9cd4b5a..079a3dab 100644 --- a/scylla-server-rust/src/controller/mod.rs +++ b/scylla-server-rust/src/controller/mod.rs @@ -1 +1,7 @@ pub mod data_controller; +pub mod data_type_controller; +pub mod driver_controller; +pub mod location_controller; +pub mod node_controller; +pub mod run_controller; +pub mod system_controller; diff --git a/scylla-server-rust/src/controller/node_controller.rs b/scylla-server-rust/src/controller/node_controller.rs new file mode 100644 index 00000000..f0f80530 --- /dev/null +++ b/scylla-server-rust/src/controller/node_controller.rs @@ -0,0 +1,36 @@ +use axum::{extract::State, Json}; +use serde::Serialize; + +use crate::{error::ScyllaError, prisma, services::node_service, Database}; + +use super::data_type_controller::DataTypeSend; + +#[derive(Serialize)] +pub struct NodeSend { + name: String, + #[serde(rename = "dataTypes")] + data_types: Vec, +} + +impl From<&prisma::node::Data> for NodeSend { + fn from(value: &prisma::node::Data) -> Self { + NodeSend { + name: value.name.clone(), + data_types: value + .data_types + .clone() + .unwrap_or_default() + .iter() + .map(DataTypeSend::from) + .collect(), + } + } +} + +pub async fn get_all_nodes(State(db): State) -> Result>, ScyllaError> { + let data = node_service::get_all_nodes(db).await?; + + let transformed_data: Vec = data.iter().map(NodeSend::from).collect(); + + Ok(Json::from(transformed_data)) +} diff --git a/scylla-server-rust/src/controller/run_controller.rs b/scylla-server-rust/src/controller/run_controller.rs new file mode 100644 index 00000000..4e6a0de2 --- /dev/null +++ b/scylla-server-rust/src/controller/run_controller.rs @@ -0,0 +1,62 @@ +use axum::{ + extract::{Path, State}, + Json, +}; +use serde::Serialize; + +use crate::{error::ScyllaError, prisma, services::run_service, Database}; + +#[derive(Serialize)] +pub struct RunSend { + pub id: i32, + #[serde(rename = "locationName")] + pub location_name: String, + #[serde(rename = "driverName")] + pub driver_name: String, + #[serde(rename = "systemName")] + pub system_name: String, + pub time: i64, +} + +impl From<&prisma::run::Data> for RunSend { + fn from(value: &prisma::run::Data) -> Self { + RunSend { + id: value.id, + location_name: value.location_name.clone().unwrap_or_default(), + driver_name: value.driver_name.clone().unwrap_or_default(), + system_name: value.system_name.clone().unwrap_or_default(), + time: value.time.timestamp_millis(), + } + } +} + +pub async fn get_all_runs(State(db): State) -> Result>, ScyllaError> { + let data = run_service::get_all_runs(db).await?; + + let transformed_data: Vec = data.iter().map(RunSend::from).collect(); + + Ok(Json::from(transformed_data)) +} + +pub async fn get_run_by_id( + State(db): State, + Path(run_id): Path, +) -> Result, ScyllaError> { + let data = run_service::get_run_by_id(db, run_id).await?; + + if data.is_none() { + return Err(ScyllaError::NotFound); + } + + let data_new = data.unwrap(); + + let transformed_data = RunSend { + id: data_new.id, + location_name: data_new.location_name.unwrap_or_default(), + driver_name: data_new.driver_name.unwrap_or_default(), + system_name: data_new.system_name.unwrap_or_default(), + time: data_new.time.timestamp_millis(), + }; + + Ok(Json::from(transformed_data)) +} diff --git a/scylla-server-rust/src/controller/system_controller.rs b/scylla-server-rust/src/controller/system_controller.rs new file mode 100644 index 00000000..d538f81c --- /dev/null +++ b/scylla-server-rust/src/controller/system_controller.rs @@ -0,0 +1,37 @@ +use axum::{extract::State, Json}; +use serde::Serialize; + +use crate::{error::ScyllaError, prisma, services::system_service, Database}; + +use super::run_controller::{self, RunSend}; + +#[derive(Serialize)] +pub struct SystemSend { + pub name: String, + pub runs: Vec, +} + +impl From<&prisma::system::Data> for SystemSend { + fn from(value: &prisma::system::Data) -> Self { + SystemSend { + name: value.name.clone(), + runs: value + .runs + .clone() + .unwrap_or_default() + .iter() + .map(RunSend::from) + .collect(), + } + } +} + +pub async fn get_all_systems( + State(db): State, +) -> Result>, ScyllaError> { + let data = system_service::get_all_systems(db).await?; + + let transformed_data: Vec = data.iter().map(SystemSend::from).collect(); + + Ok(Json::from(transformed_data)) +} diff --git a/scylla-server-rust/src/error.rs b/scylla-server-rust/src/error.rs new file mode 100644 index 00000000..9c57b350 --- /dev/null +++ b/scylla-server-rust/src/error.rs @@ -0,0 +1,37 @@ +use axum::{ + http::StatusCode, + response::{IntoResponse, Response}, +}; +use prisma_client_rust::{ + prisma_errors::query_engine::{RecordNotFound, UniqueKeyViolation}, + QueryError, +}; + +pub enum ScyllaError { + PrismaError(QueryError), + NotFound, +} + +impl From for ScyllaError { + fn from(error: QueryError) -> Self { + match error { + e if e.is_prisma_error::() => ScyllaError::NotFound, + e => ScyllaError::PrismaError(e), + } + } +} + +// This centralizes all different errors from our app in one place +impl IntoResponse for ScyllaError { + fn into_response(self) -> Response { + let status = match self { + ScyllaError::PrismaError(error) if error.is_prisma_error::() => { + StatusCode::CONFLICT + } + ScyllaError::PrismaError(_) => StatusCode::BAD_REQUEST, + ScyllaError::NotFound => StatusCode::NOT_FOUND, + }; + + status.into_response() + } +} diff --git a/scylla-server-rust/src/lib.rs b/scylla-server-rust/src/lib.rs index 837b75e1..8c572eca 100644 --- a/scylla-server-rust/src/lib.rs +++ b/scylla-server-rust/src/lib.rs @@ -1,8 +1,11 @@ pub mod controller; +pub mod error; pub mod services; #[allow(clippy::all)] #[allow(warnings)] pub mod prisma; -pub type Database = axum::Extension>; +pub mod serverdata; + +pub type Database = std::sync::Arc; diff --git a/scylla-server-rust/src/main.rs b/scylla-server-rust/src/main.rs index edff0410..9733bf65 100755 --- a/scylla-server-rust/src/main.rs +++ b/scylla-server-rust/src/main.rs @@ -1,16 +1,31 @@ use std::sync::Arc; -use axum::{routing::get, Extension, Router}; -use scylla_server_rust::{controller, prisma::PrismaClient}; +use axum::{routing::get, Router}; +use scylla_server_rust::{ + controller::{ + self, data_type_controller, driver_controller, location_controller, node_controller, + run_controller, system_controller, + }, + prisma::PrismaClient, +}; #[tokio::main] async fn main() { let client = Arc::new(PrismaClient::_builder().build().await.unwrap()); - let app = Router::new().route( - "/data/:dataTypeName/:runId", - get(controller::data_controller::get_data).layer(Extension(client)), - ); + let app = Router::new() + .route( + "/data/:dataTypeName/:runId", + get(controller::data_controller::get_data), + ) + .route("/datatypes", get(data_type_controller::get_all_data_types)) + .route("/drivers", get(driver_controller::get_all_drivers)) + .route("/locations", get(location_controller::get_all_locations)) + .route("/nodes", get(node_controller::get_all_nodes)) + .route("/runs", get(run_controller::get_all_runs)) + .route("/runs/:id", get(run_controller::get_run_by_id)) + .route("/systems", get(system_controller::get_all_systems)) + .with_state(client); let listener = tokio::net::TcpListener::bind("0.0.0.0:8000").await.unwrap(); axum::serve(listener, app).await.unwrap(); diff --git a/scylla-server-rust/src/mod.rs b/scylla-server-rust/src/mod.rs new file mode 100644 index 00000000..31b78fdf --- /dev/null +++ b/scylla-server-rust/src/mod.rs @@ -0,0 +1,3 @@ +// @generated + +pub mod serverdata; diff --git a/scylla-server-rust/src/services/data_service.rs b/scylla-server-rust/src/services/data_service.rs index 7abc06c5..8b42eca6 100644 --- a/scylla-server-rust/src/services/data_service.rs +++ b/scylla-server-rust/src/services/data_service.rs @@ -1,10 +1,17 @@ -use crate::{prisma, Database}; +use prisma_client_rust::{chrono::DateTime, QueryError}; +use crate::{ + prisma::{self}, + serverdata::ServerData, + Database, +}; + +/// Get a datapoint pub async fn get_data( db: Database, data_type_name: String, run_id: i32, -) -> Vec { +) -> Result, QueryError> { db.data() .find_many(vec![ prisma::data::data_type_name::equals(data_type_name), @@ -12,5 +19,31 @@ pub async fn get_data( ]) .exec() .await - .unwrap() +} + +/// Add a datapoint +pub async fn add_data( + db: Database, + serverdata: ServerData, + unix_time: i64, + data_type_name: String, + run_id: i32, +) -> Result { + db.data() + .create( + prisma::data_type::name::equals(data_type_name), + DateTime::from_timestamp_millis(unix_time) + .unwrap() + .fixed_offset(), + prisma::run::id::equals(run_id), + vec![prisma::data::values::set( + serverdata + .value + .iter() + .map(|f| f.parse::().unwrap()) + .collect(), + )], + ) + .exec() + .await } diff --git a/scylla-server-rust/src/services/data_type_service.rs b/scylla-server-rust/src/services/data_type_service.rs new file mode 100644 index 00000000..062ded57 --- /dev/null +++ b/scylla-server-rust/src/services/data_type_service.rs @@ -0,0 +1,31 @@ +use prisma_client_rust::QueryError; + +use crate::{prisma, Database}; + +pub async fn get_all_data_types(db: Database) -> Result, QueryError> { + db.data_type().find_many(vec![]).exec().await +} + +pub async fn upsert_data_type( + db: Database, + data_type_name: String, + unit: String, + node_name: String, +) -> Result { + db.data_type() + .upsert( + prisma::data_type::name::equals(data_type_name.clone()), + prisma::data_type::create( + data_type_name, + unit.clone(), + prisma::node::name::equals(node_name.clone()), + vec![], + ), + vec![ + prisma::data_type::unit::set(unit), + prisma::data_type::node_name::set(node_name), + ], + ) + .exec() + .await +} diff --git a/scylla-server-rust/src/services/driver_service.rs b/scylla-server-rust/src/services/driver_service.rs new file mode 100644 index 00000000..ade500b6 --- /dev/null +++ b/scylla-server-rust/src/services/driver_service.rs @@ -0,0 +1,38 @@ +use prisma_client_rust::QueryError; + +use crate::{ + prisma::{self}, + Database, +}; + +pub async fn get_all_drivers(db: Database) -> Result, QueryError> { + db.driver().find_many(vec![]).exec().await +} + +pub async fn upsert_driver( + db: Database, + driver_name: String, + run_id: i32, +) -> Result { + let drive = db + .driver() + .upsert( + prisma::driver::username::equals(driver_name.clone()), + prisma::driver::create(driver_name.clone(), vec![]), + vec![], + ) + .exec() + .await?; + + db.run() + .update( + prisma::run::id::equals(run_id), + vec![prisma::run::driver::connect( + prisma::driver::username::equals(driver_name), + )], + ) + .exec() + .await?; + + Ok(drive) +} diff --git a/scylla-server-rust/src/services/location_service.rs b/scylla-server-rust/src/services/location_service.rs new file mode 100644 index 00000000..b9bd9428 --- /dev/null +++ b/scylla-server-rust/src/services/location_service.rs @@ -0,0 +1,45 @@ +use prisma_client_rust::QueryError; + +use crate::{ + prisma::{self}, + Database, +}; + +pub async fn get_all_locations(db: Database) -> Result, QueryError> { + db.location().find_many(vec![]).exec().await +} + +pub async fn upsert_location( + db: Database, + name: String, + latitude: f64, + longitude: f64, + radius: f64, + run_id: i32, +) -> Result { + let loc = db + .location() + .upsert( + prisma::location::name::equals(name.clone()), + prisma::location::create(name.clone(), latitude, longitude, radius, vec![]), + vec![ + prisma::location::latitude::set(latitude), + prisma::location::longitude::set(longitude), + prisma::location::radius::set(radius), + ], + ) + .exec() + .await?; + + db.run() + .update( + prisma::run::id::equals(run_id), + vec![prisma::run::location::connect( + prisma::location::name::equals(name), + )], + ) + .exec() + .await?; + + Ok(loc) +} diff --git a/scylla-server-rust/src/services/mod.rs b/scylla-server-rust/src/services/mod.rs index 23e6bc6a..abaaef06 100644 --- a/scylla-server-rust/src/services/mod.rs +++ b/scylla-server-rust/src/services/mod.rs @@ -1 +1,7 @@ pub mod data_service; +pub mod data_type_service; +pub mod driver_service; +pub mod location_service; +pub mod node_service; +pub mod run_service; +pub mod system_service; diff --git a/scylla-server-rust/src/services/node_service.rs b/scylla-server-rust/src/services/node_service.rs new file mode 100644 index 00000000..80068e85 --- /dev/null +++ b/scylla-server-rust/src/services/node_service.rs @@ -0,0 +1,21 @@ +use prisma_client_rust::QueryError; + +use crate::{prisma, Database}; + +pub async fn get_all_nodes(db: Database) -> Result, QueryError> { + db.node().find_many(vec![]).exec().await +} + +pub async fn upsert_node( + db: Database, + node_name: String, +) -> Result { + db.node() + .upsert( + prisma::node::name::equals(node_name.clone()), + prisma::node::create(node_name, vec![]), + vec![], + ) + .exec() + .await +} diff --git a/scylla-server-rust/src/services/run_service.rs b/scylla-server-rust/src/services/run_service.rs new file mode 100644 index 00000000..6d3cdea1 --- /dev/null +++ b/scylla-server-rust/src/services/run_service.rs @@ -0,0 +1,50 @@ +use std::vec; + +use prisma_client_rust::{chrono::DateTime, QueryError}; + +use crate::{ + prisma::{self}, + Database, +}; + +pub async fn get_all_runs(db: Database) -> Result, QueryError> { + db.run().find_many(vec![]).exec().await +} + +pub async fn get_run_by_id( + db: Database, + run_id: i32, +) -> Result, QueryError> { + db.run() + .find_unique(prisma::run::id::equals(run_id)) + .exec() + .await +} + +pub async fn create_run(db: Database, timestamp: i64) -> Result { + db.run() + .create( + DateTime::from_timestamp_millis(timestamp) + .unwrap() + .fixed_offset(), + vec![], + ) + .exec() + .await +} + +pub async fn create_run_with_id( + db: Database, + timestamp: i64, + run_id: i32, +) -> Result { + db.run() + .create( + DateTime::from_timestamp_millis(timestamp) + .unwrap() + .fixed_offset(), + vec![prisma::run::id::set(run_id)], + ) + .exec() + .await +} diff --git a/scylla-server-rust/src/services/system_service.rs b/scylla-server-rust/src/services/system_service.rs new file mode 100644 index 00000000..1b5accb5 --- /dev/null +++ b/scylla-server-rust/src/services/system_service.rs @@ -0,0 +1,38 @@ +use prisma_client_rust::QueryError; + +use crate::{ + prisma::{self}, + Database, +}; + +pub async fn get_all_systems(db: Database) -> Result, QueryError> { + db.system().find_many(vec![]).exec().await +} + +pub async fn upsert_system( + db: Database, + system_name: String, + run_id: i32, +) -> Result { + let system = db + .system() + .upsert( + prisma::system::name::equals(system_name.clone()), + prisma::system::create(system_name.clone(), vec![]), + vec![], + ) + .exec() + .await?; + + db.run() + .update( + prisma::run::id::equals(run_id), + vec![prisma::run::system::connect(prisma::system::name::equals( + system_name, + ))], + ) + .exec() + .await?; + + Ok(system) +} From 5048fae09cc5d077f06e1b373dd97a5321fa57a6 Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Fri, 5 Jul 2024 23:13:11 -0400 Subject: [PATCH 03/68] develop name changed --- .github/workflows/scylla-rust-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/scylla-rust-ci.yml b/.github/workflows/scylla-rust-ci.yml index dddf7a73..a04ddefb 100644 --- a/.github/workflows/scylla-rust-ci.yml +++ b/.github/workflows/scylla-rust-ci.yml @@ -4,11 +4,11 @@ on: push: branches: - main - - Develop + - develop pull_request: branches: - main - - Develop + - develop jobs: build: From aa03ff588b3a61c9ff888fcf921b1ab4e946a9dd Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Fri, 5 Jul 2024 23:14:51 -0400 Subject: [PATCH 04/68] change dir --- .github/workflows/scylla-rust-ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/scylla-rust-ci.yml b/.github/workflows/scylla-rust-ci.yml index a04ddefb..2f1e8f2f 100644 --- a/.github/workflows/scylla-rust-ci.yml +++ b/.github/workflows/scylla-rust-ci.yml @@ -10,6 +10,12 @@ on: - main - develop + +defaults: + run: + working-directory: scylla-server-rust + + jobs: build: runs-on: ubuntu-latest From b7faa50f05c627abce2baac1dcbd8e5411783f2d Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Fri, 5 Jul 2024 23:17:33 -0400 Subject: [PATCH 05/68] generate prisma --- .github/workflows/scylla-rust-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/scylla-rust-ci.yml b/.github/workflows/scylla-rust-ci.yml index 2f1e8f2f..d180bd76 100644 --- a/.github/workflows/scylla-rust-ci.yml +++ b/.github/workflows/scylla-rust-ci.yml @@ -27,6 +27,8 @@ jobs: submodules: recursive - name: Install cargo-audit run: cargo install cargo-audit + - name: Generate prisma + run: cargo prisma generate - name: Build run: cargo build --verbose - name: Test From 0c49e357c3a59445270b6944b0425996d37b716f Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Sat, 6 Jul 2024 22:06:03 -0400 Subject: [PATCH 06/68] add tests, fixups and cleanups --- .github/workflows/scylla-rust-ci.yml | 2 - scylla-server-rust/Cargo.lock | 102 ++++++++++++++ scylla-server-rust/Cargo.toml | 3 +- scylla-server-rust/README.md | 10 ++ scylla-server-rust/prisma-cli/Cargo.toml | 2 +- .../src/controller/data_controller.rs | 10 +- .../src/controller/data_type_controller.rs | 4 +- .../src/controller/driver_controller.rs | 4 +- .../src/controller/location_controller.rs | 4 +- .../src/controller/node_controller.rs | 4 +- .../src/controller/run_controller.rs | 6 +- .../src/controller/system_controller.rs | 4 +- scylla-server-rust/src/lib.rs | 3 +- scylla-server-rust/src/main.rs | 13 +- .../src/services/data_service.rs | 4 +- .../src/services/data_type_service.rs | 4 +- .../src/services/driver_service.rs | 4 +- .../src/services/location_service.rs | 4 +- .../src/services/node_service.rs | 4 +- .../src/services/run_service.rs | 8 +- .../src/services/system_service.rs | 4 +- scylla-server-rust/tests/data_service_test.rs | 129 ++++++++++++++++++ .../tests/data_type_service_test.rs | 72 ++++++++++ .../tests/driver_service_test.rs | 35 +++++ .../tests/location_service_test.rs | 34 +++++ scylla-server-rust/tests/node_service_test.rs | 47 +++++++ scylla-server-rust/tests/run_service_test.rs | 35 +++++ .../tests/system_service_test.rs | 65 +++++++++ scylla-server-rust/tests/test_utils.rs | 24 ++++ 29 files changed, 604 insertions(+), 40 deletions(-) create mode 100644 scylla-server-rust/tests/data_service_test.rs create mode 100644 scylla-server-rust/tests/data_type_service_test.rs create mode 100644 scylla-server-rust/tests/driver_service_test.rs create mode 100644 scylla-server-rust/tests/location_service_test.rs create mode 100644 scylla-server-rust/tests/node_service_test.rs create mode 100644 scylla-server-rust/tests/run_service_test.rs create mode 100644 scylla-server-rust/tests/system_service_test.rs create mode 100644 scylla-server-rust/tests/test_utils.rs diff --git a/.github/workflows/scylla-rust-ci.yml b/.github/workflows/scylla-rust-ci.yml index d180bd76..0ae620c3 100644 --- a/.github/workflows/scylla-rust-ci.yml +++ b/.github/workflows/scylla-rust-ci.yml @@ -31,8 +31,6 @@ jobs: run: cargo prisma generate - name: Build run: cargo build --verbose - - name: Test - run: cargo test --verbose - name: Clippy run: cargo clippy --verbose -- -D warnings - name: Audit diff --git a/scylla-server-rust/Cargo.lock b/scylla-server-rust/Cargo.lock index fbf47381..c2936845 100755 --- a/scylla-server-rust/Cargo.lock +++ b/scylla-server-rust/Cargo.lock @@ -706,6 +706,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" + [[package]] name = "futures" version = "0.3.30" @@ -1114,6 +1120,25 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "include_dir" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "923d117408f1e49d914f1a379a309cffe4f18c05cf4e3d12e613a15fc81bd0dd" +dependencies = [ + "include_dir_macros", +] + +[[package]] +name = "include_dir_macros" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cab85a7ed0bd5f0e76d93846e0147172bed2e2d3f859bcc33a8d9699cad1a75" +dependencies = [ + "proc-macro2", + "quote", +] + [[package]] name = "indexmap" version = "1.9.3" @@ -2010,6 +2035,7 @@ dependencies = [ "dmmf", "dotenv", "futures", + "include_dir", "indexmap 1.9.3", "migration-core", "paste", @@ -2022,7 +2048,10 @@ dependencies = [ "serde", "serde-value", "serde_json", + "tempdir", "thiserror", + "tokio", + "tracing", "user-facing-errors", "uuid", ] @@ -2352,6 +2381,19 @@ dependencies = [ "nibble_vec", ] +[[package]] +name = "rand" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" +dependencies = [ + "fuchsia-cprng", + "libc", + "rand_core 0.3.1", + "rdrand", + "winapi", +] + [[package]] name = "rand" version = "0.7.3" @@ -2396,6 +2438,21 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +dependencies = [ + "rand_core 0.4.2", +] + +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + [[package]] name = "rand_core" version = "0.5.1" @@ -2432,6 +2489,15 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +dependencies = [ + "rand_core 0.3.1", +] + [[package]] name = "redox_syscall" version = "0.2.16" @@ -2490,6 +2556,15 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + [[package]] name = "request-handlers" version = "0.1.0" @@ -2651,6 +2726,7 @@ dependencies = [ "serde", "tokio", "tower", + "tower-http", ] [[package]] @@ -3052,6 +3128,16 @@ dependencies = [ "libc", ] +[[package]] +name = "tempdir" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" +dependencies = [ + "rand 0.4.6", + "remove_dir_all", +] + [[package]] name = "tempfile" version = "3.10.1" @@ -3219,6 +3305,22 @@ dependencies = [ "tracing", ] +[[package]] +name = "tower-http" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" +dependencies = [ + "bitflags 2.5.0", + "bytes", + "http 1.1.0", + "http-body 1.0.0", + "http-body-util", + "pin-project-lite", + "tower-layer", + "tower-service", +] + [[package]] name = "tower-layer" version = "0.3.2" diff --git a/scylla-server-rust/Cargo.toml b/scylla-server-rust/Cargo.toml index 7a8656d2..dbda33fb 100644 --- a/scylla-server-rust/Cargo.toml +++ b/scylla-server-rust/Cargo.toml @@ -4,13 +4,14 @@ version = "0.0.1" edition = "2021" [dependencies] -prisma-client-rust = {git = "https://github.com/Brendonovich/prisma-client-rust", tag = "0.6.11", default-features = false,features = ["postgresql"] } +prisma-client-rust = {git = "https://github.com/Brendonovich/prisma-client-rust", tag = "0.6.11", default-features = false,features = ["postgresql", "migrations"] } serde = "1.0.203" protobuf-codegen = "3.3.0" protobuf = "3.3.0" tokio = { version = "1.38.0", features = ["full"] } axum = "0.7.5" tower = { version = "0.4.13", features = ["timeout"] } +tower-http = { version = "0.5.2", features = ["cors"] } [workspace] members = ["prisma-cli"] diff --git a/scylla-server-rust/README.md b/scylla-server-rust/README.md index fe1d523f..803f3840 100644 --- a/scylla-server-rust/README.md +++ b/scylla-server-rust/README.md @@ -17,4 +17,14 @@ docker compose run -P odyssey-timescale ``` # in this directory SOURCE_DATABASE_URL=postgresql://postgres:password@127.0.0.1:5432/timescaledb cargo run +``` + +### Test this app + +Since this app uses the database for testing, you must wipe the database. +``` +docker volume rm argos_db-data +docker compose run -P odyssey-timescale +cargo prisma migrate deploy +SOURCE_DATABASE_URL=postgresql://postgres:password@127.0.0.1:5432/timescaledb cargo test -- --test-threads=1 ``` \ No newline at end of file diff --git a/scylla-server-rust/prisma-cli/Cargo.toml b/scylla-server-rust/prisma-cli/Cargo.toml index a81d9964..31208305 100755 --- a/scylla-server-rust/prisma-cli/Cargo.toml +++ b/scylla-server-rust/prisma-cli/Cargo.toml @@ -5,5 +5,5 @@ edition = "2021" [dependencies] prisma-client-rust-cli = { git = "https://github.com/Brendonovich/prisma-client-rust", tag = "0.6.11", features = [ - "postgresql" + "postgresql", "migrations" ], default-features = false } \ No newline at end of file diff --git a/scylla-server-rust/src/controller/data_controller.rs b/scylla-server-rust/src/controller/data_controller.rs index 02abf258..3edf9939 100644 --- a/scylla-server-rust/src/controller/data_controller.rs +++ b/scylla-server-rust/src/controller/data_controller.rs @@ -11,12 +11,13 @@ use crate::{ Database, }; -#[derive(Serialize, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Serialize, Debug, PartialEq, Eq, PartialOrd, Ord)] pub struct DataSend { - time: i64, - values: Vec, + pub time: i64, + pub values: Vec, } +// convert the prisma type to the client type for JSON encoding impl From<&prisma::data::Data> for DataSend { fn from(value: &prisma::data::Data) -> Self { DataSend { @@ -30,10 +31,11 @@ pub async fn get_data( State(db): State, Path((data_type_name, run_id)): Path<(String, i32)>, ) -> Result>, ScyllaError> { - let data = data_service::get_data(db, data_type_name, run_id).await?; + let data = data_service::get_data(&db, data_type_name, run_id).await?; // map data to correct types let mut transformed_data: Vec = data.iter().map(DataSend::from).collect(); + // sort it by time for the client transformed_data.sort(); Ok(Json::from(transformed_data)) diff --git a/scylla-server-rust/src/controller/data_type_controller.rs b/scylla-server-rust/src/controller/data_type_controller.rs index 157f729a..25aadafb 100644 --- a/scylla-server-rust/src/controller/data_type_controller.rs +++ b/scylla-server-rust/src/controller/data_type_controller.rs @@ -3,7 +3,7 @@ use serde::Serialize; use crate::{error::ScyllaError, prisma, services::data_type_service, Database}; -#[derive(Serialize)] +#[derive(Serialize, Debug, PartialEq)] pub struct DataTypeSend { pub name: String, pub unit: String, @@ -21,7 +21,7 @@ impl From<&prisma::data_type::Data> for DataTypeSend { pub async fn get_all_data_types( State(db): State, ) -> Result>, ScyllaError> { - let data = data_type_service::get_all_data_types(db).await?; + let data = data_type_service::get_all_data_types(&db).await?; let transformed_data: Vec = data.iter().map(DataTypeSend::from).collect(); diff --git a/scylla-server-rust/src/controller/driver_controller.rs b/scylla-server-rust/src/controller/driver_controller.rs index 699f2519..137db707 100644 --- a/scylla-server-rust/src/controller/driver_controller.rs +++ b/scylla-server-rust/src/controller/driver_controller.rs @@ -5,7 +5,7 @@ use crate::{error::ScyllaError, prisma, services::driver_service, Database}; use super::run_controller::{self, RunSend}; -#[derive(Serialize)] +#[derive(Serialize, PartialEq)] pub struct DriverSend { username: String, runs: Vec, @@ -29,7 +29,7 @@ impl From<&prisma::driver::Data> for DriverSend { pub async fn get_all_drivers( State(db): State, ) -> Result>, ScyllaError> { - let data = driver_service::get_all_drivers(db).await?; + let data = driver_service::get_all_drivers(&db).await?; let transformed_data: Vec = data.iter().map(DriverSend::from).collect(); diff --git a/scylla-server-rust/src/controller/location_controller.rs b/scylla-server-rust/src/controller/location_controller.rs index 6a68d67e..1bd861ee 100644 --- a/scylla-server-rust/src/controller/location_controller.rs +++ b/scylla-server-rust/src/controller/location_controller.rs @@ -5,7 +5,7 @@ use crate::{error::ScyllaError, prisma, services::location_service, Database}; use super::run_controller::{self, RunSend}; -#[derive(Serialize)] +#[derive(Serialize, Debug, PartialEq)] pub struct LocationSend { pub name: String, pub latitude: f64, @@ -35,7 +35,7 @@ impl From<&prisma::location::Data> for LocationSend { pub async fn get_all_locations( State(db): State, ) -> Result>, ScyllaError> { - let data = location_service::get_all_locations(db).await?; + let data = location_service::get_all_locations(&db).await?; let transformed_data: Vec = data.iter().map(LocationSend::from).collect(); diff --git a/scylla-server-rust/src/controller/node_controller.rs b/scylla-server-rust/src/controller/node_controller.rs index f0f80530..ed1743ba 100644 --- a/scylla-server-rust/src/controller/node_controller.rs +++ b/scylla-server-rust/src/controller/node_controller.rs @@ -5,7 +5,7 @@ use crate::{error::ScyllaError, prisma, services::node_service, Database}; use super::data_type_controller::DataTypeSend; -#[derive(Serialize)] +#[derive(Serialize, PartialEq)] pub struct NodeSend { name: String, #[serde(rename = "dataTypes")] @@ -28,7 +28,7 @@ impl From<&prisma::node::Data> for NodeSend { } pub async fn get_all_nodes(State(db): State) -> Result>, ScyllaError> { - let data = node_service::get_all_nodes(db).await?; + let data = node_service::get_all_nodes(&db).await?; let transformed_data: Vec = data.iter().map(NodeSend::from).collect(); diff --git a/scylla-server-rust/src/controller/run_controller.rs b/scylla-server-rust/src/controller/run_controller.rs index 4e6a0de2..7e9fe983 100644 --- a/scylla-server-rust/src/controller/run_controller.rs +++ b/scylla-server-rust/src/controller/run_controller.rs @@ -6,7 +6,7 @@ use serde::Serialize; use crate::{error::ScyllaError, prisma, services::run_service, Database}; -#[derive(Serialize)] +#[derive(Serialize, Debug, PartialEq)] pub struct RunSend { pub id: i32, #[serde(rename = "locationName")] @@ -31,7 +31,7 @@ impl From<&prisma::run::Data> for RunSend { } pub async fn get_all_runs(State(db): State) -> Result>, ScyllaError> { - let data = run_service::get_all_runs(db).await?; + let data = run_service::get_all_runs(&db).await?; let transformed_data: Vec = data.iter().map(RunSend::from).collect(); @@ -42,7 +42,7 @@ pub async fn get_run_by_id( State(db): State, Path(run_id): Path, ) -> Result, ScyllaError> { - let data = run_service::get_run_by_id(db, run_id).await?; + let data = run_service::get_run_by_id(&db, run_id).await?; if data.is_none() { return Err(ScyllaError::NotFound); diff --git a/scylla-server-rust/src/controller/system_controller.rs b/scylla-server-rust/src/controller/system_controller.rs index d538f81c..c4dcdf3b 100644 --- a/scylla-server-rust/src/controller/system_controller.rs +++ b/scylla-server-rust/src/controller/system_controller.rs @@ -5,7 +5,7 @@ use crate::{error::ScyllaError, prisma, services::system_service, Database}; use super::run_controller::{self, RunSend}; -#[derive(Serialize)] +#[derive(Serialize, Debug, PartialEq)] pub struct SystemSend { pub name: String, pub runs: Vec, @@ -29,7 +29,7 @@ impl From<&prisma::system::Data> for SystemSend { pub async fn get_all_systems( State(db): State, ) -> Result>, ScyllaError> { - let data = system_service::get_all_systems(db).await?; + let data = system_service::get_all_systems(&db).await?; let transformed_data: Vec = data.iter().map(SystemSend::from).collect(); diff --git a/scylla-server-rust/src/lib.rs b/scylla-server-rust/src/lib.rs index 8c572eca..c40a46d2 100644 --- a/scylla-server-rust/src/lib.rs +++ b/scylla-server-rust/src/lib.rs @@ -8,4 +8,5 @@ pub mod prisma; pub mod serverdata; -pub type Database = std::sync::Arc; +/// The type descriptor of the database passed to the middlelayer through axum state +pub type Database = std::sync::Arc; \ No newline at end of file diff --git a/scylla-server-rust/src/main.rs b/scylla-server-rust/src/main.rs index 9733bf65..50bd8042 100755 --- a/scylla-server-rust/src/main.rs +++ b/scylla-server-rust/src/main.rs @@ -1,17 +1,19 @@ use std::sync::Arc; -use axum::{routing::get, Router}; +use axum::{http::Method, routing::get, Router}; use scylla_server_rust::{ controller::{ self, data_type_controller, driver_controller, location_controller, node_controller, run_controller, system_controller, }, prisma::PrismaClient, + Database, }; +use tower_http::cors::{Any, CorsLayer}; #[tokio::main] async fn main() { - let client = Arc::new(PrismaClient::_builder().build().await.unwrap()); + let client: Database = Arc::new(PrismaClient::_builder().build().await.unwrap()); let app = Router::new() .route( @@ -25,6 +27,13 @@ async fn main() { .route("/runs", get(run_controller::get_all_runs)) .route("/runs/:id", get(run_controller::get_run_by_id)) .route("/systems", get(system_controller::get_all_systems)) + .layer( + CorsLayer::new() + // allow `GET` + .allow_methods([Method::GET]) + // allow requests from any origin + .allow_origin(Any), + ) .with_state(client); let listener = tokio::net::TcpListener::bind("0.0.0.0:8000").await.unwrap(); diff --git a/scylla-server-rust/src/services/data_service.rs b/scylla-server-rust/src/services/data_service.rs index 8b42eca6..e37070cd 100644 --- a/scylla-server-rust/src/services/data_service.rs +++ b/scylla-server-rust/src/services/data_service.rs @@ -8,7 +8,7 @@ use crate::{ /// Get a datapoint pub async fn get_data( - db: Database, + db: &Database, data_type_name: String, run_id: i32, ) -> Result, QueryError> { @@ -23,7 +23,7 @@ pub async fn get_data( /// Add a datapoint pub async fn add_data( - db: Database, + db: &Database, serverdata: ServerData, unix_time: i64, data_type_name: String, diff --git a/scylla-server-rust/src/services/data_type_service.rs b/scylla-server-rust/src/services/data_type_service.rs index 062ded57..64f3e23f 100644 --- a/scylla-server-rust/src/services/data_type_service.rs +++ b/scylla-server-rust/src/services/data_type_service.rs @@ -2,12 +2,12 @@ use prisma_client_rust::QueryError; use crate::{prisma, Database}; -pub async fn get_all_data_types(db: Database) -> Result, QueryError> { +pub async fn get_all_data_types(db: &Database) -> Result, QueryError> { db.data_type().find_many(vec![]).exec().await } pub async fn upsert_data_type( - db: Database, + db: &Database, data_type_name: String, unit: String, node_name: String, diff --git a/scylla-server-rust/src/services/driver_service.rs b/scylla-server-rust/src/services/driver_service.rs index ade500b6..c316be24 100644 --- a/scylla-server-rust/src/services/driver_service.rs +++ b/scylla-server-rust/src/services/driver_service.rs @@ -5,12 +5,12 @@ use crate::{ Database, }; -pub async fn get_all_drivers(db: Database) -> Result, QueryError> { +pub async fn get_all_drivers(db: &Database) -> Result, QueryError> { db.driver().find_many(vec![]).exec().await } pub async fn upsert_driver( - db: Database, + db: &Database, driver_name: String, run_id: i32, ) -> Result { diff --git a/scylla-server-rust/src/services/location_service.rs b/scylla-server-rust/src/services/location_service.rs index b9bd9428..a677139b 100644 --- a/scylla-server-rust/src/services/location_service.rs +++ b/scylla-server-rust/src/services/location_service.rs @@ -5,12 +5,12 @@ use crate::{ Database, }; -pub async fn get_all_locations(db: Database) -> Result, QueryError> { +pub async fn get_all_locations(db: &Database) -> Result, QueryError> { db.location().find_many(vec![]).exec().await } pub async fn upsert_location( - db: Database, + db: &Database, name: String, latitude: f64, longitude: f64, diff --git a/scylla-server-rust/src/services/node_service.rs b/scylla-server-rust/src/services/node_service.rs index 80068e85..2381ad5c 100644 --- a/scylla-server-rust/src/services/node_service.rs +++ b/scylla-server-rust/src/services/node_service.rs @@ -2,12 +2,12 @@ use prisma_client_rust::QueryError; use crate::{prisma, Database}; -pub async fn get_all_nodes(db: Database) -> Result, QueryError> { +pub async fn get_all_nodes(db: &Database) -> Result, QueryError> { db.node().find_many(vec![]).exec().await } pub async fn upsert_node( - db: Database, + db: &Database, node_name: String, ) -> Result { db.node() diff --git a/scylla-server-rust/src/services/run_service.rs b/scylla-server-rust/src/services/run_service.rs index 6d3cdea1..77d33b37 100644 --- a/scylla-server-rust/src/services/run_service.rs +++ b/scylla-server-rust/src/services/run_service.rs @@ -7,12 +7,12 @@ use crate::{ Database, }; -pub async fn get_all_runs(db: Database) -> Result, QueryError> { +pub async fn get_all_runs(db: &Database) -> Result, QueryError> { db.run().find_many(vec![]).exec().await } pub async fn get_run_by_id( - db: Database, + db: &Database, run_id: i32, ) -> Result, QueryError> { db.run() @@ -21,7 +21,7 @@ pub async fn get_run_by_id( .await } -pub async fn create_run(db: Database, timestamp: i64) -> Result { +pub async fn create_run(db: &Database, timestamp: i64) -> Result { db.run() .create( DateTime::from_timestamp_millis(timestamp) @@ -34,7 +34,7 @@ pub async fn create_run(db: Database, timestamp: i64) -> Result Result { diff --git a/scylla-server-rust/src/services/system_service.rs b/scylla-server-rust/src/services/system_service.rs index 1b5accb5..5d31acf1 100644 --- a/scylla-server-rust/src/services/system_service.rs +++ b/scylla-server-rust/src/services/system_service.rs @@ -5,12 +5,12 @@ use crate::{ Database, }; -pub async fn get_all_systems(db: Database) -> Result, QueryError> { +pub async fn get_all_systems(db: &Database) -> Result, QueryError> { db.system().find_many(vec![]).exec().await } pub async fn upsert_system( - db: Database, + db: &Database, system_name: String, run_id: i32, ) -> Result { diff --git a/scylla-server-rust/tests/data_service_test.rs b/scylla-server-rust/tests/data_service_test.rs new file mode 100644 index 00000000..0a15f71d --- /dev/null +++ b/scylla-server-rust/tests/data_service_test.rs @@ -0,0 +1,129 @@ +#[path = "test_utils.rs"] +mod test_utils; + +use prisma_client_rust::QueryError; +use protobuf::SpecialFields; +use scylla_server_rust::{ + controller::data_controller::DataSend, + serverdata::ServerData, + services::{data_service, data_type_service, node_service, run_service}, +}; +use test_utils::cleanup_and_prepare; + +const DATA_TYPE_NAME: &str = "test"; + +#[tokio::test] +async fn test_data_service() -> Result<(), QueryError> { + let db = cleanup_and_prepare().await?; + + run_service::create_run_with_id(&db, 0, 0).await?; + node_service::upsert_node(&db, DATA_TYPE_NAME.to_owned()).await?; + data_type_service::upsert_data_type( + &db, + DATA_TYPE_NAME.to_owned(), + "joe_mama".to_owned(), + DATA_TYPE_NAME.to_owned(), + ) + .await?; + data_service::get_data(&db, DATA_TYPE_NAME.to_owned(), 0).await?; + + Ok(()) +} + +#[tokio::test] +async fn test_data_add() -> Result<(), QueryError> { + let db = cleanup_and_prepare().await?; + + node_service::upsert_node(&db, DATA_TYPE_NAME.to_owned()).await?; + data_type_service::upsert_data_type( + &db, + DATA_TYPE_NAME.to_owned(), + "joe mama".to_owned(), + DATA_TYPE_NAME.to_owned(), + ) + .await?; + let run_data = run_service::create_run(&db, 999).await?; + + let data = data_service::add_data( + &db, + ServerData { + value: vec!["0".to_owned()], + unit: "A".to_owned(), + special_fields: SpecialFields::new(), + }, + 1000, + DATA_TYPE_NAME.to_owned(), + run_data.id, + ) + .await?; + + assert_eq!( + DataSend::from(&data), + DataSend { + time: 1000, + values: vec!["0".to_owned()] + } + ); + + Ok(()) +} + +#[tokio::test] +async fn test_data_fetch_empty() -> Result<(), QueryError> { + let db = cleanup_and_prepare().await?; + + // should be empty, nothing was added to run + let data = data_service::get_data(&db, DATA_TYPE_NAME.to_owned(), 0).await?; + + assert!(data.is_empty()); + + Ok(()) +} + +#[tokio::test] +async fn test_data_no_prereqs() -> Result<(), QueryError> { + let db = cleanup_and_prepare().await?; + + // should err out as data type name doesnt exist yet + data_service::add_data( + &db, + ServerData { + value: vec!["0".to_owned()], + unit: "A".to_owned(), + special_fields: SpecialFields::new(), + }, + 1000, + DATA_TYPE_NAME.to_owned(), + 0, + ) + .await + .expect_err("Should have errored, datatype doesnt exist!"); + + // now add the node, datatype, and run + node_service::upsert_node(&db, DATA_TYPE_NAME.to_owned()).await?; + data_type_service::upsert_data_type( + &db, + DATA_TYPE_NAME.to_owned(), + "ur mom".to_owned(), + DATA_TYPE_NAME.to_owned(), + ) + .await?; + run_service::create_run_with_id(&db, 1000, 0).await?; + + // now shouldnt fail as it and node does exist + data_service::add_data( + &db, + ServerData { + value: vec!["0".to_owned()], + unit: "A".to_owned(), + special_fields: SpecialFields::new(), + }, + 1000, + DATA_TYPE_NAME.to_owned(), + 0, + ) + .await?; + + Ok(()) +} + diff --git a/scylla-server-rust/tests/data_type_service_test.rs b/scylla-server-rust/tests/data_type_service_test.rs new file mode 100644 index 00000000..172f48e4 --- /dev/null +++ b/scylla-server-rust/tests/data_type_service_test.rs @@ -0,0 +1,72 @@ +#[path = "test_utils.rs"] +mod test_utils; + +use prisma_client_rust::QueryError; +use scylla_server_rust::{ + controller::data_type_controller::DataTypeSend, + prisma, + services::{data_type_service, node_service}, +}; +use test_utils::cleanup_and_prepare; + +const DATA_TYPE_NAME: &str = "test"; + +#[tokio::test] +async fn test_get_all_datatypes() -> Result<(), QueryError> { + let db = cleanup_and_prepare().await?; + + // ensure datatypes is empty + assert!(data_type_service::get_all_data_types(&db).await?.is_empty()); + + Ok(()) +} + +#[tokio::test] +async fn test_datatype_fail_upsert_no_node() -> Result<(), QueryError> { + let db = cleanup_and_prepare().await?; + + // should fail since no node exists + data_type_service::upsert_data_type( + &db, + DATA_TYPE_NAME.to_owned(), + "hello wurld".to_owned(), + DATA_TYPE_NAME.to_owned(), + ) + .await + .expect_err("Should error, as no node exists"); + + Ok(()) +} + +#[tokio::test] +async fn test_datatype_create() -> Result<(), QueryError> { + let data_type_name: String = "test".to_owned(); + let unit: String = "testUnitCreation".to_owned(); + let node_name: String = "testNode".to_owned(); + + let db = cleanup_and_prepare().await?; + + // make node + node_service::upsert_node(&db, node_name.clone()).await?; + // upsert + data_type_service::upsert_data_type(&db, data_type_name.clone(), unit.clone(), node_name) + .await?; + + // fetch + let data = db + .data_type() + .find_unique(prisma::data_type::name::equals(data_type_name.clone())) + .exec() + .await? + .expect("This should not be empty"); + + assert_eq!( + DataTypeSend::from(&data), + DataTypeSend { + name: data_type_name, + unit: unit + } + ); + + Ok(()) +} diff --git a/scylla-server-rust/tests/driver_service_test.rs b/scylla-server-rust/tests/driver_service_test.rs new file mode 100644 index 00000000..270fd72b --- /dev/null +++ b/scylla-server-rust/tests/driver_service_test.rs @@ -0,0 +1,35 @@ +use prisma_client_rust::QueryError; +use scylla_server_rust::services::{driver_service, run_service}; +use test_utils::cleanup_and_prepare; + +#[path = "test_utils.rs"] +mod test_utils; + +const DATA_TYPE_NAME: &str = "test"; + +#[tokio::test] +async fn test_get_all_drivers() -> Result<(), QueryError> { + let db = cleanup_and_prepare().await?; + + // ensure drivers is empty + assert!(driver_service::get_all_drivers(&db).await?.is_empty()); + + Ok(()) +} + +#[tokio::test] +async fn test_create_driver() -> Result<(), QueryError> { + let db = cleanup_and_prepare().await?; + + driver_service::upsert_driver( + &db, + DATA_TYPE_NAME.to_owned(), + run_service::create_run(&db, 10001).await?.id, + ) + .await?; + + // ensure drivers is now not empty + assert!(!driver_service::get_all_drivers(&db).await?.is_empty()); + + Ok(()) +} diff --git a/scylla-server-rust/tests/location_service_test.rs b/scylla-server-rust/tests/location_service_test.rs new file mode 100644 index 00000000..5c0e570c --- /dev/null +++ b/scylla-server-rust/tests/location_service_test.rs @@ -0,0 +1,34 @@ +use prisma_client_rust::QueryError; +use scylla_server_rust::{ + prisma, + services::{location_service, run_service}, +}; +use test_utils::cleanup_and_prepare; + +#[path = "test_utils.rs"] +mod test_utils; + +const DATA_TYPE_NAME: &str = "test"; + +#[tokio::test] +async fn test_get_all_locations_and_upsert() -> Result<(), QueryError> { + let db = cleanup_and_prepare().await?; + + location_service::upsert_location( + &db, + DATA_TYPE_NAME.to_owned(), + 100.0, + 200.0, + 300.0, + run_service::create_run(&db, 10001).await?.id, + ) + .await?; + + db.location() + .find_unique(prisma::location::name::equals(DATA_TYPE_NAME.to_owned())) + .exec() + .await? + .expect("Location exist as was just upserted"); + + Ok(()) +} diff --git a/scylla-server-rust/tests/node_service_test.rs b/scylla-server-rust/tests/node_service_test.rs new file mode 100644 index 00000000..e6462d0f --- /dev/null +++ b/scylla-server-rust/tests/node_service_test.rs @@ -0,0 +1,47 @@ +use prisma_client_rust::QueryError; +use scylla_server_rust::{prisma, services::node_service}; +use test_utils::cleanup_and_prepare; + +#[path = "test_utils.rs"] +mod test_utils; + +const DATA_TYPE_NAME: &str = "test"; + +#[tokio::test] +async fn test_upsert_node() -> Result<(), QueryError> { + let db = cleanup_and_prepare().await?; + + node_service::upsert_node(&db, DATA_TYPE_NAME.to_owned()).await?; + + db.node() + .find_unique(prisma::node::name::equals(DATA_TYPE_NAME.to_owned())) + .exec() + .await? + .expect("There should be a node, one was just upserted"); + + Ok(()) +} + +#[tokio::test] +async fn test_get_all_nodes() -> Result<(), QueryError> { + let db = cleanup_and_prepare().await?; + + // ensure nodes is empty + assert!(node_service::get_all_nodes(&db).await?.is_empty()); + + Ok(()) +} + +#[tokio::test] +async fn test_upsert_node_twice() -> Result<(), QueryError> { + let db = cleanup_and_prepare().await?; + + let all_nodes = node_service::get_all_nodes(&db).await?; + node_service::upsert_node(&db, DATA_TYPE_NAME.to_owned()).await?; + node_service::upsert_node(&db, DATA_TYPE_NAME.to_owned()).await?; + let all_nodes_after = node_service::get_all_nodes(&db).await?; + + assert_eq!(all_nodes.len(), all_nodes_after.len() - 1); + + Ok(()) +} diff --git a/scylla-server-rust/tests/run_service_test.rs b/scylla-server-rust/tests/run_service_test.rs new file mode 100644 index 00000000..93d4e546 --- /dev/null +++ b/scylla-server-rust/tests/run_service_test.rs @@ -0,0 +1,35 @@ +use prisma_client_rust::QueryError; +use scylla_server_rust::{controller::run_controller::RunSend, services::run_service}; +use test_utils::cleanup_and_prepare; + +#[path = "test_utils.rs"] +mod test_utils; + +//const DATA_TYPE_NAME: &str = "test"; + +#[tokio::test] +async fn test_get_all_runs() -> Result<(), QueryError> { + let db = cleanup_and_prepare().await?; + + // ensure runs is empty + assert!(run_service::get_all_runs(&db).await?.is_empty()); + + Ok(()) +} + +#[tokio::test] +async fn test_get_run_by_id() -> Result<(), QueryError> { + let db = cleanup_and_prepare().await?; + + // add a run + let run_c = run_service::create_run(&db, 1).await?; + + // get that run + let run = run_service::get_run_by_id(&db, run_c.id) + .await? + .expect("Run should exist was upserted "); + + assert_eq!(RunSend::from(&run_c), RunSend::from(&run)); + + Ok(()) +} diff --git a/scylla-server-rust/tests/system_service_test.rs b/scylla-server-rust/tests/system_service_test.rs new file mode 100644 index 00000000..d7b33a5b --- /dev/null +++ b/scylla-server-rust/tests/system_service_test.rs @@ -0,0 +1,65 @@ +use prisma_client_rust::QueryError; +use scylla_server_rust::{ + controller::system_controller::SystemSend, + prisma, + services::{run_service, system_service}, +}; +use test_utils::cleanup_and_prepare; + +#[path = "test_utils.rs"] +mod test_utils; + +const DATA_TYPE_NAME: &str = "test"; + +#[tokio::test] +async fn test_upsert_system_create() -> Result<(), QueryError> { + let db = cleanup_and_prepare().await?; + + let res_c = system_service::upsert_system( + &db, + DATA_TYPE_NAME.to_owned(), + run_service::create_run(&db, 101).await?.id, + ) + .await?; + + let res = db + .system() + .find_unique(prisma::system::name::equals(DATA_TYPE_NAME.to_owned())) + .exec() + .await? + .expect("System should exist, was just upserted!"); + + assert_eq!(SystemSend::from(&res_c), SystemSend::from(&res)); + + Ok(()) +} + +#[tokio::test] +async fn test_get_all_systems() -> Result<(), QueryError> { + let db = cleanup_and_prepare().await?; + + // ensure runs is empty + assert!(system_service::get_all_systems(&db).await?.is_empty()); + + Ok(()) +} + +#[tokio::test] +async fn test_get_upsert_system() -> Result<(), QueryError> { + let db = cleanup_and_prepare().await?; + + system_service::upsert_system( + &db, + DATA_TYPE_NAME.to_owned(), + run_service::create_run(&db, 101).await?.id, + ) + .await?; + + let sys = system_service::get_all_systems(&db).await?; + + sys.iter() + .find(|&f| f.name == DATA_TYPE_NAME.to_owned()) + .expect("System of the added name should exist in the list of systems"); + + Ok(()) +} diff --git a/scylla-server-rust/tests/test_utils.rs b/scylla-server-rust/tests/test_utils.rs new file mode 100644 index 00000000..bc3db82f --- /dev/null +++ b/scylla-server-rust/tests/test_utils.rs @@ -0,0 +1,24 @@ +use std::sync::Arc; + +use prisma_client_rust::QueryError; +use scylla_server_rust::{prisma::PrismaClient, Database}; + +pub async fn cleanup_and_prepare() -> Result { + let client = Arc::new(PrismaClient::_builder().build().await.unwrap()); + + client.data().delete_many(vec![]).exec().await?; + + client.data_type().delete_many(vec![]).exec().await?; + + client.driver().delete_many(vec![]).exec().await?; + + client.location().delete_many(vec![]).exec().await?; + + client.node().delete_many(vec![]).exec().await?; + + client.run().delete_many(vec![]).exec().await?; + + client.system().delete_many(vec![]).exec().await?; + + Ok(client) +} From 9a620476109de0bcae93c9bf412db52558dacb93 Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Sat, 6 Jul 2024 22:14:05 -0400 Subject: [PATCH 07/68] security checks? not at ner we dont --- .github/workflows/scylla-rust-ci.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/scylla-rust-ci.yml b/.github/workflows/scylla-rust-ci.yml index 0ae620c3..8b3a7c22 100644 --- a/.github/workflows/scylla-rust-ci.yml +++ b/.github/workflows/scylla-rust-ci.yml @@ -25,13 +25,9 @@ jobs: uses: actions/checkout@v3 with: submodules: recursive - - name: Install cargo-audit - run: cargo install cargo-audit - name: Generate prisma run: cargo prisma generate - name: Build run: cargo build --verbose - name: Clippy run: cargo clippy --verbose -- -D warnings - - name: Audit - run: cargo audit From 85f696af796caff718467d419ba5d74dce36a0a8 Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Sat, 6 Jul 2024 22:19:01 -0400 Subject: [PATCH 08/68] remove shell.nix --- scylla-server-rust/shell.nix | 30 ------------------------------ 1 file changed, 30 deletions(-) delete mode 100755 scylla-server-rust/shell.nix diff --git a/scylla-server-rust/shell.nix b/scylla-server-rust/shell.nix deleted file mode 100755 index 0db6465f..00000000 --- a/scylla-server-rust/shell.nix +++ /dev/null @@ -1,30 +0,0 @@ -# ln -s /home/jack/Documents/nixos/shell-rust.nix /home/jack/Projects/NER/Calypso/shell.nix - -{ - pkgs ? import { }, -}: -pkgs.mkShell { - nativeBuildInputs = with pkgs; [ - rustc - cargo - gcc - rustfmt - clippy - - # for paho-mqtt - openssl - pkg-config - cmake - python3 - python311Packages.ruamel-yaml-clib -python311Packages.ruamel-base -python311Packages.ruamel-yaml - # rust - rust-analyzer - ]; - - # Certain Rust tools won't work without this - # This can also be fixed by using oxalica/rust-overlay and specifying the rust-src extension - # See https://discourse.nixos.org/t/rust-src-not-found-and-other-misadventures-of-developing-rust-on-nixos/11570/3?u=samuela. for more details. - RUST_SRC_PATH = "${pkgs.rust.packages.stable.rustPlatform.rustLibSrc}"; -} From 11c3a1ae2b257ffcc51d9155fc3a4c3fa416f5bc Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Sat, 6 Jul 2024 22:23:15 -0400 Subject: [PATCH 09/68] newline fixes --- scylla-server-rust/src/lib.rs | 2 +- scylla-server-rust/tests/data_service_test.rs | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/scylla-server-rust/src/lib.rs b/scylla-server-rust/src/lib.rs index c40a46d2..2f2b8afc 100644 --- a/scylla-server-rust/src/lib.rs +++ b/scylla-server-rust/src/lib.rs @@ -9,4 +9,4 @@ pub mod prisma; pub mod serverdata; /// The type descriptor of the database passed to the middlelayer through axum state -pub type Database = std::sync::Arc; \ No newline at end of file +pub type Database = std::sync::Arc; diff --git a/scylla-server-rust/tests/data_service_test.rs b/scylla-server-rust/tests/data_service_test.rs index 0a15f71d..32d9277a 100644 --- a/scylla-server-rust/tests/data_service_test.rs +++ b/scylla-server-rust/tests/data_service_test.rs @@ -126,4 +126,3 @@ async fn test_data_no_prereqs() -> Result<(), QueryError> { Ok(()) } - From c2ebaec1b1d9173474d1d1ec86cd0e15e73eeea3 Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Sun, 7 Jul 2024 16:54:01 -0400 Subject: [PATCH 10/68] make refactoring and fixes according to PR comments --- .../src/controller/data_type_controller.rs | 29 -------------- .../src/controller/driver_controller.rs | 37 ------------------ .../src/controller/node_controller.rs | 36 ----------------- .../src/controller/system_controller.rs | 37 ------------------ .../data_controller.rs | 16 ++++---- .../src/controllers/data_type_controller.rs | 31 +++++++++++++++ .../src/controllers/driver_controller.rs | 39 +++++++++++++++++++ .../location_controller.rs | 22 ++++++----- .../src/{controller => controllers}/mod.rs | 0 .../src/controllers/node_controller.rs | 39 +++++++++++++++++++ .../run_controller.rs | 33 +++++++--------- .../src/controllers/system_controller.rs | 38 ++++++++++++++++++ scylla-server-rust/src/lib.rs | 2 +- scylla-server-rust/src/main.rs | 13 ++++++- .../src/services/data_service.rs | 14 ++++++- .../src/services/data_type_service.rs | 9 +++++ .../src/services/driver_service.rs | 8 ++++ .../src/services/location_service.rs | 11 ++++++ .../src/services/node_service.rs | 7 ++++ .../src/services/run_service.rs | 16 ++++++++ .../src/services/system_service.rs | 11 ++++++ scylla-server-rust/tests/data_service_test.rs | 36 ++++++++--------- .../tests/data_type_service_test.rs | 15 ++++--- .../tests/driver_service_test.rs | 4 +- .../tests/location_service_test.rs | 6 +-- scylla-server-rust/tests/node_service_test.rs | 10 ++--- scylla-server-rust/tests/run_service_test.rs | 6 +-- .../tests/system_service_test.rs | 14 +++---- 28 files changed, 311 insertions(+), 228 deletions(-) delete mode 100644 scylla-server-rust/src/controller/data_type_controller.rs delete mode 100644 scylla-server-rust/src/controller/driver_controller.rs delete mode 100644 scylla-server-rust/src/controller/node_controller.rs delete mode 100644 scylla-server-rust/src/controller/system_controller.rs rename scylla-server-rust/src/{controller => controllers}/data_controller.rs (62%) create mode 100644 scylla-server-rust/src/controllers/data_type_controller.rs create mode 100644 scylla-server-rust/src/controllers/driver_controller.rs rename scylla-server-rust/src/{controller => controllers}/location_controller.rs (55%) rename scylla-server-rust/src/{controller => controllers}/mod.rs (100%) create mode 100644 scylla-server-rust/src/controllers/node_controller.rs rename scylla-server-rust/src/{controller => controllers}/run_controller.rs (55%) create mode 100644 scylla-server-rust/src/controllers/system_controller.rs diff --git a/scylla-server-rust/src/controller/data_type_controller.rs b/scylla-server-rust/src/controller/data_type_controller.rs deleted file mode 100644 index 25aadafb..00000000 --- a/scylla-server-rust/src/controller/data_type_controller.rs +++ /dev/null @@ -1,29 +0,0 @@ -use axum::{extract::State, Json}; -use serde::Serialize; - -use crate::{error::ScyllaError, prisma, services::data_type_service, Database}; - -#[derive(Serialize, Debug, PartialEq)] -pub struct DataTypeSend { - pub name: String, - pub unit: String, -} - -impl From<&prisma::data_type::Data> for DataTypeSend { - fn from(value: &prisma::data_type::Data) -> Self { - DataTypeSend { - name: value.name.clone(), - unit: value.unit.clone(), - } - } -} - -pub async fn get_all_data_types( - State(db): State, -) -> Result>, ScyllaError> { - let data = data_type_service::get_all_data_types(&db).await?; - - let transformed_data: Vec = data.iter().map(DataTypeSend::from).collect(); - - Ok(Json::from(transformed_data)) -} diff --git a/scylla-server-rust/src/controller/driver_controller.rs b/scylla-server-rust/src/controller/driver_controller.rs deleted file mode 100644 index 137db707..00000000 --- a/scylla-server-rust/src/controller/driver_controller.rs +++ /dev/null @@ -1,37 +0,0 @@ -use axum::{extract::State, Json}; -use serde::Serialize; - -use crate::{error::ScyllaError, prisma, services::driver_service, Database}; - -use super::run_controller::{self, RunSend}; - -#[derive(Serialize, PartialEq)] -pub struct DriverSend { - username: String, - runs: Vec, -} - -impl From<&prisma::driver::Data> for DriverSend { - fn from(value: &prisma::driver::Data) -> Self { - DriverSend { - username: value.username.clone(), - runs: value - .runs - .clone() - .unwrap_or_default() - .iter() - .map(RunSend::from) - .collect(), - } - } -} - -pub async fn get_all_drivers( - State(db): State, -) -> Result>, ScyllaError> { - let data = driver_service::get_all_drivers(&db).await?; - - let transformed_data: Vec = data.iter().map(DriverSend::from).collect(); - - Ok(Json::from(transformed_data)) -} diff --git a/scylla-server-rust/src/controller/node_controller.rs b/scylla-server-rust/src/controller/node_controller.rs deleted file mode 100644 index ed1743ba..00000000 --- a/scylla-server-rust/src/controller/node_controller.rs +++ /dev/null @@ -1,36 +0,0 @@ -use axum::{extract::State, Json}; -use serde::Serialize; - -use crate::{error::ScyllaError, prisma, services::node_service, Database}; - -use super::data_type_controller::DataTypeSend; - -#[derive(Serialize, PartialEq)] -pub struct NodeSend { - name: String, - #[serde(rename = "dataTypes")] - data_types: Vec, -} - -impl From<&prisma::node::Data> for NodeSend { - fn from(value: &prisma::node::Data) -> Self { - NodeSend { - name: value.name.clone(), - data_types: value - .data_types - .clone() - .unwrap_or_default() - .iter() - .map(DataTypeSend::from) - .collect(), - } - } -} - -pub async fn get_all_nodes(State(db): State) -> Result>, ScyllaError> { - let data = node_service::get_all_nodes(&db).await?; - - let transformed_data: Vec = data.iter().map(NodeSend::from).collect(); - - Ok(Json::from(transformed_data)) -} diff --git a/scylla-server-rust/src/controller/system_controller.rs b/scylla-server-rust/src/controller/system_controller.rs deleted file mode 100644 index c4dcdf3b..00000000 --- a/scylla-server-rust/src/controller/system_controller.rs +++ /dev/null @@ -1,37 +0,0 @@ -use axum::{extract::State, Json}; -use serde::Serialize; - -use crate::{error::ScyllaError, prisma, services::system_service, Database}; - -use super::run_controller::{self, RunSend}; - -#[derive(Serialize, Debug, PartialEq)] -pub struct SystemSend { - pub name: String, - pub runs: Vec, -} - -impl From<&prisma::system::Data> for SystemSend { - fn from(value: &prisma::system::Data) -> Self { - SystemSend { - name: value.name.clone(), - runs: value - .runs - .clone() - .unwrap_or_default() - .iter() - .map(RunSend::from) - .collect(), - } - } -} - -pub async fn get_all_systems( - State(db): State, -) -> Result>, ScyllaError> { - let data = system_service::get_all_systems(&db).await?; - - let transformed_data: Vec = data.iter().map(SystemSend::from).collect(); - - Ok(Json::from(transformed_data)) -} diff --git a/scylla-server-rust/src/controller/data_controller.rs b/scylla-server-rust/src/controllers/data_controller.rs similarity index 62% rename from scylla-server-rust/src/controller/data_controller.rs rename to scylla-server-rust/src/controllers/data_controller.rs index 3edf9939..55fd9fbe 100644 --- a/scylla-server-rust/src/controller/data_controller.rs +++ b/scylla-server-rust/src/controllers/data_controller.rs @@ -11,16 +11,17 @@ use crate::{ Database, }; +/// The struct defining the data format sent to the client #[derive(Serialize, Debug, PartialEq, Eq, PartialOrd, Ord)] -pub struct DataSend { +pub struct PublicData { pub time: i64, pub values: Vec, } -// convert the prisma type to the client type for JSON encoding -impl From<&prisma::data::Data> for DataSend { +/// convert the prisma type to the client type for JSON encoding +impl From<&prisma::data::Data> for PublicData { fn from(value: &prisma::data::Data) -> Self { - DataSend { + PublicData { values: value.values.iter().map(|f| f.to_string()).collect(), time: value.time.timestamp_millis(), } @@ -30,12 +31,11 @@ impl From<&prisma::data::Data> for DataSend { pub async fn get_data( State(db): State, Path((data_type_name, run_id)): Path<(String, i32)>, -) -> Result>, ScyllaError> { +) -> Result>, ScyllaError> { let data = data_service::get_data(&db, data_type_name, run_id).await?; - // map data to correct types - let mut transformed_data: Vec = data.iter().map(DataSend::from).collect(); - // sort it by time for the client + // map data to frontend data types according to the From func of the client struct + let mut transformed_data: Vec = data.iter().map(PublicData::from).collect(); transformed_data.sort(); Ok(Json::from(transformed_data)) diff --git a/scylla-server-rust/src/controllers/data_type_controller.rs b/scylla-server-rust/src/controllers/data_type_controller.rs new file mode 100644 index 00000000..279857ad --- /dev/null +++ b/scylla-server-rust/src/controllers/data_type_controller.rs @@ -0,0 +1,31 @@ +use axum::{extract::State, Json}; +use serde::Serialize; + +use crate::{error::ScyllaError, prisma, services::data_type_service, Database}; + +/// The struct defining the data type format sent to the client +#[derive(Serialize, Debug, PartialEq)] +pub struct PublicDataType { + pub name: String, + pub unit: String, +} + +impl From<&prisma::data_type::Data> for PublicDataType { + fn from(value: &prisma::data_type::Data) -> Self { + PublicDataType { + name: value.name.clone(), + unit: value.unit.clone(), + } + } +} + +pub async fn get_all_data_types( + State(db): State, +) -> Result>, ScyllaError> { + let data_types = data_type_service::get_all_data_types(&db).await?; + + let transformed_data_types: Vec = + data_types.iter().map(PublicDataType::from).collect(); + + Ok(Json::from(transformed_data_types)) +} diff --git a/scylla-server-rust/src/controllers/driver_controller.rs b/scylla-server-rust/src/controllers/driver_controller.rs new file mode 100644 index 00000000..283ddff7 --- /dev/null +++ b/scylla-server-rust/src/controllers/driver_controller.rs @@ -0,0 +1,39 @@ +use axum::{extract::State, Json}; +use serde::Serialize; + +use crate::{error::ScyllaError, prisma, services::driver_service, Database}; + +use super::run_controller::{self, PublicRun}; + +/// The struct defining the driver format sent to the client +#[derive(Serialize, PartialEq)] +pub struct PublicDriver { + username: String, + runs: Vec, +} + +impl From<&prisma::driver::Data> for PublicDriver { + fn from(value: &prisma::driver::Data) -> Self { + PublicDriver { + username: value.username.clone(), + runs: value + .runs + .clone() + .unwrap_or_default() + .iter() + .map(PublicRun::from) + .collect(), + } + } +} + +pub async fn get_all_drivers( + State(db): State, +) -> Result>, ScyllaError> { + let driver_data = driver_service::get_all_drivers(&db).await?; + + let transformed_driver_data: Vec = + driver_data.iter().map(PublicDriver::from).collect(); + + Ok(Json::from(transformed_driver_data)) +} diff --git a/scylla-server-rust/src/controller/location_controller.rs b/scylla-server-rust/src/controllers/location_controller.rs similarity index 55% rename from scylla-server-rust/src/controller/location_controller.rs rename to scylla-server-rust/src/controllers/location_controller.rs index 1bd861ee..92f44758 100644 --- a/scylla-server-rust/src/controller/location_controller.rs +++ b/scylla-server-rust/src/controllers/location_controller.rs @@ -3,20 +3,21 @@ use serde::Serialize; use crate::{error::ScyllaError, prisma, services::location_service, Database}; -use super::run_controller::{self, RunSend}; +use super::run_controller::{self, PublicRun}; +/// The struct defining the location format sent to the client #[derive(Serialize, Debug, PartialEq)] -pub struct LocationSend { +pub struct PublicLocation { pub name: String, pub latitude: f64, pub longitude: f64, pub radius: f64, - pub runs: Vec, + pub runs: Vec, } -impl From<&prisma::location::Data> for LocationSend { +impl From<&prisma::location::Data> for PublicLocation { fn from(value: &prisma::location::Data) -> Self { - LocationSend { + PublicLocation { name: value.name.clone(), latitude: value.latitude, longitude: value.longitude, @@ -26,7 +27,7 @@ impl From<&prisma::location::Data> for LocationSend { .clone() .unwrap_or_default() .iter() - .map(RunSend::from) + .map(PublicRun::from) .collect(), } } @@ -34,10 +35,11 @@ impl From<&prisma::location::Data> for LocationSend { pub async fn get_all_locations( State(db): State, -) -> Result>, ScyllaError> { - let data = location_service::get_all_locations(&db).await?; +) -> Result>, ScyllaError> { + let loc_data = location_service::get_all_locations(&db).await?; - let transformed_data: Vec = data.iter().map(LocationSend::from).collect(); + let transformed_loc_data: Vec = + loc_data.iter().map(PublicLocation::from).collect(); - Ok(Json::from(transformed_data)) + Ok(Json::from(transformed_loc_data)) } diff --git a/scylla-server-rust/src/controller/mod.rs b/scylla-server-rust/src/controllers/mod.rs similarity index 100% rename from scylla-server-rust/src/controller/mod.rs rename to scylla-server-rust/src/controllers/mod.rs diff --git a/scylla-server-rust/src/controllers/node_controller.rs b/scylla-server-rust/src/controllers/node_controller.rs new file mode 100644 index 00000000..2ba88179 --- /dev/null +++ b/scylla-server-rust/src/controllers/node_controller.rs @@ -0,0 +1,39 @@ +use axum::{extract::State, Json}; +use serde::Serialize; + +use crate::{error::ScyllaError, prisma, services::node_service, Database}; + +use super::data_type_controller::PublicDataType; + +/// The struct defining the node format sent to the client +#[derive(Serialize, PartialEq)] +pub struct PublicNode { + name: String, + #[serde(rename = "dataTypes")] + data_types: Vec, +} + +impl From<&prisma::node::Data> for PublicNode { + fn from(value: &prisma::node::Data) -> Self { + PublicNode { + name: value.name.clone(), + data_types: value + .data_types + .clone() + .unwrap_or_default() + .iter() + .map(PublicDataType::from) + .collect(), + } + } +} + +pub async fn get_all_nodes( + State(db): State, +) -> Result>, ScyllaError> { + let node_data = node_service::get_all_nodes(&db).await?; + + let transformed_node_data: Vec = node_data.iter().map(PublicNode::from).collect(); + + Ok(Json::from(transformed_node_data)) +} diff --git a/scylla-server-rust/src/controller/run_controller.rs b/scylla-server-rust/src/controllers/run_controller.rs similarity index 55% rename from scylla-server-rust/src/controller/run_controller.rs rename to scylla-server-rust/src/controllers/run_controller.rs index 7e9fe983..c1a2568f 100644 --- a/scylla-server-rust/src/controller/run_controller.rs +++ b/scylla-server-rust/src/controllers/run_controller.rs @@ -6,8 +6,9 @@ use serde::Serialize; use crate::{error::ScyllaError, prisma, services::run_service, Database}; +/// The struct defining the run format sent to the client #[derive(Serialize, Debug, PartialEq)] -pub struct RunSend { +pub struct PublicRun { pub id: i32, #[serde(rename = "locationName")] pub location_name: String, @@ -18,9 +19,9 @@ pub struct RunSend { pub time: i64, } -impl From<&prisma::run::Data> for RunSend { +impl From<&prisma::run::Data> for PublicRun { fn from(value: &prisma::run::Data) -> Self { - RunSend { + PublicRun { id: value.id, location_name: value.location_name.clone().unwrap_or_default(), driver_name: value.driver_name.clone().unwrap_or_default(), @@ -30,33 +31,27 @@ impl From<&prisma::run::Data> for RunSend { } } -pub async fn get_all_runs(State(db): State) -> Result>, ScyllaError> { - let data = run_service::get_all_runs(&db).await?; +pub async fn get_all_runs(State(db): State) -> Result>, ScyllaError> { + let run_data = run_service::get_all_runs(&db).await?; - let transformed_data: Vec = data.iter().map(RunSend::from).collect(); + let transformed_run_data: Vec = run_data.iter().map(PublicRun::from).collect(); - Ok(Json::from(transformed_data)) + Ok(Json::from(transformed_run_data)) } pub async fn get_run_by_id( State(db): State, Path(run_id): Path, -) -> Result, ScyllaError> { - let data = run_service::get_run_by_id(&db, run_id).await?; +) -> Result, ScyllaError> { + let run_data = run_service::get_run_by_id(&db, run_id).await?; - if data.is_none() { + if run_data.is_none() { return Err(ScyllaError::NotFound); } - let data_new = data.unwrap(); + let run_data_safe = run_data.unwrap(); - let transformed_data = RunSend { - id: data_new.id, - location_name: data_new.location_name.unwrap_or_default(), - driver_name: data_new.driver_name.unwrap_or_default(), - system_name: data_new.system_name.unwrap_or_default(), - time: data_new.time.timestamp_millis(), - }; + let transformed_run_data = PublicRun::from(&run_data_safe); - Ok(Json::from(transformed_data)) + Ok(Json::from(transformed_run_data)) } diff --git a/scylla-server-rust/src/controllers/system_controller.rs b/scylla-server-rust/src/controllers/system_controller.rs new file mode 100644 index 00000000..4e51d5e8 --- /dev/null +++ b/scylla-server-rust/src/controllers/system_controller.rs @@ -0,0 +1,38 @@ +use axum::{extract::State, Json}; +use serde::Serialize; + +use crate::{error::ScyllaError, prisma, services::system_service, Database}; + +use super::run_controller::{self, PublicRun}; + +/// The struct defining the system format sent to the client +#[derive(Serialize, Debug, PartialEq)] +pub struct PublicSystem { + pub name: String, + pub runs: Vec, +} + +impl From<&prisma::system::Data> for PublicSystem { + fn from(value: &prisma::system::Data) -> Self { + PublicSystem { + name: value.name.clone(), + runs: value + .runs + .clone() + .unwrap_or_default() + .iter() + .map(PublicRun::from) + .collect(), + } + } +} + +pub async fn get_all_systems( + State(db): State, +) -> Result>, ScyllaError> { + let run_data = system_service::get_all_systems(&db).await?; + + let transformed_run_data: Vec = run_data.iter().map(PublicSystem::from).collect(); + + Ok(Json::from(transformed_run_data)) +} diff --git a/scylla-server-rust/src/lib.rs b/scylla-server-rust/src/lib.rs index 2f2b8afc..2c9cff5f 100644 --- a/scylla-server-rust/src/lib.rs +++ b/scylla-server-rust/src/lib.rs @@ -1,4 +1,4 @@ -pub mod controller; +pub mod controllers; pub mod error; pub mod services; diff --git a/scylla-server-rust/src/main.rs b/scylla-server-rust/src/main.rs index 50bd8042..533d9f76 100755 --- a/scylla-server-rust/src/main.rs +++ b/scylla-server-rust/src/main.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use axum::{http::Method, routing::get, Router}; use scylla_server_rust::{ - controller::{ + controllers::{ self, data_type_controller, driver_controller, location_controller, node_controller, run_controller, system_controller, }, @@ -16,16 +16,25 @@ async fn main() { let client: Database = Arc::new(PrismaClient::_builder().build().await.unwrap()); let app = Router::new() + // get all data with the name dataTypeName and runID as specified .route( "/data/:dataTypeName/:runId", - get(controller::data_controller::get_data), + get(controllers::data_controller::get_data), ) + // get all datatypes .route("/datatypes", get(data_type_controller::get_all_data_types)) + // get all drivers .route("/drivers", get(driver_controller::get_all_drivers)) + // get all locations .route("/locations", get(location_controller::get_all_locations)) + // get all nodes .route("/nodes", get(node_controller::get_all_nodes)) + // runs: + // get all runs .route("/runs", get(run_controller::get_all_runs)) + // get run with id .route("/runs/:id", get(run_controller::get_run_by_id)) + // get all systems .route("/systems", get(system_controller::get_all_systems)) .layer( CorsLayer::new() diff --git a/scylla-server-rust/src/services/data_service.rs b/scylla-server-rust/src/services/data_service.rs index e37070cd..3283b1db 100644 --- a/scylla-server-rust/src/services/data_service.rs +++ b/scylla-server-rust/src/services/data_service.rs @@ -6,7 +6,11 @@ use crate::{ Database, }; -/// Get a datapoint +/// Get datapoints that mach criteria +/// * `db` - The prisma client to make the call to +/// * `data_type_name` - The data type name to filter the data by +/// * `run_id` - The run id to filter the data by +/// returns: A result containing the data or the QueryError propogated by the db pub async fn get_data( db: &Database, data_type_name: String, @@ -21,7 +25,13 @@ pub async fn get_data( .await } -/// Add a datapoint +/// Adds a datapoint +/// * `db` - The prisma client to make the call to +/// * `serverdata` - The protobuf message to parse, note the unit is ignored! +/// * `unix_time` - The time im miliseconds since unix epoch of the message +/// * `data_type_name` - The name of the data type, note this data type must already exist! +/// * `rin_id` - The run id to assign the data point to, note this run must already exist! +/// returns: A result containing the data or the QueryError propogated by the db pub async fn add_data( db: &Database, serverdata: ServerData, diff --git a/scylla-server-rust/src/services/data_type_service.rs b/scylla-server-rust/src/services/data_type_service.rs index 64f3e23f..00bf25e2 100644 --- a/scylla-server-rust/src/services/data_type_service.rs +++ b/scylla-server-rust/src/services/data_type_service.rs @@ -2,10 +2,19 @@ use prisma_client_rust::QueryError; use crate::{prisma, Database}; +/// Gets all datatypes +/// * `db` - The prisma client to make the call to +/// returns: A result containing the data or the QueryError propogated by the db pub async fn get_all_data_types(db: &Database) -> Result, QueryError> { db.data_type().find_many(vec![]).exec().await } +/// Upserts a datatype, either creating or updating one depending on its existence +/// * `db` - The prisma client to make the call to +/// * `data_type_name` - The data type name to upsert +/// * `unit` - The unit of the data +/// * `node_name` - The name of the node linked to the data type, must already exist! +/// returns: A result containing the data or the QueryError propogated by the db pub async fn upsert_data_type( db: &Database, data_type_name: String, diff --git a/scylla-server-rust/src/services/driver_service.rs b/scylla-server-rust/src/services/driver_service.rs index c316be24..18145aca 100644 --- a/scylla-server-rust/src/services/driver_service.rs +++ b/scylla-server-rust/src/services/driver_service.rs @@ -5,10 +5,18 @@ use crate::{ Database, }; +/// Gets all drivers +/// * `db` - The prisma client to make the call to +/// returns: A result containing the data or the QueryError propogated by the db pub async fn get_all_drivers(db: &Database) -> Result, QueryError> { db.driver().find_many(vec![]).exec().await } +/// Upserts a driver, either creating or updating one depending on its existence +/// * `db` - The prisma client to make the call to +/// * `driver_name` - The name of the driver to upsert +/// * `run_id` - The id of the run to link to the driver, must already exist! +/// returns: A result containing the data or the QueryError propogated by the db pub async fn upsert_driver( db: &Database, driver_name: String, diff --git a/scylla-server-rust/src/services/location_service.rs b/scylla-server-rust/src/services/location_service.rs index a677139b..2e1483d8 100644 --- a/scylla-server-rust/src/services/location_service.rs +++ b/scylla-server-rust/src/services/location_service.rs @@ -5,10 +5,21 @@ use crate::{ Database, }; +/// Gets all locations +/// * `db` - The prisma client to make the call to +/// returns: A result containing the data or the QueryError propogated by the db pub async fn get_all_locations(db: &Database) -> Result, QueryError> { db.location().find_many(vec![]).exec().await } +/// Upserts a location, either creating or updating one depending on its existence +/// * `db` - The prisma client to make the call to +/// * `name` - The name of the location to upsert +/// * `latitude` - The latitude of the location +/// * `longitude` - The longitude of the location +/// * `radius` - The radius of the locations bounds +/// * `run_id` - The run at the location, must already exist! +/// returns: A result containing the data or the QueryError propogated by the db pub async fn upsert_location( db: &Database, name: String, diff --git a/scylla-server-rust/src/services/node_service.rs b/scylla-server-rust/src/services/node_service.rs index 2381ad5c..5f8608af 100644 --- a/scylla-server-rust/src/services/node_service.rs +++ b/scylla-server-rust/src/services/node_service.rs @@ -2,10 +2,17 @@ use prisma_client_rust::QueryError; use crate::{prisma, Database}; +/// Gets all nodes +/// * `db` - The prisma client to make the call to +/// returns: A result containing the data or the QueryError propogated by the db pub async fn get_all_nodes(db: &Database) -> Result, QueryError> { db.node().find_many(vec![]).exec().await } +/// Upserts a node, either creating or updating one depending on its existence +/// * `db` - The prisma client to make the call to +/// * `node_name` - The name of the node linked to the data type +/// returns: A result containing the data or the QueryError propogated by the db pub async fn upsert_node( db: &Database, node_name: String, diff --git a/scylla-server-rust/src/services/run_service.rs b/scylla-server-rust/src/services/run_service.rs index 77d33b37..1a126040 100644 --- a/scylla-server-rust/src/services/run_service.rs +++ b/scylla-server-rust/src/services/run_service.rs @@ -7,10 +7,17 @@ use crate::{ Database, }; +/// Gets all runs +/// * `db` - The prisma client to make the call to +/// returns: A result containing the data or the QueryError propogated by the db pub async fn get_all_runs(db: &Database) -> Result, QueryError> { db.run().find_many(vec![]).exec().await } +/// Gets a single run by its id +/// * `db` - The prisma client to make the call to +/// * `run_id` - The id of the run to search for +/// returns: A result containing the data (or None if the `run_id` was not a valid run) or the QueryError propogated by the db pub async fn get_run_by_id( db: &Database, run_id: i32, @@ -21,6 +28,10 @@ pub async fn get_run_by_id( .await } +/// Creates a run +/// * `db` - The prisma client to make the call to +/// * `timestamp` - The unix time since epoch in miliseconds when the run starts +/// returns: A result containing the data or the QueryError propogated by the db pub async fn create_run(db: &Database, timestamp: i64) -> Result { db.run() .create( @@ -33,6 +44,11 @@ pub async fn create_run(db: &Database, timestamp: i64) -> Result Result, QueryError> { db.system().find_many(vec![]).exec().await } +/// Upserts a system, either creating or updating one depending on its existence +/// * `db` - The prisma client to make the call to +/// * `system_name` - The system name name to upsert +/// * `run_id` - The id of the run to link to the system, must already exist +/// returns: A result containing the data or the QueryError propogated by the db pub async fn upsert_system( db: &Database, system_name: String, diff --git a/scylla-server-rust/tests/data_service_test.rs b/scylla-server-rust/tests/data_service_test.rs index 32d9277a..6f420963 100644 --- a/scylla-server-rust/tests/data_service_test.rs +++ b/scylla-server-rust/tests/data_service_test.rs @@ -4,28 +4,28 @@ mod test_utils; use prisma_client_rust::QueryError; use protobuf::SpecialFields; use scylla_server_rust::{ - controller::data_controller::DataSend, + controllers::data_controller::PublicData, serverdata::ServerData, services::{data_service, data_type_service, node_service, run_service}, }; use test_utils::cleanup_and_prepare; -const DATA_TYPE_NAME: &str = "test"; +const TEST_KEYWORD: &str = "test"; #[tokio::test] async fn test_data_service() -> Result<(), QueryError> { let db = cleanup_and_prepare().await?; run_service::create_run_with_id(&db, 0, 0).await?; - node_service::upsert_node(&db, DATA_TYPE_NAME.to_owned()).await?; + node_service::upsert_node(&db, TEST_KEYWORD.to_owned()).await?; data_type_service::upsert_data_type( &db, - DATA_TYPE_NAME.to_owned(), + TEST_KEYWORD.to_owned(), "joe_mama".to_owned(), - DATA_TYPE_NAME.to_owned(), + TEST_KEYWORD.to_owned(), ) .await?; - data_service::get_data(&db, DATA_TYPE_NAME.to_owned(), 0).await?; + data_service::get_data(&db, TEST_KEYWORD.to_owned(), 0).await?; Ok(()) } @@ -34,12 +34,12 @@ async fn test_data_service() -> Result<(), QueryError> { async fn test_data_add() -> Result<(), QueryError> { let db = cleanup_and_prepare().await?; - node_service::upsert_node(&db, DATA_TYPE_NAME.to_owned()).await?; + node_service::upsert_node(&db, TEST_KEYWORD.to_owned()).await?; data_type_service::upsert_data_type( &db, - DATA_TYPE_NAME.to_owned(), + TEST_KEYWORD.to_owned(), "joe mama".to_owned(), - DATA_TYPE_NAME.to_owned(), + TEST_KEYWORD.to_owned(), ) .await?; let run_data = run_service::create_run(&db, 999).await?; @@ -52,14 +52,14 @@ async fn test_data_add() -> Result<(), QueryError> { special_fields: SpecialFields::new(), }, 1000, - DATA_TYPE_NAME.to_owned(), + TEST_KEYWORD.to_owned(), run_data.id, ) .await?; assert_eq!( - DataSend::from(&data), - DataSend { + PublicData::from(&data), + PublicData { time: 1000, values: vec!["0".to_owned()] } @@ -73,7 +73,7 @@ async fn test_data_fetch_empty() -> Result<(), QueryError> { let db = cleanup_and_prepare().await?; // should be empty, nothing was added to run - let data = data_service::get_data(&db, DATA_TYPE_NAME.to_owned(), 0).await?; + let data = data_service::get_data(&db, TEST_KEYWORD.to_owned(), 0).await?; assert!(data.is_empty()); @@ -93,19 +93,19 @@ async fn test_data_no_prereqs() -> Result<(), QueryError> { special_fields: SpecialFields::new(), }, 1000, - DATA_TYPE_NAME.to_owned(), + TEST_KEYWORD.to_owned(), 0, ) .await .expect_err("Should have errored, datatype doesnt exist!"); // now add the node, datatype, and run - node_service::upsert_node(&db, DATA_TYPE_NAME.to_owned()).await?; + node_service::upsert_node(&db, TEST_KEYWORD.to_owned()).await?; data_type_service::upsert_data_type( &db, - DATA_TYPE_NAME.to_owned(), + TEST_KEYWORD.to_owned(), "ur mom".to_owned(), - DATA_TYPE_NAME.to_owned(), + TEST_KEYWORD.to_owned(), ) .await?; run_service::create_run_with_id(&db, 1000, 0).await?; @@ -119,7 +119,7 @@ async fn test_data_no_prereqs() -> Result<(), QueryError> { special_fields: SpecialFields::new(), }, 1000, - DATA_TYPE_NAME.to_owned(), + TEST_KEYWORD.to_owned(), 0, ) .await?; diff --git a/scylla-server-rust/tests/data_type_service_test.rs b/scylla-server-rust/tests/data_type_service_test.rs index 172f48e4..e0dc44ed 100644 --- a/scylla-server-rust/tests/data_type_service_test.rs +++ b/scylla-server-rust/tests/data_type_service_test.rs @@ -3,13 +3,13 @@ mod test_utils; use prisma_client_rust::QueryError; use scylla_server_rust::{ - controller::data_type_controller::DataTypeSend, + controllers::data_type_controller::PublicDataType, prisma, services::{data_type_service, node_service}, }; use test_utils::cleanup_and_prepare; -const DATA_TYPE_NAME: &str = "test"; +const TEST_KEYWORD: &str = "test"; #[tokio::test] async fn test_get_all_datatypes() -> Result<(), QueryError> { @@ -28,12 +28,11 @@ async fn test_datatype_fail_upsert_no_node() -> Result<(), QueryError> { // should fail since no node exists data_type_service::upsert_data_type( &db, - DATA_TYPE_NAME.to_owned(), + TEST_KEYWORD.to_owned(), "hello wurld".to_owned(), - DATA_TYPE_NAME.to_owned(), + TEST_KEYWORD.to_owned(), ) - .await - .expect_err("Should error, as no node exists"); + .await?; Ok(()) } @@ -61,8 +60,8 @@ async fn test_datatype_create() -> Result<(), QueryError> { .expect("This should not be empty"); assert_eq!( - DataTypeSend::from(&data), - DataTypeSend { + PublicDataType::from(&data), + PublicDataType { name: data_type_name, unit: unit } diff --git a/scylla-server-rust/tests/driver_service_test.rs b/scylla-server-rust/tests/driver_service_test.rs index 270fd72b..805ba047 100644 --- a/scylla-server-rust/tests/driver_service_test.rs +++ b/scylla-server-rust/tests/driver_service_test.rs @@ -5,7 +5,7 @@ use test_utils::cleanup_and_prepare; #[path = "test_utils.rs"] mod test_utils; -const DATA_TYPE_NAME: &str = "test"; +const TEST_KEYWORD: &str = "test"; #[tokio::test] async fn test_get_all_drivers() -> Result<(), QueryError> { @@ -23,7 +23,7 @@ async fn test_create_driver() -> Result<(), QueryError> { driver_service::upsert_driver( &db, - DATA_TYPE_NAME.to_owned(), + TEST_KEYWORD.to_owned(), run_service::create_run(&db, 10001).await?.id, ) .await?; diff --git a/scylla-server-rust/tests/location_service_test.rs b/scylla-server-rust/tests/location_service_test.rs index 5c0e570c..ca9abd10 100644 --- a/scylla-server-rust/tests/location_service_test.rs +++ b/scylla-server-rust/tests/location_service_test.rs @@ -8,7 +8,7 @@ use test_utils::cleanup_and_prepare; #[path = "test_utils.rs"] mod test_utils; -const DATA_TYPE_NAME: &str = "test"; +const TEST_KEYWORD: &str = "test"; #[tokio::test] async fn test_get_all_locations_and_upsert() -> Result<(), QueryError> { @@ -16,7 +16,7 @@ async fn test_get_all_locations_and_upsert() -> Result<(), QueryError> { location_service::upsert_location( &db, - DATA_TYPE_NAME.to_owned(), + TEST_KEYWORD.to_owned(), 100.0, 200.0, 300.0, @@ -25,7 +25,7 @@ async fn test_get_all_locations_and_upsert() -> Result<(), QueryError> { .await?; db.location() - .find_unique(prisma::location::name::equals(DATA_TYPE_NAME.to_owned())) + .find_unique(prisma::location::name::equals(TEST_KEYWORD.to_owned())) .exec() .await? .expect("Location exist as was just upserted"); diff --git a/scylla-server-rust/tests/node_service_test.rs b/scylla-server-rust/tests/node_service_test.rs index e6462d0f..7ce84cab 100644 --- a/scylla-server-rust/tests/node_service_test.rs +++ b/scylla-server-rust/tests/node_service_test.rs @@ -5,16 +5,16 @@ use test_utils::cleanup_and_prepare; #[path = "test_utils.rs"] mod test_utils; -const DATA_TYPE_NAME: &str = "test"; +const TEST_KEYWORD: &str = "test"; #[tokio::test] async fn test_upsert_node() -> Result<(), QueryError> { let db = cleanup_and_prepare().await?; - node_service::upsert_node(&db, DATA_TYPE_NAME.to_owned()).await?; + node_service::upsert_node(&db, TEST_KEYWORD.to_owned()).await?; db.node() - .find_unique(prisma::node::name::equals(DATA_TYPE_NAME.to_owned())) + .find_unique(prisma::node::name::equals(TEST_KEYWORD.to_owned())) .exec() .await? .expect("There should be a node, one was just upserted"); @@ -37,8 +37,8 @@ async fn test_upsert_node_twice() -> Result<(), QueryError> { let db = cleanup_and_prepare().await?; let all_nodes = node_service::get_all_nodes(&db).await?; - node_service::upsert_node(&db, DATA_TYPE_NAME.to_owned()).await?; - node_service::upsert_node(&db, DATA_TYPE_NAME.to_owned()).await?; + node_service::upsert_node(&db, TEST_KEYWORD.to_owned()).await?; + node_service::upsert_node(&db, TEST_KEYWORD.to_owned()).await?; let all_nodes_after = node_service::get_all_nodes(&db).await?; assert_eq!(all_nodes.len(), all_nodes_after.len() - 1); diff --git a/scylla-server-rust/tests/run_service_test.rs b/scylla-server-rust/tests/run_service_test.rs index 93d4e546..1d4770f4 100644 --- a/scylla-server-rust/tests/run_service_test.rs +++ b/scylla-server-rust/tests/run_service_test.rs @@ -1,12 +1,10 @@ use prisma_client_rust::QueryError; -use scylla_server_rust::{controller::run_controller::RunSend, services::run_service}; +use scylla_server_rust::{controllers::run_controller::PublicRun, services::run_service}; use test_utils::cleanup_and_prepare; #[path = "test_utils.rs"] mod test_utils; -//const DATA_TYPE_NAME: &str = "test"; - #[tokio::test] async fn test_get_all_runs() -> Result<(), QueryError> { let db = cleanup_and_prepare().await?; @@ -29,7 +27,7 @@ async fn test_get_run_by_id() -> Result<(), QueryError> { .await? .expect("Run should exist was upserted "); - assert_eq!(RunSend::from(&run_c), RunSend::from(&run)); + assert_eq!(PublicRun::from(&run_c), PublicRun::from(&run)); Ok(()) } diff --git a/scylla-server-rust/tests/system_service_test.rs b/scylla-server-rust/tests/system_service_test.rs index d7b33a5b..c9437082 100644 --- a/scylla-server-rust/tests/system_service_test.rs +++ b/scylla-server-rust/tests/system_service_test.rs @@ -1,6 +1,6 @@ use prisma_client_rust::QueryError; use scylla_server_rust::{ - controller::system_controller::SystemSend, + controllers::system_controller::PublicSystem, prisma, services::{run_service, system_service}, }; @@ -9,7 +9,7 @@ use test_utils::cleanup_and_prepare; #[path = "test_utils.rs"] mod test_utils; -const DATA_TYPE_NAME: &str = "test"; +const TEST_KEYWORD: &str = "test"; #[tokio::test] async fn test_upsert_system_create() -> Result<(), QueryError> { @@ -17,19 +17,19 @@ async fn test_upsert_system_create() -> Result<(), QueryError> { let res_c = system_service::upsert_system( &db, - DATA_TYPE_NAME.to_owned(), + TEST_KEYWORD.to_owned(), run_service::create_run(&db, 101).await?.id, ) .await?; let res = db .system() - .find_unique(prisma::system::name::equals(DATA_TYPE_NAME.to_owned())) + .find_unique(prisma::system::name::equals(TEST_KEYWORD.to_owned())) .exec() .await? .expect("System should exist, was just upserted!"); - assert_eq!(SystemSend::from(&res_c), SystemSend::from(&res)); + assert_eq!(PublicSystem::from(&res_c), PublicSystem::from(&res)); Ok(()) } @@ -50,7 +50,7 @@ async fn test_get_upsert_system() -> Result<(), QueryError> { system_service::upsert_system( &db, - DATA_TYPE_NAME.to_owned(), + TEST_KEYWORD.to_owned(), run_service::create_run(&db, 101).await?.id, ) .await?; @@ -58,7 +58,7 @@ async fn test_get_upsert_system() -> Result<(), QueryError> { let sys = system_service::get_all_systems(&db).await?; sys.iter() - .find(|&f| f.name == DATA_TYPE_NAME.to_owned()) + .find(|&f| f.name == TEST_KEYWORD.to_owned()) .expect("System of the added name should exist in the list of systems"); Ok(()) From 52d6db5a08c70aef18165ae66e8824495231e2ed Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Sun, 7 Jul 2024 17:04:32 -0400 Subject: [PATCH 11/68] make transformers own folder --- .../src/controllers/data_controller.rs | 22 +----------- .../src/controllers/data_type_controller.rs | 22 +++--------- .../src/controllers/driver_controller.rs | 30 +++------------- .../src/controllers/location_controller.rs | 36 +++---------------- .../src/controllers/node_controller.rs | 31 +++------------- .../src/controllers/run_controller.rs | 30 ++-------------- .../src/controllers/system_controller.rs | 30 +++------------- scylla-server-rust/src/lib.rs | 1 + .../src/transformers/data_transformer.rs | 20 +++++++++++ .../src/transformers/data_type_transformer.rs | 19 ++++++++++ .../src/transformers/driver_transformer.rs | 27 ++++++++++++++ .../src/transformers/location_transformer.rs | 33 +++++++++++++++++ scylla-server-rust/src/transformers/mod.rs | 7 ++++ .../src/transformers/node_transformer.rs | 28 +++++++++++++++ .../src/transformers/run_transformer.rs | 28 +++++++++++++++ .../src/transformers/system_transformer.rs | 27 ++++++++++++++ scylla-server-rust/tests/data_service_test.rs | 2 +- .../tests/data_type_service_test.rs | 2 +- scylla-server-rust/tests/run_service_test.rs | 2 +- .../tests/system_service_test.rs | 2 +- 20 files changed, 218 insertions(+), 181 deletions(-) create mode 100644 scylla-server-rust/src/transformers/data_transformer.rs create mode 100644 scylla-server-rust/src/transformers/data_type_transformer.rs create mode 100644 scylla-server-rust/src/transformers/driver_transformer.rs create mode 100644 scylla-server-rust/src/transformers/location_transformer.rs create mode 100644 scylla-server-rust/src/transformers/mod.rs create mode 100644 scylla-server-rust/src/transformers/node_transformer.rs create mode 100644 scylla-server-rust/src/transformers/run_transformer.rs create mode 100644 scylla-server-rust/src/transformers/system_transformer.rs diff --git a/scylla-server-rust/src/controllers/data_controller.rs b/scylla-server-rust/src/controllers/data_controller.rs index 55fd9fbe..6731246a 100644 --- a/scylla-server-rust/src/controllers/data_controller.rs +++ b/scylla-server-rust/src/controllers/data_controller.rs @@ -2,32 +2,12 @@ use axum::{ extract::{Path, State}, Json, }; -use serde::Serialize; use crate::{ - error::ScyllaError, - prisma::{self}, - services::data_service, + error::ScyllaError, services::data_service, transformers::data_transformer::PublicData, Database, }; -/// The struct defining the data format sent to the client -#[derive(Serialize, Debug, PartialEq, Eq, PartialOrd, Ord)] -pub struct PublicData { - pub time: i64, - pub values: Vec, -} - -/// convert the prisma type to the client type for JSON encoding -impl From<&prisma::data::Data> for PublicData { - fn from(value: &prisma::data::Data) -> Self { - PublicData { - values: value.values.iter().map(|f| f.to_string()).collect(), - time: value.time.timestamp_millis(), - } - } -} - pub async fn get_data( State(db): State, Path((data_type_name, run_id)): Path<(String, i32)>, diff --git a/scylla-server-rust/src/controllers/data_type_controller.rs b/scylla-server-rust/src/controllers/data_type_controller.rs index 279857ad..d80ed459 100644 --- a/scylla-server-rust/src/controllers/data_type_controller.rs +++ b/scylla-server-rust/src/controllers/data_type_controller.rs @@ -1,23 +1,9 @@ use axum::{extract::State, Json}; -use serde::Serialize; -use crate::{error::ScyllaError, prisma, services::data_type_service, Database}; - -/// The struct defining the data type format sent to the client -#[derive(Serialize, Debug, PartialEq)] -pub struct PublicDataType { - pub name: String, - pub unit: String, -} - -impl From<&prisma::data_type::Data> for PublicDataType { - fn from(value: &prisma::data_type::Data) -> Self { - PublicDataType { - name: value.name.clone(), - unit: value.unit.clone(), - } - } -} +use crate::{ + error::ScyllaError, services::data_type_service, + transformers::data_type_transformer::PublicDataType, Database, +}; pub async fn get_all_data_types( State(db): State, diff --git a/scylla-server-rust/src/controllers/driver_controller.rs b/scylla-server-rust/src/controllers/driver_controller.rs index 283ddff7..66d67009 100644 --- a/scylla-server-rust/src/controllers/driver_controller.rs +++ b/scylla-server-rust/src/controllers/driver_controller.rs @@ -1,31 +1,9 @@ use axum::{extract::State, Json}; -use serde::Serialize; -use crate::{error::ScyllaError, prisma, services::driver_service, Database}; - -use super::run_controller::{self, PublicRun}; - -/// The struct defining the driver format sent to the client -#[derive(Serialize, PartialEq)] -pub struct PublicDriver { - username: String, - runs: Vec, -} - -impl From<&prisma::driver::Data> for PublicDriver { - fn from(value: &prisma::driver::Data) -> Self { - PublicDriver { - username: value.username.clone(), - runs: value - .runs - .clone() - .unwrap_or_default() - .iter() - .map(PublicRun::from) - .collect(), - } - } -} +use crate::{ + error::ScyllaError, services::driver_service, transformers::driver_transformer::PublicDriver, + Database, +}; pub async fn get_all_drivers( State(db): State, diff --git a/scylla-server-rust/src/controllers/location_controller.rs b/scylla-server-rust/src/controllers/location_controller.rs index 92f44758..24e46337 100644 --- a/scylla-server-rust/src/controllers/location_controller.rs +++ b/scylla-server-rust/src/controllers/location_controller.rs @@ -1,37 +1,9 @@ use axum::{extract::State, Json}; -use serde::Serialize; -use crate::{error::ScyllaError, prisma, services::location_service, Database}; - -use super::run_controller::{self, PublicRun}; - -/// The struct defining the location format sent to the client -#[derive(Serialize, Debug, PartialEq)] -pub struct PublicLocation { - pub name: String, - pub latitude: f64, - pub longitude: f64, - pub radius: f64, - pub runs: Vec, -} - -impl From<&prisma::location::Data> for PublicLocation { - fn from(value: &prisma::location::Data) -> Self { - PublicLocation { - name: value.name.clone(), - latitude: value.latitude, - longitude: value.longitude, - radius: value.radius, - runs: value - .runs - .clone() - .unwrap_or_default() - .iter() - .map(PublicRun::from) - .collect(), - } - } -} +use crate::{ + error::ScyllaError, services::location_service, + transformers::location_transformer::PublicLocation, Database, +}; pub async fn get_all_locations( State(db): State, diff --git a/scylla-server-rust/src/controllers/node_controller.rs b/scylla-server-rust/src/controllers/node_controller.rs index 2ba88179..3f03f110 100644 --- a/scylla-server-rust/src/controllers/node_controller.rs +++ b/scylla-server-rust/src/controllers/node_controller.rs @@ -1,32 +1,9 @@ use axum::{extract::State, Json}; -use serde::Serialize; -use crate::{error::ScyllaError, prisma, services::node_service, Database}; - -use super::data_type_controller::PublicDataType; - -/// The struct defining the node format sent to the client -#[derive(Serialize, PartialEq)] -pub struct PublicNode { - name: String, - #[serde(rename = "dataTypes")] - data_types: Vec, -} - -impl From<&prisma::node::Data> for PublicNode { - fn from(value: &prisma::node::Data) -> Self { - PublicNode { - name: value.name.clone(), - data_types: value - .data_types - .clone() - .unwrap_or_default() - .iter() - .map(PublicDataType::from) - .collect(), - } - } -} +use crate::{ + error::ScyllaError, services::node_service, transformers::node_transformer::PublicNode, + Database, +}; pub async fn get_all_nodes( State(db): State, diff --git a/scylla-server-rust/src/controllers/run_controller.rs b/scylla-server-rust/src/controllers/run_controller.rs index c1a2568f..4c26eac4 100644 --- a/scylla-server-rust/src/controllers/run_controller.rs +++ b/scylla-server-rust/src/controllers/run_controller.rs @@ -2,34 +2,10 @@ use axum::{ extract::{Path, State}, Json, }; -use serde::Serialize; - -use crate::{error::ScyllaError, prisma, services::run_service, Database}; - -/// The struct defining the run format sent to the client -#[derive(Serialize, Debug, PartialEq)] -pub struct PublicRun { - pub id: i32, - #[serde(rename = "locationName")] - pub location_name: String, - #[serde(rename = "driverName")] - pub driver_name: String, - #[serde(rename = "systemName")] - pub system_name: String, - pub time: i64, -} -impl From<&prisma::run::Data> for PublicRun { - fn from(value: &prisma::run::Data) -> Self { - PublicRun { - id: value.id, - location_name: value.location_name.clone().unwrap_or_default(), - driver_name: value.driver_name.clone().unwrap_or_default(), - system_name: value.system_name.clone().unwrap_or_default(), - time: value.time.timestamp_millis(), - } - } -} +use crate::{ + error::ScyllaError, services::run_service, transformers::run_transformer::PublicRun, Database, +}; pub async fn get_all_runs(State(db): State) -> Result>, ScyllaError> { let run_data = run_service::get_all_runs(&db).await?; diff --git a/scylla-server-rust/src/controllers/system_controller.rs b/scylla-server-rust/src/controllers/system_controller.rs index 4e51d5e8..5c9d1a01 100644 --- a/scylla-server-rust/src/controllers/system_controller.rs +++ b/scylla-server-rust/src/controllers/system_controller.rs @@ -1,31 +1,9 @@ use axum::{extract::State, Json}; -use serde::Serialize; -use crate::{error::ScyllaError, prisma, services::system_service, Database}; - -use super::run_controller::{self, PublicRun}; - -/// The struct defining the system format sent to the client -#[derive(Serialize, Debug, PartialEq)] -pub struct PublicSystem { - pub name: String, - pub runs: Vec, -} - -impl From<&prisma::system::Data> for PublicSystem { - fn from(value: &prisma::system::Data) -> Self { - PublicSystem { - name: value.name.clone(), - runs: value - .runs - .clone() - .unwrap_or_default() - .iter() - .map(PublicRun::from) - .collect(), - } - } -} +use crate::{ + error::ScyllaError, services::system_service, transformers::system_transformer::PublicSystem, + Database, +}; pub async fn get_all_systems( State(db): State, diff --git a/scylla-server-rust/src/lib.rs b/scylla-server-rust/src/lib.rs index 2c9cff5f..df6b8493 100644 --- a/scylla-server-rust/src/lib.rs +++ b/scylla-server-rust/src/lib.rs @@ -1,6 +1,7 @@ pub mod controllers; pub mod error; pub mod services; +pub mod transformers; #[allow(clippy::all)] #[allow(warnings)] diff --git a/scylla-server-rust/src/transformers/data_transformer.rs b/scylla-server-rust/src/transformers/data_transformer.rs new file mode 100644 index 00000000..ab183948 --- /dev/null +++ b/scylla-server-rust/src/transformers/data_transformer.rs @@ -0,0 +1,20 @@ +use serde::Serialize; + +use crate::prisma; + +/// The struct defining the data format sent to the client +#[derive(Serialize, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct PublicData { + pub time: i64, + pub values: Vec, +} + +/// convert the prisma type to the client type for JSON encoding +impl From<&prisma::data::Data> for PublicData { + fn from(value: &prisma::data::Data) -> Self { + PublicData { + values: value.values.iter().map(|f| f.to_string()).collect(), + time: value.time.timestamp_millis(), + } + } +} diff --git a/scylla-server-rust/src/transformers/data_type_transformer.rs b/scylla-server-rust/src/transformers/data_type_transformer.rs new file mode 100644 index 00000000..34a92615 --- /dev/null +++ b/scylla-server-rust/src/transformers/data_type_transformer.rs @@ -0,0 +1,19 @@ +use serde::Serialize; + +use crate::prisma; + +/// The struct defining the data type format sent to the client +#[derive(Serialize, Debug, PartialEq)] +pub struct PublicDataType { + pub name: String, + pub unit: String, +} + +impl From<&prisma::data_type::Data> for PublicDataType { + fn from(value: &prisma::data_type::Data) -> Self { + PublicDataType { + name: value.name.clone(), + unit: value.unit.clone(), + } + } +} diff --git a/scylla-server-rust/src/transformers/driver_transformer.rs b/scylla-server-rust/src/transformers/driver_transformer.rs new file mode 100644 index 00000000..b7a0520f --- /dev/null +++ b/scylla-server-rust/src/transformers/driver_transformer.rs @@ -0,0 +1,27 @@ +use serde::Serialize; + +use crate::prisma; + +use super::run_transformer::PublicRun; + +/// The struct defining the driver format sent to the client +#[derive(Serialize, PartialEq)] +pub struct PublicDriver { + username: String, + runs: Vec, +} + +impl From<&prisma::driver::Data> for PublicDriver { + fn from(value: &prisma::driver::Data) -> Self { + PublicDriver { + username: value.username.clone(), + runs: value + .runs + .clone() + .unwrap_or_default() + .iter() + .map(PublicRun::from) + .collect(), + } + } +} diff --git a/scylla-server-rust/src/transformers/location_transformer.rs b/scylla-server-rust/src/transformers/location_transformer.rs new file mode 100644 index 00000000..c8dedc90 --- /dev/null +++ b/scylla-server-rust/src/transformers/location_transformer.rs @@ -0,0 +1,33 @@ +use serde::Serialize; + +use crate::prisma; + +use super::run_transformer::PublicRun; + +/// The struct defining the location format sent to the client +#[derive(Serialize, Debug, PartialEq)] +pub struct PublicLocation { + pub name: String, + pub latitude: f64, + pub longitude: f64, + pub radius: f64, + pub runs: Vec, +} + +impl From<&prisma::location::Data> for PublicLocation { + fn from(value: &prisma::location::Data) -> Self { + PublicLocation { + name: value.name.clone(), + latitude: value.latitude, + longitude: value.longitude, + radius: value.radius, + runs: value + .runs + .clone() + .unwrap_or_default() + .iter() + .map(PublicRun::from) + .collect(), + } + } +} diff --git a/scylla-server-rust/src/transformers/mod.rs b/scylla-server-rust/src/transformers/mod.rs new file mode 100644 index 00000000..3538f453 --- /dev/null +++ b/scylla-server-rust/src/transformers/mod.rs @@ -0,0 +1,7 @@ +pub mod data_transformer; +pub mod data_type_transformer; +pub mod driver_transformer; +pub mod location_transformer; +pub mod node_transformer; +pub mod run_transformer; +pub mod system_transformer; diff --git a/scylla-server-rust/src/transformers/node_transformer.rs b/scylla-server-rust/src/transformers/node_transformer.rs new file mode 100644 index 00000000..c8ebbca4 --- /dev/null +++ b/scylla-server-rust/src/transformers/node_transformer.rs @@ -0,0 +1,28 @@ +use serde::Serialize; + +use crate::prisma; + +use super::data_type_transformer::PublicDataType; + +/// The struct defining the node format sent to the client +#[derive(Serialize, PartialEq)] +pub struct PublicNode { + name: String, + #[serde(rename = "dataTypes")] + data_types: Vec, +} + +impl From<&prisma::node::Data> for PublicNode { + fn from(value: &prisma::node::Data) -> Self { + PublicNode { + name: value.name.clone(), + data_types: value + .data_types + .clone() + .unwrap_or_default() + .iter() + .map(PublicDataType::from) + .collect(), + } + } +} diff --git a/scylla-server-rust/src/transformers/run_transformer.rs b/scylla-server-rust/src/transformers/run_transformer.rs new file mode 100644 index 00000000..0f2ca2bf --- /dev/null +++ b/scylla-server-rust/src/transformers/run_transformer.rs @@ -0,0 +1,28 @@ +use serde::Serialize; + +use crate::prisma; + +/// The struct defining the run format sent to the client +#[derive(Serialize, Debug, PartialEq)] +pub struct PublicRun { + pub id: i32, + #[serde(rename = "locationName")] + pub location_name: String, + #[serde(rename = "driverName")] + pub driver_name: String, + #[serde(rename = "systemName")] + pub system_name: String, + pub time: i64, +} + +impl From<&prisma::run::Data> for PublicRun { + fn from(value: &prisma::run::Data) -> Self { + PublicRun { + id: value.id, + location_name: value.location_name.clone().unwrap_or_default(), + driver_name: value.driver_name.clone().unwrap_or_default(), + system_name: value.system_name.clone().unwrap_or_default(), + time: value.time.timestamp_millis(), + } + } +} diff --git a/scylla-server-rust/src/transformers/system_transformer.rs b/scylla-server-rust/src/transformers/system_transformer.rs new file mode 100644 index 00000000..34f76d3f --- /dev/null +++ b/scylla-server-rust/src/transformers/system_transformer.rs @@ -0,0 +1,27 @@ +use serde::Serialize; + +use crate::prisma; + +use super::run_transformer::PublicRun; + +/// The struct defining the system format sent to the client +#[derive(Serialize, Debug, PartialEq)] +pub struct PublicSystem { + pub name: String, + pub runs: Vec, +} + +impl From<&prisma::system::Data> for PublicSystem { + fn from(value: &prisma::system::Data) -> Self { + PublicSystem { + name: value.name.clone(), + runs: value + .runs + .clone() + .unwrap_or_default() + .iter() + .map(PublicRun::from) + .collect(), + } + } +} diff --git a/scylla-server-rust/tests/data_service_test.rs b/scylla-server-rust/tests/data_service_test.rs index 6f420963..d32b58ea 100644 --- a/scylla-server-rust/tests/data_service_test.rs +++ b/scylla-server-rust/tests/data_service_test.rs @@ -4,9 +4,9 @@ mod test_utils; use prisma_client_rust::QueryError; use protobuf::SpecialFields; use scylla_server_rust::{ - controllers::data_controller::PublicData, serverdata::ServerData, services::{data_service, data_type_service, node_service, run_service}, + transformers::data_transformer::PublicData, }; use test_utils::cleanup_and_prepare; diff --git a/scylla-server-rust/tests/data_type_service_test.rs b/scylla-server-rust/tests/data_type_service_test.rs index e0dc44ed..42c26471 100644 --- a/scylla-server-rust/tests/data_type_service_test.rs +++ b/scylla-server-rust/tests/data_type_service_test.rs @@ -3,9 +3,9 @@ mod test_utils; use prisma_client_rust::QueryError; use scylla_server_rust::{ - controllers::data_type_controller::PublicDataType, prisma, services::{data_type_service, node_service}, + transformers::data_type_transformer::PublicDataType, }; use test_utils::cleanup_and_prepare; diff --git a/scylla-server-rust/tests/run_service_test.rs b/scylla-server-rust/tests/run_service_test.rs index 1d4770f4..e13bd2c3 100644 --- a/scylla-server-rust/tests/run_service_test.rs +++ b/scylla-server-rust/tests/run_service_test.rs @@ -1,5 +1,5 @@ use prisma_client_rust::QueryError; -use scylla_server_rust::{controllers::run_controller::PublicRun, services::run_service}; +use scylla_server_rust::{services::run_service, transformers::run_transformer::PublicRun}; use test_utils::cleanup_and_prepare; #[path = "test_utils.rs"] diff --git a/scylla-server-rust/tests/system_service_test.rs b/scylla-server-rust/tests/system_service_test.rs index c9437082..aa43d3c6 100644 --- a/scylla-server-rust/tests/system_service_test.rs +++ b/scylla-server-rust/tests/system_service_test.rs @@ -1,8 +1,8 @@ use prisma_client_rust::QueryError; use scylla_server_rust::{ - controllers::system_controller::PublicSystem, prisma, services::{run_service, system_service}, + transformers::system_transformer::PublicSystem, }; use test_utils::cleanup_and_prepare; From 0b9156f30a3d44df2797defdd9e252117c9d8d6b Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Sun, 7 Jul 2024 17:14:58 -0400 Subject: [PATCH 12/68] add a scylla rust test runner --- .github/workflows/scylla-rust-tests.yml | 35 +++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .github/workflows/scylla-rust-tests.yml diff --git a/.github/workflows/scylla-rust-tests.yml b/.github/workflows/scylla-rust-tests.yml new file mode 100644 index 00000000..855ba2ec --- /dev/null +++ b/.github/workflows/scylla-rust-tests.yml @@ -0,0 +1,35 @@ +name: Scylla Rust Tests + +on: + push: + branches: + - main + - develop + pull_request: + branches: + - main + - develop + - 'feature/**' + +env: + SOURCE_DATABASE_URL: postgresql://postgres:password@127.0.0.1:5432/timescaledb cargo run + +jobs: + run-tests: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + submodules: true + - name: Run db + run: docker compose run -P -d odyssey-timescale + - name: Generate prisma + run: cargo prisma generate + working-directory: scylla-server-rust + - name: Deploy prisma + run: cargo prisma migrate deploy + - name: Run tests + working-directory: scylla-server-rust + run: cargo test -- --test-threads=1 From 1c295c9338de1e353eb0075df6f71f78d3cfa53f Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Sun, 7 Jul 2024 17:17:25 -0400 Subject: [PATCH 13/68] test fix --- .github/workflows/scylla-rust-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/scylla-rust-tests.yml b/.github/workflows/scylla-rust-tests.yml index 855ba2ec..2888fc7c 100644 --- a/.github/workflows/scylla-rust-tests.yml +++ b/.github/workflows/scylla-rust-tests.yml @@ -30,6 +30,7 @@ jobs: working-directory: scylla-server-rust - name: Deploy prisma run: cargo prisma migrate deploy + working-directory: scylla-server-rust - name: Run tests working-directory: scylla-server-rust run: cargo test -- --test-threads=1 From 704af85fa138f2d7b6b234c452180a09ab7aed29 Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Sun, 7 Jul 2024 17:20:58 -0400 Subject: [PATCH 14/68] add locations to specifc tests so they dont all trigger --- .github/workflows/lint-check.yml | 2 ++ .github/workflows/scylla-rust-ci.yml | 2 ++ .github/workflows/scylla-rust-tests.yml | 2 ++ 3 files changed, 6 insertions(+) diff --git a/.github/workflows/lint-check.yml b/.github/workflows/lint-check.yml index de4a638c..c9a1ad8d 100644 --- a/.github/workflows/lint-check.yml +++ b/.github/workflows/lint-check.yml @@ -4,11 +4,13 @@ on: branches: - main - develop + paths: ['angular-client/**', '.github/***'] pull_request: branches: - main - develop - 'feature/**' + paths: ['angular-client/**', '.github/***'] jobs: run-linting-check: runs-on: ubuntu-latest diff --git a/.github/workflows/scylla-rust-ci.yml b/.github/workflows/scylla-rust-ci.yml index 8b3a7c22..549e144a 100644 --- a/.github/workflows/scylla-rust-ci.yml +++ b/.github/workflows/scylla-rust-ci.yml @@ -5,10 +5,12 @@ on: branches: - main - develop + paths: ['scylla-server-rust/**', '.github/***'] pull_request: branches: - main - develop + paths: ['scylla-server-rust/**', '.github/**'] defaults: diff --git a/.github/workflows/scylla-rust-tests.yml b/.github/workflows/scylla-rust-tests.yml index 2888fc7c..b130be3c 100644 --- a/.github/workflows/scylla-rust-tests.yml +++ b/.github/workflows/scylla-rust-tests.yml @@ -5,11 +5,13 @@ on: branches: - main - develop + paths: ['scylla-server-rust/**', '.github/***'] pull_request: branches: - main - develop - 'feature/**' + paths: ['scylla-server-rust/**', '.github/**'] env: SOURCE_DATABASE_URL: postgresql://postgres:password@127.0.0.1:5432/timescaledb cargo run From 482fe23bb622762f6493763917069133c5838f5a Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Sun, 7 Jul 2024 17:25:38 -0400 Subject: [PATCH 15/68] fixes to test ci --- .github/workflows/prettier-check.yml | 7 +++++++ .github/workflows/scylla-rust-ci.yml | 8 ++++++-- .github/workflows/scylla-rust-tests.yml | 4 +++- scylla-server-rust/src/transformers/data_transformer.rs | 2 +- 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/.github/workflows/prettier-check.yml b/.github/workflows/prettier-check.yml index 8a63502e..690e8995 100644 --- a/.github/workflows/prettier-check.yml +++ b/.github/workflows/prettier-check.yml @@ -4,11 +4,18 @@ on: branches: - main - develop + paths: + - 'angular-client/**' + - '.github/**' pull_request: branches: - main - develop - 'feature/**' + paths: + - 'angular-client/**' + - '.github/**' + jobs: run-prettier-check: runs-on: ubuntu-latest diff --git a/.github/workflows/scylla-rust-ci.yml b/.github/workflows/scylla-rust-ci.yml index 549e144a..83f01970 100644 --- a/.github/workflows/scylla-rust-ci.yml +++ b/.github/workflows/scylla-rust-ci.yml @@ -5,12 +5,16 @@ on: branches: - main - develop - paths: ['scylla-server-rust/**', '.github/***'] + paths: + - 'scylla-server-rust/**' + - '.github/**' pull_request: branches: - main - develop - paths: ['scylla-server-rust/**', '.github/**'] + paths: + - 'scylla-server-rust/**' + - '.github/**' defaults: diff --git a/.github/workflows/scylla-rust-tests.yml b/.github/workflows/scylla-rust-tests.yml index b130be3c..98412fdd 100644 --- a/.github/workflows/scylla-rust-tests.yml +++ b/.github/workflows/scylla-rust-tests.yml @@ -11,7 +11,9 @@ on: - main - develop - 'feature/**' - paths: ['scylla-server-rust/**', '.github/**'] + paths: + - 'scylla-server-rust/**' + - '.github/**' env: SOURCE_DATABASE_URL: postgresql://postgres:password@127.0.0.1:5432/timescaledb cargo run diff --git a/scylla-server-rust/src/transformers/data_transformer.rs b/scylla-server-rust/src/transformers/data_transformer.rs index ab183948..a4ce39d5 100644 --- a/scylla-server-rust/src/transformers/data_transformer.rs +++ b/scylla-server-rust/src/transformers/data_transformer.rs @@ -2,7 +2,7 @@ use serde::Serialize; use crate::prisma; -/// The struct defining the data format sent to the client +/// The struct defining the data format sent to the client #[derive(Serialize, Debug, PartialEq, Eq, PartialOrd, Ord)] pub struct PublicData { pub time: i64, From 0a0afb602084cab942c3f7bcaf76215eb9517a69 Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Sun, 7 Jul 2024 17:27:32 -0400 Subject: [PATCH 16/68] fix test issue --- .github/workflows/scylla-rust-tests.yml | 4 +++- scylla-server-rust/src/transformers/data_transformer.rs | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/scylla-rust-tests.yml b/.github/workflows/scylla-rust-tests.yml index 98412fdd..6037597f 100644 --- a/.github/workflows/scylla-rust-tests.yml +++ b/.github/workflows/scylla-rust-tests.yml @@ -5,7 +5,9 @@ on: branches: - main - develop - paths: ['scylla-server-rust/**', '.github/***'] + paths: + - 'scylla-server-rust/**' + - '.github/**' pull_request: branches: - main diff --git a/scylla-server-rust/src/transformers/data_transformer.rs b/scylla-server-rust/src/transformers/data_transformer.rs index a4ce39d5..ab183948 100644 --- a/scylla-server-rust/src/transformers/data_transformer.rs +++ b/scylla-server-rust/src/transformers/data_transformer.rs @@ -2,7 +2,7 @@ use serde::Serialize; use crate::prisma; -/// The struct defining the data format sent to the client +/// The struct defining the data format sent to the client #[derive(Serialize, Debug, PartialEq, Eq, PartialOrd, Ord)] pub struct PublicData { pub time: i64, From 0bf4b277d6ff8da12c3447b0eac8ecd0f6f9dee8 Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Sun, 7 Jul 2024 17:32:35 -0400 Subject: [PATCH 17/68] last ci check, fix test fail --- .github/workflows/scylla-rust-ci.yml | 1 + scylla-server-rust/tests/data_type_service_test.rs | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/scylla-rust-ci.yml b/.github/workflows/scylla-rust-ci.yml index 83f01970..3097d8e3 100644 --- a/.github/workflows/scylla-rust-ci.yml +++ b/.github/workflows/scylla-rust-ci.yml @@ -12,6 +12,7 @@ on: branches: - main - develop + - 'feature/**' paths: - 'scylla-server-rust/**' - '.github/**' diff --git a/scylla-server-rust/tests/data_type_service_test.rs b/scylla-server-rust/tests/data_type_service_test.rs index 42c26471..86037f0f 100644 --- a/scylla-server-rust/tests/data_type_service_test.rs +++ b/scylla-server-rust/tests/data_type_service_test.rs @@ -32,7 +32,8 @@ async fn test_datatype_fail_upsert_no_node() -> Result<(), QueryError> { "hello wurld".to_owned(), TEST_KEYWORD.to_owned(), ) - .await?; + .await + .expect_err("Test should fail, no node exists"); Ok(()) } From 04cc67b3c81b2686bb418b8c0528b1d18b3608ab Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Sun, 7 Jul 2024 17:44:55 -0400 Subject: [PATCH 18/68] add a script for integration testing --- scylla-server-rust/README.md | 4 ++-- scylla-server-rust/integration_test.sh | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) create mode 100755 scylla-server-rust/integration_test.sh diff --git a/scylla-server-rust/README.md b/scylla-server-rust/README.md index 803f3840..e39a5082 100644 --- a/scylla-server-rust/README.md +++ b/scylla-server-rust/README.md @@ -21,10 +21,10 @@ SOURCE_DATABASE_URL=postgresql://postgres:password@127.0.0.1:5432/timescaledb ca ### Test this app -Since this app uses the database for testing, you must wipe the database. +Since this app uses the database for testing, you must follow these steps, or run `./integration_test.sh`: ``` docker volume rm argos_db-data -docker compose run -P odyssey-timescale +docker compose run -Pd odyssey-timescale cargo prisma migrate deploy SOURCE_DATABASE_URL=postgresql://postgres:password@127.0.0.1:5432/timescaledb cargo test -- --test-threads=1 ``` \ No newline at end of file diff --git a/scylla-server-rust/integration_test.sh b/scylla-server-rust/integration_test.sh new file mode 100755 index 00000000..1e99b6b5 --- /dev/null +++ b/scylla-server-rust/integration_test.sh @@ -0,0 +1,14 @@ +#!/bin/sh + +echo "Starting db" +cd .. +docker compose run -Pd odyssey-timescale + +echo "Deploying prisma" +cd ./scylla-server-rust +cargo prisma migrate deploy + +echo "Running tests" +SOURCE_DATABASE_URL=postgresql://postgres:password@127.0.0.1:5432/timescaledb cargo test -- --test-threads=1 + +echo "Please stop your db in docker" From ffb5cb323cfd55daabbdb62560c422dd12fcbbe0 Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Sun, 7 Jul 2024 17:47:13 -0400 Subject: [PATCH 19/68] fix workflows --- .github/workflows/lint-check.yml | 4 ++-- .github/workflows/prettier-check.yml | 2 -- .github/workflows/scylla-rust-ci.yml | 2 -- .github/workflows/scylla-rust-tests.yml | 2 -- 4 files changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/lint-check.yml b/.github/workflows/lint-check.yml index c9a1ad8d..c7bcce7a 100644 --- a/.github/workflows/lint-check.yml +++ b/.github/workflows/lint-check.yml @@ -4,13 +4,13 @@ on: branches: - main - develop - paths: ['angular-client/**', '.github/***'] + paths: ['angular-client/**'] pull_request: branches: - main - develop - 'feature/**' - paths: ['angular-client/**', '.github/***'] + paths: ['angular-client/**'] jobs: run-linting-check: runs-on: ubuntu-latest diff --git a/.github/workflows/prettier-check.yml b/.github/workflows/prettier-check.yml index 690e8995..855ae69d 100644 --- a/.github/workflows/prettier-check.yml +++ b/.github/workflows/prettier-check.yml @@ -6,7 +6,6 @@ on: - develop paths: - 'angular-client/**' - - '.github/**' pull_request: branches: - main @@ -14,7 +13,6 @@ on: - 'feature/**' paths: - 'angular-client/**' - - '.github/**' jobs: run-prettier-check: diff --git a/.github/workflows/scylla-rust-ci.yml b/.github/workflows/scylla-rust-ci.yml index 3097d8e3..f7d9b86b 100644 --- a/.github/workflows/scylla-rust-ci.yml +++ b/.github/workflows/scylla-rust-ci.yml @@ -7,7 +7,6 @@ on: - develop paths: - 'scylla-server-rust/**' - - '.github/**' pull_request: branches: - main @@ -15,7 +14,6 @@ on: - 'feature/**' paths: - 'scylla-server-rust/**' - - '.github/**' defaults: diff --git a/.github/workflows/scylla-rust-tests.yml b/.github/workflows/scylla-rust-tests.yml index 6037597f..80fbe92f 100644 --- a/.github/workflows/scylla-rust-tests.yml +++ b/.github/workflows/scylla-rust-tests.yml @@ -7,7 +7,6 @@ on: - develop paths: - 'scylla-server-rust/**' - - '.github/**' pull_request: branches: - main @@ -15,7 +14,6 @@ on: - 'feature/**' paths: - 'scylla-server-rust/**' - - '.github/**' env: SOURCE_DATABASE_URL: postgresql://postgres:password@127.0.0.1:5432/timescaledb cargo run From ee1f5934d34056e06dd0b09ee9bd206953a54a8b Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Sun, 7 Jul 2024 17:51:09 -0400 Subject: [PATCH 20/68] fuck yaml --- .github/workflows/lint-check.yml | 3 +-- .github/workflows/prettier-check.yml | 5 +---- .github/workflows/scylla-rust-ci.yml | 4 ---- .github/workflows/scylla-rust-tests.yml | 4 ---- 4 files changed, 2 insertions(+), 14 deletions(-) diff --git a/.github/workflows/lint-check.yml b/.github/workflows/lint-check.yml index c7bcce7a..4faf907f 100644 --- a/.github/workflows/lint-check.yml +++ b/.github/workflows/lint-check.yml @@ -4,13 +4,12 @@ on: branches: - main - develop - paths: ['angular-client/**'] pull_request: branches: - main - develop - 'feature/**' - paths: ['angular-client/**'] + jobs: run-linting-check: runs-on: ubuntu-latest diff --git a/.github/workflows/prettier-check.yml b/.github/workflows/prettier-check.yml index 855ae69d..c41943b1 100644 --- a/.github/workflows/prettier-check.yml +++ b/.github/workflows/prettier-check.yml @@ -4,15 +4,12 @@ on: branches: - main - develop - paths: - - 'angular-client/**' pull_request: branches: - main - develop - 'feature/**' - paths: - - 'angular-client/**' + jobs: run-prettier-check: diff --git a/.github/workflows/scylla-rust-ci.yml b/.github/workflows/scylla-rust-ci.yml index f7d9b86b..5c57d4f1 100644 --- a/.github/workflows/scylla-rust-ci.yml +++ b/.github/workflows/scylla-rust-ci.yml @@ -5,15 +5,11 @@ on: branches: - main - develop - paths: - - 'scylla-server-rust/**' pull_request: branches: - main - develop - 'feature/**' - paths: - - 'scylla-server-rust/**' defaults: diff --git a/.github/workflows/scylla-rust-tests.yml b/.github/workflows/scylla-rust-tests.yml index 80fbe92f..2888fc7c 100644 --- a/.github/workflows/scylla-rust-tests.yml +++ b/.github/workflows/scylla-rust-tests.yml @@ -5,15 +5,11 @@ on: branches: - main - develop - paths: - - 'scylla-server-rust/**' pull_request: branches: - main - develop - 'feature/**' - paths: - - 'scylla-server-rust/**' env: SOURCE_DATABASE_URL: postgresql://postgres:password@127.0.0.1:5432/timescaledb cargo run From e80173dd7291ce0cedb8cc6a352ce10efa7fc77d Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Mon, 8 Jul 2024 20:36:24 -0400 Subject: [PATCH 21/68] basic mqtt working --- docker-compose.yml | 40 --- scylla-server-rust/Cargo.lock | 253 +++++++++++++++++- scylla-server-rust/Cargo.toml | 2 + .../src/controllers/data_controller.rs | 2 +- .../src/controllers/data_type_controller.rs | 2 +- .../src/controllers/run_controller.rs | 4 +- scylla-server-rust/src/lib.rs | 1 + scylla-server-rust/src/main.rs | 32 ++- .../src/services/data_service.rs | 49 ++-- .../src/services/data_type_service.rs | 19 +- .../src/services/node_service.rs | 6 +- .../src/services/run_service.rs | 47 +++- scylla-server-rust/src/socket/mod.rs | 3 + .../src/socket/mqtt_reciever.rs | 195 ++++++++++++++ .../src/socket/socket_handler.rs | 28 ++ .../src/transformers/data_transformer.rs | 12 +- scylla-server-rust/tests/data_service_test.rs | 40 ++- .../tests/data_type_service_test.rs | 4 +- scylla-server-rust/tests/run_service_test.rs | 6 +- 19 files changed, 638 insertions(+), 107 deletions(-) create mode 100644 scylla-server-rust/src/socket/mod.rs create mode 100644 scylla-server-rust/src/socket/mqtt_reciever.rs create mode 100644 scylla-server-rust/src/socket/socket_handler.rs diff --git a/docker-compose.yml b/docker-compose.yml index 85ed3d71..44edeb30 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,30 +17,6 @@ services: mem_limit: 3gb stop_grace_period: 1m - scylla-server: - container_name: scylla-server - restart: unless-stopped - image: ghcr.io/northeastern-electric-racing/argos:Develop-scylla-server - build: - context: ./scylla-server - target: production - dockerfile: Dockerfile - ports: - - 8000:8000 - depends_on: - - odyssey-timescale - - siren - environment: - - SOURCE_DATABASE_URL=postgresql://postgres:password@odyssey-timescale:5432/timescaledb - - PROD_SIREN_HOST_URL=siren - - PROD=true - # extra_hosts: - # - "host.docker.internal:host-gateway" # for external siren - cpu_shares: 1024 - mem_limit: 2gb - stop_grace_period: 10s - - client: container_name: client restart: unless-stopped @@ -55,25 +31,9 @@ services: dockerfile: Dockerfile ports: - 80:80 - depends_on: - - scylla-server cpu_shares: 512 mem_limit: 1gb - siren: - container_name: siren - restart: unless-stopped - image: eclipse-mosquitto:latest - ports: - - 1883:1883 - - 9001:9001 # why? - expose: - - 1883 - volumes: - - ./siren-base/mosquitto/mosquitto.conf:/mosquitto/config/mosquitto.conf - cpu_shares: 2048 - mem_limit: 2gb - oom_kill_disable: true volumes: diff --git a/scylla-server-rust/Cargo.lock b/scylla-server-rust/Cargo.lock index c2936845..1f0ffd18 100755 --- a/scylla-server-rust/Cargo.lock +++ b/scylla-server-rust/Cargo.lock @@ -115,7 +115,7 @@ dependencies = [ "hyper 1.4.0", "hyper-util", "itoa", - "matchit", + "matchit 0.7.3", "memchr", "mime", "percent-encoding", @@ -203,6 +203,12 @@ version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "bigdecimal" version = "0.3.1" @@ -283,6 +289,9 @@ name = "bytes" version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" +dependencies = [ + "serde", +] [[package]] name = "cc" @@ -469,6 +478,12 @@ dependencies = [ "sha3", ] +[[package]] +name = "data-encoding" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" + [[package]] name = "datamodel-renderer" version = "0.1.0" @@ -590,6 +605,32 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" +[[package]] +name = "engineioxide" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33b9cfc311d0ac3237b8177d2ee8962aa5bb4cfa22faf284356f2ebaf9d698f0" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-core", + "futures-util", + "http 1.1.0", + "http-body 1.0.0", + "http-body-util", + "hyper 1.4.0", + "hyper-util", + "pin-project-lite", + "rand 0.8.5", + "serde", + "serde_json", + "smallvec", + "thiserror", + "tokio", + "tokio-tungstenite", + "tower", +] + [[package]] name = "enumflags2" version = "0.7.10" @@ -676,6 +717,17 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "flume" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55ac459de2512911e4b674ce33cf20befaba382d05b62b008afc1c8b57cbf181" +dependencies = [ + "futures-core", + "futures-sink", + "spin", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1357,6 +1409,12 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" +[[package]] +name = "matchit" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d3c2fcf089c060eb333302d80c5f3ffa8297abecf220f788e4a09ef85f59420" + [[package]] name = "md-5" version = "0.10.6" @@ -2611,7 +2669,7 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls-pemfile", + "rustls-pemfile 1.0.4", "serde", "serde_json", "serde_urlencoded", @@ -2627,6 +2685,39 @@ dependencies = [ "winreg", ] +[[package]] +name = "ring" +version = "0.17.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.15", + "libc", + "spin", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rumqttc" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1568e15fab2d546f940ed3a21f48bbbd1c494c90c99c4481339364a497f94a9" +dependencies = [ + "bytes", + "flume", + "futures-util", + "log", + "rustls-native-certs", + "rustls-pemfile 2.1.2", + "rustls-webpki", + "thiserror", + "tokio", + "tokio-rustls", +] + [[package]] name = "rustc-demangle" version = "0.1.24" @@ -2646,6 +2737,33 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rustls" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf4ef73721ac7bcd79b2b315da7779d8fc09718c6b3d2d1b2d94850eb8c18432" +dependencies = [ + "log", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a88d6d420651b496bdd98684116959239430022a115c1240e6c3993be0b15fba" +dependencies = [ + "openssl-probe", + "rustls-pemfile 2.1.2", + "rustls-pki-types", + "schannel", + "security-framework", +] + [[package]] name = "rustls-pemfile" version = "1.0.4" @@ -2655,6 +2773,33 @@ dependencies = [ "base64 0.21.7", ] +[[package]] +name = "rustls-pemfile" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d" +dependencies = [ + "base64 0.22.1", + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" + +[[package]] +name = "rustls-webpki" +version = "0.102.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9a6fccd794a42c2c105b513a2f62bc3fd8f3ba57a4593677ceb0bd035164d78" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "rustversion" version = "1.0.17" @@ -2723,7 +2868,9 @@ dependencies = [ "prisma-client-rust", "protobuf", "protobuf-codegen", + "rumqttc", "serde", + "socketioxide", "tokio", "tower", "tower-http", @@ -2827,6 +2974,17 @@ dependencies = [ "serde", ] +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest 0.10.7", +] + [[package]] name = "sha2" version = "0.9.9" @@ -2926,6 +3084,38 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "socketioxide" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f50a295325631d230022f1562fde3d1351edf4d8eac73265f657cc762f655c" +dependencies = [ + "bytes", + "engineioxide", + "futures-core", + "futures-util", + "http 1.1.0", + "http-body 1.0.0", + "hyper 1.4.0", + "itoa", + "matchit 0.8.3", + "pin-project-lite", + "serde", + "serde_json", + "thiserror", + "tokio", + "tower", +] + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + [[package]] name = "sql-ddl" version = "0.1.0" @@ -3267,6 +3457,29 @@ dependencies = [ "tokio-util", ] +[[package]] +name = "tokio-rustls" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" +dependencies = [ + "rustls", + "rustls-pki-types", + "tokio", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6989540ced10490aaf14e6bad2e3d33728a2813310a0c71d1574304c49631cd" +dependencies = [ + "futures-util", + "log", + "tokio", + "tungstenite", +] + [[package]] name = "tokio-util" version = "0.7.11" @@ -3442,6 +3655,24 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "tungstenite" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2e2ce1e47ed2994fd43b04c8f618008d4cabdd5ee34027cf14f9d918edd9c8" +dependencies = [ + "byteorder", + "bytes", + "data-encoding", + "http 1.1.0", + "httparse", + "log", + "rand 0.8.5", + "sha1", + "thiserror", + "utf-8", +] + [[package]] name = "typenum" version = "1.17.0" @@ -3508,6 +3739,12 @@ dependencies = [ "void", ] +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "url" version = "2.5.2" @@ -3544,6 +3781,12 @@ dependencies = [ "user-facing-error-macros", ] +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + [[package]] name = "uuid" version = "1.8.0" @@ -3872,3 +4115,9 @@ dependencies = [ "cfg-if", "windows-sys 0.48.0", ] + +[[package]] +name = "zeroize" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" diff --git a/scylla-server-rust/Cargo.toml b/scylla-server-rust/Cargo.toml index dbda33fb..e140cccf 100644 --- a/scylla-server-rust/Cargo.toml +++ b/scylla-server-rust/Cargo.toml @@ -12,6 +12,8 @@ tokio = { version = "1.38.0", features = ["full"] } axum = "0.7.5" tower = { version = "0.4.13", features = ["timeout"] } tower-http = { version = "0.5.2", features = ["cors"] } +socketioxide = "0.14.0" +rumqttc = "0.24.0" [workspace] members = ["prisma-cli"] diff --git a/scylla-server-rust/src/controllers/data_controller.rs b/scylla-server-rust/src/controllers/data_controller.rs index 6731246a..ebc30013 100644 --- a/scylla-server-rust/src/controllers/data_controller.rs +++ b/scylla-server-rust/src/controllers/data_controller.rs @@ -12,7 +12,7 @@ pub async fn get_data( State(db): State, Path((data_type_name, run_id)): Path<(String, i32)>, ) -> Result>, ScyllaError> { - let data = data_service::get_data(&db, data_type_name, run_id).await?; + let data = data_service::get_data(&db, data_type_name, run_id, true, true).await?; // map data to frontend data types according to the From func of the client struct let mut transformed_data: Vec = data.iter().map(PublicData::from).collect(); diff --git a/scylla-server-rust/src/controllers/data_type_controller.rs b/scylla-server-rust/src/controllers/data_type_controller.rs index d80ed459..bb725577 100644 --- a/scylla-server-rust/src/controllers/data_type_controller.rs +++ b/scylla-server-rust/src/controllers/data_type_controller.rs @@ -8,7 +8,7 @@ use crate::{ pub async fn get_all_data_types( State(db): State, ) -> Result>, ScyllaError> { - let data_types = data_type_service::get_all_data_types(&db).await?; + let data_types = data_type_service::get_all_data_types(&db, true).await?; let transformed_data_types: Vec = data_types.iter().map(PublicDataType::from).collect(); diff --git a/scylla-server-rust/src/controllers/run_controller.rs b/scylla-server-rust/src/controllers/run_controller.rs index 4c26eac4..ed01cfa2 100644 --- a/scylla-server-rust/src/controllers/run_controller.rs +++ b/scylla-server-rust/src/controllers/run_controller.rs @@ -8,7 +8,7 @@ use crate::{ }; pub async fn get_all_runs(State(db): State) -> Result>, ScyllaError> { - let run_data = run_service::get_all_runs(&db).await?; + let run_data = run_service::get_all_runs(&db, true, true, true).await?; let transformed_run_data: Vec = run_data.iter().map(PublicRun::from).collect(); @@ -19,7 +19,7 @@ pub async fn get_run_by_id( State(db): State, Path(run_id): Path, ) -> Result, ScyllaError> { - let run_data = run_service::get_run_by_id(&db, run_id).await?; + let run_data = run_service::get_run_by_id(&db, true, true, true, run_id).await?; if run_data.is_none() { return Err(ScyllaError::NotFound); diff --git a/scylla-server-rust/src/lib.rs b/scylla-server-rust/src/lib.rs index df6b8493..6c8532ef 100644 --- a/scylla-server-rust/src/lib.rs +++ b/scylla-server-rust/src/lib.rs @@ -1,6 +1,7 @@ pub mod controllers; pub mod error; pub mod services; +pub mod socket; pub mod transformers; #[allow(clippy::all)] diff --git a/scylla-server-rust/src/main.rs b/scylla-server-rust/src/main.rs index 533d9f76..3f01bc98 100755 --- a/scylla-server-rust/src/main.rs +++ b/scylla-server-rust/src/main.rs @@ -7,13 +7,34 @@ use scylla_server_rust::{ run_controller, system_controller, }, prisma::PrismaClient, + socket::{ + mqtt_reciever::{recieve_mqtt, MqttReciever}, + socket_handler, + }, Database, }; +use socketioxide::SocketIo; +use tokio::sync::mpsc; +use tower::ServiceBuilder; use tower_http::cors::{Any, CorsLayer}; #[tokio::main] async fn main() { - let client: Database = Arc::new(PrismaClient::_builder().build().await.unwrap()); + let db: Database = Arc::new(PrismaClient::_builder().build().await.unwrap()); + + let (socket_layer, io) = SocketIo::new_layer(); + + // channel to pass the mqtt data + // TODO tune buffer size + let (tx, rx) = mpsc::channel::(32); + + // spawn the socket handler + tokio::spawn(socket_handler::handle_socket(io, rx)); + + // create and spawn the mock handler + let recv = MqttReciever::new(tx, "localhost:1883", db.clone()).await; + recv.siren_connect().await; + tokio::spawn(recieve_mqtt(recv)); let app = Router::new() // get all data with the name dataTypeName and runID as specified @@ -36,6 +57,7 @@ async fn main() { .route("/runs/:id", get(run_controller::get_run_by_id)) // get all systems .route("/systems", get(system_controller::get_all_systems)) + // for CORS handling .layer( CorsLayer::new() // allow `GET` @@ -43,7 +65,13 @@ async fn main() { // allow requests from any origin .allow_origin(Any), ) - .with_state(client); + // for socketio integration + .layer( + ServiceBuilder::new() + .layer(CorsLayer::permissive()) + .layer(socket_layer), + ) + .with_state(db.clone()); let listener = tokio::net::TcpListener::bind("0.0.0.0:8000").await.unwrap(); axum::serve(listener, app).await.unwrap(); diff --git a/scylla-server-rust/src/services/data_service.rs b/scylla-server-rust/src/services/data_service.rs index 3283b1db..fe183f7c 100644 --- a/scylla-server-rust/src/services/data_service.rs +++ b/scylla-server-rust/src/services/data_service.rs @@ -1,28 +1,36 @@ use prisma_client_rust::{chrono::DateTime, QueryError}; -use crate::{ - prisma::{self}, - serverdata::ServerData, - Database, -}; +use crate::{prisma, socket::socket_handler::ClientData, Database}; /// Get datapoints that mach criteria /// * `db` - The prisma client to make the call to /// * `data_type_name` - The data type name to filter the data by -/// * `run_id` - The run id to filter the data by +/// * `run_id` - The run id to filter the data +/// * `fetch_run` whether to fetch the run assocaited with this data +/// * `fetch_data_type` whether to fetch the data type associated with this data /// returns: A result containing the data or the QueryError propogated by the db pub async fn get_data( db: &Database, data_type_name: String, run_id: i32, + fetch_run: bool, + fetch_data_type: bool, ) -> Result, QueryError> { - db.data() - .find_many(vec![ - prisma::data::data_type_name::equals(data_type_name), - prisma::data::run_id::equals(run_id), - ]) - .exec() - .await + let mut find_q = db.data().find_many(vec![ + prisma::data::data_type_name::equals(data_type_name), + prisma::data::run_id::equals(run_id), + ]); + + // add a with statement to fetch runs + + if fetch_run { + find_q = find_q.with(prisma::data::run::fetch()); + } + if fetch_data_type { + find_q = find_q.with(prisma::data::data_type::fetch()); + } + + find_q.exec().await } /// Adds a datapoint @@ -34,21 +42,18 @@ pub async fn get_data( /// returns: A result containing the data or the QueryError propogated by the db pub async fn add_data( db: &Database, - serverdata: ServerData, - unix_time: i64, - data_type_name: String, - run_id: i32, + client_data: ClientData, ) -> Result { db.data() .create( - prisma::data_type::name::equals(data_type_name), - DateTime::from_timestamp_millis(unix_time) + prisma::data_type::name::equals(client_data.name), + DateTime::from_timestamp_millis(client_data.timestamp) .unwrap() .fixed_offset(), - prisma::run::id::equals(run_id), + prisma::run::id::equals(client_data.run_id), vec![prisma::data::values::set( - serverdata - .value + client_data + .values .iter() .map(|f| f.parse::().unwrap()) .collect(), diff --git a/scylla-server-rust/src/services/data_type_service.rs b/scylla-server-rust/src/services/data_type_service.rs index 00bf25e2..d7b793e1 100644 --- a/scylla-server-rust/src/services/data_type_service.rs +++ b/scylla-server-rust/src/services/data_type_service.rs @@ -4,9 +4,19 @@ use crate::{prisma, Database}; /// Gets all datatypes /// * `db` - The prisma client to make the call to +/// * `fetch_node` - Whether to fetch the node associated with the data type /// returns: A result containing the data or the QueryError propogated by the db -pub async fn get_all_data_types(db: &Database) -> Result, QueryError> { - db.data_type().find_many(vec![]).exec().await +pub async fn get_all_data_types( + db: &Database, + fetch_node: bool, +) -> Result, QueryError> { + let mut find_q = db.data_type().find_many(vec![]); + + if fetch_node { + find_q = find_q.with(prisma::data_type::node::fetch()); + } + + find_q.exec().await } /// Upserts a datatype, either creating or updating one depending on its existence @@ -25,14 +35,15 @@ pub async fn upsert_data_type( .upsert( prisma::data_type::name::equals(data_type_name.clone()), prisma::data_type::create( - data_type_name, + data_type_name.clone(), unit.clone(), prisma::node::name::equals(node_name.clone()), vec![], ), vec![ prisma::data_type::unit::set(unit), - prisma::data_type::node_name::set(node_name), + prisma::data_type::name::set(data_type_name.clone()), + prisma::data_type::node::connect(prisma::node::name::equals(node_name.clone())), ], ) .exec() diff --git a/scylla-server-rust/src/services/node_service.rs b/scylla-server-rust/src/services/node_service.rs index 5f8608af..c359b3fb 100644 --- a/scylla-server-rust/src/services/node_service.rs +++ b/scylla-server-rust/src/services/node_service.rs @@ -6,7 +6,11 @@ use crate::{prisma, Database}; /// * `db` - The prisma client to make the call to /// returns: A result containing the data or the QueryError propogated by the db pub async fn get_all_nodes(db: &Database) -> Result, QueryError> { - db.node().find_many(vec![]).exec().await + db.node() + .find_many(vec![]) + .with(prisma::node::data_types::fetch(vec![])) + .exec() + .await } /// Upserts a node, either creating or updating one depending on its existence diff --git a/scylla-server-rust/src/services/run_service.rs b/scylla-server-rust/src/services/run_service.rs index 1a126040..ee9c21f3 100644 --- a/scylla-server-rust/src/services/run_service.rs +++ b/scylla-server-rust/src/services/run_service.rs @@ -9,23 +9,58 @@ use crate::{ /// Gets all runs /// * `db` - The prisma client to make the call to +/// * `fetch_loc` - Whether to fetch the location data +/// * `fetch_driver` - Whether to fetch the driver data +/// * `fetch_system` - Whether to fetch the system data /// returns: A result containing the data or the QueryError propogated by the db -pub async fn get_all_runs(db: &Database) -> Result, QueryError> { - db.run().find_many(vec![]).exec().await +pub async fn get_all_runs( + db: &Database, + fetch_loc: bool, + fetch_driver: bool, + fetch_system: bool, +) -> Result, QueryError> { + let mut find_q = db.run().find_many(vec![]); + + if fetch_loc { + find_q = find_q.with(prisma::run::location::fetch()); + } + if fetch_driver { + find_q = find_q.with(prisma::run::driver::fetch()); + } + if fetch_system { + find_q = find_q.with(prisma::run::system::fetch()); + } + + find_q.exec().await } /// Gets a single run by its id /// * `db` - The prisma client to make the call to +/// * `fetch_loc` - Whether to fetch the location data +/// * `fetch_driver` - Whether to fetch the driver data +/// * `fetch_system` - Whether to fetch the system data /// * `run_id` - The id of the run to search for /// returns: A result containing the data (or None if the `run_id` was not a valid run) or the QueryError propogated by the db pub async fn get_run_by_id( db: &Database, + fetch_loc: bool, + fetch_driver: bool, + fetch_system: bool, run_id: i32, ) -> Result, QueryError> { - db.run() - .find_unique(prisma::run::id::equals(run_id)) - .exec() - .await + let mut find_q = db.run().find_unique(prisma::run::id::equals(run_id)); + + if fetch_loc { + find_q = find_q.with(prisma::run::location::fetch()); + } + if fetch_driver { + find_q = find_q.with(prisma::run::driver::fetch()); + } + if fetch_system { + find_q = find_q.with(prisma::run::system::fetch()); + } + + find_q.exec().await } /// Creates a run diff --git a/scylla-server-rust/src/socket/mod.rs b/scylla-server-rust/src/socket/mod.rs new file mode 100644 index 00000000..fc95aeb4 --- /dev/null +++ b/scylla-server-rust/src/socket/mod.rs @@ -0,0 +1,3 @@ +pub mod mock_reciever; +pub mod mqtt_reciever; +pub mod socket_handler; diff --git a/scylla-server-rust/src/socket/mqtt_reciever.rs b/scylla-server-rust/src/socket/mqtt_reciever.rs new file mode 100644 index 00000000..4096f328 --- /dev/null +++ b/scylla-server-rust/src/socket/mqtt_reciever.rs @@ -0,0 +1,195 @@ +use core::fmt; +use std::time::Duration; + +use prisma_client_rust::{chrono, QueryError}; +use protobuf::Message; +use tokio::sync::mpsc::Sender; + +use crate::{ + serverdata, + services::{data_service, data_type_service, node_service, run_service}, + Database, +}; + +use super::socket_handler::ClientData; +use paho_mqtt::MQTT_VERSION_5; + +pub struct MqttReciever { + channel: Sender, + client: paho_mqtt::AsyncClient, + recv_stream: paho_mqtt::AsyncReceiver>, + database: Database, + current_run: i32, +} + +impl MqttReciever { + /// Creates a new mqtt reciever + /// * `channel` - The mpsc channel to send the socket.io data to + /// * `mqtt_path` - The mqtt URI (without the mqtt://) to subscribe to + /// * `db` - The database to store the data in + pub async fn new(channel: Sender, mqtt_path: &str, db: Database) -> MqttReciever { + // create the mqtt client and configure it + let create_opts = paho_mqtt::CreateOptionsBuilder::new() + .server_uri(format!("mqtt://{}", mqtt_path)) + .client_id("ScyllaServer") + .finalize(); + + let mut client = + paho_mqtt::AsyncClient::new(create_opts).expect("Could not create MQTT client!"); + + // TODO tune limit + let cli_stream = client.get_stream(30); + + // creates the initial run + let curr_run = run_service::create_run(&db, chrono::offset::Utc::now().timestamp_millis()) + .await + .expect("Could not create initial run!"); + + MqttReciever { + channel, + client, + recv_stream: cli_stream, + database: db, + current_run: curr_run.id, + } + } + + /// connect to siren or panicing if siren cannot be found + /// Uses lwt and begins the subscription if the connection was successful + pub async fn siren_connect(&self) { + let lwt = paho_mqtt::Message::new("Scylla/Status", "Scylla Disconnected", 2); + let conn_opts = paho_mqtt::ConnectOptionsBuilder::with_mqtt_version(MQTT_VERSION_5) + .will_message(lwt) + .keep_alive_interval(Duration::from_secs(20)) + .clean_session(false) + .automatic_reconnect(Duration::from_secs(1), Duration::from_secs(30)) + .finalize(); + + self.client.connect(conn_opts).await.expect(&format!( + "Could not connect to Siren at {}", + self.client.server_uri() + )); + + self.client + .subscribe("#", 2) + .await + .expect("Could not subscribe to Siren"); + } + + /// Parse the message + /// * `msg` - The mqtt message to parse + /// returns the ClientData and the node name, or the Err of something that can be debug printed + async fn parse_msg( + &self, + msg: paho_mqtt::Message, + ) -> Result<(ClientData, String), impl fmt::Debug> { + println!("Recved msg!"); + + let data = serverdata::ServerData::parse_from_bytes(msg.payload()) + .map_err(|f| format!("Could not parse message: {}", f.to_string()))?; + + let split = msg.topic().split_once("/").ok_or("Could not parse topic")?; + + let node = split.0; + + let data_type = split.1.replace("/", "-"); + + println!( + "{:?}, a: {:?}", + msg.properties(), + msg.properties().find_user_property("ts") + ); + + let unix_time = msg + .properties() + .find_user_property("ts") + .unwrap_or_else(|| chrono::offset::Utc::now().timestamp_millis().to_string()); + + let Ok(time_clean) = unix_time.parse::() else { + return Err(format!("Invalid timestamp {}", unix_time)); + }; + if time_clean < 963014966000 { + println!("Hit ts"); + return Err("Timestamp before year 2000!".to_owned()); + } + + Ok(( + ClientData { + run_id: self.current_run, + name: data_type, + unit: data.unit, + values: data.value, + timestamp: time_clean, + }, + node.to_string(), + )) + } + + /// Send a message to the channel, printing and IGNORING any error that may occur + /// * `client_data` - The cliet data to send over the socket + async fn send_msg(&self, client_data: &ClientData) { + println!("{:?}", client_data); + + let _ = self + .channel + .send(client_data.clone()) + .await + .inspect_err(|f| println!("Error sending through channel: {:?}", f)); + } + + /// Stores the data in the database + /// * `client_data` - The client data to store + /// * `node` - The name of the node to store it under + /// returns Ok or the QueryError + async fn store_data(&self, client_data: ClientData, node: String) -> Result<(), QueryError> { + node_service::upsert_node(&self.database, node.clone()).await?; + data_type_service::upsert_data_type( + &self.database, + client_data.name.clone(), + client_data.unit.clone(), + node, + ) + .await?; + + match client_data.name { + _ => { + data_service::add_data(&self.database, client_data).await?; + } + } + + Ok(()) + } +} + +/// This handles the reception of mqtt messages +/// This would make more sense to go into the Impl, however the lifetime of the struct would no longer be static! +pub async fn recieve_mqtt(rec: MqttReciever) { + // iterate over messages + while let Ok(msg) = rec.recv_stream.recv().await { + // safe unwrap the message + if let Some(msg) = msg { + // parse the message into the data and the node name it falls under + let item_data = match rec.parse_msg(msg).await { + Ok(msg) => msg, + Err(err) => { + println!("Message parse error: {:?}", err); + continue; + } + }; + // send the message over the channel to the socketio consumer + rec.send_msg(&item_data.0).await; + // store the data or print any storage error that occurs + if let Err(store) = rec.store_data(item_data.0, item_data.1).await { + println!("Storage error: {:?}", store); + } + } else { + // basic and somewhat untested reconnection logic + println!("Lost connection. Attempting reconnect."); + while let Err(err) = rec.client.reconnect().await { + println!("Error reconnecting: {}", err); + + tokio::time::sleep(Duration::from_millis(1000)).await; + } + } + } +} diff --git a/scylla-server-rust/src/socket/socket_handler.rs b/scylla-server-rust/src/socket/socket_handler.rs new file mode 100644 index 00000000..e7f58dcc --- /dev/null +++ b/scylla-server-rust/src/socket/socket_handler.rs @@ -0,0 +1,28 @@ +use prisma_client_rust::serde_json; +use serde::Serialize; +use socketioxide::{extract::SocketRef, SocketIo}; +use tokio::sync::mpsc::Receiver; + +/// Represents the client data +#[derive(Serialize, Clone, Debug)] +pub struct ClientData { + pub run_id: i32, + pub name: String, + pub unit: String, + pub values: Vec, + pub timestamp: i64, +} + +pub async fn handle_socket(io: SocketIo, mut channel: Receiver) { + io.ns("/", |s: SocketRef| { + s.on_disconnect(|_: SocketRef| println!("Socket: Client disconnected from socket")) + }); + + // await a new message to send to client + while let Some(cmd) = channel.recv().await { + match io.emit("message", serde_json::to_string(&cmd).unwrap()) { + Ok(_) => (), + Err(err) => println!("Socket: Broadcast error: {}", err), + } + } +} diff --git a/scylla-server-rust/src/transformers/data_transformer.rs b/scylla-server-rust/src/transformers/data_transformer.rs index ab183948..7d1d499d 100644 --- a/scylla-server-rust/src/transformers/data_transformer.rs +++ b/scylla-server-rust/src/transformers/data_transformer.rs @@ -1,6 +1,6 @@ use serde::Serialize; -use crate::prisma; +use crate::{prisma, socket::socket_handler::ClientData}; /// The struct defining the data format sent to the client #[derive(Serialize, Debug, PartialEq, Eq, PartialOrd, Ord)] @@ -18,3 +18,13 @@ impl From<&prisma::data::Data> for PublicData { } } } + +/// convert from the client (socket) type to the client type, for debugging and testing only probably +impl From for PublicData { + fn from(value: ClientData) -> Self { + PublicData { + time: value.timestamp, + values: value.values, + } + } +} diff --git a/scylla-server-rust/tests/data_service_test.rs b/scylla-server-rust/tests/data_service_test.rs index d32b58ea..c6043d47 100644 --- a/scylla-server-rust/tests/data_service_test.rs +++ b/scylla-server-rust/tests/data_service_test.rs @@ -2,10 +2,9 @@ mod test_utils; use prisma_client_rust::QueryError; -use protobuf::SpecialFields; use scylla_server_rust::{ - serverdata::ServerData, services::{data_service, data_type_service, node_service, run_service}, + socket::socket_handler::ClientData, transformers::data_transformer::PublicData, }; use test_utils::cleanup_and_prepare; @@ -25,7 +24,7 @@ async fn test_data_service() -> Result<(), QueryError> { TEST_KEYWORD.to_owned(), ) .await?; - data_service::get_data(&db, TEST_KEYWORD.to_owned(), 0).await?; + data_service::get_data(&db, TEST_KEYWORD.to_owned(), 0, true, true).await?; Ok(()) } @@ -46,14 +45,13 @@ async fn test_data_add() -> Result<(), QueryError> { let data = data_service::add_data( &db, - ServerData { - value: vec!["0".to_owned()], + ClientData { + values: vec!["0".to_owned()], unit: "A".to_owned(), - special_fields: SpecialFields::new(), + run_id: run_data.id, + name: TEST_KEYWORD.to_owned(), + timestamp: 1000, }, - 1000, - TEST_KEYWORD.to_owned(), - run_data.id, ) .await?; @@ -73,7 +71,7 @@ async fn test_data_fetch_empty() -> Result<(), QueryError> { let db = cleanup_and_prepare().await?; // should be empty, nothing was added to run - let data = data_service::get_data(&db, TEST_KEYWORD.to_owned(), 0).await?; + let data = data_service::get_data(&db, TEST_KEYWORD.to_owned(), 0, true, true).await?; assert!(data.is_empty()); @@ -87,14 +85,13 @@ async fn test_data_no_prereqs() -> Result<(), QueryError> { // should err out as data type name doesnt exist yet data_service::add_data( &db, - ServerData { - value: vec!["0".to_owned()], + ClientData { + values: vec!["0".to_owned()], unit: "A".to_owned(), - special_fields: SpecialFields::new(), + run_id: 0, + name: TEST_KEYWORD.to_owned(), + timestamp: 1000, }, - 1000, - TEST_KEYWORD.to_owned(), - 0, ) .await .expect_err("Should have errored, datatype doesnt exist!"); @@ -113,14 +110,13 @@ async fn test_data_no_prereqs() -> Result<(), QueryError> { // now shouldnt fail as it and node does exist data_service::add_data( &db, - ServerData { - value: vec!["0".to_owned()], + ClientData { + values: vec!["0".to_owned()], unit: "A".to_owned(), - special_fields: SpecialFields::new(), + run_id: 0, + name: TEST_KEYWORD.to_owned(), + timestamp: 1000, }, - 1000, - TEST_KEYWORD.to_owned(), - 0, ) .await?; diff --git a/scylla-server-rust/tests/data_type_service_test.rs b/scylla-server-rust/tests/data_type_service_test.rs index 86037f0f..1c3167e3 100644 --- a/scylla-server-rust/tests/data_type_service_test.rs +++ b/scylla-server-rust/tests/data_type_service_test.rs @@ -16,7 +16,9 @@ async fn test_get_all_datatypes() -> Result<(), QueryError> { let db = cleanup_and_prepare().await?; // ensure datatypes is empty - assert!(data_type_service::get_all_data_types(&db).await?.is_empty()); + assert!(data_type_service::get_all_data_types(&db, true) + .await? + .is_empty()); Ok(()) } diff --git a/scylla-server-rust/tests/run_service_test.rs b/scylla-server-rust/tests/run_service_test.rs index e13bd2c3..7f40cc6a 100644 --- a/scylla-server-rust/tests/run_service_test.rs +++ b/scylla-server-rust/tests/run_service_test.rs @@ -10,7 +10,9 @@ async fn test_get_all_runs() -> Result<(), QueryError> { let db = cleanup_and_prepare().await?; // ensure runs is empty - assert!(run_service::get_all_runs(&db).await?.is_empty()); + assert!(run_service::get_all_runs(&db, true, true, true) + .await? + .is_empty()); Ok(()) } @@ -23,7 +25,7 @@ async fn test_get_run_by_id() -> Result<(), QueryError> { let run_c = run_service::create_run(&db, 1).await?; // get that run - let run = run_service::get_run_by_id(&db, run_c.id) + let run = run_service::get_run_by_id(&db, true, true, true, run_c.id) .await? .expect("Run should exist was upserted "); From 4fa870c9f41ea1def49f2b10b784d17ddb124404 Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Mon, 8 Jul 2024 21:52:18 -0400 Subject: [PATCH 22/68] single recv working --- scylla-server-rust/src/main.rs | 5 +- .../src/socket/mqtt_reciever.rs | 167 +++++++++--------- 2 files changed, 87 insertions(+), 85 deletions(-) diff --git a/scylla-server-rust/src/main.rs b/scylla-server-rust/src/main.rs index 3f01bc98..e731e1c0 100755 --- a/scylla-server-rust/src/main.rs +++ b/scylla-server-rust/src/main.rs @@ -32,9 +32,8 @@ async fn main() { tokio::spawn(socket_handler::handle_socket(io, rx)); // create and spawn the mock handler - let recv = MqttReciever::new(tx, "localhost:1883", db.clone()).await; - recv.siren_connect().await; - tokio::spawn(recieve_mqtt(recv)); + let (recv, opts) = MqttReciever::new(tx, "localhost:1883", db.clone()).await; + tokio::spawn(recieve_mqtt(recv, opts)); let app = Router::new() // get all data with the name dataTypeName and runID as specified diff --git a/scylla-server-rust/src/socket/mqtt_reciever.rs b/scylla-server-rust/src/socket/mqtt_reciever.rs index 4096f328..a8b72e55 100644 --- a/scylla-server-rust/src/socket/mqtt_reciever.rs +++ b/scylla-server-rust/src/socket/mqtt_reciever.rs @@ -3,6 +3,10 @@ use std::time::Duration; use prisma_client_rust::{chrono, QueryError}; use protobuf::Message; +use rumqttc::v5::{ + mqttbytes::v5::{LastWill, Packet, Publish}, + AsyncClient, Event, MqttOptions, +}; use tokio::sync::mpsc::Sender; use crate::{ @@ -12,110 +16,109 @@ use crate::{ }; use super::socket_handler::ClientData; -use paho_mqtt::MQTT_VERSION_5; +use std::borrow::Cow; pub struct MqttReciever { channel: Sender, - client: paho_mqtt::AsyncClient, - recv_stream: paho_mqtt::AsyncReceiver>, database: Database, - current_run: i32, + curr_run: i32, } impl MqttReciever { /// Creates a new mqtt reciever /// * `channel` - The mpsc channel to send the socket.io data to - /// * `mqtt_path` - The mqtt URI (without the mqtt://) to subscribe to + /// * `mqtt_path` - The mqtt URI, including port, (without the mqtt://) to subscribe to /// * `db` - The database to store the data in - pub async fn new(channel: Sender, mqtt_path: &str, db: Database) -> MqttReciever { + pub async fn new( + channel: Sender, + mqtt_path: &str, + db: Database, + ) -> (MqttReciever, MqttOptions) { // create the mqtt client and configure it - let create_opts = paho_mqtt::CreateOptionsBuilder::new() - .server_uri(format!("mqtt://{}", mqtt_path)) - .client_id("ScyllaServer") - .finalize(); - - let mut client = - paho_mqtt::AsyncClient::new(create_opts).expect("Could not create MQTT client!"); - - // TODO tune limit - let cli_stream = client.get_stream(30); + let mut create_opts = MqttOptions::new( + "ScyllaServer", + mqtt_path.split_once(":").expect("Invalid Siren URL").0, + mqtt_path + .split_once(":") + .unwrap() + .1 + .parse::() + .expect("Invalid Siren port"), + ); + create_opts.set_keep_alive(Duration::from_secs(20)); + create_opts.set_last_will(LastWill::new( + "Scylla/Status", + "Scylla has disconnected!", + rumqttc::v5::mqttbytes::QoS::ExactlyOnce, + true, + None, + )); + create_opts.set_clean_start(false); // creates the initial run let curr_run = run_service::create_run(&db, chrono::offset::Utc::now().timestamp_millis()) .await .expect("Could not create initial run!"); - MqttReciever { - channel, - client, - recv_stream: cli_stream, - database: db, - current_run: curr_run.id, - } - } - - /// connect to siren or panicing if siren cannot be found - /// Uses lwt and begins the subscription if the connection was successful - pub async fn siren_connect(&self) { - let lwt = paho_mqtt::Message::new("Scylla/Status", "Scylla Disconnected", 2); - let conn_opts = paho_mqtt::ConnectOptionsBuilder::with_mqtt_version(MQTT_VERSION_5) - .will_message(lwt) - .keep_alive_interval(Duration::from_secs(20)) - .clean_session(false) - .automatic_reconnect(Duration::from_secs(1), Duration::from_secs(30)) - .finalize(); - - self.client.connect(conn_opts).await.expect(&format!( - "Could not connect to Siren at {}", - self.client.server_uri() - )); - - self.client - .subscribe("#", 2) - .await - .expect("Could not subscribe to Siren"); + ( + MqttReciever { + channel, + database: db, + curr_run: curr_run.id, + }, + create_opts, + ) } /// Parse the message /// * `msg` - The mqtt message to parse /// returns the ClientData and the node name, or the Err of something that can be debug printed - async fn parse_msg( - &self, - msg: paho_mqtt::Message, - ) -> Result<(ClientData, String), impl fmt::Debug> { - println!("Recved msg!"); - let data = serverdata::ServerData::parse_from_bytes(msg.payload()) - .map_err(|f| format!("Could not parse message: {}", f.to_string()))?; + async fn parse_msg(&self, msg: Publish) -> Result<(ClientData, String), impl fmt::Debug> { + let data = serverdata::ServerData::parse_from_bytes(&msg.payload).map_err(|f| { + format!( + "Could not parse message topic:{:?} error: {}", + msg.topic, + f.to_string() + ) + })?; - let split = msg.topic().split_once("/").ok_or("Could not parse topic")?; + let split = std::str::from_utf8(&msg.topic) + .expect(&format!("Could not parse topic: {:?}", msg.topic)) + .split_once("/") + .ok_or(&format!("Could not parse nesting: {:?}", msg.topic))?; let node = split.0; let data_type = split.1.replace("/", "-"); - println!( - "{:?}, a: {:?}", - msg.properties(), - msg.properties().find_user_property("ts") - ); - + // extract the unix time, returning the current time instead if needed + // note the Cow magic involves the return from the map is a borrow, but the unwrap cannot as we dont own it let unix_time = msg - .properties() - .find_user_property("ts") - .unwrap_or_else(|| chrono::offset::Utc::now().timestamp_millis().to_string()); - - let Ok(time_clean) = unix_time.parse::() else { - return Err(format!("Invalid timestamp {}", unix_time)); + .properties + .unwrap_or_default() + .user_properties + .iter() + .map(Cow::Borrowed) + .find(|f| f.0 == "ts") + .unwrap_or_else(|| { + Cow::Owned(( + "ts".to_string(), + chrono::offset::Utc::now().timestamp_millis().to_string(), + )) + }) + .into_owned(); + + let Ok(time_clean) = unix_time.1.parse::() else { + return Err(format!("Invalid timestamp: {}", unix_time.1)); }; if time_clean < 963014966000 { - println!("Hit ts"); - return Err("Timestamp before year 2000!".to_owned()); + return Err(format!("Timestamp before year 2000: {}", unix_time.1)); } Ok(( ClientData { - run_id: self.current_run, + run_id: self.curr_run, name: data_type, unit: data.unit, values: data.value, @@ -128,8 +131,6 @@ impl MqttReciever { /// Send a message to the channel, printing and IGNORING any error that may occur /// * `client_data` - The cliet data to send over the socket async fn send_msg(&self, client_data: &ClientData) { - println!("{:?}", client_data); - let _ = self .channel .send(client_data.clone()) @@ -163,11 +164,21 @@ impl MqttReciever { /// This handles the reception of mqtt messages /// This would make more sense to go into the Impl, however the lifetime of the struct would no longer be static! -pub async fn recieve_mqtt(rec: MqttReciever) { +/// * `rec` - The mqtt reciever created by new +/// * `client_opts` - The client options returned by new +pub async fn recieve_mqtt(rec: MqttReciever, client_opts: MqttOptions) { + // TODO mess with incoming message cap if db, etc. cannot keep up + let (client, mut connect) = AsyncClient::new(client_opts, 30); + + client + .subscribe("#", rumqttc::v5::mqttbytes::QoS::ExactlyOnce) + .await + .expect("Could not subscribe to Siren"); + // iterate over messages - while let Ok(msg) = rec.recv_stream.recv().await { - // safe unwrap the message - if let Some(msg) = msg { + while let Ok(msg) = connect.poll().await { + // safe parse the message + if let Event::Incoming(Packet::Publish(msg)) = msg { // parse the message into the data and the node name it falls under let item_data = match rec.parse_msg(msg).await { Ok(msg) => msg, @@ -182,14 +193,6 @@ pub async fn recieve_mqtt(rec: MqttReciever) { if let Err(store) = rec.store_data(item_data.0, item_data.1).await { println!("Storage error: {:?}", store); } - } else { - // basic and somewhat untested reconnection logic - println!("Lost connection. Attempting reconnect."); - while let Err(err) = rec.client.reconnect().await { - println!("Error reconnecting: {}", err); - - tokio::time::sleep(Duration::from_millis(1000)).await; - } } } } From 0c83453eb10ef7d589f69f64577067281bc730cf Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Wed, 10 Jul 2024 00:17:43 -0400 Subject: [PATCH 23/68] most insert logic add special types upsert, upsert checks, batching, two db threads and clean shutdown --- scylla-server-rust/Cargo.lock | 55 +++- scylla-server-rust/Cargo.toml | 1 + scylla-server-rust/src/error.rs | 1 + scylla-server-rust/src/lib.rs | 2 +- scylla-server-rust/src/main.rs | 59 ++++- scylla-server-rust/src/reciever/db_handler.rs | 248 ++++++++++++++++++ .../src/reciever/mock_reciever.rs | 22 ++ scylla-server-rust/src/reciever/mod.rs | 21 ++ .../src/reciever/mqtt_reciever.rs | 164 ++++++++++++ .../{socket => reciever}/socket_handler.rs | 15 +- .../src/services/data_service.rs | 30 ++- scylla-server-rust/src/socket/mod.rs | 3 - .../src/socket/mqtt_reciever.rs | 198 -------------- .../src/transformers/data_transformer.rs | 2 +- scylla-server-rust/tests/data_service_test.rs | 5 +- 15 files changed, 594 insertions(+), 232 deletions(-) create mode 100644 scylla-server-rust/src/reciever/db_handler.rs create mode 100644 scylla-server-rust/src/reciever/mock_reciever.rs create mode 100644 scylla-server-rust/src/reciever/mod.rs create mode 100644 scylla-server-rust/src/reciever/mqtt_reciever.rs rename scylla-server-rust/src/{socket => reciever}/socket_handler.rs (62%) delete mode 100644 scylla-server-rust/src/socket/mod.rs delete mode 100644 scylla-server-rust/src/socket/mqtt_reciever.rs diff --git a/scylla-server-rust/Cargo.lock b/scylla-server-rust/Cargo.lock index 1f0ffd18..71d7eeaa 100755 --- a/scylla-server-rust/Cargo.lock +++ b/scylla-server-rust/Cargo.lock @@ -28,6 +28,18 @@ dependencies = [ "version_check", ] +[[package]] +name = "ahash" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "0.7.20" @@ -46,6 +58,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" + [[package]] name = "android-tzdata" version = "0.1.1" @@ -932,7 +950,7 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" dependencies = [ - "ahash", + "ahash 0.7.8", ] [[package]] @@ -941,7 +959,7 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" dependencies = [ - "ahash", + "ahash 0.7.8", ] [[package]] @@ -949,6 +967,10 @@ name = "hashbrown" version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash 0.8.11", + "allocator-api2", +] [[package]] name = "heck" @@ -1437,7 +1459,7 @@ version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e52eb6380b6d2a10eb3434aec0885374490f5b82c8aaf5cd487a183c98be834" dependencies = [ - "ahash", + "ahash 0.7.8", "metrics-macros", ] @@ -1447,7 +1469,7 @@ version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "142c53885123b68d94108295a09d4afe1a1388ed95b54d5dacd9a454753030f2" dependencies = [ - "ahash", + "ahash 0.7.8", "metrics-macros", ] @@ -2872,6 +2894,7 @@ dependencies = [ "serde", "socketioxide", "tokio", + "tokio-util", "tower", "tower-http", ] @@ -3488,8 +3511,12 @@ checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" dependencies = [ "bytes", "futures-core", + "futures-io", "futures-sink", + "futures-util", + "hashbrown 0.14.5", "pin-project-lite", + "slab", "tokio", ] @@ -4116,6 +4143,26 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "zerocopy" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.67", +] + [[package]] name = "zeroize" version = "1.8.1" diff --git a/scylla-server-rust/Cargo.toml b/scylla-server-rust/Cargo.toml index e140cccf..d0ce6fd5 100644 --- a/scylla-server-rust/Cargo.toml +++ b/scylla-server-rust/Cargo.toml @@ -14,6 +14,7 @@ tower = { version = "0.4.13", features = ["timeout"] } tower-http = { version = "0.5.2", features = ["cors"] } socketioxide = "0.14.0" rumqttc = "0.24.0" +tokio-util = { version= "0.7.11", features = ["full"] } [workspace] members = ["prisma-cli"] diff --git a/scylla-server-rust/src/error.rs b/scylla-server-rust/src/error.rs index 9c57b350..734949d3 100644 --- a/scylla-server-rust/src/error.rs +++ b/scylla-server-rust/src/error.rs @@ -14,6 +14,7 @@ pub enum ScyllaError { impl From for ScyllaError { fn from(error: QueryError) -> Self { + println!("Query error: {:?}", error); match error { e if e.is_prisma_error::() => ScyllaError::NotFound, e => ScyllaError::PrismaError(e), diff --git a/scylla-server-rust/src/lib.rs b/scylla-server-rust/src/lib.rs index 6c8532ef..fe2033bd 100644 --- a/scylla-server-rust/src/lib.rs +++ b/scylla-server-rust/src/lib.rs @@ -1,7 +1,7 @@ pub mod controllers; pub mod error; +pub mod reciever; pub mod services; -pub mod socket; pub mod transformers; #[allow(clippy::all)] diff --git a/scylla-server-rust/src/main.rs b/scylla-server-rust/src/main.rs index e731e1c0..918865c3 100755 --- a/scylla-server-rust/src/main.rs +++ b/scylla-server-rust/src/main.rs @@ -7,33 +7,56 @@ use scylla_server_rust::{ run_controller, system_controller, }, prisma::PrismaClient, - socket::{ - mqtt_reciever::{recieve_mqtt, MqttReciever}, - socket_handler, - }, + reciever::{db_handler, mqtt_reciever::MqttReciever, socket_handler, ClientData}, Database, }; use socketioxide::SocketIo; -use tokio::sync::mpsc; +use tokio::{ + signal, + sync::{broadcast as Broadcaster, mpsc}, +}; +use tokio_util::{sync::CancellationToken, task::TaskTracker}; use tower::ServiceBuilder; use tower_http::cors::{Any, CorsLayer}; #[tokio::main] async fn main() { + // create the database stuff let db: Database = Arc::new(PrismaClient::_builder().build().await.unwrap()); + // create the socket stuff let (socket_layer, io) = SocketIo::new_layer(); // channel to pass the mqtt data // TODO tune buffer size - let (tx, rx) = mpsc::channel::(32); + let (tx, rx) = Broadcaster::channel::(1000); + let rx2 = tx.subscribe(); + + // channel to pass the processed data to the db thread + // TODO tune buffer size + let (tx_proc, rx_proc) = mpsc::channel::>(1000000); // spawn the socket handler tokio::spawn(socket_handler::handle_socket(io, rx)); - // create and spawn the mock handler - let (recv, opts) = MqttReciever::new(tx, "localhost:1883", db.clone()).await; - tokio::spawn(recieve_mqtt(recv, opts)); + // the below two threads need to cancel cleanly to ensure all queued messages are sent. therefore they are part of the a task tracker group. + // create a task tracker and cancellation token + let task_tracker = TaskTracker::new(); + let token = CancellationToken::new(); + // spawn the database handler + task_tracker.spawn( + db_handler::DbHandler::new(rx2, Arc::clone(&db)).handling_loop(tx_proc, token.clone()), + ); + // spawm the database inserter + task_tracker.spawn(db_handler::DbHandler::batching_loop( + rx_proc, + Arc::clone(&db), + token.clone(), + )); + + // create and spawn the mqtt reciever + let (recv, eloop) = MqttReciever::new(tx, "localhost:1883", db.clone()).await; + tokio::spawn(recv.recieve_mqtt(eloop)); let app = Router::new() // get all data with the name dataTypeName and runID as specified @@ -73,5 +96,21 @@ async fn main() { .with_state(db.clone()); let listener = tokio::net::TcpListener::bind("0.0.0.0:8000").await.unwrap(); - axum::serve(listener, app).await.unwrap(); + let axum_token = token.clone(); + tokio::spawn(async { + axum::serve(listener, app) + .with_graceful_shutdown(async move { + _ = axum_token.cancelled().await; + }) + .await + .expect("Failed shutdown init for axum"); + }); + + task_tracker.close(); + // listen for ctrl_c, then cancel, close, and await for all tasks in the tracker. Other tasks cancel vai the default tokio system + signal::ctrl_c() + .await + .expect("Could not read cancellation trigger (ctr+c)"); + token.cancel(); + task_tracker.wait().await; } diff --git a/scylla-server-rust/src/reciever/db_handler.rs b/scylla-server-rust/src/reciever/db_handler.rs new file mode 100644 index 00000000..23d3fca2 --- /dev/null +++ b/scylla-server-rust/src/reciever/db_handler.rs @@ -0,0 +1,248 @@ +use tokio::sync::mpsc::Receiver; + +use tokio::{sync::mpsc::Sender, time::Duration}; + +use tokio::sync::broadcast::Receiver as BroadcastReceiver; +use tokio_util::sync::CancellationToken; + +use crate::{ + services::{ + data_service, data_type_service, driver_service, location_service, node_service, + system_service, + }, + Database, +}; + +const UPLOAD_INTERVAL: u64 = 5000; + +use super::ClientData; + +struct LocLock { + location_name: Option, + points: Option<(f64, f64)>, + radius: Option, +} + +struct Loc { + location_name: String, + lat: f64, + long: f64, + radius: f64, +} + +impl LocLock { + pub fn new() -> LocLock { + LocLock { + location_name: None, + points: None, + radius: None, + } + } + + pub fn add_loc_name(&mut self, loc_name: String) { + self.location_name = Some(loc_name); + } + + pub fn add_points(&mut self, lat: f64, long: f64) { + self.points = Some((lat, long)); + } + + pub fn add_radius(&mut self, radius: f64) { + self.radius = Some(radius); + } + + pub fn finalize(&mut self) -> Option { + if self.location_name.is_some() && self.points.is_some() && self.radius.is_some() { + self.clear(); + return Some(Loc { + location_name: self.location_name.clone().unwrap(), + lat: self.points.unwrap().0, + long: self.points.unwrap().1, + radius: self.radius.unwrap(), + }); + } + None + } + + fn clear(&mut self) { + self.location_name = None; + self.points = None; + self.radius = None; + } +} + +pub struct DbHandler { + node_list: Vec, + datatype_list: Vec, + reciever: BroadcastReceiver, + db: Database, + loc_lock: LocLock, + is_loc: bool, +} + +impl DbHandler { + /// Make a new db handler + /// * `recv` - the broadcast reciver of which clientdata will be sent + pub fn new(reciever: BroadcastReceiver, db: Database) -> DbHandler { + DbHandler { + node_list: vec![], + datatype_list: vec![], + reciever, + db, + loc_lock: LocLock::new(), + is_loc: false, + } + } + + pub async fn batching_loop( + mut data_queue: Receiver>, + database: Database, + cancel_token: CancellationToken, + ) { + loop { + tokio::select! { + _ = cancel_token.cancelled() => { + let final_msgs = data_queue.recv().await.expect("DB send could not shutdown cleanly!"); + println!( + "Final Batch uploaded: {:?}", + data_service::add_many(&database, final_msgs).await + ); + break; + }, + Some(msg) = data_queue.recv() => { + println!( + "Batch uploaded: {:?}", + data_service::add_many(&database, msg).await + ); + } + } + } + } + + pub async fn handling_loop( + mut self, + data_channel: Sender>, + cancel_token: CancellationToken, + ) { + let mut data_queue: Vec = vec![]; + let mut last_time = tokio::time::Instant::now(); + loop { + tokio::select! { + _ = cancel_token.cancelled() => { + println!("Pushing final messages to queue"); + data_channel.send(data_queue.clone()).await.expect("Could not comm data to db thread, shutdown"); + data_queue.clear(); + break; + }, + Ok(msg) = self.reciever.recv() => { + + if tokio::time::Instant::now().duration_since(last_time) + > Duration::from_millis(UPLOAD_INTERVAL) + { + data_channel.send(data_queue.clone()).await.expect("Could not comm data to db thread"); + data_queue.clear(); + last_time = tokio::time::Instant::now(); + } + + // upsert if not present, a sort of cache of upserted types really + if !self.node_list.contains(&msg.node) { + println!("Upserting node: {}", msg.node); + if let Err(msg) = node_service::upsert_node(&self.db, msg.node.clone()).await { + println!("DB error node upsert: {:?}", msg); + } + self.node_list.push(msg.node.clone()); + } + if !self.datatype_list.contains(&msg.name) { + println!("Upserting data type: {}", msg.name); + if let Err(msg) = data_type_service::upsert_data_type( + &self.db, + msg.name.clone(), + msg.unit.clone(), + msg.node.clone(), + ) + .await { + println!("DB error datatype upsert: {:?}", msg); + } + self.datatype_list.push(msg.name.clone()); + } + + // if data has some special meanings, push them to the database immediately, otherwise enter batching logic + match msg.name.as_str() { + "Driver" => { + let _ = driver_service::upsert_driver( + &self.db, + msg.values + .first() + .unwrap_or(&"PizzaTheHut".to_string()) + .to_string(), + msg.run_id, + ) + .await; + } + "location" => { + self.loc_lock.add_loc_name( + msg.values + .first() + .unwrap_or(&"PizzaTheHut".to_string()) + .to_string(), + ); + self.is_loc = true; + } + "system" => { + let _ = system_service::upsert_system( + &self.db, + msg.values + .first() + .unwrap_or(&"PizzaTheHut".to_string()) + .to_string(), + msg.run_id, + ) + .await; + } + "GPS-Location" => { + self.loc_lock.add_points( + msg.values + .first() + .unwrap_or(&"PizzaTheHut".to_string()) + .parse::() + .unwrap_or_default(), + msg.values + .get(1) + .unwrap_or(&"PizzaTheHut".to_string()) + .parse::() + .unwrap_or_default(), + ); + self.is_loc = true; + } + "Radius" => { + self.loc_lock.add_radius( + msg.values + .first() + .unwrap_or(&"PizzaTheHut".to_string()) + .parse::() + .unwrap_or_default(), + ); + self.is_loc = true; + } + _ => { + data_queue.push(msg.clone()); + } + } + if self.is_loc { + if let Some(loc) = self.loc_lock.finalize() { + let _ = location_service::upsert_location( + &self.db, + loc.location_name, + loc.lat, + loc.long, + loc.radius, + msg.run_id, + ) + .await; + } + } + } + } + } + } +} diff --git a/scylla-server-rust/src/reciever/mock_reciever.rs b/scylla-server-rust/src/reciever/mock_reciever.rs new file mode 100644 index 00000000..59a52baf --- /dev/null +++ b/scylla-server-rust/src/reciever/mock_reciever.rs @@ -0,0 +1,22 @@ +// use std::time::Duration; + +// use prisma_client_rust::chrono::{self}; +// use tokio::sync::mpsc::Sender; + +// use super::ClientData; + +// pub async fn recieve_mock(channel: Sender) { +// loop { +// tokio::time::sleep(Duration::from_millis(300)).await; +// let a = channel +// .send(ClientData { +// run_id: 0, +// name: "Acceleration".to_owned(), +// unit: "abc".to_owned(), +// values: vec!["1".to_owned()], +// timestamp: chrono::offset::Utc::now().timestamp_millis(), +// node: "ABC".to_string(), +// }) +// .await; +// } +// } diff --git a/scylla-server-rust/src/reciever/mod.rs b/scylla-server-rust/src/reciever/mod.rs new file mode 100644 index 00000000..0ad6d7ca --- /dev/null +++ b/scylla-server-rust/src/reciever/mod.rs @@ -0,0 +1,21 @@ +pub mod db_handler; +pub mod mock_reciever; +pub mod mqtt_reciever; +pub mod socket_handler; + +/// Represents the client data +/// This has the dual purposes of +/// * - representing the packet sent over the socket for live data +/// * - representing the struct for the service layer to unpack for insertion +/// Note: node name is only considered for database storage and convenience, it is not serialized in a socket packet +#[derive(serde::Serialize, Clone, Debug)] +pub struct ClientData { + pub run_id: i32, + pub name: String, + pub unit: String, + pub values: Vec, + pub timestamp: i64, + + #[serde(skip_serializing)] + pub node: String, +} diff --git a/scylla-server-rust/src/reciever/mqtt_reciever.rs b/scylla-server-rust/src/reciever/mqtt_reciever.rs new file mode 100644 index 00000000..c4631a93 --- /dev/null +++ b/scylla-server-rust/src/reciever/mqtt_reciever.rs @@ -0,0 +1,164 @@ +use core::fmt; +use std::time::Duration; + +use prisma_client_rust::chrono; +use protobuf::Message; +use rumqttc::v5::{ + mqttbytes::v5::{LastWill, Packet, Publish}, + AsyncClient, Event, EventLoop, MqttOptions, +}; +use tokio::sync::broadcast::Sender; + +use crate::{serverdata, services::run_service, Database}; + +use super::ClientData; +use std::borrow::Cow; + +pub struct MqttReciever { + channel: Sender, + curr_run: i32, +} + +impl MqttReciever { + /// Creates a new mqtt reciever + /// * `channel` - The mpsc channel to send the socket.io data to + /// * `mqtt_path` - The mqtt URI, including port, (without the mqtt://) to subscribe to + /// * `db` - The database to store the data in + /// + /// This is async as it creates the initial run and gets the ID, as well as connecting to and subbing Siren + /// Returns the instance and the event loop, which can be passed into the recieve_mqtt func to begin recieiving + pub async fn new( + channel: Sender, + mqtt_path: &str, + db: Database, + ) -> (MqttReciever, EventLoop) { + // create the mqtt client and configure it + let mut create_opts = MqttOptions::new( + "ScyllaServer", + mqtt_path.split_once(':').expect("Invalid Siren URL").0, + mqtt_path + .split_once(':') + .unwrap() + .1 + .parse::() + .expect("Invalid Siren port"), + ); + create_opts.set_keep_alive(Duration::from_secs(20)); + create_opts.set_last_will(LastWill::new( + "Scylla/Status", + "Scylla has disconnected!", + rumqttc::v5::mqttbytes::QoS::ExactlyOnce, + true, + None, + )); + create_opts.set_clean_start(false); + + // creates the initial run + let curr_run = run_service::create_run(&db, chrono::offset::Utc::now().timestamp_millis()) + .await + .expect("Could not create initial run!"); + + // TODO mess with incoming message cap if db, etc. cannot keep up + let (client, connect) = AsyncClient::new(create_opts, 1000); + + client + .subscribe("#", rumqttc::v5::mqttbytes::QoS::ExactlyOnce) + .await + .expect("Could not subscribe to Siren"); + + ( + MqttReciever { + channel, + curr_run: curr_run.id, + }, + connect, + ) + } + + /// This handles the reception of mqtt messages, will not return + /// * `connect` - The eventloop returned by ::new to connect to + pub async fn recieve_mqtt(self, mut connect: EventLoop) { + // process over messages, non blocking + while let Ok(msg) = connect.poll().await { + // safe parse the message + if let Event::Incoming(Packet::Publish(msg)) = msg { + // parse the message into the data and the node name it falls under + let item_data = match self.parse_msg(msg).await { + Ok(msg) => msg, + Err(err) => { + println!("Message parse error: {:?}", err); + continue; + } + }; + // send the message over the channel to the socketio and database consumers + self.send_msg(item_data).await; + } + } + } + + /// Parse the message + /// * `msg` - The mqtt message to parse + /// returns the ClientData, or the Err of something that can be debug printed + async fn parse_msg(&self, msg: Publish) -> Result { + let data = serverdata::ServerData::parse_from_bytes(&msg.payload).map_err(|f| { + format!( + "Could not parse message topic:{:?} error: {}", + msg.topic, + f + ) + })?; + + let split = std::str::from_utf8(&msg.topic) + .unwrap_or_else(|_| panic!("Could not parse topic: {:?}", msg.topic)) + .split_once('/') + .ok_or(&format!("Could not parse nesting: {:?}", msg.topic))?; + + let node = split.0; + + let data_type = split.1.replace('/', "-"); + + // extract the unix time, returning the current time instead if needed + // note the Cow magic involves the return from the map is a borrow, but the unwrap cannot as we dont own it + let unix_time = msg + .properties + .unwrap_or_default() + .user_properties + .iter() + .map(Cow::Borrowed) + .find(|f| f.0 == "ts") + .unwrap_or_else(|| { + Cow::Owned(( + "ts".to_string(), + chrono::offset::Utc::now().timestamp_millis().to_string(), + )) + }) + .into_owned(); + + // parse time, if time isnt present use sys time (see above) + let Ok(time_clean) = unix_time.1.parse::() else { + return Err(format!("Invalid timestamp: {}", unix_time.1)); + }; + // ts check for bad sources of time which may return 1970 + if time_clean < 963014966000 { + return Err(format!("Timestamp before year 2000: {}", unix_time.1)); + } + + Ok(ClientData { + run_id: self.curr_run, + name: data_type, + unit: data.unit, + values: data.value, + timestamp: time_clean, + node: node.to_string(), + }) + } + + /// Send a message to the channel, printing and IGNORING any error that may occur + /// * `client_data` - The cliet data to send over the broadcast + async fn send_msg(&self, client_data: ClientData) { + let _ = self + .channel + .send(client_data) + .inspect_err(|f| println!("Error sending through channel: {:?}", f)); + } +} diff --git a/scylla-server-rust/src/socket/socket_handler.rs b/scylla-server-rust/src/reciever/socket_handler.rs similarity index 62% rename from scylla-server-rust/src/socket/socket_handler.rs rename to scylla-server-rust/src/reciever/socket_handler.rs index e7f58dcc..aa7dcf22 100644 --- a/scylla-server-rust/src/socket/socket_handler.rs +++ b/scylla-server-rust/src/reciever/socket_handler.rs @@ -1,17 +1,8 @@ use prisma_client_rust::serde_json; -use serde::Serialize; use socketioxide::{extract::SocketRef, SocketIo}; -use tokio::sync::mpsc::Receiver; +use tokio::sync::broadcast::Receiver; -/// Represents the client data -#[derive(Serialize, Clone, Debug)] -pub struct ClientData { - pub run_id: i32, - pub name: String, - pub unit: String, - pub values: Vec, - pub timestamp: i64, -} +use super::ClientData; pub async fn handle_socket(io: SocketIo, mut channel: Receiver) { io.ns("/", |s: SocketRef| { @@ -19,7 +10,7 @@ pub async fn handle_socket(io: SocketIo, mut channel: Receiver) { }); // await a new message to send to client - while let Some(cmd) = channel.recv().await { + while let Ok(cmd) = channel.recv().await { match io.emit("message", serde_json::to_string(&cmd).unwrap()) { Ok(_) => (), Err(err) => println!("Socket: Broadcast error: {}", err), diff --git a/scylla-server-rust/src/services/data_service.rs b/scylla-server-rust/src/services/data_service.rs index fe183f7c..7e711c11 100644 --- a/scylla-server-rust/src/services/data_service.rs +++ b/scylla-server-rust/src/services/data_service.rs @@ -1,6 +1,6 @@ use prisma_client_rust::{chrono::DateTime, QueryError}; -use crate::{prisma, socket::socket_handler::ClientData, Database}; +use crate::{prisma, reciever::ClientData, Database}; /// Get datapoints that mach criteria /// * `db` - The prisma client to make the call to @@ -55,10 +55,36 @@ pub async fn add_data( client_data .values .iter() - .map(|f| f.parse::().unwrap()) + .map(|f| f.parse::().unwrap_or_default()) .collect(), )], ) .exec() .await } + +pub async fn add_many(db: &Database, client_data: Vec) -> Result { + db.data() + .create_many( + client_data + .iter() + .map(|f| { + prisma::data::create_unchecked( + f.name.to_string(), + DateTime::from_timestamp_millis(f.timestamp) + .unwrap() + .fixed_offset(), + f.run_id, + vec![prisma::data::values::set( + f.values + .iter() + .map(|f| f.parse::().unwrap_or_default()) + .collect(), + )], + ) + }) + .collect(), + ) + .exec() + .await +} diff --git a/scylla-server-rust/src/socket/mod.rs b/scylla-server-rust/src/socket/mod.rs deleted file mode 100644 index fc95aeb4..00000000 --- a/scylla-server-rust/src/socket/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub mod mock_reciever; -pub mod mqtt_reciever; -pub mod socket_handler; diff --git a/scylla-server-rust/src/socket/mqtt_reciever.rs b/scylla-server-rust/src/socket/mqtt_reciever.rs deleted file mode 100644 index a8b72e55..00000000 --- a/scylla-server-rust/src/socket/mqtt_reciever.rs +++ /dev/null @@ -1,198 +0,0 @@ -use core::fmt; -use std::time::Duration; - -use prisma_client_rust::{chrono, QueryError}; -use protobuf::Message; -use rumqttc::v5::{ - mqttbytes::v5::{LastWill, Packet, Publish}, - AsyncClient, Event, MqttOptions, -}; -use tokio::sync::mpsc::Sender; - -use crate::{ - serverdata, - services::{data_service, data_type_service, node_service, run_service}, - Database, -}; - -use super::socket_handler::ClientData; -use std::borrow::Cow; - -pub struct MqttReciever { - channel: Sender, - database: Database, - curr_run: i32, -} - -impl MqttReciever { - /// Creates a new mqtt reciever - /// * `channel` - The mpsc channel to send the socket.io data to - /// * `mqtt_path` - The mqtt URI, including port, (without the mqtt://) to subscribe to - /// * `db` - The database to store the data in - pub async fn new( - channel: Sender, - mqtt_path: &str, - db: Database, - ) -> (MqttReciever, MqttOptions) { - // create the mqtt client and configure it - let mut create_opts = MqttOptions::new( - "ScyllaServer", - mqtt_path.split_once(":").expect("Invalid Siren URL").0, - mqtt_path - .split_once(":") - .unwrap() - .1 - .parse::() - .expect("Invalid Siren port"), - ); - create_opts.set_keep_alive(Duration::from_secs(20)); - create_opts.set_last_will(LastWill::new( - "Scylla/Status", - "Scylla has disconnected!", - rumqttc::v5::mqttbytes::QoS::ExactlyOnce, - true, - None, - )); - create_opts.set_clean_start(false); - - // creates the initial run - let curr_run = run_service::create_run(&db, chrono::offset::Utc::now().timestamp_millis()) - .await - .expect("Could not create initial run!"); - - ( - MqttReciever { - channel, - database: db, - curr_run: curr_run.id, - }, - create_opts, - ) - } - - /// Parse the message - /// * `msg` - The mqtt message to parse - /// returns the ClientData and the node name, or the Err of something that can be debug printed - - async fn parse_msg(&self, msg: Publish) -> Result<(ClientData, String), impl fmt::Debug> { - let data = serverdata::ServerData::parse_from_bytes(&msg.payload).map_err(|f| { - format!( - "Could not parse message topic:{:?} error: {}", - msg.topic, - f.to_string() - ) - })?; - - let split = std::str::from_utf8(&msg.topic) - .expect(&format!("Could not parse topic: {:?}", msg.topic)) - .split_once("/") - .ok_or(&format!("Could not parse nesting: {:?}", msg.topic))?; - - let node = split.0; - - let data_type = split.1.replace("/", "-"); - - // extract the unix time, returning the current time instead if needed - // note the Cow magic involves the return from the map is a borrow, but the unwrap cannot as we dont own it - let unix_time = msg - .properties - .unwrap_or_default() - .user_properties - .iter() - .map(Cow::Borrowed) - .find(|f| f.0 == "ts") - .unwrap_or_else(|| { - Cow::Owned(( - "ts".to_string(), - chrono::offset::Utc::now().timestamp_millis().to_string(), - )) - }) - .into_owned(); - - let Ok(time_clean) = unix_time.1.parse::() else { - return Err(format!("Invalid timestamp: {}", unix_time.1)); - }; - if time_clean < 963014966000 { - return Err(format!("Timestamp before year 2000: {}", unix_time.1)); - } - - Ok(( - ClientData { - run_id: self.curr_run, - name: data_type, - unit: data.unit, - values: data.value, - timestamp: time_clean, - }, - node.to_string(), - )) - } - - /// Send a message to the channel, printing and IGNORING any error that may occur - /// * `client_data` - The cliet data to send over the socket - async fn send_msg(&self, client_data: &ClientData) { - let _ = self - .channel - .send(client_data.clone()) - .await - .inspect_err(|f| println!("Error sending through channel: {:?}", f)); - } - - /// Stores the data in the database - /// * `client_data` - The client data to store - /// * `node` - The name of the node to store it under - /// returns Ok or the QueryError - async fn store_data(&self, client_data: ClientData, node: String) -> Result<(), QueryError> { - node_service::upsert_node(&self.database, node.clone()).await?; - data_type_service::upsert_data_type( - &self.database, - client_data.name.clone(), - client_data.unit.clone(), - node, - ) - .await?; - - match client_data.name { - _ => { - data_service::add_data(&self.database, client_data).await?; - } - } - - Ok(()) - } -} - -/// This handles the reception of mqtt messages -/// This would make more sense to go into the Impl, however the lifetime of the struct would no longer be static! -/// * `rec` - The mqtt reciever created by new -/// * `client_opts` - The client options returned by new -pub async fn recieve_mqtt(rec: MqttReciever, client_opts: MqttOptions) { - // TODO mess with incoming message cap if db, etc. cannot keep up - let (client, mut connect) = AsyncClient::new(client_opts, 30); - - client - .subscribe("#", rumqttc::v5::mqttbytes::QoS::ExactlyOnce) - .await - .expect("Could not subscribe to Siren"); - - // iterate over messages - while let Ok(msg) = connect.poll().await { - // safe parse the message - if let Event::Incoming(Packet::Publish(msg)) = msg { - // parse the message into the data and the node name it falls under - let item_data = match rec.parse_msg(msg).await { - Ok(msg) => msg, - Err(err) => { - println!("Message parse error: {:?}", err); - continue; - } - }; - // send the message over the channel to the socketio consumer - rec.send_msg(&item_data.0).await; - // store the data or print any storage error that occurs - if let Err(store) = rec.store_data(item_data.0, item_data.1).await { - println!("Storage error: {:?}", store); - } - } - } -} diff --git a/scylla-server-rust/src/transformers/data_transformer.rs b/scylla-server-rust/src/transformers/data_transformer.rs index 7d1d499d..dc024f06 100644 --- a/scylla-server-rust/src/transformers/data_transformer.rs +++ b/scylla-server-rust/src/transformers/data_transformer.rs @@ -1,6 +1,6 @@ use serde::Serialize; -use crate::{prisma, socket::socket_handler::ClientData}; +use crate::{prisma, reciever::ClientData}; /// The struct defining the data format sent to the client #[derive(Serialize, Debug, PartialEq, Eq, PartialOrd, Ord)] diff --git a/scylla-server-rust/tests/data_service_test.rs b/scylla-server-rust/tests/data_service_test.rs index c6043d47..ad8272b4 100644 --- a/scylla-server-rust/tests/data_service_test.rs +++ b/scylla-server-rust/tests/data_service_test.rs @@ -3,8 +3,8 @@ mod test_utils; use prisma_client_rust::QueryError; use scylla_server_rust::{ + reciever::ClientData, services::{data_service, data_type_service, node_service, run_service}, - socket::socket_handler::ClientData, transformers::data_transformer::PublicData, }; use test_utils::cleanup_and_prepare; @@ -51,6 +51,7 @@ async fn test_data_add() -> Result<(), QueryError> { run_id: run_data.id, name: TEST_KEYWORD.to_owned(), timestamp: 1000, + node: "Irrelevant".to_string(), }, ) .await?; @@ -91,6 +92,7 @@ async fn test_data_no_prereqs() -> Result<(), QueryError> { run_id: 0, name: TEST_KEYWORD.to_owned(), timestamp: 1000, + node: "Irrelevant".to_string(), }, ) .await @@ -116,6 +118,7 @@ async fn test_data_no_prereqs() -> Result<(), QueryError> { run_id: 0, name: TEST_KEYWORD.to_owned(), timestamp: 1000, + node: "Irrelevant".to_string(), }, ) .await?; From 5d1cac3acf53c11e701534e4d2dab945cc5a7450 Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Wed, 10 Jul 2024 21:42:03 -0400 Subject: [PATCH 24/68] add compose stuff, args, comments, improve err todo/still wrong with this - mqtt reconnect sucks and there is no disconnect/reconnect printout - there is no mock stuff yet --- .github/workflows/build-image.yml | 6 +- docker-compose.yml | 38 ++++++++++- scylla-server-rust/.dockerignore | 25 +++++++ scylla-server-rust/Dockerfile | 4 ++ scylla-server-rust/README.md | 11 ++- scylla-server-rust/src/main.rs | 11 ++- scylla-server-rust/src/reciever/db_handler.rs | 68 +++++++++++++------ scylla-server-rust/src/reciever/mod.rs | 9 +++ .../src/reciever/mqtt_reciever.rs | 21 ++---- .../src/reciever/socket_handler.rs | 2 +- .../src/services/data_service.rs | 6 +- .../src/services/run_service.rs | 8 +-- 12 files changed, 158 insertions(+), 51 deletions(-) create mode 100644 scylla-server-rust/.dockerignore diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index 728ca589..99d81ab6 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -59,14 +59,14 @@ jobs: PROD=true BACKEND_URL=http://192.168.100.1:8000 MAP_ACCESS_TOKEN=pk.eyJ1IjoibWNrZWVwIiwiYSI6ImNscXBrcmU1ZTBscWIya284cDFyYjR3Nm8ifQ.6TQHlxhAJzptZyV-W28dnw - - name: Build and push Docker image for scylla server + - name: Build and push Docker image for scylla server rust uses: docker/build-push-action@v5.4.0 with: - context: ./scylla-server + context: ./scylla-server-rust push: true target: production platforms: linux/arm64,linux/amd64 - tags: ${{ steps.meta.outputs.tags }}-scylla-server # argos-scylla-server + tags: ${{ steps.meta.outputs.tags }}-scylla-server-rust # argos-scylla-server labels: ${{ steps.meta.outputs.labels }} # for caching # cache-from: type=gha diff --git a/docker-compose.yml b/docker-compose.yml index 44edeb30..9c0e9a3f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,6 +17,27 @@ services: mem_limit: 3gb stop_grace_period: 1m + scylla-server-rust: + container_name: scylla-server-rust + image: ghcr.io/northeastern-electric-racing/argos:Develop-scylla-server-rust + build: + context: ./scylla-server-rust + restart: unless-stopped + ports: + - 8000:8000 + depends_on: + - odyssey-timescale + - siren + environment: + - SOURCE_DATABASE_URL=postgresql://postgres:password@odyssey-timescale:5432/timescaledb + - PROD_SIREN_HOST_URL=siren:1883 + - PROD_SCYLLA=true + cpu_shares: 1024 + mem_limit: 2gb + stop_grace_period: 10s + + + client: container_name: client restart: unless-stopped @@ -25,7 +46,7 @@ services: context: ./angular-client args: PROD: "true" - BACKEND_URL: http://192.168.100.1:8000 + BACKEND_URL: http://localhost:8000 MAP_ACCESS_TOKEN: pk.eyJ1IjoibWNrZWVwIiwiYSI6ImNscXBrcmU1ZTBscWIya284cDFyYjR3Nm8ifQ.6TQHlxhAJzptZyV-W28dnw target: production dockerfile: Dockerfile @@ -34,6 +55,21 @@ services: cpu_shares: 512 mem_limit: 1gb + siren: + container_name: siren + restart: unless-stopped + image: eclipse-mosquitto:latest + ports: + - 1883:1883 + - 9001:9001 # why? + expose: + - 1883 + volumes: + - ./siren-base/mosquitto/mosquitto.conf:/mosquitto/config/mosquitto.conf + cpu_shares: 2048 + mem_limit: 2gb + oom_kill_disable: true + volumes: diff --git a/scylla-server-rust/.dockerignore b/scylla-server-rust/.dockerignore new file mode 100644 index 00000000..5b90b998 --- /dev/null +++ b/scylla-server-rust/.dockerignore @@ -0,0 +1,25 @@ +# editor things +.idea/ +.zed/ +.vscode/ + +# misc things +.DS_Store +.gitignore +*.nix + +# python things +pyrightconfig.json +__pycache__/ + +# Added by cargo (rust things) +/target +build/ +dist/ +logs/ + +# prisma +prisma.rs + +# protobuf +serverdata.rs diff --git a/scylla-server-rust/Dockerfile b/scylla-server-rust/Dockerfile index 2f03ccd9..33180360 100755 --- a/scylla-server-rust/Dockerfile +++ b/scylla-server-rust/Dockerfile @@ -2,3 +2,7 @@ FROM rust:latest WORKDIR /usr/src/myapp +COPY . . +RUN cargo prisma generate +RUN cargo build --release +CMD ["sh", "-c", "cargo prisma migrate deploy && /usr/src/myapp/target/release/scylla-server-rust"] \ No newline at end of file diff --git a/scylla-server-rust/README.md b/scylla-server-rust/README.md index e39a5082..8699edac 100644 --- a/scylla-server-rust/README.md +++ b/scylla-server-rust/README.md @@ -27,4 +27,13 @@ docker volume rm argos_db-data docker compose run -Pd odyssey-timescale cargo prisma migrate deploy SOURCE_DATABASE_URL=postgresql://postgres:password@127.0.0.1:5432/timescaledb cargo test -- --test-threads=1 -``` \ No newline at end of file +``` + +### Deploy this app + +Use the docker compose above to build & deploy. Note the CI prebuilds arm64 and amd64 images upon request in the actions tab of this repository's github page. +``` +docker compose build +docker compose up # use -d to fork to background +``` +A database migration is triggered upon every bootup. diff --git a/scylla-server-rust/src/main.rs b/scylla-server-rust/src/main.rs index 918865c3..68acc15d 100755 --- a/scylla-server-rust/src/main.rs +++ b/scylla-server-rust/src/main.rs @@ -22,7 +22,7 @@ use tower_http::cors::{Any, CorsLayer}; #[tokio::main] async fn main() { // create the database stuff - let db: Database = Arc::new(PrismaClient::_builder().build().await.unwrap()); + let db: Database = Arc::new(PrismaClient::_builder().build().await.expect("Could not build prisma DB")); // create the socket stuff let (socket_layer, io) = SocketIo::new_layer(); @@ -55,7 +55,12 @@ async fn main() { )); // create and spawn the mqtt reciever - let (recv, eloop) = MqttReciever::new(tx, "localhost:1883", db.clone()).await; + let (recv, eloop) = MqttReciever::new( + tx, + std::env::var("PROD_SIREN_HOST_URL").unwrap_or("localhost:1883".to_string()), + db.clone(), + ) + .await; tokio::spawn(recv.recieve_mqtt(eloop)); let app = Router::new() @@ -95,7 +100,7 @@ async fn main() { ) .with_state(db.clone()); - let listener = tokio::net::TcpListener::bind("0.0.0.0:8000").await.unwrap(); + let listener = tokio::net::TcpListener::bind("0.0.0.0:8000").await.expect("Could not bind to 8000!"); let axum_token = token.clone(); tokio::spawn(async { axum::serve(listener, app) diff --git a/scylla-server-rust/src/reciever/db_handler.rs b/scylla-server-rust/src/reciever/db_handler.rs index 23d3fca2..cd8e2510 100644 --- a/scylla-server-rust/src/reciever/db_handler.rs +++ b/scylla-server-rust/src/reciever/db_handler.rs @@ -13,23 +13,18 @@ use crate::{ Database, }; +/// The upload interval for batch uploads const UPLOAD_INTERVAL: u64 = 5000; -use super::ClientData; +use super::{ClientData, LocationData}; +/// A struct defining an in progress location packet struct LocLock { location_name: Option, points: Option<(f64, f64)>, radius: Option, } -struct Loc { - location_name: String, - lat: f64, - long: f64, - radius: f64, -} - impl LocLock { pub fn new() -> LocLock { LocLock { @@ -39,22 +34,26 @@ impl LocLock { } } + /// Add the location name to the packet pub fn add_loc_name(&mut self, loc_name: String) { self.location_name = Some(loc_name); } + /// Add points to the packet pub fn add_points(&mut self, lat: f64, long: f64) { self.points = Some((lat, long)); } + /// Add a radius to the packet pub fn add_radius(&mut self, radius: f64) { self.radius = Some(radius); } - pub fn finalize(&mut self) -> Option { + /// Attempt to finalize the packet, returning a location data and clearing this object or None if still in progress + pub fn finalize(&mut self) -> Option { if self.location_name.is_some() && self.points.is_some() && self.radius.is_some() { self.clear(); - return Some(Loc { + return Some(LocationData { location_name: self.location_name.clone().unwrap(), lat: self.points.unwrap().0, long: self.points.unwrap().1, @@ -64,6 +63,7 @@ impl LocLock { None } + /// Clear the internal state fn clear(&mut self) { self.location_name = None; self.points = None; @@ -71,12 +71,20 @@ impl LocLock { } } +/// A few threads to manage the processing and inserting of special types, +/// upserting of metadata for data, and batch uploading the database pub struct DbHandler { + /// The list of nodes seen by this instance, used for when to upsert node_list: Vec, + /// The list of data types seen by this instance, used for when to upsert datatype_list: Vec, + /// The broadcast channel which provides serial datapoints for processing reciever: BroadcastReceiver, + /// The database db: Database, + /// An internal state of an in progress location packet loc_lock: LocLock, + /// Whether the location has been modified this loop is_loc: bool, } @@ -94,6 +102,9 @@ impl DbHandler { } } + /// This loop handles batch uploading, and has no internal state or requirements + /// It uses the queue from data queue to insert to the database specified + /// On cancellation, will await one final queue message to cleanup anything remaining in the channel pub async fn batching_loop( mut data_queue: Receiver>, database: Database, @@ -119,6 +130,11 @@ impl DbHandler { } } + /// A loop which uses self and a sender channel to process data + /// If the data is special, i.e. coordinates, driver, etc. it will store it in its special location of the db immediately + /// For all data points it will add the to the data_channel for batch uploading logic when a certain time has elapsed + /// Before this time the data is stored in an internal queue. + /// On cancellation, the messages currently in the queue will be sent as a final flush of any remaining messages recieved before cancellation pub async fn handling_loop( mut self, data_channel: Sender>, @@ -136,8 +152,9 @@ impl DbHandler { }, Ok(msg) = self.reciever.recv() => { + // If the time is greater than upload interval, push to batch upload thread and clear queue if tokio::time::Instant::now().duration_since(last_time) - > Duration::from_millis(UPLOAD_INTERVAL) + > Duration::from_millis(UPLOAD_INTERVAL) && !data_queue.is_empty() { data_channel.send(data_queue.clone()).await.expect("Could not comm data to db thread"); data_queue.clear(); @@ -166,10 +183,10 @@ impl DbHandler { self.datatype_list.push(msg.name.clone()); } - // if data has some special meanings, push them to the database immediately, otherwise enter batching logic + // if data has some special meanings, push them to the database immediately, notably no matter what also enter batching logic match msg.name.as_str() { "Driver" => { - let _ = driver_service::upsert_driver( + if let Err(err) = driver_service::upsert_driver( &self.db, msg.values .first() @@ -177,7 +194,9 @@ impl DbHandler { .to_string(), msg.run_id, ) - .await; + .await { + println!("Driver upsert error: {:?}", err); + } } "location" => { self.loc_lock.add_loc_name( @@ -189,7 +208,7 @@ impl DbHandler { self.is_loc = true; } "system" => { - let _ = system_service::upsert_system( + if let Err(err) = system_service::upsert_system( &self.db, msg.values .first() @@ -197,7 +216,9 @@ impl DbHandler { .to_string(), msg.run_id, ) - .await; + .await { + println!("System upsert error: {:?}", err); + } } "GPS-Location" => { self.loc_lock.add_points( @@ -224,13 +245,12 @@ impl DbHandler { ); self.is_loc = true; } - _ => { - data_queue.push(msg.clone()); - } + _ => {} } + // if location has been modified, push a new location of the loc lock object returns Some if self.is_loc { if let Some(loc) = self.loc_lock.finalize() { - let _ = location_service::upsert_location( + if let Err(err) = location_service::upsert_location( &self.db, loc.location_name, loc.lat, @@ -238,9 +258,15 @@ impl DbHandler { loc.radius, msg.run_id, ) - .await; + .await { + println!("Location upsert error: {:?}", err); + } } + self.is_loc = false; } + + // no matter what, batch upload the message + data_queue.push(msg); } } } diff --git a/scylla-server-rust/src/reciever/mod.rs b/scylla-server-rust/src/reciever/mod.rs index 0ad6d7ca..9ebe8f97 100644 --- a/scylla-server-rust/src/reciever/mod.rs +++ b/scylla-server-rust/src/reciever/mod.rs @@ -19,3 +19,12 @@ pub struct ClientData { #[serde(skip_serializing)] pub node: String, } + +/// A final location packet +/// This has the purpose of representing the struct for the service layer to unpack for insertion, and therefore is not serialized +struct LocationData { + location_name: String, + lat: f64, + long: f64, + radius: f64, +} diff --git a/scylla-server-rust/src/reciever/mqtt_reciever.rs b/scylla-server-rust/src/reciever/mqtt_reciever.rs index c4631a93..770c9a9d 100644 --- a/scylla-server-rust/src/reciever/mqtt_reciever.rs +++ b/scylla-server-rust/src/reciever/mqtt_reciever.rs @@ -29,7 +29,7 @@ impl MqttReciever { /// Returns the instance and the event loop, which can be passed into the recieve_mqtt func to begin recieiving pub async fn new( channel: Sender, - mqtt_path: &str, + mqtt_path: String, db: Database, ) -> (MqttReciever, EventLoop) { // create the mqtt client and configure it @@ -62,8 +62,7 @@ impl MqttReciever { let (client, connect) = AsyncClient::new(create_opts, 1000); client - .subscribe("#", rumqttc::v5::mqttbytes::QoS::ExactlyOnce) - .await + .try_subscribe("#", rumqttc::v5::mqttbytes::QoS::ExactlyOnce) .expect("Could not subscribe to Siren"); ( @@ -100,13 +99,8 @@ impl MqttReciever { /// * `msg` - The mqtt message to parse /// returns the ClientData, or the Err of something that can be debug printed async fn parse_msg(&self, msg: Publish) -> Result { - let data = serverdata::ServerData::parse_from_bytes(&msg.payload).map_err(|f| { - format!( - "Could not parse message topic:{:?} error: {}", - msg.topic, - f - ) - })?; + let data = serverdata::ServerData::parse_from_bytes(&msg.payload) + .map_err(|f| format!("Could not parse message topic:{:?} error: {}", msg.topic, f))?; let split = std::str::from_utf8(&msg.topic) .unwrap_or_else(|_| panic!("Could not parse topic: {:?}", msg.topic)) @@ -156,9 +150,8 @@ impl MqttReciever { /// Send a message to the channel, printing and IGNORING any error that may occur /// * `client_data` - The cliet data to send over the broadcast async fn send_msg(&self, client_data: ClientData) { - let _ = self - .channel - .send(client_data) - .inspect_err(|f| println!("Error sending through channel: {:?}", f)); + if let Err(err) = self.channel.send(client_data) { + println!("Error sending through channel: {:?}", err) + } } } diff --git a/scylla-server-rust/src/reciever/socket_handler.rs b/scylla-server-rust/src/reciever/socket_handler.rs index aa7dcf22..6d8b3162 100644 --- a/scylla-server-rust/src/reciever/socket_handler.rs +++ b/scylla-server-rust/src/reciever/socket_handler.rs @@ -11,7 +11,7 @@ pub async fn handle_socket(io: SocketIo, mut channel: Receiver) { // await a new message to send to client while let Ok(cmd) = channel.recv().await { - match io.emit("message", serde_json::to_string(&cmd).unwrap()) { + match io.emit("message", serde_json::to_string(&cmd).expect("Could not serialize ClientData")) { Ok(_) => (), Err(err) => println!("Socket: Broadcast error: {}", err), } diff --git a/scylla-server-rust/src/services/data_service.rs b/scylla-server-rust/src/services/data_service.rs index 7e711c11..b6d3b3da 100644 --- a/scylla-server-rust/src/services/data_service.rs +++ b/scylla-server-rust/src/services/data_service.rs @@ -48,7 +48,7 @@ pub async fn add_data( .create( prisma::data_type::name::equals(client_data.name), DateTime::from_timestamp_millis(client_data.timestamp) - .unwrap() + .expect("Could not parse timestamp") .fixed_offset(), prisma::run::id::equals(client_data.run_id), vec![prisma::data::values::set( @@ -72,8 +72,8 @@ pub async fn add_many(db: &Database, client_data: Vec) -> Result Result Date: Wed, 10 Jul 2024 22:17:43 -0400 Subject: [PATCH 25/68] fix image --- .github/workflows/build-image.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index 99d81ab6..39bc4ed2 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -64,7 +64,6 @@ jobs: with: context: ./scylla-server-rust push: true - target: production platforms: linux/arm64,linux/amd64 tags: ${{ steps.meta.outputs.tags }}-scylla-server-rust # argos-scylla-server labels: ${{ steps.meta.outputs.labels }} From 4fd469e34656c5a59256dd08ba3168f87ccde549 Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Wed, 10 Jul 2024 22:54:28 -0400 Subject: [PATCH 26/68] fix docker wierdness --- docker-compose-dev.yml | 79 -------------------------------- docker-compose.yml | 10 ++-- scylla-server-rust/Dockerfile | 4 +- scylla-server-rust/README.md | 4 +- scylla-server-rust/docker_run.sh | 3 ++ 5 files changed, 13 insertions(+), 87 deletions(-) delete mode 100644 docker-compose-dev.yml create mode 100755 scylla-server-rust/docker_run.sh diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml deleted file mode 100644 index 03f8f3ba..00000000 --- a/docker-compose-dev.yml +++ /dev/null @@ -1,79 +0,0 @@ -services: - odyssey-timescale: - container_name: odyssey-timescale - image: timescale/timescaledb:2.13.1-pg15 - restart: unless-stopped - environment: - POSTGRES_HOST_AUTH_METHOD: trust - ports: # needs to be published for charybdis, which runs outside of docker - - 5432:5432 - expose: - - 5432 - cpu_shares: 1024 - mem_limit: 3gb - cpuset: 0-3 - stop_grace_period: 1m - - scylla-server: - container_name: scylla-server - restart: unless-stopped - # image: ghcr.io/northeastern-electric-racing/argos:ghcr-action-scylla-server - build: - context: ./scylla-server - target: production - dockerfile: Dockerfile - ports: - - 8000:8000 - depends_on: - - odyssey-timescale - - siren - environment: - - SOURCE_DATABASE_URL=postgresql://postgres:password@odyssey-timescale:5432/timescaledb - - PROD_SIREN_HOST_URL=siren - - PROD=true - # extra_hosts: - # - "host.docker.internal:host-gateway" # for external siren - cpu_shares: 1024 - cpuset: 0-3 - mem_limit: 2gb - stop_grace_period: 10s - - - client: - container_name: client - restart: unless-stopped - # image: ghcr.io/northeastern-electric-racing/argos:ghcr-action-angular-client - build: - context: ./angular-client - args: - PROD: "true" - BACKEND_URL: http://localhost:8000 - MAP_ACCESS_TOKEN: pk.eyJ1IjoibWNrZWVwIiwiYSI6ImNscXBrcmU1ZTBscWIya284cDFyYjR3Nm8ifQ.6TQHlxhAJzptZyV-W28dnw - target: production - dockerfile: Dockerfile - ports: - - 80:80 - depends_on: - - scylla-server - cpu_shares: 512 - cpuset: 0-3 - mem_limit: 1gb - - siren: - container_name: siren - restart: unless-stopped - image: eclipse-mosquitto:latest - ports: - - 1883:1883 - - 9001:9001 # why? - expose: - - 1883 - volumes: - - ./siren-base/mosquitto/mosquitto.conf:/mosquitto/config/mosquitto.conf - cpu_shares: 2048 - cpuset: 0-3 - mem_limit: 2gb - oom_kill_disable: true - - - diff --git a/docker-compose.yml b/docker-compose.yml index 9c0e9a3f..0dc259f5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,7 +19,7 @@ services: scylla-server-rust: container_name: scylla-server-rust - image: ghcr.io/northeastern-electric-racing/argos:Develop-scylla-server-rust + image: ghcr.io/northeastern-electric-racing/argos:rust-rewrite-socket-scylla-server-rust build: context: ./scylla-server-rust restart: unless-stopped @@ -35,18 +35,20 @@ services: cpu_shares: 1024 mem_limit: 2gb stop_grace_period: 10s - + stop_signal: SIGINT + init: true + entrypoint: ["./docker_run.sh"] client: container_name: client restart: unless-stopped - image: ghcr.io/northeastern-electric-racing/argos:Develop-angular-client + image: ghcr.io/northeastern-electric-racing/argos:develop-angular-client build: context: ./angular-client args: PROD: "true" - BACKEND_URL: http://localhost:8000 + BACKEND_URL: http://192.168.100.1:8000 MAP_ACCESS_TOKEN: pk.eyJ1IjoibWNrZWVwIiwiYSI6ImNscXBrcmU1ZTBscWIya284cDFyYjR3Nm8ifQ.6TQHlxhAJzptZyV-W28dnw target: production dockerfile: Dockerfile diff --git a/scylla-server-rust/Dockerfile b/scylla-server-rust/Dockerfile index 33180360..e4cee263 100755 --- a/scylla-server-rust/Dockerfile +++ b/scylla-server-rust/Dockerfile @@ -4,5 +4,5 @@ WORKDIR /usr/src/myapp COPY . . RUN cargo prisma generate -RUN cargo build --release -CMD ["sh", "-c", "cargo prisma migrate deploy && /usr/src/myapp/target/release/scylla-server-rust"] \ No newline at end of file +RUN cargo build --release --locked +ENTRYPOINT ["./docker_run.sh"] \ No newline at end of file diff --git a/scylla-server-rust/README.md b/scylla-server-rust/README.md index 8699edac..cb730e3e 100644 --- a/scylla-server-rust/README.md +++ b/scylla-server-rust/README.md @@ -11,7 +11,7 @@ cargo prisma generate ``` # in argos proper -docker compose run -P odyssey-timescale +docker compose up odyssey-timescale ``` ``` @@ -24,7 +24,7 @@ SOURCE_DATABASE_URL=postgresql://postgres:password@127.0.0.1:5432/timescaledb ca Since this app uses the database for testing, you must follow these steps, or run `./integration_test.sh`: ``` docker volume rm argos_db-data -docker compose run -Pd odyssey-timescale +docker compose up odyssey-timescale cargo prisma migrate deploy SOURCE_DATABASE_URL=postgresql://postgres:password@127.0.0.1:5432/timescaledb cargo test -- --test-threads=1 ``` diff --git a/scylla-server-rust/docker_run.sh b/scylla-server-rust/docker_run.sh new file mode 100755 index 00000000..b84d990b --- /dev/null +++ b/scylla-server-rust/docker_run.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +cargo prisma migrate deploy && exec /usr/src/myapp/target/release/scylla-server-rust \ No newline at end of file From 61918e71ff8a201a2cba5d9193913054c0859d5a Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Fri, 12 Jul 2024 20:28:15 -0400 Subject: [PATCH 27/68] move socket to reciever thread, split image workflows --- .../{build-image.yml => build-client.yml} | 13 ----- .github/workflows/scylla-rust-build.yml | 58 +++++++++++++++++++ scylla-server-rust/src/main.rs | 38 +++++++----- scylla-server-rust/src/reciever/db_handler.rs | 7 +-- scylla-server-rust/src/reciever/mod.rs | 1 - .../src/reciever/mqtt_reciever.rs | 22 +++++-- .../src/reciever/socket_handler.rs | 19 ------ .../src/services/data_service.rs | 4 +- .../src/services/run_service.rs | 8 +-- 9 files changed, 108 insertions(+), 62 deletions(-) rename .github/workflows/{build-image.yml => build-client.yml} (84%) create mode 100644 .github/workflows/scylla-rust-build.yml delete mode 100644 scylla-server-rust/src/reciever/socket_handler.rs diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-client.yml similarity index 84% rename from .github/workflows/build-image.yml rename to .github/workflows/build-client.yml index 39bc4ed2..592a5d62 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-client.yml @@ -59,17 +59,4 @@ jobs: PROD=true BACKEND_URL=http://192.168.100.1:8000 MAP_ACCESS_TOKEN=pk.eyJ1IjoibWNrZWVwIiwiYSI6ImNscXBrcmU1ZTBscWIya284cDFyYjR3Nm8ifQ.6TQHlxhAJzptZyV-W28dnw - - name: Build and push Docker image for scylla server rust - uses: docker/build-push-action@v5.4.0 - with: - context: ./scylla-server-rust - push: true - platforms: linux/arm64,linux/amd64 - tags: ${{ steps.meta.outputs.tags }}-scylla-server-rust # argos-scylla-server - labels: ${{ steps.meta.outputs.labels }} - # for caching - # cache-from: type=gha - # cache-to: type=gha,mode=max - # https://github.com/docker/build-push-action/issues/820 - provenance: false diff --git a/.github/workflows/scylla-rust-build.yml b/.github/workflows/scylla-rust-build.yml new file mode 100644 index 00000000..0decd961 --- /dev/null +++ b/.github/workflows/scylla-rust-build.yml @@ -0,0 +1,58 @@ +# +name: Create and publish a Docker image + +on: + workflow_dispatch: + + + +# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. +jobs: + build-and-push-image: + runs-on: ubuntu-latest + # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. + permissions: + contents: read + packages: write + # + steps: + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3.3.0 + - name: Checkout repository + uses: actions/checkout@v4 + # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. + - name: Log in to the Container registry + uses: docker/login-action@v3.2.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels. + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5.5.1 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. + # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. + # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. + - name: Build and push Docker image for scylla server rust + uses: docker/build-push-action@v5.4.0 + with: + context: ./scylla-server-rust + push: true + platforms: linux/arm64,linux/amd64 + tags: ${{ steps.meta.outputs.tags }}-scylla-server-rust # argos-scylla-server + labels: ${{ steps.meta.outputs.labels }} + # for caching + # cache-from: type=gha + # cache-to: type=gha,mode=max + # https://github.com/docker/build-push-action/issues/820 + provenance: false + + diff --git a/scylla-server-rust/src/main.rs b/scylla-server-rust/src/main.rs index 68acc15d..a2fe2578 100755 --- a/scylla-server-rust/src/main.rs +++ b/scylla-server-rust/src/main.rs @@ -7,45 +7,48 @@ use scylla_server_rust::{ run_controller, system_controller, }, prisma::PrismaClient, - reciever::{db_handler, mqtt_reciever::MqttReciever, socket_handler, ClientData}, + reciever::{db_handler, mqtt_reciever::MqttReciever, ClientData}, Database, }; -use socketioxide::SocketIo; -use tokio::{ - signal, - sync::{broadcast as Broadcaster, mpsc}, -}; +use socketioxide::{extract::SocketRef, SocketIo}; +use tokio::{signal, sync::mpsc}; use tokio_util::{sync::CancellationToken, task::TaskTracker}; use tower::ServiceBuilder; use tower_http::cors::{Any, CorsLayer}; #[tokio::main] async fn main() { + println!("Initializing scylla server..."); + // create the database stuff - let db: Database = Arc::new(PrismaClient::_builder().build().await.expect("Could not build prisma DB")); + let db: Database = Arc::new( + PrismaClient::_builder() + .build() + .await + .expect("Could not build prisma DB"), + ); // create the socket stuff let (socket_layer, io) = SocketIo::new_layer(); + io.ns("/", |s: SocketRef| { + s.on_disconnect(|_: SocketRef| println!("Socket: Client disconnected from socket")) + }); // channel to pass the mqtt data // TODO tune buffer size - let (tx, rx) = Broadcaster::channel::(1000); - let rx2 = tx.subscribe(); + let (tx, rx) = mpsc::channel::(1000); // channel to pass the processed data to the db thread // TODO tune buffer size let (tx_proc, rx_proc) = mpsc::channel::>(1000000); - // spawn the socket handler - tokio::spawn(socket_handler::handle_socket(io, rx)); - // the below two threads need to cancel cleanly to ensure all queued messages are sent. therefore they are part of the a task tracker group. // create a task tracker and cancellation token let task_tracker = TaskTracker::new(); let token = CancellationToken::new(); // spawn the database handler task_tracker.spawn( - db_handler::DbHandler::new(rx2, Arc::clone(&db)).handling_loop(tx_proc, token.clone()), + db_handler::DbHandler::new(rx, Arc::clone(&db)).handling_loop(tx_proc, token.clone()), ); // spawm the database inserter task_tracker.spawn(db_handler::DbHandler::batching_loop( @@ -59,6 +62,7 @@ async fn main() { tx, std::env::var("PROD_SIREN_HOST_URL").unwrap_or("localhost:1883".to_string()), db.clone(), + io, ) .await; tokio::spawn(recv.recieve_mqtt(eloop)); @@ -100,7 +104,9 @@ async fn main() { ) .with_state(db.clone()); - let listener = tokio::net::TcpListener::bind("0.0.0.0:8000").await.expect("Could not bind to 8000!"); + let listener = tokio::net::TcpListener::bind("0.0.0.0:8000") + .await + .expect("Could not bind to 8000!"); let axum_token = token.clone(); tokio::spawn(async { axum::serve(listener, app) @@ -112,6 +118,10 @@ async fn main() { }); task_tracker.close(); + + println!("Initialization complete, ready..."); + println!("Use Ctrl+C or SIGINT to exit cleanly!"); + // listen for ctrl_c, then cancel, close, and await for all tasks in the tracker. Other tasks cancel vai the default tokio system signal::ctrl_c() .await diff --git a/scylla-server-rust/src/reciever/db_handler.rs b/scylla-server-rust/src/reciever/db_handler.rs index cd8e2510..f58a7e0b 100644 --- a/scylla-server-rust/src/reciever/db_handler.rs +++ b/scylla-server-rust/src/reciever/db_handler.rs @@ -2,7 +2,6 @@ use tokio::sync::mpsc::Receiver; use tokio::{sync::mpsc::Sender, time::Duration}; -use tokio::sync::broadcast::Receiver as BroadcastReceiver; use tokio_util::sync::CancellationToken; use crate::{ @@ -79,7 +78,7 @@ pub struct DbHandler { /// The list of data types seen by this instance, used for when to upsert datatype_list: Vec, /// The broadcast channel which provides serial datapoints for processing - reciever: BroadcastReceiver, + reciever: Receiver, /// The database db: Database, /// An internal state of an in progress location packet @@ -91,7 +90,7 @@ pub struct DbHandler { impl DbHandler { /// Make a new db handler /// * `recv` - the broadcast reciver of which clientdata will be sent - pub fn new(reciever: BroadcastReceiver, db: Database) -> DbHandler { + pub fn new(reciever: Receiver, db: Database) -> DbHandler { DbHandler { node_list: vec![], datatype_list: vec![], @@ -150,7 +149,7 @@ impl DbHandler { data_queue.clear(); break; }, - Ok(msg) = self.reciever.recv() => { + Some(msg) = self.reciever.recv() => { // If the time is greater than upload interval, push to batch upload thread and clear queue if tokio::time::Instant::now().duration_since(last_time) diff --git a/scylla-server-rust/src/reciever/mod.rs b/scylla-server-rust/src/reciever/mod.rs index 9ebe8f97..b1713044 100644 --- a/scylla-server-rust/src/reciever/mod.rs +++ b/scylla-server-rust/src/reciever/mod.rs @@ -1,7 +1,6 @@ pub mod db_handler; pub mod mock_reciever; pub mod mqtt_reciever; -pub mod socket_handler; /// Represents the client data /// This has the dual purposes of diff --git a/scylla-server-rust/src/reciever/mqtt_reciever.rs b/scylla-server-rust/src/reciever/mqtt_reciever.rs index 770c9a9d..3bd17415 100644 --- a/scylla-server-rust/src/reciever/mqtt_reciever.rs +++ b/scylla-server-rust/src/reciever/mqtt_reciever.rs @@ -1,13 +1,14 @@ use core::fmt; use std::time::Duration; -use prisma_client_rust::chrono; +use prisma_client_rust::{chrono, serde_json}; use protobuf::Message; use rumqttc::v5::{ mqttbytes::v5::{LastWill, Packet, Publish}, AsyncClient, Event, EventLoop, MqttOptions, }; -use tokio::sync::broadcast::Sender; +use socketioxide::SocketIo; +use tokio::sync::mpsc::Sender; use crate::{serverdata, services::run_service, Database}; @@ -17,13 +18,15 @@ use std::borrow::Cow; pub struct MqttReciever { channel: Sender, curr_run: i32, + io: SocketIo, } impl MqttReciever { - /// Creates a new mqtt reciever - /// * `channel` - The mpsc channel to send the socket.io data to + /// Creates a new mqtt reciever and socketio sender + /// * `channel` - The mpsc channel to send the database data to /// * `mqtt_path` - The mqtt URI, including port, (without the mqtt://) to subscribe to /// * `db` - The database to store the data in + /// * `io` - The socketio layer to send the data to /// /// This is async as it creates the initial run and gets the ID, as well as connecting to and subbing Siren /// Returns the instance and the event loop, which can be passed into the recieve_mqtt func to begin recieiving @@ -31,6 +34,7 @@ impl MqttReciever { channel: Sender, mqtt_path: String, db: Database, + io: SocketIo, ) -> (MqttReciever, EventLoop) { // create the mqtt client and configure it let mut create_opts = MqttOptions::new( @@ -69,6 +73,7 @@ impl MqttReciever { MqttReciever { channel, curr_run: curr_run.id, + io, }, connect, ) @@ -150,8 +155,15 @@ impl MqttReciever { /// Send a message to the channel, printing and IGNORING any error that may occur /// * `client_data` - The cliet data to send over the broadcast async fn send_msg(&self, client_data: ClientData) { - if let Err(err) = self.channel.send(client_data) { + if let Err(err) = self.channel.send(client_data.clone()).await { println!("Error sending through channel: {:?}", err) } + match self.io.emit( + "message", + serde_json::to_string(&client_data).expect("Could not serialize ClientData"), + ) { + Ok(_) => (), + Err(err) => println!("Socket: Broadcast error: {}", err), + } } } diff --git a/scylla-server-rust/src/reciever/socket_handler.rs b/scylla-server-rust/src/reciever/socket_handler.rs deleted file mode 100644 index 6d8b3162..00000000 --- a/scylla-server-rust/src/reciever/socket_handler.rs +++ /dev/null @@ -1,19 +0,0 @@ -use prisma_client_rust::serde_json; -use socketioxide::{extract::SocketRef, SocketIo}; -use tokio::sync::broadcast::Receiver; - -use super::ClientData; - -pub async fn handle_socket(io: SocketIo, mut channel: Receiver) { - io.ns("/", |s: SocketRef| { - s.on_disconnect(|_: SocketRef| println!("Socket: Client disconnected from socket")) - }); - - // await a new message to send to client - while let Ok(cmd) = channel.recv().await { - match io.emit("message", serde_json::to_string(&cmd).expect("Could not serialize ClientData")) { - Ok(_) => (), - Err(err) => println!("Socket: Broadcast error: {}", err), - } - } -} diff --git a/scylla-server-rust/src/services/data_service.rs b/scylla-server-rust/src/services/data_service.rs index b6d3b3da..12bde6ed 100644 --- a/scylla-server-rust/src/services/data_service.rs +++ b/scylla-server-rust/src/services/data_service.rs @@ -72,8 +72,8 @@ pub async fn add_many(db: &Database, client_data: Vec) -> Result Result Date: Sun, 14 Jul 2024 21:59:09 -0400 Subject: [PATCH 28/68] switch to select/include --- .../src/controllers/data_controller.rs | 2 +- .../src/controllers/data_type_controller.rs | 2 +- .../src/controllers/run_controller.rs | 4 +- .../src/services/data_service.rs | 35 +++++----- .../src/services/data_type_service.rs | 26 +++---- .../src/services/driver_service.rs | 22 +++++- .../src/services/location_service.rs | 25 ++++++- .../src/services/node_service.rs | 13 +++- .../src/services/run_service.rs | 68 +++++++------------ .../src/services/system_service.rs | 22 +++++- .../src/transformers/data_transformer.rs | 8 +-- .../src/transformers/data_type_transformer.rs | 15 +++- .../src/transformers/driver_transformer.rs | 14 ++-- .../src/transformers/location_transformer.rs | 14 ++-- .../src/transformers/node_transformer.rs | 7 +- .../src/transformers/run_transformer.rs | 47 ++++++++++++- .../src/transformers/system_transformer.rs | 14 ++-- scylla-server-rust/tests/data_service_test.rs | 4 +- .../tests/data_type_service_test.rs | 10 +-- scylla-server-rust/tests/run_service_test.rs | 6 +- .../tests/system_service_test.rs | 20 +++--- 21 files changed, 222 insertions(+), 156 deletions(-) diff --git a/scylla-server-rust/src/controllers/data_controller.rs b/scylla-server-rust/src/controllers/data_controller.rs index ebc30013..6731246a 100644 --- a/scylla-server-rust/src/controllers/data_controller.rs +++ b/scylla-server-rust/src/controllers/data_controller.rs @@ -12,7 +12,7 @@ pub async fn get_data( State(db): State, Path((data_type_name, run_id)): Path<(String, i32)>, ) -> Result>, ScyllaError> { - let data = data_service::get_data(&db, data_type_name, run_id, true, true).await?; + let data = data_service::get_data(&db, data_type_name, run_id).await?; // map data to frontend data types according to the From func of the client struct let mut transformed_data: Vec = data.iter().map(PublicData::from).collect(); diff --git a/scylla-server-rust/src/controllers/data_type_controller.rs b/scylla-server-rust/src/controllers/data_type_controller.rs index bb725577..d80ed459 100644 --- a/scylla-server-rust/src/controllers/data_type_controller.rs +++ b/scylla-server-rust/src/controllers/data_type_controller.rs @@ -8,7 +8,7 @@ use crate::{ pub async fn get_all_data_types( State(db): State, ) -> Result>, ScyllaError> { - let data_types = data_type_service::get_all_data_types(&db, true).await?; + let data_types = data_type_service::get_all_data_types(&db).await?; let transformed_data_types: Vec = data_types.iter().map(PublicDataType::from).collect(); diff --git a/scylla-server-rust/src/controllers/run_controller.rs b/scylla-server-rust/src/controllers/run_controller.rs index ed01cfa2..4c26eac4 100644 --- a/scylla-server-rust/src/controllers/run_controller.rs +++ b/scylla-server-rust/src/controllers/run_controller.rs @@ -8,7 +8,7 @@ use crate::{ }; pub async fn get_all_runs(State(db): State) -> Result>, ScyllaError> { - let run_data = run_service::get_all_runs(&db, true, true, true).await?; + let run_data = run_service::get_all_runs(&db).await?; let transformed_run_data: Vec = run_data.iter().map(PublicRun::from).collect(); @@ -19,7 +19,7 @@ pub async fn get_run_by_id( State(db): State, Path(run_id): Path, ) -> Result, ScyllaError> { - let run_data = run_service::get_run_by_id(&db, true, true, true, run_id).await?; + let run_data = run_service::get_run_by_id(&db, run_id).await?; if run_data.is_none() { return Err(ScyllaError::NotFound); diff --git a/scylla-server-rust/src/services/data_service.rs b/scylla-server-rust/src/services/data_service.rs index 12bde6ed..b2b43048 100644 --- a/scylla-server-rust/src/services/data_service.rs +++ b/scylla-server-rust/src/services/data_service.rs @@ -2,6 +2,11 @@ use prisma_client_rust::{chrono::DateTime, QueryError}; use crate::{prisma, reciever::ClientData, Database}; +prisma::data::select! {public_data { + time + values +}} + /// Get datapoints that mach criteria /// * `db` - The prisma client to make the call to /// * `data_type_name` - The data type name to filter the data by @@ -13,24 +18,15 @@ pub async fn get_data( db: &Database, data_type_name: String, run_id: i32, - fetch_run: bool, - fetch_data_type: bool, -) -> Result, QueryError> { - let mut find_q = db.data().find_many(vec![ - prisma::data::data_type_name::equals(data_type_name), - prisma::data::run_id::equals(run_id), - ]); - - // add a with statement to fetch runs - - if fetch_run { - find_q = find_q.with(prisma::data::run::fetch()); - } - if fetch_data_type { - find_q = find_q.with(prisma::data::data_type::fetch()); - } - - find_q.exec().await +) -> Result, QueryError> { + db.data() + .find_many(vec![ + prisma::data::data_type_name::equals(data_type_name), + prisma::data::run_id::equals(run_id), + ]) + .select(public_data::select()) + .exec() + .await } /// Adds a datapoint @@ -43,7 +39,7 @@ pub async fn get_data( pub async fn add_data( db: &Database, client_data: ClientData, -) -> Result { +) -> Result { db.data() .create( prisma::data_type::name::equals(client_data.name), @@ -59,6 +55,7 @@ pub async fn add_data( .collect(), )], ) + .select(public_data::select()) .exec() .await } diff --git a/scylla-server-rust/src/services/data_type_service.rs b/scylla-server-rust/src/services/data_type_service.rs index d7b793e1..8e1aabb5 100644 --- a/scylla-server-rust/src/services/data_type_service.rs +++ b/scylla-server-rust/src/services/data_type_service.rs @@ -2,21 +2,20 @@ use prisma_client_rust::QueryError; use crate::{prisma, Database}; +prisma::data_type::select! {public_datatype { + name + unit +}} + /// Gets all datatypes /// * `db` - The prisma client to make the call to -/// * `fetch_node` - Whether to fetch the node associated with the data type /// returns: A result containing the data or the QueryError propogated by the db -pub async fn get_all_data_types( - db: &Database, - fetch_node: bool, -) -> Result, QueryError> { - let mut find_q = db.data_type().find_many(vec![]); - - if fetch_node { - find_q = find_q.with(prisma::data_type::node::fetch()); - } - - find_q.exec().await +pub async fn get_all_data_types(db: &Database) -> Result, QueryError> { + db.data_type() + .find_many(vec![]) + .select(public_datatype::select()) + .exec() + .await } /// Upserts a datatype, either creating or updating one depending on its existence @@ -30,7 +29,7 @@ pub async fn upsert_data_type( data_type_name: String, unit: String, node_name: String, -) -> Result { +) -> Result { db.data_type() .upsert( prisma::data_type::name::equals(data_type_name.clone()), @@ -46,6 +45,7 @@ pub async fn upsert_data_type( prisma::data_type::node::connect(prisma::node::name::equals(node_name.clone())), ], ) + .select(public_datatype::select()) .exec() .await } diff --git a/scylla-server-rust/src/services/driver_service.rs b/scylla-server-rust/src/services/driver_service.rs index 18145aca..247fd179 100644 --- a/scylla-server-rust/src/services/driver_service.rs +++ b/scylla-server-rust/src/services/driver_service.rs @@ -5,11 +5,26 @@ use crate::{ Database, }; +prisma::driver::select! { public_driver{ + username + runs: select { + id + location_name + driver_name + system_name + time + } +}} + /// Gets all drivers /// * `db` - The prisma client to make the call to /// returns: A result containing the data or the QueryError propogated by the db -pub async fn get_all_drivers(db: &Database) -> Result, QueryError> { - db.driver().find_many(vec![]).exec().await +pub async fn get_all_drivers(db: &Database) -> Result, QueryError> { + db.driver() + .find_many(vec![]) + .select(public_driver::select()) + .exec() + .await } /// Upserts a driver, either creating or updating one depending on its existence @@ -21,7 +36,7 @@ pub async fn upsert_driver( db: &Database, driver_name: String, run_id: i32, -) -> Result { +) -> Result { let drive = db .driver() .upsert( @@ -29,6 +44,7 @@ pub async fn upsert_driver( prisma::driver::create(driver_name.clone(), vec![]), vec![], ) + .select(public_driver::select()) .exec() .await?; diff --git a/scylla-server-rust/src/services/location_service.rs b/scylla-server-rust/src/services/location_service.rs index 2e1483d8..04cf5796 100644 --- a/scylla-server-rust/src/services/location_service.rs +++ b/scylla-server-rust/src/services/location_service.rs @@ -5,11 +5,29 @@ use crate::{ Database, }; +prisma::location::select! {public_location{ + name + latitude + longitude + radius + runs: select { + id + location_name + driver_name + system_name + time + } +}} + /// Gets all locations /// * `db` - The prisma client to make the call to /// returns: A result containing the data or the QueryError propogated by the db -pub async fn get_all_locations(db: &Database) -> Result, QueryError> { - db.location().find_many(vec![]).exec().await +pub async fn get_all_locations(db: &Database) -> Result, QueryError> { + db.location() + .find_many(vec![]) + .select(public_location::select()) + .exec() + .await } /// Upserts a location, either creating or updating one depending on its existence @@ -27,7 +45,7 @@ pub async fn upsert_location( longitude: f64, radius: f64, run_id: i32, -) -> Result { +) -> Result { let loc = db .location() .upsert( @@ -39,6 +57,7 @@ pub async fn upsert_location( prisma::location::radius::set(radius), ], ) + .select(public_location::select()) .exec() .await?; diff --git a/scylla-server-rust/src/services/node_service.rs b/scylla-server-rust/src/services/node_service.rs index c359b3fb..4ecd9430 100644 --- a/scylla-server-rust/src/services/node_service.rs +++ b/scylla-server-rust/src/services/node_service.rs @@ -2,13 +2,21 @@ use prisma_client_rust::QueryError; use crate::{prisma, Database}; +prisma::node::include! {public_node{ + data_types: select { + name + unit + } +}} + /// Gets all nodes /// * `db` - The prisma client to make the call to /// returns: A result containing the data or the QueryError propogated by the db -pub async fn get_all_nodes(db: &Database) -> Result, QueryError> { +pub async fn get_all_nodes(db: &Database) -> Result, QueryError> { db.node() .find_many(vec![]) .with(prisma::node::data_types::fetch(vec![])) + .include(public_node::include()) .exec() .await } @@ -20,13 +28,14 @@ pub async fn get_all_nodes(db: &Database) -> Result, Que pub async fn upsert_node( db: &Database, node_name: String, -) -> Result { +) -> Result { db.node() .upsert( prisma::node::name::equals(node_name.clone()), prisma::node::create(node_name, vec![]), vec![], ) + .include(public_node::include()) .exec() .await } diff --git a/scylla-server-rust/src/services/run_service.rs b/scylla-server-rust/src/services/run_service.rs index 2c802116..3e1581fa 100644 --- a/scylla-server-rust/src/services/run_service.rs +++ b/scylla-server-rust/src/services/run_service.rs @@ -7,67 +7,45 @@ use crate::{ Database, }; +prisma::run::select! {public_run{ + id + location_name + driver_name + system_name + time +}} + /// Gets all runs /// * `db` - The prisma client to make the call to -/// * `fetch_loc` - Whether to fetch the location data -/// * `fetch_driver` - Whether to fetch the driver data -/// * `fetch_system` - Whether to fetch the system data /// returns: A result containing the data or the QueryError propogated by the db -pub async fn get_all_runs( - db: &Database, - fetch_loc: bool, - fetch_driver: bool, - fetch_system: bool, -) -> Result, QueryError> { - let mut find_q = db.run().find_many(vec![]); - - if fetch_loc { - find_q = find_q.with(prisma::run::location::fetch()); - } - if fetch_driver { - find_q = find_q.with(prisma::run::driver::fetch()); - } - if fetch_system { - find_q = find_q.with(prisma::run::system::fetch()); - } - - find_q.exec().await +pub async fn get_all_runs(db: &Database) -> Result, QueryError> { + db.run() + .find_many(vec![]) + .select(public_run::select()) + .exec() + .await } /// Gets a single run by its id /// * `db` - The prisma client to make the call to -/// * `fetch_loc` - Whether to fetch the location data -/// * `fetch_driver` - Whether to fetch the driver data -/// * `fetch_system` - Whether to fetch the system data /// * `run_id` - The id of the run to search for /// returns: A result containing the data (or None if the `run_id` was not a valid run) or the QueryError propogated by the db pub async fn get_run_by_id( db: &Database, - fetch_loc: bool, - fetch_driver: bool, - fetch_system: bool, run_id: i32, -) -> Result, QueryError> { - let mut find_q = db.run().find_unique(prisma::run::id::equals(run_id)); - - if fetch_loc { - find_q = find_q.with(prisma::run::location::fetch()); - } - if fetch_driver { - find_q = find_q.with(prisma::run::driver::fetch()); - } - if fetch_system { - find_q = find_q.with(prisma::run::system::fetch()); - } - - find_q.exec().await +) -> Result, QueryError> { + db.run() + .find_unique(prisma::run::id::equals(run_id)) + .select(public_run::select()) + .exec() + .await } /// Creates a run /// * `db` - The prisma client to make the call to /// * `timestamp` - The unix time since epoch in miliseconds when the run starts /// returns: A result containing the data or the QueryError propogated by the db -pub async fn create_run(db: &Database, timestamp: i64) -> Result { +pub async fn create_run(db: &Database, timestamp: i64) -> Result { db.run() .create( DateTime::from_timestamp_millis(timestamp) @@ -75,6 +53,7 @@ pub async fn create_run(db: &Database, timestamp: i64) -> Result Result { +) -> Result { db.run() .create( DateTime::from_timestamp_millis(timestamp) @@ -96,6 +75,7 @@ pub async fn create_run_with_id( .fixed_offset(), vec![prisma::run::id::set(run_id)], ) + .select(public_run::select()) .exec() .await } diff --git a/scylla-server-rust/src/services/system_service.rs b/scylla-server-rust/src/services/system_service.rs index 54f7ca08..a8b18e2c 100644 --- a/scylla-server-rust/src/services/system_service.rs +++ b/scylla-server-rust/src/services/system_service.rs @@ -5,14 +5,29 @@ use crate::{ Database, }; +prisma::system::select! {public_system{ + name + runs: select { + id + location_name + driver_name + system_name + time + } +}} + /// Upserts a datatype, either creating or updating one depending on its existence /// * `db` - The prisma client to make the call to /// * `data_type_name` - The data type name to upsert /// * `unit` - The unit of the data /// * `node_name` - The name of the node linked to the data type, must already exist! /// returns: A result containing the data or the QueryError propogated by the db -pub async fn get_all_systems(db: &Database) -> Result, QueryError> { - db.system().find_many(vec![]).exec().await +pub async fn get_all_systems(db: &Database) -> Result, QueryError> { + db.system() + .find_many(vec![]) + .select(public_system::select()) + .exec() + .await } /// Upserts a system, either creating or updating one depending on its existence @@ -24,7 +39,7 @@ pub async fn upsert_system( db: &Database, system_name: String, run_id: i32, -) -> Result { +) -> Result { let system = db .system() .upsert( @@ -32,6 +47,7 @@ pub async fn upsert_system( prisma::system::create(system_name.clone(), vec![]), vec![], ) + .select(public_system::select()) .exec() .await?; diff --git a/scylla-server-rust/src/transformers/data_transformer.rs b/scylla-server-rust/src/transformers/data_transformer.rs index dc024f06..f59c3722 100644 --- a/scylla-server-rust/src/transformers/data_transformer.rs +++ b/scylla-server-rust/src/transformers/data_transformer.rs @@ -1,6 +1,6 @@ use serde::Serialize; -use crate::{prisma, reciever::ClientData}; +use crate::{reciever::ClientData, services::data_service}; /// The struct defining the data format sent to the client #[derive(Serialize, Debug, PartialEq, Eq, PartialOrd, Ord)] @@ -10,10 +10,10 @@ pub struct PublicData { } /// convert the prisma type to the client type for JSON encoding -impl From<&prisma::data::Data> for PublicData { - fn from(value: &prisma::data::Data) -> Self { +impl From<&data_service::public_data::Data> for PublicData { + fn from(value: &data_service::public_data::Data) -> Self { PublicData { - values: value.values.iter().map(|f| f.to_string()).collect(), + values: value.values.iter().map(f64::to_string).collect(), time: value.time.timestamp_millis(), } } diff --git a/scylla-server-rust/src/transformers/data_type_transformer.rs b/scylla-server-rust/src/transformers/data_type_transformer.rs index 34a92615..7075250e 100644 --- a/scylla-server-rust/src/transformers/data_type_transformer.rs +++ b/scylla-server-rust/src/transformers/data_type_transformer.rs @@ -1,6 +1,6 @@ use serde::Serialize; -use crate::prisma; +use crate::services::{data_type_service, node_service}; /// The struct defining the data type format sent to the client #[derive(Serialize, Debug, PartialEq)] @@ -9,8 +9,17 @@ pub struct PublicDataType { pub unit: String, } -impl From<&prisma::data_type::Data> for PublicDataType { - fn from(value: &prisma::data_type::Data) -> Self { +impl From<&data_type_service::public_datatype::Data> for PublicDataType { + fn from(value: &data_type_service::public_datatype::Data) -> Self { + PublicDataType { + name: value.name.clone(), + unit: value.unit.clone(), + } + } +} + +impl From<&node_service::public_node::data_types::Data> for PublicDataType { + fn from(value: &node_service::public_node::data_types::Data) -> Self { PublicDataType { name: value.name.clone(), unit: value.unit.clone(), diff --git a/scylla-server-rust/src/transformers/driver_transformer.rs b/scylla-server-rust/src/transformers/driver_transformer.rs index b7a0520f..5bba8b4b 100644 --- a/scylla-server-rust/src/transformers/driver_transformer.rs +++ b/scylla-server-rust/src/transformers/driver_transformer.rs @@ -1,6 +1,6 @@ use serde::Serialize; -use crate::prisma; +use crate::services::driver_service; use super::run_transformer::PublicRun; @@ -11,17 +11,11 @@ pub struct PublicDriver { runs: Vec, } -impl From<&prisma::driver::Data> for PublicDriver { - fn from(value: &prisma::driver::Data) -> Self { +impl From<&driver_service::public_driver::Data> for PublicDriver { + fn from(value: &driver_service::public_driver::Data) -> Self { PublicDriver { username: value.username.clone(), - runs: value - .runs - .clone() - .unwrap_or_default() - .iter() - .map(PublicRun::from) - .collect(), + runs: value.runs.clone().iter().map(PublicRun::from).collect(), } } } diff --git a/scylla-server-rust/src/transformers/location_transformer.rs b/scylla-server-rust/src/transformers/location_transformer.rs index c8dedc90..7fef3d21 100644 --- a/scylla-server-rust/src/transformers/location_transformer.rs +++ b/scylla-server-rust/src/transformers/location_transformer.rs @@ -1,6 +1,6 @@ use serde::Serialize; -use crate::prisma; +use crate::services::location_service; use super::run_transformer::PublicRun; @@ -14,20 +14,14 @@ pub struct PublicLocation { pub runs: Vec, } -impl From<&prisma::location::Data> for PublicLocation { - fn from(value: &prisma::location::Data) -> Self { +impl From<&location_service::public_location::Data> for PublicLocation { + fn from(value: &location_service::public_location::Data) -> Self { PublicLocation { name: value.name.clone(), latitude: value.latitude, longitude: value.longitude, radius: value.radius, - runs: value - .runs - .clone() - .unwrap_or_default() - .iter() - .map(PublicRun::from) - .collect(), + runs: value.runs.clone().iter().map(PublicRun::from).collect(), } } } diff --git a/scylla-server-rust/src/transformers/node_transformer.rs b/scylla-server-rust/src/transformers/node_transformer.rs index c8ebbca4..d1551e9a 100644 --- a/scylla-server-rust/src/transformers/node_transformer.rs +++ b/scylla-server-rust/src/transformers/node_transformer.rs @@ -1,6 +1,6 @@ use serde::Serialize; -use crate::prisma; +use crate::services::node_service; use super::data_type_transformer::PublicDataType; @@ -12,14 +12,13 @@ pub struct PublicNode { data_types: Vec, } -impl From<&prisma::node::Data> for PublicNode { - fn from(value: &prisma::node::Data) -> Self { +impl From<&node_service::public_node::Data> for PublicNode { + fn from(value: &node_service::public_node::Data) -> Self { PublicNode { name: value.name.clone(), data_types: value .data_types .clone() - .unwrap_or_default() .iter() .map(PublicDataType::from) .collect(), diff --git a/scylla-server-rust/src/transformers/run_transformer.rs b/scylla-server-rust/src/transformers/run_transformer.rs index 0f2ca2bf..b1e18b6b 100644 --- a/scylla-server-rust/src/transformers/run_transformer.rs +++ b/scylla-server-rust/src/transformers/run_transformer.rs @@ -1,6 +1,9 @@ use serde::Serialize; -use crate::prisma; +use crate::services::{ + driver_service::public_driver, location_service::public_location, run_service::public_run, + system_service::public_system, +}; /// The struct defining the run format sent to the client #[derive(Serialize, Debug, PartialEq)] @@ -15,8 +18,46 @@ pub struct PublicRun { pub time: i64, } -impl From<&prisma::run::Data> for PublicRun { - fn from(value: &prisma::run::Data) -> Self { +impl From<&public_run::Data> for PublicRun { + fn from(value: &public_run::Data) -> Self { + PublicRun { + id: value.id, + location_name: value.location_name.clone().unwrap_or_default(), + driver_name: value.driver_name.clone().unwrap_or_default(), + system_name: value.system_name.clone().unwrap_or_default(), + time: value.time.timestamp_millis(), + } + } +} + +// why are these three needed? basically the nested relations via select do not "share" nested relations elsewhere. +// ultimately this means structs with identical fields have non identical types, and as they are macro generated they cannot be derived together +impl From<&public_driver::runs::Data> for PublicRun { + fn from(value: &public_driver::runs::Data) -> Self { + PublicRun { + id: value.id, + location_name: value.location_name.clone().unwrap_or_default(), + driver_name: value.driver_name.clone().unwrap_or_default(), + system_name: value.system_name.clone().unwrap_or_default(), + time: value.time.timestamp_millis(), + } + } +} + +impl From<&public_location::runs::Data> for PublicRun { + fn from(value: &public_location::runs::Data) -> Self { + PublicRun { + id: value.id, + location_name: value.location_name.clone().unwrap_or_default(), + driver_name: value.driver_name.clone().unwrap_or_default(), + system_name: value.system_name.clone().unwrap_or_default(), + time: value.time.timestamp_millis(), + } + } +} + +impl From<&public_system::runs::Data> for PublicRun { + fn from(value: &public_system::runs::Data) -> Self { PublicRun { id: value.id, location_name: value.location_name.clone().unwrap_or_default(), diff --git a/scylla-server-rust/src/transformers/system_transformer.rs b/scylla-server-rust/src/transformers/system_transformer.rs index 34f76d3f..8985239c 100644 --- a/scylla-server-rust/src/transformers/system_transformer.rs +++ b/scylla-server-rust/src/transformers/system_transformer.rs @@ -1,6 +1,6 @@ use serde::Serialize; -use crate::prisma; +use crate::services::system_service; use super::run_transformer::PublicRun; @@ -11,17 +11,11 @@ pub struct PublicSystem { pub runs: Vec, } -impl From<&prisma::system::Data> for PublicSystem { - fn from(value: &prisma::system::Data) -> Self { +impl From<&system_service::public_system::Data> for PublicSystem { + fn from(value: &system_service::public_system::Data) -> Self { PublicSystem { name: value.name.clone(), - runs: value - .runs - .clone() - .unwrap_or_default() - .iter() - .map(PublicRun::from) - .collect(), + runs: value.runs.clone().iter().map(PublicRun::from).collect(), } } } diff --git a/scylla-server-rust/tests/data_service_test.rs b/scylla-server-rust/tests/data_service_test.rs index ad8272b4..bf3717ea 100644 --- a/scylla-server-rust/tests/data_service_test.rs +++ b/scylla-server-rust/tests/data_service_test.rs @@ -24,7 +24,7 @@ async fn test_data_service() -> Result<(), QueryError> { TEST_KEYWORD.to_owned(), ) .await?; - data_service::get_data(&db, TEST_KEYWORD.to_owned(), 0, true, true).await?; + data_service::get_data(&db, TEST_KEYWORD.to_owned(), 0).await?; Ok(()) } @@ -72,7 +72,7 @@ async fn test_data_fetch_empty() -> Result<(), QueryError> { let db = cleanup_and_prepare().await?; // should be empty, nothing was added to run - let data = data_service::get_data(&db, TEST_KEYWORD.to_owned(), 0, true, true).await?; + let data = data_service::get_data(&db, TEST_KEYWORD.to_owned(), 0).await?; assert!(data.is_empty()); diff --git a/scylla-server-rust/tests/data_type_service_test.rs b/scylla-server-rust/tests/data_type_service_test.rs index 1c3167e3..36c156b0 100644 --- a/scylla-server-rust/tests/data_type_service_test.rs +++ b/scylla-server-rust/tests/data_type_service_test.rs @@ -4,7 +4,10 @@ mod test_utils; use prisma_client_rust::QueryError; use scylla_server_rust::{ prisma, - services::{data_type_service, node_service}, + services::{ + data_type_service::{self, public_datatype}, + node_service, + }, transformers::data_type_transformer::PublicDataType, }; use test_utils::cleanup_and_prepare; @@ -16,9 +19,7 @@ async fn test_get_all_datatypes() -> Result<(), QueryError> { let db = cleanup_and_prepare().await?; // ensure datatypes is empty - assert!(data_type_service::get_all_data_types(&db, true) - .await? - .is_empty()); + assert!(data_type_service::get_all_data_types(&db).await?.is_empty()); Ok(()) } @@ -58,6 +59,7 @@ async fn test_datatype_create() -> Result<(), QueryError> { let data = db .data_type() .find_unique(prisma::data_type::name::equals(data_type_name.clone())) + .select(public_datatype::select()) .exec() .await? .expect("This should not be empty"); diff --git a/scylla-server-rust/tests/run_service_test.rs b/scylla-server-rust/tests/run_service_test.rs index 7f40cc6a..e13bd2c3 100644 --- a/scylla-server-rust/tests/run_service_test.rs +++ b/scylla-server-rust/tests/run_service_test.rs @@ -10,9 +10,7 @@ async fn test_get_all_runs() -> Result<(), QueryError> { let db = cleanup_and_prepare().await?; // ensure runs is empty - assert!(run_service::get_all_runs(&db, true, true, true) - .await? - .is_empty()); + assert!(run_service::get_all_runs(&db).await?.is_empty()); Ok(()) } @@ -25,7 +23,7 @@ async fn test_get_run_by_id() -> Result<(), QueryError> { let run_c = run_service::create_run(&db, 1).await?; // get that run - let run = run_service::get_run_by_id(&db, true, true, true, run_c.id) + let run = run_service::get_run_by_id(&db, run_c.id) .await? .expect("Run should exist was upserted "); diff --git a/scylla-server-rust/tests/system_service_test.rs b/scylla-server-rust/tests/system_service_test.rs index aa43d3c6..82e45d1c 100644 --- a/scylla-server-rust/tests/system_service_test.rs +++ b/scylla-server-rust/tests/system_service_test.rs @@ -1,8 +1,10 @@ use prisma_client_rust::QueryError; use scylla_server_rust::{ prisma, - services::{run_service, system_service}, - transformers::system_transformer::PublicSystem, + services::{ + run_service, + system_service::{self, public_system}, + }, }; use test_utils::cleanup_and_prepare; @@ -15,22 +17,18 @@ const TEST_KEYWORD: &str = "test"; async fn test_upsert_system_create() -> Result<(), QueryError> { let db = cleanup_and_prepare().await?; - let res_c = system_service::upsert_system( - &db, - TEST_KEYWORD.to_owned(), - run_service::create_run(&db, 101).await?.id, - ) - .await?; + let run = run_service::create_run(&db, 101).await?; + + let _ = system_service::upsert_system(&db, TEST_KEYWORD.to_owned(), run.id).await?; - let res = db + let _ = db .system() .find_unique(prisma::system::name::equals(TEST_KEYWORD.to_owned())) + .select(public_system::select()) .exec() .await? .expect("System should exist, was just upserted!"); - assert_eq!(PublicSystem::from(&res_c), PublicSystem::from(&res)); - Ok(()) } From 4b526cede55110dfa46a9f7a00a55f86eb550f46 Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Mon, 15 Jul 2024 00:03:30 -0400 Subject: [PATCH 29/68] add logging and basic performance benchmarking framework --- docker-compose.yml | 1 + scylla-server-rust/Cargo.lock | 37 ++- scylla-server-rust/Cargo.toml | 4 +- scylla-server-rust/src/error.rs | 3 +- scylla-server-rust/src/main.rs | 31 +- scylla-server-rust/src/reciever/db_handler.rs | 288 ++++++++++-------- scylla-server-rust/src/reciever/mod.rs | 1 + .../src/reciever/mqtt_reciever.rs | 12 +- 8 files changed, 241 insertions(+), 136 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 0dc259f5..5b07cfce 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -32,6 +32,7 @@ services: - SOURCE_DATABASE_URL=postgresql://postgres:password@odyssey-timescale:5432/timescaledb - PROD_SIREN_HOST_URL=siren:1883 - PROD_SCYLLA=true + - RUST_LOG=none,scylla_server_rust=INFO # default log setting for docker cpu_shares: 1024 mem_limit: 2gb stop_grace_period: 10s diff --git a/scylla-server-rust/Cargo.lock b/scylla-server-rust/Cargo.lock index 71d7eeaa..480aff31 100755 --- a/scylla-server-rust/Cargo.lock +++ b/scylla-server-rust/Cargo.lock @@ -1425,6 +1425,15 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata 0.1.10", +] + [[package]] name = "matchit" version = "0.7.3" @@ -2615,8 +2624,17 @@ checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" dependencies = [ "aho-corasick 1.1.3", "memchr", - "regex-automata", - "regex-syntax", + "regex-automata 0.4.7", + "regex-syntax 0.8.4", +] + +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax 0.6.29", ] [[package]] @@ -2627,9 +2645,15 @@ checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" dependencies = [ "aho-corasick 1.1.3", "memchr", - "regex-syntax", + "regex-syntax 0.8.4", ] +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + [[package]] name = "regex-syntax" version = "0.8.4" @@ -2897,6 +2921,8 @@ dependencies = [ "tokio-util", "tower", "tower-http", + "tracing", + "tracing-subscriber", ] [[package]] @@ -3424,6 +3450,7 @@ dependencies = [ "signal-hook-registry", "socket2 0.5.7", "tokio-macros", + "tracing", "windows-sys 0.48.0", ] @@ -3668,10 +3695,14 @@ version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" dependencies = [ + "matchers", "nu-ansi-term", + "once_cell", + "regex", "sharded-slab", "smallvec", "thread_local", + "tracing", "tracing-core", "tracing-log 0.2.0", ] diff --git a/scylla-server-rust/Cargo.toml b/scylla-server-rust/Cargo.toml index d0ce6fd5..74bd5f05 100644 --- a/scylla-server-rust/Cargo.toml +++ b/scylla-server-rust/Cargo.toml @@ -8,13 +8,15 @@ prisma-client-rust = {git = "https://github.com/Brendonovich/prisma-client-rust" serde = "1.0.203" protobuf-codegen = "3.3.0" protobuf = "3.3.0" -tokio = { version = "1.38.0", features = ["full"] } +tokio = { version = "1.38.0", features = ["full", "tracing"] } axum = "0.7.5" tower = { version = "0.4.13", features = ["timeout"] } tower-http = { version = "0.5.2", features = ["cors"] } socketioxide = "0.14.0" rumqttc = "0.24.0" tokio-util = { version= "0.7.11", features = ["full"] } +tracing = "0.1.40" +tracing-subscriber = { version = "0.3.18", features = ["ansi", "env-filter"] } [workspace] members = ["prisma-cli"] diff --git a/scylla-server-rust/src/error.rs b/scylla-server-rust/src/error.rs index 734949d3..221b4f13 100644 --- a/scylla-server-rust/src/error.rs +++ b/scylla-server-rust/src/error.rs @@ -6,6 +6,7 @@ use prisma_client_rust::{ prisma_errors::query_engine::{RecordNotFound, UniqueKeyViolation}, QueryError, }; +use tracing::warn; pub enum ScyllaError { PrismaError(QueryError), @@ -14,7 +15,7 @@ pub enum ScyllaError { impl From for ScyllaError { fn from(error: QueryError) -> Self { - println!("Query error: {:?}", error); + warn!("Query error: {:?}", error); match error { e if e.is_prisma_error::() => ScyllaError::NotFound, e => ScyllaError::PrismaError(e), diff --git a/scylla-server-rust/src/main.rs b/scylla-server-rust/src/main.rs index a2fe2578..ac70c8e6 100755 --- a/scylla-server-rust/src/main.rs +++ b/scylla-server-rust/src/main.rs @@ -15,10 +15,28 @@ use tokio::{signal, sync::mpsc}; use tokio_util::{sync::CancellationToken, task::TaskTracker}; use tower::ServiceBuilder; use tower_http::cors::{Any, CorsLayer}; +use tracing::{debug, info, level_filters::LevelFilter}; +use tracing_subscriber::{fmt::format::FmtSpan, EnvFilter}; #[tokio::main] async fn main() { println!("Initializing scylla server..."); + // construct a subscriber that prints formatted traces to stdout + // if RUST_LOG is not set, defaults to loglevel INFO + let subscriber = tracing_subscriber::fmt() + .pretty() + .with_thread_ids(true) + .with_ansi(true) + .with_thread_names(true) + .with_span_events(FmtSpan::CLOSE) + .with_env_filter( + EnvFilter::builder() + .with_default_directive(LevelFilter::INFO.into()) + .from_env_lossy(), + ) + .finish(); + // use that subscriber to process traces emitted after this point + tracing::subscriber::set_global_default(subscriber).expect("Could not init tracing"); // create the database stuff let db: Database = Arc::new( @@ -31,16 +49,16 @@ async fn main() { // create the socket stuff let (socket_layer, io) = SocketIo::new_layer(); io.ns("/", |s: SocketRef| { - s.on_disconnect(|_: SocketRef| println!("Socket: Client disconnected from socket")) + s.on_disconnect(|_: SocketRef| debug!("Socket: Client disconnected from socket")) }); // channel to pass the mqtt data // TODO tune buffer size - let (tx, rx) = mpsc::channel::(1000); + let (tx, rx) = mpsc::channel::(10000); // channel to pass the processed data to the db thread // TODO tune buffer size - let (tx_proc, rx_proc) = mpsc::channel::>(1000000); + let (tx_proc, rx_proc) = mpsc::channel::>(1000); // the below two threads need to cancel cleanly to ensure all queued messages are sent. therefore they are part of the a task tracker group. // create a task tracker and cancellation token @@ -50,7 +68,7 @@ async fn main() { task_tracker.spawn( db_handler::DbHandler::new(rx, Arc::clone(&db)).handling_loop(tx_proc, token.clone()), ); - // spawm the database inserter + // spawn the database inserter task_tracker.spawn(db_handler::DbHandler::batching_loop( rx_proc, Arc::clone(&db), @@ -119,13 +137,14 @@ async fn main() { task_tracker.close(); - println!("Initialization complete, ready..."); - println!("Use Ctrl+C or SIGINT to exit cleanly!"); + info!("Initialization complete, ready..."); + info!("Use Ctrl+C or SIGINT to exit cleanly!"); // listen for ctrl_c, then cancel, close, and await for all tasks in the tracker. Other tasks cancel vai the default tokio system signal::ctrl_c() .await .expect("Could not read cancellation trigger (ctr+c)"); + info!("Recieved exit signal, shutting down!"); token.cancel(); task_tracker.wait().await; } diff --git a/scylla-server-rust/src/reciever/db_handler.rs b/scylla-server-rust/src/reciever/db_handler.rs index f58a7e0b..aea7213c 100644 --- a/scylla-server-rust/src/reciever/db_handler.rs +++ b/scylla-server-rust/src/reciever/db_handler.rs @@ -3,6 +3,7 @@ use tokio::sync::mpsc::Receiver; use tokio::{sync::mpsc::Sender, time::Duration}; use tokio_util::sync::CancellationToken; +use tracing::{debug, info, instrument, trace, warn, Level}; use crate::{ services::{ @@ -85,6 +86,10 @@ pub struct DbHandler { loc_lock: LocLock, /// Whether the location has been modified this loop is_loc: bool, + /// the queue of data + data_queue: Vec, + /// the time since last batch + last_time: tokio::time::Instant, } impl DbHandler { @@ -98,6 +103,8 @@ impl DbHandler { db, loc_lock: LocLock::new(), is_loc: false, + data_queue: vec![], + last_time: tokio::time::Instant::now(), } } @@ -112,23 +119,36 @@ impl DbHandler { loop { tokio::select! { _ = cancel_token.cancelled() => { - let final_msgs = data_queue.recv().await.expect("DB send could not shutdown cleanly!"); - println!( + if let Some(final_msgs) = data_queue.recv().await { + info!( "Final Batch uploaded: {:?}", data_service::add_many(&database, final_msgs).await ); + } else { + info!("No messages to cleanup.") + } break; }, - Some(msg) = data_queue.recv() => { - println!( - "Batch uploaded: {:?}", - data_service::add_many(&database, msg).await + Some(msgs) = data_queue.recv() => { + Self::batch_upload(msgs, &database).await; + trace!( + "DB send: {} of {}", + data_queue.len(), + data_queue.max_capacity() ); } } } } + #[instrument(level = Level::DEBUG)] + async fn batch_upload(msg: Vec, db: &Database) { + info!( + "Batch uploaded: {:?}", + data_service::add_many(db, msg).await + ); + } + /// A loop which uses self and a sender channel to process data /// If the data is special, i.e. coordinates, driver, etc. it will store it in its special location of the db immediately /// For all data points it will add the to the data_channel for batch uploading logic when a certain time has elapsed @@ -139,135 +159,159 @@ impl DbHandler { data_channel: Sender>, cancel_token: CancellationToken, ) { - let mut data_queue: Vec = vec![]; - let mut last_time = tokio::time::Instant::now(); loop { tokio::select! { _ = cancel_token.cancelled() => { - println!("Pushing final messages to queue"); - data_channel.send(data_queue.clone()).await.expect("Could not comm data to db thread, shutdown"); - data_queue.clear(); + debug!("Pushing final messages to queue"); + data_channel.send(self.data_queue.clone()).await.expect("Could not comm data to db thread, shutdown"); + self.data_queue.clear(); break; }, Some(msg) = self.reciever.recv() => { - - // If the time is greater than upload interval, push to batch upload thread and clear queue - if tokio::time::Instant::now().duration_since(last_time) - > Duration::from_millis(UPLOAD_INTERVAL) && !data_queue.is_empty() - { - data_channel.send(data_queue.clone()).await.expect("Could not comm data to db thread"); - data_queue.clear(); - last_time = tokio::time::Instant::now(); + self.handle_msg(msg, &data_channel).await; } + } + } + } - // upsert if not present, a sort of cache of upserted types really - if !self.node_list.contains(&msg.node) { - println!("Upserting node: {}", msg.node); - if let Err(msg) = node_service::upsert_node(&self.db, msg.node.clone()).await { - println!("DB error node upsert: {:?}", msg); - } - self.node_list.push(msg.node.clone()); - } - if !self.datatype_list.contains(&msg.name) { - println!("Upserting data type: {}", msg.name); - if let Err(msg) = data_type_service::upsert_data_type( - &self.db, - msg.name.clone(), - msg.unit.clone(), - msg.node.clone(), - ) - .await { - println!("DB error datatype upsert: {:?}", msg); - } - self.datatype_list.push(msg.name.clone()); - } + #[instrument(skip(self), level = Level::TRACE)] + async fn handle_msg(&mut self, msg: ClientData, data_channel: &Sender>) { + debug!( + "Mqtt dispatcher: {} of {}", + self.reciever.len(), + self.reciever.max_capacity() + ); + + // If the time is greater than upload interval, push to batch upload thread and clear queue + if tokio::time::Instant::now().duration_since(self.last_time) + > Duration::from_millis(UPLOAD_INTERVAL) + && !self.data_queue.is_empty() + { + data_channel + .send(self.data_queue.clone()) + .await + .expect("Could not comm data to db thread"); + self.data_queue.clear(); + self.last_time = tokio::time::Instant::now(); + } + + // upsert if not present, a sort of cache of upserted types really + if !self.node_list.contains(&msg.node) { + info!("Upserting node: {}", msg.node); + if let Err(msg) = node_service::upsert_node(&self.db, msg.node.clone()).await { + warn!("DB error node upsert: {:?}", msg); + } + self.node_list.push(msg.node.clone()); + } + if !self.datatype_list.contains(&msg.name) { + info!("Upserting data type: {}", msg.name); + if let Err(msg) = data_type_service::upsert_data_type( + &self.db, + msg.name.clone(), + msg.unit.clone(), + msg.node.clone(), + ) + .await + { + warn!("DB error datatype upsert: {:?}", msg); + } + self.datatype_list.push(msg.name.clone()); + } - // if data has some special meanings, push them to the database immediately, notably no matter what also enter batching logic - match msg.name.as_str() { - "Driver" => { - if let Err(err) = driver_service::upsert_driver( - &self.db, - msg.values - .first() - .unwrap_or(&"PizzaTheHut".to_string()) - .to_string(), - msg.run_id, - ) - .await { - println!("Driver upsert error: {:?}", err); - } - } - "location" => { - self.loc_lock.add_loc_name( - msg.values - .first() - .unwrap_or(&"PizzaTheHut".to_string()) - .to_string(), - ); - self.is_loc = true; - } - "system" => { - if let Err(err) = system_service::upsert_system( - &self.db, - msg.values - .first() - .unwrap_or(&"PizzaTheHut".to_string()) - .to_string(), - msg.run_id, - ) - .await { - println!("System upsert error: {:?}", err); - } - } - "GPS-Location" => { - self.loc_lock.add_points( - msg.values - .first() - .unwrap_or(&"PizzaTheHut".to_string()) - .parse::() - .unwrap_or_default(), - msg.values - .get(1) - .unwrap_or(&"PizzaTheHut".to_string()) - .parse::() - .unwrap_or_default(), - ); - self.is_loc = true; - } - "Radius" => { - self.loc_lock.add_radius( - msg.values - .first() - .unwrap_or(&"PizzaTheHut".to_string()) - .parse::() - .unwrap_or_default(), - ); - self.is_loc = true; - } - _ => {} + // if data has some special meanings, push them to the database immediately, notably no matter what also enter batching logic + match msg.name.as_str() { + "Driver" => { + debug!("Upserting driver: {:?}", msg.values); + if let Err(err) = driver_service::upsert_driver( + &self.db, + msg.values + .first() + .unwrap_or(&"PizzaTheHut".to_string()) + .to_string(), + msg.run_id, + ) + .await + { + warn!("Driver upsert error: {:?}", err); } - // if location has been modified, push a new location of the loc lock object returns Some - if self.is_loc { - if let Some(loc) = self.loc_lock.finalize() { - if let Err(err) = location_service::upsert_location( - &self.db, - loc.location_name, - loc.lat, - loc.long, - loc.radius, - msg.run_id, - ) - .await { - println!("Location upsert error: {:?}", err); - } - } - self.is_loc = false; + } + "location" => { + debug!("Upserting location name: {:?}", msg.values); + self.loc_lock.add_loc_name( + msg.values + .first() + .unwrap_or(&"PizzaTheHut".to_string()) + .to_string(), + ); + self.is_loc = true; + } + "system" => { + debug!("Upserting system: {:?}", msg.values); + if let Err(err) = system_service::upsert_system( + &self.db, + msg.values + .first() + .unwrap_or(&"PizzaTheHut".to_string()) + .to_string(), + msg.run_id, + ) + .await + { + warn!("System upsert error: {:?}", err); } - - // no matter what, batch upload the message - data_queue.push(msg); + } + "GPS-Location" => { + debug!("Upserting location points: {:?}", msg.values); + self.loc_lock.add_points( + msg.values + .first() + .unwrap_or(&"PizzaTheHut".to_string()) + .parse::() + .unwrap_or_default(), + msg.values + .get(1) + .unwrap_or(&"PizzaTheHut".to_string()) + .parse::() + .unwrap_or_default(), + ); + self.is_loc = true; + } + "Radius" => { + debug!("Upserting location radius: {:?}", msg.values); + self.loc_lock.add_radius( + msg.values + .first() + .unwrap_or(&"PizzaTheHut".to_string()) + .parse::() + .unwrap_or_default(), + ); + self.is_loc = true; + } + _ => {} + } + // if location has been modified, push a new location of the loc lock object returns Some + if self.is_loc { + trace!("Checking location status..."); + if let Some(loc) = self.loc_lock.finalize() { + debug!("Upserting location: {:?}", loc); + if let Err(err) = location_service::upsert_location( + &self.db, + loc.location_name, + loc.lat, + loc.long, + loc.radius, + msg.run_id, + ) + .await + { + warn!("Location upsert error: {:?}", err); } } + self.is_loc = false; } + + // no matter what, batch upload the message + trace!("Pushing msg to queue: {:?}", msg); + self.data_queue.push(msg); } } diff --git a/scylla-server-rust/src/reciever/mod.rs b/scylla-server-rust/src/reciever/mod.rs index b1713044..599a3ed0 100644 --- a/scylla-server-rust/src/reciever/mod.rs +++ b/scylla-server-rust/src/reciever/mod.rs @@ -21,6 +21,7 @@ pub struct ClientData { /// A final location packet /// This has the purpose of representing the struct for the service layer to unpack for insertion, and therefore is not serialized +#[derive(Debug)] struct LocationData { location_name: String, lat: f64, diff --git a/scylla-server-rust/src/reciever/mqtt_reciever.rs b/scylla-server-rust/src/reciever/mqtt_reciever.rs index 3bd17415..0c2fd270 100644 --- a/scylla-server-rust/src/reciever/mqtt_reciever.rs +++ b/scylla-server-rust/src/reciever/mqtt_reciever.rs @@ -9,6 +9,7 @@ use rumqttc::v5::{ }; use socketioxide::SocketIo; use tokio::sync::mpsc::Sender; +use tracing::{debug, instrument, trace, warn, Level}; use crate::{serverdata, services::run_service, Database}; @@ -61,10 +62,12 @@ impl MqttReciever { let curr_run = run_service::create_run(&db, chrono::offset::Utc::now().timestamp_millis()) .await .expect("Could not create initial run!"); + debug!("Configuring current run: {:?}", curr_run); // TODO mess with incoming message cap if db, etc. cannot keep up let (client, connect) = AsyncClient::new(create_opts, 1000); + debug!("Subscribing to siren"); client .try_subscribe("#", rumqttc::v5::mqttbytes::QoS::ExactlyOnce) .expect("Could not subscribe to Siren"); @@ -86,11 +89,12 @@ impl MqttReciever { while let Ok(msg) = connect.poll().await { // safe parse the message if let Event::Incoming(Packet::Publish(msg)) = msg { + trace!("Recieved mqtt message: {:?}", msg); // parse the message into the data and the node name it falls under let item_data = match self.parse_msg(msg).await { Ok(msg) => msg, Err(err) => { - println!("Message parse error: {:?}", err); + warn!("Message parse error: {:?}", err); continue; } }; @@ -103,6 +107,7 @@ impl MqttReciever { /// Parse the message /// * `msg` - The mqtt message to parse /// returns the ClientData, or the Err of something that can be debug printed + #[instrument(skip(self), level = Level::TRACE)] async fn parse_msg(&self, msg: Publish) -> Result { let data = serverdata::ServerData::parse_from_bytes(&msg.payload) .map_err(|f| format!("Could not parse message topic:{:?} error: {}", msg.topic, f))?; @@ -126,6 +131,7 @@ impl MqttReciever { .map(Cow::Borrowed) .find(|f| f.0 == "ts") .unwrap_or_else(|| { + debug!("Could not find timestamp in mqtt, using current time"); Cow::Owned(( "ts".to_string(), chrono::offset::Utc::now().timestamp_millis().to_string(), @@ -156,14 +162,14 @@ impl MqttReciever { /// * `client_data` - The cliet data to send over the broadcast async fn send_msg(&self, client_data: ClientData) { if let Err(err) = self.channel.send(client_data.clone()).await { - println!("Error sending through channel: {:?}", err) + warn!("Error sending through channel: {:?}", err) } match self.io.emit( "message", serde_json::to_string(&client_data).expect("Could not serialize ClientData"), ) { Ok(_) => (), - Err(err) => println!("Socket: Broadcast error: {}", err), + Err(err) => warn!("Socket: Broadcast error: {}", err), } } } From 5fb4c027dd890e070c92e8939a39fcd57f56915c Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Tue, 16 Jul 2024 21:13:45 -0400 Subject: [PATCH 30/68] add the mock ability --- scylla-server-rust/Cargo.lock | 1 + scylla-server-rust/Cargo.toml | 1 + scylla-server-rust/README.md | 28 ++ scylla-server-rust/src/main.rs | 29 +- .../src/reciever/mock_reciever.rs | 251 ++++++++++++++++-- 5 files changed, 278 insertions(+), 32 deletions(-) diff --git a/scylla-server-rust/Cargo.lock b/scylla-server-rust/Cargo.lock index 480aff31..7a5604ed 100755 --- a/scylla-server-rust/Cargo.lock +++ b/scylla-server-rust/Cargo.lock @@ -2914,6 +2914,7 @@ dependencies = [ "prisma-client-rust", "protobuf", "protobuf-codegen", + "rand 0.8.5", "rumqttc", "serde", "socketioxide", diff --git a/scylla-server-rust/Cargo.toml b/scylla-server-rust/Cargo.toml index 74bd5f05..58e24371 100644 --- a/scylla-server-rust/Cargo.toml +++ b/scylla-server-rust/Cargo.toml @@ -17,6 +17,7 @@ rumqttc = "0.24.0" tokio-util = { version= "0.7.11", features = ["full"] } tracing = "0.1.40" tracing-subscriber = { version = "0.3.18", features = ["ansi", "env-filter"] } +rand = "0.8.5" [workspace] members = ["prisma-cli"] diff --git a/scylla-server-rust/README.md b/scylla-server-rust/README.md index cb730e3e..965db72c 100644 --- a/scylla-server-rust/README.md +++ b/scylla-server-rust/README.md @@ -21,6 +21,9 @@ SOURCE_DATABASE_URL=postgresql://postgres:password@127.0.0.1:5432/timescaledb ca ### Test this app + +#### Integration tests + Since this app uses the database for testing, you must follow these steps, or run `./integration_test.sh`: ``` docker volume rm argos_db-data @@ -29,6 +32,31 @@ cargo prisma migrate deploy SOURCE_DATABASE_URL=postgresql://postgres:password@127.0.0.1:5432/timescaledb cargo test -- --test-threads=1 ``` +#### Test it yourself! + +You can send your own messages to the app running in production mode, and test how the client, database, etc. reacts! + +Follow this confluence doc: https://nerdocs.atlassian.net/wiki/spaces/NER/pages/473727054/How+to+run+data+through+Argos+without+the+car + +#### Debug logging + +#### Activate logs +Modify the RUST_LOG env variable. Usually you dont want third party crate logs, so `RUST_LOG=none,scylla_server_rust=trace`. You can replace both none and trace with the levels you want. The levels are: +- none: not a darn thing +- trace: very verbose, could print on every message, which would flood the log especially if run on a server receiving millions of the cars messages +- debug: helpful info not constantly printed in high load situations, good for periodic task info or init/end status checks +- info: user-facing info that is optional, usually to notify of a setting change or whatnot +- warn: info the user should take note of as an error may have occured +- error: a critical byt survivable issue in the application + +#### Develop with logs + +When developing, take advantage of easy logging. Use `debug!`, `trace!` etc macros. with anything you may need, but be sure to abide by the conventions above when making a final PR. + +Have an async function that takes time and is somewhat important for performance? Use tracing::instrument macro on the function definition to capture performance data. + + + ### Deploy this app Use the docker compose above to build & deploy. Note the CI prebuilds arm64 and amd64 images upon request in the actions tab of this repository's github page. diff --git a/scylla-server-rust/src/main.rs b/scylla-server-rust/src/main.rs index ac70c8e6..5a76006f 100755 --- a/scylla-server-rust/src/main.rs +++ b/scylla-server-rust/src/main.rs @@ -7,7 +7,7 @@ use scylla_server_rust::{ run_controller, system_controller, }, prisma::PrismaClient, - reciever::{db_handler, mqtt_reciever::MqttReciever, ClientData}, + reciever::{db_handler, mock_reciever::MockReciever, mqtt_reciever::MqttReciever, ClientData}, Database, }; use socketioxide::{extract::SocketRef, SocketIo}; @@ -75,15 +75,24 @@ async fn main() { token.clone(), )); - // create and spawn the mqtt reciever - let (recv, eloop) = MqttReciever::new( - tx, - std::env::var("PROD_SIREN_HOST_URL").unwrap_or("localhost:1883".to_string()), - db.clone(), - io, - ) - .await; - tokio::spawn(recv.recieve_mqtt(eloop)); + // if PROD_SCYLLA=false + if std::env::var("PROD_SCYLLA").is_ok_and(|f| f == "false") { + info!("Running reciever in mock mode, no data will be stored"); + let recv = MockReciever::new(io); + tokio::spawn(recv.generate_mock()); + } else { + // run prod if this isnt present + // create and spawn the mqtt reciever + info!("Running reciever in MQTT (production) mode"); + let (recv, eloop) = MqttReciever::new( + tx, + std::env::var("PROD_SIREN_HOST_URL").unwrap_or("localhost:1883".to_string()), + db.clone(), + io, + ) + .await; + tokio::spawn(recv.recieve_mqtt(eloop)); + } let app = Router::new() // get all data with the name dataTypeName and runID as specified diff --git a/scylla-server-rust/src/reciever/mock_reciever.rs b/scylla-server-rust/src/reciever/mock_reciever.rs index 59a52baf..b78e3086 100644 --- a/scylla-server-rust/src/reciever/mock_reciever.rs +++ b/scylla-server-rust/src/reciever/mock_reciever.rs @@ -1,22 +1,229 @@ -// use std::time::Duration; - -// use prisma_client_rust::chrono::{self}; -// use tokio::sync::mpsc::Sender; - -// use super::ClientData; - -// pub async fn recieve_mock(channel: Sender) { -// loop { -// tokio::time::sleep(Duration::from_millis(300)).await; -// let a = channel -// .send(ClientData { -// run_id: 0, -// name: "Acceleration".to_owned(), -// unit: "abc".to_owned(), -// values: vec!["1".to_owned()], -// timestamp: chrono::offset::Utc::now().timestamp_millis(), -// node: "ABC".to_string(), -// }) -// .await; -// } -// } +use std::time::Duration; + +use prisma_client_rust::{chrono, serde_json}; +use rand::Rng; +use socketioxide::SocketIo; +use tracing::warn; + +use super::ClientData; + +#[derive(Clone, Copy)] +struct MockData { + name: &'static str, + unit: &'static str, + num_of_vals: u8, + min: f64, + max: f64, +} + +impl MockData { + fn get_values(&self) -> Vec { + let mut val_vec: Vec = vec![]; + // for each point, get a random number in the range + for _ in 0..self.num_of_vals { + val_vec.push( + rand::thread_rng() + .gen_range((self.min)..(self.max)) + .to_string(), + ); + } + + val_vec + } +} + +const BASE_MOCK_DATA: [MockData; 17] = [ + MockData { + name: "Status-Temp_Average", + unit: "C", + num_of_vals: 1, + min: -20.0, + max: 54.0, + }, + MockData { + name: "Temps-Motor_Temperature", + unit: "C", + num_of_vals: 1, + min: -20.0, + max: 54.0, + }, + MockData { + name: "Pack-SOC", + unit: "%", + num_of_vals: 1, + min: 0.0, + max: 100.0, + }, + MockData { + name: "Sense-Accel", + unit: "G", + num_of_vals: 3, + min: -6.0, + max: 6.0, + }, + MockData { + name: "GPS-Location", + unit: "coordinates", + num_of_vals: 2, + min: -90.0, + max: 90.0, + }, + MockData { + name: "Sense-SteeringAngle", + unit: "degrees", + num_of_vals: 1, + min: 0.0, + max: 360.0, + }, + MockData { + name: "Pack-Voltage", + unit: "V", + num_of_vals: 1, + min: 0.0, + max: 5.0, + }, + MockData { + name: "OnBoard-CpuUsage", + unit: "%", + num_of_vals: 1, + min: 0.0, + max: 100.0, + }, + MockData { + name: "OnBoard-CpuTemp", + unit: "C", + num_of_vals: 1, + min: 0.0, + max: 100.0, + }, + MockData { + name: "OnBoard-MemAvailable", + unit: "mb", + num_of_vals: 1, + min: 0.0, + max: 8000.0, + }, + MockData { + name: "HaLow-RSSI", + unit: "dbm", + num_of_vals: 1, + min: -150.0, + max: 80.0, + }, + MockData { + name: "HaLow-StaMCS", + unit: "", + num_of_vals: 1, + min: 0.0, + max: 10.0, + }, + MockData { + name: "Status/MPH", + unit: "mph", + num_of_vals: 1, + min: 0.0, + max: 88.0, + }, + MockData { + name: "Pack-CCL", + unit: "A", + num_of_vals: 1, + min: -35.0, + max: 0.0, + }, + MockData { + name: "Pack-DCL", + unit: "A", + num_of_vals: 1, + min: 0.0, + max: 550.0, + }, + MockData { + name: "Pedals-Brake1", + unit: "", + num_of_vals: 1, + min: 0.0, + max: 3000.0, + }, + MockData { + name: "Power-AC_Current", + unit: "A", + num_of_vals: 1, + min: 0.0, + max: 600.0, + }, +]; + +#[derive(Clone, Copy)] +struct MockStringData { + name: &'static str, + unit: &'static str, + vals: &'static str, +} + +const BASE_MOCK_STRING_DATA: [MockStringData; 2] = [ + MockStringData { + name: "Driver", + unit: "String", + vals: "Fergus", + }, + MockStringData { + name: "Location", + unit: "String", + vals: "Max", + }, +]; + +pub struct MockReciever { + curr_run: i32, + io: SocketIo, +} + +impl MockReciever { + pub fn new(io: SocketIo) -> Self { + MockReciever { curr_run: 1, io } + } + + pub async fn generate_mock(self) { + loop { + // get a random mock datapoint the first 0 to len of number mock data is for the non string and x to len of string mocks is a string mock index. + let index = rand::thread_rng() + .gen_range(0..(BASE_MOCK_DATA.len() + BASE_MOCK_STRING_DATA.len())); + + let client_data: ClientData; + // if we are doing non-string mock this loop + if index < BASE_MOCK_DATA.len() { + let dat = BASE_MOCK_DATA[index]; + + client_data = ClientData { + run_id: self.curr_run, + name: dat.name.to_string(), + unit: dat.unit.to_string(), + values: dat.get_values(), + timestamp: chrono::offset::Utc::now().timestamp_millis(), + node: "".to_string(), // uneeded for socket use only + }; + // do a string mock + } else { + let dat = BASE_MOCK_STRING_DATA[index - BASE_MOCK_DATA.len()]; + client_data = ClientData { + run_id: self.curr_run, + name: dat.name.to_string(), + unit: dat.unit.to_string(), + values: vec![dat.vals.to_string()], + timestamp: chrono::offset::Utc::now().timestamp_millis(), + node: "".to_string(), // uneeded for socket use only + }; + } + + match self.io.emit( + "message", + serde_json::to_string(&client_data).expect("Could not serialize ClientData"), + ) { + Ok(_) => (), + Err(err) => warn!("Socket: Broadcast error: {}", err), + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + } +} From 37b3d9d5a422ff0ff5f15d3f1d50df083e774c55 Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Tue, 16 Jul 2024 21:15:59 -0400 Subject: [PATCH 31/68] clippy fix --- scylla-server-rust/src/reciever/mock_reciever.rs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/scylla-server-rust/src/reciever/mock_reciever.rs b/scylla-server-rust/src/reciever/mock_reciever.rs index b78e3086..4f26a80c 100644 --- a/scylla-server-rust/src/reciever/mock_reciever.rs +++ b/scylla-server-rust/src/reciever/mock_reciever.rs @@ -190,31 +190,30 @@ impl MockReciever { let index = rand::thread_rng() .gen_range(0..(BASE_MOCK_DATA.len() + BASE_MOCK_STRING_DATA.len())); - let client_data: ClientData; // if we are doing non-string mock this loop - if index < BASE_MOCK_DATA.len() { + let client_data: ClientData = if index < BASE_MOCK_DATA.len() { let dat = BASE_MOCK_DATA[index]; - client_data = ClientData { + ClientData { run_id: self.curr_run, name: dat.name.to_string(), unit: dat.unit.to_string(), values: dat.get_values(), timestamp: chrono::offset::Utc::now().timestamp_millis(), node: "".to_string(), // uneeded for socket use only - }; + } // do a string mock } else { let dat = BASE_MOCK_STRING_DATA[index - BASE_MOCK_DATA.len()]; - client_data = ClientData { + ClientData { run_id: self.curr_run, name: dat.name.to_string(), unit: dat.unit.to_string(), values: vec![dat.vals.to_string()], timestamp: chrono::offset::Utc::now().timestamp_millis(), node: "".to_string(), // uneeded for socket use only - }; - } + } + }; match self.io.emit( "message", From 172b9bdaaef5b05640a04ac04307b3e641439151 Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Tue, 16 Jul 2024 21:50:36 -0400 Subject: [PATCH 32/68] use processor instead of receiver, other cleanup --- scylla-server-rust/README.md | 15 ++++--- scylla-server-rust/src/lib.rs | 2 +- scylla-server-rust/src/main.rs | 34 ++++++++-------- .../{reciever => processors}/db_handler.rs | 40 +++++++++---------- .../mock_processor.rs} | 6 +-- .../src/{reciever => processors}/mod.rs | 4 +- .../mqtt_processor.rs} | 16 ++++---- .../src/services/data_service.rs | 2 +- .../src/transformers/data_transformer.rs | 2 +- scylla-server-rust/tests/data_service_test.rs | 2 +- 10 files changed, 63 insertions(+), 60 deletions(-) rename scylla-server-rust/src/{reciever => processors}/db_handler.rs (92%) rename scylla-server-rust/src/{reciever/mock_reciever.rs => processors/mock_processor.rs} (98%) rename scylla-server-rust/src/{reciever => processors}/mod.rs (94%) rename scylla-server-rust/src/{reciever/mqtt_reciever.rs => processors/mqtt_processor.rs} (94%) diff --git a/scylla-server-rust/README.md b/scylla-server-rust/README.md index 965db72c..1ea369de 100644 --- a/scylla-server-rust/README.md +++ b/scylla-server-rust/README.md @@ -59,9 +59,12 @@ Have an async function that takes time and is somewhat important for performance ### Deploy this app -Use the docker compose above to build & deploy. Note the CI prebuilds arm64 and amd64 images upon request in the actions tab of this repository's github page. -``` -docker compose build -docker compose up # use -d to fork to background -``` -A database migration is triggered upon every bootup. +See main README. + + +#### Env variables + +- `SOURCE_DATABASE_URL` The timescale URL +- `PROD_SCYLLA` false=use mock instead of production (mqtt) as source of data +- `RUST_LOG=none,scylla_server_rust` levels of logging for this create, see above +- `PROD_SIREN_HOST_URL` URL:Port of the MQTT server, using when `PROD_SCYLLA=/false` diff --git a/scylla-server-rust/src/lib.rs b/scylla-server-rust/src/lib.rs index fe2033bd..d0b47a4c 100644 --- a/scylla-server-rust/src/lib.rs +++ b/scylla-server-rust/src/lib.rs @@ -1,6 +1,6 @@ pub mod controllers; pub mod error; -pub mod reciever; +pub mod processors; pub mod services; pub mod transformers; diff --git a/scylla-server-rust/src/main.rs b/scylla-server-rust/src/main.rs index 5a76006f..3c7a9244 100755 --- a/scylla-server-rust/src/main.rs +++ b/scylla-server-rust/src/main.rs @@ -7,7 +7,9 @@ use scylla_server_rust::{ run_controller, system_controller, }, prisma::PrismaClient, - reciever::{db_handler, mock_reciever::MockReciever, mqtt_reciever::MqttReciever, ClientData}, + processors::{ + db_handler, mock_processor::MockProcessor, mqtt_processor::MqttProcessor, ClientData, + }, Database, }; use socketioxide::{extract::SocketRef, SocketIo}; @@ -77,43 +79,41 @@ async fn main() { // if PROD_SCYLLA=false if std::env::var("PROD_SCYLLA").is_ok_and(|f| f == "false") { - info!("Running reciever in mock mode, no data will be stored"); - let recv = MockReciever::new(io); + info!("Running processor in mock mode, no data will be stored"); + let recv = MockProcessor::new(io); tokio::spawn(recv.generate_mock()); } else { // run prod if this isnt present - // create and spawn the mqtt reciever - info!("Running reciever in MQTT (production) mode"); - let (recv, eloop) = MqttReciever::new( + // create and spawn the mqtt processor + info!("Running processor in MQTT (production) mode"); + let (recv, eloop) = MqttProcessor::new( tx, std::env::var("PROD_SIREN_HOST_URL").unwrap_or("localhost:1883".to_string()), db.clone(), io, ) .await; - tokio::spawn(recv.recieve_mqtt(eloop)); + tokio::spawn(recv.process_mqtt(eloop)); } let app = Router::new() - // get all data with the name dataTypeName and runID as specified + // DATA ROUTES .route( "/data/:dataTypeName/:runId", get(controllers::data_controller::get_data), ) - // get all datatypes + // DATA TYPE ROUTES .route("/datatypes", get(data_type_controller::get_all_data_types)) - // get all drivers + // DRIVERS .route("/drivers", get(driver_controller::get_all_drivers)) - // get all locations + // LOCATIONS .route("/locations", get(location_controller::get_all_locations)) - // get all nodes + // NODES .route("/nodes", get(node_controller::get_all_nodes)) - // runs: - // get all runs + // RUNS .route("/runs", get(run_controller::get_all_runs)) - // get run with id .route("/runs/:id", get(run_controller::get_run_by_id)) - // get all systems + // SYSTEMS .route("/systems", get(system_controller::get_all_systems)) // for CORS handling .layer( @@ -153,7 +153,7 @@ async fn main() { signal::ctrl_c() .await .expect("Could not read cancellation trigger (ctr+c)"); - info!("Recieved exit signal, shutting down!"); + info!("Received exit signal, shutting down!"); token.cancel(); task_tracker.wait().await; } diff --git a/scylla-server-rust/src/reciever/db_handler.rs b/scylla-server-rust/src/processors/db_handler.rs similarity index 92% rename from scylla-server-rust/src/reciever/db_handler.rs rename to scylla-server-rust/src/processors/db_handler.rs index aea7213c..aeda364a 100644 --- a/scylla-server-rust/src/reciever/db_handler.rs +++ b/scylla-server-rust/src/processors/db_handler.rs @@ -79,13 +79,13 @@ pub struct DbHandler { /// The list of data types seen by this instance, used for when to upsert datatype_list: Vec, /// The broadcast channel which provides serial datapoints for processing - reciever: Receiver, + receiver: Receiver, /// The database db: Database, /// An internal state of an in progress location packet - loc_lock: LocLock, + location_lock: LocLock, /// Whether the location has been modified this loop - is_loc: bool, + is_location: bool, /// the queue of data data_queue: Vec, /// the time since last batch @@ -95,14 +95,14 @@ pub struct DbHandler { impl DbHandler { /// Make a new db handler /// * `recv` - the broadcast reciver of which clientdata will be sent - pub fn new(reciever: Receiver, db: Database) -> DbHandler { + pub fn new(receiver: Receiver, db: Database) -> DbHandler { DbHandler { node_list: vec![], datatype_list: vec![], - reciever, + receiver, db, - loc_lock: LocLock::new(), - is_loc: false, + location_lock: LocLock::new(), + is_location: false, data_queue: vec![], last_time: tokio::time::Instant::now(), } @@ -153,7 +153,7 @@ impl DbHandler { /// If the data is special, i.e. coordinates, driver, etc. it will store it in its special location of the db immediately /// For all data points it will add the to the data_channel for batch uploading logic when a certain time has elapsed /// Before this time the data is stored in an internal queue. - /// On cancellation, the messages currently in the queue will be sent as a final flush of any remaining messages recieved before cancellation + /// On cancellation, the messages currently in the queue will be sent as a final flush of any remaining messages received before cancellation pub async fn handling_loop( mut self, data_channel: Sender>, @@ -167,7 +167,7 @@ impl DbHandler { self.data_queue.clear(); break; }, - Some(msg) = self.reciever.recv() => { + Some(msg) = self.receiver.recv() => { self.handle_msg(msg, &data_channel).await; } } @@ -178,8 +178,8 @@ impl DbHandler { async fn handle_msg(&mut self, msg: ClientData, data_channel: &Sender>) { debug!( "Mqtt dispatcher: {} of {}", - self.reciever.len(), - self.reciever.max_capacity() + self.receiver.len(), + self.receiver.max_capacity() ); // If the time is greater than upload interval, push to batch upload thread and clear queue @@ -237,13 +237,13 @@ impl DbHandler { } "location" => { debug!("Upserting location name: {:?}", msg.values); - self.loc_lock.add_loc_name( + self.location_lock.add_loc_name( msg.values .first() .unwrap_or(&"PizzaTheHut".to_string()) .to_string(), ); - self.is_loc = true; + self.is_location = true; } "system" => { debug!("Upserting system: {:?}", msg.values); @@ -262,7 +262,7 @@ impl DbHandler { } "GPS-Location" => { debug!("Upserting location points: {:?}", msg.values); - self.loc_lock.add_points( + self.location_lock.add_points( msg.values .first() .unwrap_or(&"PizzaTheHut".to_string()) @@ -274,25 +274,25 @@ impl DbHandler { .parse::() .unwrap_or_default(), ); - self.is_loc = true; + self.is_location = true; } "Radius" => { debug!("Upserting location radius: {:?}", msg.values); - self.loc_lock.add_radius( + self.location_lock.add_radius( msg.values .first() .unwrap_or(&"PizzaTheHut".to_string()) .parse::() .unwrap_or_default(), ); - self.is_loc = true; + self.is_location = true; } _ => {} } // if location has been modified, push a new location of the loc lock object returns Some - if self.is_loc { + if self.is_location { trace!("Checking location status..."); - if let Some(loc) = self.loc_lock.finalize() { + if let Some(loc) = self.location_lock.finalize() { debug!("Upserting location: {:?}", loc); if let Err(err) = location_service::upsert_location( &self.db, @@ -307,7 +307,7 @@ impl DbHandler { warn!("Location upsert error: {:?}", err); } } - self.is_loc = false; + self.is_location = false; } // no matter what, batch upload the message diff --git a/scylla-server-rust/src/reciever/mock_reciever.rs b/scylla-server-rust/src/processors/mock_processor.rs similarity index 98% rename from scylla-server-rust/src/reciever/mock_reciever.rs rename to scylla-server-rust/src/processors/mock_processor.rs index 4f26a80c..b9faad54 100644 --- a/scylla-server-rust/src/reciever/mock_reciever.rs +++ b/scylla-server-rust/src/processors/mock_processor.rs @@ -174,14 +174,14 @@ const BASE_MOCK_STRING_DATA: [MockStringData; 2] = [ }, ]; -pub struct MockReciever { +pub struct MockProcessor { curr_run: i32, io: SocketIo, } -impl MockReciever { +impl MockProcessor { pub fn new(io: SocketIo) -> Self { - MockReciever { curr_run: 1, io } + MockProcessor { curr_run: 1, io } } pub async fn generate_mock(self) { diff --git a/scylla-server-rust/src/reciever/mod.rs b/scylla-server-rust/src/processors/mod.rs similarity index 94% rename from scylla-server-rust/src/reciever/mod.rs rename to scylla-server-rust/src/processors/mod.rs index 599a3ed0..eae82b8d 100644 --- a/scylla-server-rust/src/reciever/mod.rs +++ b/scylla-server-rust/src/processors/mod.rs @@ -1,6 +1,6 @@ pub mod db_handler; -pub mod mock_reciever; -pub mod mqtt_reciever; +pub mod mock_processor; +pub mod mqtt_processor; /// Represents the client data /// This has the dual purposes of diff --git a/scylla-server-rust/src/reciever/mqtt_reciever.rs b/scylla-server-rust/src/processors/mqtt_processor.rs similarity index 94% rename from scylla-server-rust/src/reciever/mqtt_reciever.rs rename to scylla-server-rust/src/processors/mqtt_processor.rs index 0c2fd270..d33c0e33 100644 --- a/scylla-server-rust/src/reciever/mqtt_reciever.rs +++ b/scylla-server-rust/src/processors/mqtt_processor.rs @@ -16,27 +16,27 @@ use crate::{serverdata, services::run_service, Database}; use super::ClientData; use std::borrow::Cow; -pub struct MqttReciever { +pub struct MqttProcessor { channel: Sender, curr_run: i32, io: SocketIo, } -impl MqttReciever { - /// Creates a new mqtt reciever and socketio sender +impl MqttProcessor { + /// Creates a new mqtt receiver and socketio and db sender /// * `channel` - The mpsc channel to send the database data to /// * `mqtt_path` - The mqtt URI, including port, (without the mqtt://) to subscribe to /// * `db` - The database to store the data in /// * `io` - The socketio layer to send the data to /// /// This is async as it creates the initial run and gets the ID, as well as connecting to and subbing Siren - /// Returns the instance and the event loop, which can be passed into the recieve_mqtt func to begin recieiving + /// Returns the instance and the event loop, which can be passed into the process_mqtt func to begin recieiving pub async fn new( channel: Sender, mqtt_path: String, db: Database, io: SocketIo, - ) -> (MqttReciever, EventLoop) { + ) -> (MqttProcessor, EventLoop) { // create the mqtt client and configure it let mut create_opts = MqttOptions::new( "ScyllaServer", @@ -73,7 +73,7 @@ impl MqttReciever { .expect("Could not subscribe to Siren"); ( - MqttReciever { + MqttProcessor { channel, curr_run: curr_run.id, io, @@ -84,12 +84,12 @@ impl MqttReciever { /// This handles the reception of mqtt messages, will not return /// * `connect` - The eventloop returned by ::new to connect to - pub async fn recieve_mqtt(self, mut connect: EventLoop) { + pub async fn process_mqtt(self, mut connect: EventLoop) { // process over messages, non blocking while let Ok(msg) = connect.poll().await { // safe parse the message if let Event::Incoming(Packet::Publish(msg)) = msg { - trace!("Recieved mqtt message: {:?}", msg); + trace!("Received mqtt message: {:?}", msg); // parse the message into the data and the node name it falls under let item_data = match self.parse_msg(msg).await { Ok(msg) => msg, diff --git a/scylla-server-rust/src/services/data_service.rs b/scylla-server-rust/src/services/data_service.rs index b2b43048..d3a65be1 100644 --- a/scylla-server-rust/src/services/data_service.rs +++ b/scylla-server-rust/src/services/data_service.rs @@ -1,6 +1,6 @@ use prisma_client_rust::{chrono::DateTime, QueryError}; -use crate::{prisma, reciever::ClientData, Database}; +use crate::{prisma, processors::ClientData, Database}; prisma::data::select! {public_data { time diff --git a/scylla-server-rust/src/transformers/data_transformer.rs b/scylla-server-rust/src/transformers/data_transformer.rs index f59c3722..6fd20b3d 100644 --- a/scylla-server-rust/src/transformers/data_transformer.rs +++ b/scylla-server-rust/src/transformers/data_transformer.rs @@ -1,6 +1,6 @@ use serde::Serialize; -use crate::{reciever::ClientData, services::data_service}; +use crate::{processors::ClientData, services::data_service}; /// The struct defining the data format sent to the client #[derive(Serialize, Debug, PartialEq, Eq, PartialOrd, Ord)] diff --git a/scylla-server-rust/tests/data_service_test.rs b/scylla-server-rust/tests/data_service_test.rs index bf3717ea..327836d6 100644 --- a/scylla-server-rust/tests/data_service_test.rs +++ b/scylla-server-rust/tests/data_service_test.rs @@ -3,7 +3,7 @@ mod test_utils; use prisma_client_rust::QueryError; use scylla_server_rust::{ - reciever::ClientData, + processors::ClientData, services::{data_service, data_type_service, node_service, run_service}, transformers::data_transformer::PublicData, }; From 54cf482ca854e2ddcb57b3917340905a1cf85cb1 Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Tue, 16 Jul 2024 22:29:06 -0400 Subject: [PATCH 33/68] adds viewership data as a concurrent future updated every few seconds --- scylla-server-rust/Cargo.lock | 2 + scylla-server-rust/Cargo.toml | 2 +- .../src/processors/mqtt_processor.rs | 63 ++++++++++++++----- 3 files changed, 49 insertions(+), 18 deletions(-) diff --git a/scylla-server-rust/Cargo.lock b/scylla-server-rust/Cargo.lock index 7a5604ed..4664328a 100755 --- a/scylla-server-rust/Cargo.lock +++ b/scylla-server-rust/Cargo.lock @@ -647,6 +647,7 @@ dependencies = [ "tokio", "tokio-tungstenite", "tower", + "tracing", ] [[package]] @@ -3155,6 +3156,7 @@ dependencies = [ "thiserror", "tokio", "tower", + "tracing", ] [[package]] diff --git a/scylla-server-rust/Cargo.toml b/scylla-server-rust/Cargo.toml index 58e24371..526fcaa7 100644 --- a/scylla-server-rust/Cargo.toml +++ b/scylla-server-rust/Cargo.toml @@ -12,7 +12,7 @@ tokio = { version = "1.38.0", features = ["full", "tracing"] } axum = "0.7.5" tower = { version = "0.4.13", features = ["timeout"] } tower-http = { version = "0.5.2", features = ["cors"] } -socketioxide = "0.14.0" +socketioxide = { version = "0.14.0", features = ["tracing"] } rumqttc = "0.24.0" tokio-util = { version= "0.7.11", features = ["full"] } tracing = "0.1.40" diff --git a/scylla-server-rust/src/processors/mqtt_processor.rs b/scylla-server-rust/src/processors/mqtt_processor.rs index d33c0e33..1441a8a0 100644 --- a/scylla-server-rust/src/processors/mqtt_processor.rs +++ b/scylla-server-rust/src/processors/mqtt_processor.rs @@ -85,22 +85,46 @@ impl MqttProcessor { /// This handles the reception of mqtt messages, will not return /// * `connect` - The eventloop returned by ::new to connect to pub async fn process_mqtt(self, mut connect: EventLoop) { + let mut spec_interval = tokio::time::interval(Duration::from_secs(3)); // process over messages, non blocking - while let Ok(msg) = connect.poll().await { - // safe parse the message - if let Event::Incoming(Packet::Publish(msg)) = msg { - trace!("Received mqtt message: {:?}", msg); - // parse the message into the data and the node name it falls under - let item_data = match self.parse_msg(msg).await { - Ok(msg) => msg, - Err(err) => { - warn!("Message parse error: {:?}", err); - continue; + loop { + tokio::select! { + Ok(msg) = connect.poll() => { + // safe parse the message + if let Event::Incoming(Packet::Publish(msg)) = msg { + trace!("Received mqtt message: {:?}", msg); + // parse the message into the data and the node name it falls under + let msg = match self.parse_msg(msg) { + Ok(msg) => msg, + Err(err) => { + warn!("Message parse error: {:?}", err); + continue; + } + }; + self.send_db_msg(msg.clone()).await; + self.send_socket_msg(msg).await; + } + + }, + _ = spec_interval.tick() => { + trace!("Updating viewership data!"); + if let Ok(sockets) = self.io.sockets() { + let client_data = ClientData { + name: "Viewers".to_string(), + node: "Internal".to_string(), + unit: "".to_string(), + run_id: self.curr_run, + timestamp: chrono::offset::Utc::now().timestamp_millis(), + values: vec![sockets.len().to_string()] + }; + self.send_socket_msg(client_data).await; + + } else { + warn!("Could not fetch socket count"); + } } - }; - // send the message over the channel to the socketio and database consumers - self.send_msg(item_data).await; - } + + } } } @@ -108,7 +132,7 @@ impl MqttProcessor { /// * `msg` - The mqtt message to parse /// returns the ClientData, or the Err of something that can be debug printed #[instrument(skip(self), level = Level::TRACE)] - async fn parse_msg(&self, msg: Publish) -> Result { + fn parse_msg(&self, msg: Publish) -> Result { let data = serverdata::ServerData::parse_from_bytes(&msg.payload) .map_err(|f| format!("Could not parse message topic:{:?} error: {}", msg.topic, f))?; @@ -159,11 +183,16 @@ impl MqttProcessor { } /// Send a message to the channel, printing and IGNORING any error that may occur - /// * `client_data` - The cliet data to send over the broadcast - async fn send_msg(&self, client_data: ClientData) { + /// * `client_data` - The client data to send over the broadcast + async fn send_db_msg(&self, client_data: ClientData) { if let Err(err) = self.channel.send(client_data.clone()).await { warn!("Error sending through channel: {:?}", err) } + } + + /// Sends a message to the socket, printing and IGNORING any error that may occur + /// * `client_data` - The client data to send over the broadcast + async fn send_socket_msg(&self, client_data: ClientData) { match self.io.emit( "message", serde_json::to_string(&client_data).expect("Could not serialize ClientData"), From 468f3275b47e36be31b8b3269859518d8264e843 Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Fri, 19 Jul 2024 16:13:07 -0400 Subject: [PATCH 34/68] fix gitignore --- scylla-server-rust/.gitignore | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 scylla-server-rust/.gitignore diff --git a/scylla-server-rust/.gitignore b/scylla-server-rust/.gitignore new file mode 100644 index 00000000..11d3f83a --- /dev/null +++ b/scylla-server-rust/.gitignore @@ -0,0 +1,24 @@ +# editor things +.idea/ +.zed/ +.vscode/ + +# misc things +.DS_Store +*.nix + +# python things +pyrightconfig.json +__pycache__/ + +# Added by cargo (rust things) +/target +build/ +dist/ +logs/ + +# prisma +prisma.rs + +# protobuf +serverdata.rs From 544bbbe6ca05ad9ae5cc55a5350785e80aa0f611 Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Sat, 20 Jul 2024 14:46:17 -0400 Subject: [PATCH 35/68] switch to profile based compose and update readme --- .gitignore | 5 +++- README.md | 39 +++++++++++++++++-------------- angular-client/compose.client.yml | 17 ++++++++++++++ compose.client-dev.yml | 14 +++++++++++ compose.router.yml | 16 +++++++++++++ compose.scylla-dev.yml | 21 +++++++++++++++++ compose.tpu.yml | 6 +++++ docker-compose.yml => compose.yml | 38 +----------------------------- siren-base/compose.siren.yml | 15 ++++++++++++ 9 files changed, 116 insertions(+), 55 deletions(-) create mode 100644 angular-client/compose.client.yml create mode 100644 compose.client-dev.yml create mode 100644 compose.router.yml create mode 100644 compose.scylla-dev.yml create mode 100644 compose.tpu.yml rename docker-compose.yml => compose.yml (55%) create mode 100644 siren-base/compose.siren.yml diff --git a/.gitignore b/.gitignore index 65cc1e4f..0b475aa1 100644 --- a/.gitignore +++ b/.gitignore @@ -95,4 +95,7 @@ fastlane/test_output # After new code Injection tools there's a generated folder /iOSInjectionProject # https://github.com/johnno1962/injectionforxcode -iOSInjectionProject/ \ No newline at end of file +iOSInjectionProject/ + +# user compose override +compose.override.yml diff --git a/README.md b/README.md index f5996cfa..03d7eb3f 100644 --- a/README.md +++ b/README.md @@ -17,34 +17,39 @@ Once you've sucessfully setup Scylla and the Client, you can either run them sep ## Production -### Running the Project in Prod Mode - -There is a `docker-compose-dev.yml` file for a dev which varies from the router deployment: +The base docker compose (`compose.yml`) contains some important features to note. However, it is useless standalone. Please read the profile customization selection below before using the base compose. - It matches the number of CPUs as the router to roughly simulate router CPU (your CPU is still faster) -- You must build it locally first! -- It does not persist the database between `down` commands +- It persists the database between `down` commands via a volume called `argos_db-data`. Delete it with `docker volume rm argos_db-data` to start with a new database next `up`. +- It weighs the CPU usage of siren higher, so it is prioritized in CPU starvation scenarios. +- It limits memory according to the capacity of the router. + + +### Customizing runtime profiles of the project via docker compose + +This project uses docker compose overrides to secify configurations. Therefore there are multiple "profiles" to choose from when running in production, and there are some profiles for development testing. Also, there are fragment files for siren and client in `siren-base` and `angular-client` respectively, as they are services only used in certain cases. These profiles are specified via the command line on top of the base `compose.yml` file as follows. -Note that both compose files limit memory to the same amount. However, the disk I/O of the router is **much** slower than yours. +``` +docker compose -f compose.yml -f +``` -This will build the docker images that will be run: +Additionally if you need to create your own options, you can create a `compose.override.yml` file in this directory and specify what settings should be changed, which is ignored by git. If you think the override would become useful, document it here and name it `compose..yml`. Here is the current list of overrides, designed so only one is used at any given time: -`docker-compose -f ./docker-compose-dev.yml build` +- `scylla-dev`*: Testing the client and interactions of scylla (scylla prod, siren local, client pt local) +- `client-dev`*: Testing the client development using the scylla mock data mode (scylla mock, client pt local) +- `router`: For production deployment to the base gateway node (scylla prod, siren local, client pt 192.168.100.1) +- `tpu`: Production deployment to the TPU on the car (no client, no siren, scylla pt siren external) -This will start the containers and output all the outputs from both of them to the terminal. If the container is not already an image through docker-compose build, it will attempt to pull the images from docker hub. +***Note that since client settings are changed via rebuild, overrides with a * must be rebuilt via `docker compose -f compose.yml -f compose..yml build client`. Further, a build should be done when reverting to profiles without stars. ** -`docker-compose up` +Examples: -If changes are made to either the client or scylla you will need to rebuild and push to docker hub then pull on the router. Contact the current Argos lead or Chief Software Engineer to get access to the docker hub. +Router deploy and send to background: `docker compose -f compose.yml -f compose.router.yml up -d` -### Running on the Openwrt router -The `docker-compose.yml` file is made for the router. When you push a commit it automatically gets built for the router in 20-30 minutes. -To use a non-standard branch edit the docker-compose.yml file to the name of the tag specified by the name [here](https://github.com/Northeastern-Electric-Racing/Argos/pkgs/container/argos). -Then do `docker compose down` and `docker compose pull` and `docker compose up -d`. -**The database is stored in a volume called `argos_db-data`, delete the volume to start the database fresh!** +## Codegen Protobuf Types (client only) -## Codegen Protobuf Types +Server protobuf generation is automatic. See below for client protobuf generation. ##### Mac diff --git a/angular-client/compose.client.yml b/angular-client/compose.client.yml new file mode 100644 index 00000000..7a3b6636 --- /dev/null +++ b/angular-client/compose.client.yml @@ -0,0 +1,17 @@ +services: + client: + container_name: client + restart: unless-stopped + image: ghcr.io/northeastern-electric-racing/argos:develop-angular-client + build: + context: . + args: + PROD: "true" + BACKEND_URL: http://192.168.100.1:8000 + MAP_ACCESS_TOKEN: pk.eyJ1IjoibWNrZWVwIiwiYSI6ImNscXBrcmU1ZTBscWIya284cDFyYjR3Nm8ifQ.6TQHlxhAJzptZyV-W28dnw + target: production + dockerfile: Dockerfile + ports: + - 80:80 + cpu_shares: 512 + mem_limit: 1gb diff --git a/compose.client-dev.yml b/compose.client-dev.yml new file mode 100644 index 00000000..0030c37c --- /dev/null +++ b/compose.client-dev.yml @@ -0,0 +1,14 @@ +services: + scylla-server-rust: + environment: + - PROD_SIREN_HOST_URL=siren:1883 + - PROD_SCYLLA=false + + client: + extends: + file: ./angular-client/compose.client.yml + service: client + build: + context: ./angular-client + args: + BACKEND_URL: http://localhost:8000 diff --git a/compose.router.yml b/compose.router.yml new file mode 100644 index 00000000..3061c642 --- /dev/null +++ b/compose.router.yml @@ -0,0 +1,16 @@ +services: + scylla-server-rust: + depends_on: + - siren + environment: + - PROD_SIREN_HOST_URL=siren:1883 + + client: + extends: + file: ./angular-client/compose.client.yml + service: client + + siren: + extends: + file: ./siren-base/compose.siren.yml + service: siren diff --git a/compose.scylla-dev.yml b/compose.scylla-dev.yml new file mode 100644 index 00000000..1392f26a --- /dev/null +++ b/compose.scylla-dev.yml @@ -0,0 +1,21 @@ + services: + scylla-server-rust: + depends_on: + - siren + environment: + - PROD_SIREN_HOST_URL=siren:1883 + - RUST_LOG=none,scylla_server_rust=DEBUG + + client: + extends: + file: ./angular-client/compose.client.yml + service: client + build: + context: ./angular-client + args: + BACKEND_URL: http://localhost:8000 + + siren: + extends: + file: ./siren-base/compose.siren.yml + service: siren diff --git a/compose.tpu.yml b/compose.tpu.yml new file mode 100644 index 00000000..d94fe873 --- /dev/null +++ b/compose.tpu.yml @@ -0,0 +1,6 @@ +services: + scylla-server-rust: + environment: + - PROD_SIREN_HOST_URL=host.docker.internal:1883 + extra_hosts: + - "host.docker.internal:host-gateway" # for external siren diff --git a/docker-compose.yml b/compose.yml similarity index 55% rename from docker-compose.yml rename to compose.yml index 5b07cfce..cd64dda3 100644 --- a/docker-compose.yml +++ b/compose.yml @@ -1,5 +1,3 @@ -version: "3.8" - services: odyssey-timescale: container_name: odyssey-timescale @@ -27,10 +25,9 @@ services: - 8000:8000 depends_on: - odyssey-timescale - - siren environment: - SOURCE_DATABASE_URL=postgresql://postgres:password@odyssey-timescale:5432/timescaledb - - PROD_SIREN_HOST_URL=siren:1883 + # - PROD_SIREN_HOST_URL=siren:1883 - PROD_SCYLLA=true - RUST_LOG=none,scylla_server_rust=INFO # default log setting for docker cpu_shares: 1024 @@ -41,39 +38,6 @@ services: entrypoint: ["./docker_run.sh"] - client: - container_name: client - restart: unless-stopped - image: ghcr.io/northeastern-electric-racing/argos:develop-angular-client - build: - context: ./angular-client - args: - PROD: "true" - BACKEND_URL: http://192.168.100.1:8000 - MAP_ACCESS_TOKEN: pk.eyJ1IjoibWNrZWVwIiwiYSI6ImNscXBrcmU1ZTBscWIya284cDFyYjR3Nm8ifQ.6TQHlxhAJzptZyV-W28dnw - target: production - dockerfile: Dockerfile - ports: - - 80:80 - cpu_shares: 512 - mem_limit: 1gb - - siren: - container_name: siren - restart: unless-stopped - image: eclipse-mosquitto:latest - ports: - - 1883:1883 - - 9001:9001 # why? - expose: - - 1883 - volumes: - - ./siren-base/mosquitto/mosquitto.conf:/mosquitto/config/mosquitto.conf - cpu_shares: 2048 - mem_limit: 2gb - oom_kill_disable: true - - volumes: db-data: diff --git a/siren-base/compose.siren.yml b/siren-base/compose.siren.yml new file mode 100644 index 00000000..3fde4fda --- /dev/null +++ b/siren-base/compose.siren.yml @@ -0,0 +1,15 @@ +services: + siren: + container_name: siren + restart: unless-stopped + image: eclipse-mosquitto:latest + ports: + - 1883:1883 + - 9001:9001 # why? + expose: + - 1883 + volumes: + - ./mosquitto/mosquitto.conf:/mosquitto/config/mosquitto.conf + cpu_shares: 2048 + mem_limit: 2gb + oom_kill_disable: true From e6ba7cfe0f4befcef60009b59033cac2f35f7cea Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Sat, 20 Jul 2024 14:50:05 -0400 Subject: [PATCH 36/68] move mock data to own file --- .../src/processors/mock_data.rs | 136 +++++++++++++++ .../src/processors/mock_processor.rs | 160 ++---------------- scylla-server-rust/src/processors/mod.rs | 1 + 3 files changed, 151 insertions(+), 146 deletions(-) create mode 100644 scylla-server-rust/src/processors/mock_data.rs diff --git a/scylla-server-rust/src/processors/mock_data.rs b/scylla-server-rust/src/processors/mock_data.rs new file mode 100644 index 00000000..0ca1c447 --- /dev/null +++ b/scylla-server-rust/src/processors/mock_data.rs @@ -0,0 +1,136 @@ +use super::mock_processor::{MockData, MockStringData}; + +pub const BASE_MOCK_DATA: [MockData; 17] = [ + MockData { + name: "Status-Temp_Average", + unit: "C", + num_of_vals: 1, + min: -20.0, + max: 54.0, + }, + MockData { + name: "Temps-Motor_Temperature", + unit: "C", + num_of_vals: 1, + min: -20.0, + max: 54.0, + }, + MockData { + name: "Pack-SOC", + unit: "%", + num_of_vals: 1, + min: 0.0, + max: 100.0, + }, + MockData { + name: "Sense-Accel", + unit: "G", + num_of_vals: 3, + min: -6.0, + max: 6.0, + }, + MockData { + name: "GPS-Location", + unit: "coordinates", + num_of_vals: 2, + min: -90.0, + max: 90.0, + }, + MockData { + name: "Sense-SteeringAngle", + unit: "degrees", + num_of_vals: 1, + min: 0.0, + max: 360.0, + }, + MockData { + name: "Pack-Voltage", + unit: "V", + num_of_vals: 1, + min: 0.0, + max: 5.0, + }, + MockData { + name: "OnBoard-CpuUsage", + unit: "%", + num_of_vals: 1, + min: 0.0, + max: 100.0, + }, + MockData { + name: "OnBoard-CpuTemp", + unit: "C", + num_of_vals: 1, + min: 0.0, + max: 100.0, + }, + MockData { + name: "OnBoard-MemAvailable", + unit: "mb", + num_of_vals: 1, + min: 0.0, + max: 8000.0, + }, + MockData { + name: "HaLow-RSSI", + unit: "dbm", + num_of_vals: 1, + min: -150.0, + max: 80.0, + }, + MockData { + name: "HaLow-StaMCS", + unit: "", + num_of_vals: 1, + min: 0.0, + max: 10.0, + }, + MockData { + name: "Status/MPH", + unit: "mph", + num_of_vals: 1, + min: 0.0, + max: 88.0, + }, + MockData { + name: "Pack-CCL", + unit: "A", + num_of_vals: 1, + min: -35.0, + max: 0.0, + }, + MockData { + name: "Pack-DCL", + unit: "A", + num_of_vals: 1, + min: 0.0, + max: 550.0, + }, + MockData { + name: "Pedals-Brake1", + unit: "", + num_of_vals: 1, + min: 0.0, + max: 3000.0, + }, + MockData { + name: "Power-AC_Current", + unit: "A", + num_of_vals: 1, + min: 0.0, + max: 600.0, + }, +]; + +pub const BASE_MOCK_STRING_DATA: [MockStringData; 2] = [ + MockStringData { + name: "Driver", + unit: "String", + vals: "Fergus", + }, + MockStringData { + name: "Location", + unit: "String", + vals: "Max", + }, +]; diff --git a/scylla-server-rust/src/processors/mock_processor.rs b/scylla-server-rust/src/processors/mock_processor.rs index b9faad54..c6f8b789 100644 --- a/scylla-server-rust/src/processors/mock_processor.rs +++ b/scylla-server-rust/src/processors/mock_processor.rs @@ -5,15 +5,18 @@ use rand::Rng; use socketioxide::SocketIo; use tracing::warn; -use super::ClientData; +use super::{ + mock_data::{BASE_MOCK_DATA, BASE_MOCK_STRING_DATA}, + ClientData, +}; #[derive(Clone, Copy)] -struct MockData { - name: &'static str, - unit: &'static str, - num_of_vals: u8, - min: f64, - max: f64, +pub struct MockData { + pub name: &'static str, + pub unit: &'static str, + pub num_of_vals: u8, + pub min: f64, + pub max: f64, } impl MockData { @@ -32,148 +35,13 @@ impl MockData { } } -const BASE_MOCK_DATA: [MockData; 17] = [ - MockData { - name: "Status-Temp_Average", - unit: "C", - num_of_vals: 1, - min: -20.0, - max: 54.0, - }, - MockData { - name: "Temps-Motor_Temperature", - unit: "C", - num_of_vals: 1, - min: -20.0, - max: 54.0, - }, - MockData { - name: "Pack-SOC", - unit: "%", - num_of_vals: 1, - min: 0.0, - max: 100.0, - }, - MockData { - name: "Sense-Accel", - unit: "G", - num_of_vals: 3, - min: -6.0, - max: 6.0, - }, - MockData { - name: "GPS-Location", - unit: "coordinates", - num_of_vals: 2, - min: -90.0, - max: 90.0, - }, - MockData { - name: "Sense-SteeringAngle", - unit: "degrees", - num_of_vals: 1, - min: 0.0, - max: 360.0, - }, - MockData { - name: "Pack-Voltage", - unit: "V", - num_of_vals: 1, - min: 0.0, - max: 5.0, - }, - MockData { - name: "OnBoard-CpuUsage", - unit: "%", - num_of_vals: 1, - min: 0.0, - max: 100.0, - }, - MockData { - name: "OnBoard-CpuTemp", - unit: "C", - num_of_vals: 1, - min: 0.0, - max: 100.0, - }, - MockData { - name: "OnBoard-MemAvailable", - unit: "mb", - num_of_vals: 1, - min: 0.0, - max: 8000.0, - }, - MockData { - name: "HaLow-RSSI", - unit: "dbm", - num_of_vals: 1, - min: -150.0, - max: 80.0, - }, - MockData { - name: "HaLow-StaMCS", - unit: "", - num_of_vals: 1, - min: 0.0, - max: 10.0, - }, - MockData { - name: "Status/MPH", - unit: "mph", - num_of_vals: 1, - min: 0.0, - max: 88.0, - }, - MockData { - name: "Pack-CCL", - unit: "A", - num_of_vals: 1, - min: -35.0, - max: 0.0, - }, - MockData { - name: "Pack-DCL", - unit: "A", - num_of_vals: 1, - min: 0.0, - max: 550.0, - }, - MockData { - name: "Pedals-Brake1", - unit: "", - num_of_vals: 1, - min: 0.0, - max: 3000.0, - }, - MockData { - name: "Power-AC_Current", - unit: "A", - num_of_vals: 1, - min: 0.0, - max: 600.0, - }, -]; - #[derive(Clone, Copy)] -struct MockStringData { - name: &'static str, - unit: &'static str, - vals: &'static str, +pub struct MockStringData { + pub name: &'static str, + pub unit: &'static str, + pub vals: &'static str, } -const BASE_MOCK_STRING_DATA: [MockStringData; 2] = [ - MockStringData { - name: "Driver", - unit: "String", - vals: "Fergus", - }, - MockStringData { - name: "Location", - unit: "String", - vals: "Max", - }, -]; - pub struct MockProcessor { curr_run: i32, io: SocketIo, diff --git a/scylla-server-rust/src/processors/mod.rs b/scylla-server-rust/src/processors/mod.rs index eae82b8d..e28de9c8 100644 --- a/scylla-server-rust/src/processors/mod.rs +++ b/scylla-server-rust/src/processors/mod.rs @@ -1,4 +1,5 @@ pub mod db_handler; +mod mock_data; pub mod mock_processor; pub mod mqtt_processor; From 7007d6e66f014460e998dbc395a22667a2168658 Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Sat, 20 Jul 2024 15:28:05 -0400 Subject: [PATCH 37/68] document push, also rename variables --- README.md | 12 ++++++++++++ scylla-server-rust/src/main.rs | 11 ++++++----- scylla-server-rust/src/processors/mqtt_processor.rs | 2 +- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 03d7eb3f..759253c5 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,18 @@ Examples: Router deploy and send to background: `docker compose -f compose.yml -f compose.router.yml up -d` +### Build and deploy + +Using the above profiles, one can `build` the app. Then, with correct permissions, you can `push` the app then `pull` it elsewhere for usage. Note that you must `push` and `pull` on the same architecture, so you cannot use for example a dell laptop to build for the router! To get `push` permissions, create a PAT [here](https://github.com/settings/tokens/new?scopes=write:packages) and copy the token into this command: + +``` +sudo docker login ghcr.io -u -p +``` + +Now you can update the image on a remote server. Note to save time you can just specify which service to upload, like `scylla-server-rust` or `client`. +``` +sudo docker compose -f compose.yml -f compose.router.yml build && sudo docker compose -f compose.yml -f compose.router.yml push +``` ## Codegen Protobuf Types (client only) diff --git a/scylla-server-rust/src/main.rs b/scylla-server-rust/src/main.rs index 3c7a9244..b4e7756e 100755 --- a/scylla-server-rust/src/main.rs +++ b/scylla-server-rust/src/main.rs @@ -56,11 +56,11 @@ async fn main() { // channel to pass the mqtt data // TODO tune buffer size - let (tx, rx) = mpsc::channel::(10000); + let (mqtt_receive, mqtt_send) = mpsc::channel::(10000); // channel to pass the processed data to the db thread // TODO tune buffer size - let (tx_proc, rx_proc) = mpsc::channel::>(1000); + let (db_receive, db_send) = mpsc::channel::>(1000); // the below two threads need to cancel cleanly to ensure all queued messages are sent. therefore they are part of the a task tracker group. // create a task tracker and cancellation token @@ -68,11 +68,12 @@ async fn main() { let token = CancellationToken::new(); // spawn the database handler task_tracker.spawn( - db_handler::DbHandler::new(rx, Arc::clone(&db)).handling_loop(tx_proc, token.clone()), + db_handler::DbHandler::new(mqtt_send, Arc::clone(&db)) + .handling_loop(db_receive, token.clone()), ); // spawn the database inserter task_tracker.spawn(db_handler::DbHandler::batching_loop( - rx_proc, + db_send, Arc::clone(&db), token.clone(), )); @@ -87,7 +88,7 @@ async fn main() { // create and spawn the mqtt processor info!("Running processor in MQTT (production) mode"); let (recv, eloop) = MqttProcessor::new( - tx, + mqtt_receive, std::env::var("PROD_SIREN_HOST_URL").unwrap_or("localhost:1883".to_string()), db.clone(), io, diff --git a/scylla-server-rust/src/processors/mqtt_processor.rs b/scylla-server-rust/src/processors/mqtt_processor.rs index 1441a8a0..093743c6 100644 --- a/scylla-server-rust/src/processors/mqtt_processor.rs +++ b/scylla-server-rust/src/processors/mqtt_processor.rs @@ -163,7 +163,7 @@ impl MqttProcessor { }) .into_owned(); - // parse time, if time isnt present use sys time (see above) + // parse time, if invalid time error out let Ok(time_clean) = unix_time.1.parse::() else { return Err(format!("Invalid timestamp: {}", unix_time.1)); }; From 1cec94eac420a65268753e7ee9bf25da1c8091bd Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Sat, 20 Jul 2024 15:35:38 -0400 Subject: [PATCH 38/68] attempt argos split --- .github/workflows/scylla-rust-build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/scylla-rust-build.yml b/.github/workflows/scylla-rust-build.yml index 0decd961..1d5731ec 100644 --- a/.github/workflows/scylla-rust-build.yml +++ b/.github/workflows/scylla-rust-build.yml @@ -37,7 +37,7 @@ jobs: id: meta uses: docker/metadata-action@v5.5.1 with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-scylla # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. @@ -47,7 +47,7 @@ jobs: context: ./scylla-server-rust push: true platforms: linux/arm64,linux/amd64 - tags: ${{ steps.meta.outputs.tags }}-scylla-server-rust # argos-scylla-server + tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} # for caching # cache-from: type=gha From 1454dcb70b1293e6b6784c200480284a8dc87732 Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Sat, 20 Jul 2024 15:36:54 -0400 Subject: [PATCH 39/68] change name for testing --- .github/workflows/{scylla-rust-build.yml => build-image.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{scylla-rust-build.yml => build-image.yml} (100%) diff --git a/.github/workflows/scylla-rust-build.yml b/.github/workflows/build-image.yml similarity index 100% rename from .github/workflows/scylla-rust-build.yml rename to .github/workflows/build-image.yml From 582ed83e5dc0de5fccf61be28d97194eb1937083 Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Sat, 20 Jul 2024 17:33:07 -0400 Subject: [PATCH 40/68] attempt a macos build --- .github/workflows/build-image.yml | 8 +++++--- compose.yml | 2 +- scylla-server-rust/.cargo/config.toml | 2 +- scylla-server-rust/.dockerignore | 6 ++++++ scylla-server-rust/Cargo.toml | 1 + scylla-server-rust/Dockerfile | 2 +- scylla-server-rust/prisma-cli/Cargo.toml | 5 ++++- 7 files changed, 19 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index 1d5731ec..52ff1600 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -13,8 +13,11 @@ env: # There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. jobs: - build-and-push-image: - runs-on: ubuntu-latest + build-and-push-image_matrix: + strategy: + matrix: + os: [macos-latest, ubuntu-latest] + runs-on: ${{ matrix.os }} # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. permissions: contents: read @@ -46,7 +49,6 @@ jobs: with: context: ./scylla-server-rust push: true - platforms: linux/arm64,linux/amd64 tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} # for caching diff --git a/compose.yml b/compose.yml index cd64dda3..49665376 100644 --- a/compose.yml +++ b/compose.yml @@ -17,7 +17,7 @@ services: scylla-server-rust: container_name: scylla-server-rust - image: ghcr.io/northeastern-electric-racing/argos:rust-rewrite-socket-scylla-server-rust + image: ghcr.io/northeastern-electric-racing/argos-scylla:rust-rewrite-socket build: context: ./scylla-server-rust restart: unless-stopped diff --git a/scylla-server-rust/.cargo/config.toml b/scylla-server-rust/.cargo/config.toml index eea34c5d..ba1f1a45 100644 --- a/scylla-server-rust/.cargo/config.toml +++ b/scylla-server-rust/.cargo/config.toml @@ -1,2 +1,2 @@ [alias] -prisma = "run --package prisma-cli --" \ No newline at end of file +prisma = "run --release --package prisma-cli --" \ No newline at end of file diff --git a/scylla-server-rust/.dockerignore b/scylla-server-rust/.dockerignore index 5b90b998..690dff8c 100644 --- a/scylla-server-rust/.dockerignore +++ b/scylla-server-rust/.dockerignore @@ -23,3 +23,9 @@ prisma.rs # protobuf serverdata.rs + +target +Dockerfile +.dockerignore +.git +.gitignore diff --git a/scylla-server-rust/Cargo.toml b/scylla-server-rust/Cargo.toml index 526fcaa7..b0225476 100644 --- a/scylla-server-rust/Cargo.toml +++ b/scylla-server-rust/Cargo.toml @@ -29,3 +29,4 @@ protobuf-codegen = "3.3.0" lto = true codegen-units = 1 panic = "abort" +strip = true # Automatically strip symbols from the binary. diff --git a/scylla-server-rust/Dockerfile b/scylla-server-rust/Dockerfile index e4cee263..19c57654 100755 --- a/scylla-server-rust/Dockerfile +++ b/scylla-server-rust/Dockerfile @@ -5,4 +5,4 @@ WORKDIR /usr/src/myapp COPY . . RUN cargo prisma generate RUN cargo build --release --locked -ENTRYPOINT ["./docker_run.sh"] \ No newline at end of file +ENTRYPOINT ["./docker_run.sh"] diff --git a/scylla-server-rust/prisma-cli/Cargo.toml b/scylla-server-rust/prisma-cli/Cargo.toml index 31208305..a477c6cf 100755 --- a/scylla-server-rust/prisma-cli/Cargo.toml +++ b/scylla-server-rust/prisma-cli/Cargo.toml @@ -6,4 +6,7 @@ edition = "2021" [dependencies] prisma-client-rust-cli = { git = "https://github.com/Brendonovich/prisma-client-rust", tag = "0.6.11", features = [ "postgresql", "migrations" -], default-features = false } \ No newline at end of file +], default-features = false } + +[profile.release] +strip = true # Automatically strip symbols from the binary. \ No newline at end of file From fa469dcf9df3d455a22cedd1d4dc3472386b06e3 Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Sat, 20 Jul 2024 17:44:56 -0400 Subject: [PATCH 41/68] fix client, remove macos build as license wrong --- .../workflows/{build-client.yml => client-build.yml} | 6 +++--- .../{build-image.yml => scylla-server-rust-build.yml} | 11 ++++------- angular-client/compose.client.yml | 2 +- 3 files changed, 8 insertions(+), 11 deletions(-) rename .github/workflows/{build-client.yml => client-build.yml} (94%) rename .github/workflows/{build-image.yml => scylla-server-rust-build.yml} (93%) diff --git a/.github/workflows/build-client.yml b/.github/workflows/client-build.yml similarity index 94% rename from .github/workflows/build-client.yml rename to .github/workflows/client-build.yml index 592a5d62..cc5fa273 100644 --- a/.github/workflows/build-client.yml +++ b/.github/workflows/client-build.yml @@ -1,5 +1,5 @@ # -name: Create and publish a Docker image +name: Create and publish client docker image on: workflow_dispatch: @@ -37,7 +37,7 @@ jobs: id: meta uses: docker/metadata-action@v5.5.1 with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-client # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. @@ -48,7 +48,7 @@ jobs: push: true target: production platforms: linux/arm64,linux/amd64 - tags: ${{ steps.meta.outputs.tags }}-angular-client # argos-angular-client + tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} # for caching # cache-from: type=gha,scope=global diff --git a/.github/workflows/build-image.yml b/.github/workflows/scylla-server-rust-build.yml similarity index 93% rename from .github/workflows/build-image.yml rename to .github/workflows/scylla-server-rust-build.yml index 52ff1600..42bdb18b 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/scylla-server-rust-build.yml @@ -1,5 +1,5 @@ # -name: Create and publish a Docker image +name: Create and publish scylla docker image on: workflow_dispatch: @@ -13,11 +13,8 @@ env: # There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. jobs: - build-and-push-image_matrix: - strategy: - matrix: - os: [macos-latest, ubuntu-latest] - runs-on: ${{ matrix.os }} + build-and-push-image: + runs-on: ubuntu-latest # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. permissions: contents: read @@ -49,6 +46,7 @@ jobs: with: context: ./scylla-server-rust push: true + platforms: linux/arm64,linux/amd64 tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} # for caching @@ -57,4 +55,3 @@ jobs: # https://github.com/docker/build-push-action/issues/820 provenance: false - diff --git a/angular-client/compose.client.yml b/angular-client/compose.client.yml index 7a3b6636..472f12e5 100644 --- a/angular-client/compose.client.yml +++ b/angular-client/compose.client.yml @@ -2,7 +2,7 @@ services: client: container_name: client restart: unless-stopped - image: ghcr.io/northeastern-electric-racing/argos:develop-angular-client + image: ghcr.io/northeastern-electric-racing/argos-client:rust-rewrite-socket build: context: . args: From 5061c8fa7375cd18d1dbca253df9c7181ac7e6c5 Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Sat, 20 Jul 2024 20:57:01 -0400 Subject: [PATCH 42/68] misc fixes, reconnect logic fix in siren and timestamp logic fix --- scylla-server-rust/Cargo.lock | 25 ++- scylla-server-rust/Cargo.toml | 2 +- scylla-server-rust/prisma-cli/Cargo.toml | 5 +- scylla-server-rust/src/main.rs | 4 +- .../src/processors/db_handler.rs | 4 +- .../src/processors/mqtt_processor.rs | 154 ++++++++++-------- siren-base/mosquitto/mosquitto.conf | 14 +- 7 files changed, 121 insertions(+), 87 deletions(-) diff --git a/scylla-server-rust/Cargo.lock b/scylla-server-rust/Cargo.lock index 4664328a..b3a84563 100755 --- a/scylla-server-rust/Cargo.lock +++ b/scylla-server-rust/Cargo.lock @@ -726,6 +726,12 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86d4de0081402f5e88cdac65c8dcdcc73118c1a7a465e2a05f0da05843a8ea33" +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + [[package]] name = "flate2" version = "1.0.30" @@ -1996,7 +2002,7 @@ version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c3659d1ee90221741f65dd128d9998311b0e40c5d3c23a62445938214abce4f" dependencies = [ - "fixedbitset", + "fixedbitset 0.1.9", "ordermap", ] @@ -2750,10 +2756,10 @@ dependencies = [ [[package]] name = "rumqttc" version = "0.24.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1568e15fab2d546f940ed3a21f48bbbd1c494c90c99c4481339364a497f94a9" +source = "git+https://github.com/bytebeamio/rumqtt?branch=main#db1f261dd5cd6c69bbfd1058ba69ea8ef5f4fc38" dependencies = [ "bytes", + "fixedbitset 0.5.7", "flume", "futures-util", "log", @@ -2763,6 +2769,8 @@ dependencies = [ "thiserror", "tokio", "tokio-rustls", + "tokio-stream", + "tokio-util", ] [[package]] @@ -3521,6 +3529,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-stream" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-tungstenite" version = "0.23.1" diff --git a/scylla-server-rust/Cargo.toml b/scylla-server-rust/Cargo.toml index b0225476..e223e7d9 100644 --- a/scylla-server-rust/Cargo.toml +++ b/scylla-server-rust/Cargo.toml @@ -13,7 +13,7 @@ axum = "0.7.5" tower = { version = "0.4.13", features = ["timeout"] } tower-http = { version = "0.5.2", features = ["cors"] } socketioxide = { version = "0.14.0", features = ["tracing"] } -rumqttc = "0.24.0" +rumqttc = { git = "https://github.com/bytebeamio/rumqtt", branch = "main"} tokio-util = { version= "0.7.11", features = ["full"] } tracing = "0.1.40" tracing-subscriber = { version = "0.3.18", features = ["ansi", "env-filter"] } diff --git a/scylla-server-rust/prisma-cli/Cargo.toml b/scylla-server-rust/prisma-cli/Cargo.toml index a477c6cf..31208305 100755 --- a/scylla-server-rust/prisma-cli/Cargo.toml +++ b/scylla-server-rust/prisma-cli/Cargo.toml @@ -6,7 +6,4 @@ edition = "2021" [dependencies] prisma-client-rust-cli = { git = "https://github.com/Brendonovich/prisma-client-rust", tag = "0.6.11", features = [ "postgresql", "migrations" -], default-features = false } - -[profile.release] -strip = true # Automatically strip symbols from the binary. \ No newline at end of file +], default-features = false } \ No newline at end of file diff --git a/scylla-server-rust/src/main.rs b/scylla-server-rust/src/main.rs index b4e7756e..7169afd7 100755 --- a/scylla-server-rust/src/main.rs +++ b/scylla-server-rust/src/main.rs @@ -87,14 +87,14 @@ async fn main() { // run prod if this isnt present // create and spawn the mqtt processor info!("Running processor in MQTT (production) mode"); - let (recv, eloop) = MqttProcessor::new( + let recv = MqttProcessor::new( mqtt_receive, std::env::var("PROD_SIREN_HOST_URL").unwrap_or("localhost:1883".to_string()), db.clone(), io, ) .await; - tokio::spawn(recv.process_mqtt(eloop)); + tokio::spawn(recv.process_mqtt()); } let app = Router::new() diff --git a/scylla-server-rust/src/processors/db_handler.rs b/scylla-server-rust/src/processors/db_handler.rs index aeda364a..9d611f2b 100644 --- a/scylla-server-rust/src/processors/db_handler.rs +++ b/scylla-server-rust/src/processors/db_handler.rs @@ -141,7 +141,7 @@ impl DbHandler { } } - #[instrument(level = Level::DEBUG)] + #[instrument(level = Level::DEBUG, skip(msg))] async fn batch_upload(msg: Vec, db: &Database) { info!( "Batch uploaded: {:?}", @@ -176,7 +176,7 @@ impl DbHandler { #[instrument(skip(self), level = Level::TRACE)] async fn handle_msg(&mut self, msg: ClientData, data_channel: &Sender>) { - debug!( + trace!( "Mqtt dispatcher: {} of {}", self.receiver.len(), self.receiver.max_capacity() diff --git a/scylla-server-rust/src/processors/mqtt_processor.rs b/scylla-server-rust/src/processors/mqtt_processor.rs index 093743c6..ca8695fc 100644 --- a/scylla-server-rust/src/processors/mqtt_processor.rs +++ b/scylla-server-rust/src/processors/mqtt_processor.rs @@ -5,7 +5,7 @@ use prisma_client_rust::{chrono, serde_json}; use protobuf::Message; use rumqttc::v5::{ mqttbytes::v5::{LastWill, Packet, Publish}, - AsyncClient, Event, EventLoop, MqttOptions, + AsyncClient, Event, MqttOptions, }; use socketioxide::SocketIo; use tokio::sync::mpsc::Sender; @@ -20,6 +20,7 @@ pub struct MqttProcessor { channel: Sender, curr_run: i32, io: SocketIo, + mqtt_ops: MqttOptions, } impl MqttProcessor { @@ -36,7 +37,7 @@ impl MqttProcessor { mqtt_path: String, db: Database, io: SocketIo, - ) -> (MqttProcessor, EventLoop) { + ) -> MqttProcessor { // create the mqtt client and configure it let mut create_opts = MqttOptions::new( "ScyllaServer", @@ -48,15 +49,19 @@ impl MqttProcessor { .parse::() .expect("Invalid Siren port"), ); - create_opts.set_keep_alive(Duration::from_secs(20)); - create_opts.set_last_will(LastWill::new( - "Scylla/Status", - "Scylla has disconnected!", - rumqttc::v5::mqttbytes::QoS::ExactlyOnce, - true, - None, - )); - create_opts.set_clean_start(false); + create_opts + .set_last_will(LastWill::new( + "Scylla/Status", + "Scylla has disconnected!", + rumqttc::v5::mqttbytes::QoS::ExactlyOnce, + true, + None, + )) + .set_keep_alive(Duration::from_secs(20)) + .set_clean_start(false) + .set_connection_timeout(3) + .set_session_expiry_interval(Some(u32::MAX)) + .set_topic_alias_max(Some(600)); // creates the initial run let curr_run = run_service::create_run(&db, chrono::offset::Utc::now().timestamp_millis()) @@ -64,67 +69,65 @@ impl MqttProcessor { .expect("Could not create initial run!"); debug!("Configuring current run: {:?}", curr_run); - // TODO mess with incoming message cap if db, etc. cannot keep up - let (client, connect) = AsyncClient::new(create_opts, 1000); - - debug!("Subscribing to siren"); - client - .try_subscribe("#", rumqttc::v5::mqttbytes::QoS::ExactlyOnce) - .expect("Could not subscribe to Siren"); - - ( - MqttProcessor { - channel, - curr_run: curr_run.id, - io, - }, - connect, - ) + MqttProcessor { + channel, + curr_run: curr_run.id, + io, + mqtt_ops: create_opts, + } } /// This handles the reception of mqtt messages, will not return /// * `connect` - The eventloop returned by ::new to connect to - pub async fn process_mqtt(self, mut connect: EventLoop) { + pub async fn process_mqtt(self) { let mut spec_interval = tokio::time::interval(Duration::from_secs(3)); // process over messages, non blocking + // TODO mess with incoming message cap if db, etc. cannot keep up + let (client, mut connect) = AsyncClient::new(self.mqtt_ops.clone(), 600); + + debug!("Subscribing to siren"); + client + .subscribe("#", rumqttc::v5::mqttbytes::QoS::ExactlyOnce) + .await + .expect("Could not subscribe to Siren"); + loop { + #[rustfmt::skip] // rust cannot format this macro for some reason tokio::select! { - Ok(msg) = connect.poll() => { - // safe parse the message - if let Event::Incoming(Packet::Publish(msg)) = msg { - trace!("Received mqtt message: {:?}", msg); - // parse the message into the data and the node name it falls under - let msg = match self.parse_msg(msg) { - Ok(msg) => msg, - Err(err) => { - warn!("Message parse error: {:?}", err); - continue; - } - }; - self.send_db_msg(msg.clone()).await; - self.send_socket_msg(msg).await; - } - - }, - _ = spec_interval.tick() => { - trace!("Updating viewership data!"); - if let Ok(sockets) = self.io.sockets() { - let client_data = ClientData { - name: "Viewers".to_string(), - node: "Internal".to_string(), - unit: "".to_string(), - run_id: self.curr_run, - timestamp: chrono::offset::Utc::now().timestamp_millis(), - values: vec![sockets.len().to_string()] + msg = connect.poll() => match msg { + Ok(Event::Incoming(Packet::Publish(msg))) => { + trace!("Received mqtt message: {:?}", msg); + // parse the message into the data and the node name it falls under + let msg = match self.parse_msg(msg) { + Ok(msg) => msg, + Err(err) => { + warn!("Message parse error: {:?}", err); + continue; + } }; - self.send_socket_msg(client_data).await; - + self.send_db_msg(msg.clone()).await; + self.send_socket_msg(msg); + }, + Err(msg) => trace!("Received mqtt error: {:?}", msg), + _ => trace!("Received misc mqtt: {:?}", msg), + }, + _ = spec_interval.tick() => { + trace!("Updating viewership data!"); + if let Ok(sockets) = self.io.sockets() { + let client_data = ClientData { + name: "Viewers".to_string(), + node: "Internal".to_string(), + unit: "".to_string(), + run_id: self.curr_run, + timestamp: chrono::offset::Utc::now().timestamp_millis(), + values: vec![sockets.len().to_string()] + }; + self.send_socket_msg(client_data); } else { warn!("Could not fetch socket count"); } - } - } + } } } @@ -145,7 +148,7 @@ impl MqttProcessor { let data_type = split.1.replace('/', "-"); - // extract the unix time, returning the current time instead if needed + // extract the unix time, returning the current time instead if either the "ts" user property isnt present or it isnt parsable // note the Cow magic involves the return from the map is a borrow, but the unwrap cannot as we dont own it let unix_time = msg .properties @@ -155,29 +158,38 @@ impl MqttProcessor { .map(Cow::Borrowed) .find(|f| f.0 == "ts") .unwrap_or_else(|| { - debug!("Could not find timestamp in mqtt, using current time"); + debug!("Could not find timestamp in mqtt, using system time"); Cow::Owned(( "ts".to_string(), chrono::offset::Utc::now().timestamp_millis().to_string(), )) }) - .into_owned(); + .1 + .parse::() + .unwrap_or_else(|err| { + warn!("Invalid timestamp in mqtt, using system time: {}", err); + chrono::offset::Utc::now().timestamp_millis() + }); - // parse time, if invalid time error out - let Ok(time_clean) = unix_time.1.parse::() else { - return Err(format!("Invalid timestamp: {}", unix_time.1)); - }; // ts check for bad sources of time which may return 1970 - if time_clean < 963014966000 { - return Err(format!("Timestamp before year 2000: {}", unix_time.1)); - } + // if both system time and packet timestamp are before year 2000, the message cannot be recorded + let unix_clean = if unix_time < 963014966000 { + debug!("Timestamp before year 2000: {}", unix_time); + let sys_time = chrono::offset::Utc::now().timestamp_millis(); + if sys_time < 963014966000 { + return Err("System has no good time, discarding message!".to_string()); + } + sys_time + } else { + unix_time + }; Ok(ClientData { run_id: self.curr_run, name: data_type, unit: data.unit, values: data.value, - timestamp: time_clean, + timestamp: unix_clean, node: node.to_string(), }) } @@ -192,7 +204,7 @@ impl MqttProcessor { /// Sends a message to the socket, printing and IGNORING any error that may occur /// * `client_data` - The client data to send over the broadcast - async fn send_socket_msg(&self, client_data: ClientData) { + fn send_socket_msg(&self, client_data: ClientData) { match self.io.emit( "message", serde_json::to_string(&client_data).expect("Could not serialize ClientData"), diff --git a/siren-base/mosquitto/mosquitto.conf b/siren-base/mosquitto/mosquitto.conf index f4cee72f..e281ff79 100755 --- a/siren-base/mosquitto/mosquitto.conf +++ b/siren-base/mosquitto/mosquitto.conf @@ -109,21 +109,27 @@ autosave_interval 30 #autosave_on_changes false -#persistence false +# *** diff from tpu +persistence true #persistence_file mosquitto.db -#persistence_location +# *** diff from tpu +persistence_location /mosquitto/data # ================================================================= # Logging # ================================================================= # *** diff from tpu (for docker) -log_dest file /mosquitto/log/mosquitto.log - +log_dest stdout +log_type error +log_type warning log_type notice +log_type information +log_type subscribe +log_type unsubscribe #log_type information connection_messages true From 85c42b8a586fea4dab518cca091db28440d720d7 Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Sun, 21 Jul 2024 12:19:28 -0400 Subject: [PATCH 43/68] instructions and feature config for tokio-console top --- scylla-server-rust/Cargo.lock | 214 ++++++++++++++++++++++++++++++++- scylla-server-rust/Cargo.toml | 4 + scylla-server-rust/README.md | 6 + scylla-server-rust/src/main.rs | 43 ++++--- 4 files changed, 249 insertions(+), 18 deletions(-) diff --git a/scylla-server-rust/Cargo.lock b/scylla-server-rust/Cargo.lock index b3a84563..2bfc4121 100755 --- a/scylla-server-rust/Cargo.lock +++ b/scylla-server-rust/Cargo.lock @@ -91,6 +91,28 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eab1c04a571841102f5345a8fc0f6bb3d31c315dec879b5c6e42e40ce7ffa34e" +[[package]] +name = "async-stream" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.67", +] + [[package]] name = "async-trait" version = "0.1.80" @@ -117,6 +139,34 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +[[package]] +name = "axum" +version = "0.6.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" +dependencies = [ + "async-trait", + "axum-core 0.3.4", + "bitflags 1.3.2", + "bytes", + "futures-util", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.29", + "itoa", + "matchit 0.7.3", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper 0.1.2", + "tower", + "tower-layer", + "tower-service", +] + [[package]] name = "axum" version = "0.7.5" @@ -124,7 +174,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf" dependencies = [ "async-trait", - "axum-core", + "axum-core 0.4.3", "bytes", "futures-util", "http 1.1.0", @@ -151,6 +201,23 @@ dependencies = [ "tracing", ] +[[package]] +name = "axum-core" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http 0.2.12", + "http-body 0.4.6", + "mime", + "rustversion", + "tower-layer", + "tower-service", +] + [[package]] name = "axum-core" version = "0.4.3" @@ -371,6 +438,44 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "console-api" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a257c22cd7e487dd4a13d413beabc512c5052f0bc048db0da6a84c3d8a6142fd" +dependencies = [ + "futures-core", + "prost", + "prost-types", + "tonic", + "tracing-core", +] + +[[package]] +name = "console-subscriber" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31c4cc54bae66f7d9188996404abdf7fdfa23034ef8e43478c8810828abad758" +dependencies = [ + "console-api", + "crossbeam-channel", + "crossbeam-utils", + "futures-task", + "hdrhistogram", + "humantime", + "prost", + "prost-types", + "serde", + "serde_json", + "thread_local", + "tokio", + "tokio-stream", + "tonic", + "tracing", + "tracing-core", + "tracing-subscriber", +] + [[package]] name = "convert_case" version = "0.5.0" @@ -979,6 +1084,19 @@ dependencies = [ "allocator-api2", ] +[[package]] +name = "hdrhistogram" +version = "7.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "765c9198f173dd59ce26ff9f95ef0aafd0a0fe01fb9d72841bc5066a4c06511d" +dependencies = [ + "base64 0.21.7", + "byteorder", + "flate2", + "nom", + "num-traits", +] + [[package]] name = "heck" version = "0.3.3" @@ -1097,6 +1215,12 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + [[package]] name = "hyper" version = "0.14.29" @@ -1140,6 +1264,18 @@ dependencies = [ "tokio", ] +[[package]] +name = "hyper-timeout" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +dependencies = [ + "hyper 0.14.29", + "pin-project-lite", + "tokio", + "tokio-io-timeout", +] + [[package]] name = "hyper-tls" version = "0.5.0" @@ -2244,6 +2380,38 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prost" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 2.0.67", +] + +[[package]] +name = "prost-types" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9091c90b0a32608e984ff2fa4091273cbdd755d54935c51d520887f4a1dbd5b0" +dependencies = [ + "prost", +] + [[package]] name = "protobuf" version = "3.5.0" @@ -2919,7 +3087,8 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" name = "scylla-server-rust" version = "0.0.1" dependencies = [ - "axum", + "axum 0.7.5", + "console-subscriber", "prisma-client-rust", "protobuf", "protobuf-codegen", @@ -3465,6 +3634,16 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "tokio-io-timeout" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" +dependencies = [ + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-macros" version = "2.3.0" @@ -3578,6 +3757,33 @@ dependencies = [ "serde", ] +[[package]] +name = "tonic" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76c4eb7a4e9ef9d4763600161f12f5070b92a578e1b634db88a6887844c91a13" +dependencies = [ + "async-stream", + "async-trait", + "axum 0.6.20", + "base64 0.21.7", + "bytes", + "h2", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.29", + "hyper-timeout", + "percent-encoding", + "pin-project", + "prost", + "tokio", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "tower" version = "0.4.13" @@ -3586,9 +3792,13 @@ checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" dependencies = [ "futures-core", "futures-util", + "indexmap 1.9.3", "pin-project", "pin-project-lite", + "rand 0.8.5", + "slab", "tokio", + "tokio-util", "tower-layer", "tower-service", "tracing", diff --git a/scylla-server-rust/Cargo.toml b/scylla-server-rust/Cargo.toml index e223e7d9..dd811eb9 100644 --- a/scylla-server-rust/Cargo.toml +++ b/scylla-server-rust/Cargo.toml @@ -18,6 +18,10 @@ tokio-util = { version= "0.7.11", features = ["full"] } tracing = "0.1.40" tracing-subscriber = { version = "0.3.18", features = ["ansi", "env-filter"] } rand = "0.8.5" +console-subscriber = { version = "0.3.0", optional = true } + +[features] +top = ["dep:console-subscriber"] [workspace] members = ["prisma-cli"] diff --git a/scylla-server-rust/README.md b/scylla-server-rust/README.md index 1ea369de..7935d281 100644 --- a/scylla-server-rust/README.md +++ b/scylla-server-rust/README.md @@ -38,6 +38,12 @@ You can send your own messages to the app running in production mode, and test h Follow this confluence doc: https://nerdocs.atlassian.net/wiki/spaces/NER/pages/473727054/How+to+run+data+through+Argos+without+the+car +#### View threads and resources + +1. Build or run as: `RUSTFLAGS="--cfg tokio_unstable" cargo run --features top` +2. Install tokio console: ex `cargo install --locked tokio-console` +3. Run app and `tokio-console` + #### Debug logging #### Activate logs diff --git a/scylla-server-rust/src/main.rs b/scylla-server-rust/src/main.rs index 7169afd7..a2f66bb6 100755 --- a/scylla-server-rust/src/main.rs +++ b/scylla-server-rust/src/main.rs @@ -23,22 +23,33 @@ use tracing_subscriber::{fmt::format::FmtSpan, EnvFilter}; #[tokio::main] async fn main() { println!("Initializing scylla server..."); - // construct a subscriber that prints formatted traces to stdout - // if RUST_LOG is not set, defaults to loglevel INFO - let subscriber = tracing_subscriber::fmt() - .pretty() - .with_thread_ids(true) - .with_ansi(true) - .with_thread_names(true) - .with_span_events(FmtSpan::CLOSE) - .with_env_filter( - EnvFilter::builder() - .with_default_directive(LevelFilter::INFO.into()) - .from_env_lossy(), - ) - .finish(); - // use that subscriber to process traces emitted after this point - tracing::subscriber::set_global_default(subscriber).expect("Could not init tracing"); + + #[cfg(feature = "top")] + { + println!("Initializing tokio console subscriber"); + console_subscriber::init(); + } + + #[cfg(not(feature = "top"))] + { + println!("Initializing fmt subscriber"); + // construct a subscriber that prints formatted traces to stdout + // if RUST_LOG is not set, defaults to loglevel INFO + let subscriber = tracing_subscriber::fmt() + .pretty() + .with_thread_ids(true) + .with_ansi(true) + .with_thread_names(true) + .with_span_events(FmtSpan::CLOSE) + .with_env_filter( + EnvFilter::builder() + .with_default_directive(LevelFilter::INFO.into()) + .from_env_lossy(), + ) + .finish(); + // use that subscriber to process traces emitted after this point + tracing::subscriber::set_global_default(subscriber).expect("Could not init tracing"); + } // create the database stuff let db: Database = Arc::new( From 17cff48125147c12c9c7af15691bc8f005fb732d Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Sun, 21 Jul 2024 20:12:19 -0400 Subject: [PATCH 44/68] add latency send 0.25 seconds 20 message ringbuffer --- scylla-server-rust/Cargo.lock | 7 +++++ scylla-server-rust/Cargo.toml | 1 + .../src/processors/mqtt_processor.rs | 31 +++++++++++++++++-- 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/scylla-server-rust/Cargo.lock b/scylla-server-rust/Cargo.lock index 2bfc4121..76589e94 100755 --- a/scylla-server-rust/Cargo.lock +++ b/scylla-server-rust/Cargo.lock @@ -2921,6 +2921,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "ringbuffer" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3df6368f71f205ff9c33c076d170dd56ebf68e8161c733c0caa07a7a5509ed53" + [[package]] name = "rumqttc" version = "0.24.0" @@ -3093,6 +3099,7 @@ dependencies = [ "protobuf", "protobuf-codegen", "rand 0.8.5", + "ringbuffer", "rumqttc", "serde", "socketioxide", diff --git a/scylla-server-rust/Cargo.toml b/scylla-server-rust/Cargo.toml index dd811eb9..37fd4f13 100644 --- a/scylla-server-rust/Cargo.toml +++ b/scylla-server-rust/Cargo.toml @@ -19,6 +19,7 @@ tracing = "0.1.40" tracing-subscriber = { version = "0.3.18", features = ["ansi", "env-filter"] } rand = "0.8.5" console-subscriber = { version = "0.3.0", optional = true } +ringbuffer = "0.15.0" [features] top = ["dep:console-subscriber"] diff --git a/scylla-server-rust/src/processors/mqtt_processor.rs b/scylla-server-rust/src/processors/mqtt_processor.rs index ca8695fc..80ad9ece 100644 --- a/scylla-server-rust/src/processors/mqtt_processor.rs +++ b/scylla-server-rust/src/processors/mqtt_processor.rs @@ -1,8 +1,9 @@ use core::fmt; use std::time::Duration; -use prisma_client_rust::{chrono, serde_json}; +use prisma_client_rust::{bigdecimal::ToPrimitive, chrono, serde_json}; use protobuf::Message; +use ringbuffer::RingBuffer; use rumqttc::v5::{ mqttbytes::v5::{LastWill, Packet, Publish}, AsyncClient, Event, MqttOptions, @@ -80,7 +81,11 @@ impl MqttProcessor { /// This handles the reception of mqtt messages, will not return /// * `connect` - The eventloop returned by ::new to connect to pub async fn process_mqtt(self) { - let mut spec_interval = tokio::time::interval(Duration::from_secs(3)); + let mut view_interval = tokio::time::interval(Duration::from_secs(3)); + + let mut latency_interval = tokio::time::interval(Duration::from_millis(250)); + let mut latency_ringbuffer = ringbuffer::AllocRingBuffer::::new(20); + // process over messages, non blocking // TODO mess with incoming message cap if db, etc. cannot keep up let (client, mut connect) = AsyncClient::new(self.mqtt_ops.clone(), 600); @@ -105,13 +110,14 @@ impl MqttProcessor { continue; } }; + latency_ringbuffer.push(chrono::offset::Utc::now().timestamp_millis() - msg.timestamp); self.send_db_msg(msg.clone()).await; self.send_socket_msg(msg); }, Err(msg) => trace!("Received mqtt error: {:?}", msg), _ => trace!("Received misc mqtt: {:?}", msg), }, - _ = spec_interval.tick() => { + _ = view_interval.tick() => { trace!("Updating viewership data!"); if let Ok(sockets) = self.io.sockets() { let client_data = ClientData { @@ -127,6 +133,25 @@ impl MqttProcessor { warn!("Could not fetch socket count"); } } + _ = latency_interval.tick() => { + // set latency to 0 if no messages are in buffer + let avg_latency = if latency_ringbuffer.is_empty() { + 0 + } else { + latency_ringbuffer.iter().sum::() / latency_ringbuffer.len().to_i64().unwrap_or_default() + }; + + let client_data = ClientData { + name: "Latency".to_string(), + node: "Internal".to_string(), + unit: "ms".to_string(), + run_id: self.curr_run, + timestamp: chrono::offset::Utc::now().timestamp_millis(), + values: vec![avg_latency.to_string()] + }; + trace!("Latency update sending: {}", client_data.values.first().unwrap_or(&"n/a".to_string())); + self.send_socket_msg(client_data); + } } } } From 716677b355907d0c89b1f51e0a2dfee74acc00dc Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Tue, 23 Jul 2024 14:21:50 -0400 Subject: [PATCH 45/68] add new run, fix some error handling and prints, add axum trace --- scylla-server-rust/Cargo.lock | 1 + scylla-server-rust/Cargo.toml | 2 +- .../src/controllers/run_controller.rs | 20 ++++++++- scylla-server-rust/src/main.rs | 44 ++++++++++++++----- .../src/processors/db_handler.rs | 8 ++-- .../src/processors/mqtt_processor.rs | 25 ++++++----- 6 files changed, 72 insertions(+), 28 deletions(-) diff --git a/scylla-server-rust/Cargo.lock b/scylla-server-rust/Cargo.lock index 76589e94..22090201 100755 --- a/scylla-server-rust/Cargo.lock +++ b/scylla-server-rust/Cargo.lock @@ -3825,6 +3825,7 @@ dependencies = [ "pin-project-lite", "tower-layer", "tower-service", + "tracing", ] [[package]] diff --git a/scylla-server-rust/Cargo.toml b/scylla-server-rust/Cargo.toml index 37fd4f13..49dc8cfa 100644 --- a/scylla-server-rust/Cargo.toml +++ b/scylla-server-rust/Cargo.toml @@ -11,7 +11,7 @@ protobuf = "3.3.0" tokio = { version = "1.38.0", features = ["full", "tracing"] } axum = "0.7.5" tower = { version = "0.4.13", features = ["timeout"] } -tower-http = { version = "0.5.2", features = ["cors"] } +tower-http = { version = "0.5.2", features = ["cors", "trace"] } socketioxide = { version = "0.14.0", features = ["tracing"] } rumqttc = { git = "https://github.com/bytebeamio/rumqtt", branch = "main"} tokio-util = { version= "0.7.11", features = ["full"] } diff --git a/scylla-server-rust/src/controllers/run_controller.rs b/scylla-server-rust/src/controllers/run_controller.rs index 4c26eac4..984f6db2 100644 --- a/scylla-server-rust/src/controllers/run_controller.rs +++ b/scylla-server-rust/src/controllers/run_controller.rs @@ -1,7 +1,10 @@ use axum::{ extract::{Path, State}, - Json, + Extension, Json, }; +use prisma_client_rust::chrono; +use tokio::sync::mpsc; +use tracing::warn; use crate::{ error::ScyllaError, services::run_service, transformers::run_transformer::PublicRun, Database, @@ -31,3 +34,18 @@ pub async fn get_run_by_id( Ok(Json::from(transformed_run_data)) } + +pub async fn new_run( + State(db): State, + Extension(channel): Extension>, +) -> Result, ScyllaError> { + let run_data = + run_service::create_run(&db, chrono::offset::Utc::now().timestamp_millis()).await?; + + // notify the mqtt receiver a new run has been created + if let Err(err) = channel.send(run_data.clone()).await { + warn!("Could not notify system about an updated run: {}", err); + } + + Ok(Json::from(PublicRun::from(&run_data))) +} diff --git a/scylla-server-rust/src/main.rs b/scylla-server-rust/src/main.rs index a2f66bb6..52acea93 100755 --- a/scylla-server-rust/src/main.rs +++ b/scylla-server-rust/src/main.rs @@ -1,6 +1,11 @@ use std::sync::Arc; -use axum::{http::Method, routing::get, Router}; +use axum::{ + http::Method, + routing::{get, post}, + Extension, Router, +}; +use prisma_client_rust::chrono; use scylla_server_rust::{ controllers::{ self, data_type_controller, driver_controller, location_controller, node_controller, @@ -10,13 +15,17 @@ use scylla_server_rust::{ processors::{ db_handler, mock_processor::MockProcessor, mqtt_processor::MqttProcessor, ClientData, }, + services::run_service::{self, public_run}, Database, }; use socketioxide::{extract::SocketRef, SocketIo}; use tokio::{signal, sync::mpsc}; use tokio_util::{sync::CancellationToken, task::TaskTracker}; use tower::ServiceBuilder; -use tower_http::cors::{Any, CorsLayer}; +use tower_http::{ + cors::{Any, CorsLayer}, + trace::TraceLayer, +}; use tracing::{debug, info, level_filters::LevelFilter}; use tracing_subscriber::{fmt::format::FmtSpan, EnvFilter}; @@ -67,11 +76,14 @@ async fn main() { // channel to pass the mqtt data // TODO tune buffer size - let (mqtt_receive, mqtt_send) = mpsc::channel::(10000); + let (mqtt_send, mqtt_receive) = mpsc::channel::(10000); // channel to pass the processed data to the db thread // TODO tune buffer size - let (db_receive, db_send) = mpsc::channel::>(1000); + let (db_send, db_receive) = mpsc::channel::>(1000); + + // channel to update the run to a new value + let (new_run_send, new_run_receive) = mpsc::channel::(5); // the below two threads need to cancel cleanly to ensure all queued messages are sent. therefore they are part of the a task tracker group. // create a task tracker and cancellation token @@ -79,12 +91,12 @@ async fn main() { let token = CancellationToken::new(); // spawn the database handler task_tracker.spawn( - db_handler::DbHandler::new(mqtt_send, Arc::clone(&db)) - .handling_loop(db_receive, token.clone()), + db_handler::DbHandler::new(mqtt_receive, Arc::clone(&db)) + .handling_loop(db_send, token.clone()), ); // spawn the database inserter task_tracker.spawn(db_handler::DbHandler::batching_loop( - db_send, + db_receive, Arc::clone(&db), token.clone(), )); @@ -95,13 +107,20 @@ async fn main() { let recv = MockProcessor::new(io); tokio::spawn(recv.generate_mock()); } else { + // creates the initial run + let curr_run = run_service::create_run(&db, chrono::offset::Utc::now().timestamp_millis()) + .await + .expect("Could not create initial run!"); + debug!("Configuring current run: {:?}", curr_run); + // run prod if this isnt present // create and spawn the mqtt processor info!("Running processor in MQTT (production) mode"); let recv = MqttProcessor::new( - mqtt_receive, + mqtt_send, + new_run_receive, std::env::var("PROD_SIREN_HOST_URL").unwrap_or("localhost:1883".to_string()), - db.clone(), + curr_run.id, io, ) .await; @@ -125,13 +144,17 @@ async fn main() { // RUNS .route("/runs", get(run_controller::get_all_runs)) .route("/runs/:id", get(run_controller::get_run_by_id)) + .route( + "/runs/new", + post(run_controller::new_run).layer(Extension(new_run_send)), + ) // SYSTEMS .route("/systems", get(system_controller::get_all_systems)) // for CORS handling .layer( CorsLayer::new() // allow `GET` - .allow_methods([Method::GET]) + .allow_methods([Method::GET, Method::POST]) // allow requests from any origin .allow_origin(Any), ) @@ -141,6 +164,7 @@ async fn main() { .layer(CorsLayer::permissive()) .layer(socket_layer), ) + .layer(TraceLayer::new_for_http()) .with_state(db.clone()); let listener = tokio::net::TcpListener::bind("0.0.0.0:8000") diff --git a/scylla-server-rust/src/processors/db_handler.rs b/scylla-server-rust/src/processors/db_handler.rs index 9d611f2b..5ad20d2b 100644 --- a/scylla-server-rust/src/processors/db_handler.rs +++ b/scylla-server-rust/src/processors/db_handler.rs @@ -143,10 +143,10 @@ impl DbHandler { #[instrument(level = Level::DEBUG, skip(msg))] async fn batch_upload(msg: Vec, db: &Database) { - info!( - "Batch uploaded: {:?}", - data_service::add_many(db, msg).await - ); + match data_service::add_many(db, msg).await { + Ok(count) => info!("Batch uploaded: {:?}", count), + Err(err) => warn!("Error in batch upload: {:?}", err), + } } /// A loop which uses self and a sender channel to process data diff --git a/scylla-server-rust/src/processors/mqtt_processor.rs b/scylla-server-rust/src/processors/mqtt_processor.rs index 80ad9ece..b824df0d 100644 --- a/scylla-server-rust/src/processors/mqtt_processor.rs +++ b/scylla-server-rust/src/processors/mqtt_processor.rs @@ -9,16 +9,17 @@ use rumqttc::v5::{ AsyncClient, Event, MqttOptions, }; use socketioxide::SocketIo; -use tokio::sync::mpsc::Sender; +use tokio::sync::mpsc::{Receiver, Sender}; use tracing::{debug, instrument, trace, warn, Level}; -use crate::{serverdata, services::run_service, Database}; +use crate::{serverdata, services::run_service}; use super::ClientData; use std::borrow::Cow; pub struct MqttProcessor { channel: Sender, + new_run_channel: Receiver, curr_run: i32, io: SocketIo, mqtt_ops: MqttOptions, @@ -35,8 +36,9 @@ impl MqttProcessor { /// Returns the instance and the event loop, which can be passed into the process_mqtt func to begin recieiving pub async fn new( channel: Sender, + new_run_channel: Receiver, mqtt_path: String, - db: Database, + initial_run: i32, io: SocketIo, ) -> MqttProcessor { // create the mqtt client and configure it @@ -64,15 +66,10 @@ impl MqttProcessor { .set_session_expiry_interval(Some(u32::MAX)) .set_topic_alias_max(Some(600)); - // creates the initial run - let curr_run = run_service::create_run(&db, chrono::offset::Utc::now().timestamp_millis()) - .await - .expect("Could not create initial run!"); - debug!("Configuring current run: {:?}", curr_run); - MqttProcessor { channel, - curr_run: curr_run.id, + new_run_channel, + curr_run: initial_run, io, mqtt_ops: create_opts, } @@ -80,7 +77,7 @@ impl MqttProcessor { /// This handles the reception of mqtt messages, will not return /// * `connect` - The eventloop returned by ::new to connect to - pub async fn process_mqtt(self) { + pub async fn process_mqtt(mut self) { let mut view_interval = tokio::time::interval(Duration::from_secs(3)); let mut latency_interval = tokio::time::interval(Duration::from_millis(250)); @@ -152,6 +149,10 @@ impl MqttProcessor { trace!("Latency update sending: {}", client_data.values.first().unwrap_or(&"n/a".to_string())); self.send_socket_msg(client_data); } + Some(new_run) = self.new_run_channel.recv() => { + trace!("New run: {:?}", new_run); + self.curr_run = new_run.id; + } } } } @@ -165,7 +166,7 @@ impl MqttProcessor { .map_err(|f| format!("Could not parse message topic:{:?} error: {}", msg.topic, f))?; let split = std::str::from_utf8(&msg.topic) - .unwrap_or_else(|_| panic!("Could not parse topic: {:?}", msg.topic)) + .map_err(|f| format!("Could not parse topic: {}, topic: {:?}", f, msg.topic))? .split_once('/') .ok_or(&format!("Could not parse nesting: {:?}", msg.topic))?; From ecbb5884602a37580a17d85ab393e895ec6bb345 Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Tue, 23 Jul 2024 17:28:10 -0400 Subject: [PATCH 46/68] add cargo prisma-seed --- scylla-server-rust/.cargo/config.toml | 3 +- scylla-server-rust/Cargo.toml | 9 + scylla-server-rust/README.md | 4 + scylla-server-rust/prisma/seed.rs | 229 ++++++++++++++++++++++++++ 4 files changed, 244 insertions(+), 1 deletion(-) create mode 100644 scylla-server-rust/prisma/seed.rs diff --git a/scylla-server-rust/.cargo/config.toml b/scylla-server-rust/.cargo/config.toml index ba1f1a45..09ffa116 100644 --- a/scylla-server-rust/.cargo/config.toml +++ b/scylla-server-rust/.cargo/config.toml @@ -1,2 +1,3 @@ [alias] -prisma = "run --release --package prisma-cli --" \ No newline at end of file +prisma = "run --release --package prisma-cli --" +prisma-seed = "run --release --bin seed" \ No newline at end of file diff --git a/scylla-server-rust/Cargo.toml b/scylla-server-rust/Cargo.toml index 49dc8cfa..e7532679 100644 --- a/scylla-server-rust/Cargo.toml +++ b/scylla-server-rust/Cargo.toml @@ -2,6 +2,7 @@ name = "scylla-server-rust" version = "0.0.1" edition = "2021" +default-run = "scylla-server-rust" [dependencies] prisma-client-rust = {git = "https://github.com/Brendonovich/prisma-client-rust", tag = "0.6.11", default-features = false,features = ["postgresql", "migrations"] } @@ -35,3 +36,11 @@ lto = true codegen-units = 1 panic = "abort" strip = true # Automatically strip symbols from the binary. + +[[bin]] +name = "scylla-server-rust" +path = "src/main.rs" + +[[bin]] +name = "seed" +path = "prisma/seed.rs" \ No newline at end of file diff --git a/scylla-server-rust/README.md b/scylla-server-rust/README.md index 7935d281..7bee2651 100644 --- a/scylla-server-rust/README.md +++ b/scylla-server-rust/README.md @@ -21,6 +21,10 @@ SOURCE_DATABASE_URL=postgresql://postgres:password@127.0.0.1:5432/timescaledb ca ### Test this app +#### Seed some data + +Run `cargo prisma-seed` + #### Integration tests diff --git a/scylla-server-rust/prisma/seed.rs b/scylla-server-rust/prisma/seed.rs new file mode 100644 index 00000000..42f99871 --- /dev/null +++ b/scylla-server-rust/prisma/seed.rs @@ -0,0 +1,229 @@ +use std::{sync::Arc, time::Duration}; + +use prisma_client_rust::{chrono, QueryError}; +use scylla_server_rust::{ + prisma::PrismaClient, + processors::ClientData, + services::{ + data_service, data_type_service, driver_service, location_service, node_service, + run_service, system_service, + }, + Database, +}; + +#[tokio::main] +async fn main() -> Result<(), QueryError> { + println!("Connecting and seeding!"); + let client: Database = Arc::new( + PrismaClient::_builder() + .build() + .await + .expect("Could not build prisma DB"), + ); + + client.data().delete_many(vec![]).exec().await?; + + client.data_type().delete_many(vec![]).exec().await?; + + client.driver().delete_many(vec![]).exec().await?; + + client.location().delete_many(vec![]).exec().await?; + + client.node().delete_many(vec![]).exec().await?; + + client.run().delete_many(vec![]).exec().await?; + + client.system().delete_many(vec![]).exec().await?; + + let created_run = + run_service::create_run(&client, chrono::offset::Utc::now().timestamp_millis()).await?; + + system_service::upsert_system(&client, "Data And Controls".to_string(), created_run.id).await?; + driver_service::upsert_driver(&client, "Fergus".to_string(), created_run.id).await?; + location_service::upsert_location( + &client, + "Gainsborough".to_string(), + 1.0, + 1.0, + 1.0, + created_run.id, + ) + .await?; + + node_service::upsert_node(&client, "BMS".to_string()).await?; + node_service::upsert_node(&client, "MPU".to_string()).await?; + + data_type_service::upsert_data_type( + &client, + "Pack-Temp".to_string(), + "C".to_string(), + "BMS".to_string(), + ) + .await?; + data_service::add_many( + &client, + vec![ + ClientData { + run_id: created_run.id, + name: "Pack-Temp".to_string(), + unit: "C".to_string(), + values: vec!["20".to_string()], + timestamp: chrono::offset::Utc::now().timestamp_millis(), + node: "BMS".to_string(), + }, + ClientData { + run_id: created_run.id, + name: "Pack-Temp".to_string(), + unit: "C".to_string(), + values: vec!["21".to_string()], + timestamp: chrono::offset::Utc::now().timestamp_millis() + 1000, + node: "BMS".to_string(), + }, + ClientData { + run_id: created_run.id, + name: "Pack-Temp".to_string(), + unit: "C".to_string(), + values: vec!["22".to_string()], + timestamp: chrono::offset::Utc::now().timestamp_millis() + 2000, + node: "BMS".to_string(), + }, + ClientData { + run_id: created_run.id, + name: "Pack-Temp".to_string(), + unit: "C".to_string(), + values: vec!["17".to_string()], + timestamp: chrono::offset::Utc::now().timestamp_millis() + 3000, + node: "BMS".to_string(), + }, + ClientData { + run_id: created_run.id, + name: "Pack-Temp".to_string(), + unit: "C".to_string(), + values: vec!["25".to_string()], + timestamp: chrono::offset::Utc::now().timestamp_millis() + 4000, + node: "BMS".to_string(), + }, + ClientData { + run_id: created_run.id, + name: "Pack-Temp".to_string(), + unit: "C".to_string(), + values: vec!["30".to_string()], + timestamp: chrono::offset::Utc::now().timestamp_millis() + 5000, + node: "BMS".to_string(), + }, + ClientData { + run_id: created_run.id, + name: "Pack-Temp".to_string(), + unit: "C".to_string(), + values: vec!["38".to_string()], + timestamp: chrono::offset::Utc::now().timestamp_millis() + 6000, + node: "BMS".to_string(), + }, + ClientData { + run_id: created_run.id, + name: "Pack-Temp".to_string(), + unit: "C".to_string(), + values: vec!["32".to_string()], + timestamp: chrono::offset::Utc::now().timestamp_millis() + 7000, + node: "BMS".to_string(), + }, + ClientData { + run_id: created_run.id, + name: "Pack-Temp".to_string(), + unit: "C".to_string(), + values: vec!["26".to_string()], + timestamp: chrono::offset::Utc::now().timestamp_millis() + 8000, + node: "BMS".to_string(), + }, + ], + ) + .await?; + + data_type_service::upsert_data_type( + &client, + "Pack-Voltage".to_string(), + "V".to_string(), + "BMS".to_string(), + ) + .await?; + data_type_service::upsert_data_type( + &client, + "Pack-SOC".to_string(), + "%".to_string(), + "BMS".to_string(), + ) + .await?; + data_type_service::upsert_data_type( + &client, + "Pack-Current".to_string(), + "A".to_string(), + "BMS".to_string(), + ) + .await?; + data_type_service::upsert_data_type( + &client, + "Sense-Accel".to_string(), + "G".to_string(), + "MPU".to_string(), + ) + .await?; + data_type_service::upsert_data_type( + &client, + "Sense-Temperature".to_string(), + "C".to_string(), + "MPU".to_string(), + ) + .await?; + data_type_service::upsert_data_type( + &client, + "State-Speed".to_string(), + "mph".to_string(), + "MPU".to_string(), + ) + .await?; + + node_service::upsert_node(&client, "TPU".to_string()).await?; + data_type_service::upsert_data_type( + &client, + "Points".to_string(), + "coords".to_string(), + "TPU".to_string(), + ) + .await?; + + simulate_route(client, created_run.id).await?; + + Ok(()) +} + +// lat,long +const NYC_COORDS: (f64, f64) = (40.7128, -74.006); +const LA_COORDS: (f64, f64) = (34.0522, -118.2437); +const STEP_NUM: u8 = 10; +async fn simulate_route(db: Database, curr_run: i32) -> Result<(), QueryError> { + let step_lat = (LA_COORDS.0 - NYC_COORDS.0) / STEP_NUM as f64; + let step_long = (LA_COORDS.1 - NYC_COORDS.1) / STEP_NUM as f64; + + for i in 0..STEP_NUM { + // clamp to [-90,90] + let inter_lat = f64::min(f64::max(NYC_COORDS.0 + step_lat * i as f64, -90.0), 90.0); + let inter_long = NYC_COORDS.1 + step_long * i as f64; + + data_service::add_data( + &db, + ClientData { + run_id: curr_run, + name: "Points".to_string(), + unit: "Coord".to_string(), + values: vec![inter_lat.to_string(), inter_long.to_string()], + timestamp: chrono::offset::Utc::now().timestamp_millis(), + node: "TPU".to_string(), + }, + ) + .await?; + + tokio::time::sleep(Duration::from_secs(1)).await; + } + + Ok(()) +} From 7c3d19c185a18b72ce16df078f720642003d4eb9 Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Tue, 23 Jul 2024 17:38:55 -0400 Subject: [PATCH 47/68] remedy clippy (for some reason gone on local) --- scylla-server-rust/prisma/seed.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scylla-server-rust/prisma/seed.rs b/scylla-server-rust/prisma/seed.rs index 42f99871..d8af8827 100644 --- a/scylla-server-rust/prisma/seed.rs +++ b/scylla-server-rust/prisma/seed.rs @@ -206,7 +206,7 @@ async fn simulate_route(db: Database, curr_run: i32) -> Result<(), QueryError> { for i in 0..STEP_NUM { // clamp to [-90,90] - let inter_lat = f64::min(f64::max(NYC_COORDS.0 + step_lat * i as f64, -90.0), 90.0); + let inter_lat = (NYC_COORDS.0 + step_lat * i as f64).clamp(-90.0, 90.0); let inter_long = NYC_COORDS.1 + step_long * i as f64; data_service::add_data( From 87f651888beb65aa972bcc48a9127fde72e306d9 Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Wed, 24 Jul 2024 10:36:08 -0400 Subject: [PATCH 48/68] run on push once --- .github/workflows/client-build.yml | 1 + .github/workflows/scylla-server-rust-build.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/client-build.yml b/.github/workflows/client-build.yml index cc5fa273..9148cfff 100644 --- a/.github/workflows/client-build.yml +++ b/.github/workflows/client-build.yml @@ -3,6 +3,7 @@ name: Create and publish client docker image on: workflow_dispatch: + push: diff --git a/.github/workflows/scylla-server-rust-build.yml b/.github/workflows/scylla-server-rust-build.yml index 42bdb18b..f0198a02 100644 --- a/.github/workflows/scylla-server-rust-build.yml +++ b/.github/workflows/scylla-server-rust-build.yml @@ -3,6 +3,7 @@ name: Create and publish scylla docker image on: workflow_dispatch: + push: From 5a615e6eabfd9ddfa419e9328d502130d973a8c9 Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Wed, 24 Jul 2024 13:13:51 -0400 Subject: [PATCH 49/68] misc compose fixups --- .github/workflows/client-build.yml | 1 - .github/workflows/scylla-server-rust-build.yml | 1 - angular-client/compose.client.yml | 2 +- compose.tpu.yml | 1 + compose.yml | 4 ++-- 5 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/client-build.yml b/.github/workflows/client-build.yml index 9148cfff..cc5fa273 100644 --- a/.github/workflows/client-build.yml +++ b/.github/workflows/client-build.yml @@ -3,7 +3,6 @@ name: Create and publish client docker image on: workflow_dispatch: - push: diff --git a/.github/workflows/scylla-server-rust-build.yml b/.github/workflows/scylla-server-rust-build.yml index f0198a02..42bdb18b 100644 --- a/.github/workflows/scylla-server-rust-build.yml +++ b/.github/workflows/scylla-server-rust-build.yml @@ -3,7 +3,6 @@ name: Create and publish scylla docker image on: workflow_dispatch: - push: diff --git a/angular-client/compose.client.yml b/angular-client/compose.client.yml index 472f12e5..e1a44d50 100644 --- a/angular-client/compose.client.yml +++ b/angular-client/compose.client.yml @@ -2,7 +2,7 @@ services: client: container_name: client restart: unless-stopped - image: ghcr.io/northeastern-electric-racing/argos-client:rust-rewrite-socket + image: ghcr.io/northeastern-electric-racing/argos-client:feature-scylla-rust-server build: context: . args: diff --git a/compose.tpu.yml b/compose.tpu.yml index d94fe873..f380d15a 100644 --- a/compose.tpu.yml +++ b/compose.tpu.yml @@ -4,3 +4,4 @@ services: - PROD_SIREN_HOST_URL=host.docker.internal:1883 extra_hosts: - "host.docker.internal:host-gateway" # for external siren + init: false # not supported on buildroot for some reason, further investigation needed diff --git a/compose.yml b/compose.yml index 49665376..f34d9fe1 100644 --- a/compose.yml +++ b/compose.yml @@ -17,7 +17,7 @@ services: scylla-server-rust: container_name: scylla-server-rust - image: ghcr.io/northeastern-electric-racing/argos-scylla:rust-rewrite-socket + image: ghcr.io/northeastern-electric-racing/argos-scylla:feature-scylla-rust-server build: context: ./scylla-server-rust restart: unless-stopped @@ -29,7 +29,7 @@ services: - SOURCE_DATABASE_URL=postgresql://postgres:password@odyssey-timescale:5432/timescaledb # - PROD_SIREN_HOST_URL=siren:1883 - PROD_SCYLLA=true - - RUST_LOG=none,scylla_server_rust=INFO # default log setting for docker + - RUST_LOG=none,scylla_server_rust=info # default log setting for docker cpu_shares: 1024 mem_limit: 2gb stop_grace_period: 10s From bfda4e0f96ac0a06f72f5c4b87e632d11d74818f Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Mon, 29 Jul 2024 15:53:45 -0400 Subject: [PATCH 50/68] change db name to postgres --- compose.yml | 2 +- scylla-server-rust/README.md | 4 ++-- scylla-server-rust/integration_test.sh | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/compose.yml b/compose.yml index f34d9fe1..1aa74b21 100644 --- a/compose.yml +++ b/compose.yml @@ -26,7 +26,7 @@ services: depends_on: - odyssey-timescale environment: - - SOURCE_DATABASE_URL=postgresql://postgres:password@odyssey-timescale:5432/timescaledb + - SOURCE_DATABASE_URL=postgresql://postgres:password@odyssey-timescale:5432/postgres # - PROD_SIREN_HOST_URL=siren:1883 - PROD_SCYLLA=true - RUST_LOG=none,scylla_server_rust=info # default log setting for docker diff --git a/scylla-server-rust/README.md b/scylla-server-rust/README.md index 7bee2651..cc650ad6 100644 --- a/scylla-server-rust/README.md +++ b/scylla-server-rust/README.md @@ -16,7 +16,7 @@ docker compose up odyssey-timescale ``` # in this directory -SOURCE_DATABASE_URL=postgresql://postgres:password@127.0.0.1:5432/timescaledb cargo run +SOURCE_DATABASE_URL=postgresql://postgres:password@127.0.0.1:5432/postgres cargo run ``` ### Test this app @@ -33,7 +33,7 @@ Since this app uses the database for testing, you must follow these steps, or ru docker volume rm argos_db-data docker compose up odyssey-timescale cargo prisma migrate deploy -SOURCE_DATABASE_URL=postgresql://postgres:password@127.0.0.1:5432/timescaledb cargo test -- --test-threads=1 +SOURCE_DATABASE_URL=postgresql://postgres:password@127.0.0.1:5432/postgres cargo test -- --test-threads=1 ``` #### Test it yourself! diff --git a/scylla-server-rust/integration_test.sh b/scylla-server-rust/integration_test.sh index 1e99b6b5..042756ed 100755 --- a/scylla-server-rust/integration_test.sh +++ b/scylla-server-rust/integration_test.sh @@ -9,6 +9,6 @@ cd ./scylla-server-rust cargo prisma migrate deploy echo "Running tests" -SOURCE_DATABASE_URL=postgresql://postgres:password@127.0.0.1:5432/timescaledb cargo test -- --test-threads=1 +SOURCE_DATABASE_URL=postgresql://postgres:password@127.0.0.1:5432/postgres cargo test -- --test-threads=1 echo "Please stop your db in docker" From 7f44f4301212e9808f8737593ae34947c9d146b3 Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Mon, 29 Jul 2024 21:32:45 -0400 Subject: [PATCH 51/68] add docker help scripts, clap cli and more settings to batching --- README.md | 10 +- compose.client-dev.yml | 4 +- compose.router.yml | 2 +- compose.scylla-dev.yml | 2 +- compose.tpu.yml | 2 +- compose.yml | 2 +- router-up.sh | 3 + scylla-server-rust/Cargo.lock | 122 +++++++++++++++++- scylla-server-rust/Cargo.toml | 3 +- scylla-server-rust/README.md | 6 +- scylla-server-rust/src/main.rs | 52 ++++++-- .../src/processors/db_handler.rs | 43 +++--- .../src/processors/mqtt_processor.rs | 15 ++- scylla.sh | 5 + 14 files changed, 232 insertions(+), 39 deletions(-) create mode 100755 router-up.sh create mode 100755 scylla.sh diff --git a/README.md b/README.md index 759253c5..9cd7330d 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,12 @@ Once you've sucessfully setup Scylla and the Client, you can either run them sep ## Production +### Quick start shortcuts + +The `scylla.sh` can be used on POSIX compliant OSes to quickly get up and running without worrying about profiles. Its syntax is `./scylla.sh ` where profile could be router, tpu, scylla-dev, or client-dev and the cmd is the normal argument passed into docker compose, such as `up -d` to start the process and fork to background. **Make sure to `down` with the same profile that you used `up` with!** + +### Customizing and more info + The base docker compose (`compose.yml`) contains some important features to note. However, it is useless standalone. Please read the profile customization selection below before using the base compose. - It matches the number of CPUs as the router to roughly simulate router CPU (your CPU is still faster) - It persists the database between `down` commands via a volume called `argos_db-data`. Delete it with `docker volume rm argos_db-data` to start with a new database next `up`. @@ -24,7 +30,7 @@ The base docker compose (`compose.yml`) contains some important features to note - It limits memory according to the capacity of the router. -### Customizing runtime profiles of the project via docker compose +#### Customizing runtime profiles of the project via docker compose This project uses docker compose overrides to secify configurations. Therefore there are multiple "profiles" to choose from when running in production, and there are some profiles for development testing. Also, there are fragment files for siren and client in `siren-base` and `angular-client` respectively, as they are services only used in certain cases. These profiles are specified via the command line on top of the base `compose.yml` file as follows. @@ -71,6 +77,6 @@ Server protobuf generation is automatic. See below for client protobuf generati #### Codegen `npm run build:proto` -### Siren +## Siren The configuration for the Mosquitto MQTT server on the router is in the siren-base folder. Note that the configuration is used in the docker compose file, but the configuration on the TPU is stored in [Odysseus.](https://github.com/Northeastern-Electric-Racing/Odysseus/tree/cb12fb3240d5fd58adfeae26262e158ad6dd889b/odysseus_tree/overlays/rootfs_overlay_tpu/etc/mosquitto) diff --git a/compose.client-dev.yml b/compose.client-dev.yml index 0030c37c..057a7f1f 100644 --- a/compose.client-dev.yml +++ b/compose.client-dev.yml @@ -1,8 +1,8 @@ services: scylla-server-rust: environment: - - PROD_SIREN_HOST_URL=siren:1883 - - PROD_SCYLLA=false + - SCYLLA_SIREN_HOST_URL=siren:1883 + - SCYLLA_PROD=false client: extends: diff --git a/compose.router.yml b/compose.router.yml index 3061c642..2ceca407 100644 --- a/compose.router.yml +++ b/compose.router.yml @@ -3,7 +3,7 @@ services: depends_on: - siren environment: - - PROD_SIREN_HOST_URL=siren:1883 + - SCYLLA_SIREN_HOST_URL=siren:1883 client: extends: diff --git a/compose.scylla-dev.yml b/compose.scylla-dev.yml index 1392f26a..3730b752 100644 --- a/compose.scylla-dev.yml +++ b/compose.scylla-dev.yml @@ -3,7 +3,7 @@ depends_on: - siren environment: - - PROD_SIREN_HOST_URL=siren:1883 + - SCYLLA_SIREN_HOST_URL=siren:1883 - RUST_LOG=none,scylla_server_rust=DEBUG client: diff --git a/compose.tpu.yml b/compose.tpu.yml index f380d15a..16c4e05e 100644 --- a/compose.tpu.yml +++ b/compose.tpu.yml @@ -1,7 +1,7 @@ services: scylla-server-rust: environment: - - PROD_SIREN_HOST_URL=host.docker.internal:1883 + - SCYLLA_SIREN_HOST_URL=host.docker.internal:1883 extra_hosts: - "host.docker.internal:host-gateway" # for external siren init: false # not supported on buildroot for some reason, further investigation needed diff --git a/compose.yml b/compose.yml index 1aa74b21..02e9fb70 100644 --- a/compose.yml +++ b/compose.yml @@ -28,7 +28,7 @@ services: environment: - SOURCE_DATABASE_URL=postgresql://postgres:password@odyssey-timescale:5432/postgres # - PROD_SIREN_HOST_URL=siren:1883 - - PROD_SCYLLA=true + - SCYLLA_PROD=true - RUST_LOG=none,scylla_server_rust=info # default log setting for docker cpu_shares: 1024 mem_limit: 2gb diff --git a/router-up.sh b/router-up.sh new file mode 100755 index 00000000..d682cb03 --- /dev/null +++ b/router-up.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +./scylla.sh router up -d diff --git a/scylla-server-rust/Cargo.lock b/scylla-server-rust/Cargo.lock index 22090201..51f2f9df 100755 --- a/scylla-server-rust/Cargo.lock +++ b/scylla-server-rust/Cargo.lock @@ -79,6 +79,55 @@ dependencies = [ "libc", ] +[[package]] +name = "anstream" +version = "0.6.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1" + +[[package]] +name = "anstyle-parse" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb47de1e80c2b463c735db5b217a0ddc39d612e7ac9e2e96a5aed1f57616c1cb" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8" +dependencies = [ + "anstyle", + "windows-sys 0.52.0", +] + [[package]] name = "anyhow" version = "1.0.86" @@ -405,6 +454,52 @@ dependencies = [ "windows-targets 0.52.5", ] +[[package]] +name = "clap" +version = "4.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35723e6a11662c2afb578bcf0b88bf6ea8e21282a953428f240574fcc3a2b5b3" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49eb96cbfa7cfa35017b7cd548c75b14c3118c98b423041d70562665e07fb0fa" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d029b67f89d30bbb547c89fd5161293c0aec155fc691d7924b64550662db93e" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.67", +] + +[[package]] +name = "clap_lex" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" + +[[package]] +name = "colorchoice" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0" + [[package]] name = "colored" version = "2.1.0" @@ -1106,6 +1201,12 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + [[package]] name = "hermit-abi" version = "0.3.9" @@ -1413,6 +1514,12 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + [[package]] name = "itertools" version = "0.10.5" @@ -1443,7 +1550,7 @@ version = "0.1.0" source = "git+https://github.com/Brendonovich/prisma-engines?tag=pcr-0.6.10#c4aeef82dbae310e974d6122160c7e3b5fb6df53" dependencies = [ "backtrace", - "heck", + "heck 0.3.3", "serde", "toml", ] @@ -3094,6 +3201,7 @@ name = "scylla-server-rust" version = "0.0.1" dependencies = [ "axum 0.7.5", + "clap", "console-subscriber", "prisma-client-rust", "protobuf", @@ -3481,6 +3589,12 @@ dependencies = [ "unicode-properties", ] +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "subtle" version = "2.6.0" @@ -4085,6 +4199,12 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "uuid" version = "1.8.0" diff --git a/scylla-server-rust/Cargo.toml b/scylla-server-rust/Cargo.toml index e7532679..f7d11bd8 100644 --- a/scylla-server-rust/Cargo.toml +++ b/scylla-server-rust/Cargo.toml @@ -21,6 +21,7 @@ tracing-subscriber = { version = "0.3.18", features = ["ansi", "env-filter"] } rand = "0.8.5" console-subscriber = { version = "0.3.0", optional = true } ringbuffer = "0.15.0" +clap = { version = "4.5.11", features = ["derive", "env"] } [features] top = ["dep:console-subscriber"] @@ -43,4 +44,4 @@ path = "src/main.rs" [[bin]] name = "seed" -path = "prisma/seed.rs" \ No newline at end of file +path = "prisma/seed.rs" diff --git a/scylla-server-rust/README.md b/scylla-server-rust/README.md index cc650ad6..6e9f0b7d 100644 --- a/scylla-server-rust/README.md +++ b/scylla-server-rust/README.md @@ -72,9 +72,9 @@ Have an async function that takes time and is somewhat important for performance See main README. -#### Env variables +#### Env variables & CLI Customization - `SOURCE_DATABASE_URL` The timescale URL -- `PROD_SCYLLA` false=use mock instead of production (mqtt) as source of data - `RUST_LOG=none,scylla_server_rust` levels of logging for this create, see above -- `PROD_SIREN_HOST_URL` URL:Port of the MQTT server, using when `PROD_SCYLLA=/false` + +**See `cargo run -- -h` for other variables and settings to change!** \ No newline at end of file diff --git a/scylla-server-rust/src/main.rs b/scylla-server-rust/src/main.rs index 52acea93..5453cd0c 100755 --- a/scylla-server-rust/src/main.rs +++ b/scylla-server-rust/src/main.rs @@ -1,10 +1,11 @@ -use std::sync::Arc; +use std::{sync::Arc, time::Duration}; use axum::{ http::Method, routing::{get, post}, Extension, Router, }; +use clap::Parser; use prisma_client_rust::chrono; use scylla_server_rust::{ controllers::{ @@ -29,8 +30,39 @@ use tower_http::{ use tracing::{debug, info, level_filters::LevelFilter}; use tracing_subscriber::{fmt::format::FmtSpan, EnvFilter}; +/// Scylla command line arguments +#[derive(Parser, Debug)] +#[command(version)] +struct ScyllaArgs { + /// Whether to enable the Scylla production mode + #[arg(short = 'p', long, env = "SCYLLA_PROD")] + prod: bool, + + /// Whether to enable batch saturation (parallel batching) + #[arg(short = 's', long, env = "SCYLLA_SATURATE_BATCH")] + saturate_batch: bool, + + // /// Whether to enable the socket io server in Scylla + // #[arg(short, long, env = "SCYLLA_SOCKET")] + // socket: bool, + /// The host url of the siren, including port and excluding protocol prefix + #[arg( + short = 'u', + long, + env = "SCYLLA_SIREN_HOST_URL", + default_value = "localhost:1883" + )] + siren_host_url: String, + + /// The time, in seconds between collection for a batch upsert + #[arg(short = 't', long, env = "SCYLLA_BATCH_UPSERT_TIME", default_value = "10")] + batch_upsert_time: u64, +} + #[tokio::main] async fn main() { + let cli = ScyllaArgs::parse(); + println!("Initializing scylla server..."); #[cfg(feature = "top")] @@ -45,7 +77,6 @@ async fn main() { // construct a subscriber that prints formatted traces to stdout // if RUST_LOG is not set, defaults to loglevel INFO let subscriber = tracing_subscriber::fmt() - .pretty() .with_thread_ids(true) .with_ansi(true) .with_thread_names(true) @@ -69,7 +100,12 @@ async fn main() { ); // create the socket stuff - let (socket_layer, io) = SocketIo::new_layer(); + let (socket_layer, io) = SocketIo::builder() + .max_buffer_size(4096) // TODO tune values + .connect_timeout(Duration::from_secs(5)) // may be unecessary + .ping_timeout(Duration::from_secs(5)) // may be unecessary + .ack_timeout(Duration::from_millis(1500)) // this should be well below the time to fill max buffer size above + .build_layer(); io.ns("/", |s: SocketRef| { s.on_disconnect(|_: SocketRef| debug!("Socket: Client disconnected from socket")) }); @@ -91,18 +127,19 @@ async fn main() { let token = CancellationToken::new(); // spawn the database handler task_tracker.spawn( - db_handler::DbHandler::new(mqtt_receive, Arc::clone(&db)) + db_handler::DbHandler::new(mqtt_receive, Arc::clone(&db), cli.batch_upsert_time * 1000) .handling_loop(db_send, token.clone()), ); // spawn the database inserter task_tracker.spawn(db_handler::DbHandler::batching_loop( db_receive, Arc::clone(&db), + cli.saturate_batch, token.clone(), )); // if PROD_SCYLLA=false - if std::env::var("PROD_SCYLLA").is_ok_and(|f| f == "false") { + if !cli.prod { info!("Running processor in mock mode, no data will be stored"); let recv = MockProcessor::new(io); tokio::spawn(recv.generate_mock()); @@ -119,11 +156,10 @@ async fn main() { let recv = MqttProcessor::new( mqtt_send, new_run_receive, - std::env::var("PROD_SIREN_HOST_URL").unwrap_or("localhost:1883".to_string()), + cli.siren_host_url, curr_run.id, io, - ) - .await; + ); tokio::spawn(recv.process_mqtt()); } diff --git a/scylla-server-rust/src/processors/db_handler.rs b/scylla-server-rust/src/processors/db_handler.rs index 5ad20d2b..c65f09b9 100644 --- a/scylla-server-rust/src/processors/db_handler.rs +++ b/scylla-server-rust/src/processors/db_handler.rs @@ -13,9 +13,6 @@ use crate::{ Database, }; -/// The upload interval for batch uploads -const UPLOAD_INTERVAL: u64 = 5000; - use super::{ClientData, LocationData}; /// A struct defining an in progress location packet @@ -90,12 +87,14 @@ pub struct DbHandler { data_queue: Vec, /// the time since last batch last_time: tokio::time::Instant, + /// upload interval + upload_interval: u64, } impl DbHandler { /// Make a new db handler /// * `recv` - the broadcast reciver of which clientdata will be sent - pub fn new(receiver: Receiver, db: Database) -> DbHandler { + pub fn new(receiver: Receiver, db: Database, upload_interval: u64) -> DbHandler { DbHandler { node_list: vec![], datatype_list: vec![], @@ -105,6 +104,7 @@ impl DbHandler { is_location: false, data_queue: vec![], last_time: tokio::time::Instant::now(), + upload_interval, } } @@ -114,24 +114,37 @@ impl DbHandler { pub async fn batching_loop( mut data_queue: Receiver>, database: Database, + saturate_batches: bool, cancel_token: CancellationToken, ) { loop { tokio::select! { _ = cancel_token.cancelled() => { - if let Some(final_msgs) = data_queue.recv().await { - info!( - "Final Batch uploaded: {:?}", - data_service::add_many(&database, final_msgs).await - ); - } else { - info!("No messages to cleanup.") + loop { + info!("{} batches remaining!", data_queue.len()); + if let Some(final_msgs) = data_queue.recv().await { + info!( + "A cleanup batch uploaded: {:?}", + data_service::add_many(&database, final_msgs).await + ); + } else { + info!("No more messages to cleanup."); + break; + } + } - break; + break; }, Some(msgs) = data_queue.recv() => { - Self::batch_upload(msgs, &database).await; - trace!( + if saturate_batches { + let shared_db = database.clone(); + tokio::spawn(async move { + Self::batch_upload(msgs, &shared_db).await; + }); + } else { + Self::batch_upload(msgs, &database).await; + } + debug!( "DB send: {} of {}", data_queue.len(), data_queue.max_capacity() @@ -184,7 +197,7 @@ impl DbHandler { // If the time is greater than upload interval, push to batch upload thread and clear queue if tokio::time::Instant::now().duration_since(self.last_time) - > Duration::from_millis(UPLOAD_INTERVAL) + > Duration::from_millis(self.upload_interval) && !self.data_queue.is_empty() { data_channel diff --git a/scylla-server-rust/src/processors/mqtt_processor.rs b/scylla-server-rust/src/processors/mqtt_processor.rs index b824df0d..7f34c0a6 100644 --- a/scylla-server-rust/src/processors/mqtt_processor.rs +++ b/scylla-server-rust/src/processors/mqtt_processor.rs @@ -32,9 +32,8 @@ impl MqttProcessor { /// * `db` - The database to store the data in /// * `io` - The socketio layer to send the data to /// - /// This is async as it creates the initial run and gets the ID, as well as connecting to and subbing Siren /// Returns the instance and the event loop, which can be passed into the process_mqtt func to begin recieiving - pub async fn new( + pub fn new( channel: Sender, new_run_channel: Receiver, mqtt_path: String, @@ -236,7 +235,17 @@ impl MqttProcessor { serde_json::to_string(&client_data).expect("Could not serialize ClientData"), ) { Ok(_) => (), - Err(err) => warn!("Socket: Broadcast error: {}", err), + Err(err) => match err { + socketioxide::BroadcastError::Socket(e) => { + trace!("Socket: Transmit error: {:?}", e); + } + socketioxide::BroadcastError::Serialize(_) => { + warn!("Socket: Serialize error: {}", err) + } + socketioxide::BroadcastError::Adapter(_) => { + warn!("Socket: Adapter error: {}", err) + } + }, } } } diff --git a/scylla.sh b/scylla.sh new file mode 100755 index 00000000..d58f5591 --- /dev/null +++ b/scylla.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +profile=$1 +shift 1 +docker compose -f compose.yml -f "compose.$profile.yml" "$@" From 174a6517631adc63f38c2c89c76343169849988a Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Sat, 10 Aug 2024 21:13:54 -0400 Subject: [PATCH 52/68] Scylla Calypso (#173) * basic proof of concept * misc improvements and cleanup of cursory processing * implement reids review suggestions * try and fix clippy * and again fix clippy * make data query optional to reset calypso to default if calypso receives a short or emtpy protobuf command_data, it will fill the extra values with the default value specified in the YAML. so we should allow the client to specify blank which could end up meaning a "Reset to Default" button. * clippy is trolling me * fixup error handling for plaintext error message * implement reid fixes * remove not prod error --- scylla-server-rust/.gitignore | 1 + scylla-server-rust/Cargo.lock | 37 ++++++++++ scylla-server-rust/Cargo.toml | 1 + scylla-server-rust/build.rs | 1 + .../src/controllers/car_command_controller.rs | 70 +++++++++++++++++++ scylla-server-rust/src/controllers/mod.rs | 1 + .../src/controllers/run_controller.rs | 2 +- scylla-server-rust/src/error.rs | 43 ++++++++---- scylla-server-rust/src/lib.rs | 1 + scylla-server-rust/src/main.rs | 31 ++++++-- scylla-server-rust/src/mod.rs | 1 + .../src/processors/db_handler.rs | 30 ++++---- scylla-server-rust/src/processors/mod.rs | 2 +- .../src/processors/mqtt_processor.rs | 64 ++++++++++------- .../src/proto/command_data.proto | 7 ++ .../src/services/data_service.rs | 4 +- .../src/services/data_type_service.rs | 4 +- .../src/services/driver_service.rs | 4 +- .../src/services/location_service.rs | 4 +- .../src/services/node_service.rs | 4 +- .../src/services/run_service.rs | 8 +-- .../src/services/system_service.rs | 4 +- 22 files changed, 243 insertions(+), 81 deletions(-) create mode 100644 scylla-server-rust/src/controllers/car_command_controller.rs create mode 100644 scylla-server-rust/src/proto/command_data.proto diff --git a/scylla-server-rust/.gitignore b/scylla-server-rust/.gitignore index 11d3f83a..a9a01cca 100644 --- a/scylla-server-rust/.gitignore +++ b/scylla-server-rust/.gitignore @@ -22,3 +22,4 @@ prisma.rs # protobuf serverdata.rs +command_data.rs diff --git a/scylla-server-rust/Cargo.lock b/scylla-server-rust/Cargo.lock index 51f2f9df..432320e4 100755 --- a/scylla-server-rust/Cargo.lock +++ b/scylla-server-rust/Cargo.lock @@ -288,6 +288,29 @@ dependencies = [ "tracing", ] +[[package]] +name = "axum-extra" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0be6ea09c9b96cb5076af0de2e383bd2bc0c18f827cf1967bdd353e0b910d733" +dependencies = [ + "axum 0.7.5", + "axum-core 0.4.3", + "bytes", + "futures-util", + "http 1.1.0", + "http-body 1.0.0", + "http-body-util", + "mime", + "pin-project-lite", + "serde", + "serde_html_form", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "backtrace" version = "0.3.73" @@ -3201,6 +3224,7 @@ name = "scylla-server-rust" version = "0.0.1" dependencies = [ "axum 0.7.5", + "axum-extra", "clap", "console-subscriber", "prisma-client-rust", @@ -3272,6 +3296,19 @@ dependencies = [ "syn 2.0.67", ] +[[package]] +name = "serde_html_form" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de514ef58196f1fc96dcaef80fe6170a1ce6215df9687a93fe8300e773fefc5" +dependencies = [ + "form_urlencoded", + "indexmap 2.2.6", + "itoa", + "ryu", + "serde", +] + [[package]] name = "serde_json" version = "1.0.117" diff --git a/scylla-server-rust/Cargo.toml b/scylla-server-rust/Cargo.toml index f7d11bd8..c5ec9fd2 100644 --- a/scylla-server-rust/Cargo.toml +++ b/scylla-server-rust/Cargo.toml @@ -22,6 +22,7 @@ rand = "0.8.5" console-subscriber = { version = "0.3.0", optional = true } ringbuffer = "0.15.0" clap = { version = "4.5.11", features = ["derive", "env"] } +axum-extra = { version = "0.9.3", features = ["query"] } [features] top = ["dep:console-subscriber"] diff --git a/scylla-server-rust/build.rs b/scylla-server-rust/build.rs index a57e70ec..44b9825e 100644 --- a/scylla-server-rust/build.rs +++ b/scylla-server-rust/build.rs @@ -8,6 +8,7 @@ fn main() { .includes(["src/proto"]) // Inputs must reside in some of include paths. .input("src/proto/serverdata.proto") + .input("src/proto/command_data.proto") // Specify output directory relative to Cargo output directory. .out_dir("src") .run_from_script(); diff --git a/scylla-server-rust/src/controllers/car_command_controller.rs b/scylla-server-rust/src/controllers/car_command_controller.rs new file mode 100644 index 00000000..d36ec09a --- /dev/null +++ b/scylla-server-rust/src/controllers/car_command_controller.rs @@ -0,0 +1,70 @@ +use std::sync::Arc; + +use axum::{extract::Path, Extension}; +use axum_extra::extract::Query; +use protobuf::Message; +use rumqttc::v5::AsyncClient; +use serde::Deserialize; +use tracing::{info, warn}; + +use crate::{command_data::CommandData, error::ScyllaError}; + +/// the prefix for the calypso topic, so topic of cmd is this plus the key appended on +pub const CALYPSO_BIDIR_CMD_PREFIX: &str = "Calypso/Bidir/Command/"; + +#[derive(Deserialize, Debug)] +pub struct ConfigRequest { + pub data: Option>, +} + +/// Sends a configuration to the car over MQTT +/// * `key` - The key of the configuration, as defined in the cangen YAML +/// * `data_query` - The data of the configuration, a URI query list of data=. If empty or too short, filled with cangen YAMl defaults +/// * `client` - The MQTT client to be used to send the data +/// +/// More info: This follows the specification of sending a command_data object over siren to topic CALYPSO_BIDIR_CMD_PREFIX/ +pub async fn send_config_command( + Path(key): Path, + Query(data_query): Query, + Extension(client): Extension>>, +) -> Result<(), ScyllaError> { + info!( + "Sending car config with key: {}, and values: {:?}", + key, data_query.data + ); + // disable scylla if not prod, as there will be None mqtt client + let Some(client) = client else { + warn!("Cannot use config endpoint in dev mode!"); + return Ok(()) + }; + + // the protobuf calypso converts into CAN + let mut payload = CommandData::new(); + // empty "data" in the protobuf tells calypso to use the default value + if let Some(data) = data_query.data { + payload.data = data; + } + let Ok(bytes) = payload.write_to_bytes() else { + return Err(ScyllaError::InvalidEncoding( + "Payload could not be written!".to_string(), + )); + }; + + // publish the message to the topic that calypso's encoder is susbcribed to + if let Err(err) = client + .publish( + format!("{}{}", CALYPSO_BIDIR_CMD_PREFIX, key), + rumqttc::v5::mqttbytes::QoS::ExactlyOnce, + false, + bytes, + ) + .await + { + warn!("Could not publish instruction: {}", err); + return Err(ScyllaError::CommFailure( + "Siren publish for instruction failed!".to_string(), + )); + } + + Ok(()) +} diff --git a/scylla-server-rust/src/controllers/mod.rs b/scylla-server-rust/src/controllers/mod.rs index 079a3dab..cf82adb2 100644 --- a/scylla-server-rust/src/controllers/mod.rs +++ b/scylla-server-rust/src/controllers/mod.rs @@ -1,3 +1,4 @@ +pub mod car_command_controller; pub mod data_controller; pub mod data_type_controller; pub mod driver_controller; diff --git a/scylla-server-rust/src/controllers/run_controller.rs b/scylla-server-rust/src/controllers/run_controller.rs index 984f6db2..528c4405 100644 --- a/scylla-server-rust/src/controllers/run_controller.rs +++ b/scylla-server-rust/src/controllers/run_controller.rs @@ -25,7 +25,7 @@ pub async fn get_run_by_id( let run_data = run_service::get_run_by_id(&db, run_id).await?; if run_data.is_none() { - return Err(ScyllaError::NotFound); + return Err(ScyllaError::EmptyResult); } let run_data_safe = run_data.unwrap(); diff --git a/scylla-server-rust/src/error.rs b/scylla-server-rust/src/error.rs index 221b4f13..8b7563ad 100644 --- a/scylla-server-rust/src/error.rs +++ b/scylla-server-rust/src/error.rs @@ -9,31 +9,48 @@ use prisma_client_rust::{ use tracing::warn; pub enum ScyllaError { + /// Any prisma query which errors out PrismaError(QueryError), - NotFound, + /// An instruction was not encodable + InvalidEncoding(String), + /// Could not communicate to car + CommFailure(String), + /// A query turned up empty that should not have + EmptyResult, } impl From for ScyllaError { fn from(error: QueryError) -> Self { - warn!("Query error: {:?}", error); - match error { - e if e.is_prisma_error::() => ScyllaError::NotFound, - e => ScyllaError::PrismaError(e), - } + ScyllaError::PrismaError(error) } } // This centralizes all different errors from our app in one place impl IntoResponse for ScyllaError { fn into_response(self) -> Response { - let status = match self { - ScyllaError::PrismaError(error) if error.is_prisma_error::() => { - StatusCode::CONFLICT - } - ScyllaError::PrismaError(_) => StatusCode::BAD_REQUEST, - ScyllaError::NotFound => StatusCode::NOT_FOUND, + let (status, reason) = match self { + ScyllaError::PrismaError(error) if error.is_prisma_error::() => ( + StatusCode::CONFLICT, + format!("Unique Key Violation: {}", error), + ), + ScyllaError::PrismaError(error) if error.is_prisma_error::() => ( + StatusCode::NOT_FOUND, + format!("Record Not Found: {}", error), + ), + ScyllaError::PrismaError(error) => ( + StatusCode::BAD_REQUEST, + format!("Misc query error: {}", error), + ), + ScyllaError::InvalidEncoding(reason) => (StatusCode::UNPROCESSABLE_ENTITY, reason), + ScyllaError::CommFailure(reason) => (StatusCode::BAD_GATEWAY, reason), + ScyllaError::EmptyResult => ( + StatusCode::NOT_FOUND, + "Fetched an empty result that should not be!".to_string(), + ), }; - status.into_response() + warn!("Routing error: {}: {}", status, reason); + + (status, reason).into_response() } } diff --git a/scylla-server-rust/src/lib.rs b/scylla-server-rust/src/lib.rs index d0b47a4c..f3bd395c 100644 --- a/scylla-server-rust/src/lib.rs +++ b/scylla-server-rust/src/lib.rs @@ -8,6 +8,7 @@ pub mod transformers; #[allow(warnings)] pub mod prisma; +pub mod command_data; pub mod serverdata; /// The type descriptor of the database passed to the middlelayer through axum state diff --git a/scylla-server-rust/src/main.rs b/scylla-server-rust/src/main.rs index 5453cd0c..71c53d49 100755 --- a/scylla-server-rust/src/main.rs +++ b/scylla-server-rust/src/main.rs @@ -7,9 +7,12 @@ use axum::{ }; use clap::Parser; use prisma_client_rust::chrono; +use rumqttc::v5::AsyncClient; use scylla_server_rust::{ controllers::{ - self, data_type_controller, driver_controller, location_controller, node_controller, + self, + car_command_controller::{self}, + data_type_controller, driver_controller, location_controller, node_controller, run_controller, system_controller, }, prisma::PrismaClient, @@ -55,7 +58,12 @@ struct ScyllaArgs { siren_host_url: String, /// The time, in seconds between collection for a batch upsert - #[arg(short = 't', long, env = "SCYLLA_BATCH_UPSERT_TIME", default_value = "10")] + #[arg( + short = 't', + long, + env = "SCYLLA_BATCH_UPSERT_TIME", + default_value = "10" + )] batch_upsert_time: u64, } @@ -138,11 +146,12 @@ async fn main() { token.clone(), )); - // if PROD_SCYLLA=false - if !cli.prod { + // if PROD_SCYLLA=false, also procur a client for use in the config state + let client: Option> = if !cli.prod { info!("Running processor in mock mode, no data will be stored"); let recv = MockProcessor::new(io); tokio::spawn(recv.generate_mock()); + None } else { // creates the initial run let curr_run = run_service::create_run(&db, chrono::offset::Utc::now().timestamp_millis()) @@ -153,15 +162,18 @@ async fn main() { // run prod if this isnt present // create and spawn the mqtt processor info!("Running processor in MQTT (production) mode"); - let recv = MqttProcessor::new( + let (recv, opts) = MqttProcessor::new( mqtt_send, new_run_receive, cli.siren_host_url, curr_run.id, io, ); - tokio::spawn(recv.process_mqtt()); - } + let (client, eventloop) = AsyncClient::new(opts, 600); + let client_sharable: Arc = Arc::new(client); + tokio::spawn(recv.process_mqtt(client_sharable.clone(), eventloop)); + Some(client_sharable) + }; let app = Router::new() // DATA ROUTES @@ -186,6 +198,11 @@ async fn main() { ) // SYSTEMS .route("/systems", get(system_controller::get_all_systems)) + // CONFIG + .route( + "/config/set/:configKey", + post(car_command_controller::send_config_command).layer(Extension(client)), + ) // for CORS handling .layer( CorsLayer::new() diff --git a/scylla-server-rust/src/mod.rs b/scylla-server-rust/src/mod.rs index 31b78fdf..b7017c61 100644 --- a/scylla-server-rust/src/mod.rs +++ b/scylla-server-rust/src/mod.rs @@ -1,3 +1,4 @@ // @generated +pub mod command_data; pub mod serverdata; diff --git a/scylla-server-rust/src/processors/db_handler.rs b/scylla-server-rust/src/processors/db_handler.rs index c65f09b9..6541d24d 100644 --- a/scylla-server-rust/src/processors/db_handler.rs +++ b/scylla-server-rust/src/processors/db_handler.rs @@ -112,7 +112,7 @@ impl DbHandler { /// It uses the queue from data queue to insert to the database specified /// On cancellation, will await one final queue message to cleanup anything remaining in the channel pub async fn batching_loop( - mut data_queue: Receiver>, + mut batch_queue: Receiver>, database: Database, saturate_batches: bool, cancel_token: CancellationToken, @@ -120,22 +120,18 @@ impl DbHandler { loop { tokio::select! { _ = cancel_token.cancelled() => { - loop { - info!("{} batches remaining!", data_queue.len()); - if let Some(final_msgs) = data_queue.recv().await { - info!( - "A cleanup batch uploaded: {:?}", - data_service::add_many(&database, final_msgs).await - ); - } else { - info!("No more messages to cleanup."); - break; + // cleanup all remaining messages if batches start backing up + while let Some(final_msgs) = batch_queue.recv().await { + info!("{} batches remaining!", batch_queue.len()+1); + info!( + "A cleanup batch uploaded: {:?}", + data_service::add_many(&database, final_msgs).await + ); } - - } - break; + info!("No more messages to cleanup."); + break; }, - Some(msgs) = data_queue.recv() => { + Some(msgs) = batch_queue.recv() => { if saturate_batches { let shared_db = database.clone(); tokio::spawn(async move { @@ -146,8 +142,8 @@ impl DbHandler { } debug!( "DB send: {} of {}", - data_queue.len(), - data_queue.max_capacity() + batch_queue.len(), + batch_queue.max_capacity() ); } } diff --git a/scylla-server-rust/src/processors/mod.rs b/scylla-server-rust/src/processors/mod.rs index e28de9c8..afb3bbef 100644 --- a/scylla-server-rust/src/processors/mod.rs +++ b/scylla-server-rust/src/processors/mod.rs @@ -7,7 +7,7 @@ pub mod mqtt_processor; /// This has the dual purposes of /// * - representing the packet sent over the socket for live data /// * - representing the struct for the service layer to unpack for insertion -/// Note: node name is only considered for database storage and convenience, it is not serialized in a socket packet +/// Note: node name is only considered for database storage and convenience, it is not serialized in a socket packet #[derive(serde::Serialize, Clone, Debug)] pub struct ClientData { pub run_id: i32, diff --git a/scylla-server-rust/src/processors/mqtt_processor.rs b/scylla-server-rust/src/processors/mqtt_processor.rs index 7f34c0a6..df2d214c 100644 --- a/scylla-server-rust/src/processors/mqtt_processor.rs +++ b/scylla-server-rust/src/processors/mqtt_processor.rs @@ -1,18 +1,21 @@ use core::fmt; -use std::time::Duration; +use std::{sync::Arc, time::Duration}; use prisma_client_rust::{bigdecimal::ToPrimitive, chrono, serde_json}; use protobuf::Message; use ringbuffer::RingBuffer; use rumqttc::v5::{ mqttbytes::v5::{LastWill, Packet, Publish}, - AsyncClient, Event, MqttOptions, + AsyncClient, Event, EventLoop, MqttOptions, }; use socketioxide::SocketIo; use tokio::sync::mpsc::{Receiver, Sender}; use tracing::{debug, instrument, trace, warn, Level}; -use crate::{serverdata, services::run_service}; +use crate::{ + controllers::car_command_controller::CALYPSO_BIDIR_CMD_PREFIX, serverdata, + services::run_service, +}; use super::ClientData; use std::borrow::Cow; @@ -22,7 +25,6 @@ pub struct MqttProcessor { new_run_channel: Receiver, curr_run: i32, io: SocketIo, - mqtt_ops: MqttOptions, } impl MqttProcessor { @@ -32,16 +34,16 @@ impl MqttProcessor { /// * `db` - The database to store the data in /// * `io` - The socketio layer to send the data to /// - /// Returns the instance and the event loop, which can be passed into the process_mqtt func to begin recieiving + /// Returns the instance and options to create a client, which is then used in the process_mqtt loop pub fn new( channel: Sender, new_run_channel: Receiver, mqtt_path: String, initial_run: i32, io: SocketIo, - ) -> MqttProcessor { + ) -> (MqttProcessor, MqttOptions) { // create the mqtt client and configure it - let mut create_opts = MqttOptions::new( + let mut mqtt_opts = MqttOptions::new( "ScyllaServer", mqtt_path.split_once(':').expect("Invalid Siren URL").0, mqtt_path @@ -51,7 +53,7 @@ impl MqttProcessor { .parse::() .expect("Invalid Siren port"), ); - create_opts + mqtt_opts .set_last_will(LastWill::new( "Scylla/Status", "Scylla has disconnected!", @@ -65,27 +67,28 @@ impl MqttProcessor { .set_session_expiry_interval(Some(u32::MAX)) .set_topic_alias_max(Some(600)); - MqttProcessor { - channel, - new_run_channel, - curr_run: initial_run, - io, - mqtt_ops: create_opts, - } + // TODO mess with incoming message cap if db, etc. cannot keep up + + ( + MqttProcessor { + channel, + new_run_channel, + curr_run: initial_run, + io, + }, + mqtt_opts, + ) } /// This handles the reception of mqtt messages, will not return - /// * `connect` - The eventloop returned by ::new to connect to - pub async fn process_mqtt(mut self) { + /// * `eventloop` - The eventloop returned by ::new to connect to. The loop isnt sync so this is the best that can be done + /// * `client` - The async mqttt v5 client to use for subscriptions + pub async fn process_mqtt(mut self, client: Arc, mut eventloop: EventLoop) { let mut view_interval = tokio::time::interval(Duration::from_secs(3)); let mut latency_interval = tokio::time::interval(Duration::from_millis(250)); let mut latency_ringbuffer = ringbuffer::AllocRingBuffer::::new(20); - // process over messages, non blocking - // TODO mess with incoming message cap if db, etc. cannot keep up - let (client, mut connect) = AsyncClient::new(self.mqtt_ops.clone(), 600); - debug!("Subscribing to siren"); client .subscribe("#", rumqttc::v5::mqttbytes::QoS::ExactlyOnce) @@ -95,7 +98,7 @@ impl MqttProcessor { loop { #[rustfmt::skip] // rust cannot format this macro for some reason tokio::select! { - msg = connect.poll() => match msg { + msg = eventloop.poll() => match msg { Ok(Event::Incoming(Packet::Publish(msg))) => { trace!("Received mqtt message: {:?}", msg); // parse the message into the data and the node name it falls under @@ -161,14 +164,23 @@ impl MqttProcessor { /// returns the ClientData, or the Err of something that can be debug printed #[instrument(skip(self), level = Level::TRACE)] fn parse_msg(&self, msg: Publish) -> Result { - let data = serverdata::ServerData::parse_from_bytes(&msg.payload) - .map_err(|f| format!("Could not parse message topic:{:?} error: {}", msg.topic, f))?; + let topic = std::str::from_utf8(&msg.topic) + .map_err(|f| format!("Could not parse topic: {}, topic: {:?}", f, msg.topic))?; + + // ignore command messages, less confusing in logs than just failing to decode protobuf + if topic.starts_with(CALYPSO_BIDIR_CMD_PREFIX) { + return Err(format!("Skipping command message: {}", topic)); + } - let split = std::str::from_utf8(&msg.topic) - .map_err(|f| format!("Could not parse topic: {}, topic: {:?}", f, msg.topic))? + let split = topic .split_once('/') .ok_or(&format!("Could not parse nesting: {:?}", msg.topic))?; + // look at data after topic as if we dont have a topic the protobuf is useless anyways + let data = serverdata::ServerData::parse_from_bytes(&msg.payload) + .map_err(|f| format!("Could not parse message topic:{:?} error: {}", msg.topic, f))?; + + // get the node and datatype from the topic extracted at the beginning let node = split.0; let data_type = split.1.replace('/', "-"); diff --git a/scylla-server-rust/src/proto/command_data.proto b/scylla-server-rust/src/proto/command_data.proto new file mode 100644 index 00000000..fe6b7744 --- /dev/null +++ b/scylla-server-rust/src/proto/command_data.proto @@ -0,0 +1,7 @@ +syntax = "proto3"; + +package command_data.v1; + +message CommandData { + repeated float data = 1; +} diff --git a/scylla-server-rust/src/services/data_service.rs b/scylla-server-rust/src/services/data_service.rs index d3a65be1..729c9f43 100644 --- a/scylla-server-rust/src/services/data_service.rs +++ b/scylla-server-rust/src/services/data_service.rs @@ -13,7 +13,7 @@ prisma::data::select! {public_data { /// * `run_id` - The run id to filter the data /// * `fetch_run` whether to fetch the run assocaited with this data /// * `fetch_data_type` whether to fetch the data type associated with this data -/// returns: A result containing the data or the QueryError propogated by the db +/// returns: A result containing the data or the QueryError propogated by the db pub async fn get_data( db: &Database, data_type_name: String, @@ -35,7 +35,7 @@ pub async fn get_data( /// * `unix_time` - The time im miliseconds since unix epoch of the message /// * `data_type_name` - The name of the data type, note this data type must already exist! /// * `rin_id` - The run id to assign the data point to, note this run must already exist! -/// returns: A result containing the data or the QueryError propogated by the db +/// returns: A result containing the data or the QueryError propogated by the db pub async fn add_data( db: &Database, client_data: ClientData, diff --git a/scylla-server-rust/src/services/data_type_service.rs b/scylla-server-rust/src/services/data_type_service.rs index 8e1aabb5..2cf87278 100644 --- a/scylla-server-rust/src/services/data_type_service.rs +++ b/scylla-server-rust/src/services/data_type_service.rs @@ -9,7 +9,7 @@ prisma::data_type::select! {public_datatype { /// Gets all datatypes /// * `db` - The prisma client to make the call to -/// returns: A result containing the data or the QueryError propogated by the db +/// returns: A result containing the data or the QueryError propogated by the db pub async fn get_all_data_types(db: &Database) -> Result, QueryError> { db.data_type() .find_many(vec![]) @@ -23,7 +23,7 @@ pub async fn get_all_data_types(db: &Database) -> Result Result, QueryError> { db.driver() .find_many(vec![]) @@ -31,7 +31,7 @@ pub async fn get_all_drivers(db: &Database) -> Result, /// * `db` - The prisma client to make the call to /// * `driver_name` - The name of the driver to upsert /// * `run_id` - The id of the run to link to the driver, must already exist! -/// returns: A result containing the data or the QueryError propogated by the db +/// returns: A result containing the data or the QueryError propogated by the db pub async fn upsert_driver( db: &Database, driver_name: String, diff --git a/scylla-server-rust/src/services/location_service.rs b/scylla-server-rust/src/services/location_service.rs index 04cf5796..717045ba 100644 --- a/scylla-server-rust/src/services/location_service.rs +++ b/scylla-server-rust/src/services/location_service.rs @@ -21,7 +21,7 @@ prisma::location::select! {public_location{ /// Gets all locations /// * `db` - The prisma client to make the call to -/// returns: A result containing the data or the QueryError propogated by the db +/// returns: A result containing the data or the QueryError propogated by the db pub async fn get_all_locations(db: &Database) -> Result, QueryError> { db.location() .find_many(vec![]) @@ -37,7 +37,7 @@ pub async fn get_all_locations(db: &Database) -> Result Result, QueryError> { db.node() .find_many(vec![]) @@ -24,7 +24,7 @@ pub async fn get_all_nodes(db: &Database) -> Result, Quer /// Upserts a node, either creating or updating one depending on its existence /// * `db` - The prisma client to make the call to /// * `node_name` - The name of the node linked to the data type -/// returns: A result containing the data or the QueryError propogated by the db +/// returns: A result containing the data or the QueryError propogated by the db pub async fn upsert_node( db: &Database, node_name: String, diff --git a/scylla-server-rust/src/services/run_service.rs b/scylla-server-rust/src/services/run_service.rs index 3e1581fa..5520851e 100644 --- a/scylla-server-rust/src/services/run_service.rs +++ b/scylla-server-rust/src/services/run_service.rs @@ -17,7 +17,7 @@ prisma::run::select! {public_run{ /// Gets all runs /// * `db` - The prisma client to make the call to -/// returns: A result containing the data or the QueryError propogated by the db +/// returns: A result containing the data or the QueryError propogated by the db pub async fn get_all_runs(db: &Database) -> Result, QueryError> { db.run() .find_many(vec![]) @@ -29,7 +29,7 @@ pub async fn get_all_runs(db: &Database) -> Result, QueryE /// Gets a single run by its id /// * `db` - The prisma client to make the call to /// * `run_id` - The id of the run to search for -/// returns: A result containing the data (or None if the `run_id` was not a valid run) or the QueryError propogated by the db +/// returns: A result containing the data (or None if the `run_id` was not a valid run) or the QueryError propogated by the db pub async fn get_run_by_id( db: &Database, run_id: i32, @@ -44,7 +44,7 @@ pub async fn get_run_by_id( /// Creates a run /// * `db` - The prisma client to make the call to /// * `timestamp` - The unix time since epoch in miliseconds when the run starts -/// returns: A result containing the data or the QueryError propogated by the db +/// returns: A result containing the data or the QueryError propogated by the db pub async fn create_run(db: &Database, timestamp: i64) -> Result { db.run() .create( @@ -62,7 +62,7 @@ pub async fn create_run(db: &Database, timestamp: i64) -> Result Result, QueryError> { db.system() .find_many(vec![]) @@ -34,7 +34,7 @@ pub async fn get_all_systems(db: &Database) -> Result, /// * `db` - The prisma client to make the call to /// * `system_name` - The system name name to upsert /// * `run_id` - The id of the run to link to the system, must already exist -/// returns: A result containing the data or the QueryError propogated by the db +/// returns: A result containing the data or the QueryError propogated by the db pub async fn upsert_system( db: &Database, system_name: String, From 3e63d0a05ad8d2e58924da79598a2aa84056e57d Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Sat, 24 Aug 2024 20:04:19 -0400 Subject: [PATCH 53/68] rename misleading script, fix some docs --- README.md | 8 +++----- scylla.sh => argos.sh | 0 router-up.sh | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) rename scylla.sh => argos.sh (100%) diff --git a/README.md b/README.md index 9cd7330d..8196bea7 100644 --- a/README.md +++ b/README.md @@ -9,17 +9,15 @@ To initialize the odyssey submodule run `git submodule update --init` Setup angular-client and scylla-server: [Angular Client](./angular-client/README.md)\ -[Scylla Server](./scylla-server/README.md) +[Scylla Server](./scylla-server-rust/README.md) -Once you've sucessfully setup Scylla and the Client, you can either run them separately, or run the following command to run them concurrently (the outputs from both will be output to the same terminal): - -`npm run start` +Once you've sucessfully setup Scylla and the Client, you can either run them separately, or follow the docker commands below to run them together. ## Production ### Quick start shortcuts -The `scylla.sh` can be used on POSIX compliant OSes to quickly get up and running without worrying about profiles. Its syntax is `./scylla.sh ` where profile could be router, tpu, scylla-dev, or client-dev and the cmd is the normal argument passed into docker compose, such as `up -d` to start the process and fork to background. **Make sure to `down` with the same profile that you used `up` with!** +The `argos.sh` can be used on POSIX compliant OSes to quickly get up and running without worrying about profiles. Its syntax is `./argos.sh ` where profile could be router, tpu, scylla-dev, or client-dev and the cmd is the normal argument passed into docker compose, such as `up -d` to start the process and fork to background. **Make sure to `down` with the same profile that you used `up` with!** ### Customizing and more info diff --git a/scylla.sh b/argos.sh similarity index 100% rename from scylla.sh rename to argos.sh diff --git a/router-up.sh b/router-up.sh index d682cb03..0b2a289b 100755 --- a/router-up.sh +++ b/router-up.sh @@ -1,3 +1,3 @@ #!/bin/sh -./scylla.sh router up -d +./argos.sh router up -d From 78ceca388087f54dc80fe1cbcc4f76f10eda6d84 Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Wed, 28 Aug 2024 16:14:33 -0400 Subject: [PATCH 54/68] add tokenized halt of mqtt processor to prevent race conditions --- compose.yml | 2 +- .../src/controllers/car_command_controller.rs | 2 +- scylla-server-rust/src/main.rs | 3 ++- .../src/processors/mqtt_processor.rs | 14 +++++++++++--- 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/compose.yml b/compose.yml index 02e9fb70..95b2a163 100644 --- a/compose.yml +++ b/compose.yml @@ -29,7 +29,7 @@ services: - SOURCE_DATABASE_URL=postgresql://postgres:password@odyssey-timescale:5432/postgres # - PROD_SIREN_HOST_URL=siren:1883 - SCYLLA_PROD=true - - RUST_LOG=none,scylla_server_rust=info # default log setting for docker + - RUST_LOG=warn,scylla_server_rust=info # default log setting for docker cpu_shares: 1024 mem_limit: 2gb stop_grace_period: 10s diff --git a/scylla-server-rust/src/controllers/car_command_controller.rs b/scylla-server-rust/src/controllers/car_command_controller.rs index d36ec09a..ad3d12c4 100644 --- a/scylla-server-rust/src/controllers/car_command_controller.rs +++ b/scylla-server-rust/src/controllers/car_command_controller.rs @@ -35,7 +35,7 @@ pub async fn send_config_command( // disable scylla if not prod, as there will be None mqtt client let Some(client) = client else { warn!("Cannot use config endpoint in dev mode!"); - return Ok(()) + return Ok(()); }; // the protobuf calypso converts into CAN diff --git a/scylla-server-rust/src/main.rs b/scylla-server-rust/src/main.rs index 71c53d49..82d3afbf 100755 --- a/scylla-server-rust/src/main.rs +++ b/scylla-server-rust/src/main.rs @@ -168,10 +168,11 @@ async fn main() { cli.siren_host_url, curr_run.id, io, + token.clone(), ); let (client, eventloop) = AsyncClient::new(opts, 600); let client_sharable: Arc = Arc::new(client); - tokio::spawn(recv.process_mqtt(client_sharable.clone(), eventloop)); + task_tracker.spawn(recv.process_mqtt(client_sharable.clone(), eventloop)); Some(client_sharable) }; diff --git a/scylla-server-rust/src/processors/mqtt_processor.rs b/scylla-server-rust/src/processors/mqtt_processor.rs index df2d214c..608d9cdc 100644 --- a/scylla-server-rust/src/processors/mqtt_processor.rs +++ b/scylla-server-rust/src/processors/mqtt_processor.rs @@ -10,6 +10,7 @@ use rumqttc::v5::{ }; use socketioxide::SocketIo; use tokio::sync::mpsc::{Receiver, Sender}; +use tokio_util::sync::CancellationToken; use tracing::{debug, instrument, trace, warn, Level}; use crate::{ @@ -25,6 +26,7 @@ pub struct MqttProcessor { new_run_channel: Receiver, curr_run: i32, io: SocketIo, + cancel_token: CancellationToken, } impl MqttProcessor { @@ -33,14 +35,15 @@ impl MqttProcessor { /// * `mqtt_path` - The mqtt URI, including port, (without the mqtt://) to subscribe to /// * `db` - The database to store the data in /// * `io` - The socketio layer to send the data to - /// - /// Returns the instance and options to create a client, which is then used in the process_mqtt loop + /// * `cancel_token` - The token which indicates cancellation of the task + /// Returns the instance and options to create a client, which is then used in the process_mqtt loop pub fn new( channel: Sender, new_run_channel: Receiver, mqtt_path: String, initial_run: i32, io: SocketIo, + cancel_token: CancellationToken, ) -> (MqttProcessor, MqttOptions) { // create the mqtt client and configure it let mut mqtt_opts = MqttOptions::new( @@ -75,6 +78,7 @@ impl MqttProcessor { new_run_channel, curr_run: initial_run, io, + cancel_token, }, mqtt_opts, ) @@ -98,6 +102,10 @@ impl MqttProcessor { loop { #[rustfmt::skip] // rust cannot format this macro for some reason tokio::select! { + _ = self.cancel_token.cancelled() => { + debug!("Shutting down MQTT processor!"); + break; + }, msg = eventloop.poll() => match msg { Ok(Event::Incoming(Packet::Publish(msg))) => { trace!("Received mqtt message: {:?}", msg); @@ -235,7 +243,7 @@ impl MqttProcessor { /// * `client_data` - The client data to send over the broadcast async fn send_db_msg(&self, client_data: ClientData) { if let Err(err) = self.channel.send(client_data.clone()).await { - warn!("Error sending through channel: {:?}", err) + warn!("Error sending through channel: {:?}", err); } } From 4aaa5ee7f2c445bc6e0bc0116658e7d4c18b0060 Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Wed, 28 Aug 2024 16:36:42 -0400 Subject: [PATCH 55/68] ignore yaml --- .prettierignore | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.prettierignore b/.prettierignore index ac9b8cc4..b1480eed 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,8 +1,9 @@ .github .vscode -docker-compose.yml -docker-compose-dev.yml +# ignore docker, etc +*.yml + .eslintrc.js tsconfig.json From 6b2b1598eb72329ce691c29a92ba96dc4f5124dc Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Thu, 29 Aug 2024 21:25:10 -0400 Subject: [PATCH 56/68] add one missing doc --- scylla-server-rust/src/services/data_service.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scylla-server-rust/src/services/data_service.rs b/scylla-server-rust/src/services/data_service.rs index 729c9f43..2dd63617 100644 --- a/scylla-server-rust/src/services/data_service.rs +++ b/scylla-server-rust/src/services/data_service.rs @@ -60,6 +60,11 @@ pub async fn add_data( .await } + +/// Adds many datapoints via a batch insert +/// * `db` - The prisma client to make the call to +/// * `client_data` - A list of data to batch insert +/// returns: A result containing the number of rows inserted or the QueryError propogated by the db pub async fn add_many(db: &Database, client_data: Vec) -> Result { db.data() .create_many( From f126c48860d4fec22d96256b04a76d3c4a4afa44 Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Thu, 29 Aug 2024 21:29:45 -0400 Subject: [PATCH 57/68] update compose --- angular-client/compose.client.yml | 1 - compose.yml | 6 ++---- siren-base/compose.siren.yml | 1 - 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/angular-client/compose.client.yml b/angular-client/compose.client.yml index e1a44d50..eee5f658 100644 --- a/angular-client/compose.client.yml +++ b/angular-client/compose.client.yml @@ -14,4 +14,3 @@ services: ports: - 80:80 cpu_shares: 512 - mem_limit: 1gb diff --git a/compose.yml b/compose.yml index 95b2a163..be21cf5b 100644 --- a/compose.yml +++ b/compose.yml @@ -12,8 +12,7 @@ services: volumes: - db-data:/var/lib/postgresql/data cpu_shares: 1024 - mem_limit: 3gb - stop_grace_period: 1m + stop_grace_period: 2m scylla-server-rust: container_name: scylla-server-rust @@ -31,8 +30,7 @@ services: - SCYLLA_PROD=true - RUST_LOG=warn,scylla_server_rust=info # default log setting for docker cpu_shares: 1024 - mem_limit: 2gb - stop_grace_period: 10s + stop_grace_period: 2m stop_signal: SIGINT init: true entrypoint: ["./docker_run.sh"] diff --git a/siren-base/compose.siren.yml b/siren-base/compose.siren.yml index 3fde4fda..69013f70 100644 --- a/siren-base/compose.siren.yml +++ b/siren-base/compose.siren.yml @@ -11,5 +11,4 @@ services: volumes: - ./mosquitto/mosquitto.conf:/mosquitto/config/mosquitto.conf cpu_shares: 2048 - mem_limit: 2gb oom_kill_disable: true From 876239e8b4f9e4dbffad8bfb8cb30b928bdc032a Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Sun, 1 Sep 2024 16:48:41 -0400 Subject: [PATCH 58/68] fix typo --- scylla-server-rust/src/processors/mqtt_processor.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scylla-server-rust/src/processors/mqtt_processor.rs b/scylla-server-rust/src/processors/mqtt_processor.rs index 608d9cdc..cca16e67 100644 --- a/scylla-server-rust/src/processors/mqtt_processor.rs +++ b/scylla-server-rust/src/processors/mqtt_processor.rs @@ -186,7 +186,7 @@ impl MqttProcessor { // look at data after topic as if we dont have a topic the protobuf is useless anyways let data = serverdata::ServerData::parse_from_bytes(&msg.payload) - .map_err(|f| format!("Could not parse message topic:{:?} error: {}", msg.topic, f))?; + .map_err(|f| format!("Could not parse message payload:{:?} error: {}", msg.topic, f))?; // get the node and datatype from the topic extracted at the beginning let node = split.0; From 6ec3c6aa7bccbfd6dfa189c67f2858a7af57cad0 Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Mon, 2 Sep 2024 14:22:59 -0400 Subject: [PATCH 59/68] flag to disable batch uploading --- scylla-server-rust/src/main.rs | 25 +++++++++++++------ .../src/processors/db_handler.rs | 18 +++++++++++++ .../src/processors/mqtt_processor.rs | 8 ++++-- .../src/services/data_service.rs | 1 - 4 files changed, 42 insertions(+), 10 deletions(-) diff --git a/scylla-server-rust/src/main.rs b/scylla-server-rust/src/main.rs index 82d3afbf..acc40159 100755 --- a/scylla-server-rust/src/main.rs +++ b/scylla-server-rust/src/main.rs @@ -45,6 +45,10 @@ struct ScyllaArgs { #[arg(short = 's', long, env = "SCYLLA_SATURATE_BATCH")] saturate_batch: bool, + /// Whether to disable batch data uploading (will not disable upsertion of special types) + #[arg(long, env = "SCYLLA_DATA_UPLOAD_DISABLE")] + disable_data_upload: bool, + // /// Whether to enable the socket io server in Scylla // #[arg(short, long, env = "SCYLLA_SOCKET")] // socket: bool, @@ -138,13 +142,20 @@ async fn main() { db_handler::DbHandler::new(mqtt_receive, Arc::clone(&db), cli.batch_upsert_time * 1000) .handling_loop(db_send, token.clone()), ); - // spawn the database inserter - task_tracker.spawn(db_handler::DbHandler::batching_loop( - db_receive, - Arc::clone(&db), - cli.saturate_batch, - token.clone(), - )); + // spawn the database inserter, if we have it enabled + if !cli.disable_data_upload { + task_tracker.spawn(db_handler::DbHandler::batching_loop( + db_receive, + Arc::clone(&db), + cli.saturate_batch, + token.clone(), + )); + } else { + task_tracker.spawn(db_handler::DbHandler::fake_batching_loop( + db_receive, + token.clone(), + )); + } // if PROD_SCYLLA=false, also procur a client for use in the config state let client: Option> = if !cli.prod { diff --git a/scylla-server-rust/src/processors/db_handler.rs b/scylla-server-rust/src/processors/db_handler.rs index 6541d24d..dedd9c59 100644 --- a/scylla-server-rust/src/processors/db_handler.rs +++ b/scylla-server-rust/src/processors/db_handler.rs @@ -150,6 +150,24 @@ impl DbHandler { } } + /// A batching loop that consumes messages but does not upload anything + pub async fn fake_batching_loop( + mut batch_queue: Receiver>, + cancel_token: CancellationToken, + ) { + loop { + tokio::select! { + _ = cancel_token.cancelled() => { + warn!("Cancelling fake upload with {} batches left in queue!", batch_queue.len()); + break; + }, + Some(msgs) = batch_queue.recv() => { + warn!("NOT UPLOADING {} MESSAGES", msgs.len()); + }, + } + } + } + #[instrument(level = Level::DEBUG, skip(msg))] async fn batch_upload(msg: Vec, db: &Database) { match data_service::add_many(db, msg).await { diff --git a/scylla-server-rust/src/processors/mqtt_processor.rs b/scylla-server-rust/src/processors/mqtt_processor.rs index cca16e67..6d62064c 100644 --- a/scylla-server-rust/src/processors/mqtt_processor.rs +++ b/scylla-server-rust/src/processors/mqtt_processor.rs @@ -185,8 +185,12 @@ impl MqttProcessor { .ok_or(&format!("Could not parse nesting: {:?}", msg.topic))?; // look at data after topic as if we dont have a topic the protobuf is useless anyways - let data = serverdata::ServerData::parse_from_bytes(&msg.payload) - .map_err(|f| format!("Could not parse message payload:{:?} error: {}", msg.topic, f))?; + let data = serverdata::ServerData::parse_from_bytes(&msg.payload).map_err(|f| { + format!( + "Could not parse message payload:{:?} error: {}", + msg.topic, f + ) + })?; // get the node and datatype from the topic extracted at the beginning let node = split.0; diff --git a/scylla-server-rust/src/services/data_service.rs b/scylla-server-rust/src/services/data_service.rs index 2dd63617..dd5c212e 100644 --- a/scylla-server-rust/src/services/data_service.rs +++ b/scylla-server-rust/src/services/data_service.rs @@ -60,7 +60,6 @@ pub async fn add_data( .await } - /// Adds many datapoints via a batch insert /// * `db` - The prisma client to make the call to /// * `client_data` - A list of data to batch insert From df5d9351f36227e43c5f6125c1f2db01025229af Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Mon, 2 Sep 2024 22:09:57 -0400 Subject: [PATCH 60/68] add simple socket discarding --- scylla-server-rust/src/main.rs | 10 ++++ .../src/processors/mqtt_processor.rs | 51 +++++++++++-------- 2 files changed, 41 insertions(+), 20 deletions(-) diff --git a/scylla-server-rust/src/main.rs b/scylla-server-rust/src/main.rs index acc40159..6df7b497 100755 --- a/scylla-server-rust/src/main.rs +++ b/scylla-server-rust/src/main.rs @@ -69,6 +69,15 @@ struct ScyllaArgs { default_value = "10" )] batch_upsert_time: u64, + + /// The + #[arg( + short = 'd', + long, + env = "SCYLLA_SOCKET_DISCARD_PERCENT", + default_value = "0" + )] + socketio_discard_percent: u8, } #[tokio::main] @@ -180,6 +189,7 @@ async fn main() { curr_run.id, io, token.clone(), + ((cli.socketio_discard_percent as f32 / 100.0) * 255.0) as u8, ); let (client, eventloop) = AsyncClient::new(opts, 600); let client_sharable: Arc = Arc::new(client); diff --git a/scylla-server-rust/src/processors/mqtt_processor.rs b/scylla-server-rust/src/processors/mqtt_processor.rs index 6d62064c..fd37ece2 100644 --- a/scylla-server-rust/src/processors/mqtt_processor.rs +++ b/scylla-server-rust/src/processors/mqtt_processor.rs @@ -27,6 +27,8 @@ pub struct MqttProcessor { curr_run: i32, io: SocketIo, cancel_token: CancellationToken, + /// Upload ratio, below is not uploaded above is uploaded + upload_ratio: u8, } impl MqttProcessor { @@ -44,6 +46,7 @@ impl MqttProcessor { initial_run: i32, io: SocketIo, cancel_token: CancellationToken, + upload_ratio: u8, ) -> (MqttProcessor, MqttOptions) { // create the mqtt client and configure it let mut mqtt_opts = MqttOptions::new( @@ -79,6 +82,7 @@ impl MqttProcessor { curr_run: initial_run, io, cancel_token, + upload_ratio, }, mqtt_opts, ) @@ -93,6 +97,8 @@ impl MqttProcessor { let mut latency_interval = tokio::time::interval(Duration::from_millis(250)); let mut latency_ringbuffer = ringbuffer::AllocRingBuffer::::new(20); + let mut upload_counter: u8 = 0; + debug!("Subscribing to siren"); client .subscribe("#", rumqttc::v5::mqttbytes::QoS::ExactlyOnce) @@ -119,7 +125,7 @@ impl MqttProcessor { }; latency_ringbuffer.push(chrono::offset::Utc::now().timestamp_millis() - msg.timestamp); self.send_db_msg(msg.clone()).await; - self.send_socket_msg(msg); + self.send_socket_msg(msg, &mut upload_counter); }, Err(msg) => trace!("Received mqtt error: {:?}", msg), _ => trace!("Received misc mqtt: {:?}", msg), @@ -135,7 +141,7 @@ impl MqttProcessor { timestamp: chrono::offset::Utc::now().timestamp_millis(), values: vec![sockets.len().to_string()] }; - self.send_socket_msg(client_data); + self.send_socket_msg(client_data, &mut upload_counter); } else { warn!("Could not fetch socket count"); } @@ -157,7 +163,7 @@ impl MqttProcessor { values: vec![avg_latency.to_string()] }; trace!("Latency update sending: {}", client_data.values.first().unwrap_or(&"n/a".to_string())); - self.send_socket_msg(client_data); + self.send_socket_msg(client_data, &mut upload_counter); } Some(new_run) = self.new_run_channel.recv() => { trace!("New run: {:?}", new_run); @@ -253,23 +259,28 @@ impl MqttProcessor { /// Sends a message to the socket, printing and IGNORING any error that may occur /// * `client_data` - The client data to send over the broadcast - fn send_socket_msg(&self, client_data: ClientData) { - match self.io.emit( - "message", - serde_json::to_string(&client_data).expect("Could not serialize ClientData"), - ) { - Ok(_) => (), - Err(err) => match err { - socketioxide::BroadcastError::Socket(e) => { - trace!("Socket: Transmit error: {:?}", e); - } - socketioxide::BroadcastError::Serialize(_) => { - warn!("Socket: Serialize error: {}", err) - } - socketioxide::BroadcastError::Adapter(_) => { - warn!("Socket: Adapter error: {}", err) - } - }, + fn send_socket_msg(&self, client_data: ClientData, upload_counter: &mut u8) { + *upload_counter = upload_counter.wrapping_add(1); + if *upload_counter >= self.upload_ratio { + match self.io.emit( + "message", + serde_json::to_string(&client_data).expect("Could not serialize ClientData"), + ) { + Ok(_) => (), + Err(err) => match err { + socketioxide::BroadcastError::Socket(e) => { + trace!("Socket: Transmit error: {:?}", e); + } + socketioxide::BroadcastError::Serialize(_) => { + warn!("Socket: Serialize error: {}", err) + } + socketioxide::BroadcastError::Adapter(_) => { + warn!("Socket: Adapter error: {}", err) + } + }, + } + } else { + trace!("Discarding message with topic {}", client_data.name); } } } From df5b8de989ca075f169b8b4587cc588551b30d72 Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Thu, 5 Sep 2024 19:27:27 -0400 Subject: [PATCH 61/68] fix docs --- scylla-server-rust/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scylla-server-rust/src/main.rs b/scylla-server-rust/src/main.rs index 6df7b497..f149a606 100755 --- a/scylla-server-rust/src/main.rs +++ b/scylla-server-rust/src/main.rs @@ -70,7 +70,7 @@ struct ScyllaArgs { )] batch_upsert_time: u64, - /// The + /// The percent of messages discarded when sent from the socket #[arg( short = 'd', long, From 7a3d239222c4555c9c59916601e02388fef3246c Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Sun, 8 Sep 2024 20:35:17 -0400 Subject: [PATCH 62/68] remove last will --- scylla-server-rust/src/processors/mqtt_processor.rs | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/scylla-server-rust/src/processors/mqtt_processor.rs b/scylla-server-rust/src/processors/mqtt_processor.rs index fd37ece2..08bc4ca4 100644 --- a/scylla-server-rust/src/processors/mqtt_processor.rs +++ b/scylla-server-rust/src/processors/mqtt_processor.rs @@ -5,7 +5,7 @@ use prisma_client_rust::{bigdecimal::ToPrimitive, chrono, serde_json}; use protobuf::Message; use ringbuffer::RingBuffer; use rumqttc::v5::{ - mqttbytes::v5::{LastWill, Packet, Publish}, + mqttbytes::v5::{Packet, Publish}, AsyncClient, Event, EventLoop, MqttOptions, }; use socketioxide::SocketIo; @@ -60,13 +60,6 @@ impl MqttProcessor { .expect("Invalid Siren port"), ); mqtt_opts - .set_last_will(LastWill::new( - "Scylla/Status", - "Scylla has disconnected!", - rumqttc::v5::mqttbytes::QoS::ExactlyOnce, - true, - None, - )) .set_keep_alive(Duration::from_secs(20)) .set_clean_start(false) .set_connection_timeout(3) From 15347e78578e12dd62192272a430370f65ff43ee Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Tue, 10 Sep 2024 12:59:01 -0400 Subject: [PATCH 63/68] add brick --- compose.brick.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 compose.brick.yml diff --git a/compose.brick.yml b/compose.brick.yml new file mode 100644 index 00000000..26c866df --- /dev/null +++ b/compose.brick.yml @@ -0,0 +1,7 @@ +services: + scylla-server-rust: + depends_on: + - siren + environment: + - SCYLLA_SIREN_HOST_URL=192.168.100.1:1883 + From 92563d5fb9926b91812f9919fe56b5cb9b74073e Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Tue, 10 Sep 2024 12:59:42 -0400 Subject: [PATCH 64/68] fix brick --- compose.brick.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/compose.brick.yml b/compose.brick.yml index 26c866df..b8d23fab 100644 --- a/compose.brick.yml +++ b/compose.brick.yml @@ -1,7 +1,5 @@ services: scylla-server-rust: - depends_on: - - siren environment: - SCYLLA_SIREN_HOST_URL=192.168.100.1:1883 From 6c184615e2d4fe14c7b5b252243eef4c1fc2291d Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Mon, 16 Sep 2024 18:35:46 -0400 Subject: [PATCH 65/68] rename scylla-server-rust to scylla-server --- .../{scylla-rust-ci.yml => scylla-ci.yml} | 4 +- ...scylla-rust-tests.yml => scylla-tests.yml} | 6 +- .gitignore | 4 +- .gitmodules | 4 +- README.md | 4 +- compose.brick.yml | 2 +- compose.client-dev.yml | 2 +- compose.router.yml | 2 +- compose.scylla-dev.yml | 4 +- compose.tpu.yml | 2 +- compose.yml | 8 +- scylla-server-rust/.dockerignore | 31 ------- scylla-server-rust/.gitignore | 25 ------ scylla-server-rust/Dockerfile | 8 -- scylla-server-rust/README.md | 80 ------------------- scylla-server-typescript/.dockerignore | 1 + scylla-server-typescript/.gitignore | 6 ++ scylla-server-typescript/Dockerfile | 26 ++++++ scylla-server-typescript/README.md | 39 +++++++++ .../package-lock.json | 0 .../package.json | 0 .../src/controllers/data.controller.ts | 0 .../src/controllers/datatype.controller.ts | 0 .../src/controllers/driver.controller.ts | 0 .../src/controllers/location.controller.ts | 0 .../src/controllers/node.controller.ts | 0 .../src/controllers/run.controller.ts | 0 .../src/controllers/system.controller.ts | 0 .../src/index.ts | 0 .../src/odyssey-base | 0 .../src/routes/data.routes.ts | 0 .../src/routes/datatype.routes.ts | 0 .../src/routes/driver.routes.ts | 0 .../src/routes/location.routes.ts | 0 .../src/routes/node.routes.ts | 0 .../src/routes/run.routes.ts | 0 .../src/routes/system.routes.ts | 0 .../src/socket/mock-proxy-client.ts | 0 .../src/socket/prod-proxy-client.ts | 0 .../src/socket/proxy-client.ts | 0 .../src/socket/proxy-server.ts | 0 .../src/utils/data.utils.ts | 0 .../src/utils/message.utils.ts | 0 .../tsconfig.json | 0 .../vite.config.js | 0 .../.cargo/config.toml | 0 scylla-server/.dockerignore | 32 +++++++- scylla-server/.gitignore | 27 ++++++- .../Cargo.lock | 2 +- .../Cargo.toml | 6 +- scylla-server/Dockerfile | 30 ++----- scylla-server/README.md | 79 +++++++++++++----- .../build.rs | 0 .../docker_run.sh | 0 .../integration_test.sh | 0 .../prisma-cli/Cargo.toml | 0 .../prisma-cli/src/main.rs | 0 .../20240212220042_init/migration.sql | 0 .../20240212220355_init/migration.sql | 0 .../prisma/migrations/migration_lock.toml | 0 .../prisma/schema.prisma | 0 .../prisma/seed.rs | 2 +- .../src/controllers/car_command_controller.rs | 0 .../src/controllers/data_controller.rs | 0 .../src/controllers/data_type_controller.rs | 0 .../src/controllers/driver_controller.rs | 0 .../src/controllers/location_controller.rs | 0 .../src/controllers/mod.rs | 0 .../src/controllers/node_controller.rs | 0 .../src/controllers/run_controller.rs | 0 .../src/controllers/system_controller.rs | 0 .../src/error.rs | 0 .../src/lib.rs | 0 .../src/main.rs | 2 +- .../src/mod.rs | 0 .../src/processors/db_handler.rs | 0 .../src/processors/mock_data.rs | 0 .../src/processors/mock_processor.rs | 0 .../src/processors/mod.rs | 0 .../src/processors/mqtt_processor.rs | 0 .../src/proto/command_data.proto | 0 .../src/proto/serverdata.proto | 0 .../src/services/data_service.rs | 0 .../src/services/data_type_service.rs | 0 .../src/services/driver_service.rs | 0 .../src/services/location_service.rs | 0 .../src/services/mod.rs | 0 .../src/services/node_service.rs | 0 .../src/services/run_service.rs | 0 .../src/services/system_service.rs | 0 .../src/transformers/data_transformer.rs | 0 .../src/transformers/data_type_transformer.rs | 0 .../src/transformers/driver_transformer.rs | 0 .../src/transformers/location_transformer.rs | 0 .../src/transformers/mod.rs | 0 .../src/transformers/node_transformer.rs | 0 .../src/transformers/run_transformer.rs | 0 .../src/transformers/system_transformer.rs | 0 .../tests/data_service_test.rs | 2 +- .../tests/data_type_service_test.rs | 2 +- .../tests/driver_service_test.rs | 2 +- .../tests/location_service_test.rs | 2 +- .../tests/node_service_test.rs | 2 +- .../tests/run_service_test.rs | 2 +- .../tests/system_service_test.rs | 2 +- .../tests/test_utils.rs | 2 +- 106 files changed, 227 insertions(+), 227 deletions(-) rename .github/workflows/{scylla-rust-ci.yml => scylla-ci.yml} (89%) rename .github/workflows/{scylla-rust-tests.yml => scylla-tests.yml} (84%) delete mode 100644 scylla-server-rust/.dockerignore delete mode 100644 scylla-server-rust/.gitignore delete mode 100755 scylla-server-rust/Dockerfile delete mode 100644 scylla-server-rust/README.md create mode 100644 scylla-server-typescript/.dockerignore create mode 100644 scylla-server-typescript/.gitignore create mode 100644 scylla-server-typescript/Dockerfile create mode 100644 scylla-server-typescript/README.md rename {scylla-server => scylla-server-typescript}/package-lock.json (100%) rename {scylla-server => scylla-server-typescript}/package.json (100%) rename {scylla-server => scylla-server-typescript}/src/controllers/data.controller.ts (100%) rename {scylla-server => scylla-server-typescript}/src/controllers/datatype.controller.ts (100%) rename {scylla-server => scylla-server-typescript}/src/controllers/driver.controller.ts (100%) rename {scylla-server => scylla-server-typescript}/src/controllers/location.controller.ts (100%) rename {scylla-server => scylla-server-typescript}/src/controllers/node.controller.ts (100%) rename {scylla-server => scylla-server-typescript}/src/controllers/run.controller.ts (100%) rename {scylla-server => scylla-server-typescript}/src/controllers/system.controller.ts (100%) rename {scylla-server => scylla-server-typescript}/src/index.ts (100%) rename {scylla-server => scylla-server-typescript}/src/odyssey-base (100%) rename {scylla-server => scylla-server-typescript}/src/routes/data.routes.ts (100%) rename {scylla-server => scylla-server-typescript}/src/routes/datatype.routes.ts (100%) rename {scylla-server => scylla-server-typescript}/src/routes/driver.routes.ts (100%) rename {scylla-server => scylla-server-typescript}/src/routes/location.routes.ts (100%) rename {scylla-server => scylla-server-typescript}/src/routes/node.routes.ts (100%) rename {scylla-server => scylla-server-typescript}/src/routes/run.routes.ts (100%) rename {scylla-server => scylla-server-typescript}/src/routes/system.routes.ts (100%) rename {scylla-server => scylla-server-typescript}/src/socket/mock-proxy-client.ts (100%) rename {scylla-server => scylla-server-typescript}/src/socket/prod-proxy-client.ts (100%) rename {scylla-server => scylla-server-typescript}/src/socket/proxy-client.ts (100%) rename {scylla-server => scylla-server-typescript}/src/socket/proxy-server.ts (100%) rename {scylla-server => scylla-server-typescript}/src/utils/data.utils.ts (100%) rename {scylla-server => scylla-server-typescript}/src/utils/message.utils.ts (100%) rename {scylla-server => scylla-server-typescript}/tsconfig.json (100%) rename {scylla-server => scylla-server-typescript}/vite.config.js (100%) rename {scylla-server-rust => scylla-server}/.cargo/config.toml (100%) rename {scylla-server-rust => scylla-server}/Cargo.lock (99%) rename {scylla-server-rust => scylla-server}/Cargo.toml (93%) mode change 100644 => 100755 scylla-server/Dockerfile rename {scylla-server-rust => scylla-server}/build.rs (100%) rename {scylla-server-rust => scylla-server}/docker_run.sh (100%) rename {scylla-server-rust => scylla-server}/integration_test.sh (100%) rename {scylla-server-rust => scylla-server}/prisma-cli/Cargo.toml (100%) rename {scylla-server-rust => scylla-server}/prisma-cli/src/main.rs (100%) rename {scylla-server-rust => scylla-server}/prisma/migrations/20240212220042_init/migration.sql (100%) rename {scylla-server-rust => scylla-server}/prisma/migrations/20240212220355_init/migration.sql (100%) rename {scylla-server-rust => scylla-server}/prisma/migrations/migration_lock.toml (100%) rename {scylla-server-rust => scylla-server}/prisma/schema.prisma (100%) rename {scylla-server-rust => scylla-server}/prisma/seed.rs (99%) rename {scylla-server-rust => scylla-server}/src/controllers/car_command_controller.rs (100%) rename {scylla-server-rust => scylla-server}/src/controllers/data_controller.rs (100%) rename {scylla-server-rust => scylla-server}/src/controllers/data_type_controller.rs (100%) rename {scylla-server-rust => scylla-server}/src/controllers/driver_controller.rs (100%) rename {scylla-server-rust => scylla-server}/src/controllers/location_controller.rs (100%) rename {scylla-server-rust => scylla-server}/src/controllers/mod.rs (100%) rename {scylla-server-rust => scylla-server}/src/controllers/node_controller.rs (100%) rename {scylla-server-rust => scylla-server}/src/controllers/run_controller.rs (100%) rename {scylla-server-rust => scylla-server}/src/controllers/system_controller.rs (100%) rename {scylla-server-rust => scylla-server}/src/error.rs (100%) rename {scylla-server-rust => scylla-server}/src/lib.rs (100%) rename {scylla-server-rust => scylla-server}/src/main.rs (99%) rename {scylla-server-rust => scylla-server}/src/mod.rs (100%) rename {scylla-server-rust => scylla-server}/src/processors/db_handler.rs (100%) rename {scylla-server-rust => scylla-server}/src/processors/mock_data.rs (100%) rename {scylla-server-rust => scylla-server}/src/processors/mock_processor.rs (100%) rename {scylla-server-rust => scylla-server}/src/processors/mod.rs (100%) rename {scylla-server-rust => scylla-server}/src/processors/mqtt_processor.rs (100%) rename {scylla-server-rust => scylla-server}/src/proto/command_data.proto (100%) rename {scylla-server-rust => scylla-server}/src/proto/serverdata.proto (100%) rename {scylla-server-rust => scylla-server}/src/services/data_service.rs (100%) rename {scylla-server-rust => scylla-server}/src/services/data_type_service.rs (100%) rename {scylla-server-rust => scylla-server}/src/services/driver_service.rs (100%) rename {scylla-server-rust => scylla-server}/src/services/location_service.rs (100%) rename {scylla-server-rust => scylla-server}/src/services/mod.rs (100%) rename {scylla-server-rust => scylla-server}/src/services/node_service.rs (100%) rename {scylla-server-rust => scylla-server}/src/services/run_service.rs (100%) rename {scylla-server-rust => scylla-server}/src/services/system_service.rs (100%) rename {scylla-server-rust => scylla-server}/src/transformers/data_transformer.rs (100%) rename {scylla-server-rust => scylla-server}/src/transformers/data_type_transformer.rs (100%) rename {scylla-server-rust => scylla-server}/src/transformers/driver_transformer.rs (100%) rename {scylla-server-rust => scylla-server}/src/transformers/location_transformer.rs (100%) rename {scylla-server-rust => scylla-server}/src/transformers/mod.rs (100%) rename {scylla-server-rust => scylla-server}/src/transformers/node_transformer.rs (100%) rename {scylla-server-rust => scylla-server}/src/transformers/run_transformer.rs (100%) rename {scylla-server-rust => scylla-server}/src/transformers/system_transformer.rs (100%) rename {scylla-server-rust => scylla-server}/tests/data_service_test.rs (99%) rename {scylla-server-rust => scylla-server}/tests/data_type_service_test.rs (98%) rename {scylla-server-rust => scylla-server}/tests/driver_service_test.rs (92%) rename {scylla-server-rust => scylla-server}/tests/location_service_test.rs (96%) rename {scylla-server-rust => scylla-server}/tests/node_service_test.rs (95%) rename {scylla-server-rust => scylla-server}/tests/run_service_test.rs (89%) rename {scylla-server-rust => scylla-server}/tests/system_service_test.rs (98%) rename {scylla-server-rust => scylla-server}/tests/test_utils.rs (91%) diff --git a/.github/workflows/scylla-rust-ci.yml b/.github/workflows/scylla-ci.yml similarity index 89% rename from .github/workflows/scylla-rust-ci.yml rename to .github/workflows/scylla-ci.yml index 5c57d4f1..98842bf1 100644 --- a/.github/workflows/scylla-rust-ci.yml +++ b/.github/workflows/scylla-ci.yml @@ -1,4 +1,4 @@ -name: Scylla Rust CI +name: Scylla CI on: push: @@ -14,7 +14,7 @@ on: defaults: run: - working-directory: scylla-server-rust + working-directory: scylla-server jobs: diff --git a/.github/workflows/scylla-rust-tests.yml b/.github/workflows/scylla-tests.yml similarity index 84% rename from .github/workflows/scylla-rust-tests.yml rename to .github/workflows/scylla-tests.yml index 2888fc7c..280c97f9 100644 --- a/.github/workflows/scylla-rust-tests.yml +++ b/.github/workflows/scylla-tests.yml @@ -27,10 +27,10 @@ jobs: run: docker compose run -P -d odyssey-timescale - name: Generate prisma run: cargo prisma generate - working-directory: scylla-server-rust + working-directory: scylla-server - name: Deploy prisma run: cargo prisma migrate deploy - working-directory: scylla-server-rust + working-directory: scylla-server - name: Run tests - working-directory: scylla-server-rust + working-directory: scylla-server run: cargo test -- --test-threads=1 diff --git a/.gitignore b/.gitignore index 0b475aa1..00e420dd 100644 --- a/.gitignore +++ b/.gitignore @@ -3,8 +3,8 @@ # environment config **/.env -scylla-server/src/prisma/mydatabase.db -scylla-server/src/prisma/mydatabase.db-journal +scylla-server-typescript/src/prisma/mydatabase.db +scylla-server-typescript/src/prisma/mydatabase.db-journal # Xcode # diff --git a/.gitmodules b/.gitmodules index 884dbc4e..398a7052 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ -[submodule "scylla-server/src/odyssey-base"] - path = scylla-server/src/odyssey-base +[submodule "scylla-server-typescript/src/odyssey-base"] + path = scylla-server-typescript/src/odyssey-base url = https://github.com/Northeastern-Electric-Racing/Odyssey-Base.git diff --git a/README.md b/README.md index 8196bea7..2f44e247 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ To initialize the odyssey submodule run `git submodule update --init` Setup angular-client and scylla-server: [Angular Client](./angular-client/README.md)\ -[Scylla Server](./scylla-server-rust/README.md) +[Scylla Server](./scylla-server/README.md) Once you've sucessfully setup Scylla and the Client, you can either run them separately, or follow the docker commands below to run them together. @@ -57,7 +57,7 @@ Using the above profiles, one can `build` the app. Then, with correct permissio sudo docker login ghcr.io -u -p ``` -Now you can update the image on a remote server. Note to save time you can just specify which service to upload, like `scylla-server-rust` or `client`. +Now you can update the image on a remote server. Note to save time you can just specify which service to upload, like `scylla-server` or `client`. ``` sudo docker compose -f compose.yml -f compose.router.yml build && sudo docker compose -f compose.yml -f compose.router.yml push ``` diff --git a/compose.brick.yml b/compose.brick.yml index b8d23fab..40a015ef 100644 --- a/compose.brick.yml +++ b/compose.brick.yml @@ -1,5 +1,5 @@ services: - scylla-server-rust: + scylla-server: environment: - SCYLLA_SIREN_HOST_URL=192.168.100.1:1883 diff --git a/compose.client-dev.yml b/compose.client-dev.yml index 057a7f1f..98d60845 100644 --- a/compose.client-dev.yml +++ b/compose.client-dev.yml @@ -1,5 +1,5 @@ services: - scylla-server-rust: + scylla-server: environment: - SCYLLA_SIREN_HOST_URL=siren:1883 - SCYLLA_PROD=false diff --git a/compose.router.yml b/compose.router.yml index 2ceca407..d65fa5df 100644 --- a/compose.router.yml +++ b/compose.router.yml @@ -1,5 +1,5 @@ services: - scylla-server-rust: + scylla-server: depends_on: - siren environment: diff --git a/compose.scylla-dev.yml b/compose.scylla-dev.yml index 3730b752..63d1f5ef 100644 --- a/compose.scylla-dev.yml +++ b/compose.scylla-dev.yml @@ -1,10 +1,10 @@ services: - scylla-server-rust: + scylla-server: depends_on: - siren environment: - SCYLLA_SIREN_HOST_URL=siren:1883 - - RUST_LOG=none,scylla_server_rust=DEBUG + - RUST_LOG=none,scylla_server=DEBUG client: extends: diff --git a/compose.tpu.yml b/compose.tpu.yml index 16c4e05e..a68135b5 100644 --- a/compose.tpu.yml +++ b/compose.tpu.yml @@ -1,5 +1,5 @@ services: - scylla-server-rust: + scylla-server: environment: - SCYLLA_SIREN_HOST_URL=host.docker.internal:1883 extra_hosts: diff --git a/compose.yml b/compose.yml index be21cf5b..94058960 100644 --- a/compose.yml +++ b/compose.yml @@ -14,11 +14,11 @@ services: cpu_shares: 1024 stop_grace_period: 2m - scylla-server-rust: - container_name: scylla-server-rust + scylla-server: + container_name: scylla-server image: ghcr.io/northeastern-electric-racing/argos-scylla:feature-scylla-rust-server build: - context: ./scylla-server-rust + context: ./scylla-server restart: unless-stopped ports: - 8000:8000 @@ -28,7 +28,7 @@ services: - SOURCE_DATABASE_URL=postgresql://postgres:password@odyssey-timescale:5432/postgres # - PROD_SIREN_HOST_URL=siren:1883 - SCYLLA_PROD=true - - RUST_LOG=warn,scylla_server_rust=info # default log setting for docker + - RUST_LOG=warn,scylla_server=debug cpu_shares: 1024 stop_grace_period: 2m stop_signal: SIGINT diff --git a/scylla-server-rust/.dockerignore b/scylla-server-rust/.dockerignore deleted file mode 100644 index 690dff8c..00000000 --- a/scylla-server-rust/.dockerignore +++ /dev/null @@ -1,31 +0,0 @@ -# editor things -.idea/ -.zed/ -.vscode/ - -# misc things -.DS_Store -.gitignore -*.nix - -# python things -pyrightconfig.json -__pycache__/ - -# Added by cargo (rust things) -/target -build/ -dist/ -logs/ - -# prisma -prisma.rs - -# protobuf -serverdata.rs - -target -Dockerfile -.dockerignore -.git -.gitignore diff --git a/scylla-server-rust/.gitignore b/scylla-server-rust/.gitignore deleted file mode 100644 index a9a01cca..00000000 --- a/scylla-server-rust/.gitignore +++ /dev/null @@ -1,25 +0,0 @@ -# editor things -.idea/ -.zed/ -.vscode/ - -# misc things -.DS_Store -*.nix - -# python things -pyrightconfig.json -__pycache__/ - -# Added by cargo (rust things) -/target -build/ -dist/ -logs/ - -# prisma -prisma.rs - -# protobuf -serverdata.rs -command_data.rs diff --git a/scylla-server-rust/Dockerfile b/scylla-server-rust/Dockerfile deleted file mode 100755 index 19c57654..00000000 --- a/scylla-server-rust/Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -FROM rust:latest - -WORKDIR /usr/src/myapp - -COPY . . -RUN cargo prisma generate -RUN cargo build --release --locked -ENTRYPOINT ["./docker_run.sh"] diff --git a/scylla-server-rust/README.md b/scylla-server-rust/README.md deleted file mode 100644 index 6e9f0b7d..00000000 --- a/scylla-server-rust/README.md +++ /dev/null @@ -1,80 +0,0 @@ -# Scylla Server (Rust) - - -### Generate prisma - -``` -cargo prisma generate -``` - -### Run the app - -``` -# in argos proper -docker compose up odyssey-timescale -``` - -``` -# in this directory -SOURCE_DATABASE_URL=postgresql://postgres:password@127.0.0.1:5432/postgres cargo run -``` - -### Test this app - -#### Seed some data - -Run `cargo prisma-seed` - - -#### Integration tests - -Since this app uses the database for testing, you must follow these steps, or run `./integration_test.sh`: -``` -docker volume rm argos_db-data -docker compose up odyssey-timescale -cargo prisma migrate deploy -SOURCE_DATABASE_URL=postgresql://postgres:password@127.0.0.1:5432/postgres cargo test -- --test-threads=1 -``` - -#### Test it yourself! - -You can send your own messages to the app running in production mode, and test how the client, database, etc. reacts! - -Follow this confluence doc: https://nerdocs.atlassian.net/wiki/spaces/NER/pages/473727054/How+to+run+data+through+Argos+without+the+car - -#### View threads and resources - -1. Build or run as: `RUSTFLAGS="--cfg tokio_unstable" cargo run --features top` -2. Install tokio console: ex `cargo install --locked tokio-console` -3. Run app and `tokio-console` - -#### Debug logging - -#### Activate logs -Modify the RUST_LOG env variable. Usually you dont want third party crate logs, so `RUST_LOG=none,scylla_server_rust=trace`. You can replace both none and trace with the levels you want. The levels are: -- none: not a darn thing -- trace: very verbose, could print on every message, which would flood the log especially if run on a server receiving millions of the cars messages -- debug: helpful info not constantly printed in high load situations, good for periodic task info or init/end status checks -- info: user-facing info that is optional, usually to notify of a setting change or whatnot -- warn: info the user should take note of as an error may have occured -- error: a critical byt survivable issue in the application - -#### Develop with logs - -When developing, take advantage of easy logging. Use `debug!`, `trace!` etc macros. with anything you may need, but be sure to abide by the conventions above when making a final PR. - -Have an async function that takes time and is somewhat important for performance? Use tracing::instrument macro on the function definition to capture performance data. - - - -### Deploy this app - -See main README. - - -#### Env variables & CLI Customization - -- `SOURCE_DATABASE_URL` The timescale URL -- `RUST_LOG=none,scylla_server_rust` levels of logging for this create, see above - -**See `cargo run -- -h` for other variables and settings to change!** \ No newline at end of file diff --git a/scylla-server-typescript/.dockerignore b/scylla-server-typescript/.dockerignore new file mode 100644 index 00000000..84c057a1 --- /dev/null +++ b/scylla-server-typescript/.dockerignore @@ -0,0 +1 @@ +src/odyssey-base \ No newline at end of file diff --git a/scylla-server-typescript/.gitignore b/scylla-server-typescript/.gitignore new file mode 100644 index 00000000..c3020db5 --- /dev/null +++ b/scylla-server-typescript/.gitignore @@ -0,0 +1,6 @@ +# dependencies +/node_modules + +# production +/dist + diff --git a/scylla-server-typescript/Dockerfile b/scylla-server-typescript/Dockerfile new file mode 100644 index 00000000..f69fd6f2 --- /dev/null +++ b/scylla-server-typescript/Dockerfile @@ -0,0 +1,26 @@ +FROM node:18.18.0 AS Production + +ENV NODE_ENV=production + +WORKDIR /usr/src/api + +COPY package.json . +COPY package-lock.json . + +RUN npm ci --only=production + +COPY . . + +# Initialize for git submodules +RUN git init +RUN git submodule add -f https://github.com/Northeastern-Electric-Racing/Odyssey-Base.git ./src/odyssey-base && \ + git submodule update --init --recursive --remote + +RUN npm run prisma:generate +RUN npm run build +# RUN echo "SOURCE_DATABASE_URL=postgresql://postgres:password@127.0.0.1:5432/timescaledb" >> .env + +EXPOSE 8000 + +CMD ["sh", "-c", "npm run prisma:migrate:prod && \ + npm run start:production"] diff --git a/scylla-server-typescript/README.md b/scylla-server-typescript/README.md new file mode 100644 index 00000000..4c022350 --- /dev/null +++ b/scylla-server-typescript/README.md @@ -0,0 +1,39 @@ +# Scylla Server + +The express backend for Argos. Handles storing Telemetry data from MQTT into a TimeScaleDB running on the router and forwarding the data to both of the frontends (argos-ios and angular-client). + +## Local Development + +IMPORTANT - Before doing anything make sure you've run: + +`git submodule update --init` + +Make sure you're in the `scylla-server` directory. + +Create a file named .env in the scylla-server directory with the following content for local development: +``` +SOURCE_DATABASE_URL="postgresql://postgres:password@localhost:5432/timescaledb" +``` + +Note: If connecting to MQTT, set PROD=true in your .env variabe and add the PROD_SIREN_HOST_URL to your .env file as well, this is the host name for connecting to the mqtt server. + +To install dependencies run: + +`npm install` + +To setup the database in docker run: + +`docker-compose up -d odyssey-timescale` + +Then: + +`npm run prisma:generate` + +`npm run prisma:migrate` + +Then to actually run the server in development mode, run: + +`npm start` + + +The server will be exposed on `http://localhost:8000/`. If you have the client running, you should see message about connecting to a socket. Once you see that, you're all setup! diff --git a/scylla-server/package-lock.json b/scylla-server-typescript/package-lock.json similarity index 100% rename from scylla-server/package-lock.json rename to scylla-server-typescript/package-lock.json diff --git a/scylla-server/package.json b/scylla-server-typescript/package.json similarity index 100% rename from scylla-server/package.json rename to scylla-server-typescript/package.json diff --git a/scylla-server/src/controllers/data.controller.ts b/scylla-server-typescript/src/controllers/data.controller.ts similarity index 100% rename from scylla-server/src/controllers/data.controller.ts rename to scylla-server-typescript/src/controllers/data.controller.ts diff --git a/scylla-server/src/controllers/datatype.controller.ts b/scylla-server-typescript/src/controllers/datatype.controller.ts similarity index 100% rename from scylla-server/src/controllers/datatype.controller.ts rename to scylla-server-typescript/src/controllers/datatype.controller.ts diff --git a/scylla-server/src/controllers/driver.controller.ts b/scylla-server-typescript/src/controllers/driver.controller.ts similarity index 100% rename from scylla-server/src/controllers/driver.controller.ts rename to scylla-server-typescript/src/controllers/driver.controller.ts diff --git a/scylla-server/src/controllers/location.controller.ts b/scylla-server-typescript/src/controllers/location.controller.ts similarity index 100% rename from scylla-server/src/controllers/location.controller.ts rename to scylla-server-typescript/src/controllers/location.controller.ts diff --git a/scylla-server/src/controllers/node.controller.ts b/scylla-server-typescript/src/controllers/node.controller.ts similarity index 100% rename from scylla-server/src/controllers/node.controller.ts rename to scylla-server-typescript/src/controllers/node.controller.ts diff --git a/scylla-server/src/controllers/run.controller.ts b/scylla-server-typescript/src/controllers/run.controller.ts similarity index 100% rename from scylla-server/src/controllers/run.controller.ts rename to scylla-server-typescript/src/controllers/run.controller.ts diff --git a/scylla-server/src/controllers/system.controller.ts b/scylla-server-typescript/src/controllers/system.controller.ts similarity index 100% rename from scylla-server/src/controllers/system.controller.ts rename to scylla-server-typescript/src/controllers/system.controller.ts diff --git a/scylla-server/src/index.ts b/scylla-server-typescript/src/index.ts similarity index 100% rename from scylla-server/src/index.ts rename to scylla-server-typescript/src/index.ts diff --git a/scylla-server/src/odyssey-base b/scylla-server-typescript/src/odyssey-base similarity index 100% rename from scylla-server/src/odyssey-base rename to scylla-server-typescript/src/odyssey-base diff --git a/scylla-server/src/routes/data.routes.ts b/scylla-server-typescript/src/routes/data.routes.ts similarity index 100% rename from scylla-server/src/routes/data.routes.ts rename to scylla-server-typescript/src/routes/data.routes.ts diff --git a/scylla-server/src/routes/datatype.routes.ts b/scylla-server-typescript/src/routes/datatype.routes.ts similarity index 100% rename from scylla-server/src/routes/datatype.routes.ts rename to scylla-server-typescript/src/routes/datatype.routes.ts diff --git a/scylla-server/src/routes/driver.routes.ts b/scylla-server-typescript/src/routes/driver.routes.ts similarity index 100% rename from scylla-server/src/routes/driver.routes.ts rename to scylla-server-typescript/src/routes/driver.routes.ts diff --git a/scylla-server/src/routes/location.routes.ts b/scylla-server-typescript/src/routes/location.routes.ts similarity index 100% rename from scylla-server/src/routes/location.routes.ts rename to scylla-server-typescript/src/routes/location.routes.ts diff --git a/scylla-server/src/routes/node.routes.ts b/scylla-server-typescript/src/routes/node.routes.ts similarity index 100% rename from scylla-server/src/routes/node.routes.ts rename to scylla-server-typescript/src/routes/node.routes.ts diff --git a/scylla-server/src/routes/run.routes.ts b/scylla-server-typescript/src/routes/run.routes.ts similarity index 100% rename from scylla-server/src/routes/run.routes.ts rename to scylla-server-typescript/src/routes/run.routes.ts diff --git a/scylla-server/src/routes/system.routes.ts b/scylla-server-typescript/src/routes/system.routes.ts similarity index 100% rename from scylla-server/src/routes/system.routes.ts rename to scylla-server-typescript/src/routes/system.routes.ts diff --git a/scylla-server/src/socket/mock-proxy-client.ts b/scylla-server-typescript/src/socket/mock-proxy-client.ts similarity index 100% rename from scylla-server/src/socket/mock-proxy-client.ts rename to scylla-server-typescript/src/socket/mock-proxy-client.ts diff --git a/scylla-server/src/socket/prod-proxy-client.ts b/scylla-server-typescript/src/socket/prod-proxy-client.ts similarity index 100% rename from scylla-server/src/socket/prod-proxy-client.ts rename to scylla-server-typescript/src/socket/prod-proxy-client.ts diff --git a/scylla-server/src/socket/proxy-client.ts b/scylla-server-typescript/src/socket/proxy-client.ts similarity index 100% rename from scylla-server/src/socket/proxy-client.ts rename to scylla-server-typescript/src/socket/proxy-client.ts diff --git a/scylla-server/src/socket/proxy-server.ts b/scylla-server-typescript/src/socket/proxy-server.ts similarity index 100% rename from scylla-server/src/socket/proxy-server.ts rename to scylla-server-typescript/src/socket/proxy-server.ts diff --git a/scylla-server/src/utils/data.utils.ts b/scylla-server-typescript/src/utils/data.utils.ts similarity index 100% rename from scylla-server/src/utils/data.utils.ts rename to scylla-server-typescript/src/utils/data.utils.ts diff --git a/scylla-server/src/utils/message.utils.ts b/scylla-server-typescript/src/utils/message.utils.ts similarity index 100% rename from scylla-server/src/utils/message.utils.ts rename to scylla-server-typescript/src/utils/message.utils.ts diff --git a/scylla-server/tsconfig.json b/scylla-server-typescript/tsconfig.json similarity index 100% rename from scylla-server/tsconfig.json rename to scylla-server-typescript/tsconfig.json diff --git a/scylla-server/vite.config.js b/scylla-server-typescript/vite.config.js similarity index 100% rename from scylla-server/vite.config.js rename to scylla-server-typescript/vite.config.js diff --git a/scylla-server-rust/.cargo/config.toml b/scylla-server/.cargo/config.toml similarity index 100% rename from scylla-server-rust/.cargo/config.toml rename to scylla-server/.cargo/config.toml diff --git a/scylla-server/.dockerignore b/scylla-server/.dockerignore index 84c057a1..690dff8c 100644 --- a/scylla-server/.dockerignore +++ b/scylla-server/.dockerignore @@ -1 +1,31 @@ -src/odyssey-base \ No newline at end of file +# editor things +.idea/ +.zed/ +.vscode/ + +# misc things +.DS_Store +.gitignore +*.nix + +# python things +pyrightconfig.json +__pycache__/ + +# Added by cargo (rust things) +/target +build/ +dist/ +logs/ + +# prisma +prisma.rs + +# protobuf +serverdata.rs + +target +Dockerfile +.dockerignore +.git +.gitignore diff --git a/scylla-server/.gitignore b/scylla-server/.gitignore index c3020db5..a9a01cca 100644 --- a/scylla-server/.gitignore +++ b/scylla-server/.gitignore @@ -1,6 +1,25 @@ -# dependencies -/node_modules +# editor things +.idea/ +.zed/ +.vscode/ -# production -/dist +# misc things +.DS_Store +*.nix +# python things +pyrightconfig.json +__pycache__/ + +# Added by cargo (rust things) +/target +build/ +dist/ +logs/ + +# prisma +prisma.rs + +# protobuf +serverdata.rs +command_data.rs diff --git a/scylla-server-rust/Cargo.lock b/scylla-server/Cargo.lock similarity index 99% rename from scylla-server-rust/Cargo.lock rename to scylla-server/Cargo.lock index 432320e4..903c506f 100755 --- a/scylla-server-rust/Cargo.lock +++ b/scylla-server/Cargo.lock @@ -3220,7 +3220,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] -name = "scylla-server-rust" +name = "scylla-server" version = "0.0.1" dependencies = [ "axum 0.7.5", diff --git a/scylla-server-rust/Cargo.toml b/scylla-server/Cargo.toml similarity index 93% rename from scylla-server-rust/Cargo.toml rename to scylla-server/Cargo.toml index c5ec9fd2..3ca59c86 100644 --- a/scylla-server-rust/Cargo.toml +++ b/scylla-server/Cargo.toml @@ -1,8 +1,8 @@ [package] -name = "scylla-server-rust" +name = "scylla-server" version = "0.0.1" edition = "2021" -default-run = "scylla-server-rust" +default-run = "scylla-server" [dependencies] prisma-client-rust = {git = "https://github.com/Brendonovich/prisma-client-rust", tag = "0.6.11", default-features = false,features = ["postgresql", "migrations"] } @@ -40,7 +40,7 @@ panic = "abort" strip = true # Automatically strip symbols from the binary. [[bin]] -name = "scylla-server-rust" +name = "scylla-server" path = "src/main.rs" [[bin]] diff --git a/scylla-server/Dockerfile b/scylla-server/Dockerfile old mode 100644 new mode 100755 index f69fd6f2..19c57654 --- a/scylla-server/Dockerfile +++ b/scylla-server/Dockerfile @@ -1,26 +1,8 @@ -FROM node:18.18.0 AS Production +FROM rust:latest -ENV NODE_ENV=production +WORKDIR /usr/src/myapp -WORKDIR /usr/src/api - -COPY package.json . -COPY package-lock.json . - -RUN npm ci --only=production - -COPY . . - -# Initialize for git submodules -RUN git init -RUN git submodule add -f https://github.com/Northeastern-Electric-Racing/Odyssey-Base.git ./src/odyssey-base && \ - git submodule update --init --recursive --remote - -RUN npm run prisma:generate -RUN npm run build -# RUN echo "SOURCE_DATABASE_URL=postgresql://postgres:password@127.0.0.1:5432/timescaledb" >> .env - -EXPOSE 8000 - -CMD ["sh", "-c", "npm run prisma:migrate:prod && \ - npm run start:production"] +COPY . . +RUN cargo prisma generate +RUN cargo build --release --locked +ENTRYPOINT ["./docker_run.sh"] diff --git a/scylla-server/README.md b/scylla-server/README.md index 4c022350..4f368f12 100644 --- a/scylla-server/README.md +++ b/scylla-server/README.md @@ -1,39 +1,80 @@ -# Scylla Server +# Scylla Server (Rust) -The express backend for Argos. Handles storing Telemetry data from MQTT into a TimeScaleDB running on the router and forwarding the data to both of the frontends (argos-ios and angular-client). -## Local Development +### Generate prisma -IMPORTANT - Before doing anything make sure you've run: +``` +cargo prisma generate +``` + +### Run the app + +``` +# in argos proper +docker compose up odyssey-timescale +``` + +``` +# in this directory +SOURCE_DATABASE_URL=postgresql://postgres:password@127.0.0.1:5432/postgres cargo run +``` -`git submodule update --init` +### Test this app -Make sure you're in the `scylla-server` directory. +#### Seed some data -Create a file named .env in the scylla-server directory with the following content for local development: +Run `cargo prisma-seed` + + +#### Integration tests + +Since this app uses the database for testing, you must follow these steps, or run `./integration_test.sh`: ``` -SOURCE_DATABASE_URL="postgresql://postgres:password@localhost:5432/timescaledb" +docker volume rm argos_db-data +docker compose up odyssey-timescale +cargo prisma migrate deploy +SOURCE_DATABASE_URL=postgresql://postgres:password@127.0.0.1:5432/postgres cargo test -- --test-threads=1 ``` -Note: If connecting to MQTT, set PROD=true in your .env variabe and add the PROD_SIREN_HOST_URL to your .env file as well, this is the host name for connecting to the mqtt server. +#### Test it yourself! + +You can send your own messages to the app running in production mode, and test how the client, database, etc. reacts! + +Follow this confluence doc: https://nerdocs.atlassian.net/wiki/spaces/NER/pages/473727054/How+to+run+data+through+Argos+without+the+car + +#### View threads and resources + +1. Build or run as: `RUSTFLAGS="--cfg tokio_unstable" cargo run --features top` +2. Install tokio console: ex `cargo install --locked tokio-console` +3. Run app and `tokio-console` + +#### Debug logging + +#### Activate logs +Modify the RUST_LOG env variable. Usually you dont want third party crate logs, so `RUST_LOG=none,scylla_server=trace`. You can replace both none and trace with the levels you want. The levels are: +- none: not a darn thing +- trace: very verbose, could print on every message, which would flood the log especially if run on a server receiving millions of the cars messages +- debug: helpful info not constantly printed in high load situations, good for periodic task info or init/end status checks +- info: user-facing info that is optional, usually to notify of a setting change or whatnot +- warn: info the user should take note of as an error may have occured +- error: a critical byt survivable issue in the application -To install dependencies run: +#### Develop with logs -`npm install` +When developing, take advantage of easy logging. Use `debug!`, `trace!` etc macros. with anything you may need, but be sure to abide by the conventions above when making a final PR. -To setup the database in docker run: +Have an async function that takes time and is somewhat important for performance? Use tracing::instrument macro on the function definition to capture performance data. -`docker-compose up -d odyssey-timescale` -Then: -`npm run prisma:generate` +### Deploy this app -`npm run prisma:migrate` +See main README. -Then to actually run the server in development mode, run: -`npm start` +#### Env variables & CLI Customization +- `SOURCE_DATABASE_URL` The timescale URL +- `RUST_LOG=none,scylla_server=` levels of logging for this create, see above -The server will be exposed on `http://localhost:8000/`. If you have the client running, you should see message about connecting to a socket. Once you see that, you're all setup! +**See `cargo run -- -h` for other variables and settings to change!** \ No newline at end of file diff --git a/scylla-server-rust/build.rs b/scylla-server/build.rs similarity index 100% rename from scylla-server-rust/build.rs rename to scylla-server/build.rs diff --git a/scylla-server-rust/docker_run.sh b/scylla-server/docker_run.sh similarity index 100% rename from scylla-server-rust/docker_run.sh rename to scylla-server/docker_run.sh diff --git a/scylla-server-rust/integration_test.sh b/scylla-server/integration_test.sh similarity index 100% rename from scylla-server-rust/integration_test.sh rename to scylla-server/integration_test.sh diff --git a/scylla-server-rust/prisma-cli/Cargo.toml b/scylla-server/prisma-cli/Cargo.toml similarity index 100% rename from scylla-server-rust/prisma-cli/Cargo.toml rename to scylla-server/prisma-cli/Cargo.toml diff --git a/scylla-server-rust/prisma-cli/src/main.rs b/scylla-server/prisma-cli/src/main.rs similarity index 100% rename from scylla-server-rust/prisma-cli/src/main.rs rename to scylla-server/prisma-cli/src/main.rs diff --git a/scylla-server-rust/prisma/migrations/20240212220042_init/migration.sql b/scylla-server/prisma/migrations/20240212220042_init/migration.sql similarity index 100% rename from scylla-server-rust/prisma/migrations/20240212220042_init/migration.sql rename to scylla-server/prisma/migrations/20240212220042_init/migration.sql diff --git a/scylla-server-rust/prisma/migrations/20240212220355_init/migration.sql b/scylla-server/prisma/migrations/20240212220355_init/migration.sql similarity index 100% rename from scylla-server-rust/prisma/migrations/20240212220355_init/migration.sql rename to scylla-server/prisma/migrations/20240212220355_init/migration.sql diff --git a/scylla-server-rust/prisma/migrations/migration_lock.toml b/scylla-server/prisma/migrations/migration_lock.toml similarity index 100% rename from scylla-server-rust/prisma/migrations/migration_lock.toml rename to scylla-server/prisma/migrations/migration_lock.toml diff --git a/scylla-server-rust/prisma/schema.prisma b/scylla-server/prisma/schema.prisma similarity index 100% rename from scylla-server-rust/prisma/schema.prisma rename to scylla-server/prisma/schema.prisma diff --git a/scylla-server-rust/prisma/seed.rs b/scylla-server/prisma/seed.rs similarity index 99% rename from scylla-server-rust/prisma/seed.rs rename to scylla-server/prisma/seed.rs index d8af8827..6ed5f8f1 100644 --- a/scylla-server-rust/prisma/seed.rs +++ b/scylla-server/prisma/seed.rs @@ -1,7 +1,7 @@ use std::{sync::Arc, time::Duration}; use prisma_client_rust::{chrono, QueryError}; -use scylla_server_rust::{ +use scylla_server::{ prisma::PrismaClient, processors::ClientData, services::{ diff --git a/scylla-server-rust/src/controllers/car_command_controller.rs b/scylla-server/src/controllers/car_command_controller.rs similarity index 100% rename from scylla-server-rust/src/controllers/car_command_controller.rs rename to scylla-server/src/controllers/car_command_controller.rs diff --git a/scylla-server-rust/src/controllers/data_controller.rs b/scylla-server/src/controllers/data_controller.rs similarity index 100% rename from scylla-server-rust/src/controllers/data_controller.rs rename to scylla-server/src/controllers/data_controller.rs diff --git a/scylla-server-rust/src/controllers/data_type_controller.rs b/scylla-server/src/controllers/data_type_controller.rs similarity index 100% rename from scylla-server-rust/src/controllers/data_type_controller.rs rename to scylla-server/src/controllers/data_type_controller.rs diff --git a/scylla-server-rust/src/controllers/driver_controller.rs b/scylla-server/src/controllers/driver_controller.rs similarity index 100% rename from scylla-server-rust/src/controllers/driver_controller.rs rename to scylla-server/src/controllers/driver_controller.rs diff --git a/scylla-server-rust/src/controllers/location_controller.rs b/scylla-server/src/controllers/location_controller.rs similarity index 100% rename from scylla-server-rust/src/controllers/location_controller.rs rename to scylla-server/src/controllers/location_controller.rs diff --git a/scylla-server-rust/src/controllers/mod.rs b/scylla-server/src/controllers/mod.rs similarity index 100% rename from scylla-server-rust/src/controllers/mod.rs rename to scylla-server/src/controllers/mod.rs diff --git a/scylla-server-rust/src/controllers/node_controller.rs b/scylla-server/src/controllers/node_controller.rs similarity index 100% rename from scylla-server-rust/src/controllers/node_controller.rs rename to scylla-server/src/controllers/node_controller.rs diff --git a/scylla-server-rust/src/controllers/run_controller.rs b/scylla-server/src/controllers/run_controller.rs similarity index 100% rename from scylla-server-rust/src/controllers/run_controller.rs rename to scylla-server/src/controllers/run_controller.rs diff --git a/scylla-server-rust/src/controllers/system_controller.rs b/scylla-server/src/controllers/system_controller.rs similarity index 100% rename from scylla-server-rust/src/controllers/system_controller.rs rename to scylla-server/src/controllers/system_controller.rs diff --git a/scylla-server-rust/src/error.rs b/scylla-server/src/error.rs similarity index 100% rename from scylla-server-rust/src/error.rs rename to scylla-server/src/error.rs diff --git a/scylla-server-rust/src/lib.rs b/scylla-server/src/lib.rs similarity index 100% rename from scylla-server-rust/src/lib.rs rename to scylla-server/src/lib.rs diff --git a/scylla-server-rust/src/main.rs b/scylla-server/src/main.rs similarity index 99% rename from scylla-server-rust/src/main.rs rename to scylla-server/src/main.rs index f149a606..2db796b2 100755 --- a/scylla-server-rust/src/main.rs +++ b/scylla-server/src/main.rs @@ -8,7 +8,7 @@ use axum::{ use clap::Parser; use prisma_client_rust::chrono; use rumqttc::v5::AsyncClient; -use scylla_server_rust::{ +use scylla_server::{ controllers::{ self, car_command_controller::{self}, diff --git a/scylla-server-rust/src/mod.rs b/scylla-server/src/mod.rs similarity index 100% rename from scylla-server-rust/src/mod.rs rename to scylla-server/src/mod.rs diff --git a/scylla-server-rust/src/processors/db_handler.rs b/scylla-server/src/processors/db_handler.rs similarity index 100% rename from scylla-server-rust/src/processors/db_handler.rs rename to scylla-server/src/processors/db_handler.rs diff --git a/scylla-server-rust/src/processors/mock_data.rs b/scylla-server/src/processors/mock_data.rs similarity index 100% rename from scylla-server-rust/src/processors/mock_data.rs rename to scylla-server/src/processors/mock_data.rs diff --git a/scylla-server-rust/src/processors/mock_processor.rs b/scylla-server/src/processors/mock_processor.rs similarity index 100% rename from scylla-server-rust/src/processors/mock_processor.rs rename to scylla-server/src/processors/mock_processor.rs diff --git a/scylla-server-rust/src/processors/mod.rs b/scylla-server/src/processors/mod.rs similarity index 100% rename from scylla-server-rust/src/processors/mod.rs rename to scylla-server/src/processors/mod.rs diff --git a/scylla-server-rust/src/processors/mqtt_processor.rs b/scylla-server/src/processors/mqtt_processor.rs similarity index 100% rename from scylla-server-rust/src/processors/mqtt_processor.rs rename to scylla-server/src/processors/mqtt_processor.rs diff --git a/scylla-server-rust/src/proto/command_data.proto b/scylla-server/src/proto/command_data.proto similarity index 100% rename from scylla-server-rust/src/proto/command_data.proto rename to scylla-server/src/proto/command_data.proto diff --git a/scylla-server-rust/src/proto/serverdata.proto b/scylla-server/src/proto/serverdata.proto similarity index 100% rename from scylla-server-rust/src/proto/serverdata.proto rename to scylla-server/src/proto/serverdata.proto diff --git a/scylla-server-rust/src/services/data_service.rs b/scylla-server/src/services/data_service.rs similarity index 100% rename from scylla-server-rust/src/services/data_service.rs rename to scylla-server/src/services/data_service.rs diff --git a/scylla-server-rust/src/services/data_type_service.rs b/scylla-server/src/services/data_type_service.rs similarity index 100% rename from scylla-server-rust/src/services/data_type_service.rs rename to scylla-server/src/services/data_type_service.rs diff --git a/scylla-server-rust/src/services/driver_service.rs b/scylla-server/src/services/driver_service.rs similarity index 100% rename from scylla-server-rust/src/services/driver_service.rs rename to scylla-server/src/services/driver_service.rs diff --git a/scylla-server-rust/src/services/location_service.rs b/scylla-server/src/services/location_service.rs similarity index 100% rename from scylla-server-rust/src/services/location_service.rs rename to scylla-server/src/services/location_service.rs diff --git a/scylla-server-rust/src/services/mod.rs b/scylla-server/src/services/mod.rs similarity index 100% rename from scylla-server-rust/src/services/mod.rs rename to scylla-server/src/services/mod.rs diff --git a/scylla-server-rust/src/services/node_service.rs b/scylla-server/src/services/node_service.rs similarity index 100% rename from scylla-server-rust/src/services/node_service.rs rename to scylla-server/src/services/node_service.rs diff --git a/scylla-server-rust/src/services/run_service.rs b/scylla-server/src/services/run_service.rs similarity index 100% rename from scylla-server-rust/src/services/run_service.rs rename to scylla-server/src/services/run_service.rs diff --git a/scylla-server-rust/src/services/system_service.rs b/scylla-server/src/services/system_service.rs similarity index 100% rename from scylla-server-rust/src/services/system_service.rs rename to scylla-server/src/services/system_service.rs diff --git a/scylla-server-rust/src/transformers/data_transformer.rs b/scylla-server/src/transformers/data_transformer.rs similarity index 100% rename from scylla-server-rust/src/transformers/data_transformer.rs rename to scylla-server/src/transformers/data_transformer.rs diff --git a/scylla-server-rust/src/transformers/data_type_transformer.rs b/scylla-server/src/transformers/data_type_transformer.rs similarity index 100% rename from scylla-server-rust/src/transformers/data_type_transformer.rs rename to scylla-server/src/transformers/data_type_transformer.rs diff --git a/scylla-server-rust/src/transformers/driver_transformer.rs b/scylla-server/src/transformers/driver_transformer.rs similarity index 100% rename from scylla-server-rust/src/transformers/driver_transformer.rs rename to scylla-server/src/transformers/driver_transformer.rs diff --git a/scylla-server-rust/src/transformers/location_transformer.rs b/scylla-server/src/transformers/location_transformer.rs similarity index 100% rename from scylla-server-rust/src/transformers/location_transformer.rs rename to scylla-server/src/transformers/location_transformer.rs diff --git a/scylla-server-rust/src/transformers/mod.rs b/scylla-server/src/transformers/mod.rs similarity index 100% rename from scylla-server-rust/src/transformers/mod.rs rename to scylla-server/src/transformers/mod.rs diff --git a/scylla-server-rust/src/transformers/node_transformer.rs b/scylla-server/src/transformers/node_transformer.rs similarity index 100% rename from scylla-server-rust/src/transformers/node_transformer.rs rename to scylla-server/src/transformers/node_transformer.rs diff --git a/scylla-server-rust/src/transformers/run_transformer.rs b/scylla-server/src/transformers/run_transformer.rs similarity index 100% rename from scylla-server-rust/src/transformers/run_transformer.rs rename to scylla-server/src/transformers/run_transformer.rs diff --git a/scylla-server-rust/src/transformers/system_transformer.rs b/scylla-server/src/transformers/system_transformer.rs similarity index 100% rename from scylla-server-rust/src/transformers/system_transformer.rs rename to scylla-server/src/transformers/system_transformer.rs diff --git a/scylla-server-rust/tests/data_service_test.rs b/scylla-server/tests/data_service_test.rs similarity index 99% rename from scylla-server-rust/tests/data_service_test.rs rename to scylla-server/tests/data_service_test.rs index 327836d6..68c027fe 100644 --- a/scylla-server-rust/tests/data_service_test.rs +++ b/scylla-server/tests/data_service_test.rs @@ -2,7 +2,7 @@ mod test_utils; use prisma_client_rust::QueryError; -use scylla_server_rust::{ +use scylla_server::{ processors::ClientData, services::{data_service, data_type_service, node_service, run_service}, transformers::data_transformer::PublicData, diff --git a/scylla-server-rust/tests/data_type_service_test.rs b/scylla-server/tests/data_type_service_test.rs similarity index 98% rename from scylla-server-rust/tests/data_type_service_test.rs rename to scylla-server/tests/data_type_service_test.rs index 36c156b0..1f5007e0 100644 --- a/scylla-server-rust/tests/data_type_service_test.rs +++ b/scylla-server/tests/data_type_service_test.rs @@ -2,7 +2,7 @@ mod test_utils; use prisma_client_rust::QueryError; -use scylla_server_rust::{ +use scylla_server::{ prisma, services::{ data_type_service::{self, public_datatype}, diff --git a/scylla-server-rust/tests/driver_service_test.rs b/scylla-server/tests/driver_service_test.rs similarity index 92% rename from scylla-server-rust/tests/driver_service_test.rs rename to scylla-server/tests/driver_service_test.rs index 805ba047..f5233c72 100644 --- a/scylla-server-rust/tests/driver_service_test.rs +++ b/scylla-server/tests/driver_service_test.rs @@ -1,5 +1,5 @@ use prisma_client_rust::QueryError; -use scylla_server_rust::services::{driver_service, run_service}; +use scylla_server::services::{driver_service, run_service}; use test_utils::cleanup_and_prepare; #[path = "test_utils.rs"] diff --git a/scylla-server-rust/tests/location_service_test.rs b/scylla-server/tests/location_service_test.rs similarity index 96% rename from scylla-server-rust/tests/location_service_test.rs rename to scylla-server/tests/location_service_test.rs index ca9abd10..72f476aa 100644 --- a/scylla-server-rust/tests/location_service_test.rs +++ b/scylla-server/tests/location_service_test.rs @@ -1,5 +1,5 @@ use prisma_client_rust::QueryError; -use scylla_server_rust::{ +use scylla_server::{ prisma, services::{location_service, run_service}, }; diff --git a/scylla-server-rust/tests/node_service_test.rs b/scylla-server/tests/node_service_test.rs similarity index 95% rename from scylla-server-rust/tests/node_service_test.rs rename to scylla-server/tests/node_service_test.rs index 7ce84cab..54a1b3ec 100644 --- a/scylla-server-rust/tests/node_service_test.rs +++ b/scylla-server/tests/node_service_test.rs @@ -1,5 +1,5 @@ use prisma_client_rust::QueryError; -use scylla_server_rust::{prisma, services::node_service}; +use scylla_server::{prisma, services::node_service}; use test_utils::cleanup_and_prepare; #[path = "test_utils.rs"] diff --git a/scylla-server-rust/tests/run_service_test.rs b/scylla-server/tests/run_service_test.rs similarity index 89% rename from scylla-server-rust/tests/run_service_test.rs rename to scylla-server/tests/run_service_test.rs index e13bd2c3..1f4aeba5 100644 --- a/scylla-server-rust/tests/run_service_test.rs +++ b/scylla-server/tests/run_service_test.rs @@ -1,5 +1,5 @@ use prisma_client_rust::QueryError; -use scylla_server_rust::{services::run_service, transformers::run_transformer::PublicRun}; +use scylla_server::{services::run_service, transformers::run_transformer::PublicRun}; use test_utils::cleanup_and_prepare; #[path = "test_utils.rs"] diff --git a/scylla-server-rust/tests/system_service_test.rs b/scylla-server/tests/system_service_test.rs similarity index 98% rename from scylla-server-rust/tests/system_service_test.rs rename to scylla-server/tests/system_service_test.rs index 82e45d1c..2bf2ee1b 100644 --- a/scylla-server-rust/tests/system_service_test.rs +++ b/scylla-server/tests/system_service_test.rs @@ -1,5 +1,5 @@ use prisma_client_rust::QueryError; -use scylla_server_rust::{ +use scylla_server::{ prisma, services::{ run_service, diff --git a/scylla-server-rust/tests/test_utils.rs b/scylla-server/tests/test_utils.rs similarity index 91% rename from scylla-server-rust/tests/test_utils.rs rename to scylla-server/tests/test_utils.rs index bc3db82f..8ac0d24b 100644 --- a/scylla-server-rust/tests/test_utils.rs +++ b/scylla-server/tests/test_utils.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use prisma_client_rust::QueryError; -use scylla_server_rust::{prisma::PrismaClient, Database}; +use scylla_server::{prisma::PrismaClient, Database}; pub async fn cleanup_and_prepare() -> Result { let client = Arc::new(PrismaClient::_builder().build().await.unwrap()); From f0956c254de1f1de5aa5d8228f1dc1b4cc30b4af Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Mon, 16 Sep 2024 19:33:20 -0400 Subject: [PATCH 66/68] add comments, fix tests, rename more files, fix ci --- .../{scylla-server-rust-build.yml => scylla-build.yml} | 2 +- scylla-server/integration_test.sh | 7 ++++--- scylla-server/src/controllers/data_controller.rs | 1 + scylla-server/src/controllers/data_type_controller.rs | 1 + scylla-server/src/controllers/driver_controller.rs | 1 + scylla-server/src/controllers/location_controller.rs | 1 + scylla-server/src/controllers/node_controller.rs | 1 + scylla-server/src/controllers/run_controller.rs | 4 ++++ scylla-server/src/controllers/system_controller.rs | 1 + scylla-server/src/main.rs | 4 ++-- 10 files changed, 17 insertions(+), 6 deletions(-) rename .github/workflows/{scylla-server-rust-build.yml => scylla-build.yml} (98%) diff --git a/.github/workflows/scylla-server-rust-build.yml b/.github/workflows/scylla-build.yml similarity index 98% rename from .github/workflows/scylla-server-rust-build.yml rename to .github/workflows/scylla-build.yml index 42bdb18b..912381c3 100644 --- a/.github/workflows/scylla-server-rust-build.yml +++ b/.github/workflows/scylla-build.yml @@ -44,7 +44,7 @@ jobs: - name: Build and push Docker image for scylla server rust uses: docker/build-push-action@v5.4.0 with: - context: ./scylla-server-rust + context: ./scylla-server push: true platforms: linux/arm64,linux/amd64 tags: ${{ steps.meta.outputs.tags }} diff --git a/scylla-server/integration_test.sh b/scylla-server/integration_test.sh index 042756ed..829d7c7b 100755 --- a/scylla-server/integration_test.sh +++ b/scylla-server/integration_test.sh @@ -2,13 +2,14 @@ echo "Starting db" cd .. -docker compose run -Pd odyssey-timescale +docker compose up -d odyssey-timescale echo "Deploying prisma" -cd ./scylla-server-rust +cd ./scylla-server cargo prisma migrate deploy echo "Running tests" SOURCE_DATABASE_URL=postgresql://postgres:password@127.0.0.1:5432/postgres cargo test -- --test-threads=1 -echo "Please stop your db in docker" +cd .. +docker compose down \ No newline at end of file diff --git a/scylla-server/src/controllers/data_controller.rs b/scylla-server/src/controllers/data_controller.rs index 6731246a..4e74d9d9 100644 --- a/scylla-server/src/controllers/data_controller.rs +++ b/scylla-server/src/controllers/data_controller.rs @@ -8,6 +8,7 @@ use crate::{ Database, }; +/// Get all of the data points of a certain data type name and run ID pub async fn get_data( State(db): State, Path((data_type_name, run_id)): Path<(String, i32)>, diff --git a/scylla-server/src/controllers/data_type_controller.rs b/scylla-server/src/controllers/data_type_controller.rs index d80ed459..1277492d 100644 --- a/scylla-server/src/controllers/data_type_controller.rs +++ b/scylla-server/src/controllers/data_type_controller.rs @@ -5,6 +5,7 @@ use crate::{ transformers::data_type_transformer::PublicDataType, Database, }; +/// Get a list of data types pub async fn get_all_data_types( State(db): State, ) -> Result>, ScyllaError> { diff --git a/scylla-server/src/controllers/driver_controller.rs b/scylla-server/src/controllers/driver_controller.rs index 66d67009..50de4bb8 100644 --- a/scylla-server/src/controllers/driver_controller.rs +++ b/scylla-server/src/controllers/driver_controller.rs @@ -5,6 +5,7 @@ use crate::{ Database, }; +/// Get a list of drivers pub async fn get_all_drivers( State(db): State, ) -> Result>, ScyllaError> { diff --git a/scylla-server/src/controllers/location_controller.rs b/scylla-server/src/controllers/location_controller.rs index 24e46337..679756b8 100644 --- a/scylla-server/src/controllers/location_controller.rs +++ b/scylla-server/src/controllers/location_controller.rs @@ -5,6 +5,7 @@ use crate::{ transformers::location_transformer::PublicLocation, Database, }; +/// get a list of locations pub async fn get_all_locations( State(db): State, ) -> Result>, ScyllaError> { diff --git a/scylla-server/src/controllers/node_controller.rs b/scylla-server/src/controllers/node_controller.rs index 3f03f110..4460caaf 100644 --- a/scylla-server/src/controllers/node_controller.rs +++ b/scylla-server/src/controllers/node_controller.rs @@ -5,6 +5,7 @@ use crate::{ Database, }; +/// get a list of nodes pub async fn get_all_nodes( State(db): State, ) -> Result>, ScyllaError> { diff --git a/scylla-server/src/controllers/run_controller.rs b/scylla-server/src/controllers/run_controller.rs index 528c4405..31cb517c 100644 --- a/scylla-server/src/controllers/run_controller.rs +++ b/scylla-server/src/controllers/run_controller.rs @@ -10,6 +10,7 @@ use crate::{ error::ScyllaError, services::run_service, transformers::run_transformer::PublicRun, Database, }; +/// get a list of runs pub async fn get_all_runs(State(db): State) -> Result>, ScyllaError> { let run_data = run_service::get_all_runs(&db).await?; @@ -18,6 +19,7 @@ pub async fn get_all_runs(State(db): State) -> Result, Path(run_id): Path, @@ -35,6 +37,8 @@ pub async fn get_run_by_id( Ok(Json::from(transformed_run_data)) } +/// create a new run with an auto-incremented ID +/// note the new run must be updated so the channel passed in notifies the data processor to use the new run pub async fn new_run( State(db): State, Extension(channel): Extension>, diff --git a/scylla-server/src/controllers/system_controller.rs b/scylla-server/src/controllers/system_controller.rs index 5c9d1a01..5922efa9 100644 --- a/scylla-server/src/controllers/system_controller.rs +++ b/scylla-server/src/controllers/system_controller.rs @@ -5,6 +5,7 @@ use crate::{ Database, }; +/// get a list of systems pub async fn get_all_systems( State(db): State, ) -> Result>, ScyllaError> { diff --git a/scylla-server/src/main.rs b/scylla-server/src/main.rs index 2db796b2..818aaf0c 100755 --- a/scylla-server/src/main.rs +++ b/scylla-server/src/main.rs @@ -198,12 +198,12 @@ async fn main() { }; let app = Router::new() - // DATA ROUTES + // DATA .route( "/data/:dataTypeName/:runId", get(controllers::data_controller::get_data), ) - // DATA TYPE ROUTES + // DATA TYPE .route("/datatypes", get(data_type_controller::get_all_data_types)) // DRIVERS .route("/drivers", get(driver_controller::get_all_drivers)) From 86e464c475c5c324b2cff6edc7e047bae01dd3be Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Mon, 16 Sep 2024 19:41:31 -0400 Subject: [PATCH 67/68] bump protobuf, please clippy --- scylla-server/Cargo.lock | 16 ++++++++-------- scylla-server/Cargo.toml | 6 +++--- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/scylla-server/Cargo.lock b/scylla-server/Cargo.lock index 903c506f..b3704400 100755 --- a/scylla-server/Cargo.lock +++ b/scylla-server/Cargo.lock @@ -2544,9 +2544,9 @@ dependencies = [ [[package]] name = "protobuf" -version = "3.5.0" +version = "3.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df67496db1a89596beaced1579212e9b7c53c22dca1d9745de00ead76573d514" +checksum = "0bcc343da15609eaecd65f8aa76df8dc4209d325131d8219358c0aaaebab0bf6" dependencies = [ "once_cell", "protobuf-support", @@ -2555,9 +2555,9 @@ dependencies = [ [[package]] name = "protobuf-codegen" -version = "3.5.0" +version = "3.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eab09155fad2d39333d3796f67845d43e29b266eea74f7bc93f153f707f126dc" +checksum = "c4d0cde5642ea4df842b13eb9f59ea6fafa26dcb43e3e1ee49120e9757556189" dependencies = [ "anyhow", "once_cell", @@ -2570,9 +2570,9 @@ dependencies = [ [[package]] name = "protobuf-parse" -version = "3.5.0" +version = "3.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a16027030d4ec33e423385f73bb559821827e9ec18c50e7874e4d6de5a4e96f" +checksum = "1b0e9b447d099ae2c4993c0cbb03c7a9d6c937b17f2d56cfc0b1550e6fcfdb76" dependencies = [ "anyhow", "indexmap 2.2.6", @@ -2586,9 +2586,9 @@ dependencies = [ [[package]] name = "protobuf-support" -version = "3.5.0" +version = "3.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70e2d30ab1878b2e72d1e2fc23ff5517799c9929e2cf81a8516f9f4dcf2b9cf3" +checksum = "f0766e3675a627c327e4b3964582594b0e8741305d628a98a5de75a1d15f99b9" dependencies = [ "thiserror", ] diff --git a/scylla-server/Cargo.toml b/scylla-server/Cargo.toml index 3ca59c86..4980650d 100644 --- a/scylla-server/Cargo.toml +++ b/scylla-server/Cargo.toml @@ -7,8 +7,8 @@ default-run = "scylla-server" [dependencies] prisma-client-rust = {git = "https://github.com/Brendonovich/prisma-client-rust", tag = "0.6.11", default-features = false,features = ["postgresql", "migrations"] } serde = "1.0.203" -protobuf-codegen = "3.3.0" -protobuf = "3.3.0" +protobuf-codegen = "3.5.1" +protobuf = "3.5.1" tokio = { version = "1.38.0", features = ["full", "tracing"] } axum = "0.7.5" tower = { version = "0.4.13", features = ["timeout"] } @@ -31,7 +31,7 @@ top = ["dep:console-subscriber"] members = ["prisma-cli"] [build-dependencies] -protobuf-codegen = "3.3.0" +protobuf-codegen = "3.5.1" [profile.release] lto = true From d636316ae1d5d87410300659424388e0207481d4 Mon Sep 17 00:00:00 2001 From: Jack Rubacha Date: Mon, 16 Sep 2024 19:52:52 -0400 Subject: [PATCH 68/68] rename branch to develop in CI --- angular-client/compose.client.yml | 2 +- compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/angular-client/compose.client.yml b/angular-client/compose.client.yml index eee5f658..2d8ae082 100644 --- a/angular-client/compose.client.yml +++ b/angular-client/compose.client.yml @@ -2,7 +2,7 @@ services: client: container_name: client restart: unless-stopped - image: ghcr.io/northeastern-electric-racing/argos-client:feature-scylla-rust-server + image: ghcr.io/northeastern-electric-racing/argos-client:develop build: context: . args: diff --git a/compose.yml b/compose.yml index 94058960..473a7ed2 100644 --- a/compose.yml +++ b/compose.yml @@ -16,7 +16,7 @@ services: scylla-server: container_name: scylla-server - image: ghcr.io/northeastern-electric-racing/argos-scylla:feature-scylla-rust-server + image: ghcr.io/northeastern-electric-racing/argos-scylla:develop build: context: ./scylla-server restart: unless-stopped