diff --git a/.env.example b/.env.example deleted file mode 100644 index 6ad3a2aec..000000000 --- a/.env.example +++ /dev/null @@ -1,35 +0,0 @@ -# Docker-compose specific settings - -## Docker Postrges settings - -### Postgres log directory path on host machine (absolute or relative) -### This folder needs to be created before exec docker-compose -DOCKER_PG_HOST_LOGS_DIR=./pg_logs - -### Postgres port -DOCKER_PG_HOST_PORT=5006 - -### Webserver port -GEOV_WEBSERVER_PORT=3001 -DOMAIN_NAME=https://geovistory.com - -### nginx toolbox port -NGINX_TOOLBOX_PORT=3002 - -### Command executed at startup (use it to override default postgresql.config settings) -DOCKER_PG_COMMAND="postgres -c logging_collector=on -c log_destination=stderr -c log_directory=/logs" -# -c log_statement=all -# -c log_duration=1 -# -c log_min_duration_statement=200 - -## Stack level settings - -### database created by postgres and used by warehouse and webserver when launched by docker compose full stack -GEOV_DB=gv_test_db -WH_DB=wh_test_db - -# SMTP connection params -GEOV_EMAIL_HOST= -GEOV_EMAIL_PORT= -GEOV_EMAIL_ADDRESS=info@geovistory.org -GEOV_EMAIL_PASSWORD=*** diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 031542cd7..000000000 --- a/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -node_modules -dev-logs - -# postgres logs -pg_logs - -# file for customizing environment vars -.env \ No newline at end of file diff --git a/Aptfile b/Aptfile deleted file mode 100644 index 3d52ee91e..000000000 --- a/Aptfile +++ /dev/null @@ -1,4 +0,0 @@ -# documentation: https://elements.heroku.com/buildpacks/heroku/heroku-buildpack-apt - -# you can list packages -# pgloader diff --git a/DEVELOPING.md b/DEVELOPING.md deleted file mode 100644 index f62061c5a..000000000 --- a/DEVELOPING.md +++ /dev/null @@ -1,105 +0,0 @@ -# Developer's Guide - -We use Visual Studio Code for developing LoopBack and recommend the same to our -users. - -## Docker - -Docker is used to create images for the different components of Geovistory: - -- webserver -- warehouse - -(The postgres database is part of the Geovistory stack but we can use existing docker images for setting up postgres.) - -## Docker compose - -Docker-compose helps to setup a stack. This is usefull for local development. - -### Docker-compose files - -- `docker-compose.no-webserver.yaml` contains only the layers we don't develop ourselfs. Currently only postgres. This is useful for **development of webserver**. -- `docker-compose.no-warehouse.yaml` contains only the layers we don't develop ourselfs. Currently only postgres. This is useful for **development of warehouse**. -- `docker-compose.base.yaml` contains only the layers we don't develop ourselfs. Currently only postgres. This is useful for **development of webserver and warehouse**. -- `docker-compose.all-local.yaml` contains all layers of stack including webserver and warehouse. This is useful to verify if the built images run. - -## Environment variables - -To configure docker-compose we use the approched discussed here: -http://blog.code4hire.com/2018/06/define-named-volume-with-host-mount-in-the-docker-compose-file/ - -The docker-compose files substitute certain values with env vars. These env vars can be defined in .env file. - -### Docker-compose commands - -To build a project: - -```sh -docker-compose --project-name geov_stack --file docker-compose.all-local.yaml up -d -docker-compose --project-name geov_base --file docker-compose.base.yaml up -d -``` - -To rebuild a project (its images): - -```sh -# just rebuild -docker-compose --project-name geov_stack --file docker-compose.all-local.yaml build -docker-compose --project-name geov_base --file docker-compose.base.yaml build -# rebuild and run in background -docker-compose --project-name geov_stack --file docker-compose.all-local.yaml up --build -d -docker-compose --project-name geov_base --file docker-compose.base.yaml up --build -d -``` - -To stop project: - -```sh -docker-compose --project-name geov_stack --file docker-compose.all-local.yaml down -docker-compose --project-name geov_base --file docker-compose.base.yaml down -``` - -NOTE: When you start the first time it may lead to an error because the database is not ready while beeing accessed from webserver. Just stop and start again. - -## Access the database - -To see the port of the database, starte the stack (see above) and run - -```sh -# lists the running containers -docker ps - -# output (similar): see here geovistory_db_1 is exposed at port 5005 to host machine - -# CONTAINER ID IMAGE COMMAND ... PORTS NAMES -# d3a0aa46d80b geovistory_web "docker-en…" ... 0.0.0.0:3000->3000/tcp geovistory_web_1 -# 02ea63d4cec2 postgis/postgis:12-2.5-alpine "docker-en…" ... 0.0.0.0:5005->5432/tcp geovistory_db_1 -``` - -## Postgres - -### Reporting and Logging - -Official docs: - -- https://www.postgresql.org/docs/12/runtime-config-logging.html - -Explanation and best practices: - -- https://www.2ndquadrant.com/en/blog/how-to-get-the-best-out-of-postgresql-logs/ - -To change settings on db level, you can use docker-compose, change the env var DOCKER_PG_COMMAND accordingly and rebuild the project (see above). - -To change settings in a connection (client) you can use SET (e.g.: `SET log_statement='all'`) to alter the settings for this client connection only. - -If for some reason you need to see every statement (also successfull select statements), use the log_statement parameter: - -`log_statement = 'off' | 'ddl' | 'mod' | 'all'` - -Controls which SQL statements are logged. Valid values are - -- **none** (off) -- **ddl** logs all data definition statements, such as CREATE, ALTER, and DROP statements. -- **mod** logs all ddl statements, plus data-modifying statements such as INSERT, UPDATE, DELETE, TRUNCATE, and COPY FROM. PREPARE, EXECUTE, and EXPLAIN ANALYZE statements are also logged if their contained command is of an appropriate type. For clients using extended query protocol, logging occurs when an Execute message is received, and values of the Bind parameters are included (with any embedded single-quote marks doubled). -- **all** (all statements). - -Docs: -https://www.postgresql.org/docs/12/runtime-config-logging.html#GUC-LOG-STATEMENT diff --git a/Pipfile b/Pipfile deleted file mode 100644 index b9ba84f67..000000000 --- a/Pipfile +++ /dev/null @@ -1,11 +0,0 @@ -[[source]] -url = "https://pypi.org/simple" -verify_ssl = true -name = "pypi" - -[packages] - -[dev-packages] - -[requires] -python_version = "3.7" diff --git a/Procfile b/Procfile deleted file mode 100644 index 584484fce..000000000 --- a/Procfile +++ /dev/null @@ -1,3 +0,0 @@ -release: bash release.bash -web: bash web.bash -worker: bash worker.bash \ No newline at end of file diff --git a/README.md b/README.md index 5a98a4ef0..2c4ff5489 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,65 @@ -# geovistory +# Geovistory Toolbox -[![LoopBack](https://github.com/strongloop/loopback-next/raw/master/docs/site/imgs/branding/Powered-by-LoopBack-Badge-(blue)-@2x.png)](http://loopback.io/) +The Virtual Research Environment of the Geovistory Ecosystem. + +## Project Layout + +This is a monorepo for server and client of the toolbox. + +- `/server` contains all server code (node.js / LoopBack) +- `/client` contains all client code (angular) + +## Development + +### Setup the dev stack + +Developing the toolbox server and client requires external components. +These are packed into a docker-compose file in the GitHub repo geovistory/dev-stack. + +Requirements: + +- Docker (and docker-compose) installed. +- 10 GB RAM for docker +- 4 CPUs for docker +- 100 GB disk space for docker + +Donwload the stack: + +```bash +git clone https://github.com/geovistory/dev-stack.git +``` + +Change the directory: + +```bash +cd dev-stack +``` + +Start the stack (using docker-compose under the hood) + +```bash +bash scripts/build +``` + +See more on https://github.com/geovistory/dev-stack. + +### Client Development + +To develop the toolbox client, we need to set two env vars to match the URLs +of the dev stack. + +- API_URL: The URL of the toolbox server api (for development usually hosted by dev-stack). +- ASSETS_URL: The URL of the server hosting the front-end assets (for development usually hosted by angular dev-server). + +By default, the URLs in client/.env.example match the dev-stack and angular dev-server default configuration. + +If you did not change the defaults, run: + +```bash +cd client +npm run serve +``` + +This will copy the .env.example to .env and create the correct env.js in the angular app. + +If you need to modify the API_URL or ASSETS_URL, edit the gitignored client/.env file. diff --git a/app.json b/app.json deleted file mode 100644 index 3cdde6f2d..000000000 --- a/app.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "name": "geovistory", - "scripts": {}, - "env": { - "DB_ENV": "review", - "HEROKU_APP_NAME": { - "required": true - }, - "GV_DB_SSL": "sslmode=require", - "WH_DB_SSL": "sslmode=require", - "WAREHOUSE_GV_DB_POOL_SIZE":"5", - "WAREHOUSE_WH_DB_POOL_SIZE":"5", - "WH_DATABASE_URL": "${DATABASE_URL}" - }, - "formation": {}, - "addons": [ - { - "plan": "heroku-postgresql:hobby-basic", - "options": { - "version": "10" - } - }, - { - "plan": "heroku-postgresql:hobby-basic", - "options": { - "version": "10" - } - } - ], - "buildpacks": [ - { - "url": "heroku/nodejs" - } - ] -} \ No newline at end of file diff --git a/client/.env.example b/client/.env.example new file mode 100644 index 000000000..5e8bb777c --- /dev/null +++ b/client/.env.example @@ -0,0 +1,5 @@ +# URL of toolbox server api +API_URL=http://localhost:1130 + +# URL of server hosting angular assets +ASSETS_URL=http://localhost:4200 diff --git a/client/.gitignore b/client/.gitignore index 849d950fa..a3ec7659e 100644 --- a/client/.gitignore +++ b/client/.gitignore @@ -1,8 +1,12 @@ -report.*json -documentation +# environment +.env +/projects/app-toolbox/src/assets/env.js # Dependency directories node_modules/ +# Auto generated src/__tests__/helpers/data/auto-gen/* dist/ +report.*json +documentation diff --git a/client/package.json b/client/package.json index d6588ca44..ffef7197d 100644 --- a/client/package.json +++ b/client/package.json @@ -7,7 +7,7 @@ }, "scripts": { "ng": "node --max_old_space_size=4096 node_modules/@angular/cli/bin/ng", - "preserve": "node ./scripts/check-version.js", + "preserve": "node ./scripts/preserve.js", "serve": "node --max_old_space_size=4096 node_modules/@angular/cli/bin/ng serve", "start": "ng serve", "build:libs": "sh ./scripts/ng-build-libs.sh", @@ -129,4 +129,4 @@ "typescript": "~4.2.4", "typings-quill-delta": "0.0.2" } -} \ No newline at end of file +} diff --git a/client/projects/app-toolbox/src/app/shared/components/entity-preview/entity-preview.component.ts b/client/projects/app-toolbox/src/app/shared/components/entity-preview/entity-preview.component.ts index 155cc56de..a90220a30 100644 --- a/client/projects/app-toolbox/src/app/shared/components/entity-preview/entity-preview.component.ts +++ b/client/projects/app-toolbox/src/app/shared/components/entity-preview/entity-preview.component.ts @@ -38,7 +38,8 @@ export class EntityPreviewComponent implements OnInit, OnDestroy { .pipe(takeUntil(this.destroy$)) .subscribe(preview => { this.preview = preview - this.urls = [...getUrls(preview.entity_label)] + // extract urls from string + this.urls = typeof preview.entity_label === 'string' ? [...getUrls(preview.entity_label)] : [] this.ref.detectChanges() }) } diff --git a/client/projects/app-toolbox/src/assets/env.js b/client/projects/app-toolbox/src/assets/env.js deleted file mode 100644 index 4ebf3c7b5..000000000 --- a/client/projects/app-toolbox/src/assets/env.js +++ /dev/null @@ -1,9 +0,0 @@ -// see: https://pumpingco.de/blog/environment-variables-angular-docker/ -(function (window) { - window["env"] = window["env"] || {}; - - // Environment variable for the rest api url - window["env"]["apiUrl"] = "http://localhost:3000"; - // Environment variable for the url serving static assets of angular dist folder - window["env"]["assetsUrl"] = "http://localhost:3000"; -})(this); diff --git a/client/scripts/check-version.js b/client/scripts/check-version.js deleted file mode 100644 index d1479de54..000000000 --- a/client/scripts/check-version.js +++ /dev/null @@ -1,8 +0,0 @@ -let semver = require('semver'); -let engines = require('../package.json').engines; - -const version = engines.node; -if (!semver.satisfies(process.version, version)) { - console.log(`Required node version ${version} not satisfied with current version ${process.version}.\n`); - process.exit(1); -} \ No newline at end of file diff --git a/client/scripts/preserve.js b/client/scripts/preserve.js new file mode 100644 index 000000000..b4687846d --- /dev/null +++ b/client/scripts/preserve.js @@ -0,0 +1,2 @@ +require("./validate-node-version"); +require("./setup-env"); diff --git a/client/scripts/setup-env.js b/client/scripts/setup-env.js new file mode 100644 index 000000000..f03da0ece --- /dev/null +++ b/client/scripts/setup-env.js @@ -0,0 +1,54 @@ +const fs = require("fs"); +const path = require("path"); + +const envPath = path.join(__dirname, "../.env"); +const envExamplePath = path.join(__dirname, "../.env.example"); +const envJsPath = path.join( + __dirname, + "../projects/app-toolbox/src/assets/env.js" +); +const envJsTemplatePath = path.join( + __dirname, + "../projects/app-toolbox/src/assets/env.template.js" +); + +const copyEnvExampleIfNeeded = () => { + if (!fs.existsSync(envPath)) { + fs.copyFileSync(envExamplePath, envPath); + console.log(".env file created by copying .env.example"); + } +}; + +const generateEnvJs = () => { + if (fs.existsSync(envPath)) { + const envData = fs.readFileSync(envPath, "utf8"); + let envJs = fs.readFileSync(envJsTemplatePath, "utf8"); + + console.log("creating env.js file with the environment variables:"); + envData + .split("\n") + .filter((line) => { + const l = line.trim(); + // filter comments + if (l.startsWith("#")) return false; + // filter empty lines + if (l.length === 0) return false; + return true; + }) + .forEach((line) => { + const [key, value] = line.split("="); + console.log(" " + key, value); + const keyRegex = new RegExp("\\${" + key.trim() + "}", "g"); + envJs = envJs.replace(keyRegex, value.trim()); + }); + + fs.writeFileSync(envJsPath, envJs, "utf8"); + } else { + console.error( + ".env file not found. Please create one by copying .env.example first." + ); + } +}; + +copyEnvExampleIfNeeded(); +generateEnvJs(); diff --git a/client/scripts/validate-node-version.js b/client/scripts/validate-node-version.js new file mode 100644 index 000000000..279667f9e --- /dev/null +++ b/client/scripts/validate-node-version.js @@ -0,0 +1,13 @@ +let semver = require("semver"); +let engines = require("../package.json").engines; + +/** + * Validate the node version + */ +const version = engines.node; +if (!semver.satisfies(process.version, version)) { + console.log( + `Required node version ${version} not satisfied with current version ${process.version}.\n` + ); + process.exit(1); +} diff --git a/db/db.dockerfile b/db/db.dockerfile deleted file mode 100644 index 913a9bb02..000000000 --- a/db/db.dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM postgis/postgis:12-2.5-alpine -RUN mkdir /logs -RUN chmod -R 777 /logs -COPY docker-db-init.sh /docker-entrypoint-initdb.d diff --git a/db/docker-db-init.sh b/db/docker-db-init.sh deleted file mode 100644 index 3bf00e8de..000000000 --- a/db/docker-db-init.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -set -e - -psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL - CREATE DATABASE $GEOVISTORY_DB; -EOSQL diff --git a/deployment/create-warehouse-compat-list.sh b/deployment/create-warehouse-compat-list.sh deleted file mode 100644 index 6389d02d4..000000000 --- a/deployment/create-warehouse-compat-list.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash - -# This file is used by .github/workflows/warehouse-compat.yml -# to create server/warehouse-compat-list.txt -# The script generates a list of commits from the current commit -# backwards in history in order to list commits where warehouse -# was unchanged compared to current commit. - -# exit when any command fails -set -e - -# keep track of the last executed command -trap 'last_command=$current_command; current_command=$BASH_COMMAND' DEBUG - -# echo an error message before exiting -trap 'echo "\"${last_command}\" command filed with exit code $?."' EXIT - -# assign variables -path=server/src/warehouse -output=server/warehouse-compat-list.txt -currentCommit=$(git rev-parse --short HEAD) -i=0 - -# check if warehouse directory exists -[ ! -d "$path" ] && echo "Directory $path DOES NOT exists." && exit 1 - -# remove output file (clean up) -rm -f $output - -# loop over old commits as long as no changes to current commit detected -# first iteration is over the current commit, so that the list starts with current commit -while [ $i -le 1000 ]; do - commit=$(git rev-parse --short @~$i) - echo "i: $i, currentCommit: $currentCommit, commit: $commit, path: $path" - diff=$(git diff $currentCommit $commit -- $path) - - # if [ $? -eq 0 ]; then - if [ -z "$diff" ]; then - # No modifications found in directory $path between $currentCommit and $commit. - # Adding commit to list of commits where warehouse is compatible with current comit - echo "found no diff to $commit" - echo $commit >>$output - else - # Modifications found in directory $path between $currentCommit and $commit - # exit loop because from now on all next iterations will return a difference between the commits - echo "warehouse-compat-list.txt created" - exit - fi - i=$(($i + 1)) -done diff --git a/deployment/dev_db_restore.sh b/deployment/dev_db_restore.sh deleted file mode 100755 index fa2e03ced..000000000 --- a/deployment/dev_db_restore.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/bin/bash -# -# Helper script for developers: -# Restore reviewdb.backup on a local database -# - -read -p 'Username [postgres]: ' user -user=${user:-postgres} -echo $user -echo - -read -p 'Password [GuruAtMarly]: ' password -password=${password:-GuruAtMarly} -echo $password -echo - -read -p 'Host [localhost]: ' host -host=${host:-localhost} -echo $host -echo - -read -p 'Port [5432]: ' port -port=${port:-5432} -echo $port -echo - -read -p 'Database [dev1]: ' database -database=${database:-dev1} -echo $database -echo - -read -p 'File [reviewdb.backup]: ' filepath -filepath=${filepath:-reviewdb.backup} -echo $filepath -echo - -read -p 'Number of parallel jobs [none]: ' jobs -jobs=${jobs:-none} -echo $jobs -echo - -echo Your command: pg_restore -j 6 --no-owner -d postgres://$user:$password@$host:$port/$database $filepath --verbose -echo Attention: this action is irreversible! Schemas are dropped first -echo -while true; do - read -p "Do you want to execute the command [y/n]?" yn - case $yn in - [Yy]*) - - echo '================= Drop schemas of target DB ==============================' - # We need to drop schemas before restoring because the review database may be - # in an earlier state, created by deployment after previeous commit on the - # same pull request - time psql $DATABASE_URL -f dropSchemas.sql - echo '================= Drop schemas done! =====================================' - echo - echo '======== Restore Database (from deployment/reviewdb.backup) =================' - - if [ $jobs = 'none' ]; then - time pg_restore --no-owner -d postgres://$user:$password@$host:$port/$database $filepath --verbose - else - time pg_restore -j $jobs --no-owner -d postgres://$user:$password@$host:$port/$database $filepath --verbose - fi - echo '======== Database restored ==============================================' - echo 'Consider to migrate up and to recreate warehouse' - - break - ;; - [Nn]*) exit ;; - *) echo "Please answer yes or no." ;; - esac -done diff --git a/deployment/drop-schemas.sql b/deployment/drop-schemas.sql deleted file mode 100644 index b74edab5f..000000000 --- a/deployment/drop-schemas.sql +++ /dev/null @@ -1,25 +0,0 @@ - ---SET search_path = _global, pg_catalog; - -CREATE OR REPLACE FUNCTION public.drop_all () - RETURNS VOID AS - $$ - DECLARE rec RECORD; - BEGIN - -- Get all the schemas - FOR rec IN - select schema_name - from information_schema.schemata - - -- You can exclude the schema which you don't want to drop by adding another condition here - where schema_name !~* 'information_schema|pg_catalog|pg_toast' - LOOP - EXECUTE 'DROP SCHEMA ' || rec.schema_name || ' CASCADE'; - END LOOP; - RETURN; - END; - $$ LANGUAGE plpgsql; - -select public.drop_all(); - -CREATE SCHEMA public; \ No newline at end of file diff --git a/deployment/get-commit-of-folder.sh b/deployment/get-commit-of-folder.sh deleted file mode 100644 index 8b8005e73..000000000 --- a/deployment/get-commit-of-folder.sh +++ /dev/null @@ -1,38 +0,0 @@ -# #!/bin/bash - -# This script returns the commit hash of the last commit -# that introduced a change to the directory given by -p flag - -# usage example -# bash get-commit-of-folder.sh -p ../server/src/ - -# fetch argument from flag -p -while getopts p: flag; do - case "${flag}" in - p) path=${OPTARG} ;; - esac -done -# echo "Path: $path" - -# assign variables -currentCommit=$(git rev-parse HEAD) -latestCommitWithoutChanges=$currentCommit -i=0 - -# check if directory exists -[ ! -d "$path" ] && echo "Directory $path DOES NOT exists." && exit 1 - -# loop over old commits as long as no changes to current commit detected -# first iteration is over the current commit, so that the list starts with current commit -while [ $i -le 1000 ]; do - commit=$(git rev-parse HEAD~$i) - git diff --quiet $currentCommit $commit -- $path - if [ $? -eq 1 ]; then - # Modifications found in directory $path between $currentCommit and $commit - # exit loop because from now on all next iterations will return a difference between the commits - echo $latestCommitWithoutChanges - exit 0 - fi - i=$(($i + 1)) - latestCommitWithoutChanges=$commit -done diff --git a/deployment/reviewdb.backup b/deployment/reviewdb.backup deleted file mode 100644 index 16d77fb1e..000000000 Binary files a/deployment/reviewdb.backup and /dev/null differ diff --git a/deployment/update-staging-db.sh b/deployment/update-staging-db.sh deleted file mode 100644 index 5c5e53ca5..000000000 --- a/deployment/update-staging-db.sh +++ /dev/null @@ -1,35 +0,0 @@ -# command to execute on heroku scheduler: -# bash deployment/update-staging-db.sh - -if [[ "$DB_ENV" == "staging" ]]; then - # get day of week (1: Monday, ..., 7: sunday) - day=$(date +"%u") - - if [[ "$day" == "5" ]]; then # on fridays - - # maintenante mode on stagin ON - heroku maintenance:on -a geovistory-staging - - # turn other workers OFF - heroku ps:scale worker=0 -a geovistory-staging - - # clean the warehouse: drop the schema: warcache_githash || drop all schema starting with warcache_ -c - name=$(heroku pg:psql -a geovistory-staging -c "SELECT schema_name FROM information_schema.schemata WHERE schema_name LIKE 'war_cache_%'" WH_DATABASE_URL) - name=${name/schema_name/''} # remove the string 'schema_name' - name=${name//-/''} # remove the dashs - name=${name/(1 ligne)/''} # remove the string '(1 ligne)', if FR env - name=${name/(1 row)/''} # remove the string '(1 row)', if EN env - name=`echo $name | xargs` - heroku pg:psql -a geovistory-staging -c "DROP SCHEMA IF EXISTS ${name} CASCADE;" WH_DATABASE_URL - - # copy production db to staging db - heroku pg:copy geovistory-production::DATABASE_URL DATABASE_URL -a geovistory-staging --confirm=geovistory-staging - - # turn other workers ON - heroku ps:restart -a geovistory-stating - heroku ps:scale worker=1 -a geovistory-staging - - # maintenante mode on stagin OFF - heroku maintenance:off -a geovistory-staging - fi -fi diff --git a/deployment/update-temp-db.sh b/deployment/update-temp-db.sh deleted file mode 100644 index b06c1dccd..000000000 --- a/deployment/update-temp-db.sh +++ /dev/null @@ -1,31 +0,0 @@ -echo $DB_ENV -if [[ "$DB_ENV" == "temp" ]]; then - # get day of week (1: Monday, 7: sunday) - day=$(date +"%u") - - if [[ "$day" == "1" ]]; then - # maintenante mode on stagin ON - heroku maintenance:on -a geovistory-temp - - # turn other workers OFF - heroku ps:scale worker=0 -a geovistory-temp - - # clean the warehouse: drop the schema: warcache_githash || drop all schema starting with warcache_ -c - name=$(heroku pg:psql -a geovistory-temp -c "SELECT schema_name FROM information_schema.schemata WHERE schema_name LIKE 'war_cache_%'" WH_DATABASE_URL) - name=${name/schema_name/''} # remove the string 'schema_name' - name=${name//-/''} # remove the dashs - name=${name/(1 ligne)/''} # remove the string '(1 ligne)', if FR env - name=${name/(1 row)/''} # remove the string '(1 row)', if EN env - name=`echo $name | xargs` - heroku pg:psql -a geovistory-temp -c "DROP SCHEMA IF EXISTS ${name} CASCADE;" WH_DATABASE_URL - - # copy dev db to temp db - heroku pg:copy geovistory-dev::DATABASE_URL DATABASE_URL -a geovistory-temp --confirm=confirm - - # turn other workers ON - heroku ps:scale worker=1 -a geovistory-temp - - # maintenante mode on stagin OFF - heroku maintenance:off -a geovistory-temp - fi -fi \ No newline at end of file diff --git a/docker-compose.all-local.yaml b/docker-compose.all-local.yaml deleted file mode 100644 index 587041a06..000000000 --- a/docker-compose.all-local.yaml +++ /dev/null @@ -1,60 +0,0 @@ -version: '3.8' - -services: - # db access from host machine: postgres://postgres:local_pw@{DOCKER_IP}:5005 - db: - build: - context: ./db - dockerfile: db.dockerfile - shm_size: 1g - environment: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: local_pw - GEOVISTORY_DB: ${GEOV_DB} - command: ${DOCKER_PG_COMMAND} - ports: - - ${DOCKER_PG_HOST_PORT}:5432 - restart: always - volumes: - - 'postgis-data:/var/lib/postgresql/data' - - 'postgis-logs:/logs' - - # web access from host machine: {DOCKER_IP}:${DOCKER_PG_HOST_PORT} - toolbox: - build: - context: ./client - dockerfile: Dockerfile - environment: - API_URL: ${DOMAIN_NAME}:${GEOV_WEBSERVER_PORT} - ASSETS_URL: ${DOMAIN_NAME}:${GEOV_WEBSERVER_PORT} - ports: - - ${NGINX_TOOLBOX_PORT}:80 - - # web access from host machine: {DOCKER_IP}:${DOCKER_PG_HOST_PORT} - web: - build: - context: ./server - dockerfile: webserver.dockerfile - environment: - # db:5432 points to service db and its cointainer port - DATABASE_URL: postgres://postgres:local_pw@db:5432/${GEOV_DB} - ports: - - ${GEOV_WEBSERVER_PORT}:3000 - - warehouse: - build: - context: ./server - dockerfile: warehouse.dockerfile - environment: - # db:5432 points to service db and its cointainer port - WH_DATABASE_URL: postgres://postgres:local_pw@db:5432/${WH_DB} - DATABASE_URL: postgres://postgres:local_pw@db:5432/${GEOV_DB} - -volumes: - postgis-data: - postgis-logs: - driver: local - driver_opts: - type: none - device: ${DOCKER_PG_HOST_LOGS_DIR} - o: bind diff --git a/docker-compose.base.yaml b/docker-compose.base.yaml deleted file mode 100644 index 034438836..000000000 --- a/docker-compose.base.yaml +++ /dev/null @@ -1,29 +0,0 @@ -version: "3.8" - -services: - # db access from host machine: postgres://postgres:local_pw@{DOCKER_IP}:5005 - db: - build: - context: ./db - dockerfile: db.dockerfile - shm_size: 1g - environment: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: local_pw - GEOVISTORY_DB: ${DEFAULT_DB} - command: ${DOCKER_PG_COMMAND} - ports: - - ${DOCKER_PG_HOST_PORT}:5432 - restart: always - volumes: - - "postgis-data:/var/lib/postgresql/data" - - "postgis-logs:/logs" - -volumes: - postgis-data: - postgis-logs: - driver: local - driver_opts: - type: none - device: ${DOCKER_PG_HOST_LOGS_DIR} - o: bind \ No newline at end of file diff --git a/heroku-postbuild.sh b/heroku-postbuild.sh deleted file mode 100644 index 6a9cd7ba4..000000000 --- a/heroku-postbuild.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -echo '================ Start of heroku-postbuild.sh ==============================' - -echo '================= Set environment variables for client =====================' -envsubst <./client/dist/app-toolbox/assets/env.template.js >./client/dist/app-toolbox/assets/env.js -echo 'client env vars' -cat ./client/dist/app-toolbox/assets/env.js - -echo '================= Compile Server ==============================' -cd ./server - -echo './server: npm run build' -npm run build - -echo './server: npm prune --production' -npm prune --production - -echo '================= On dev ==============================' -if [ $DB_ENV = 'dev' ]; then - echo 'touch .env' - touch .env - - echo 'npm i --production=false' - npm i --production=false -fi - -cd .. - -echo '================ End of heroku-postbuild.sh ================================' diff --git a/package-lock.json b/package-lock.json index b13801b4d..b840371b7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { - "name": "geovistory-root", - "version": "0.2.3", - "lockfileVersion": 2, - "requires": true, - "packages": { - "": { - "name": "geovistory-root", - "version": "0.2.3" + "name": "geovistory-root", + "version": "0.2.4-pr-295.1", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "name": "geovistory-root", + "version": "0.2.4-pr-295.1" + } } - } } diff --git a/package.json b/package.json index b28519cce..2d4ce8929 100644 --- a/package.json +++ b/package.json @@ -1,16 +1,7 @@ { - "name": "geovistory-root", - "description": "package.json for DevOps scripts", - "version": "0.2.3", - "scripts": { - "docker": "node ./server/scripts/docker.js", - "loopack-build:watch": "cd server && npm run build:watch", - "heroku-postbuild": "bash heroku-postbuild.sh" - }, - "private": true, - "main": "index.js", - "cacheDirectories": [ - "server/node_modules", - "client/node_modules" - ] + "name": "geovistory-root", + "description": "root package.json of the monorepo", + "version": "0.2.4-pr-295.1", + "scripts": {}, + "private": true } diff --git a/release.bash b/release.bash deleted file mode 100644 index 9277eda92..000000000 --- a/release.bash +++ /dev/null @@ -1 +0,0 @@ -bash ./server/db-migrate/up.sh \ No newline at end of file diff --git a/server/src/__tests__/acceptance/warehouse/aggregator-ds/entity-preview/EntityPreviewService.test.ts b/server/src/__tests__/acceptance/warehouse/aggregator-ds/entity-preview/EntityPreviewService.test.ts deleted file mode 100644 index f9bf75362..000000000 --- a/server/src/__tests__/acceptance/warehouse/aggregator-ds/entity-preview/EntityPreviewService.test.ts +++ /dev/null @@ -1,373 +0,0 @@ -/* eslint-disable @typescript-eslint/camelcase */ -import 'reflect-metadata'; -import {PClassFieldLabelService} from '../../../../../warehouse/aggregator-ds/class-field-label/p-class-field-label/PClassFieldLabelService'; -import {RClassFieldLabelService} from '../../../../../warehouse/aggregator-ds/class-field-label/r-class-field-label/RClassFieldLabelService'; -import {PClassLabelService} from '../../../../../warehouse/aggregator-ds/class-label/p-class-label/PClassLabelService'; -import {RClassLabelService} from '../../../../../warehouse/aggregator-ds/class-label/r-class-label/RClassLabelService'; -import {PEntityClassLabelService} from '../../../../../warehouse/aggregator-ds/entity-class-label/p-entity-class-label/PEntityClassLabelService'; -import {REntityClassLabelService} from '../../../../../warehouse/aggregator-ds/entity-class-label/r-entity-class-label/REntityClassLabelService'; -import {PEntityFullTextService} from '../../../../../warehouse/aggregator-ds/entity-full-text/p-entity-full-text/PEntityFullTextService'; -import {REntityFullTextService} from '../../../../../warehouse/aggregator-ds/entity-full-text/r-entity-full-text/REntityFullTextService'; -import {PEntityLabelService} from '../../../../../warehouse/aggregator-ds/entity-label/p-entity-label/PEntityLabelService'; -import {REntityLabelService} from '../../../../../warehouse/aggregator-ds/entity-label/r-entity-label/REntityLabelService'; -import {EntityPreviewService} from '../../../../../warehouse/aggregator-ds/entity-preview/EntityPreviewService'; -import {PEntityTimeSpanService} from '../../../../../warehouse/aggregator-ds/entity-time-span/p-entity-time-span/PEntityTimeSpanService'; -import {REntityTimeSpanService} from '../../../../../warehouse/aggregator-ds/entity-time-span/r-entity-time-span/REntityTimeSpanService'; -import {PEntityTypeService} from '../../../../../warehouse/aggregator-ds/entity-type/p-entity-type/PEntityTypeService'; -import {REntityTypeService} from '../../../../../warehouse/aggregator-ds/entity-type/r-entity-type/REntityTypeService'; -import {PClassService} from '../../../../../warehouse/primary-ds/class/PClassService'; -import {RClassService} from '../../../../../warehouse/primary-ds/class/RClassService'; -import {DfhClassHasTypePropertyService} from '../../../../../warehouse/primary-ds/DfhClassHasTypePropertyService'; -import {DfhClassLabelService} from '../../../../../warehouse/primary-ds/DfhClassLabelService'; -import {DfhOutgoingPropertyService} from '../../../../../warehouse/primary-ds/DfhOutgoingPropertyService'; -import {DfhPropertyLabelService} from '../../../../../warehouse/primary-ds/DfhPropertyLabelService'; -import {PEdgeService} from '../../../../../warehouse/primary-ds/edge/PEdgeService'; -import {REdgeService} from '../../../../../warehouse/primary-ds/edge/REdgeService'; -import {PEntityService} from '../../../../../warehouse/primary-ds/entity/PEntityService'; -import {REntityService} from '../../../../../warehouse/primary-ds/entity/REntityService'; -import {ProClassFieldsConfigService} from '../../../../../warehouse/primary-ds/ProClassFieldsConfigService'; -import {ProClassLabelService} from '../../../../../warehouse/primary-ds/ProClassLabelService'; -import {ProEntityLabelConfigService} from '../../../../../warehouse/primary-ds/ProEntityLabelConfigService'; -import {PPropertyService} from '../../../../../warehouse/primary-ds/property/PPropertyService'; -import {RPropertyService} from '../../../../../warehouse/primary-ds/property/RPropertyService'; -import {ProProjectService} from '../../../../../warehouse/primary-ds/ProProjectService'; -import {ProPropertyLabelService} from '../../../../../warehouse/primary-ds/ProPropertyLabelService'; -import {Warehouse} from '../../../../../warehouse/Warehouse'; -import {searchUntilSatisfy, setupCleanAndStartWarehouse, stopWarehouse, truncateWarehouseTables} from '../../../../helpers/warehouse-helpers'; -import {PEntityClassLabel} from '../p-entity-class-label/PEntityClassLabelService.test'; -import {PEntityFullText} from '../p-entity-full-text/PEntityFullTextService.test'; -import {PEntityLabel} from '../p-entity-label/PEntityLabelService.test'; -import {PEntityTimeSpanMock} from '../p-entity-time-span/PEntityTimeSpanService.test'; -import {PEntityTypeMock} from '../p-entity-type/PEntityTypeService.test'; -import {cleanDb} from '../../../../helpers/meta/clean-db.helper'; - -/** - * Testing whole stack from postgres to warehouse - */ -describe('EntityPreviewService', function () { - let wh: Warehouse; - let s: EntityPreviewService; - describe('EntityLabel', function () { - - before(async function () { - // eslint-disable-next-line @typescript-eslint/no-invalid-this - this.timeout(50000); // A very long environment setup. - const injector = await setupCleanAndStartWarehouse({ - primaryDataServices: [ - PEntityService, - REntityService, - REdgeService, - PEdgeService, - DfhOutgoingPropertyService, - ProEntityLabelConfigService, - ], - aggDataServices: [ - // IdentifyingPropertyService, - PEntityLabelService, - REntityLabelService, - EntityPreviewService - ] - }) - wh = injector.get(Warehouse) - s = injector.get(EntityPreviewService) - }) - beforeEach(async () => { - await cleanDb() - await truncateWarehouseTables(wh) - }) - after(async function () { - await stopWarehouse(wh) - }) - - it('should create project entity label', async () => { - const project = await PEntityLabel.createProject(); - const {naming, appellation} = await PEntityLabel.createNamingMock(); - - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx({ - pkEntity: naming.pk_entity ?? -1, - fkProject: project.pk_entity ?? -1 - }), - compare: (val) => val?.entityLabel === appellation.string - }) - }) - - it('should create repo entity label', async () => { - await PEntityLabel.createProject(); - const {naming, appellation} = await PEntityLabel.createNamingMock(); - - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx({ - pkEntity: naming.pk_entity ?? -1, - fkProject: 0 - }), - compare: (val) => val?.entityLabel === appellation.string - }) - }) - }) - - describe('EntityClassLabel', function () { - - before(async function () { - // eslint-disable-next-line @typescript-eslint/no-invalid-this - this.timeout(50000); // A very long environment setup. - const injector = await setupCleanAndStartWarehouse({ - primaryDataServices: [ - REntityService, - PEntityService, - PClassService, - RClassService, - ProProjectService, - DfhClassLabelService, - ProClassLabelService, - ], - aggDataServices: [ - PClassLabelService, - RClassLabelService, - PEntityClassLabelService, - REntityClassLabelService, - EntityPreviewService - ] - }) - wh = injector.get(Warehouse) - s = injector.get(EntityPreviewService) - }) - beforeEach(async () => { - await cleanDb() - await truncateWarehouseTables(wh) - }) - after(async function () { - await stopWarehouse(wh) - }) - - it('should create project entity class label', async () => { - const {prel, pers, cla} = await PEntityClassLabel.createBasicMock(); - - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx({ - pkEntity: pers.pk_entity ?? -1, - fkProject: prel.fk_project ?? -1 - }), - compare: (val) => val?.classLabel === cla.dfh_class_label - }) - }) - - it('should create repo entity class label', async () => { - const {pers, cla} = await PEntityClassLabel.createBasicMock(); - - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx({ - pkEntity: pers.pk_entity ?? -1, - fkProject: 0 - }), - compare: (val) => val?.classLabel === cla.dfh_class_label - }) - }) - }) - describe('EntityTypeLabel', function () { - - before(async function () { - // eslint-disable-next-line @typescript-eslint/no-invalid-this - this.timeout(50000); // A very long environment setup. - const injector = await setupCleanAndStartWarehouse({ - primaryDataServices: [ - DfhOutgoingPropertyService, - ProEntityLabelConfigService, - PEntityService, - PEdgeService, - REntityService, - REdgeService, - DfhClassHasTypePropertyService - ], - aggDataServices: [ - // IdentifyingPropertyService, - PEntityLabelService, - REntityLabelService, - REntityTypeService, - PEntityTypeService, - EntityPreviewService - ] - }) - wh = injector.get(Warehouse) - s = injector.get(EntityPreviewService) - }) - beforeEach(async () => { - await cleanDb() - await truncateWarehouseTables(wh) - }) - after(async function () { - await stopWarehouse(wh) - }) - - it('should create project entity type label', async () => { - const {madrid, project, cityTypeProjRel, cityTypeAppe} = await PEntityTypeMock.createMock(); - - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx({ - pkEntity: madrid.pk_entity ?? -1, - fkProject: project.pk_entity ?? -1 - }), - compare: (val) => { - return val?.fkType === cityTypeProjRel.fk_entity - && val?.typeLabel === cityTypeAppe.string - } - }) - }) - - it('should create repo entity type label', async () => { - const {madrid, cityTypeProjRel, cityTypeAppe} = await PEntityTypeMock.createMock(); - - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx({ - pkEntity: madrid.pk_entity ?? -1, - fkProject: 0 - }), - compare: (val) => { - return val?.fkType === cityTypeProjRel.fk_entity - && val?.typeLabel === cityTypeAppe.string - } - }) - }) - }) - describe('EntityTimeSpan', function () { - - before(async function () { - // eslint-disable-next-line @typescript-eslint/no-invalid-this - this.timeout(50000); // A very long environment setup. - const injector = await setupCleanAndStartWarehouse({ - primaryDataServices: [ - REntityService, - PEntityService, - PEdgeService, - REdgeService, - ], - aggDataServices: [ - PEntityTimeSpanService, - REntityTimeSpanService, - EntityPreviewService - ] - }) - wh = injector.get(Warehouse) - s = injector.get(EntityPreviewService) - }) - beforeEach(async () => { - await cleanDb() - await truncateWarehouseTables(wh) - }) - after(async function () { - await stopWarehouse(wh) - }) - - it('should create project entity time span', async () => { - const {shipVoyage, project} = await PEntityTimeSpanMock.createMock(); - - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx({ - pkEntity: shipVoyage.pk_entity ?? -1, - fkProject: project.pk_entity ?? -1, - }), - compare: (val) => !!val?.timeSpan?.p82 - }) - }) - - it('should create repo entity time span', async () => { - const {shipVoyage} = await PEntityTimeSpanMock.createMock(); - - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx({ - pkEntity: shipVoyage.pk_entity ?? -1, - fkProject: 0, - }), - compare: (val) => !!val?.timeSpan?.p82 - }) - }) - }) - describe('EntityFullText', function () { - - before(async function () { - // eslint-disable-next-line @typescript-eslint/no-invalid-this - this.timeout(50000); // A very long environment setup. - const injector = await setupCleanAndStartWarehouse({ - primaryDataServices: [ - PEntityService, - PEdgeService, - REdgeService, - REntityService, - ProClassFieldsConfigService, - PPropertyService, - RPropertyService, - ProProjectService, - DfhPropertyLabelService, - ProPropertyLabelService, - ProEntityLabelConfigService, - DfhOutgoingPropertyService, - PClassService, - RClassService, - DfhClassLabelService, - ProClassLabelService - ], - aggDataServices: [ - // IdentifyingPropertyService, - PClassFieldLabelService, - RClassFieldLabelService, - PEntityLabelService, - REntityLabelService, - PClassLabelService, - RClassLabelService, - PEntityFullTextService, - REntityFullTextService, - EntityPreviewService - ], - }) - wh = injector.get(Warehouse) - s = injector.get(EntityPreviewService) - }) - beforeEach(async () => { - await cleanDb() - await truncateWarehouseTables(wh) - }) - after(async function () { - await stopWarehouse(wh) - }) - - it('should create project entity full text', async () => { - const {naming, project} = await PEntityFullText.createNamingMock(); - - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx({ - pkEntity: naming.pk_entity ?? -1, - fkProject: project.pk_entity ?? -1 - }), - compare: (val) => { - return val?.fullText === `Appellation in a language (time-indexed) – refers to name: 'Jack the foo'` - && val?.entityLabel === 'Jack the foo' - } - }) - }) - - it('should create repo entity full text', async () => { - const {naming} = await PEntityFullText.createNamingMock(); - - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx({ - pkEntity: naming.pk_entity ?? -1, - fkProject: 0 - }), - compare: (val) => { - return val?.fullText === `Appellation in a language (time-indexed) – refers to name: 'Jack the foo'` - && val?.entityLabel === 'Jack the foo' - } - }) - }) - }) -}) diff --git a/server/src/__tests__/acceptance/warehouse/aggregator-ds/identifying-property/IdentifyingPropertyService.test.ts b/server/src/__tests__/acceptance/warehouse/aggregator-ds/identifying-property/IdentifyingPropertyService.test.ts deleted file mode 100644 index 27fc1c7fd..000000000 --- a/server/src/__tests__/acceptance/warehouse/aggregator-ds/identifying-property/IdentifyingPropertyService.test.ts +++ /dev/null @@ -1,69 +0,0 @@ -/* eslint-disable @typescript-eslint/camelcase */ -import 'reflect-metadata'; -import {expect} from '@loopback/testlab'; -import {IdentifyingPropertyService} from '../../../../../warehouse/aggregator-ds/identifying-property/IdentifyingPropertyService'; -import {Warehouse} from '../../../../../warehouse/Warehouse'; -import {createDfhApiProperty} from '../../../../helpers/atomic/dfh-api-property.helper'; -import {cleanDb} from '../../../../helpers/meta/clean-db.helper'; -import {DfhApiPropertyMock} from '../../../../helpers/data/gvDB/DfhApiPropertyMock'; -import {searchUntilSatisfy, setupCleanAndStartWarehouse, stopWarehouse, truncateWarehouseTables} from '../../../../helpers/warehouse-helpers'; -import {WarehouseStubs} from '../../../../../warehouse/createWarehouse'; -import {DfhOutgoingPropertyService} from '../../../../../warehouse/primary-ds/DfhOutgoingPropertyService'; -const stubs: WarehouseStubs = { - primaryDataServices: [DfhOutgoingPropertyService], - aggDataServices: [IdentifyingPropertyService] -} - -describe('IdentifyingPropertyService', function () { - - let wh: Warehouse; - let s: IdentifyingPropertyService; - before(async function () { - // eslint-disable-next-line @typescript-eslint/no-invalid-this - this.timeout(5000); // A very long environment setup. - const injector = await setupCleanAndStartWarehouse(stubs) - wh = injector.get(Warehouse) - s = injector.get(IdentifyingPropertyService) - }) - beforeEach(async () => { - await cleanDb() - await truncateWarehouseTables(wh) - }) - after(async function () { - await stopWarehouse(wh) - }) - it('should have one identifying property for birth', async () => { - await Promise.all([ - createDfhApiProperty(DfhApiPropertyMock.EN_86_BROUGHT_INTO_LIFE), - createDfhApiProperty(DfhApiPropertyMock.EN_1435_STEMS_FROM) - ]) - - const a = await searchUntilSatisfy({ - notifier$: s.afterChange$, - compare: (val) => val?.length === 1, - getFn: () => s.index.getFromIdx({pkClass: 61}) - }) - - expect(a) - }) - - - it('should have two identifying properties for appellation in a language (time-indexed)', async () => { - await Promise.all([ - createDfhApiProperty(DfhApiPropertyMock.EN_1111_IS_APPE_OF), - createDfhApiProperty(DfhApiPropertyMock.EN_1113_REFERS_TO_NAME), - createDfhApiProperty(DfhApiPropertyMock.EN_1112_USED_IN_LANGUAGE), - createDfhApiProperty(DfhApiPropertyMock.EN_86_BROUGHT_INTO_LIFE), - createDfhApiProperty(DfhApiPropertyMock.EN_1435_STEMS_FROM) - ]) - - const a = await searchUntilSatisfy({ - notifier$: s.afterChange$, - compare: (val) => val?.length === 3, - getFn: () => s.index.getFromIdx({pkClass: 365}) - }) - - expect(a) - }) - -}) diff --git a/server/src/__tests__/acceptance/warehouse/aggregator-ds/p-class-field-label/PClassFieldLabelService.test.ts b/server/src/__tests__/acceptance/warehouse/aggregator-ds/p-class-field-label/PClassFieldLabelService.test.ts deleted file mode 100644 index c3204db10..000000000 --- a/server/src/__tests__/acceptance/warehouse/aggregator-ds/p-class-field-label/PClassFieldLabelService.test.ts +++ /dev/null @@ -1,290 +0,0 @@ -/* eslint-disable @typescript-eslint/camelcase */ -import 'reflect-metadata'; -import {PClassFieldLabelId, PClassFieldLabelService} from '../../../../../warehouse/aggregator-ds/class-field-label/p-class-field-label/PClassFieldLabelService'; -import {WarehouseStubs} from '../../../../../warehouse/createWarehouse'; -import {DfhPropertyLabelService} from '../../../../../warehouse/primary-ds/DfhPropertyLabelService'; -import {PPropertyService} from '../../../../../warehouse/primary-ds/property/PPropertyService'; -import {ProProjectService} from '../../../../../warehouse/primary-ds/ProProjectService'; -import {ProPropertyLabelService} from '../../../../../warehouse/primary-ds/ProPropertyLabelService'; -import {Warehouse} from '../../../../../warehouse/Warehouse'; -import {createDfhApiClass} from '../../../../helpers/atomic/dfh-api-class.helper'; -import {createDfhApiProperty} from '../../../../helpers/atomic/dfh-api-property.helper'; -import {createInfLanguage} from '../../../../helpers/atomic/inf-language.helper'; -import {createProDfhProfileProjRel} from '../../../../helpers/atomic/pro-dfh-profile-proj-rel.helper'; -import {createProProject} from '../../../../helpers/atomic/pro-project.helper'; -import {createProTextProperty, deleteProTextProperty, updateProTextProperty} from '../../../../helpers/atomic/pro-text-property.helper'; -import {DfhApiClassMock} from '../../../../helpers/data/gvDB/DfhApiClassMock'; -import {DfhApiPropertyMock} from '../../../../helpers/data/gvDB/DfhApiPropertyMock'; -import {InfLanguageMock} from '../../../../helpers/data/gvDB/InfLanguageMock'; -import {ProDfhProfileProjRelMock} from '../../../../helpers/data/gvDB/ProDfhProfileProjRelMock'; -import {ProProjectMock} from '../../../../helpers/data/gvDB/ProProjectMock'; -import {ProTextPropertyMock} from '../../../../helpers/data/gvDB/ProTextPropertyMock'; -import {cleanDb} from '../../../../helpers/meta/clean-db.helper'; -import {createTypes} from '../../../../helpers/meta/model.helper'; -import {searchUntilSatisfy, setupCleanAndStartWarehouse, stopWarehouse, truncateWarehouseTables} from '../../../../helpers/warehouse-helpers'; - -const stubs: WarehouseStubs = { - primaryDataServices: [ - PPropertyService, - ProProjectService, - DfhPropertyLabelService, - ProPropertyLabelService - ], - aggDataServices: [PClassFieldLabelService] -} - -describe('PClassFieldLabelService', function () { - - let wh: Warehouse; - let s: PClassFieldLabelService; - before(async function () { - // eslint-disable-next-line @typescript-eslint/no-invalid-this - this.timeout(35000); // A very long environment setup. - const injector = await setupCleanAndStartWarehouse(stubs) - wh = injector.get(Warehouse) - s = injector.get(PClassFieldLabelService) - }) - beforeEach(async () => { - await cleanDb() - await truncateWarehouseTables(wh) - }) - after(async function () { - await stopWarehouse(wh) - }) - - - it('should create outgoing property label for en from ontome', async () => { - const {prel, dfhProp} = await createDfhLabelMock(); - const id: PClassFieldLabelId = { - fkProject: prel.fk_project, - fkClass: dfhProp.dfh_property_domain, - fkProperty: dfhProp.dfh_pk_property, - isOutgoing: true, - } - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => val?.label === DfhApiPropertyMock.EN_86_BROUGHT_INTO_LIFE.dfh_property_label - }) - }) - - - it('should create incoming property label for en from ontome', async () => { - const {prel, dfhProp} = await createDfhLabelMock(); - const expected = DfhApiPropertyMock.EN_86_BROUGHT_INTO_LIFE.dfh_property_inverse_label - const id: PClassFieldLabelId = { - fkProject: prel.fk_project, - fkClass: dfhProp.dfh_property_range, - fkProperty: dfhProp.dfh_pk_property, - isOutgoing: false, - } - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => val?.label === expected - }) - }) - - it('should create outgoing property label for de from geovistory', async () => { - const {prel, dfhProp, gvTxt} = await createGeovistoryLabelMock(); - const expected = gvTxt.string - const id: PClassFieldLabelId = { - fkProject: prel.fk_project, - fkClass: dfhProp.dfh_property_domain, - fkProperty: dfhProp.dfh_pk_property, - isOutgoing: true, - } - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => val?.label === expected - }) - }) - - - it('should create incoming property label for de from geovistory', async () => { - const {prel, dfhProp, gvTxt} = await createGeovistoryLabelIncomingMock(); - const expected = gvTxt.string - const id: PClassFieldLabelId = { - fkProject: prel.fk_project, - fkClass: dfhProp.dfh_property_range, - fkProperty: dfhProp.dfh_pk_property, - isOutgoing: false, - } - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => val?.label === expected - }) - }) - - it('should create outgoing property label de from project', async () => { - const {prel, dfhProp, proTxt} = await createProjectLabelMock(); - const expected = proTxt.string - const id: PClassFieldLabelId = { - fkProject: prel.fk_project, - fkClass: dfhProp.dfh_property_domain, - fkProperty: dfhProp.dfh_pk_property, - isOutgoing: true, - } - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => val?.label === expected - }) - }) - - - it('should create incoming property label de from project', async () => { - const {prel, dfhProp, proTxt} = await createProjectLabelIncomingMock(); - const expected = proTxt.string - const id: PClassFieldLabelId = { - fkProject: prel.fk_project, - fkClass: dfhProp.dfh_property_range, - fkProperty: dfhProp.dfh_pk_property, - isOutgoing: false, - } - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => val?.label === expected - }) - }) - - it('should create incoming property label of person "has appellations"', async () => { - const {project, apiProp: dfhProp, hasAppePropLabel} = await createPersonMock(); - const expected = hasAppePropLabel.string - const id: PClassFieldLabelId = { - fkProject: project.pk_entity ?? -1, - fkClass: DfhApiClassMock.EN_21_PERSON.dfh_pk_class, - fkProperty: dfhProp.dfh_pk_property, - isOutgoing: false, - } - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => val?.label === expected - }) - }) - - it('should update outgoing property label de from project', async () => { - const {prel, dfhProp, proTxt} = await createProjectLabelMock(); - const expected = 'Brachte zur Welt 2' - await updateProTextProperty( - proTxt.pk_entity ?? -1, - {string: expected} - ) - const id: PClassFieldLabelId = { - fkProject: prel.fk_project, - fkClass: dfhProp.dfh_property_domain, - fkProperty: dfhProp.dfh_pk_property, - isOutgoing: true, - } - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => val?.label === expected - }) - - }) - - it('should switch outgoing property label de-project to de-geovistory', async () => { - const {prel, dfhProp, gvTxt, proTxt} = await createProjectLabelMock(); - const id: PClassFieldLabelId = { - fkProject: prel.fk_project, - fkClass: dfhProp.dfh_property_domain, - fkProperty: dfhProp.dfh_pk_property, - isOutgoing: true, - } - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => val?.label === proTxt.string - }) - - - await deleteProTextProperty(proTxt.pk_entity ?? -1) - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => val?.label === gvTxt.string - }) - - - }) - - it('should switch outgoing property label from de-project to de-ontome', async () => { - const {prel, dfhProp, proTxt, gvTxt} = await createProjectLabelMock(); - const id: PClassFieldLabelId = { - fkProject: prel.fk_project, - fkClass: dfhProp.dfh_property_domain, - fkProperty: dfhProp.dfh_pk_property, - isOutgoing: true, - } - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => val?.label === proTxt.string - }) - - await deleteProTextProperty(proTxt.pk_entity ?? -1) - await deleteProTextProperty(gvTxt.pk_entity ?? -1) - - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => val?.label === dfhProp.dfh_property_label - }) - - }) - - -}) -async function createDfhLabelMock() { - await createInfLanguage(InfLanguageMock.GERMAN); - await createProProject(ProProjectMock.PROJECT_1); - const prel = await createProDfhProfileProjRel(ProDfhProfileProjRelMock.PROJ_1_PROFILE_4); - const dfhProp = await createDfhApiProperty(DfhApiPropertyMock.EN_86_BROUGHT_INTO_LIFE); - return {prel, dfhProp} -} - -async function createGeovistoryLabelMock() { - const {prel, dfhProp} = await createDfhLabelMock(); - await createTypes() - const gvTxt = await createProTextProperty(ProTextPropertyMock.PROJ_DEF_DE_PROPERTY_BROUGHT_INTO_LIFE) - return {prel, dfhProp, gvTxt} -} - -async function createGeovistoryLabelIncomingMock() { - const {prel, dfhProp} = await createDfhLabelMock(); - await createTypes() - const gvTxt = await createProTextProperty(ProTextPropertyMock.PROJ_DEF_DE_PROPERTY_BROUGHT_INTO_LIFE_REVERSE) - return {prel, dfhProp, gvTxt} -} - -async function createProjectLabelMock() { - const {prel, dfhProp, gvTxt} = await createGeovistoryLabelMock(); - const proTxt = await createProTextProperty(ProTextPropertyMock.PROJ_1_PROPERTY_BROUGHT_INTO_LIFE) - return {prel, dfhProp, gvTxt, proTxt} -} - - -async function createProjectLabelIncomingMock() { - const {prel, dfhProp, gvTxt} = await createGeovistoryLabelMock(); - const proTxt = await createProTextProperty(ProTextPropertyMock.PROJ_1_PROPERTY_BROUGHT_INTO_LIFE_REVERSE) - return {prel, dfhProp, gvTxt, proTxt} -} - - -async function createPersonMock() { - await createTypes() - await createInfLanguage(InfLanguageMock.GERMAN); - const project = await createProProject(ProProjectMock.PROJECT_1); - await createProProject(ProProjectMock.DEFAULT_PROJECT); - await createProDfhProfileProjRel(ProDfhProfileProjRelMock.PROJ_1_PROFILE_4); - await createDfhApiClass(DfhApiClassMock.EN_21_PERSON); - - const apiProp = await createDfhApiProperty(DfhApiPropertyMock.EN_1111_IS_APPE_OF); - const hasAppePropLabel = await createProTextProperty(ProTextPropertyMock.PROJ_1_PROPERTY_PERSON_HAS_APPELLATION) - return {project, apiProp, hasAppePropLabel}; -} diff --git a/server/src/__tests__/acceptance/warehouse/aggregator-ds/p-class-label/PClassLabelService.test.ts b/server/src/__tests__/acceptance/warehouse/aggregator-ds/p-class-label/PClassLabelService.test.ts deleted file mode 100644 index 58987d9bf..000000000 --- a/server/src/__tests__/acceptance/warehouse/aggregator-ds/p-class-label/PClassLabelService.test.ts +++ /dev/null @@ -1,157 +0,0 @@ -/* eslint-disable @typescript-eslint/camelcase */ -import 'reflect-metadata'; -import {expect} from '@loopback/testlab'; -import {PClassLabelService} from '../../../../../warehouse/aggregator-ds/class-label/p-class-label/PClassLabelService'; -import {WarehouseStubs} from '../../../../../warehouse/createWarehouse'; -import {PClassService} from '../../../../../warehouse/primary-ds/class/PClassService'; -import {DfhClassLabelService} from '../../../../../warehouse/primary-ds/DfhClassLabelService'; -import {ProClassLabelService} from '../../../../../warehouse/primary-ds/ProClassLabelService'; -import {ProProjectService} from '../../../../../warehouse/primary-ds/ProProjectService'; -import {PK_DEFAULT_CONFIG_PROJECT, Warehouse} from '../../../../../warehouse/Warehouse'; -import {createDfhApiClass} from '../../../../helpers/atomic/dfh-api-class.helper'; -import {createInfLanguage} from '../../../../helpers/atomic/inf-language.helper'; -import {createProDfhProfileProjRel} from '../../../../helpers/atomic/pro-dfh-profile-proj-rel.helper'; -import {createProProject} from '../../../../helpers/atomic/pro-project.helper'; -import {createProTextPropertyClassLabel, deleteProTextProperty, updateProTextProperty} from '../../../../helpers/atomic/pro-text-property.helper'; -import {createTypes} from '../../../../helpers/meta/model.helper'; -import {InfLanguageMock} from '../../../../helpers/data/gvDB/InfLanguageMock'; -import {ProDfhProfileProjRelMock} from '../../../../helpers/data/gvDB/ProDfhProfileProjRelMock'; -import {ProProjectMock} from '../../../../helpers/data/gvDB/ProProjectMock'; -import {cleanDb} from '../../../../helpers/meta/clean-db.helper'; -import {setupCleanAndStartWarehouse, stopWarehouse, truncateWarehouseTables, waitForClassPreview} from '../../../../helpers/warehouse-helpers'; -const pClassLabelService: WarehouseStubs = { - primaryDataServices: [ - PClassService, - ProProjectService, - DfhClassLabelService, - ProClassLabelService - ], - aggDataServices: [ - PClassLabelService - ] -} -describe('PClassLabelService', function () { - - let wh: Warehouse; - - before(async function () { - // eslint-disable-next-line @typescript-eslint/no-invalid-this - this.timeout(5000); // A very long environment setup. - const injector = await setupCleanAndStartWarehouse(pClassLabelService) - wh = injector.get(Warehouse) - }) - beforeEach(async () => { - await cleanDb() - await truncateWarehouseTables(wh) - }) - after(async function () { - await stopWarehouse(wh) - }) - - it('should create class label of Person: de-ontome', async () => { - const { prel, cla } = await createBasicMock(); - const result = await waitForClassPreview(wh, [ - { fk_class: { eq: cla.dfh_pk_class } }, - { fk_project: { eq: prel.fk_project } }, - { label: { eq: 'Foo' } }, - ]) - expect(result.label).to.equal('Foo') - }) - it('should create class label of Person: de-geovistory', async () => { - const { prel, cla, gvTxt } = await createGeovistoryLabelMock(); - const result = await waitForClassPreview(wh, [ - { fk_class: { eq: cla.dfh_pk_class } }, - { fk_project: { eq: prel.fk_project } }, - { label: { eq: gvTxt.string } }, - ]) - expect(result.label).to.equal('Geburt (GV default)') - }) - it('should create class label of Person: de-project', async () => { - const { prel, cla, proTxt } = await createProjectLabelMock(); - const result = await waitForClassPreview(wh, [ - { fk_class: { eq: cla.dfh_pk_class } }, - { fk_project: { eq: prel.fk_project } }, - { label: { eq: proTxt.string } }, - ]) - expect(result.label).to.equal('Geburt') - }) - it('should update class label of Person: de-project', async () => { - const { prel, cla, proTxt } = await createProjectLabelMock(); - await updateProTextProperty( - proTxt.pk_entity ?? -1, - { string: 'Niederkunft' } - ) - const result = await waitForClassPreview(wh, [ - { fk_class: { eq: cla.dfh_pk_class } }, - { fk_project: { eq: prel.fk_project } }, - { label: { eq: 'Niederkunft' } }, - ]) - expect(result.label).to.equal('Niederkunft') - }) - - it('should switch class label of Person: de-project to de-geovistory', async () => { - const { prel, cla, proTxt } = await createProjectLabelMock(); - - await deleteProTextProperty(proTxt.pk_entity ?? -1) - const result = await waitForClassPreview(wh, [ - { fk_class: { eq: cla.dfh_pk_class } }, - { fk_project: { eq: prel.fk_project } }, - { label: { eq: 'Geburt (GV default)' } }, - ]) - expect(result.label).to.equal('Geburt (GV default)') - - }) - - it('should switch class label of Person: de-project to de-ontome', async () => { - const { prel, cla, proTxt, gvTxt } = await createProjectLabelMock(); - - await Promise.all([ - deleteProTextProperty(proTxt.pk_entity ?? -1), - deleteProTextProperty(gvTxt.pk_entity ?? -1), - waitForClassPreview(wh, [ - {fk_class: {eq: cla.dfh_pk_class}}, - {fk_project: {eq: prel.fk_project}}, - {label: {eq: 'Foo'}}, - ]) - ]) - - }) - - -}) -async function createBasicMock() { - await createInfLanguage(InfLanguageMock.GERMAN); - await createProProject(ProProjectMock.PROJECT_1); - const prel = await createProDfhProfileProjRel(ProDfhProfileProjRelMock.PROJ_1_PROFILE_4); - const cla = await createDfhApiClass({ - dfh_pk_class: 1, - dfh_class_label: 'Foo', - dfh_class_label_language: 'de', - dfh_fk_profile: ProDfhProfileProjRelMock.PROJ_1_PROFILE_4.fk_profile - }); - return { prel, cla } -} - -async function createGeovistoryLabelMock() { - const { prel, cla } = await createBasicMock(); - await createTypes() - const gvTxt = await createProTextPropertyClassLabel( - PK_DEFAULT_CONFIG_PROJECT, - 1, - 'Geburt (GV default)', - 18605 //german - ) - return { prel, cla, gvTxt } -} - - -async function createProjectLabelMock() { - const { prel, cla, gvTxt } = await createGeovistoryLabelMock(); - const proTxt = await createProTextPropertyClassLabel( - prel.fk_project ?? -1, - 1, - 'Geburt', - 18605 //german - ) - return { prel, cla, gvTxt, proTxt } -} diff --git a/server/src/__tests__/acceptance/warehouse/aggregator-ds/p-entity-class-label/PEntityClassLabelService.test.ts b/server/src/__tests__/acceptance/warehouse/aggregator-ds/p-entity-class-label/PEntityClassLabelService.test.ts deleted file mode 100644 index f7ad3e712..000000000 --- a/server/src/__tests__/acceptance/warehouse/aggregator-ds/p-entity-class-label/PEntityClassLabelService.test.ts +++ /dev/null @@ -1,108 +0,0 @@ -/* eslint-disable @typescript-eslint/camelcase */ -import 'reflect-metadata'; -import {expect} from '@loopback/testlab'; -import {PEntityClassLabelService} from '../../../../../warehouse/aggregator-ds/entity-class-label/p-entity-class-label/PEntityClassLabelService'; -import {Warehouse} from '../../../../../warehouse/Warehouse'; -import {createDfhApiClass} from '../../../../helpers/atomic/dfh-api-class.helper'; -import {createInfLanguage} from '../../../../helpers/atomic/inf-language.helper'; -import {createProDfhProfileProjRel} from '../../../../helpers/atomic/pro-dfh-profile-proj-rel.helper'; -import {createProInfoProjRel, updateProInfoProjRel} from '../../../../helpers/atomic/pro-info-proj-rel.helper'; -import {createProProject} from '../../../../helpers/atomic/pro-project.helper'; -import {cleanDb} from '../../../../helpers/meta/clean-db.helper'; -import {DfhApiClassMock} from '../../../../helpers/data/gvDB/DfhApiClassMock'; -import {InfLanguageMock} from '../../../../helpers/data/gvDB/InfLanguageMock'; -import {InfResourceMock} from '../../../../helpers/data/gvDB/InfResourceMock'; -import {ProDfhProfileProjRelMock} from '../../../../helpers/data/gvDB/ProDfhProfileProjRelMock'; -import {ProInfoProjRelMock} from '../../../../helpers/data/gvDB/ProInfoProjRelMock'; -import {ProProjectMock} from '../../../../helpers/data/gvDB/ProProjectMock'; -import {searchUntilSatisfy, setupCleanAndStartWarehouse, stopWarehouse, truncateWarehouseTables, waitForEntityPreview} from '../../../../helpers/warehouse-helpers'; -import {WarehouseStubs} from '../../../../../warehouse/createWarehouse'; -import {PEntityService} from '../../../../../warehouse/primary-ds/entity/PEntityService'; -import {PClassLabelService} from '../../../../../warehouse/aggregator-ds/class-label/p-class-label/PClassLabelService'; -import {DfhClassLabelService} from '../../../../../warehouse/primary-ds/DfhClassLabelService'; -import {PClassService} from '../../../../../warehouse/primary-ds/class/PClassService'; -import {ProProjectService} from '../../../../../warehouse/primary-ds/ProProjectService'; -import {ProClassLabelService} from '../../../../../warehouse/primary-ds/ProClassLabelService'; -import {EntityPreviewService} from '../../../../../warehouse/aggregator-ds/entity-preview/EntityPreviewService'; -import {createInfResource} from '../../../../helpers/atomic/inf-resource.helper'; - -const pEntityClassLabelStub: WarehouseStubs = { - primaryDataServices: [ - PEntityService, - PClassService, - ProProjectService, - DfhClassLabelService, - ProClassLabelService, - ], - aggDataServices: [ - PClassLabelService, - PEntityClassLabelService, - EntityPreviewService - ] -} -describe('PEntityClassLabelService', function () { - - let wh: Warehouse; - let s: PEntityClassLabelService - before(async function () { - // eslint-disable-next-line @typescript-eslint/no-invalid-this - this.timeout(5000); // A very long environment setup. - const injector = await setupCleanAndStartWarehouse(pEntityClassLabelStub) - wh = injector.get(Warehouse) - s = injector.get(PEntityClassLabelService) - }) - beforeEach(async () => { - await cleanDb() - await truncateWarehouseTables(wh) - }) - after(async function () { - await stopWarehouse(wh) - }) - - it('should create entity class label of Person', async () => { - const {prel, pers, cla} = await PEntityClassLabel.createBasicMock(); - const result = await waitForEntityPreview(wh, [ - {pk_entity: {eq: pers.pk_entity}}, - {fk_project: {eq: prel.fk_project}}, - {class_label: {eq: cla.dfh_class_label}}, - ]) - expect(result.class_label).to.equal('Person') - }) - - - it('should delete entity class label from index when entity is removed from project', async () => { - const {prel, pers, cla} = await PEntityClassLabel.createBasicMock(); - - const result = await waitForEntityPreview(wh, [ - {pk_entity: {eq: pers.pk_entity}}, - {fk_project: {eq: prel.fk_project}}, - {class_label: {eq: cla.dfh_class_label}}, - ]) - expect(result.class_label).to.equal('Person') - // remove person from the project - await updateProInfoProjRel(prel.pk_entity ?? -1, {is_in_project: false}) - - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdxWithTmsps({pkEntity: pers.pk_entity ?? -1, fkProject: prel.fk_project ?? -1}), - compare: (val) => !!val?.deleted - - }) - }) - - -}) -export namespace PEntityClassLabel { - - export async function createBasicMock() { - // CLASS + LABEL - await createInfLanguage(InfLanguageMock.GERMAN); - await createProProject(ProProjectMock.PROJECT_1); - await createProDfhProfileProjRel(ProDfhProfileProjRelMock.PROJ_1_PROFILE_4); - const cla = await createDfhApiClass(DfhApiClassMock.EN_21_PERSON); - // PERSON - const pers = await createInfResource(InfResourceMock.PERSON_1) - const prel = await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_PERSON_1) - return {prel, pers, cla} - } -} diff --git a/server/src/__tests__/acceptance/warehouse/aggregator-ds/p-entity-full-text/PEntityFullTextService.test.ts b/server/src/__tests__/acceptance/warehouse/aggregator-ds/p-entity-full-text/PEntityFullTextService.test.ts deleted file mode 100644 index c90a665aa..000000000 --- a/server/src/__tests__/acceptance/warehouse/aggregator-ds/p-entity-full-text/PEntityFullTextService.test.ts +++ /dev/null @@ -1,269 +0,0 @@ -/* eslint-disable @typescript-eslint/camelcase */ -import 'reflect-metadata'; -import {expect} from '@loopback/testlab'; -import {PClassFieldLabelService} from '../../../../../warehouse/aggregator-ds/class-field-label/p-class-field-label/PClassFieldLabelService'; -import {PClassLabelService} from '../../../../../warehouse/aggregator-ds/class-label/p-class-label/PClassLabelService'; -import {PEntityFullTextService} from '../../../../../warehouse/aggregator-ds/entity-full-text/p-entity-full-text/PEntityFullTextService'; -import {PEntityLabelService} from '../../../../../warehouse/aggregator-ds/entity-label/p-entity-label/PEntityLabelService'; -import {REntityLabelService} from '../../../../../warehouse/aggregator-ds/entity-label/r-entity-label/REntityLabelService'; -import {EntityPreviewService} from '../../../../../warehouse/aggregator-ds/entity-preview/EntityPreviewService'; -import {WarehouseStubs} from '../../../../../warehouse/createWarehouse'; -import {PClassService} from '../../../../../warehouse/primary-ds/class/PClassService'; -import {DfhClassLabelService} from '../../../../../warehouse/primary-ds/DfhClassLabelService'; -import {DfhOutgoingPropertyService} from '../../../../../warehouse/primary-ds/DfhOutgoingPropertyService'; -import {DfhPropertyLabelService} from '../../../../../warehouse/primary-ds/DfhPropertyLabelService'; -import {PEdgeService} from '../../../../../warehouse/primary-ds/edge/PEdgeService'; -import {REdgeService} from '../../../../../warehouse/primary-ds/edge/REdgeService'; -import {PEntityId, PEntityService} from '../../../../../warehouse/primary-ds/entity/PEntityService'; -import {REntityService} from '../../../../../warehouse/primary-ds/entity/REntityService'; -import {ProClassFieldsConfigService} from '../../../../../warehouse/primary-ds/ProClassFieldsConfigService'; -import {ProClassLabelService} from '../../../../../warehouse/primary-ds/ProClassLabelService'; -import {ProEntityLabelConfigService} from '../../../../../warehouse/primary-ds/ProEntityLabelConfigService'; -import {PPropertyService} from '../../../../../warehouse/primary-ds/property/PPropertyService'; -import {ProProjectService} from '../../../../../warehouse/primary-ds/ProProjectService'; -import {ProPropertyLabelService} from '../../../../../warehouse/primary-ds/ProPropertyLabelService'; -import {Warehouse} from '../../../../../warehouse/Warehouse'; -import {createDfhApiClass} from '../../../../helpers/atomic/dfh-api-class.helper'; -import {createDfhApiProperty} from '../../../../helpers/atomic/dfh-api-property.helper'; -import {createInfAppellation} from '../../../../helpers/atomic/inf-appellation.helper'; -import {createInfLanguage} from '../../../../helpers/atomic/inf-language.helper'; -import {createInfStatement} from '../../../../helpers/atomic/inf-statement.helper'; -import {createInfResource} from '../../../../helpers/atomic/inf-resource.helper'; -import {createInfTimePrimitive} from '../../../../helpers/atomic/inf-time-primitive.helper'; -import {createProClassFieldConfig} from '../../../../helpers/atomic/pro-class-field-config.helper'; -import {createProDfhProfileProjRel} from '../../../../helpers/atomic/pro-dfh-profile-proj-rel.helper'; -import {addInfoToProject, updateProInfoProjRel} from '../../../../helpers/atomic/pro-info-proj-rel.helper'; -import {createProProject} from '../../../../helpers/atomic/pro-project.helper'; -import {createProTextProperty} from '../../../../helpers/atomic/pro-text-property.helper'; -import {createTypes} from '../../../../helpers/meta/model.helper'; -import {cleanDb} from '../../../../helpers/meta/clean-db.helper'; -import {DfhApiClassMock} from '../../../../helpers/data/gvDB/DfhApiClassMock'; -import {DfhApiPropertyMock} from '../../../../helpers/data/gvDB/DfhApiPropertyMock'; -import {InfAppellationMock} from '../../../../helpers/data/gvDB/InfAppellationMock'; -import {InfLanguageMock} from '../../../../helpers/data/gvDB/InfLanguageMock'; -import {InfStatementMock} from '../../../../helpers/data/gvDB/InfStatementMock'; -import {InfResourceMock} from '../../../../helpers/data/gvDB/InfResourceMock'; -import {InfTimePrimitiveMock} from '../../../../helpers/data/gvDB/InfTimePrimitiveMock'; -import {ProClassFieldConfigMock} from '../../../../helpers/data/gvDB/ProClassFieldConfigMock'; -import {ProDfhProfileProjRelMock} from '../../../../helpers/data/gvDB/ProDfhProfileProjRelMock'; -import {ProProjectMock} from '../../../../helpers/data/gvDB/ProProjectMock'; -import {ProTextPropertyMock} from '../../../../helpers/data/gvDB/ProTextPropertyMock'; -import {SysSystemTypeMock} from '../../../../helpers/data/gvDB/SysSystemTypeMock'; -import {searchUntilSatisfy, setupCleanAndStartWarehouse, stopWarehouse, truncateWarehouseTables, waitForEntityPreview, waitForEntityPreviewUntil} from '../../../../helpers/warehouse-helpers'; -const pEntityFullTextStub: WarehouseStubs = { - primaryDataServices: [ - PEntityService, - PEdgeService, - REdgeService, - REntityService, - ProClassFieldsConfigService, - PPropertyService, - ProProjectService, - DfhPropertyLabelService, - ProPropertyLabelService, - ProEntityLabelConfigService, - DfhOutgoingPropertyService, - PClassService, - DfhClassLabelService, - ProClassLabelService - ], - aggDataServices: [ - // IdentifyingPropertyService, - PClassFieldLabelService, - PEntityLabelService, - REntityLabelService, - PClassLabelService, - PEntityFullTextService, - EntityPreviewService - ], -} -/** - * Testing whole stack from postgres to warehouse - */ -describe('PEntityFullTextService', function () { - let wh: Warehouse; - let s: PEntityFullTextService; - before(async function () { - // eslint-disable-next-line @typescript-eslint/no-invalid-this - this.timeout(15000); // A very long environment setup. - const injector = await setupCleanAndStartWarehouse(pEntityFullTextStub) - wh = injector.get(Warehouse) - s = injector.get(PEntityFullTextService) - }) - beforeEach(async () => { - await cleanDb() - await truncateWarehouseTables(wh) - }) - after(async function () { - await stopWarehouse(wh) - }) - - it('should create full text of naming', async () => { - const {naming, project} = await PEntityFullText.createNamingMock(); - - const result = await waitForEntityPreviewUntil(wh, (item) => { - // console.log(item) - return item.pk_entity === naming.pk_entity - && item.fk_project === project.pk_entity - && item.full_text === `Appellation in a language (time-indexed) – refers to name: 'Jack the foo'` - }) - - expect(result).not.to.be.undefined(); - }) - - it('should create full text of naming with person', async () => { - const {naming, project} = await PEntityFullText.createNamingAndPersonMock(); - - const expected = `Appellation in a language (time-indexed) – refers to name: 'Jack the foo', is appellation for language of: 'Jack the foo'` - - await waitForEntityPreview(wh, [ - {pk_entity: {eq: naming.pk_entity}}, - {fk_project: {eq: project.pk_entity}}, - {full_text: {eq: expected}}, - ]) - - }) - - // it('should create incoming property label of person "has appellations"', async () => { - // const {project, hasAppePropLabel} = await createNamingAndPersonMock(); - // const expected = hasAppePropLabel.string - - // const result = await waitUntilSatisfy(wh.agg.pClassFieldLabel.afterPut$, (item) => { - // return item.key.fkProject === project.pk_entity - // && item.key.fkClass === DfhApiClassMock.EN_21_PERSON.dfh_pk_class - // && item.key.fkProperty === 1111 - // && item.key.isOutgoing === false - // && item.val === expected - // }) - // expect(result.val).to.equal(expected) - - // }) - it('should create full text of person', async () => { - const {person, project} = await PEntityFullText.createNamingAndPersonMock(); - - const expected = `Person – has appellations: 'Jack the foo'` - - const id: PEntityId = { - pkEntity: person.pk_entity ?? -1, - fkProject: project.pk_entity ?? -1 - } - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => { - return val?.fullText === expected - } - }) - - }) - it('should update full text of person', async () => { - const {person, project} = await PEntityFullText.createNamingAndPersonMock(); - - let expected = `Person – has appellations: 'Jack the foo'` - const id: PEntityId = { - pkEntity: person.pk_entity ?? -1, - fkProject: project.pk_entity ?? -1 - } - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => { - return val?.fullText === expected - } - }) - - await createProTextProperty({ - fk_dfh_class: 21, - fk_project: ProProjectMock.PROJECT_1.pk_entity, - fk_language: ProProjectMock.PROJECT_1.fk_language, - fk_system_type: SysSystemTypeMock.PRO_TEXT_PROPTERTY_LABEL.pk_entity, - string: 'Human' - }) - expected = `Human – has appellations: 'Jack the foo'` - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => { - return val?.fullText === expected - } - }) - - }) - - it('should delete entity full text from index when entity is removed from project', async () => { - const {naming, namingProjRel, project} = await PEntityFullText.createNamingMock(); - const id: PEntityId = { - pkEntity: naming.pk_entity ?? -1, - fkProject: project.pk_entity ?? -1 - } - const result = await waitForEntityPreviewUntil(wh, (item) => { - return item.pk_entity === naming.pk_entity - && item.fk_project === project.pk_entity - && item.full_text === `Appellation in a language (time-indexed) – refers to name: 'Jack the foo'` - }) - - expect(result).not.to.be.undefined(); - // remove person from the project - await updateProInfoProjRel(namingProjRel.pk_entity ?? -1, {is_in_project: false}) - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdxWithTmsps(id), - compare: (item) => !!item?.deleted - }) - - }) - - -}) - -export namespace PEntityFullText { - - - export async function createNamingAndPersonMock() { - // NAMING - const {naming, project, appellation, propertyRefersToName} = await createNamingMock(); - // PERSON - const {person, classPerson, hasAppePropLabel} = await createPersonMock(); - return {project, appellation, naming, person, classPerson, hasAppePropLabel, propertyRefersToName}; - } - - export async function createPersonMock() { - await createTypes() - const classPerson = await createDfhApiClass(DfhApiClassMock.EN_21_PERSON); - await createDfhApiProperty(DfhApiPropertyMock.EN_1111_IS_APPE_OF); - const hasAppePropLabel = await createProTextProperty(ProTextPropertyMock.PROJ_1_PROPERTY_PERSON_HAS_APPELLATION) - const person = await createInfResource(InfResourceMock.PERSON_1); - await addInfoToProject(person.pk_entity, ProProjectMock.PROJECT_1.pk_entity); - const stmt = await createInfStatement(InfStatementMock.NAME_1_TO_PERSON); - await addInfoToProject(stmt.pk_entity, ProProjectMock.PROJECT_1.pk_entity); - return {person, classPerson, hasAppePropLabel}; - } - - export async function createNamingMock() { - await createInfLanguage(InfLanguageMock.GERMAN); - const project = await createProProject(ProProjectMock.PROJECT_1); - await createProProject(ProProjectMock.DEFAULT_PROJECT); - await createDfhApiClass(DfhApiClassMock.EN_365_NAMING); - await createProDfhProfileProjRel(ProDfhProfileProjRelMock.PROJ_1_PROFILE_4); - const propertyRefersToName = await createDfhApiProperty(DfhApiPropertyMock.EN_1113_REFERS_TO_NAME); - - await createProClassFieldConfig(ProClassFieldConfigMock.PROJ_DEF_C365_NAMING_P1111_IS_APPE_OF) - await createProClassFieldConfig(ProClassFieldConfigMock.PROJ_DEF_C365_NAMING_P1113_REFERS_TO_NAME) - - const naming = await createInfResource(InfResourceMock.NAMING_1); - const namingProjRel = await addInfoToProject(naming.pk_entity, ProProjectMock.PROJECT_1.pk_entity); - - const appellation = await createInfAppellation(InfAppellationMock.JACK_THE_FOO); - const stmtToAppe = await createInfStatement(InfStatementMock.NAME_1_TO_APPE); - await addInfoToProject(stmtToAppe.pk_entity, ProProjectMock.PROJECT_1.pk_entity); - - await createInfTimePrimitive(InfTimePrimitiveMock.TP_1) - - const stmtToTp = await createInfStatement(InfStatementMock.NAMING_1_ONGOING_THROUGHOUT_TP_1) - await addInfoToProject(stmtToTp.pk_entity, project.pk_entity) - - return {naming, namingProjRel, project, appellation, propertyRefersToName}; - } - - -} diff --git a/server/src/__tests__/acceptance/warehouse/aggregator-ds/p-entity-label/PEntityLabelService.test.ts b/server/src/__tests__/acceptance/warehouse/aggregator-ds/p-entity-label/PEntityLabelService.test.ts deleted file mode 100644 index 38ca97630..000000000 --- a/server/src/__tests__/acceptance/warehouse/aggregator-ds/p-entity-label/PEntityLabelService.test.ts +++ /dev/null @@ -1,260 +0,0 @@ -/* eslint-disable @typescript-eslint/no-invalid-this */ -/* eslint-disable @typescript-eslint/camelcase */ -import {expect} from '@loopback/testlab'; -import 'reflect-metadata'; -import {EntityLabel, ENTITY_LABEL_MAX_LENGTH} from '../../../../../warehouse/aggregator-ds/entity-label/entity-label.commons'; -import {PEntityLabelService} from '../../../../../warehouse/aggregator-ds/entity-label/p-entity-label/PEntityLabelService'; -import {EntityPreviewService} from '../../../../../warehouse/aggregator-ds/entity-preview/EntityPreviewService'; -import {WarehouseStubs} from '../../../../../warehouse/createWarehouse'; -import {DfhOutgoingPropertyService} from '../../../../../warehouse/primary-ds/DfhOutgoingPropertyService'; -import {PEdgeService} from '../../../../../warehouse/primary-ds/edge/PEdgeService'; -import {PEntityService} from '../../../../../warehouse/primary-ds/entity/PEntityService'; -import {ProEntityLabelConfigService} from '../../../../../warehouse/primary-ds/ProEntityLabelConfigService'; -import {Warehouse} from '../../../../../warehouse/Warehouse'; -import {createDfhApiClass} from '../../../../helpers/atomic/dfh-api-class.helper'; -import {createInfAppellation} from '../../../../helpers/atomic/inf-appellation.helper'; -import {createInfLanguage} from '../../../../helpers/atomic/inf-language.helper'; -import {createInfStatement} from '../../../../helpers/atomic/inf-statement.helper'; -import {createInfResource} from '../../../../helpers/atomic/inf-resource.helper'; -import {createProInfoProjRel, updateProInfoProjRel} from '../../../../helpers/atomic/pro-info-proj-rel.helper'; -import {createProProject} from '../../../../helpers/atomic/pro-project.helper'; -import {DfhApiClassMock} from '../../../../helpers/data/gvDB/DfhApiClassMock'; -import {InfAppellationMock} from '../../../../helpers/data/gvDB/InfAppellationMock'; -import {InfLanguageMock} from '../../../../helpers/data/gvDB/InfLanguageMock'; -import {InfStatementMock} from '../../../../helpers/data/gvDB/InfStatementMock'; -import {InfResourceMock} from '../../../../helpers/data/gvDB/InfResourceMock'; -import {ProInfoProjRelMock} from '../../../../helpers/data/gvDB/ProInfoProjRelMock'; -import {ProProjectMock} from '../../../../helpers/data/gvDB/ProProjectMock'; -import {cleanDb} from '../../../../helpers/meta/clean-db.helper'; -import {searchUntilSatisfy, setupCleanAndStartWarehouse, stopWarehouse, truncateWarehouseTables, waitForEntityPreview, waitForEntityPreviewUntil} from '../../../../helpers/warehouse-helpers'; -import {createUnion2Mock} from '../r-entity-label/REntityLabelService.test'; -export const pEntityLabelStub: WarehouseStubs = { - primaryDataServices: [ - DfhOutgoingPropertyService, - ProEntityLabelConfigService, - PEntityService, - PEdgeService - ], - aggDataServices: [ - // IdentifyingPropertyService, - PEntityLabelService, - EntityPreviewService - ] -} -/** - * Testing whole stack from postgres to warehouse - */ -describe('PEntityLabelService', function () { - let wh: Warehouse; - let s: PEntityLabelService - before(async function () { - // eslint-disable-next-line @typescript-eslint/no-invalid-this - this.timeout(5000); // A very long environment setup. - const injector = await setupCleanAndStartWarehouse(pEntityLabelStub) - wh = injector.get(Warehouse) - s = injector.get(PEntityLabelService) - }) - beforeEach(async () => { - await cleanDb() - await truncateWarehouseTables(wh) - }) - after(async function () { - await stopWarehouse(wh) - }) - - it('should create entity label of naming', async () => { - const project = await PEntityLabel.createProject(); - const {naming, appellation} = await PEntityLabel.createNamingMock(); - const result = await waitForEntityPreview(wh, [ - {pk_entity: {eq: naming.pk_entity}}, - {fk_project: {eq: project.pk_entity}}, - {entity_label: {eq: appellation.string}}, - ]) - expect(result.entity_label).to.equal(appellation.string) - }) - it('should create entity label of person', async () => { - const project = await PEntityLabel.createProject(); - const {person, appellation} = await PEntityLabel.createNamingAndPersonMock(); - const result = await waitForEntityPreview(wh, [ - {pk_entity: {eq: person.pk_entity}}, - {fk_project: {eq: project.pk_entity}}, - {entity_label: {eq: appellation.string}}, - ]) - expect(result.entity_label).to.equal(appellation.string) - }) - it('should update entity label of person after removing stmt 1111 from project', async () => { - const project = await PEntityLabel.createProject(); - const {person, appellation} = await PEntityLabel.createNamingAndPersonMock(); - let result = await waitForEntityPreview(wh, [ - {pk_entity: {eq: person.pk_entity}}, - {fk_project: {eq: project.pk_entity}}, - {entity_label: {eq: appellation.string}}, - ]) - expect(result.entity_label).to.equal(appellation.string) - - await updateProInfoProjRel( - ProInfoProjRelMock.PROJ_1_STMT_NAME_1_TO_PERSON.pk_entity ?? -1 - , { - ...ProInfoProjRelMock.PROJ_1_STMT_NAME_1_TO_PERSON, - is_in_project: false - }) - - result = await waitForEntityPreview(wh, [ - {pk_entity: {eq: person.pk_entity}}, - {fk_project: {eq: project.pk_entity}}, - {entity_label: {eq: '(no label)'}}, - ]) - expect(result.entity_label).to.equal('(no label)') - }) - - it('should create entity label of naming and add person', async () => { - const project = await PEntityLabel.createProject(); - const {naming, appellation} = await PEntityLabel.createNamingMock(); - let result = await waitForEntityPreview(wh, [ - {pk_entity: {eq: naming.pk_entity}}, - {fk_project: {eq: project.pk_entity}}, - {entity_label: {eq: appellation.string}}, - ]) - expect(result.entity_label).to.equal(appellation.string) - - const person = await PEntityLabel.createPersonMock(); - result = await waitForEntityPreview(wh, [ - {pk_entity: {eq: person.pk_entity}}, - {fk_project: {eq: project.pk_entity}}, - {entity_label: {eq: appellation.string}}, - ]) - expect(result.entity_label).to.equal(appellation.string) - }) - - it('should create entity label of Union with 3 label parts', async () => { - const project = await PEntityLabel.createProject(); - await PEntityLabel.createNamingMock(); - await PEntityLabel.createPersonMock(); - const {union} = await createUnion2Mock() - const result = await waitForEntityPreviewUntil(wh, (item) => { - // console.log(item.entity_label) - return item.pk_entity === union.pk_entity - && item.fk_project === project.pk_entity - && item.entity_label === '(no label), Jack the foo, (no label)' - }) - expect(result) - }) - - it('should create entity label of birth infinit label', async function () { - this.timeout(15000) - const project = await PEntityLabel.createProject(); - await PEntityLabel.createNamingMock(); - await PEntityLabel.createPersonMock(); - const {birth} = await EntityLabel.createInfinitLabel() - - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx({pkEntity: birth.pk_entity ?? -1, fkProject: project.pk_entity ?? -1}), - compare: (item) => { - // console.log(item?.entityLabel) - // console.log(item?.entityLabel?.length) - return item?.entityLabel?.length === ENTITY_LABEL_MAX_LENGTH - } - }) - - }) - - - // it('should create entity label of Birth – E67 (-- with identifying property)', async () => { - // const project = await createProject(); - // const {appellation} = await createNamingMock(); - // await createPersonMock(); - // const birth = await createBirthMock() - // const result = await waitForEntityPreviewUntil(wh, (item) => { - // return item.pk_entity === birth.pk_entity - // && item.fk_project === project.pk_entity - // && item.entity_label === appellation.string - // }) - // expect(result) - // }) - - // it('should create entity label of Union – C9 (-- with identifying property)', async () => { - - // const project = await createProject(); - // const {appellation} = await createNamingMock(); - // await createPersonMock(); - // const union = await createUnionMock() - // const result = await waitForEntityPreviewUntil(wh, (item) => { - // return item.pk_entity === union.pk_entity - // && item.fk_project === project.pk_entity - // && item.entity_label === appellation.string - // }) - // expect(result) - // }) - - it('should delete entity label from index when entity is removed from project', async () => { - const project = await PEntityLabel.createProject(); - const {naming, namingProRel, appellation} = await PEntityLabel.createNamingMock(); - const result = await waitForEntityPreview(wh, [ - {pk_entity: {eq: naming.pk_entity}}, - {fk_project: {eq: project.pk_entity}}, - {entity_label: {eq: appellation.string}}, - ]) - expect(result) - - await updateProInfoProjRel(namingProRel.pk_entity ?? -1, {is_in_project: false}) - - const item = await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdxWithTmsps({pkEntity: naming.pk_entity ?? -1, fkProject: project.pk_entity ?? -1}), - compare: (row) => { - // console.log('test',JSON.stringify(row)) - return (!!row?.deleted) - } - }) - - expect(item?.deleted).not.to.be.undefined() - }) -}) - - - -export namespace PEntityLabel { - - export async function createNamingAndPersonMock() { - // NAMING - const {naming, appellation} = await createNamingMock(); - // PERSON - const person = await createPersonMock(); - return {naming, person, appellation}; - } - - export async function createPersonMock() { - await createDfhApiClass(DfhApiClassMock.EN_21_PERSON); - const person = await createInfResource(InfResourceMock.PERSON_1); - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_PERSON_1); - return person; - } - - export async function createProject() { - await createInfLanguage(InfLanguageMock.GERMAN); - const project = await createProProject(ProProjectMock.PROJECT_1); - await createProProject(ProProjectMock.DEFAULT_PROJECT); - return project - } - - export async function createNamingMock() { - await createDfhApiClass(DfhApiClassMock.EN_365_NAMING); - - // TeEn - const naming = await createInfResource(InfResourceMock.NAMING_1); - const namingProRel = await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_NAMING_1); - - const appellation = await createInfAppellation(InfAppellationMock.JACK_THE_FOO); - await createInfStatement(InfStatementMock.NAME_1_TO_APPE); - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_STMT_NAME_1_TO_APPE); - - await createInfStatement(InfStatementMock.NAME_1_TO_PERSON); - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_STMT_NAME_1_TO_PERSON); - return {naming, namingProRel, appellation}; - } - - - -} - diff --git a/server/src/__tests__/acceptance/warehouse/aggregator-ds/p-entity-path/PEntityPathService.test.ts b/server/src/__tests__/acceptance/warehouse/aggregator-ds/p-entity-path/PEntityPathService.test.ts deleted file mode 100644 index 885bad395..000000000 --- a/server/src/__tests__/acceptance/warehouse/aggregator-ds/p-entity-path/PEntityPathService.test.ts +++ /dev/null @@ -1,103 +0,0 @@ -// /* eslint-disable @typescript-eslint/no-invalid-this */ -// /* eslint-disable @typescript-eslint/camelcase */ -// import {expect} from '@loopback/testlab'; -// import 'reflect-metadata'; -// import {PEntityPathService} from '../../../../../warehouse/aggregator-ds/entity-paths/PEntityPathService'; -// import {WarehouseStubs} from '../../../../../warehouse/createWarehouse'; -// import {PEdgeService} from '../../../../../warehouse/primary-ds/edge/PEdgeService'; -// import {PEntityService} from '../../../../../warehouse/primary-ds/entity/PEntityService'; -// import {EntityPathConfigService} from '../../../../../warehouse/primary-ds/EntityPathConfigService'; -// import {Warehouse} from '../../../../../warehouse/Warehouse'; -// import {createDfhApiClass} from '../../../../helpers/atomic/dfh-api-class.helper'; -// import {createInfLanguage} from '../../../../helpers/atomic/inf-language.helper'; -// import {createInfResource} from '../../../../helpers/atomic/inf-resource.helper'; -// import {createInfStatement} from '../../../../helpers/atomic/inf-statement.helper'; -// import {addInfosToProject} from '../../../../helpers/atomic/pro-info-proj-rel.helper'; -// import {createProProject} from '../../../../helpers/atomic/pro-project.helper'; -// import {DfhApiClassMock} from '../../../../helpers/data/gvDB/DfhApiClassMock'; -// import {InfLanguageMock} from '../../../../helpers/data/gvDB/InfLanguageMock'; -// import {InfResourceMock} from '../../../../helpers/data/gvDB/InfResourceMock'; -// import {InfStatementMock} from '../../../../helpers/data/gvDB/InfStatementMock'; -// import {ProProjectMock} from '../../../../helpers/data/gvDB/ProProjectMock'; -// import {cleanDb} from '../../../../helpers/meta/clean-db.helper'; -// import {setupCleanAndStartWarehouse, stopWarehouse, truncateWarehouseTables, wait} from '../../../../helpers/warehouse-helpers'; -// export const stub: WarehouseStubs = { -// primaryDataServices: [ -// EntityPathConfigService, -// PEntityService, -// PEdgeService -// ], -// aggDataServices: [ -// PEntityPathService -// ] -// } -// /** -// * Testing whole stack from postgres to warehouse -// */ -// describe('PEntityPathService', function () { -// let wh: Warehouse; -// let s: PEntityPathService -// let e: PEntityService -// before(async function () { -// // eslint-disable-next-line @typescript-eslint/no-invalid-this -// // this.timeout(5000); // A very long environment setup. -// const injector = await setupCleanAndStartWarehouse(stub) -// wh = injector.get(Warehouse) -// s = injector.get(PEntityPathService) -// e = injector.get(PEntityService) -// }) -// beforeEach(async () => { -// await cleanDb() -// await truncateWarehouseTables(wh) -// }) -// after(async function () { -// await stopWarehouse(wh) -// }) - -// it('should create entity path of chapter1', async () => { -// await wh.gvPgPool.query(`select pg_notify('modified_entity_path_config', now()::text)`) - -// const {project, chapter1} = await PEntityPath.createExpressionAndPortionMock(); -// await wait(5000) - -// const pEntityChapter = await e.index.getFromIdx({fkProject: project.pk_entity, pkEntity: chapter1.pk_entity ?? -1}) - -// expect(pEntityChapter).not.undefined() -// }) - - -// }) - - - -// export namespace PEntityPath { -// export async function createProject() { -// await createInfLanguage(InfLanguageMock.GERMAN); -// const project = await createProProject(ProProjectMock.PROJECT_1); -// await createProProject(ProProjectMock.DEFAULT_PROJECT); -// return project -// } -// export async function createModel() { -// await createDfhApiClass(DfhApiClassMock.EN_218_EXPRESSION); -// await createDfhApiClass(DfhApiClassMock.EN_503_EXPRESSION_PORTION); -// } - -// export async function createExpressionAndPortionMock() { -// await createModel() -// const project = await createProject() -// // Expression portion -// const chapter1 = await createInfResource(InfResourceMock.EXPRESSION_PORTION_HABS_EMP_CHAPTER_1) -// // Is part of -// const stmtChapter1Expr = await createInfStatement(InfStatementMock.EXPR_PORTION_CHAPTER_1_IS_PART_OF_HABS_EMP_EXPR) -// // Expression -// const expr = await createInfResource(InfResourceMock.HABS_EMP_EXPR) - -// await addInfosToProject(project.pk_entity, [chapter1, stmtChapter1Expr, expr].map(x => x.pk_entity)) -// return {project, chapter1} -// } - - - - -// } - diff --git a/server/src/__tests__/acceptance/warehouse/aggregator-ds/p-entity-time-span/PEntityTimeSpanService.test.ts b/server/src/__tests__/acceptance/warehouse/aggregator-ds/p-entity-time-span/PEntityTimeSpanService.test.ts deleted file mode 100644 index d3360548e..000000000 --- a/server/src/__tests__/acceptance/warehouse/aggregator-ds/p-entity-time-span/PEntityTimeSpanService.test.ts +++ /dev/null @@ -1,273 +0,0 @@ -import { expect } from '@loopback/testlab'; -import { equals } from 'ramda'; -import 'reflect-metadata'; -import { WarEntityPreviewTimeSpan } from '../../../../../models/entity-preview/WarEntityPreviewTimeSpan'; -import { CalendarType } from '../../../../../models/enums/CalendarType'; -import { Granularity } from '../../../../../models/enums/Granularity'; -import { EntityPreviewService } from '../../../../../warehouse/aggregator-ds/entity-preview/EntityPreviewService'; -import { PEntityTimeSpanService } from '../../../../../warehouse/aggregator-ds/entity-time-span/p-entity-time-span/PEntityTimeSpanService'; -import { WarehouseStubs } from '../../../../../warehouse/createWarehouse'; -import { PEdgeService } from '../../../../../warehouse/primary-ds/edge/PEdgeService'; -import { PEntityService } from '../../../../../warehouse/primary-ds/entity/PEntityService'; -import { Warehouse } from '../../../../../warehouse/Warehouse'; -import { createDfhApiClass } from '../../../../helpers/atomic/dfh-api-class.helper'; -import { createDfhApiProperty } from '../../../../helpers/atomic/dfh-api-property.helper'; -import { createInfLanguage } from '../../../../helpers/atomic/inf-language.helper'; -import { createInfResource } from '../../../../helpers/atomic/inf-resource.helper'; -import { createInfStatement } from '../../../../helpers/atomic/inf-statement.helper'; -import { createInfTimePrimitive } from '../../../../helpers/atomic/inf-time-primitive.helper'; -import { createProInfoProjRel, updateProInfoProjRel } from '../../../../helpers/atomic/pro-info-proj-rel.helper'; -import { createProProject } from '../../../../helpers/atomic/pro-project.helper'; -import { DfhApiClassMock } from '../../../../helpers/data/gvDB/DfhApiClassMock'; -import { DfhApiPropertyMock } from '../../../../helpers/data/gvDB/DfhApiPropertyMock'; -import { InfLanguageMock } from '../../../../helpers/data/gvDB/InfLanguageMock'; -import { InfResourceMock } from '../../../../helpers/data/gvDB/InfResourceMock'; -import { InfStatementMock } from '../../../../helpers/data/gvDB/InfStatementMock'; -import { InfTimePrimitiveMock } from '../../../../helpers/data/gvDB/InfTimePrimitiveMock'; -import { ProInfoProjRelMock } from '../../../../helpers/data/gvDB/ProInfoProjRelMock'; -import { ProProjectMock } from '../../../../helpers/data/gvDB/ProProjectMock'; -import { cleanDb } from '../../../../helpers/meta/clean-db.helper'; -import { searchUntilSatisfy, setupCleanAndStartWarehouse, stopWarehouse, truncateWarehouseTables, waitForEntityPreviewUntil } from '../../../../helpers/warehouse-helpers'; -const pEntityTimeSpanStub: WarehouseStubs = { - primaryDataServices: [ - PEntityService, - PEdgeService - ], - aggDataServices: [ - PEntityTimeSpanService, - EntityPreviewService - ] -} -/** - * Testing whole stack from postgres to warehouse - */ -describe('PEntityTimeSpanService', function () { - let wh: Warehouse; - let edgeService: PEdgeService; - let s: PEntityTimeSpanService - - before(async function () { - // eslint-disable-next-line @typescript-eslint/no-invalid-this - this.timeout(5000); // A very long environment setup. - const injector = await setupCleanAndStartWarehouse(pEntityTimeSpanStub) - wh = injector.get(Warehouse) - s = injector.get(PEntityTimeSpanService) - edgeService = injector.get(PEdgeService) - }) - beforeEach(async () => { - await cleanDb() - await truncateWarehouseTables(wh) - }) - after(async function () { - await stopWarehouse(wh) - }) - it('should create edges with time primitives', async () => { - const { shipVoyage, project } = await PEntityTimeSpanMock.createMock(); - - await searchUntilSatisfy({ - notifier$: edgeService.afterChange$, - getFn: () => edgeService.index.getFromIdx({ - fkProject: project.pk_entity ?? -1, - pkEntity: shipVoyage.pk_entity ?? -1, - }), - compare: (val) => !!val?.outgoing?.[71]?.length - }) - - }) - - - it('should create timespanval of time primitive', async () => { - const { shipVoyage, project } = await PEntityTimeSpanMock.createMock(); - - const expectedTimeSpan: WarEntityPreviewTimeSpan = { - "p81": { - "calendar": CalendarType.gregorian, - "duration": Granularity['1 day'], - "julianDay": 2362729 - }, - "p82": { - "calendar": CalendarType.gregorian, - "duration": Granularity['1 day'], - "julianDay": 2362730 - }, - "p81a": { - "calendar": CalendarType.gregorian, - "duration": Granularity['1 day'], - "julianDay": 2362731 - }, - "p81b": { - "calendar": CalendarType.gregorian, - "duration": Granularity['1 day'], - "julianDay": 2362732 - }, - "p82a": { - "calendar": CalendarType.julian, - "duration": Granularity['1 day'], - "julianDay": 2362733 - }, - "p82b": { - "calendar": CalendarType.julian, - "duration": Granularity['1 day'], - "julianDay": 2362734 - } - } - const expectedFirstSec = '204139785600'; - const expectedLastSec = '204140303999'; - - const result = await waitForEntityPreviewUntil(wh, (entPreview) => { - return entPreview.pk_entity === shipVoyage.pk_entity - && entPreview.fk_project === project.pk_entity - && equals(entPreview.time_span, expectedTimeSpan) - && entPreview.first_second === expectedFirstSec - && entPreview.last_second === expectedLastSec - }) - - expect(result.time_span).to.deepEqual(expectedTimeSpan); - }) - - it('should create empty time span object {}', async () => { - // - Langage and Project - const project = await PEntityTimeSpanMock.createProjectAndModelMock(); - - // - shipVoyage - const shipVoyage = await createInfResource(InfResourceMock.SHIP_VOYAGE); - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_SHIP_VOYAGE); - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx({ pkEntity: shipVoyage.pk_entity ?? -1, fkProject: project.pk_entity ?? -1 }), - compare: (val) => { - return equals(val?.timeSpan, {}) - } - }) - }) - - - it('should delete entity time span from index when entity is removed from project', async () => { - await PEntityTimeSpanMock.createProjectAndModelMock(); - const shipVoyage = await createInfResource(InfResourceMock.SHIP_VOYAGE); - const prel = await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_SHIP_VOYAGE); - - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx({ pkEntity: shipVoyage.pk_entity ?? -1, fkProject: prel.fk_project ?? -1 }), - compare: (val) => { - return equals(val?.timeSpan, {}) - } - }) - // remove person from the project - await updateProInfoProjRel(prel.pk_entity ?? -1, { is_in_project: false }) - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdxWithTmsps({ pkEntity: shipVoyage.pk_entity ?? -1, fkProject: prel.fk_project ?? -1 }), - compare: (item) => { - return !!item?.deleted - } - }) - - }) - -}) -export namespace PEntityTimeSpanMock { - - // create the mock data: - export async function createMock() { - // - Langage and Project - const project = await createProjectAndModelMock(); - - // - shipVoyage - const shipVoyage = await createInfResource(InfResourceMock.SHIP_VOYAGE); - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_SHIP_VOYAGE); - - // TimePrimitive 1 - await createInfTimePrimitive(InfTimePrimitiveMock.TP_1); - // Stmt to TimePrimitive 1 - await createInfStatement(InfStatementMock.SHIP_VOYAGE_ONGOING_THROUGHOUT_TP_1); - // Project rel for stmt (With calender info !) - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_STMT_SHIP_VOYAGE_ONGOING_THROUGHOUT_TP_1) - - // TimePrimitive 2 - await createInfTimePrimitive(InfTimePrimitiveMock.TP_2); - // Stmt to TimePrimitive 2 - await createInfStatement(InfStatementMock.SHIP_VOYAGE_AT_SOME_TIME_WITHIN_TP_2); - // Project rel for stmt (With calender info !) - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_STMT_SHIP_VOYAGE_AT_SOME_TIME_WITHIN_TP_2) - - // TimePrimitive 3 - await createInfTimePrimitive(InfTimePrimitiveMock.TP_3); - // Stmt to TimePrimitive 3 - await createInfStatement(InfStatementMock.SHIP_VOYAGE_END_OF_THE_BEGIN_TP_3); - // Project rel for stmt (With calender info !) - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_STMT_SHIP_VOYAGE_END_OF_THE_BEGIN_TP_3) - - // TimePrimitive 4 - await createInfTimePrimitive(InfTimePrimitiveMock.TP_4); - // Stmt to TimePrimitive 4 - await createInfStatement(InfStatementMock.SHIP_VOYAGE_BEGIN_OF_THE_END_TP_4); - // Project rel for stmt (With calender info !) - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_STMT_SHIP_VOYAGE_BEGIN_OF_THE_END_TP_4) - - // TimePrimitive 5 - await createInfTimePrimitive(InfTimePrimitiveMock.TP_5); - // Stmt to TimePrimitive 5 - await createInfStatement(InfStatementMock.SHIP_VOYAGE_BEGIN_OF_THE_BEGIN_TP_5); - // Project rel for stmt (With calender info !) - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_STMT_SHIP_VOYAGE_BEGIN_OF_THE_BEGIN_TP_5) - - // TimePrimitive 6 - await createInfTimePrimitive(InfTimePrimitiveMock.TP_6); - // Stmt to TimePrimitive 6 - await createInfStatement(InfStatementMock.SHIP_VOYAGE_END_OF_THE_END_TP_6); - // Project rel for stmt (With calender info !) - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_STMT_SHIP_VOYAGE_END_OF_THE_END_TP_6) - - return { shipVoyage, project }; - } - - - - - export async function createProjectAndModelMock() { - await createInfLanguage(InfLanguageMock.GERMAN); - const project = await createProProject(ProProjectMock.PROJECT_1); - - // - Class: Ship Voyage, TimePrimitive - await createDfhApiClass(DfhApiClassMock.EN_523_SHIP_VOYAGE); - await createDfhApiClass(DfhApiClassMock.EN_335_TIME_PRIMITIVE); - - // P81 ongoing throughout - await createDfhApiProperty(DfhApiPropertyMock.EN_71_ONGOING_THROUGHOUT); - // P82 at some time within - await createDfhApiProperty(DfhApiPropertyMock.EN_72_AT_SOME_TIME_WITHIN); - // P81a end of the begin - await createDfhApiProperty(DfhApiPropertyMock.EN_150_END_OF_THE_BEGIN); - // P81b begin of the end - await createDfhApiProperty(DfhApiPropertyMock.EN_151_BEGIN_OF_THE_END); - // P82a begin of the begin - await createDfhApiProperty(DfhApiPropertyMock.EN_152_BEGIN_OF_THE_BEGIN); - // P82b end of the end - await createDfhApiProperty(DfhApiPropertyMock.EN_153_END_OF_THE_END); - return project; - } -} - -/** - -select * -FROM -information.time_primitive t1, -projects.info_proj_rel t2 -WHERE t1.pk_entity=t2.fk_entity -LIMIT 10; - - -select t2.calendar -FROM -information."statement" t0, -information.time_primitive t1, -projects.info_proj_rel t2 -WHERE -t0.fk_object_info = t1.pk_entity -AND t0.pk_entity=t2.fk_entity -LIMIT 10 - - */ diff --git a/server/src/__tests__/acceptance/warehouse/aggregator-ds/p-entity-type/PEntityTypeService.test.ts b/server/src/__tests__/acceptance/warehouse/aggregator-ds/p-entity-type/PEntityTypeService.test.ts deleted file mode 100644 index 694d4ea48..000000000 --- a/server/src/__tests__/acceptance/warehouse/aggregator-ds/p-entity-type/PEntityTypeService.test.ts +++ /dev/null @@ -1,262 +0,0 @@ -/* eslint-disable @typescript-eslint/camelcase */ -import 'reflect-metadata'; -import {expect} from '@loopback/testlab'; -import {PEntityLabelService} from '../../../../../warehouse/aggregator-ds/entity-label/p-entity-label/PEntityLabelService'; -import {REntityLabelService} from '../../../../../warehouse/aggregator-ds/entity-label/r-entity-label/REntityLabelService'; -import {EntityPreviewService} from '../../../../../warehouse/aggregator-ds/entity-preview/EntityPreviewService'; -import {PEntityTypeService} from '../../../../../warehouse/aggregator-ds/entity-type/p-entity-type/PEntityTypeService'; -import {WarehouseStubs} from '../../../../../warehouse/createWarehouse'; -import {DfhClassHasTypePropertyService} from '../../../../../warehouse/primary-ds/DfhClassHasTypePropertyService'; -import {DfhOutgoingPropertyService} from '../../../../../warehouse/primary-ds/DfhOutgoingPropertyService'; -import {PEdgeService} from '../../../../../warehouse/primary-ds/edge/PEdgeService'; -import {REdgeService} from '../../../../../warehouse/primary-ds/edge/REdgeService'; -import {PEntityService} from '../../../../../warehouse/primary-ds/entity/PEntityService'; -import {REntityService} from '../../../../../warehouse/primary-ds/entity/REntityService'; -import {ProEntityLabelConfigService} from '../../../../../warehouse/primary-ds/ProEntityLabelConfigService'; -import {Warehouse} from '../../../../../warehouse/Warehouse'; -import {createDfhApiClass} from '../../../../helpers/atomic/dfh-api-class.helper'; -import {createDfhApiProperty} from '../../../../helpers/atomic/dfh-api-property.helper'; -import {createInfAppellation} from '../../../../helpers/atomic/inf-appellation.helper'; -import {createInfLanguage} from '../../../../helpers/atomic/inf-language.helper'; -import {createInfStatement} from '../../../../helpers/atomic/inf-statement.helper'; -import {createInfResource} from '../../../../helpers/atomic/inf-resource.helper'; -import {addInfosToProject, createProInfoProjRel, removeEntityFromProject, updateProInfoProjRel} from '../../../../helpers/atomic/pro-info-proj-rel.helper'; -import {createProProject} from '../../../../helpers/atomic/pro-project.helper'; -import {DfhApiClassMock} from '../../../../helpers/data/gvDB/DfhApiClassMock'; -import {DfhApiPropertyMock} from '../../../../helpers/data/gvDB/DfhApiPropertyMock'; -import {InfAppellationMock} from '../../../../helpers/data/gvDB/InfAppellationMock'; -import {InfLanguageMock} from '../../../../helpers/data/gvDB/InfLanguageMock'; -import {InfStatementMock} from '../../../../helpers/data/gvDB/InfStatementMock'; -import {InfResourceMock} from '../../../../helpers/data/gvDB/InfResourceMock'; -import {ProInfoProjRelMock} from '../../../../helpers/data/gvDB/ProInfoProjRelMock'; -import {ProProjectMock} from '../../../../helpers/data/gvDB/ProProjectMock'; -import {createInstancesForCityType} from '../../../../helpers/graphs/cityType.helper'; -import {createInstancesForMadrid, createModelMockForMadrid} from '../../../../helpers/graphs/madrid.helper'; -import {cleanDb} from '../../../../helpers/meta/clean-db.helper'; -import {searchUntilSatisfy, setupCleanAndStartWarehouse, stopWarehouse, truncateWarehouseTables, waitForEntityPreview, waitForEntityPreviewUntil} from '../../../../helpers/warehouse-helpers'; -import {createModelForCityType, createLanguages} from '../../../../helpers/meta/model.helper'; -import {createProject1, createProject2, createProject3} from '../../../../helpers/graphs/project.helper'; -const pEntityTypeServiceStub: WarehouseStubs = { - primaryDataServices: [ - DfhOutgoingPropertyService, - ProEntityLabelConfigService, - PEntityService, - PEdgeService, - REntityService, - REdgeService, - DfhClassHasTypePropertyService - ], - aggDataServices: [ - // IdentifyingPropertyService, - PEntityLabelService, - REntityLabelService, - PEntityTypeService, - EntityPreviewService - ] -} - -/** - * Testing whole stack from postgres to warehouse - */ -describe('PEntityTypeService', function () { - let wh: Warehouse; - let s: PEntityTypeService - before(async function () { - // eslint-disable-next-line @typescript-eslint/no-invalid-this - this.timeout(5000); // A very long environment setup. - const injector = await setupCleanAndStartWarehouse(pEntityTypeServiceStub) - wh = injector.get(Warehouse) - s = injector.get(PEntityTypeService) - }) - beforeEach(async () => { - await cleanDb() - await truncateWarehouseTables(wh) - }) - after(async function () { - await stopWarehouse(wh) - }) - it('should create fk_type of geographical place', async () => { - const {madrid, project, cityTypeProjRel} = await PEntityTypeMock.createMock(); - - const result = await waitForEntityPreview(wh, [ - { pk_entity: { eq: madrid.pk_entity } }, - { fk_project: { eq: project.pk_entity } }, - { fk_type: { eq: cityTypeProjRel.fk_entity } }, - ]) - expect(result); - }) - - it('should update fk_type of geographical place', async () => { - const {madrid, project, cityTypeProjRel, cityTypeAppe} = await PEntityTypeMock.createMock(); - - let result = await waitForEntityPreview(wh, [ - { pk_entity: { eq: madrid.pk_entity } }, - { fk_project: { eq: project.pk_entity } }, - { fk_type: { eq: cityTypeProjRel.fk_entity } }, - { type_label: { eq: cityTypeAppe.string } }, - ]) - expect(result.type_label).to.equal(cityTypeAppe.string); - - await updateProInfoProjRel( - ProInfoProjRelMock.PROJ_1_STMT_MADRID_HAS_GEO_PLACE_CITY_TYPE.pk_entity ?? -1, - { is_in_project: false } - ) - - result = await waitForEntityPreview(wh, [ - { pk_entity: { eq: madrid.pk_entity } }, - { fk_project: { eq: project.pk_entity } }, - { fk_type: { eq: null } }, - { type_label: { eq: null } }, - ]) - expect(result.type_label).to.not.equal(cityTypeAppe.string); - }) - - it('should take repo-label of type, if type is not in project', async () => { - await createLanguages() - await createModelForCityType(); - - const { project1 } = await createProject1(); - - const { project2 } = await createProject2(); - - const { project3 } = await createProject3(); - - // create type instances - const { - cityType, - appeCity, - appeStadt, - namingCity1: naming1, - namingCity1RefersTo: naming1RefersTo, - namingCity1IsAppeOf: naming1IsAppeOf, - namingStadt2: naming2, - namingStadt2RefersTo: naming2RefersTo, - namingStadt2IsAppeOf: naming2IsAppeOf - } = await createInstancesForCityType(); - - // add naming 1 to project1 - await addInfosToProject(project1.pk_entity, [ - cityType.pk_entity, - naming1.pk_entity, - naming1RefersTo.pk_entity, - naming1IsAppeOf.pk_entity]) - - // add naming 2 to project2 - await addInfosToProject(project2.pk_entity, [ - cityType.pk_entity, - naming2.pk_entity, - naming2RefersTo.pk_entity, - naming2IsAppeOf.pk_entity]) - - // add naming 2 to project3 - await addInfosToProject(project3.pk_entity, [ - cityType.pk_entity, - naming2.pk_entity, - naming2RefersTo.pk_entity, - naming2IsAppeOf.pk_entity]) - - await createModelMockForMadrid() - const { madrid, madridHasType } = await createInstancesForMadrid() - - - - await addInfosToProject(project1.pk_entity, [ - madrid.pk_entity, - madridHasType.pk_entity]) - - // since naming 2 is more often used, - // -> the repo label should be 'Stadt' - await waitForEntityPreviewUntil(wh, (item) => { - return item.pk_entity === cityType.pk_entity - && item.fk_project === null - && item.entity_label === appeStadt.string - }) - - // cityType should be called 'City' in project 1 and 'Stadt' in repo - await waitForEntityPreviewUntil(wh, (item) => { - return item.pk_entity === madrid.pk_entity - && item.fk_project === project1.pk_entity - && item.type_label === appeCity.string // <- !! here is the tested type label - }) - - - // remove the cityType (peIt) from project1 - await removeEntityFromProject(project1.pk_entity, cityType.pk_entity) - - // madrid.entiy_type should be called 'City' in project 1 and 'Stadt' in repo - const madridTypeLabel2 = await waitForEntityPreviewUntil(wh, (item) => { - // console.log(item) - return item.pk_entity === madrid.pk_entity - && item.fk_project === project1.pk_entity - && item.type_label === appeStadt.string // <- !! here is the tested type label - }) - expect(madridTypeLabel2.type_label).to.equal(appeStadt.string); - }) - - it('should delete entity type from index when entity is removed from project', async () => { - const {madrid, madridProjRel, project, cityTypeAppe} = await PEntityTypeMock.createMock(); - - const result = await waitForEntityPreview(wh, [ - { pk_entity: { eq: madrid.pk_entity } }, - { fk_project: { eq: project.pk_entity } }, - { type_label: { eq: cityTypeAppe.string } }, - ]) - expect(result.type_label).to.equal(cityTypeAppe.string); - // remove madrid from the project - await updateProInfoProjRel(madridProjRel.pk_entity ?? -1, {is_in_project: false}) - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdxWithTmsps({pkEntity: madrid.pk_entity ?? -1, fkProject: project.pk_entity ?? -1}), - compare: (val) => !!val?.deleted - - }) - }) - -}) - -export namespace PEntityTypeMock { - - // create the mock data: - export async function createMock() { - // - Langage and Project - await createInfLanguage(InfLanguageMock.GERMAN); - const project = await createProProject(ProProjectMock.PROJECT_1); - - // - Class: Geo Place, Geo Place type - await createDfhApiClass(DfhApiClassMock.EN_363_GEO_PLACE); - await createDfhApiClass(DfhApiClassMock.EN_364_GEO_PLACE_TYPE); - await createDfhApiClass(DfhApiClassMock.EN_365_NAMING); - - // - Property: has geo. place type - await createDfhApiProperty(DfhApiPropertyMock.EN_1110_HAS_GEO_PLACE_TYPE); - - // - peIt (Geo Place (Madrid)) - const madrid = await createInfResource(InfResourceMock.GEO_PLACE_MADRID); - const madridProjRel = await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_MADRID); - - // - peIt (Geo Place Type X) - await createInfResource(InfResourceMock.GEO_PLACE_TYPE_CITY); - const cityTypeProjRel = await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_CITY_TYPE); - - // - stmt (has geo. place type, (Madrid has type 'City')) - await createInfStatement(InfStatementMock.MADRID_HAS_GEO_PLACE_TYPE_CITY); - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_STMT_MADRID_HAS_GEO_PLACE_CITY_TYPE); - - // - appe ('City) - const cityTypeAppe = await createInfAppellation(InfAppellationMock.CITY) - - // - teEn Y (Naming of peIt X = 'City') - await createInfResource(InfResourceMock.NAMING_1_CITY); - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_NAMING_CITY); - - // - stmt (Y refers to Name appe 'City') - await createInfStatement(InfStatementMock.NAMING_CITY_TO_APPE_CITY); - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_STMT_NAMING_CITY_TO_APPE_CITY); - - // - stmt (Y is Naming of Geo Place Type X) - await createInfStatement(InfStatementMock.NAMING_CITY_TO_GEO_PLACE_TYPE); - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_STMT_NAMING_CITY_TO_GEO_PLACE_TYPE); - - return {madrid, madridProjRel, cityTypeProjRel, project, cityTypeAppe}; - } -} diff --git a/server/src/__tests__/acceptance/warehouse/aggregator-ds/r-class-field-label/RClassFieldLabelService.test.ts b/server/src/__tests__/acceptance/warehouse/aggregator-ds/r-class-field-label/RClassFieldLabelService.test.ts deleted file mode 100644 index 8b60844db..000000000 --- a/server/src/__tests__/acceptance/warehouse/aggregator-ds/r-class-field-label/RClassFieldLabelService.test.ts +++ /dev/null @@ -1,195 +0,0 @@ -/* eslint-disable @typescript-eslint/no-invalid-this */ -import 'reflect-metadata'; -import {RClassFieldId, RClassFieldLabelService} from '../../../../../warehouse/aggregator-ds/class-field-label/r-class-field-label/RClassFieldLabelService'; -import {WarehouseStubs} from '../../../../../warehouse/createWarehouse'; -import {DfhPropertyLabelService} from '../../../../../warehouse/primary-ds/DfhPropertyLabelService'; -import {RPropertyService} from '../../../../../warehouse/primary-ds/property/RPropertyService'; -import {ProPropertyLabelService} from '../../../../../warehouse/primary-ds/ProPropertyLabelService'; -import {Warehouse} from '../../../../../warehouse/Warehouse'; -import {createDfhApiClass} from '../../../../helpers/atomic/dfh-api-class.helper'; -import {createDfhApiProperty} from '../../../../helpers/atomic/dfh-api-property.helper'; -import {createInfLanguage} from '../../../../helpers/atomic/inf-language.helper'; -import {createProProject} from '../../../../helpers/atomic/pro-project.helper'; -import {createProTextProperty, deleteProTextProperty} from '../../../../helpers/atomic/pro-text-property.helper'; -import {DfhApiClassMock} from '../../../../helpers/data/gvDB/DfhApiClassMock'; -import {DfhApiPropertyMock} from '../../../../helpers/data/gvDB/DfhApiPropertyMock'; -import {InfLanguageMock} from '../../../../helpers/data/gvDB/InfLanguageMock'; -import {ProProjectMock} from '../../../../helpers/data/gvDB/ProProjectMock'; -import {ProTextPropertyMock} from '../../../../helpers/data/gvDB/ProTextPropertyMock'; -import {cleanDb} from '../../../../helpers/meta/clean-db.helper'; -import {createTypes} from '../../../../helpers/meta/model.helper'; -import {searchUntilSatisfy, setupCleanAndStartWarehouse, stopWarehouse, truncateWarehouseTables} from '../../../../helpers/warehouse-helpers'; - -const stubs: WarehouseStubs = { - primaryDataServices: [ - RPropertyService, - DfhPropertyLabelService, - ProPropertyLabelService - ], - aggDataServices: [ - RClassFieldLabelService - ] -} - - -describe('RClassFieldLabelService', function () { - - let wh: Warehouse; - let s: RClassFieldLabelService; - before(async function () { - // eslint-disable-next-line @typescript-eslint/no-invalid-this - this.timeout(5000); // A very long environment setup. - const injector = await setupCleanAndStartWarehouse(stubs) - wh = injector.get(Warehouse) - s = injector.get(RClassFieldLabelService) - }) - beforeEach(async () => { - await cleanDb() - await truncateWarehouseTables(wh) - }) - after(async function () { - await stopWarehouse(wh) - }) - - it('should create outgoing property label for en from ontome', async () => { - const {dfhProp} = await createDfhLabelMock(); - const id: RClassFieldId = { - fkClass: dfhProp.dfh_property_domain, - fkProperty: dfhProp.dfh_pk_property, - isOutgoing: true, - } - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => val?.label === DfhApiPropertyMock.EN_86_BROUGHT_INTO_LIFE.dfh_property_label - }) - }) - - - it('should create incoming property label for en from ontome', async () => { - const {dfhProp} = await createDfhLabelMock(); - const expected = DfhApiPropertyMock.EN_86_BROUGHT_INTO_LIFE.dfh_property_inverse_label - const id: RClassFieldId = { - fkClass: dfhProp.dfh_property_range, - fkProperty: dfhProp.dfh_pk_property, - isOutgoing: false, - } - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => val?.label === expected - }) - - }) - - it('should create outgoing property label for en from geovistory', async () => { - const {dfhProp, gvTxt} = await createGeovistoryLabelMock(); - const expected = gvTxt.string - const id: RClassFieldId = { - fkClass: dfhProp.dfh_property_domain, - fkProperty: dfhProp.dfh_pk_property, - isOutgoing: true, - } - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => val?.label === expected - }) - - }) - - - it('should create incoming property label for en from geovistory', async () => { - const {dfhProp, gvTxt} = await createGeovistoryLabelIncomingMock(); - const expected = gvTxt.string - const id: RClassFieldId = { - fkClass: dfhProp.dfh_property_range, - fkProperty: dfhProp.dfh_pk_property, - isOutgoing: false, - } - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => val?.label === expected - }) - - }) - - - it('should create incoming property label has appellations (default) for en from geovistory', async () => { - const {gvTxt} = await createPersonMock(); - const expected = gvTxt.string - const id: RClassFieldId = { - fkClass: 21, - fkProperty: 1111, - isOutgoing: false, - } - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => val?.label === expected - }) - - }) - - - - it('should switch outgoing property label en-geovistory to en-ontome', async () => { - const {dfhProp, gvTxt} = await createGeovistoryLabelMock(); - - const id: RClassFieldId = { - fkClass: dfhProp.dfh_property_domain, - fkProperty: dfhProp.dfh_pk_property, - isOutgoing: true, - } - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => val?.label === gvTxt.string - }) - - await deleteProTextProperty(gvTxt.pk_entity ?? -1) - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => val?.label === dfhProp.dfh_property_label - }) - - }) - -}) -async function createDfhLabelMock() { - const dfhProp = await createDfhApiProperty(DfhApiPropertyMock.EN_86_BROUGHT_INTO_LIFE); - return {dfhProp} -} - -async function createGeovistoryLabelMock() { - const {dfhProp} = await createDfhLabelMock(); - await createTypes() - await createInfLanguage(InfLanguageMock.ENGLISH); - const gvTxt = await createProTextProperty(ProTextPropertyMock.PROJ_DEF_EN_PROPERTY_BROUGHT_INTO_LIFE) - return {dfhProp, gvTxt} -} - -async function createGeovistoryLabelIncomingMock() { - const {dfhProp} = await createDfhLabelMock(); - await createTypes() - await createInfLanguage(InfLanguageMock.ENGLISH); - const gvTxt = await createProTextProperty(ProTextPropertyMock.PROJ_DEF_EN_PROPERTY_BROUGHT_INTO_LIFE_REVERSE) - return {dfhProp, gvTxt} -} - - - - -async function createPersonMock() { - await createTypes() - await createInfLanguage(InfLanguageMock.GERMAN); - await createInfLanguage(InfLanguageMock.ENGLISH); - await createProProject(ProProjectMock.DEFAULT_PROJECT); - await createDfhApiClass(DfhApiClassMock.EN_21_PERSON); - // const dfhProp = await createDfhApiProperty(DfhApiPropertyMock.EN_1111_IS_APPE_OF); - const gvTxt = await createProTextProperty(ProTextPropertyMock.PROJ_DEF_EN_PROPERTY_PERSON_HAS_APPELLATION) - - return {gvTxt}; -} diff --git a/server/src/__tests__/acceptance/warehouse/aggregator-ds/r-class-label/RClassLabelService.test.ts b/server/src/__tests__/acceptance/warehouse/aggregator-ds/r-class-label/RClassLabelService.test.ts deleted file mode 100644 index 38711b468..000000000 --- a/server/src/__tests__/acceptance/warehouse/aggregator-ds/r-class-label/RClassLabelService.test.ts +++ /dev/null @@ -1,115 +0,0 @@ -/* eslint-disable @typescript-eslint/camelcase */ -import 'reflect-metadata'; -import {expect} from '@loopback/testlab'; -import {RClassLabelService} from '../../../../../warehouse/aggregator-ds/class-label/r-class-label/RClassLabelService'; -import {WarehouseStubs} from '../../../../../warehouse/createWarehouse'; -import {RClassService} from '../../../../../warehouse/primary-ds/class/RClassService'; -import {DfhClassLabelService} from '../../../../../warehouse/primary-ds/DfhClassLabelService'; -import {ProClassLabelService} from '../../../../../warehouse/primary-ds/ProClassLabelService'; -import {PK_DEFAULT_CONFIG_PROJECT, Warehouse} from '../../../../../warehouse/Warehouse'; -import {createDfhApiClass} from '../../../../helpers/atomic/dfh-api-class.helper'; -import {createInfLanguage} from '../../../../helpers/atomic/inf-language.helper'; -import {createProTextPropertyClassLabel, deleteProTextProperty} from '../../../../helpers/atomic/pro-text-property.helper'; -import {createTypes} from '../../../../helpers/meta/model.helper'; -import {InfLanguageMock} from '../../../../helpers/data/gvDB/InfLanguageMock'; -import {cleanDb} from '../../../../helpers/meta/clean-db.helper'; -import {searchUntilSatisfy, setupCleanAndStartWarehouse, stopWarehouse, truncateWarehouseTables, waitForClassPreview} from '../../../../helpers/warehouse-helpers'; -const rClassLabelService: WarehouseStubs = { - primaryDataServices: [ - RClassService, - DfhClassLabelService, - ProClassLabelService - ], - aggDataServices: [ - RClassLabelService - ] -} -describe('RClassLabelService', function () { - - let wh: Warehouse; - let s: RClassLabelService - before(async function () { - // eslint-disable-next-line @typescript-eslint/no-invalid-this - this.timeout(15000); // A very long environment setup. - const injector = await setupCleanAndStartWarehouse(rClassLabelService) - wh = injector.get(Warehouse) - s = injector.get(RClassLabelService) - }) - beforeEach(async () => { - await cleanDb() - await truncateWarehouseTables(wh) - }) - after(async function () { - await stopWarehouse(wh) - }) - it('should propagate rClass to rClassLabel', async () => { - const cla = await createDfhApiClass({ - dfh_pk_class: 1, - dfh_class_label: 'BIRTH ONTOME', - dfh_class_label_language: 'en', - dfh_fk_profile: 5 - }); - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx({ - pkClass: cla.dfh_pk_class - }), - compare: (item) => !!item - }) - }) - - it('should create class label of Person: en-geovistory', async () => { - const { cla, gvTxt } = await createGeovistoryLabelMock(); - const result = await waitForClassPreview(wh, [ - { fk_class: { eq: cla.dfh_pk_class } }, - { fk_project: { eq: 0 } }, - { label: { eq: gvTxt.string } }, - ]) - expect(result.label).to.equal('BIRTH GV DEFAULT') - }) - - - it('should switch class label of Person: en-geovistory to en-ontome', async () => { - const {cla, gvTxt} = await createGeovistoryLabelMock(); - await waitForClassPreview(wh, [ - {fk_class: {eq: cla.dfh_pk_class}}, - {fk_project: {eq: 0}}, - {label: {eq: gvTxt.string}}, - ]) - await deleteProTextProperty(gvTxt.pk_entity ?? -1) - const result = await waitForClassPreview(wh, [ - { fk_class: { eq: cla.dfh_pk_class } }, - { fk_project: { eq: 0 } }, - { label: { eq: 'BIRTH ONTOME' } }, - ]) - expect(result.label).to.equal('BIRTH ONTOME') - - }) - - -}) -async function createOntomeLabelMock() { - - const cla = await createDfhApiClass({ - dfh_pk_class: 1, - dfh_class_label: 'BIRTH ONTOME', - dfh_class_label_language: 'en', - dfh_fk_profile: 5 - }); - return { cla } -} - -async function createGeovistoryLabelMock() { - const { cla } = await createOntomeLabelMock(); - await createTypes() - await createInfLanguage(InfLanguageMock.ENGLISH); - - const gvTxt = await createProTextPropertyClassLabel( - PK_DEFAULT_CONFIG_PROJECT, - 1, - 'BIRTH GV DEFAULT', - 18889 //english - ) - return { cla, gvTxt } -} - diff --git a/server/src/__tests__/acceptance/warehouse/aggregator-ds/r-entity-class-label/REntityClassLabelService.test.ts b/server/src/__tests__/acceptance/warehouse/aggregator-ds/r-entity-class-label/REntityClassLabelService.test.ts deleted file mode 100644 index f0d7bfe6f..000000000 --- a/server/src/__tests__/acceptance/warehouse/aggregator-ds/r-entity-class-label/REntityClassLabelService.test.ts +++ /dev/null @@ -1,83 +0,0 @@ -/* eslint-disable @typescript-eslint/camelcase */ -import 'reflect-metadata'; -import {expect} from '@loopback/testlab'; -import {Warehouse} from '../../../../../warehouse/Warehouse'; -import {createDfhApiClass} from '../../../../helpers/atomic/dfh-api-class.helper'; -import {createInfLanguage} from '../../../../helpers/atomic/inf-language.helper'; -import {createProDfhProfileProjRel} from '../../../../helpers/atomic/pro-dfh-profile-proj-rel.helper'; -import {createProInfoProjRel} from '../../../../helpers/atomic/pro-info-proj-rel.helper'; -import {createProProject} from '../../../../helpers/atomic/pro-project.helper'; -import {cleanDb} from '../../../../helpers/meta/clean-db.helper'; -import {DfhApiClassMock} from '../../../../helpers/data/gvDB/DfhApiClassMock'; -import {InfLanguageMock} from '../../../../helpers/data/gvDB/InfLanguageMock'; -import {InfResourceMock} from '../../../../helpers/data/gvDB/InfResourceMock'; -import {ProDfhProfileProjRelMock} from '../../../../helpers/data/gvDB/ProDfhProfileProjRelMock'; -import {ProInfoProjRelMock} from '../../../../helpers/data/gvDB/ProInfoProjRelMock'; -import {ProProjectMock} from '../../../../helpers/data/gvDB/ProProjectMock'; -import {setupCleanAndStartWarehouse, stopWarehouse, truncateWarehouseTables, waitForEntityPreview} from '../../../../helpers/warehouse-helpers'; -import {WarehouseStubs} from '../../../../../warehouse/createWarehouse'; -import {REntityService} from '../../../../../warehouse/primary-ds/entity/REntityService'; -import {RClassService} from '../../../../../warehouse/primary-ds/class/RClassService'; -import {ProProjectService} from '../../../../../warehouse/primary-ds/ProProjectService'; -import {DfhClassLabelService} from '../../../../../warehouse/primary-ds/DfhClassLabelService'; -import {ProClassLabelService} from '../../../../../warehouse/primary-ds/ProClassLabelService'; -import {RClassLabelService} from '../../../../../warehouse/aggregator-ds/class-label/r-class-label/RClassLabelService'; -import {REntityClassLabelService} from '../../../../../warehouse/aggregator-ds/entity-class-label/r-entity-class-label/REntityClassLabelService'; -import {EntityPreviewService} from '../../../../../warehouse/aggregator-ds/entity-preview/EntityPreviewService'; -import {createInfResource} from '../../../../helpers/atomic/inf-resource.helper'; - -const rEntityClassLabelStub: WarehouseStubs = { - primaryDataServices: [ - REntityService, - RClassService, - ProProjectService, - DfhClassLabelService, - ProClassLabelService, - ], - aggDataServices: [ - RClassLabelService, - REntityClassLabelService, - EntityPreviewService - ] -} -describe('REntityClassLabelService', function () { - - let wh: Warehouse; - before(async function () { - // eslint-disable-next-line @typescript-eslint/no-invalid-this - this.timeout(5000); // A very long environment setup. - const injector = await setupCleanAndStartWarehouse(rEntityClassLabelStub) - wh = injector.get(Warehouse) - }) - beforeEach(async () => { - await cleanDb() - await truncateWarehouseTables(wh) - }) - after(async function () { - await stopWarehouse(wh) - }) - it('should create entity class label of Person', async () => { - const {pers, cla} = await createBasicMock(); - const result = await waitForEntityPreview(wh, [ - {pk_entity: {eq: pers.pk_entity}}, - {fk_project: {eq: null}}, - {class_label: {eq: cla.dfh_class_label}}, - ]) - expect(result.class_label).to.equal('Person') - }) - - - -}) - -async function createBasicMock() { - // CLASS + LABEL - await createInfLanguage(InfLanguageMock.GERMAN); - await createProProject(ProProjectMock.PROJECT_1); - await createProDfhProfileProjRel(ProDfhProfileProjRelMock.PROJ_1_PROFILE_4); - const cla = await createDfhApiClass(DfhApiClassMock.EN_21_PERSON); - // PERSON - const pers = await createInfResource(InfResourceMock.PERSON_1) - const prel = await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_PERSON_1) - return {prel, pers, cla} -} diff --git a/server/src/__tests__/acceptance/warehouse/aggregator-ds/r-entity-full-text/REntityFullTextService.test.ts b/server/src/__tests__/acceptance/warehouse/aggregator-ds/r-entity-full-text/REntityFullTextService.test.ts deleted file mode 100644 index 8a7a4fd45..000000000 --- a/server/src/__tests__/acceptance/warehouse/aggregator-ds/r-entity-full-text/REntityFullTextService.test.ts +++ /dev/null @@ -1,220 +0,0 @@ -/* eslint-disable @typescript-eslint/no-invalid-this */ -/* eslint-disable @typescript-eslint/camelcase */ -import 'reflect-metadata'; -import {expect} from '@loopback/testlab'; -import {RClassFieldLabelService} from '../../../../../warehouse/aggregator-ds/class-field-label/r-class-field-label/RClassFieldLabelService'; -import {RClassLabelService} from '../../../../../warehouse/aggregator-ds/class-label/r-class-label/RClassLabelService'; -import {REntityFullTextService} from '../../../../../warehouse/aggregator-ds/entity-full-text/r-entity-full-text/REntityFullTextService'; -import {REntityLabelService} from '../../../../../warehouse/aggregator-ds/entity-label/r-entity-label/REntityLabelService'; -import {EntityPreviewService} from '../../../../../warehouse/aggregator-ds/entity-preview/EntityPreviewService'; -import {WarehouseStubs} from '../../../../../warehouse/createWarehouse'; -import {RClassService} from '../../../../../warehouse/primary-ds/class/RClassService'; -import {DfhClassLabelService} from '../../../../../warehouse/primary-ds/DfhClassLabelService'; -import {DfhOutgoingPropertyService} from '../../../../../warehouse/primary-ds/DfhOutgoingPropertyService'; -import {DfhPropertyLabelService} from '../../../../../warehouse/primary-ds/DfhPropertyLabelService'; -import {REdgeService} from '../../../../../warehouse/primary-ds/edge/REdgeService'; -import {REntityId, REntityService} from '../../../../../warehouse/primary-ds/entity/REntityService'; -import {ProClassFieldsConfigService} from '../../../../../warehouse/primary-ds/ProClassFieldsConfigService'; -import {ProClassLabelService} from '../../../../../warehouse/primary-ds/ProClassLabelService'; -import {ProEntityLabelConfigService} from '../../../../../warehouse/primary-ds/ProEntityLabelConfigService'; -import {RPropertyService} from '../../../../../warehouse/primary-ds/property/RPropertyService'; -import {ProPropertyLabelService} from '../../../../../warehouse/primary-ds/ProPropertyLabelService'; -import {Warehouse} from '../../../../../warehouse/Warehouse'; -import {createDfhApiClass} from '../../../../helpers/atomic/dfh-api-class.helper'; -import {createDfhApiProperty} from '../../../../helpers/atomic/dfh-api-property.helper'; -import {createInfAppellation} from '../../../../helpers/atomic/inf-appellation.helper'; -import {createInfLanguage} from '../../../../helpers/atomic/inf-language.helper'; -import {createInfStatement} from '../../../../helpers/atomic/inf-statement.helper'; -import {createInfResource} from '../../../../helpers/atomic/inf-resource.helper'; -import {createInfTimePrimitive} from '../../../../helpers/atomic/inf-time-primitive.helper'; -import {createProClassFieldConfig} from '../../../../helpers/atomic/pro-class-field-config.helper'; -import {createProDfhProfileProjRel} from '../../../../helpers/atomic/pro-dfh-profile-proj-rel.helper'; -import {addInfoToProject} from '../../../../helpers/atomic/pro-info-proj-rel.helper'; -import {createProProject} from '../../../../helpers/atomic/pro-project.helper'; -import {createProTextProperty} from '../../../../helpers/atomic/pro-text-property.helper'; -import {createTypes} from '../../../../helpers/meta/model.helper'; -import {DfhApiClassMock} from '../../../../helpers/data/gvDB/DfhApiClassMock'; -import {DfhApiPropertyMock} from '../../../../helpers/data/gvDB/DfhApiPropertyMock'; -import {InfAppellationMock} from '../../../../helpers/data/gvDB/InfAppellationMock'; -import {InfLanguageMock} from '../../../../helpers/data/gvDB/InfLanguageMock'; -import {InfStatementMock} from '../../../../helpers/data/gvDB/InfStatementMock'; -import {InfResourceMock} from '../../../../helpers/data/gvDB/InfResourceMock'; -import {InfTimePrimitiveMock} from '../../../../helpers/data/gvDB/InfTimePrimitiveMock'; -import {ProClassFieldConfigMock} from '../../../../helpers/data/gvDB/ProClassFieldConfigMock'; -import {ProDfhProfileProjRelMock} from '../../../../helpers/data/gvDB/ProDfhProfileProjRelMock'; -import {ProProjectMock} from '../../../../helpers/data/gvDB/ProProjectMock'; -import {ProTextPropertyMock} from '../../../../helpers/data/gvDB/ProTextPropertyMock'; -import {SysSystemTypeMock} from '../../../../helpers/data/gvDB/SysSystemTypeMock'; -import {searchUntilSatisfy, setupCleanAndStartWarehouse, stopWarehouse, truncateWarehouseTables, waitForEntityPreview} from '../../../../helpers/warehouse-helpers'; -import {cleanDb} from '../../../../helpers/meta/clean-db.helper'; -const rEntityFullTextStub: WarehouseStubs = { - primaryDataServices: [ - REntityService, - REdgeService, - ProClassFieldsConfigService, - RPropertyService, - DfhPropertyLabelService, - ProPropertyLabelService, - ProEntityLabelConfigService, - DfhOutgoingPropertyService, - RClassService, - DfhClassLabelService, - ProClassLabelService, - ], - aggDataServices: [ - // IdentifyingPropertyService, - RClassFieldLabelService, - REntityLabelService, - RClassLabelService, - REntityFullTextService, - EntityPreviewService - ], -} -/** - * Testing whole stack from postgres to warehouse - */ -describe('REntityFullTextService', function () { - let wh: Warehouse; - let s: REntityFullTextService; - - before(async function () { - // eslint-disable-next-line @typescript-eslint/no-invalid-this - this.timeout(5000); // A very long environment setup. - const injector = await setupCleanAndStartWarehouse(rEntityFullTextStub) - wh = injector.get(Warehouse) - s = injector.get(REntityFullTextService) - }) - beforeEach(async () => { - await cleanDb() - await truncateWarehouseTables(wh) - }) - after(async function () { - await stopWarehouse(wh) - }) - - it('should create full text of naming', async () => { - const { naming } = await createNamingMock(); - - - const result = await waitForEntityPreview(wh, [ - {pk_entity: {eq: naming.pk_entity}}, - {fk_project: {eq: null}}, - {full_text: {eq: `Appellation in a language (time-indexed) – refers to name: 'Jack the foo'`}}, - ]) - - expect(result).not.to.be.undefined(); - }) - - it('should create full text of naming with person', async () => { - const { naming } = await createNamingAndPersonMock(); - - const expected = `Appellation in a language (time-indexed) – refers to name: 'Jack the foo', is appellation for language of: 'Jack the foo'` - - await waitForEntityPreview(wh, [ - {pk_entity: {eq: naming.pk_entity}}, - {fk_project: {eq: null}}, - {full_text: {eq: expected}}, - ]) - - }) - - - - it('should create full text of person', async () => { - const { person } = await createNamingAndPersonMock(); - - const expected = `Person – has appellations (default): 'Jack the foo'` - const id: REntityId = { - pkEntity: person.pk_entity ?? -1 - } - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => { - return val?.fullText === expected - } - }) - - }) - it('should update full text of person', async () => { - const { person } = await createNamingAndPersonMock(); - - let expected = `Person – has appellations (default): 'Jack the foo'` - const id: REntityId = { - pkEntity: person.pk_entity ?? -1 - } - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => val?.fullText === expected - }) - - await createProTextProperty({ - fk_dfh_class: 21, - fk_project: ProProjectMock.DEFAULT_PROJECT.pk_entity, - fk_language: InfLanguageMock.ENGLISH.pk_entity, - fk_system_type: SysSystemTypeMock.PRO_TEXT_PROPTERTY_LABEL.pk_entity, - string: 'Human' - }) - expected = `Human – has appellations (default): 'Jack the foo'` - - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => val?.fullText === expected - }) - - }) - - -}) - - - -async function createNamingAndPersonMock() { - // NAMING - const { naming, project, appellation, propertyRefersToName } = await createNamingMock(); - // PERSON - const { person, classPerson, hasAppePropLabel } = await createPersonMock(); - return { project, appellation, naming, person, classPerson, hasAppePropLabel, propertyRefersToName }; -} - -async function createPersonMock() { - await createTypes() - const classPerson = await createDfhApiClass(DfhApiClassMock.EN_21_PERSON); - await createDfhApiProperty(DfhApiPropertyMock.EN_1111_IS_APPE_OF); - await createInfLanguage(InfLanguageMock.ENGLISH); - const hasAppePropLabel = await createProTextProperty(ProTextPropertyMock.PROJ_DEF_EN_PROPERTY_PERSON_HAS_APPELLATION) - const person = await createInfResource(InfResourceMock.PERSON_1); - await addInfoToProject(person.pk_entity, ProProjectMock.PROJECT_1.pk_entity); - const stmt = await createInfStatement(InfStatementMock.NAME_1_TO_PERSON); - await addInfoToProject(stmt.pk_entity, ProProjectMock.PROJECT_1.pk_entity); - return { person, classPerson, hasAppePropLabel }; -} - -async function createNamingMock() { - await createInfLanguage(InfLanguageMock.GERMAN); - const project = await createProProject(ProProjectMock.PROJECT_1); - await createProProject(ProProjectMock.DEFAULT_PROJECT); - await createDfhApiClass(DfhApiClassMock.EN_365_NAMING); - await createProDfhProfileProjRel(ProDfhProfileProjRelMock.PROJ_1_PROFILE_4); - const propertyRefersToName = await createDfhApiProperty(DfhApiPropertyMock.EN_1113_REFERS_TO_NAME); - - await createProClassFieldConfig(ProClassFieldConfigMock.PROJ_DEF_C365_NAMING_P1111_IS_APPE_OF) - await createProClassFieldConfig(ProClassFieldConfigMock.PROJ_DEF_C365_NAMING_P1113_REFERS_TO_NAME) - - const naming = await createInfResource(InfResourceMock.NAMING_1); - const namingProjRel = await addInfoToProject(naming.pk_entity, ProProjectMock.PROJECT_1.pk_entity); - - const appellation = await createInfAppellation(InfAppellationMock.JACK_THE_FOO); - const stmtToAppe = await createInfStatement(InfStatementMock.NAME_1_TO_APPE); - await addInfoToProject(stmtToAppe.pk_entity, ProProjectMock.PROJECT_1.pk_entity); - - await createInfTimePrimitive(InfTimePrimitiveMock.TP_1) - - const stmtToTp = await createInfStatement(InfStatementMock.NAMING_1_ONGOING_THROUGHOUT_TP_1) - await addInfoToProject(stmtToTp.pk_entity, project.pk_entity) - - return { naming, namingProjRel, project, appellation, propertyRefersToName }; -} - diff --git a/server/src/__tests__/acceptance/warehouse/aggregator-ds/r-entity-label/REntityLabelService.test.ts b/server/src/__tests__/acceptance/warehouse/aggregator-ds/r-entity-label/REntityLabelService.test.ts deleted file mode 100644 index 25b3f200e..000000000 --- a/server/src/__tests__/acceptance/warehouse/aggregator-ds/r-entity-label/REntityLabelService.test.ts +++ /dev/null @@ -1,387 +0,0 @@ -/* eslint-disable @typescript-eslint/no-invalid-this */ -/* eslint-disable @typescript-eslint/camelcase */ -import 'reflect-metadata'; -import {expect} from '@loopback/testlab'; -import {REntityLabelService} from '../../../../../warehouse/aggregator-ds/entity-label/r-entity-label/REntityLabelService'; -import {EntityPreviewService} from '../../../../../warehouse/aggregator-ds/entity-preview/EntityPreviewService'; -import {WarehouseStubs} from '../../../../../warehouse/createWarehouse'; -import {DfhOutgoingPropertyService} from '../../../../../warehouse/primary-ds/DfhOutgoingPropertyService'; -import {REdgeService} from '../../../../../warehouse/primary-ds/edge/REdgeService'; -import {REntityService} from '../../../../../warehouse/primary-ds/entity/REntityService'; -import {ProEntityLabelConfigService} from '../../../../../warehouse/primary-ds/ProEntityLabelConfigService'; -import {Warehouse} from '../../../../../warehouse/Warehouse'; -import {createDfhApiClass} from '../../../../helpers/atomic/dfh-api-class.helper'; -import {createDfhApiProperty} from '../../../../helpers/atomic/dfh-api-property.helper'; -import {createInfAppellation} from '../../../../helpers/atomic/inf-appellation.helper'; -import {createInfLanguage} from '../../../../helpers/atomic/inf-language.helper'; -import {createInfStatement} from '../../../../helpers/atomic/inf-statement.helper'; -import {createInfResource} from '../../../../helpers/atomic/inf-resource.helper'; -import {createProEntityLabelConfig} from '../../../../helpers/atomic/pro-entity-label-config.helper'; -import {addInfosToProject, createProInfoProjRel, updateProInfoProjRel} from '../../../../helpers/atomic/pro-info-proj-rel.helper'; -import {createProProject} from '../../../../helpers/atomic/pro-project.helper'; -import {DfhApiClassMock} from '../../../../helpers/data/gvDB/DfhApiClassMock'; -import {DfhApiPropertyMock} from '../../../../helpers/data/gvDB/DfhApiPropertyMock'; -import {InfAppellationMock} from '../../../../helpers/data/gvDB/InfAppellationMock'; -import {InfLanguageMock} from '../../../../helpers/data/gvDB/InfLanguageMock'; -import {InfStatementMock} from '../../../../helpers/data/gvDB/InfStatementMock'; -import {InfResourceMock} from '../../../../helpers/data/gvDB/InfResourceMock'; -import {ProEntityLabelConfigMock} from '../../../../helpers/data/gvDB/ProEntityLabelConfigMock'; -import {ProInfoProjRelMock} from '../../../../helpers/data/gvDB/ProInfoProjRelMock'; -import {ProProjectMock} from '../../../../helpers/data/gvDB/ProProjectMock'; -import {createInstancesForCityType} from '../../../../helpers/graphs/cityType.helper'; -import {cleanDb} from '../../../../helpers/meta/clean-db.helper'; -import {createModelForCityType, createLanguages} from '../../../../helpers/meta/model.helper'; -import {setupCleanAndStartWarehouse, stopWarehouse, truncateWarehouseTables, waitForEntityPreview, waitForEntityPreviewUntil, searchUntilSatisfy} from '../../../../helpers/warehouse-helpers'; -import {createProject1, createProject2, createProject3} from '../../../../helpers/graphs/project.helper'; -import {EntityLabel, ENTITY_LABEL_MAX_LENGTH} from '../../../../../warehouse/aggregator-ds/entity-label/entity-label.commons'; - -export const rEntityLabelStub: WarehouseStubs = { - primaryDataServices: [ - DfhOutgoingPropertyService, - ProEntityLabelConfigService, - REntityService, - REdgeService - ], - aggDataServices: [ - // IdentifyingPropertyService, - REntityLabelService, - EntityPreviewService - ] -} - -/** - * Testing whole stack from postgres to warehouse - */ -describe('REntityLabelService', function () { - let wh: Warehouse; - let s: REntityLabelService - - before(async function () { - // eslint-disable-next-line @typescript-eslint/no-invalid-this - this.timeout(5000); // A very long environment setup. - const injector = await setupCleanAndStartWarehouse(rEntityLabelStub) - wh = injector.get(Warehouse) - s = injector.get(REntityLabelService) - - }) - beforeEach(async () => { - await cleanDb() - await truncateWarehouseTables(wh) - }) - after(async function () { - await stopWarehouse(wh) - }) - - it('should create entity label of naming.', async () => { - await createProject(); - await Promise.all([ - createNamingMock(), - waitForEntityPreview(wh, [ - {pk_entity: {eq: InfResourceMock.NAMING_1.pk_entity}}, - {fk_project: {eq: null}}, - {entity_label: {eq: InfAppellationMock.JACK_THE_FOO.string}}, - ]) - ]) - // expect(result.entity_label).to.equal(appellation.string) - }) - it('should create entity label of person', async () => { - await createProject(); - const {person, appellation} = await createNamingAndPersonMock(); - const result = await waitForEntityPreview(wh, [ - {pk_entity: {eq: person.pk_entity}}, - {fk_project: {eq: null}}, - {entity_label: {eq: appellation.string}}, - ]) - expect(result.entity_label).to.equal(appellation.string) - }) - it('should update entity label of person after removing stmt 1111 from project', async () => { - await createProject(); - const {person, appellation} = await createNamingAndPersonMock(); - let result = await waitForEntityPreview(wh, [ - {pk_entity: {eq: person.pk_entity}}, - {fk_project: {eq: null}}, - {entity_label: {eq: appellation.string}}, - ]) - expect(result.entity_label).to.equal(appellation.string) - - await updateProInfoProjRel( - ProInfoProjRelMock.PROJ_1_STMT_NAME_1_TO_PERSON.pk_entity ?? -1 - , { - ...ProInfoProjRelMock.PROJ_1_STMT_NAME_1_TO_PERSON, - is_in_project: false - }) - - result = await waitForEntityPreview(wh, [ - {pk_entity: {eq: person.pk_entity}}, - {fk_project: {eq: null}}, - {entity_label: {eq: '(no label)'}}, - ]) - expect(result.entity_label).to.equal('(no label)') - }) - - it('should create entity label of naming and add person.', async () => { - await createProject(); - const {naming, appellation} = await createNamingMock(); - let result = await waitForEntityPreview(wh, [ - {pk_entity: {eq: naming.pk_entity}}, - {fk_project: {eq: null}}, - {entity_label: {eq: appellation.string}}, - ]) - expect(result.entity_label).to.equal(appellation.string) - - const person = await createPersonMock(); - result = await waitForEntityPreview(wh, [ - {pk_entity: {eq: person.pk_entity}}, - {fk_project: {eq: null}}, - {entity_label: {eq: appellation.string}}, - ]) - expect(result.entity_label).to.equal(appellation.string) - }) - - - // it('should create entity label of Birth – E67 (-- with identifying property)', async () => { - // await createProject(); - // const {appellation} = await createNamingMock(); - // await createPersonMock(); - // const birth = await createBirthMock() - // const result = await waitForEntityPreviewUntil(wh, (item) => { - // return item.pk_entity === birth.pk_entity - // && item.fk_project === null - // && item.entity_label === appellation.string - // }) - // expect(result) - // }) - - // it('should create entity label of Union – C9 (-- with identifying property)', async () => { - // await createProject(); - // const {appellation} = await createNamingMock(); - // await createPersonMock(); - // const union = await createUnionMock() - // const result = await waitForEntityPreviewUntil(wh, (item) => { - // return item.pk_entity === union.pk_entity - // && item.fk_project === null - // && item.entity_label === appellation.string - // }) - // expect(result) - // }) - - it('should create entity label of Union with 3 label parts', async () => { - await createProject(); - await createNamingMock(); - await createPersonMock(); - const {union} = await createUnion2Mock() - const result = await waitForEntityPreviewUntil(wh, (item) => { - // console.log(item.entity_label) - return item.pk_entity === union.pk_entity - && item.fk_project === null - && item.entity_label === '(no label), Jack the foo, (no label)' - }) - expect(result) - }) - - it('should create entity label of birth infinit label', async function() { - this.timeout(15000); // A very long environment setup. - await createProject(); - await createNamingMock(); - await createPersonMock(); - const {birth} = await EntityLabel.createInfinitLabel() - - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx({pkEntity: birth.pk_entity ?? -1}), - compare: (item) => { - return item?.entityLabel?.length === ENTITY_LABEL_MAX_LENGTH - } - }) - }) - - /** - * tests if the entity label of the repo variant is the label that has the highest - * rank amongst the project variants - */ - it('should create entity label of Geov.Place Type – according to most used is appe. of stmt', async () => { - await createLanguages() - await createModelForCityType(); - - const {project1} = await createProject1(); - - const {project2} = await createProject2(); - - const {project3} = await createProject3(); - - // create instances - const { - cityType, - appeCity, - appeStadt, - namingCity1: naming1, - namingCity1RefersTo: naming1RefersTo, - namingCity1IsAppeOf: naming1IsAppeOf, - namingStadt2: naming2, - namingStadt2RefersTo: naming2RefersTo, - namingStadt2IsAppeOf: naming2IsAppeOf - } = await createInstancesForCityType(); - - // add naming 1 to project1 - await addInfosToProject(project1.pk_entity, [ - cityType.pk_entity, - naming1.pk_entity, - naming1RefersTo.pk_entity, - naming1IsAppeOf.pk_entity]) - - // add naming 2 to project2 - await addInfosToProject(project2.pk_entity, [ - cityType.pk_entity, - naming2.pk_entity, - naming2RefersTo.pk_entity, - naming2IsAppeOf.pk_entity]) - - // add naming 2 to project3 - await addInfosToProject(project3.pk_entity, [ - cityType.pk_entity, - naming2.pk_entity, - naming2RefersTo.pk_entity, - naming2IsAppeOf.pk_entity]) - - - - // since naming 2 is more often used, - // -> the repo label should be 'Stadt' - await waitForEntityPreviewUntil(wh, (item) => { - return item.pk_entity === cityType.pk_entity - && item.fk_project === null - && item.entity_label === appeStadt.string - }) - - // add naming 1 to project 2 - await addInfosToProject(project2.pk_entity, [ - naming1.pk_entity, - naming1RefersTo.pk_entity, - naming1IsAppeOf.pk_entity]) - // add naming 1 to project 3 - await addInfosToProject(project3.pk_entity, [ - naming1.pk_entity, - naming1RefersTo.pk_entity, - naming1IsAppeOf.pk_entity]) - - // now naming1IsAppeOf is in 3 projects, naming2IsAppeOf in 2 projects - // -> the repo label should be 'City' - const repoVUpdated = await waitForEntityPreviewUntil(wh, (item) => { - return item.pk_entity === cityType.pk_entity - && item.fk_project === null - && item.entity_label === appeCity.string - }) - expect(repoVUpdated) - - }) -}) - - -async function createNamingAndPersonMock() { - // NAMING - const {naming, appellation} = await createNamingMock(); - // PERSON - const person = await createPersonMock(); - return {naming, person, appellation}; -} - -async function createPersonMock() { - await createDfhApiClass(DfhApiClassMock.EN_21_PERSON); - const person = await createInfResource(InfResourceMock.PERSON_1); - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_PERSON_1); - return person; -} - -async function createProject() { - await createInfLanguage(InfLanguageMock.GERMAN); - const project = await createProProject(ProProjectMock.PROJECT_1); - await createProProject(ProProjectMock.DEFAULT_PROJECT); - return project -} - -async function createNamingMock() { - await createDfhApiClass(DfhApiClassMock.EN_365_NAMING); - - // TeEn - const naming = await createInfResource(InfResourceMock.NAMING_1); - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_NAMING_1); - - const appellation = await createInfAppellation(InfAppellationMock.JACK_THE_FOO); - await createInfStatement(InfStatementMock.NAME_1_TO_APPE); - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_STMT_NAME_1_TO_APPE); - - await createInfStatement(InfStatementMock.NAME_1_TO_PERSON); - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_STMT_NAME_1_TO_PERSON); - return {naming, appellation}; -} - - -// async function createBirthMock() { - -// // MODEL + LABELS -// await createDfhApiClass(DfhApiClassMock.EN_61_BIRTH); -// await createDfhApiProperty(DfhApiPropertyMock.EN_86_BROUGHT_INTO_LIFE); -// await createDfhApiProperty(DfhApiPropertyMock.EN_1435_STEMS_FROM); - -// // TeEn -// const birth = await createInfTemporalEntity(InfTemporalEntityMock.BIRTH_1); -// await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_BIRTH); - -// // Stmts -// await createInfStatement(InfStatementMock.BIRTH_1_BROUGHT_INTO_LIFE_PERSON_1); -// await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_STMT_BIRTH_1_BROUGHT_INTO_LIFE_PERON_1); -// return birth; -// } - - -// async function createUnionMock() { - -// // MODEL + LABELS -// await createDfhApiClass(DfhApiClassMock.EN_633_UNION); -// await createDfhApiProperty(DfhApiPropertyMock.EN_1436_HAS_PARTNER); - -// // TeEn -// const union = await createInfTemporalEntity(InfTemporalEntityMock.UNION_1); -// await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_UNION_1); - -// // Stmts -// await createInfStatement(InfStatementMock.UNOIN_1_HAS_PARTNER_1); -// await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_STMT_UNOIN_1_HAS_PARTNER_1); - -// await createProEntityLabelConfig(ProEntityLabelConfigMock.C633_UNION_PROJECT_DEFAULT) -// return union; -// } - -export async function createUnion2Mock() { - - // MODEL + LABELS - await createDfhApiClass(DfhApiClassMock.EN_633_UNION); - await createDfhApiProperty(DfhApiPropertyMock.EN_1436_HAS_PARTNER); - await createDfhApiClass(DfhApiClassMock.EN_61_BIRTH); - await createDfhApiProperty(DfhApiPropertyMock.EN_1435_STEMS_FROM); - - // TeEn - const union = await createInfResource(InfResourceMock.UNION_1); - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_UNION_1); - - const birth = await createInfResource(InfResourceMock.BIRTH_1); - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_BIRTH); - - // peIt - await createInfResource(InfResourceMock.ALBERT_IV); - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_ALBERT_IV); - - // Stmts - await createInfStatement(InfStatementMock.UNOIN_1_HAS_PARTNER_1); - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_STMT_UNOIN_1_HAS_PARTNER_1); - - await createInfStatement(InfStatementMock.UNOIN_1_HAS_PARTNER_2); - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_STMT_UNOIN_1_HAS_PARTNER_2); - - await createInfStatement(InfStatementMock.BIRTH_1_STEMS_FROM_UNION_1); - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_STMT_BIRTH_1_STEMS_FROM_UNION_1); - - await createProEntityLabelConfig(ProEntityLabelConfigMock.C633_UNION_PROJECT_DEFAULT_2) - return {union, birth}; -} - diff --git a/server/src/__tests__/acceptance/warehouse/aggregator-ds/r-entity-time-span/REntityTimeSpanService.test.ts b/server/src/__tests__/acceptance/warehouse/aggregator-ds/r-entity-time-span/REntityTimeSpanService.test.ts deleted file mode 100644 index 44013831e..000000000 --- a/server/src/__tests__/acceptance/warehouse/aggregator-ds/r-entity-time-span/REntityTimeSpanService.test.ts +++ /dev/null @@ -1,243 +0,0 @@ -/* eslint-disable @typescript-eslint/camelcase */ -import {expect} from '@loopback/testlab'; -import {equals} from 'ramda'; -import 'reflect-metadata'; -import {WarEntityPreviewTimeSpan} from '../../../../../models/entity-preview/WarEntityPreviewTimeSpan'; -import {CalendarType} from '../../../../../models/enums/CalendarType'; -import {Granularity} from '../../../../../models/enums/Granularity'; -import {EntityPreviewService} from '../../../../../warehouse/aggregator-ds/entity-preview/EntityPreviewService'; -import {REntityTimeSpanService} from '../../../../../warehouse/aggregator-ds/entity-time-span/r-entity-time-span/REntityTimeSpanService'; -import {WarehouseStubs} from '../../../../../warehouse/createWarehouse'; -import {REdgeService} from '../../../../../warehouse/primary-ds/edge/REdgeService'; -import {REntityService} from '../../../../../warehouse/primary-ds/entity/REntityService'; -import {Warehouse} from '../../../../../warehouse/Warehouse'; -import {createDfhApiClass} from '../../../../helpers/atomic/dfh-api-class.helper'; -import {createDfhApiProperty} from '../../../../helpers/atomic/dfh-api-property.helper'; -import {createInfLanguage} from '../../../../helpers/atomic/inf-language.helper'; -import {createInfResource} from '../../../../helpers/atomic/inf-resource.helper'; -import {createInfStatement} from '../../../../helpers/atomic/inf-statement.helper'; -import {createInfTimePrimitive} from '../../../../helpers/atomic/inf-time-primitive.helper'; -import {createProInfoProjRel} from '../../../../helpers/atomic/pro-info-proj-rel.helper'; -import {createProProject} from '../../../../helpers/atomic/pro-project.helper'; -import {DfhApiClassMock} from '../../../../helpers/data/gvDB/DfhApiClassMock'; -import {DfhApiPropertyMock} from '../../../../helpers/data/gvDB/DfhApiPropertyMock'; -import {InfLanguageMock} from '../../../../helpers/data/gvDB/InfLanguageMock'; -import {InfResourceMock} from '../../../../helpers/data/gvDB/InfResourceMock'; -import {InfStatementMock} from '../../../../helpers/data/gvDB/InfStatementMock'; -import {InfTimePrimitiveMock} from '../../../../helpers/data/gvDB/InfTimePrimitiveMock'; -import {ProInfoProjRelMock} from '../../../../helpers/data/gvDB/ProInfoProjRelMock'; -import {ProProjectMock} from '../../../../helpers/data/gvDB/ProProjectMock'; -import {cleanDb} from '../../../../helpers/meta/clean-db.helper'; -import {searchUntilSatisfy, setupCleanAndStartWarehouse, stopWarehouse, truncateWarehouseTables, waitForEntityPreviewUntil} from '../../../../helpers/warehouse-helpers'; -const rEntityTimeSpanStub: WarehouseStubs = { - primaryDataServices: [ - REntityService, - REdgeService - ], - aggDataServices: [ - REntityTimeSpanService, - EntityPreviewService - ] -} -/** - * Testing whole stack from postgres to warehouse - */ -describe('REntityTimeSpanService', function () { - let wh: Warehouse; - let edgeService: REdgeService; - let s: REntityTimeSpanService - - before(async function () { - // eslint-disable-next-line @typescript-eslint/no-invalid-this - this.timeout(5000); // A very long environment setup. - const injector = await setupCleanAndStartWarehouse(rEntityTimeSpanStub) - wh = injector.get(Warehouse) - s = injector.get(REntityTimeSpanService) - edgeService = injector.get(REdgeService) - - }) - beforeEach(async () => { - await cleanDb() - await truncateWarehouseTables(wh) - }) - after(async function () { - await stopWarehouse(wh) - }) - - it('should create edges with time primitives', async () => { - const {shipVoyage} = await createMock(); - await searchUntilSatisfy({ - notifier$: edgeService.afterChange$, - getFn: () => edgeService.index.getFromIdx({ - pkEntity: shipVoyage.pk_entity ?? -1, - }), - compare: (val) => !!val?.outgoing?.[71]?.length - }) - }) - - - it('should create timespanval of time primitive', async () => { - const {shipVoyage} = await createMock(); - const expectedTimeSpan: WarEntityPreviewTimeSpan = { - "p81": { - "calendar": CalendarType.gregorian, - "duration": Granularity['1 day'], - "julianDay": 2362729 - }, - "p82": { - "calendar": CalendarType.gregorian, - "duration": Granularity['1 day'], - "julianDay": 2362730 - }, - "p81a": { - "calendar": CalendarType.gregorian, - "duration": Granularity['1 day'], - "julianDay": 2362731 - }, - "p81b": { - "calendar": CalendarType.gregorian, - "duration": Granularity['1 day'], - "julianDay": 2362732 - }, - "p82a": { - "calendar": CalendarType.julian, - "duration": Granularity['1 day'], - "julianDay": 2362733 - }, - "p82b": { - "calendar": CalendarType.julian, - "duration": Granularity['1 day'], - "julianDay": 2362734 - } - } - const expectedFirstSec = '204139785600'; - const expectedLastSec = '204140303999'; - - const result = await waitForEntityPreviewUntil(wh, (entPreview) => { - return entPreview.pk_entity === shipVoyage.pk_entity - && entPreview.fk_project === null - && equals(entPreview.time_span, expectedTimeSpan) - && entPreview.first_second === expectedFirstSec - && entPreview.last_second === expectedLastSec - }) - - expect(result.time_span).to.deepEqual(expectedTimeSpan); - }) - - it('should create empty time span object {}', async () => { - // - Langage and Project - await createProjectAndModelMock(); - - // - shipVoyage - const shipVoyage = await createInfResource(InfResourceMock.SHIP_VOYAGE); - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_SHIP_VOYAGE); - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx({pkEntity: shipVoyage.pk_entity ?? -1}), - compare: (val) => { - return equals(val?.timeSpan, {}) - } - }) - }) -}) - -// create the mock data: -async function createMock() { - // - Langage and Project - const project = await createProjectAndModelMock(); - - // - shipVoyage - const shipVoyage = await createInfResource(InfResourceMock.SHIP_VOYAGE); - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_SHIP_VOYAGE); - - // TimePrimitive 1 - await createInfTimePrimitive(InfTimePrimitiveMock.TP_1); - // Stmt to TimePrimitive 1 - await createInfStatement(InfStatementMock.SHIP_VOYAGE_ONGOING_THROUGHOUT_TP_1); - // Project rel for stmt (With calender info !) - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_STMT_SHIP_VOYAGE_ONGOING_THROUGHOUT_TP_1) - - // TimePrimitive 2 - await createInfTimePrimitive(InfTimePrimitiveMock.TP_2); - // Stmt to TimePrimitive 2 - await createInfStatement(InfStatementMock.SHIP_VOYAGE_AT_SOME_TIME_WITHIN_TP_2); - // Project rel for stmt (With calender info !) - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_STMT_SHIP_VOYAGE_AT_SOME_TIME_WITHIN_TP_2) - - // TimePrimitive 3 - await createInfTimePrimitive(InfTimePrimitiveMock.TP_3); - // Stmt to TimePrimitive 3 - await createInfStatement(InfStatementMock.SHIP_VOYAGE_END_OF_THE_BEGIN_TP_3); - // Project rel for stmt (With calender info !) - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_STMT_SHIP_VOYAGE_END_OF_THE_BEGIN_TP_3) - - // TimePrimitive 4 - await createInfTimePrimitive(InfTimePrimitiveMock.TP_4); - // Stmt to TimePrimitive 4 - await createInfStatement(InfStatementMock.SHIP_VOYAGE_BEGIN_OF_THE_END_TP_4); - // Project rel for stmt (With calender info !) - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_STMT_SHIP_VOYAGE_BEGIN_OF_THE_END_TP_4) - - // TimePrimitive 5 - await createInfTimePrimitive(InfTimePrimitiveMock.TP_5); - // Stmt to TimePrimitive 5 - await createInfStatement(InfStatementMock.SHIP_VOYAGE_BEGIN_OF_THE_BEGIN_TP_5); - // Project rel for stmt (With calender info !) - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_STMT_SHIP_VOYAGE_BEGIN_OF_THE_BEGIN_TP_5) - - // TimePrimitive 6 - await createInfTimePrimitive(InfTimePrimitiveMock.TP_6); - // Stmt to TimePrimitive 6 - await createInfStatement(InfStatementMock.SHIP_VOYAGE_END_OF_THE_END_TP_6); - // Project rel for stmt (With calender info !) - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_STMT_SHIP_VOYAGE_END_OF_THE_END_TP_6) - - return {shipVoyage, project}; -} - - - - -async function createProjectAndModelMock() { - await createInfLanguage(InfLanguageMock.GERMAN); - const project = await createProProject(ProProjectMock.PROJECT_1); - - // - Class: Ship Voyage, TimePrimitive - await createDfhApiClass(DfhApiClassMock.EN_523_SHIP_VOYAGE); - await createDfhApiClass(DfhApiClassMock.EN_335_TIME_PRIMITIVE); - - // P81 ongoing throughout - await createDfhApiProperty(DfhApiPropertyMock.EN_71_ONGOING_THROUGHOUT); - // P82 at some time within - await createDfhApiProperty(DfhApiPropertyMock.EN_72_AT_SOME_TIME_WITHIN); - // P81a end of the begin - await createDfhApiProperty(DfhApiPropertyMock.EN_150_END_OF_THE_BEGIN); - // P81b begin of the end - await createDfhApiProperty(DfhApiPropertyMock.EN_151_BEGIN_OF_THE_END); - // P82a begin of the begin - await createDfhApiProperty(DfhApiPropertyMock.EN_152_BEGIN_OF_THE_BEGIN); - // P82b end of the end - await createDfhApiProperty(DfhApiPropertyMock.EN_153_END_OF_THE_END); - return project; -} -/** - -select * -FROM -information.time_primitive t1, -projects.info_proj_rel t2 -WHERE t1.pk_entity=t2.fk_entity -LIMIT 10; - - -select t2.calendar -FROM -information."statement" t0, -information.time_primitive t1, -projects.info_proj_rel t2 -WHERE -t0.fk_object_info = t1.pk_entity -AND t0.pk_entity=t2.fk_entity -LIMIT 10 - - */ diff --git a/server/src/__tests__/acceptance/warehouse/aggregator-ds/r-entity-type/REntityTypeService.test.ts b/server/src/__tests__/acceptance/warehouse/aggregator-ds/r-entity-type/REntityTypeService.test.ts deleted file mode 100644 index d2e1cd5df..000000000 --- a/server/src/__tests__/acceptance/warehouse/aggregator-ds/r-entity-type/REntityTypeService.test.ts +++ /dev/null @@ -1,147 +0,0 @@ -/* eslint-disable @typescript-eslint/camelcase */ -import 'reflect-metadata'; -import {expect} from '@loopback/testlab'; -import {REntityLabelService} from '../../../../../warehouse/aggregator-ds/entity-label/r-entity-label/REntityLabelService'; -import {EntityPreviewService} from '../../../../../warehouse/aggregator-ds/entity-preview/EntityPreviewService'; -import {REntityTypeService} from '../../../../../warehouse/aggregator-ds/entity-type/r-entity-type/REntityTypeService'; -import {WarehouseStubs} from '../../../../../warehouse/createWarehouse'; -import {DfhClassHasTypePropertyService} from '../../../../../warehouse/primary-ds/DfhClassHasTypePropertyService'; -import {DfhOutgoingPropertyService} from '../../../../../warehouse/primary-ds/DfhOutgoingPropertyService'; -import {REdgeService} from '../../../../../warehouse/primary-ds/edge/REdgeService'; -import {REntityService} from '../../../../../warehouse/primary-ds/entity/REntityService'; -import {ProEntityLabelConfigService} from '../../../../../warehouse/primary-ds/ProEntityLabelConfigService'; -import {Warehouse} from '../../../../../warehouse/Warehouse'; -import {createDfhApiClass} from '../../../../helpers/atomic/dfh-api-class.helper'; -import {createDfhApiProperty} from '../../../../helpers/atomic/dfh-api-property.helper'; -import {createInfAppellation} from '../../../../helpers/atomic/inf-appellation.helper'; -import {createInfLanguage} from '../../../../helpers/atomic/inf-language.helper'; -import {createInfStatement} from '../../../../helpers/atomic/inf-statement.helper'; -import {createInfResource} from '../../../../helpers/atomic/inf-resource.helper'; -import {createProInfoProjRel, updateProInfoProjRel} from '../../../../helpers/atomic/pro-info-proj-rel.helper'; -import {createProProject} from '../../../../helpers/atomic/pro-project.helper'; -import {cleanDb} from '../../../../helpers/meta/clean-db.helper'; -import {DfhApiClassMock} from '../../../../helpers/data/gvDB/DfhApiClassMock'; -import {DfhApiPropertyMock} from '../../../../helpers/data/gvDB/DfhApiPropertyMock'; -import {InfAppellationMock} from '../../../../helpers/data/gvDB/InfAppellationMock'; -import {InfLanguageMock} from '../../../../helpers/data/gvDB/InfLanguageMock'; -import {InfResourceMock} from '../../../../helpers/data/gvDB/InfResourceMock'; -import {InfStatementMock} from '../../../../helpers/data/gvDB/InfStatementMock'; -import {ProInfoProjRelMock} from '../../../../helpers/data/gvDB/ProInfoProjRelMock'; -import {ProProjectMock} from '../../../../helpers/data/gvDB/ProProjectMock'; -import {setupCleanAndStartWarehouse, stopWarehouse, truncateWarehouseTables, waitForEntityPreview} from '../../../../helpers/warehouse-helpers'; -const rEntityTypeServiceStub: WarehouseStubs = { - primaryDataServices: [ - DfhOutgoingPropertyService, - ProEntityLabelConfigService, - REntityService, - REdgeService, - DfhClassHasTypePropertyService - ], - aggDataServices: [ - // IdentifyingPropertyService, - REntityLabelService, - REntityTypeService, - EntityPreviewService - ] -} -/** - * Testing whole stack from postgres to warehouse - */ -describe('REntityTypeService', function () { - let wh: Warehouse; - - before(async function () { - // eslint-disable-next-line @typescript-eslint/no-invalid-this - this.timeout(50000); // A very long environment setup. - const injector = await setupCleanAndStartWarehouse(rEntityTypeServiceStub) - wh = injector.get(Warehouse) - }) - beforeEach(async () => { - await cleanDb() - await truncateWarehouseTables(wh) - }) - after(async function () { - await stopWarehouse(wh) - }) - // TODO: Test that the entity type most often used by projects is used - // for the repo variant (should be the case according to order by of edges) - - - it('should create fk_type of geographical place', async () => { - const {madrid, cityTypeAppe} = await createMock(); - - const result = await waitForEntityPreview(wh, [ - {pk_entity: {eq: madrid.pk_entity}}, - {project: {eq: 0}}, - {type_label: {eq: cityTypeAppe.string}}, - ]) - expect(result.type_label).to.equal(cityTypeAppe.string); - }) - - it('should update fk_type of geographical place', async () => { - const {madrid, cityTypeAppe} = await createMock(); - - let result = await waitForEntityPreview(wh, [ - {pk_entity: {eq: madrid.pk_entity}}, - {project: {eq: 0}}, - {type_label: {eq: cityTypeAppe.string}}, - ]) - expect(result.type_label).to.equal(cityTypeAppe.string); - - await updateProInfoProjRel( - ProInfoProjRelMock.PROJ_1_STMT_MADRID_HAS_GEO_PLACE_CITY_TYPE.pk_entity ?? -1, - {is_in_project: false} - ) - - result = await waitForEntityPreview(wh, [ - {pk_entity: {eq: madrid.pk_entity}}, - {project: {eq: 0}}, - ]) - expect(result.type_label).to.not.equal(cityTypeAppe.string); - }) - -}) - -// create the mock data: -async function createMock() { - // - Langage and Project - await createInfLanguage(InfLanguageMock.GERMAN); - const project = await createProProject(ProProjectMock.PROJECT_1); - - // - Class: Geo Place, Geo Place type - await createDfhApiClass(DfhApiClassMock.EN_363_GEO_PLACE); - await createDfhApiClass(DfhApiClassMock.EN_364_GEO_PLACE_TYPE); - await createDfhApiClass(DfhApiClassMock.EN_365_NAMING); - - // - Property: has geo. place type - await createDfhApiProperty(DfhApiPropertyMock.EN_1110_HAS_GEO_PLACE_TYPE); - - // - peIt (Geo Place (Madrid)) - const madrid = await createInfResource(InfResourceMock.GEO_PLACE_MADRID); - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_MADRID); - - // - peIt (Geo Place Type X) - await createInfResource(InfResourceMock.GEO_PLACE_TYPE_CITY); - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_CITY_TYPE); - - // - stmt (has geo. place type, (Madrid has type 'City')) - await createInfStatement(InfStatementMock.MADRID_HAS_GEO_PLACE_TYPE_CITY); - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_STMT_MADRID_HAS_GEO_PLACE_CITY_TYPE); - - // - appe ('City) - const cityTypeAppe = await createInfAppellation(InfAppellationMock.CITY) - - // - teEn Y (Naming of peIt X = 'City') - await createInfResource(InfResourceMock.NAMING_1_CITY); - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_NAMING_CITY); - - // - stmt (Y refers to Name appe 'City') - await createInfStatement(InfStatementMock.NAMING_CITY_TO_APPE_CITY); - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_STMT_NAMING_CITY_TO_APPE_CITY); - - // - stmt (Y is Naming of Geo Place Type X) - await createInfStatement(InfStatementMock.NAMING_CITY_TO_GEO_PLACE_TYPE); - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_STMT_NAMING_CITY_TO_GEO_PLACE_TYPE); - - return {madrid, project, cityTypeAppe}; -} diff --git a/server/src/__tests__/acceptance/warehouse/primary-ds/DfhClassHasTypePropertyService.test.ts b/server/src/__tests__/acceptance/warehouse/primary-ds/DfhClassHasTypePropertyService.test.ts deleted file mode 100644 index 64fecc3ce..000000000 --- a/server/src/__tests__/acceptance/warehouse/primary-ds/DfhClassHasTypePropertyService.test.ts +++ /dev/null @@ -1,68 +0,0 @@ -/* eslint-disable @typescript-eslint/camelcase */ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import 'reflect-metadata'; -import {DfhClassHasTypePropertyService, RClassId} from '../../../../warehouse/primary-ds/DfhClassHasTypePropertyService'; -import {Warehouse} from '../../../../warehouse/Warehouse'; -import {createDfhApiProperty, updateDfhApiProperty} from '../../../helpers/atomic/dfh-api-property.helper'; -import {DfhApiPropertyMock} from '../../../helpers/data/gvDB/DfhApiPropertyMock'; -import {searchUntilSatisfy, setupCleanAndStartWarehouse, stopWarehouse, truncateWarehouseTables} from '../../../helpers/warehouse-helpers'; -import {WarehouseStubs} from '../../../../warehouse/createWarehouse'; -import {cleanDb} from '../../../helpers/meta/clean-db.helper'; -const stubs: WarehouseStubs = { - primaryDataServices:[DfhClassHasTypePropertyService], - aggDataServices:[] -} -describe('DfhClassHasTypePropertyService', () => { - - let wh: Warehouse; - let s: DfhClassHasTypePropertyService; - - before(async function () { - // eslint-disable-next-line @typescript-eslint/no-invalid-this - this.timeout(5000); // A very long environment setup. - const injector = await setupCleanAndStartWarehouse(stubs) - wh = injector.get(Warehouse) - s = injector.get(DfhClassHasTypePropertyService) -}) - beforeEach(async () => { - await cleanDb() - await truncateWarehouseTables(wh) - }) - after(async function () { - await stopWarehouse(wh) - }) - it('should have class-has-type-property in index', async () => { - await createDfhApiProperty(DfhApiPropertyMock.EN_1110_HAS_GEO_PLACE_TYPE) - const id: RClassId = { - pkClass: DfhApiPropertyMock.EN_1110_HAS_GEO_PLACE_TYPE.dfh_property_domain - } - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => val?.fkProperty === DfhApiPropertyMock.EN_1110_HAS_GEO_PLACE_TYPE.dfh_pk_property - }) - - }) - - it('should delete class-has-type-property in index', async () => { - const prop = await createDfhApiProperty(DfhApiPropertyMock.EN_1110_HAS_GEO_PLACE_TYPE) - const id: RClassId = { - pkClass: DfhApiPropertyMock.EN_1110_HAS_GEO_PLACE_TYPE.dfh_property_domain - } - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => val?.fkProperty === DfhApiPropertyMock.EN_1110_HAS_GEO_PLACE_TYPE.dfh_pk_property - }) - await updateDfhApiProperty(prop.pk_entity, {dfh_is_has_type_subproperty: false}) - - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdxWithTmsps(id), - compare: (item) => !!item?.deleted - }) - }) - - -}); - diff --git a/server/src/__tests__/acceptance/warehouse/primary-ds/DfhClassLabelService.test.ts b/server/src/__tests__/acceptance/warehouse/primary-ds/DfhClassLabelService.test.ts deleted file mode 100644 index 7ef216bc1..000000000 --- a/server/src/__tests__/acceptance/warehouse/primary-ds/DfhClassLabelService.test.ts +++ /dev/null @@ -1,76 +0,0 @@ -/* eslint-disable @typescript-eslint/camelcase */ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import 'reflect-metadata'; -import {expect} from '@loopback/testlab'; -import {DfhClassLabelService} from '../../../../warehouse/primary-ds/DfhClassLabelService'; -import {Warehouse} from '../../../../warehouse/Warehouse'; -import {createDfhApiClass, deleteDfhApiClass, updateDfhApiClass} from '../../../helpers/atomic/dfh-api-class.helper'; -import {setupCleanAndStartWarehouse, stopWarehouse, wait, truncateWarehouseTables} from '../../../helpers/warehouse-helpers'; -import {WarehouseStubs} from '../../../../warehouse/createWarehouse'; -import {cleanDb} from '../../../helpers/meta/clean-db.helper'; -const dfhClassLabelServiceStub:WarehouseStubs={ - primaryDataServices:[DfhClassLabelService], - aggDataServices:[] -} - -describe('DfhClassLabelService', () => { - - let wh: Warehouse; - let s: DfhClassLabelService; - - before(async function () { - // eslint-disable-next-line @typescript-eslint/no-invalid-this - this.timeout(5000); // A very long environment setup. - const injector = await setupCleanAndStartWarehouse(dfhClassLabelServiceStub) - wh = injector.get(Warehouse) - s = injector.get(DfhClassLabelService) - - }) - beforeEach(async () => { - await cleanDb() - await truncateWarehouseTables(wh) - }) - after(async function () { - await stopWarehouse(wh) - }) - it('should have api class label in index after initIdx()', async () => { - const c = await createDfhApiClass({dfh_class_label: 'Foo'}) - await wait(200); - const result = await s.index.getFromIdx({pkClass: c.dfh_pk_class, language: 18889}) - expect(result?.label).to.equal('Foo') - - }) - - it('should update class label', async () => { - const c = await createDfhApiClass({dfh_class_label: 'Foo'}) - - await wait(200); - - const result = await s.index.getFromIdx({pkClass: c.dfh_pk_class, language: 18889}) - expect(result?.label).to.equal('Foo') - - const c2 = await updateDfhApiClass(c.pk_entity as any, {dfh_class_label: 'Bar'}) - expect(c2.dfh_class_label).to.equal('Bar') - - await wait(200); - const resultUpdated = await s.index.getFromIdx({pkClass: c.dfh_pk_class, language: 18889}) - expect(resultUpdated?.label).to.equal('Bar') - }) - - it('should NOT! delete class label', async () => { - const c = await createDfhApiClass({dfh_class_label: 'Foo'}) - - await wait(200); - const result = await s.index.getFromIdx({pkClass: c.dfh_pk_class, language: 18889}) - expect(result?.label).to.equal('Foo') - - await deleteDfhApiClass(c.pk_entity as any) - - - await new Promise(r => setTimeout(r, 10)); - const resultUpdated = await s.index.getFromIdx({pkClass: c.dfh_pk_class, language:18889}) - expect(resultUpdated?.label).to.equal('Foo') - }) - -}); - diff --git a/server/src/__tests__/acceptance/warehouse/primary-ds/DfhOutgoingPropertyService.test.ts b/server/src/__tests__/acceptance/warehouse/primary-ds/DfhOutgoingPropertyService.test.ts deleted file mode 100644 index 29ea02ab9..000000000 --- a/server/src/__tests__/acceptance/warehouse/primary-ds/DfhOutgoingPropertyService.test.ts +++ /dev/null @@ -1,63 +0,0 @@ -/* eslint-disable @typescript-eslint/camelcase */ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import 'reflect-metadata'; -import {expect} from '@loopback/testlab'; -import {DfhOutgoingPropertyService} from '../../../../warehouse/primary-ds/DfhOutgoingPropertyService'; -import {Warehouse} from '../../../../warehouse/Warehouse'; -import {createDfhApiProperty} from '../../../helpers/atomic/dfh-api-property.helper'; -import {cleanDb} from '../../../helpers/meta/clean-db.helper'; -import {DfhApiPropertyMock} from '../../../helpers/data/gvDB/DfhApiPropertyMock'; -import {searchUntilSatisfy, setupCleanAndStartWarehouse, stopWarehouse, truncateWarehouseTables} from '../../../helpers/warehouse-helpers'; -import {WarehouseStubs} from '../../../../warehouse/createWarehouse'; -const stubs: WarehouseStubs = { - primaryDataServices:[DfhOutgoingPropertyService], - aggDataServices:[] -} -describe('DfhOutgoingPropertyService', () => { - - let wh: Warehouse; - let s: DfhOutgoingPropertyService; - - before(async function () { - // eslint-disable-next-line @typescript-eslint/no-invalid-this - this.timeout(5000); // A very long environment setup. - const injector = await setupCleanAndStartWarehouse(stubs) - wh = injector.get(Warehouse) - s = injector.get(DfhOutgoingPropertyService) - - }) - beforeEach(async () => { - await cleanDb() - await truncateWarehouseTables(wh) - }) - after(async function () { - await stopWarehouse(wh) - }) - - it('should have two outgoing properties in index', async () => { - await Promise.all([ - createDfhApiProperty(DfhApiPropertyMock.EN_86_BROUGHT_INTO_LIFE), - createDfhApiProperty(DfhApiPropertyMock.EN_1435_STEMS_FROM) - ]) - - const [a, b] = await Promise.all([ - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx({fkDomain: 61, fkProperty: 86}), - compare: (v) => v?.dfhIdentityDefining === true - }), - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx({fkDomain: 61, fkProperty: 1435}), - compare: (v) => v?.dfhIdentityDefining === false - }) - ]) - - expect(a?.dfhIdentityDefining).to.equal(true) - expect(b?.dfhIdentityDefining).to.equal(false) - - }) - - -}); - diff --git a/server/src/__tests__/acceptance/warehouse/primary-ds/DfhPropertyLabelService.test.ts b/server/src/__tests__/acceptance/warehouse/primary-ds/DfhPropertyLabelService.test.ts deleted file mode 100644 index 4132d6b97..000000000 --- a/server/src/__tests__/acceptance/warehouse/primary-ds/DfhPropertyLabelService.test.ts +++ /dev/null @@ -1,86 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import {expect} from '@loopback/testlab'; -import 'reflect-metadata'; -import {WarehouseStubs} from '../../../../warehouse/createWarehouse'; -import {DfhPropertyLabelId, DfhPropertyLabelService} from '../../../../warehouse/primary-ds/DfhPropertyLabelService'; -import {Warehouse} from '../../../../warehouse/Warehouse'; -import {createDfhApiProperty, deleteDfhApiProperty, updateDfhApiProperty} from '../../../helpers/atomic/dfh-api-property.helper'; -import {DfhApiPropertyMock} from '../../../helpers/data/gvDB/DfhApiPropertyMock'; -import {cleanDb} from '../../../helpers/meta/clean-db.helper'; -import {searchUntilSatisfy, setupCleanAndStartWarehouse, stopWarehouse, truncateWarehouseTables, wait} from '../../../helpers/warehouse-helpers'; -const stubs: WarehouseStubs = { - primaryDataServices: [DfhPropertyLabelService], - aggDataServices: [] -} -describe('DfhPropertyLabelService', () => { - - let wh: Warehouse; - let s: DfhPropertyLabelService; - - before(async function () { - // eslint-disable-next-line @typescript-eslint/no-invalid-this - this.timeout(5000); // A very long environment setup. - const injector = await setupCleanAndStartWarehouse(stubs) - wh = injector.get(Warehouse) - s = injector.get(DfhPropertyLabelService) - }) - beforeEach(async () => { - await cleanDb() - await truncateWarehouseTables(wh) - }) - after(async function () { - await stopWarehouse(wh) - }) - it('should have api property label in index after initIdx()', async () => { - const c = await createDfhApiProperty(DfhApiPropertyMock.EN_86_BROUGHT_INTO_LIFE); - const id: DfhPropertyLabelId = {pkProperty: c.dfh_pk_property, language: 18889} - - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => val?.label === DfhApiPropertyMock.EN_86_BROUGHT_INTO_LIFE.dfh_property_label - }) - }) - - it('should update property label', async () => { - const c = await createDfhApiProperty({dfh_property_label: 'Foo', dfh_property_inverse_label: 'has Foo'}) - - const id: DfhPropertyLabelId = {pkProperty: c.dfh_pk_property, language: 18889} - - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => val?.label === 'Foo' && val?.inverseLabel === 'has Foo' - }) - - const c2 = await updateDfhApiProperty(c.pk_entity as any, {dfh_property_label: 'Bar'}) - expect(c2.dfh_property_label).to.equal('Bar') - - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => val?.label === 'Bar' - }) - }) - - it('should NOT! delete property label', async () => { - const c = await createDfhApiProperty({dfh_property_label: 'Foo'}) - const id: DfhPropertyLabelId = {pkProperty: c.dfh_pk_property, language: 18889} - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => val?.label === 'Foo' - }) - - await deleteDfhApiProperty(c.pk_entity as any) - - await wait(100) - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => val?.label === 'Foo' - }) - - }) - -}); diff --git a/server/src/__tests__/acceptance/warehouse/primary-ds/EntityPathConfigService.test.ts b/server/src/__tests__/acceptance/warehouse/primary-ds/EntityPathConfigService.test.ts deleted file mode 100644 index e04469b87..000000000 --- a/server/src/__tests__/acceptance/warehouse/primary-ds/EntityPathConfigService.test.ts +++ /dev/null @@ -1,43 +0,0 @@ -// /* eslint-disable @typescript-eslint/no-explicit-any */ -// import 'reflect-metadata'; -// import {WarehouseStubs} from '../../../../warehouse/createWarehouse'; -// import {EntityPathConfigService} from '../../../../warehouse/primary-ds/EntityPathConfigService'; -// import {Warehouse} from '../../../../warehouse/Warehouse'; -// import {cleanDb} from '../../../helpers/meta/clean-db.helper'; -// import {searchUntilSatisfy, setupCleanAndStartWarehouse, stopWarehouse, truncateWarehouseTables} from '../../../helpers/warehouse-helpers'; -// const stubs: WarehouseStubs = { -// primaryDataServices: [EntityPathConfigService], -// aggDataServices: [] -// } -// describe('EntityPathConfigService', () => { - -// let wh: Warehouse; -// let s: EntityPathConfigService; - -// before(async function () { -// // eslint-disable-next-line @typescript-eslint/no-invalid-this -// this.timeout(5000); // A very long environment setup. -// const injector = await setupCleanAndStartWarehouse(stubs) -// wh = injector.get(Warehouse) -// s = injector.get(EntityPathConfigService) -// }) -// beforeEach(async () => { -// await cleanDb() -// await truncateWarehouseTables(wh) -// }) -// after(async function () { -// await stopWarehouse(wh) -// }) -// it('should have the entity path configs', async () => { -// await wh.gvPgPool.query(`select pg_notify('modified_entity_path_config', now()::text)`) -// await searchUntilSatisfy({ -// notifier$: s.afterChange$, -// getFn: () => s.index.getFromIdx({ -// pkClass: 503, // Expression portion -// }), -// compare: (val) => val?.pathConfigs?.length === 2 -// }) -// }) - -// }); - diff --git a/server/src/__tests__/acceptance/warehouse/primary-ds/PClassFieldsConfigService.test.ts b/server/src/__tests__/acceptance/warehouse/primary-ds/PClassFieldsConfigService.test.ts deleted file mode 100644 index 9676faeb9..000000000 --- a/server/src/__tests__/acceptance/warehouse/primary-ds/PClassFieldsConfigService.test.ts +++ /dev/null @@ -1,76 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import 'reflect-metadata'; -import {PClassId, ProClassFieldsConfigService} from '../../../../warehouse/primary-ds/ProClassFieldsConfigService'; -import {Warehouse} from '../../../../warehouse/Warehouse'; -import {createInfLanguage} from '../../../helpers/atomic/inf-language.helper'; -import {createProClassFieldConfig} from '../../../helpers/atomic/pro-class-field-config.helper'; -import {createProProject} from '../../../helpers/atomic/pro-project.helper'; -import {cleanDb} from '../../../helpers/meta/clean-db.helper'; -import {InfLanguageMock} from '../../../helpers/data/gvDB/InfLanguageMock'; -import {ProClassFieldConfigMock} from '../../../helpers/data/gvDB/ProClassFieldConfigMock'; -import {ProProjectMock} from '../../../helpers/data/gvDB/ProProjectMock'; -import {searchUntilSatisfy, setupCleanAndStartWarehouse, stopWarehouse, truncateWarehouseTables} from '../../../helpers/warehouse-helpers'; - -describe('PClassFieldsConfigService', () => { - - let wh: Warehouse; - let s: ProClassFieldsConfigService; - - before(async function () { - // eslint-disable-next-line @typescript-eslint/no-invalid-this - this.timeout(5000); // A very long environment setup. - const injector = await setupCleanAndStartWarehouse({ - primaryDataServices:[ProClassFieldsConfigService], - aggDataServices:[] - }) - wh = injector.get(Warehouse) - s = injector.get(ProClassFieldsConfigService) - }) - beforeEach(async () => { - await cleanDb() - await truncateWarehouseTables(wh) - }) - after(async function () { - await stopWarehouse(wh) - }) - it('should add project-class', async () => { - const {one} = await createPClassMockData(); - const id: PClassId = { - fkProject: one.fk_project ?? -1, - pkClass: one.fk_domain_class ?? -1, - } - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => !!val?.outgoing[1111] && !!val?.outgoing[1113] - }) - }) - - - - // it('should delete project-class', async () => { - // const {prel, cla} = await createPClassMockData(); - // await waitUntilNext(s.afterPut$) - - - // await updateProDfhProfileProjRel(prel.pk_entity ?? -1, {enabled: false}) - - // await waitUntilNext(s.afterDel$) - // const result = await s.index.getFromIdx({ - // fkProject: prel.fk_project ?? -1, - // pkClass: cla.dfh_pk_class ?? -1 - // }) - // expect(result).to.be.undefined() - // }) - -}); - -async function createPClassMockData() { - await createInfLanguage(InfLanguageMock.GERMAN); - await createProProject(ProProjectMock.DEFAULT_PROJECT) - const one = await createProClassFieldConfig(ProClassFieldConfigMock.PROJ_DEF_C365_NAMING_P1111_IS_APPE_OF) - const two = await createProClassFieldConfig(ProClassFieldConfigMock.PROJ_DEF_C365_NAMING_P1113_REFERS_TO_NAME) - - return {one, two}; -} - diff --git a/server/src/__tests__/acceptance/warehouse/primary-ds/PClassService.test.ts b/server/src/__tests__/acceptance/warehouse/primary-ds/PClassService.test.ts deleted file mode 100644 index d59579a7f..000000000 --- a/server/src/__tests__/acceptance/warehouse/primary-ds/PClassService.test.ts +++ /dev/null @@ -1,116 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import 'reflect-metadata'; -import {WarehouseStubs} from '../../../../warehouse/createWarehouse'; -import {PClassService} from '../../../../warehouse/primary-ds/class/PClassService'; -import {Warehouse} from '../../../../warehouse/Warehouse'; -import {createDfhApiClass} from '../../../helpers/atomic/dfh-api-class.helper'; -import {createInfLanguage} from '../../../helpers/atomic/inf-language.helper'; -import {createProDfhProfileProjRel, updateProDfhProfileProjRel} from '../../../helpers/atomic/pro-dfh-profile-proj-rel.helper'; -import {createProProject} from '../../../helpers/atomic/pro-project.helper'; -import {createSysSystemConfig} from '../../../helpers/atomic/sys-system-config.helper'; -import {DfhApiClassMock} from '../../../helpers/data/gvDB/DfhApiClassMock'; -import {InfLanguageMock} from '../../../helpers/data/gvDB/InfLanguageMock'; -import {ProDfhProfileProjRelMock} from '../../../helpers/data/gvDB/ProDfhProfileProjRelMock'; -import {ProProjectMock} from '../../../helpers/data/gvDB/ProProjectMock'; -import {cleanDb} from '../../../helpers/meta/clean-db.helper'; -import {searchUntilSatisfy, setupCleanAndStartWarehouse, stopWarehouse, truncateWarehouseTables} from '../../../helpers/warehouse-helpers'; -const stubs: WarehouseStubs = { - primaryDataServices: [PClassService], - aggDataServices: [] -} -describe('PClassService', function () { - - let wh: Warehouse; - let s: PClassService; - - before(async function () { - // eslint-disable-next-line @typescript-eslint/no-invalid-this - this.timeout(5000); // A very long environment setup. - const injector = await setupCleanAndStartWarehouse(stubs) - wh = injector.get(Warehouse) - s = injector.get(PClassService) - }) - beforeEach(async () => { - await cleanDb() - await truncateWarehouseTables(wh) - }) - after(async function () { - await stopWarehouse(wh) - }) - - it('should add project-class', async () => { - const {prel, cla} = await createPClassMockData(); - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx({ - fkProject: prel.fk_project ?? -1, - pkClass: cla.dfh_pk_class ?? -1 - }), - compare: (val) => val?.fkClass === cla.dfh_pk_class - }) - }) - - - - it('should delete project-class', async () => { - const {prel, cla} = await createPClassMockData(); - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx({ - fkProject: prel.fk_project ?? -1, - pkClass: cla.dfh_pk_class ?? -1 - }), - compare: (val) => val?.fkClass === cla.dfh_pk_class - }) - - await updateProDfhProfileProjRel(prel.pk_entity ?? -1, {enabled: false}) - - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdxWithTmsps({ - fkProject: prel.fk_project ?? -1, - pkClass: cla.dfh_pk_class ?? -1 - }), - compare: (val) => !!val?.deleted - }) - }) - it('should have class of required profile', async () => { - await createInfLanguage(InfLanguageMock.GERMAN); - const p = await createProProject(ProProjectMock.PROJECT_1); - const cla = await createDfhApiClass(DfhApiClassMock.EN_21_PERSON); - await createSysSystemConfig({classes: {}, specialFields: {}, ontome: {requiredOntomeProfiles: [4]}}); - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx({ - fkProject: p.pk_entity ?? -1, - pkClass: cla.dfh_pk_class ?? -1 - }), - compare: (val) => val?.fkClass === cla.dfh_pk_class - }) - }) - - it('should have class of required profile after making it required', async () => { - await createInfLanguage(InfLanguageMock.GERMAN); - const p = await createProProject(ProProjectMock.PROJECT_1); - const cla = await createDfhApiClass(DfhApiClassMock.EN_21_PERSON); - createSysSystemConfig({classes: {}, specialFields: {}, ontome: {requiredOntomeProfiles: [4]}}).catch(e => e); - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx({ - fkProject: p.pk_entity ?? -1, - pkClass: cla.dfh_pk_class ?? -1 - }), - compare: (val) => val?.fkClass === cla.dfh_pk_class - }) - }) - -}); - -async function createPClassMockData() { - await createInfLanguage(InfLanguageMock.GERMAN); - await createProProject(ProProjectMock.PROJECT_1); - const prel = await createProDfhProfileProjRel(ProDfhProfileProjRelMock.PROJ_1_PROFILE_4); - const cla = await createDfhApiClass(DfhApiClassMock.EN_21_PERSON); - return {prel, cla}; -} - diff --git a/server/src/__tests__/acceptance/warehouse/primary-ds/PEdgeService.test.ts b/server/src/__tests__/acceptance/warehouse/primary-ds/PEdgeService.test.ts deleted file mode 100644 index 83e4bbeb3..000000000 --- a/server/src/__tests__/acceptance/warehouse/primary-ds/PEdgeService.test.ts +++ /dev/null @@ -1,123 +0,0 @@ -/* eslint-disable @typescript-eslint/camelcase */ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import 'reflect-metadata'; -import {PEdgeService} from '../../../../warehouse/primary-ds/edge/PEdgeService'; -import {PEntityId} from '../../../../warehouse/primary-ds/entity/PEntityService'; -import {Warehouse} from '../../../../warehouse/Warehouse'; -import {createInfAppellation} from '../../../helpers/atomic/inf-appellation.helper'; -import {createInfLanguage} from '../../../helpers/atomic/inf-language.helper'; -import {createInfStatement} from '../../../helpers/atomic/inf-statement.helper'; -import {createInfResource} from '../../../helpers/atomic/inf-resource.helper'; -import {createProInfoProjRel, updateProInfoProjRel} from '../../../helpers/atomic/pro-info-proj-rel.helper'; -import {createProProject} from '../../../helpers/atomic/pro-project.helper'; -import {cleanDb} from '../../../helpers/meta/clean-db.helper'; -import {InfAppellationMock} from '../../../helpers/data/gvDB/InfAppellationMock'; -import {InfLanguageMock} from '../../../helpers/data/gvDB/InfLanguageMock'; -import {InfStatementMock} from '../../../helpers/data/gvDB/InfStatementMock'; -import {InfResourceMock} from '../../../helpers/data/gvDB/InfResourceMock'; -import {ProInfoProjRelMock} from '../../../helpers/data/gvDB/ProInfoProjRelMock'; -import {ProProjectMock} from '../../../helpers/data/gvDB/ProProjectMock'; -import {searchUntilSatisfy, setupCleanAndStartWarehouse, stopWarehouse, truncateWarehouseTables} from '../../../helpers/warehouse-helpers'; -import {WarehouseStubs} from '../../../../warehouse/createWarehouse'; -const stubs: WarehouseStubs = { - primaryDataServices:[PEdgeService], - aggDataServices:[] -} -describe('PEdgeService', () => { - - let wh: Warehouse; - let s: PEdgeService; - - before(async function () { - // eslint-disable-next-line @typescript-eslint/no-invalid-this - this.timeout(5000); // A very long environment setup. - const injector = await setupCleanAndStartWarehouse(stubs) - wh = injector.get(Warehouse) - s = injector.get(PEdgeService) - }) - beforeEach(async () => { - await cleanDb() - await truncateWarehouseTables(wh) - }) - after(async function () { - await stopWarehouse(wh) - }) - it('should have field edges in index after initIdx()', async () => { - await createInfLanguage(InfLanguageMock.GERMAN) - const project = await createProProject(ProProjectMock.PROJECT_1) - - await createInfResource(InfResourceMock.NAMING_1) - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_NAMING_1) - await createInfAppellation(InfAppellationMock.JACK_THE_FOO) - await createInfStatement(InfStatementMock.NAME_1_TO_APPE) - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_STMT_NAME_1_TO_APPE) - await createInfStatement(InfStatementMock.NAME_1_TO_PERSON) - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_STMT_NAME_1_TO_PERSON) - await createInfResource(InfResourceMock.PERSON_1) - const id: PEntityId = { - pkEntity: InfResourceMock.NAMING_1.pk_entity ?? -1, - fkProject: project.pk_entity ?? -1 - } - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => { - return val?.outgoing?.[1113]?.[0].targetLabel === InfAppellationMock.JACK_THE_FOO.string - && val?.outgoing?.[1113]?.length === 1 - && val?.outgoing?.[1111]?.[0].fkTarget === InfStatementMock.NAME_1_TO_PERSON.fk_object_info - && val?.outgoing?.[1111]?.length === 1 - } - }); - }) - - it('should update field edges if statement is removed from project', async () => { - await createInfLanguage(InfLanguageMock.GERMAN) - const project = await createProProject(ProProjectMock.PROJECT_1) - - await createInfResource(InfResourceMock.NAMING_1) - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_NAMING_1) - await createInfAppellation(InfAppellationMock.JACK_THE_FOO) - await createInfStatement(InfStatementMock.NAME_1_TO_APPE) - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_STMT_NAME_1_TO_APPE) - await createInfStatement(InfStatementMock.NAME_1_TO_PERSON) - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_STMT_NAME_1_TO_PERSON) - await createInfResource(InfResourceMock.PERSON_1) - - const id: PEntityId = { - pkEntity: InfResourceMock.NAMING_1.pk_entity ?? -1, - fkProject: project.pk_entity ?? -1 - } - - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => { - return val?.outgoing?.[1113]?.[0].targetLabel === InfAppellationMock.JACK_THE_FOO.string - && val?.outgoing?.[1113]?.length === 1 - && val?.outgoing?.[1111]?.[0].fkTarget === InfStatementMock.NAME_1_TO_PERSON.fk_object_info - && val?.outgoing?.[1111]?.length === 1 - } - }); - - await updateProInfoProjRel( - ProInfoProjRelMock.PROJ_1_STMT_NAME_1_TO_PERSON.pk_entity ?? -1, - { - ...ProInfoProjRelMock.PROJ_1_STMT_NAME_1_TO_PERSON, - is_in_project: false - } - ) - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => { - return val?.outgoing?.[1113]?.[0].targetLabel === InfAppellationMock.JACK_THE_FOO.string - && val?.outgoing?.[1113]?.length === 1 - && val?.outgoing?.[1111] === undefined - } - }); - - }) - - -}); - diff --git a/server/src/__tests__/acceptance/warehouse/primary-ds/PEntityService.test.ts b/server/src/__tests__/acceptance/warehouse/primary-ds/PEntityService.test.ts deleted file mode 100644 index cff628627..000000000 --- a/server/src/__tests__/acceptance/warehouse/primary-ds/PEntityService.test.ts +++ /dev/null @@ -1,160 +0,0 @@ -/* eslint-disable @typescript-eslint/camelcase */ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import {expect} from '@loopback/testlab'; -import 'reflect-metadata'; -import {WarehouseStubs} from '../../../../warehouse/createWarehouse'; -import {PEntityId, PEntityService} from '../../../../warehouse/primary-ds/entity/PEntityService'; -import {Warehouse} from '../../../../warehouse/Warehouse'; -import {createDfhApiClass} from '../../../helpers/atomic/dfh-api-class.helper'; -import {createInfLanguage} from '../../../helpers/atomic/inf-language.helper'; -import {createInfResource, updateInfResource} from '../../../helpers/atomic/inf-resource.helper'; -import {createProInfoProjRel, updateProInfoProjRel} from '../../../helpers/atomic/pro-info-proj-rel.helper'; -import {createProProject} from '../../../helpers/atomic/pro-project.helper'; -import {getWarEntityPreview} from '../../../helpers/atomic/war-entity-preview.helper'; -import {DfhApiClassMock} from '../../../helpers/data/gvDB/DfhApiClassMock'; -import {InfLanguageMock} from '../../../helpers/data/gvDB/InfLanguageMock'; -import {InfResourceMock} from '../../../helpers/data/gvDB/InfResourceMock'; -import {ProInfoProjRelMock} from '../../../helpers/data/gvDB/ProInfoProjRelMock'; -import {ProProjectMock} from '../../../helpers/data/gvDB/ProProjectMock'; -import {cleanDb} from '../../../helpers/meta/clean-db.helper'; -import {setupCleanAndStartWarehouse, stopWarehouse, truncateWarehouseTables, waitForEntityPreview, waitUntilNext} from '../../../helpers/warehouse-helpers'; -const stubs: WarehouseStubs = { - primaryDataServices:[PEntityService], - aggDataServices:[] -} -describe('PEntityService', () => { - - let wh: Warehouse; - let s: PEntityService; - before(async function () { - // eslint-disable-next-line @typescript-eslint/no-invalid-this - this.timeout(5000); // A very long environment setup. - const injector = await setupCleanAndStartWarehouse(stubs) - wh = injector.get(Warehouse) - s = injector.get(PEntityService) - }) - beforeEach(async () => { - await cleanDb() - await truncateWarehouseTables(wh) - }) - after(async function () { - await stopWarehouse(wh) - }) - it('should have entity in index()', async () => { - await createDfhApiClass(DfhApiClassMock.EN_21_PERSON) - const entity = await createInfResource(InfResourceMock.PERSON_1) - await createInfLanguage(InfLanguageMock.GERMAN) - const project = await createProProject(ProProjectMock.PROJECT_1) - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_PERSON_1) - - await waitUntilNext(s.afterChange$) - - const result = await s.index.getFromIdx({ - pkEntity: InfResourceMock.PERSON_1.pk_entity ?? -1, - fkProject: project.pk_entity ?? -1 - }) - expect(result?.fkClass).to.equal(entity?.fk_class) - - }) - - it('should update entity if class changed', async () => { - await createDfhApiClass(DfhApiClassMock.EN_21_PERSON) - await createDfhApiClass(DfhApiClassMock.EN_363_GEO_PLACE) - const entity = await createInfResource(InfResourceMock.PERSON_1) - await createInfLanguage(InfLanguageMock.GERMAN) - const project = await createProProject(ProProjectMock.PROJECT_1) - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_PERSON_1) - - await waitUntilNext(s.afterChange$) - - const id = { - pkEntity: InfResourceMock.PERSON_1.pk_entity ?? -1, - fkProject: project.pk_entity ?? -1 - } - const result1 = await s.index.getFromIdxWithTmsps(id) - expect(result1?.val?.fkClass).to.equal(entity?.fk_class) - await updateInfResource( - InfResourceMock.PERSON_1.pk_entity ?? -1, - { - ...InfResourceMock.PERSON_1, - fk_class: DfhApiClassMock.EN_363_GEO_PLACE.dfh_pk_class - } - ) - - await waitUntilNext(s.afterChange$) - - const result2 = await s.index.getFromIdxWithTmsps(id) - expect(result2?.val?.fkClass).to.equal(DfhApiClassMock.EN_363_GEO_PLACE.dfh_pk_class) - - expect(((result2?.lastModification ?? 0) > (result1?.lastModification ?? 0))).to.be.true() - - }) - - it('should delete entity if removed from project', async () => { - await createDfhApiClass(DfhApiClassMock.EN_21_PERSON) - const entity = await createInfResource(InfResourceMock.PERSON_1) - await createInfLanguage(InfLanguageMock.GERMAN) - const project = await createProProject(ProProjectMock.PROJECT_1) - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_PERSON_1) - - await waitUntilNext(s.afterChange$) - - const id = { - pkEntity: InfResourceMock.PERSON_1.pk_entity ?? -1, - fkProject: project.pk_entity ?? -1 - } - const result = await s.index.getFromIdxWithTmsps(id) - expect(result?.val?.fkClass).to.equal(entity?.fk_class) - expect(result?.deleted).to.be.undefined() - - let entities = await getWarEntityPreview(id.pkEntity, id.fkProject) - expect(entities.length).to.equal(1); - - await updateProInfoProjRel( - ProInfoProjRelMock.PROJ_1_PERSON_1.pk_entity ?? -1, - { - ...ProInfoProjRelMock.PROJ_1_PERSON_1, - is_in_project: false - } - ) - - await waitUntilNext(s.afterChange$) - - const result2 = await s.index.getFromIdxWithTmsps(id) - expect(result2?.deleted).not.to.be.undefined() - - entities = await getWarEntityPreview(id.pkEntity, id.fkProject) - expect(entities.length).to.equal(0); - }) - - it('should add entity preview with fk_class after entity is added', async () => { - await createDfhApiClass(DfhApiClassMock.EN_21_PERSON) - - const entities = await getWarEntityPreview( - InfResourceMock.PERSON_1.pk_entity ?? -1, - ProProjectMock.PROJECT_1.pk_entity ?? -1 - ) - expect(entities.length).to.equal(0); - - - await createInfLanguage(InfLanguageMock.GERMAN) - await createProProject(ProProjectMock.PROJECT_1) - await createInfResource(InfResourceMock.PERSON_1) - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_PERSON_1) - const id: PEntityId = { - pkEntity: InfResourceMock.PERSON_1.pk_entity ?? -1, - fkProject: ProProjectMock.PROJECT_1.pk_entity ?? -1 - } - const e = await waitForEntityPreview(wh, [ - {pk_entity: {eq: id.pkEntity}}, - {fk_project: {eq: id.fkProject}}, - {fk_class: {eq: InfResourceMock.PERSON_1.fk_class}}, - ]) - expect(e).not.to.be.undefined(); - - }) - - - -}); - diff --git a/server/src/__tests__/acceptance/warehouse/primary-ds/PFieldChangeService.test.ts b/server/src/__tests__/acceptance/warehouse/primary-ds/PFieldChangeService.test.ts deleted file mode 100644 index bee5225a7..000000000 --- a/server/src/__tests__/acceptance/warehouse/primary-ds/PFieldChangeService.test.ts +++ /dev/null @@ -1,72 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import 'reflect-metadata'; -import {WarehouseStubs} from '../../../../warehouse/createWarehouse'; -import {PFieldChangeId, PFieldChangeService} from '../../../../warehouse/primary-ds/PFieldChangeService'; -import {Warehouse} from '../../../../warehouse/Warehouse'; -import {createInfAppellation} from '../../../helpers/atomic/inf-appellation.helper'; -import {createInfLanguage} from '../../../helpers/atomic/inf-language.helper'; -import {createInfResource} from '../../../helpers/atomic/inf-resource.helper'; -import {createInfStatement} from '../../../helpers/atomic/inf-statement.helper'; -import {createProInfoProjRel} from '../../../helpers/atomic/pro-info-proj-rel.helper'; -import {createProProject} from '../../../helpers/atomic/pro-project.helper'; -import {InfAppellationMock} from '../../../helpers/data/gvDB/InfAppellationMock'; -import {InfLanguageMock} from '../../../helpers/data/gvDB/InfLanguageMock'; -import {InfResourceMock} from '../../../helpers/data/gvDB/InfResourceMock'; -import {InfStatementMock} from '../../../helpers/data/gvDB/InfStatementMock'; -import {ProInfoProjRelMock} from '../../../helpers/data/gvDB/ProInfoProjRelMock'; -import {ProProjectMock} from '../../../helpers/data/gvDB/ProProjectMock'; -import {cleanDb} from '../../../helpers/meta/clean-db.helper'; -import {searchUntilSatisfy, setupCleanAndStartWarehouse, stopWarehouse, truncateWarehouseTables} from '../../../helpers/warehouse-helpers'; -const stubs: WarehouseStubs = { - primaryDataServices: [PFieldChangeService], - aggDataServices: [] -} -describe('PFieldChangeService', () => { - - let wh: Warehouse; - let s: PFieldChangeService; - - before(async function () { - // eslint-disable-next-line @typescript-eslint/no-invalid-this - this.timeout(5000); // A very long environment setup. - const injector = await setupCleanAndStartWarehouse(stubs) - wh = injector.get(Warehouse) - s = injector.get(PFieldChangeService) - }) - beforeEach(async () => { - await cleanDb() - await truncateWarehouseTables(wh) - }) - after(async function () { - await stopWarehouse(wh) - }) - it('should have field in index', async () => { - await createInfLanguage(InfLanguageMock.GERMAN) - const project = await createProProject(ProProjectMock.PROJECT_1) - const date = new Date() - await createInfResource(InfResourceMock.NAMING_1) - await createInfAppellation(InfAppellationMock.JACK_THE_FOO) - await createInfStatement(InfStatementMock.NAME_1_TO_APPE) - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_STMT_NAME_1_TO_APPE) - - const id: PFieldChangeId = { - fkProject: project.pk_entity ?? -1, - fkSourceInfo: InfStatementMock.NAME_1_TO_APPE.fk_subject_info ?? 0, - fkSourceTablesCell: InfStatementMock.NAME_1_TO_APPE.fk_subject_tables_cell ?? 0, - fkProperty: InfStatementMock.NAME_1_TO_APPE.fk_property ?? 0, - fkPropertyOfProperty: 0, - isOutgoing: true - } - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => { - const tmsp = new Date(val?.tmspLastModification as any) - const bool = tmsp > date - return bool; - } - }); - }) - -}); - diff --git a/server/src/__tests__/acceptance/warehouse/primary-ds/PPropertyService.test.ts b/server/src/__tests__/acceptance/warehouse/primary-ds/PPropertyService.test.ts deleted file mode 100644 index 55a6f62c5..000000000 --- a/server/src/__tests__/acceptance/warehouse/primary-ds/PPropertyService.test.ts +++ /dev/null @@ -1,103 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import 'reflect-metadata'; -import {PPropertyId, PPropertyService} from '../../../../warehouse/primary-ds/property/PPropertyService'; -import {Warehouse} from '../../../../warehouse/Warehouse'; -import {createDfhApiClass} from '../../../helpers/atomic/dfh-api-class.helper'; -import {createDfhApiProperty} from '../../../helpers/atomic/dfh-api-property.helper'; -import {createInfLanguage} from '../../../helpers/atomic/inf-language.helper'; -import {createProDfhProfileProjRel, updateProDfhProfileProjRel} from '../../../helpers/atomic/pro-dfh-profile-proj-rel.helper'; -import {createProProject} from '../../../helpers/atomic/pro-project.helper'; -import {DfhApiClassMock} from '../../../helpers/data/gvDB/DfhApiClassMock'; -import {DfhApiPropertyMock} from '../../../helpers/data/gvDB/DfhApiPropertyMock'; -import {InfLanguageMock} from '../../../helpers/data/gvDB/InfLanguageMock'; -import {ProDfhProfileProjRelMock} from '../../../helpers/data/gvDB/ProDfhProfileProjRelMock'; -import {ProProjectMock} from '../../../helpers/data/gvDB/ProProjectMock'; -import {searchUntilSatisfy, setupCleanAndStartWarehouse, stopWarehouse, truncateWarehouseTables, waitUntilNext} from '../../../helpers/warehouse-helpers'; -import {WarehouseStubs} from '../../../../warehouse/createWarehouse'; -import {cleanDb} from '../../../helpers/meta/clean-db.helper'; -const stubs: WarehouseStubs = { - primaryDataServices:[PPropertyService], - aggDataServices:[] -} -describe('PPropertyService', function () { - - let wh: Warehouse; - let s: PPropertyService; - - before(async function () { - // eslint-disable-next-line @typescript-eslint/no-invalid-this - this.timeout(5000); // A very long environment setup. - const injector = await setupCleanAndStartWarehouse(stubs) - wh = injector.get(Warehouse) - s = injector.get(PPropertyService) - }) - beforeEach(async () => { - await cleanDb() - await truncateWarehouseTables(wh) - }) - after(async function () { - await stopWarehouse(wh) - }) - it('should add project-property', async () => { - const {prel, prop} = await createPPropertyMockData(); - - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx({ - fkProject: prel.fk_project ?? -1, - pkProperty: prop.dfh_pk_property ?? -1, - fkDomain: prop.dfh_property_domain ?? -1, - fkRange: prop.dfh_property_range ?? -1 - }), - compare: (val) => val?.fkProperty === prop.dfh_pk_property - }) - }) - - - - it('should delete project-property', async () => { - const {prel, prop} = await createPPropertyMockData(); - await waitUntilNext(s.afterChange$) - - await updateProDfhProfileProjRel(prel.pk_entity ?? -1, {enabled: false}) - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdxWithTmsps({ - fkProject: prel.fk_project ?? -1, - pkProperty: prop.dfh_pk_property ?? -1, - fkDomain: prop.dfh_property_domain ?? -1, - fkRange: prop.dfh_property_range ?? -1 - }), - compare: (val) => !!val?.deleted - }) - }) - - it('should add generic incoming 1111 for class Person', async () => { - await createInfLanguage(InfLanguageMock.GERMAN); - const project = await createProProject(ProProjectMock.PROJECT_1); - await createProDfhProfileProjRel(ProDfhProfileProjRelMock.PROJ_1_PROFILE_4); - const klass = await createDfhApiClass(DfhApiClassMock.EN_21_PERSON); - - const id: PPropertyId = { - fkProject: project.pk_entity ?? -1, - pkProperty: 1111, - fkDomain: 365, - fkRange: klass.dfh_pk_class - } - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => val?.fkProperty === 1111 - }) - }) - -}); - -async function createPPropertyMockData() { - await createInfLanguage(InfLanguageMock.GERMAN); - await createProProject(ProProjectMock.PROJECT_1); - const prel = await createProDfhProfileProjRel(ProDfhProfileProjRelMock.PROJ_1_PROFILE_4); - const prop = await createDfhApiProperty(DfhApiPropertyMock.EN_1110_HAS_GEO_PLACE_TYPE); - return {prel, prop}; -} - diff --git a/server/src/__tests__/acceptance/warehouse/primary-ds/PStatementService.test.ts b/server/src/__tests__/acceptance/warehouse/primary-ds/PStatementService.test.ts deleted file mode 100644 index 58945f67b..000000000 --- a/server/src/__tests__/acceptance/warehouse/primary-ds/PStatementService.test.ts +++ /dev/null @@ -1,114 +0,0 @@ -/* eslint-disable @typescript-eslint/camelcase */ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import 'reflect-metadata'; -import {WarehouseStubs} from '../../../../warehouse/createWarehouse'; -import {PStatementId, PStatementService} from '../../../../warehouse/primary-ds/statement/PStatementService'; -import {Warehouse} from '../../../../warehouse/Warehouse'; -import {createInfAppellation} from '../../../helpers/atomic/inf-appellation.helper'; -import {createInfLanguage} from '../../../helpers/atomic/inf-language.helper'; -import {createInfStatement} from '../../../helpers/atomic/inf-statement.helper'; -import {createInfResource} from '../../../helpers/atomic/inf-resource.helper'; -import {createProInfoProjRel, updateProInfoProjRel} from '../../../helpers/atomic/pro-info-proj-rel.helper'; -import {createProProject} from '../../../helpers/atomic/pro-project.helper'; -import {InfAppellationMock} from '../../../helpers/data/gvDB/InfAppellationMock'; -import {InfLanguageMock} from '../../../helpers/data/gvDB/InfLanguageMock'; -import {InfStatementMock} from '../../../helpers/data/gvDB/InfStatementMock'; -import {InfResourceMock} from '../../../helpers/data/gvDB/InfResourceMock'; -import {ProInfoProjRelMock} from '../../../helpers/data/gvDB/ProInfoProjRelMock'; -import {ProProjectMock} from '../../../helpers/data/gvDB/ProProjectMock'; -import {cleanDb} from '../../../helpers/meta/clean-db.helper'; -import {searchUntilSatisfy, setupCleanAndStartWarehouse, stopWarehouse, truncateWarehouseTables} from '../../../helpers/warehouse-helpers'; -const stubs: WarehouseStubs = { - primaryDataServices: [PStatementService], - aggDataServices: [] -} -describe('PStatementService', () => { - - let wh: Warehouse; - let s: PStatementService; - - before(async function () { - // eslint-disable-next-line @typescript-eslint/no-invalid-this - this.timeout(5000); // A very long environment setup. - const injector = await setupCleanAndStartWarehouse(stubs) - wh = injector.get(Warehouse) - s = injector.get(PStatementService) - }) - beforeEach(async () => { - await cleanDb() - await truncateWarehouseTables(wh) - }) - after(async function () { - await stopWarehouse(wh) - }) - it('should have statement in index after initIdx()', async () => { - await createInfLanguage(InfLanguageMock.GERMAN) - const project = await createProProject(ProProjectMock.PROJECT_1) - - await createInfResource(InfResourceMock.NAMING_1) - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_NAMING_1) - await createInfAppellation(InfAppellationMock.JACK_THE_FOO) - await createInfStatement(InfStatementMock.NAME_1_TO_APPE) - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_STMT_NAME_1_TO_APPE) - await createInfStatement(InfStatementMock.NAME_1_TO_PERSON) - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_STMT_NAME_1_TO_PERSON) - await createInfResource(InfResourceMock.PERSON_1) - const id: PStatementId = { - pkEntity: InfStatementMock.NAME_1_TO_PERSON.pk_entity ?? -1, - fkProject: project.pk_entity ?? -1 - } - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => { - return val?.fkProperty === InfStatementMock.NAME_1_TO_PERSON.fk_property - } - }); - }) - - it('should delete statement if statement is removed from project', async () => { - await createInfLanguage(InfLanguageMock.GERMAN) - const project = await createProProject(ProProjectMock.PROJECT_1) - - await createInfResource(InfResourceMock.NAMING_1) - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_NAMING_1) - await createInfAppellation(InfAppellationMock.JACK_THE_FOO) - await createInfStatement(InfStatementMock.NAME_1_TO_APPE) - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_STMT_NAME_1_TO_APPE) - await createInfStatement(InfStatementMock.NAME_1_TO_PERSON) - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_STMT_NAME_1_TO_PERSON) - await createInfResource(InfResourceMock.PERSON_1) - - const id: PStatementId = { - pkEntity: InfStatementMock.NAME_1_TO_PERSON.pk_entity ?? -1, - fkProject: project.pk_entity ?? -1 - } - - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => { - return val?.fkProperty === InfStatementMock.NAME_1_TO_PERSON.fk_property - } - }); - - await updateProInfoProjRel( - ProInfoProjRelMock.PROJ_1_STMT_NAME_1_TO_PERSON.pk_entity ?? -1, - { - ...ProInfoProjRelMock.PROJ_1_STMT_NAME_1_TO_PERSON, - is_in_project: false - } - ) - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdxWithTmsps(id), - compare: (item) => { - return item?.deleted !== undefined - } - }); - - }) - - -}); - diff --git a/server/src/__tests__/acceptance/warehouse/primary-ds/ProClassLabelService.test.ts b/server/src/__tests__/acceptance/warehouse/primary-ds/ProClassLabelService.test.ts deleted file mode 100644 index dad4f9229..000000000 --- a/server/src/__tests__/acceptance/warehouse/primary-ds/ProClassLabelService.test.ts +++ /dev/null @@ -1,122 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import 'reflect-metadata'; -import {expect} from '@loopback/testlab'; -import {ProClassLabelService, ProClassLabelVal} from '../../../../warehouse/primary-ds/ProClassLabelService'; -import {Warehouse} from '../../../../warehouse/Warehouse'; -import {createProject} from '../../../helpers/atomic/pro-project.helper'; -import {createProTextPropertyClassLabel, deleteProTextProperty, updateProTextProperty} from '../../../helpers/atomic/pro-text-property.helper'; -import {createTypes, createLanguages} from '../../../helpers/meta/model.helper'; -import {searchUntilSatisfy, setupCleanAndStartWarehouse, waitUntilNext, stopWarehouse, truncateWarehouseTables} from '../../../helpers/warehouse-helpers'; -import {WarehouseStubs} from '../../../../warehouse/createWarehouse'; -import {cleanDb} from '../../../helpers/meta/clean-db.helper'; -import {InfLanguageMock} from '../../../helpers/data/gvDB/InfLanguageMock'; -const stubs: WarehouseStubs = { - primaryDataServices:[ProClassLabelService], - aggDataServices:[] -} -describe('ProClassLabelService', () => { - - let wh: Warehouse; - let s: ProClassLabelService; - - before(async function () { - // eslint-disable-next-line @typescript-eslint/no-invalid-this - this.timeout(5000); // A very long environment setup. - const injector = await setupCleanAndStartWarehouse(stubs) - wh = injector.get(Warehouse) - s = injector.get(ProClassLabelService) - }) - beforeEach(async () => { - await cleanDb() - await truncateWarehouseTables(wh) - }) - after(async function () { - await stopWarehouse(wh) - }) - it('should create pro class label in db', async () => { - await createLanguages(); - await createTypes(); - const project = await createProject(18605) //German - const pkProject = project.pk_entity ?? -1; - const str = 'FooClassLabel' - const label = await createProTextPropertyClassLabel(pkProject, 12, str, 19008) //French - expect(label?.string).to.equal(str) - }) - it('should have pro class label in index', async () => { - - await createLanguages(); - await createTypes(); - const project = await createProject(18605) //German - const pkProject = project.pk_entity ?? -1; - const str = 'FooClassLabel' - const label = await createProTextPropertyClassLabel(pkProject, 12, str, 19008) //French - - await waitUntilNext(s.afterChange$) - - const result = await s.index.getFromIdx({ - fkProject: label.fk_project ?? -1, - fkLanguage: label.fk_language ?? -1, - fkClass: label.fk_dfh_class ?? -1 - }) - expect(result?.label).to.equal(str) - - }) - - it('should update pro class label in index', async () => { - await createLanguages(); - await createTypes(); - const project = await createProject(18605) //German - const pkProject = project.pk_entity ?? -1; - const str = 'FooClassLabel' - const label = await createProTextPropertyClassLabel(pkProject, 12, str, InfLanguageMock.FRENCH.pk_entity) - const id = { - fkClass: label.fk_dfh_class ?? -1, - fkLanguage: label.fk_language, - fkProject: label.fk_project - } - let result = await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val?: ProClassLabelVal) => val?.label === str - }); - - expect(result?.label).to.equal(str) - - const str2 = 'BarClassLabel' - await updateProTextProperty(label.pk_entity ?? -1, { string: str2 }) - - result = await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val?: ProClassLabelVal) => val?.label === str2 - }); - - expect(result?.label).to.equal(str2) - }) - - it('should delete pro class label in index', async () => { - await createLanguages(); - await createTypes(); - const project = await createProject(18605) //German - const pkProject = project.pk_entity ?? -1; - const str = 'FooClassLabel' - const label = await createProTextPropertyClassLabel(pkProject, 12, str, InfLanguageMock.FRENCH.pk_entity) - await waitUntilNext(s.afterChange$) - - const id = { - fkProject: label.fk_project ?? -1, - fkLanguage: label.fk_language ?? -1, - fkClass: label.fk_dfh_class ?? -1 - } - const result = await s.index.getFromIdx(id) - expect(result?.label).to.equal(str) - - await deleteProTextProperty(label.pk_entity ?? -1) - - await waitUntilNext(s.afterChange$) - const resultUpdated = await s.index.getFromIdxWithTmsps(id) - expect(resultUpdated?.deleted).not.to.be.undefined() - }) - -}); - diff --git a/server/src/__tests__/acceptance/warehouse/primary-ds/ProEntityLabelConfigService.test.ts b/server/src/__tests__/acceptance/warehouse/primary-ds/ProEntityLabelConfigService.test.ts deleted file mode 100644 index b95b876bf..000000000 --- a/server/src/__tests__/acceptance/warehouse/primary-ds/ProEntityLabelConfigService.test.ts +++ /dev/null @@ -1,119 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import 'reflect-metadata'; -import {expect} from '@loopback/testlab'; -import {clone, equals} from 'ramda'; -import {ProEntityLabelConfigService} from '../../../../warehouse/primary-ds/ProEntityLabelConfigService'; -import {Warehouse} from '../../../../warehouse/Warehouse'; -import {createInfLanguage} from '../../../helpers/atomic/inf-language.helper'; -import {createProEntityLabelConfig, deleteProEntityLabelConfig, updateProEntityLabelConfig} from '../../../helpers/atomic/pro-entity-label-config.helper'; -import {createProProject} from '../../../helpers/atomic/pro-project.helper'; -import {createTypes} from '../../../helpers/meta/model.helper'; -import {DfhApiPropertyMock} from '../../../helpers/data/gvDB/DfhApiPropertyMock'; -import {InfLanguageMock} from '../../../helpers/data/gvDB/InfLanguageMock'; -import {ProEntityLabelConfigMock} from '../../../helpers/data/gvDB/ProEntityLabelConfigMock'; -import {ProProjectMock} from '../../../helpers/data/gvDB/ProProjectMock'; -import {searchUntilSatisfy, setupCleanAndStartWarehouse, stopWarehouse, waitUntilNext, truncateWarehouseTables} from '../../../helpers/warehouse-helpers'; -import {WarehouseStubs} from '../../../../warehouse/createWarehouse'; -import {cleanDb} from '../../../helpers/meta/clean-db.helper'; -const stubs: WarehouseStubs = { - primaryDataServices:[ProEntityLabelConfigService], - aggDataServices:[] -} -describe('ProEntityLabelConfigService', () => { - - let wh: Warehouse; - let s: ProEntityLabelConfigService; - - before(async function () { - // eslint-disable-next-line @typescript-eslint/no-invalid-this - this.timeout(5000); // A very long environment setup. - const injector = await setupCleanAndStartWarehouse(stubs) - wh = injector.get(Warehouse) - s = injector.get(ProEntityLabelConfigService) - }) - beforeEach(async () => { - await cleanDb() - await truncateWarehouseTables(wh) - }) - after(async function () { - await stopWarehouse(wh) - }) - it('should create pro entity label config in db', async () => { - await createProjectMock(); - const item = await createProEntityLabelConfig(ProEntityLabelConfigMock.C633_UNION_PROJECT_DEFAULT); - - expect(item).not.to.be.undefined() - }) - it('should have pro entity label config in index', async () => { - - await createProjectMock(); - const dbItem = await createProEntityLabelConfig(ProEntityLabelConfigMock.C633_UNION_PROJECT_DEFAULT); - - const whItem = await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx({pkClass: dbItem.fk_class, fkProject: dbItem.fk_project}), - compare: (val) => equals(val, ProEntityLabelConfigMock.C633_UNION_PROJECT_DEFAULT.config) - }); - - expect(whItem).to.deepEqual(ProEntityLabelConfigMock.C633_UNION_PROJECT_DEFAULT.config) - - }) - it('should update pro entity label config in index', async () => { - - await createProjectMock(); - const dbItem = await createProEntityLabelConfig(ProEntityLabelConfigMock.C633_UNION_PROJECT_DEFAULT); - - let whItem = await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx({pkClass: dbItem.fk_class, fkProject: dbItem.fk_project}), - compare: (val) => equals(val, ProEntityLabelConfigMock.C633_UNION_PROJECT_DEFAULT.config) - }); - const modified = clone(ProEntityLabelConfigMock.C633_UNION_PROJECT_DEFAULT) - if(modified?.config) modified.config.labelParts = [{ - ordNum: 0, - field: { - fkProperty: DfhApiPropertyMock.EN_1436_HAS_PARTNER.dfh_pk_property, - isOutgoing: true, - nrOfStatementsInLabel: 1 - } - }] - await updateProEntityLabelConfig(dbItem.pk_entity, modified); - - whItem = await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx({pkClass: dbItem.fk_class, fkProject: dbItem.fk_project}), - compare: (val) => val?.labelParts?.[0].field?.nrOfStatementsInLabel === 1 - }); - - expect(whItem).not.to.be.undefined() - - }) - - it('should delete pro Property label in index', async () => { - - await createProjectMock(); - const dbItem = await createProEntityLabelConfig(ProEntityLabelConfigMock.C633_UNION_PROJECT_DEFAULT); - - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx({pkClass: dbItem.fk_class, fkProject: dbItem.fk_project}), - compare: (val) => !!val - }); - await deleteProEntityLabelConfig(dbItem.pk_entity ?? -1) - - await waitUntilNext(s.afterChange$) - const resultUpdated = await s.index.getFromIdxWithTmsps({ - fkProject: dbItem.fk_project, - pkClass: dbItem.fk_class - }) - expect(resultUpdated?.deleted).not.to.be.undefined() - }) - -}); - -async function createProjectMock() { - await createTypes(); - await createInfLanguage(InfLanguageMock.GERMAN); - await createProProject(ProProjectMock.PROJECT_1); -} - diff --git a/server/src/__tests__/acceptance/warehouse/primary-ds/ProProjectService.test.ts b/server/src/__tests__/acceptance/warehouse/primary-ds/ProProjectService.test.ts deleted file mode 100644 index 07bf0d8c8..000000000 --- a/server/src/__tests__/acceptance/warehouse/primary-ds/ProProjectService.test.ts +++ /dev/null @@ -1,76 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import 'reflect-metadata'; -import {expect} from '@loopback/testlab'; -import {ProProjectService} from '../../../../warehouse/primary-ds/ProProjectService'; -import {Warehouse} from '../../../../warehouse/Warehouse'; -import {createProject, deleteProject, updateProjectLanguage} from '../../../helpers/atomic/pro-project.helper'; -import {searchUntilSatisfy, setupCleanAndStartWarehouse, stopWarehouse, truncateWarehouseTables} from '../../../helpers/warehouse-helpers'; -import {WarehouseStubs} from '../../../../warehouse/createWarehouse'; -import {cleanDb} from '../../../helpers/meta/clean-db.helper'; -import {InfLanguageMock} from '../../../helpers/data/gvDB/InfLanguageMock'; -import {createInfLanguage} from '../../../helpers/atomic/inf-language.helper'; -const stubs: WarehouseStubs = { - primaryDataServices: [ProProjectService], - aggDataServices: [] -} -describe('ProProjectService', function () { - - let wh: Warehouse; - let s: ProProjectService; - - before(async function () { - // eslint-disable-next-line @typescript-eslint/no-invalid-this - this.timeout(5000); // A very long environment setup. - const injector = await setupCleanAndStartWarehouse(stubs) - wh = injector.get(Warehouse) - s = injector.get(ProProjectService) - }) - beforeEach(async () => { - await cleanDb() - await truncateWarehouseTables(wh) - }) - after(async function () { - await stopWarehouse(wh) - }) - - it('should have project in index', async () => { - await createInfLanguage(InfLanguageMock.GERMAN) - const project = await createProject(InfLanguageMock.GERMAN.pk_entity ?? -1) - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx({pkProject: project.pk_entity ?? -1}), - compare: (val) => val?.fkLanguage === project?.fk_language - }) - - }) - - it('should update project', async () => { - await createInfLanguage(InfLanguageMock.GERMAN) - await createInfLanguage(InfLanguageMock.ENGLISH) - const project = await createProject(InfLanguageMock.GERMAN.pk_entity ?? -1) - - const projectUpdated = await updateProjectLanguage(project.pk_entity as any, 18889) - expect(projectUpdated.fk_language).to.not.equal(project.fk_language) - - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx({pkProject: project.pk_entity ?? -1}), - compare: (val) => val?.fkLanguage !== project?.fk_language - }) - - - }) - - it('should delete project', async () => { - await createInfLanguage(InfLanguageMock.GERMAN) - const project = await createProject(InfLanguageMock.GERMAN.pk_entity ?? -1) - await deleteProject(project.pk_entity ?? -1) - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdxWithTmsps({pkProject: project.pk_entity ?? -1}), - compare: (val) => !!val?.deleted - }) - }) - -}); - diff --git a/server/src/__tests__/acceptance/warehouse/primary-ds/ProPropertyLabelService.test.ts b/server/src/__tests__/acceptance/warehouse/primary-ds/ProPropertyLabelService.test.ts deleted file mode 100644 index fbd132240..000000000 --- a/server/src/__tests__/acceptance/warehouse/primary-ds/ProPropertyLabelService.test.ts +++ /dev/null @@ -1,161 +0,0 @@ -/* eslint-disable @typescript-eslint/no-invalid-this */ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import 'reflect-metadata'; -import {expect} from '@loopback/testlab'; -import {ProPropertyLabelId, ProPropertyLabelService} from '../../../../warehouse/primary-ds/ProPropertyLabelService'; -import {Warehouse} from '../../../../warehouse/Warehouse'; -import {createInfLanguage} from '../../../helpers/atomic/inf-language.helper'; -import {createProProject} from '../../../helpers/atomic/pro-project.helper'; -import {createProTextProperty, deleteProTextProperty} from '../../../helpers/atomic/pro-text-property.helper'; -import {createTypes} from '../../../helpers/meta/model.helper'; -import {InfLanguageMock} from '../../../helpers/data/gvDB/InfLanguageMock'; -import {ProProjectMock} from '../../../helpers/data/gvDB/ProProjectMock'; -import {ProTextPropertyMock} from '../../../helpers/data/gvDB/ProTextPropertyMock'; -import {searchUntilSatisfy, setupCleanAndStartWarehouse, stopWarehouse, waitUntilNext, truncateWarehouseTables} from '../../../helpers/warehouse-helpers'; -import {WarehouseStubs} from '../../../../warehouse/createWarehouse'; -import {cleanDb} from '../../../helpers/meta/clean-db.helper'; -const stubs: WarehouseStubs = { - primaryDataServices:[ProPropertyLabelService], - aggDataServices:[] -} -describe('ProPropertyLabelService', () => { - - let wh: Warehouse; - let s: ProPropertyLabelService; - - - before(async function () { - // eslint-disable-next-line @typescript-eslint/no-invalid-this - this.timeout(5000); // A very long environment setup. - const injector = await setupCleanAndStartWarehouse(stubs) - wh = injector.get(Warehouse) - s = injector.get(ProPropertyLabelService) - }) - beforeEach(async () => { - await cleanDb() - await truncateWarehouseTables(wh) - }) - after(async function () { - await stopWarehouse(wh) - }) - - it('should create pro Property label in db', async () => { - const txtProp = await createMock(); - expect(txtProp?.string).to.equal(ProTextPropertyMock.PROJ_1_PROPERTY_BROUGHT_INTO_LIFE.string) - }) - it('should have pro Property label in index', async () => { - - const txtProp = await createMock(); - const id: ProPropertyLabelId = { - fkProject: txtProp.fk_project ?? -1, - fkClass: txtProp.fk_dfh_property_domain ?? -1, - fkProperty: txtProp.fk_dfh_property ?? -1, - isOutgoing: true, - fkLanguage: txtProp.fk_language ?? -1, - } - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => val?.label === txtProp.string - }) - }) - - it('should have pro Property label in index', async () => { - await createTypes(); - await createInfLanguage(InfLanguageMock.GERMAN); - const txtProp = await createProTextProperty(ProTextPropertyMock.PROJ_1_PROPERTY_PERSON_HAS_APPELLATION) - const id: ProPropertyLabelId = { - fkProject: txtProp.fk_project ?? -1, - fkClass: txtProp.fk_dfh_property_range ?? -1, - fkProperty: txtProp.fk_dfh_property ?? -1, - isOutgoing: false, - fkLanguage: txtProp.fk_language ?? -1, - } - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => val?.label === txtProp.string - }) - - }) - it('should have pro Property label of default project in index', async () => { - await createTypes(); - await createInfLanguage(InfLanguageMock.ENGLISH); - const txtProp = await createProTextProperty(ProTextPropertyMock.PROJ_DEF_EN_PROPERTY_PERSON_HAS_APPELLATION) - const id: ProPropertyLabelId = { - fkProject: txtProp.fk_project ?? -1, - fkClass: txtProp.fk_dfh_property_range ?? -1, - fkProperty: txtProp.fk_dfh_property ?? -1, - isOutgoing: false, - fkLanguage: txtProp.fk_language ?? -1, - } - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => val?.label === txtProp.string - }) - - }) - - - - - // it('should update pro Property label in index', async () => { - // await createLanguages(); - // await createTypes(); - // const project = await createProject('German') - // const pkProject = project.pk_entity ?? -1; - // const str = 'FooPropertyLabel' - // const label = await createProTextPropertyPropertyLabel(pkProject, 12, str, AtmLanguages.FRENCH.id) - // await waitUntilNext(s.afterPut$) - // const id = { - // fkProject: label.fk_project ?? -1, - // fkLanguage: label.fk_language ?? -1, - // fkProperty: label.fk_dfh_property ?? -1 - // } - // const result = await s.index.getFromIdx(id) - // expect(result).to.equal(str) - - // const str2 = 'BarPropertyLabel' - // await updateProTextProperty(label.pk_entity ?? -1, {string: str2}) - - // await waitUntilNext(s.afterPut$) - - // const resultUpdated = await s.index.getFromIdx(id) - // expect(resultUpdated).to.equal(str2) - // }) - - it('should delete pro Property label in index', async () => { - - const txtProp = await createMock(); - - const id: ProPropertyLabelId = { - fkProject: txtProp.fk_project ?? -1, - fkClass: txtProp.fk_dfh_property_domain ?? -1, - fkProperty: txtProp.fk_dfh_property ?? -1, - isOutgoing: true, - fkLanguage: txtProp.fk_language ?? -1, - } - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => val?.label === txtProp.string - }) - - await deleteProTextProperty(txtProp.pk_entity ?? -1) - - await waitUntilNext(s.afterChange$) - const resultUpdated = await s.index.getFromIdxWithTmsps(id) - expect(resultUpdated?.deleted).not.to.be.undefined() - }) - -}); - -async function createMock() { - await createTypes(); - await createInfLanguage(InfLanguageMock.GERMAN); - await createProProject(ProProjectMock.PROJECT_1); - const label = await createProTextProperty(ProTextPropertyMock.PROJ_1_PROPERTY_BROUGHT_INTO_LIFE); - return label; -} - diff --git a/server/src/__tests__/acceptance/warehouse/primary-ds/RClassService.test.ts b/server/src/__tests__/acceptance/warehouse/primary-ds/RClassService.test.ts deleted file mode 100644 index 800d58c49..000000000 --- a/server/src/__tests__/acceptance/warehouse/primary-ds/RClassService.test.ts +++ /dev/null @@ -1,50 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import {expect} from '@loopback/testlab'; -import {RClassService} from '../../../../warehouse/primary-ds/class/RClassService'; -import {Warehouse} from '../../../../warehouse/Warehouse'; -import {createDfhApiClass} from '../../../helpers/atomic/dfh-api-class.helper'; -import {DfhApiClassMock} from '../../../helpers/data/gvDB/DfhApiClassMock'; -import {setupCleanAndStartWarehouse, stopWarehouse, truncateWarehouseTables, waitUntilNext} from '../../../helpers/warehouse-helpers'; -import {WarehouseStubs} from '../../../../warehouse/createWarehouse'; -import {cleanDb} from '../../../helpers/meta/clean-db.helper'; -const stubs: WarehouseStubs = { - primaryDataServices:[RClassService], - aggDataServices:[] -} -describe('RClassService', () => { - - let wh: Warehouse; - let s: RClassService; - before(async function () { - // eslint-disable-next-line @typescript-eslint/no-invalid-this - this.timeout(5000); // A very long environment setup. - const injector = await setupCleanAndStartWarehouse(stubs) - wh = injector.get(Warehouse) - s = injector.get(RClassService) - }) - beforeEach(async () => { - await cleanDb() - await truncateWarehouseTables(wh) - }) - after(async function () { - await stopWarehouse(wh) - }) - it('should add api-class', async () => { - const {cla} = await createRClassMockData(); - await waitUntilNext(s.afterChange$) - const result = await s.index.getFromIdx({ - pkClass: cla.dfh_pk_class ?? -1 - }) - expect(result?.fkClass).to.equal(cla.dfh_pk_class) - - }) - - - -}); - -async function createRClassMockData() { - const cla = await createDfhApiClass(DfhApiClassMock.EN_21_PERSON); - return { cla }; -} - diff --git a/server/src/__tests__/acceptance/warehouse/primary-ds/REdgeService.test.ts b/server/src/__tests__/acceptance/warehouse/primary-ds/REdgeService.test.ts deleted file mode 100644 index 50e5acd3e..000000000 --- a/server/src/__tests__/acceptance/warehouse/primary-ds/REdgeService.test.ts +++ /dev/null @@ -1,148 +0,0 @@ -/* eslint-disable @typescript-eslint/camelcase */ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import 'reflect-metadata'; -import { WarehouseStubs } from '../../../../warehouse/createWarehouse'; -import { REdgeService } from '../../../../warehouse/primary-ds/edge/REdgeService'; -import { REntityId } from '../../../../warehouse/primary-ds/entity/REntityService'; -import { Warehouse } from '../../../../warehouse/Warehouse'; -import { createInfAppellation } from '../../../helpers/atomic/inf-appellation.helper'; -import { createInfLangString } from '../../../helpers/atomic/inf-lang-string.helper'; -import { createInfLanguage } from '../../../helpers/atomic/inf-language.helper'; -import { createInfResource } from '../../../helpers/atomic/inf-resource.helper'; -import { createInfStatement } from '../../../helpers/atomic/inf-statement.helper'; -import { createProInfoProjRel, updateProInfoProjRel } from '../../../helpers/atomic/pro-info-proj-rel.helper'; -import { createProProject } from '../../../helpers/atomic/pro-project.helper'; -import { InfAppellationMock } from '../../../helpers/data/gvDB/InfAppellationMock'; -import { InfLangStringMock } from '../../../helpers/data/gvDB/InfLangStringMock'; -import { InfLanguageMock } from '../../../helpers/data/gvDB/InfLanguageMock'; -import { InfResourceMock } from '../../../helpers/data/gvDB/InfResourceMock'; -import { InfStatementMock } from '../../../helpers/data/gvDB/InfStatementMock'; -import { ProInfoProjRelMock } from '../../../helpers/data/gvDB/ProInfoProjRelMock'; -import { ProProjectMock } from '../../../helpers/data/gvDB/ProProjectMock'; -import { cleanDb } from '../../../helpers/meta/clean-db.helper'; -import { searchUntilSatisfy, setupCleanAndStartWarehouse, stopWarehouse, truncateWarehouseTables } from '../../../helpers/warehouse-helpers'; -const stubs: WarehouseStubs = { - primaryDataServices: [REdgeService], - aggDataServices: [] -} -describe('REdgeService', () => { - - let wh: Warehouse; - let s: REdgeService; - before(async function () { - // eslint-disable-next-line @typescript-eslint/no-invalid-this - this.timeout(5000); // A very long environment setup. - const injector = await setupCleanAndStartWarehouse(stubs) - wh = injector.get(Warehouse) - s = injector.get(REdgeService) - }) - beforeEach(async () => { - await cleanDb() - await truncateWarehouseTables(wh) - }) - after(async function () { - await stopWarehouse(wh) - }) - it('should have field edges in index after initIdx()', async () => { - await createInfLanguage(InfLanguageMock.GERMAN) - await createProProject(ProProjectMock.PROJECT_1) - - await createInfResource(InfResourceMock.NAMING_1) - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_NAMING_1) - await createInfAppellation(InfAppellationMock.JACK_THE_FOO) - await createInfStatement(InfStatementMock.NAME_1_TO_APPE) - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_STMT_NAME_1_TO_APPE) - await createInfStatement(InfStatementMock.NAME_1_TO_PERSON) - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_STMT_NAME_1_TO_PERSON) - await createInfResource(InfResourceMock.PERSON_1) - const id: REntityId = { - pkEntity: InfResourceMock.NAMING_1.pk_entity ?? -1, - } - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => { - return val?.outgoing?.[1113]?.[0].targetLabel === InfAppellationMock.JACK_THE_FOO.string - && val?.outgoing?.[1113]?.length === 1 - && val?.outgoing?.[1111]?.[0].fkTarget === InfStatementMock.NAME_1_TO_PERSON.fk_object_info - && val?.outgoing?.[1111]?.length === 1 - } - }) - - }) - - it('should have short label', async () => { - await createInfLanguage(InfLanguageMock.GERMAN) - await createInfLanguage(InfLanguageMock.ENGLISH) - await createProProject(ProProjectMock.PROJECT_1) - - await createInfResource(InfResourceMock.MANIF_SINGLETON_THE_MURDERER) - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_MANIF_SINGLETON_THE_MURDERER) - await createInfLangString(InfLangStringMock.EN_SHORT_TITLE_THE_MURDERER) - await createInfStatement(InfStatementMock.MANIF_SINGLETON_HAS_SHORT_TITLE_MURDERER) - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_STMT_MANIF_SINGLETON_HAS_SHORT_TITLE_MURDERER) - const fkProperty: number = InfStatementMock.MANIF_SINGLETON_HAS_SHORT_TITLE_MURDERER.fk_property ?? -1 - const id: REntityId = { - pkEntity: InfResourceMock.MANIF_SINGLETON_THE_MURDERER.pk_entity ?? -1, - } - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => { - return val?.outgoing?.[fkProperty]?.[0].targetLabel === InfLangStringMock.EN_SHORT_TITLE_THE_MURDERER.string - && val?.outgoing?.[fkProperty]?.length === 1 - } - }) - - }) - it('should update field edges if statement is removed from project', async () => { - await createInfLanguage(InfLanguageMock.GERMAN) - await createProProject(ProProjectMock.PROJECT_1) - - await createInfResource(InfResourceMock.NAMING_1) - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_NAMING_1) - await createInfAppellation(InfAppellationMock.JACK_THE_FOO) - await createInfStatement(InfStatementMock.NAME_1_TO_APPE) - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_STMT_NAME_1_TO_APPE) - await createInfStatement(InfStatementMock.NAME_1_TO_PERSON) - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_STMT_NAME_1_TO_PERSON) - await createInfResource(InfResourceMock.PERSON_1) - - - const id: REntityId = { - pkEntity: InfResourceMock.NAMING_1.pk_entity ?? -1, - } - - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => { - return val?.outgoing?.[1113]?.[0].targetLabel === InfAppellationMock.JACK_THE_FOO.string - && val?.outgoing?.[1113]?.length === 1 - && val?.outgoing?.[1111]?.[0].fkTarget === InfStatementMock.NAME_1_TO_PERSON.fk_object_info - && val?.outgoing?.[1111]?.length === 1 - } - }) - - await updateProInfoProjRel( - ProInfoProjRelMock.PROJ_1_STMT_NAME_1_TO_PERSON.pk_entity ?? -1, - { - ...ProInfoProjRelMock.PROJ_1_STMT_NAME_1_TO_PERSON, - is_in_project: false - } - ) - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => { - return val?.outgoing?.[1113]?.[0].targetLabel === InfAppellationMock.JACK_THE_FOO.string - && val?.outgoing?.[1113]?.length === 1 - && val?.outgoing?.[1111] === undefined - } - }) - - }) - - -}); - diff --git a/server/src/__tests__/acceptance/warehouse/primary-ds/REntityService.test.ts b/server/src/__tests__/acceptance/warehouse/primary-ds/REntityService.test.ts deleted file mode 100644 index c8bf99051..000000000 --- a/server/src/__tests__/acceptance/warehouse/primary-ds/REntityService.test.ts +++ /dev/null @@ -1,172 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import {expect} from '@loopback/testlab'; -import {clone} from 'ramda'; -import 'reflect-metadata'; -import {WarehouseStubs} from '../../../../warehouse/createWarehouse'; -import {REntityService} from '../../../../warehouse/primary-ds/entity/REntityService'; -import {Warehouse} from '../../../../warehouse/Warehouse'; -import {createDfhApiClass} from '../../../helpers/atomic/dfh-api-class.helper'; -import {createInfLanguage} from '../../../helpers/atomic/inf-language.helper'; -import {createInfResource, updateInfResource} from '../../../helpers/atomic/inf-resource.helper'; -import {createProInfoProjRel, updateProInfoProjRel} from '../../../helpers/atomic/pro-info-proj-rel.helper'; -import {createProProject} from '../../../helpers/atomic/pro-project.helper'; -import {getWarEntityPreview} from '../../../helpers/atomic/war-entity-preview.helper'; -import {DfhApiClassMock} from '../../../helpers/data/gvDB/DfhApiClassMock'; -import {InfLanguageMock} from '../../../helpers/data/gvDB/InfLanguageMock'; -import {InfResourceMock} from '../../../helpers/data/gvDB/InfResourceMock'; -import {ProInfoProjRelMock} from '../../../helpers/data/gvDB/ProInfoProjRelMock'; -import {ProProjectMock} from '../../../helpers/data/gvDB/ProProjectMock'; -import {cleanDb} from '../../../helpers/meta/clean-db.helper'; -import {searchForEntityPreview, searchUntilSatisfy, setupCleanAndStartWarehouse, stopWarehouse, truncateWarehouseTables, wait, waitForEntityPreviewUntil} from '../../../helpers/warehouse-helpers'; -const stubs: WarehouseStubs = { - primaryDataServices: [REntityService], - aggDataServices: [] -} -describe('REntityService', () => { - - let wh: Warehouse; - let s: REntityService; - - before(async function () { - // eslint-disable-next-line @typescript-eslint/no-invalid-this - this.timeout(5000); // A very long environment setup. - const injector = await setupCleanAndStartWarehouse(stubs) - wh = injector.get(Warehouse) - s = injector.get(REntityService) - }) - beforeEach(async () => { - await cleanDb() - await truncateWarehouseTables(wh) - }) - after(async function () { - await stopWarehouse(wh) - }) - - - - it('should have entity preview', async () => { - const entity = await createInfResource(InfResourceMock.PERSON_1) - await createDfhApiClass(DfhApiClassMock.EN_21_PERSON) - await createInfLanguage(InfLanguageMock.GERMAN) - await createProProject(ProProjectMock.PROJECT_1) - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_PERSON_1) - - const result = await searchForEntityPreview(wh, [ - {pk_entity: {eq: entity.pk_entity}}, - {fk_project: {eq: null}}, - ]) - expect(result?.fk_class).to.equal(entity?.fk_class) - - }) - - it('should update entity if class changed', async () => { - const entity = await createInfResource(InfResourceMock.PERSON_1) - await createDfhApiClass(DfhApiClassMock.EN_21_PERSON) - await createDfhApiClass(DfhApiClassMock.EN_244_EXPRESSION_CREATION) - await createInfLanguage(InfLanguageMock.GERMAN) - await createProProject(ProProjectMock.PROJECT_1) - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_PERSON_1) - - let result = await searchForEntityPreview(wh, [ - {pk_entity: {eq: entity.pk_entity}}, - {fk_project: {eq: null}}, - ]) - expect(result?.fk_class).to.equal(entity?.fk_class) - const newFkClass = DfhApiClassMock.EN_244_EXPRESSION_CREATION.dfh_pk_class - await updateInfResource( - InfResourceMock.PERSON_1.pk_entity ?? -1, - { - ...InfResourceMock.PERSON_1, - fk_class: newFkClass - } - ) - - result = await waitForEntityPreviewUntil(wh, item => { - return item.fk_project === null - && item.pk_entity === entity.pk_entity - && item.fk_class === newFkClass - }) - expect(result?.fk_class).to.equal(newFkClass) - - }) - - it('should have correct isInProjectCount before and after removed from project', async () => { - const entity = await createInfResource(InfResourceMock.PERSON_1) - await createDfhApiClass(DfhApiClassMock.EN_21_PERSON) - await createInfLanguage(InfLanguageMock.GERMAN) - await createProProject(ProProjectMock.PROJECT_1) - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_PERSON_1) - - const id = { - pkEntity: entity.pk_entity ?? -1 - } - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => val?.isInProjectCount === 1 - }) - await updateProInfoProjRel( - entity.pk_entity ?? -1, - { - ...ProInfoProjRelMock.PROJ_1_PERSON_1, - is_in_project: false - } - ) - - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => val?.isInProjectCount === 0 - }) - }) - - - it('should produce repo entity preview if community_visibiliy.toolbox is true', async () => { - const entity = await createInfResource(InfResourceMock.PERSON_1) - await createDfhApiClass(DfhApiClassMock.EN_21_PERSON) - await createInfLanguage(InfLanguageMock.GERMAN) - await createProProject(ProProjectMock.PROJECT_1) - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_PERSON_1) - - await wait(500) - const result = await s.index.getFromIdx({pkEntity: entity.pk_entity ?? -1}) - expect(result).not.to.be.undefined() - }) - it('should not produce repo entity preview if community_visibiliy.toolbox is false', async () => { - const mock = clone(InfResourceMock.PERSON_1) - mock.community_visibility.toolbox = false; - const entity = await createInfResource(mock) - await createDfhApiClass(DfhApiClassMock.EN_21_PERSON) - await createInfLanguage(InfLanguageMock.GERMAN) - await createProProject(ProProjectMock.PROJECT_1) - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_PERSON_1) - - await wait(500) - const result = await s.index.getFromIdx({pkEntity: entity.pk_entity ?? -1}) - expect(result).to.be.undefined() - }) - - it('should remove repo entity preview when community_visibiliy.toolbox is set to false', async () => { - const mock = clone(InfResourceMock.PERSON_1) - const entity = await createInfResource(mock) - await createDfhApiClass(DfhApiClassMock.EN_21_PERSON) - await createInfLanguage(InfLanguageMock.GERMAN) - await createProProject(ProProjectMock.PROJECT_1) - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_PERSON_1) - - await wait(500) - const res = await getWarEntityPreview(entity.pk_entity ?? -1, 0) - // const result = await s.index.getFromIdx({pkEntity: entity.pk_entity ?? -1}) - expect(res.length).to.equal(1) - - mock.community_visibility.toolbox = false; - await updateInfResource(mock.pk_entity ?? -1, mock) - - await wait(500) - const res2 = await getWarEntityPreview(entity.pk_entity ?? -1, 0) - expect(res2.length).to.equal(0) - }) - - -}); - diff --git a/server/src/__tests__/acceptance/warehouse/primary-ds/RFieldChangeService.test.ts b/server/src/__tests__/acceptance/warehouse/primary-ds/RFieldChangeService.test.ts deleted file mode 100644 index 4055f94d7..000000000 --- a/server/src/__tests__/acceptance/warehouse/primary-ds/RFieldChangeService.test.ts +++ /dev/null @@ -1,72 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import 'reflect-metadata'; -import {WarehouseStubs} from '../../../../warehouse/createWarehouse'; -import {PFieldChangeId, RFieldChangeService} from '../../../../warehouse/primary-ds/RFieldChangeService'; -import {Warehouse} from '../../../../warehouse/Warehouse'; -import {createInfAppellation} from '../../../helpers/atomic/inf-appellation.helper'; -import {createInfLanguage} from '../../../helpers/atomic/inf-language.helper'; -import {createInfResource} from '../../../helpers/atomic/inf-resource.helper'; -import {createInfStatement} from '../../../helpers/atomic/inf-statement.helper'; -import {createProInfoProjRel} from '../../../helpers/atomic/pro-info-proj-rel.helper'; -import {createProProject} from '../../../helpers/atomic/pro-project.helper'; -import {InfAppellationMock} from '../../../helpers/data/gvDB/InfAppellationMock'; -import {InfLanguageMock} from '../../../helpers/data/gvDB/InfLanguageMock'; -import {InfResourceMock} from '../../../helpers/data/gvDB/InfResourceMock'; -import {InfStatementMock} from '../../../helpers/data/gvDB/InfStatementMock'; -import {ProInfoProjRelMock} from '../../../helpers/data/gvDB/ProInfoProjRelMock'; -import {ProProjectMock} from '../../../helpers/data/gvDB/ProProjectMock'; -import {cleanDb} from '../../../helpers/meta/clean-db.helper'; -import {searchUntilSatisfy, setupCleanAndStartWarehouse, stopWarehouse, truncateWarehouseTables} from '../../../helpers/warehouse-helpers'; -const stubs: WarehouseStubs = { - primaryDataServices: [RFieldChangeService], - aggDataServices: [] -} -describe('RFieldChangeService', () => { - - let wh: Warehouse; - let s: RFieldChangeService; - - before(async function () { - // eslint-disable-next-line @typescript-eslint/no-invalid-this - this.timeout(5000); // A very long environment setup. - const injector = await setupCleanAndStartWarehouse(stubs) - wh = injector.get(Warehouse) - s = injector.get(RFieldChangeService) - }) - beforeEach(async () => { - await cleanDb() - await truncateWarehouseTables(wh) - }) - after(async function () { - await stopWarehouse(wh) - }) - it('should have field in index', async () => { - await createInfLanguage(InfLanguageMock.GERMAN) - await createProProject(ProProjectMock.PROJECT_1) - const date = new Date() - await createInfResource(InfResourceMock.NAMING_1) - await createInfAppellation(InfAppellationMock.JACK_THE_FOO) - await createInfStatement(InfStatementMock.NAME_1_TO_APPE) - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_STMT_NAME_1_TO_APPE) - - const id: PFieldChangeId = { - fkProject: 0, - fkSourceInfo: InfStatementMock.NAME_1_TO_APPE.fk_subject_info ?? 0, - fkSourceTablesCell: InfStatementMock.NAME_1_TO_APPE.fk_subject_tables_cell ?? 0, - fkProperty: InfStatementMock.NAME_1_TO_APPE.fk_property ?? 0, - fkPropertyOfProperty: 0, - isOutgoing: true - } - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => { - const tmsp = new Date(val?.tmspLastModification as any) - const bool = tmsp > date - return bool; - } - }); - }) - -}); - diff --git a/server/src/__tests__/acceptance/warehouse/primary-ds/RPropertyService.test.ts b/server/src/__tests__/acceptance/warehouse/primary-ds/RPropertyService.test.ts deleted file mode 100644 index 0d2523b72..000000000 --- a/server/src/__tests__/acceptance/warehouse/primary-ds/RPropertyService.test.ts +++ /dev/null @@ -1,71 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import 'reflect-metadata'; -import {RPropertyId, RPropertyService} from '../../../../warehouse/primary-ds/property/RPropertyService'; -import {Warehouse} from '../../../../warehouse/Warehouse'; -import {createDfhApiClass} from '../../../helpers/atomic/dfh-api-class.helper'; -import {createDfhApiProperty} from '../../../helpers/atomic/dfh-api-property.helper'; -import {DfhApiClassMock} from '../../../helpers/data/gvDB/DfhApiClassMock'; -import {DfhApiPropertyMock} from '../../../helpers/data/gvDB/DfhApiPropertyMock'; -import {searchUntilSatisfy, setupCleanAndStartWarehouse, stopWarehouse, truncateWarehouseTables} from '../../../helpers/warehouse-helpers'; -import {WarehouseStubs} from '../../../../warehouse/createWarehouse'; -import {cleanDb} from '../../../helpers/meta/clean-db.helper'; - -const stubs: WarehouseStubs = { - primaryDataServices:[RPropertyService], - aggDataServices:[] -} -describe('RPropertyService', () => { - - let wh: Warehouse; - let s: RPropertyService; - - before(async function () { - // eslint-disable-next-line @typescript-eslint/no-invalid-this - this.timeout(5000); // A very long environment setup. - const injector = await setupCleanAndStartWarehouse(stubs) - wh = injector.get(Warehouse) - s = injector.get(RPropertyService) - }) - beforeEach(async () => { - await cleanDb() - await truncateWarehouseTables(wh) - }) - after(async function () { - await stopWarehouse(wh) - }) - it('should add project-property', async () => { - const {prop} = await createPPropertyMockData(); - const id: RPropertyId = { - pkProperty: prop.dfh_pk_property ?? -1, - fkDomain: prop.dfh_property_domain ?? -1, - fkRange: prop.dfh_property_range ?? -1 - } - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => val?.fkProperty === prop.dfh_pk_property - }) - }) - - - it('should add generic incoming 1111 for class Person', async () => { - const klass = await createDfhApiClass(DfhApiClassMock.EN_21_PERSON); - const id: RPropertyId = { - pkProperty: 1111, - fkDomain: 365, - fkRange: klass.dfh_pk_class - } - await searchUntilSatisfy({ - notifier$: s.afterChange$, - getFn: () => s.index.getFromIdx(id), - compare: (val) => val?.fkProperty === 1111 - }) - }) - -}); - -async function createPPropertyMockData() { - const prop = await createDfhApiProperty(DfhApiPropertyMock.EN_1110_HAS_GEO_PLACE_TYPE); - return { prop }; -} - diff --git a/server/src/__tests__/helpers/data/whDb/ClassIdMock.ts b/server/src/__tests__/helpers/data/whDb/ClassIdMock.ts deleted file mode 100644 index fb5fa1446..000000000 --- a/server/src/__tests__/helpers/data/whDb/ClassIdMock.ts +++ /dev/null @@ -1,27 +0,0 @@ -import {PClassId} from '../../../../warehouse/primary-ds/ProClassFieldsConfigService'; -import {ProjectMock} from './ProjectMock'; -import {ClassMock} from './ClassMock'; - -/** - * Mock data related to classes - */ -export class ClassIdMock { - - static readonly NAMING_ID: PClassId = { - fkProject: ProjectMock.PROJECT_DE_ID.pkProject, - pkClass: ClassMock.NAMING_ID - }; - static readonly PERSON_ID: PClassId = { - fkProject: ProjectMock.PROJECT_DE_ID.pkProject, - pkClass: ClassMock.PERSON_ID - }; - static readonly BIRTH_ID: PClassId = { - fkProject: ProjectMock.PROJECT_DE_ID.pkProject, - pkClass: ClassMock.BIRTH_ID - }; - static readonly UNION_ID: PClassId = { - fkProject: ProjectMock.PROJECT_DE_ID.pkProject, - pkClass: ClassMock.UNION_ID - }; - -} diff --git a/server/src/__tests__/helpers/data/whDb/ClassMock.ts b/server/src/__tests__/helpers/data/whDb/ClassMock.ts deleted file mode 100644 index 48af18728..000000000 --- a/server/src/__tests__/helpers/data/whDb/ClassMock.ts +++ /dev/null @@ -1,12 +0,0 @@ - -/** - * Mock data related to classes - */ -export class ClassMock { - - static readonly NAMING_ID = 365; - static readonly PERSON_ID = 21; - static readonly BIRTH_ID = 61; - static readonly UNION_ID = 66; - -} diff --git a/server/src/__tests__/helpers/data/whDb/DfhClassLabelMock.ts b/server/src/__tests__/helpers/data/whDb/DfhClassLabelMock.ts deleted file mode 100644 index f17a19cf1..000000000 --- a/server/src/__tests__/helpers/data/whDb/DfhClassLabelMock.ts +++ /dev/null @@ -1,32 +0,0 @@ -import {DfhClassLabelId} from '../../../../warehouse/primary-ds/DfhClassLabelService' - -export class DfhClassLabelMock { - - static readonly NAMING_EN_ID: DfhClassLabelId = {language: 18889, pkClass: 365} - static readonly NAMING_EN_LABEL = 'Appellation in a language'; - - static readonly NAMING_DE_ID: DfhClassLabelId = {language: 18605, pkClass: 365} - static readonly NAMING_DE_LABEL = 'Bezeichung in einer Sprache'; - - - static readonly PERSON_EN_ID: DfhClassLabelId = {language: 18889, pkClass: 21} - static readonly PERSON_EN_LABEL = 'Person'; - - static readonly PERSON_DE_ID: DfhClassLabelId = {language: 18605, pkClass: 21} - static readonly PERSON_DE_LABEL = 'Mensch'; - - static readonly BIRTH_EN_ID: DfhClassLabelId = {language: 18889, pkClass: 61} - static readonly BIRTH_EN_LABEL = 'Birth' - - static readonly BIRTH_DE_ID: DfhClassLabelId = {language: 18605, pkClass: 61} - static readonly BIRTH_DE_LABEL = 'GEBURT' - - static readonly UNION_EN_ID: DfhClassLabelId = {language: 18889, pkClass: 66} - static readonly UNION_EN_LABEL = 'Union' - - static readonly UNION_DE_ID: DfhClassLabelId = {language: 18605, pkClass: 66} - static readonly UNION_DE_LABEL = 'Paarbeziehung' - - - -} diff --git a/server/src/__tests__/helpers/data/whDb/EdgeMock.ts b/server/src/__tests__/helpers/data/whDb/EdgeMock.ts deleted file mode 100644 index 7717a458c..000000000 --- a/server/src/__tests__/helpers/data/whDb/EdgeMock.ts +++ /dev/null @@ -1,161 +0,0 @@ -/* eslint-disable @typescript-eslint/camelcase */ -import {StatementItemToIndexate} from "../../../../warehouse/primary-ds/edge/edge.commons"; -import { EntityMock } from './EntityMock'; -/** - * Mock data related to edges - */ -export class EdgeMock { - - // Out of NAME_1 - - static readonly EDGE_NAME_1_TO_APPE: StatementItemToIndexate = { - fk_project: 591, - ord_num_of_domain: 0, - ord_num_of_range: 0, - pk_statement: 1234, - fk_property: 1113, - fk_subject_info: EntityMock.NAME_1_ID.pkEntity, - subject_table: 'resource', - fk_object_info: 999, - object_table: 'appellation', - appellation: 'Jack the Foo', - language: null, - lang_string: null - }; - static readonly EDGE_NAME_1_TO_PERSON: StatementItemToIndexate = { - fk_project: 591, - ord_num_of_domain: 0, - ord_num_of_range: 0, - pk_statement: 2345, - fk_property: 1111, - fk_subject_info: EntityMock.NAME_1_ID.pkEntity, - subject_table: 'resource', - fk_object_info: EntityMock.PERS_1_ID.pkEntity, - object_table: 'resource', - appellation: null, - language: null, - lang_string: null - }; - - // Out of NAME_2 - - static readonly EDGE_NAME_2_TO_APPE: StatementItemToIndexate = { - fk_project: 591, - ord_num_of_domain: 1, - ord_num_of_range: 0, - pk_statement: 3456, - fk_property: 1113, - fk_subject_info: EntityMock.NAME_2_ID.pkEntity, - subject_table: 'resource', - fk_object_info: 999, - object_table: 'appellation', - appellation: 'Jack', - language: null, - lang_string: null - }; - - static readonly EDGE_NAME_2_TO_PERSON: StatementItemToIndexate = { - fk_project: 591, - ord_num_of_domain: 1, - ord_num_of_range: 0, - pk_statement: 4567, - fk_property: 1111, - fk_subject_info: EntityMock.NAME_2_ID.pkEntity, - subject_table: 'resource', - fk_object_info: EntityMock.PERS_1_ID.pkEntity, - object_table: 'resource', - appellation: null, - language: null, - lang_string: null - }; - static readonly EDGE_NAME_3_TO_APPE: StatementItemToIndexate = { - fk_project: 591, - ord_num_of_domain: 1, - ord_num_of_range: 0, - pk_statement: 3345, - fk_property: 1113, - fk_subject_info: EntityMock.NAME_3_ID.pkEntity, - subject_table: 'resource', - fk_object_info: 999, - object_table: 'appellation', - appellation: 'Kiddy', - language: null, - lang_string: null - } - - static readonly EDGE_NAME_3_TO_PERSON: StatementItemToIndexate = { - fk_project: 591, - ord_num_of_domain: 1, - ord_num_of_range: 0, - pk_statement: 3346, - fk_property: 1111, // is appellation of - fk_subject_info: EntityMock.NAME_3_ID.pkEntity, - subject_table: 'resource', - fk_object_info: EntityMock.PERS_3_ID.pkEntity, - object_table: 'resource', - appellation: null, - language: null, - lang_string: null - } - - static readonly EDGE_BIRTH_1_BROUGHT_INTO_LIFE: StatementItemToIndexate = { - fk_project: 591, - ord_num_of_domain: 1, - ord_num_of_range: 0, - pk_statement: 5678, - fk_property: 41, // brought into life - fk_subject_info: EntityMock.BIRTH_1.pkEntity, - subject_table: 'resource', - fk_object_info: EntityMock.PERS_3.pkEntity, - object_table: 'resource', - appellation: null, - language: null, - lang_string: null - } - static readonly EDGE_BIRTH_1_STEMS_FROM: StatementItemToIndexate = { - fk_project: 591, - ord_num_of_domain: 1, - ord_num_of_range: 2, - pk_statement: 5679, - fk_property: 42, // stems from - fk_subject_info: EntityMock.BIRTH_1.pkEntity, - subject_table: 'resource', - fk_object_info: EntityMock.UNION_1.pkEntity, - object_table: 'resource', - appellation: null, - language: null, - lang_string: null - } - - static readonly EDGE_UNION_1_HAS_PARTNER_1: StatementItemToIndexate = { - fk_project: 591, - ord_num_of_domain: 1, - ord_num_of_range: 0, - pk_statement: 6790, - fk_property: 52, // has partner - fk_subject_info: EntityMock.UNION_1.pkEntity, - subject_table: 'resource', - fk_object_info: EntityMock.PERS_1.pkEntity, - object_table: 'resource', - appellation: null, - language: null, - lang_string: null - } - - static readonly EDGE_UNION_1_HAS_PARTNER_2: StatementItemToIndexate = { - fk_project: 591, - ord_num_of_domain: 1, - ord_num_of_range: 0, - pk_statement: 6791, - fk_property: 52, // has partner - fk_subject_info: EntityMock.UNION_1.pkEntity, - subject_table: 'resource', - fk_object_info: EntityMock.PERS_2.pkEntity, - object_table: 'resource', - appellation: null, - language: null, - lang_string: null - } - - -} diff --git a/server/src/__tests__/helpers/data/whDb/EntityLabelConfigMock.ts b/server/src/__tests__/helpers/data/whDb/EntityLabelConfigMock.ts deleted file mode 100644 index 8a3d0653d..000000000 --- a/server/src/__tests__/helpers/data/whDb/EntityLabelConfigMock.ts +++ /dev/null @@ -1,103 +0,0 @@ -import {EntityLabelConfigVal} from '../../../../warehouse/primary-ds/ProEntityLabelConfigService'; -import {PClassId} from '../../../../warehouse/primary-ds/ProClassFieldsConfigService'; -import {ClassMock} from './ClassMock'; -import {ProjectMock} from './ProjectMock'; - -/** - * Mock data related to classes - */ -export class EntityLabelConfigMock { - - // Naming - - static readonly NAMING_ID: PClassId = { - fkProject: ProjectMock.PROJECT_DE_ID.pkProject, - pkClass: ClassMock.NAMING_ID - } - - static readonly NAMING_1_STMT_VAL: EntityLabelConfigVal = { - labelParts: [{ - field: { - fkProperty: 1113, - isOutgoing: true, - nrOfStatementsInLabel: 1 - }, - ordNum: 1, - }] - } - - static readonly NAMING_2_STMT_VAL: EntityLabelConfigVal = { - labelParts: [{ - field: { - fkProperty: 1113, - isOutgoing: true, - nrOfStatementsInLabel: 2 - }, - ordNum: 1, - }] - } - - // Person - - static readonly PERSON_ID: PClassId = { - fkProject: ProjectMock.PROJECT_DE_ID.pkProject, - pkClass: ClassMock.PERSON_ID - } - - static readonly PERSON_1_STMT_VAL: EntityLabelConfigVal = { - labelParts: [{ - field: { - fkProperty: 1111, - isOutgoing: false, - }, - ordNum: 1, - }] - } - static readonly PERSON_2_STMT_VAL: EntityLabelConfigVal = { - labelParts: [{ - field: { - fkProperty: 1111, - isOutgoing: false, - nrOfStatementsInLabel: 2 - }, - ordNum: 1, - }] - } - - // Birth - static readonly BIRTH_ID: PClassId = { - fkProject: ProjectMock.PROJECT_DE_ID.pkProject, - pkClass: ClassMock.BIRTH_ID - } - static readonly BIRTH_VAL: EntityLabelConfigVal = { - labelParts: [{ - field: { - fkProperty: 41, - isOutgoing: true, - nrOfStatementsInLabel: 1 - }, - ordNum: 1, - }] - } - - // Union - - static readonly UNION_ID: PClassId = { - fkProject: ProjectMock.PROJECT_DE_ID.pkProject, - pkClass: ClassMock.UNION_ID - } - - static readonly UNION_VAL: EntityLabelConfigVal = { - labelParts: [{ - field: { - fkProperty: 52, - isOutgoing: true, - nrOfStatementsInLabel: 2 - - }, - ordNum: 1, - }] - } - - -} diff --git a/server/src/__tests__/helpers/data/whDb/EntityMock.ts b/server/src/__tests__/helpers/data/whDb/EntityMock.ts deleted file mode 100644 index 31677d498..000000000 --- a/server/src/__tests__/helpers/data/whDb/EntityMock.ts +++ /dev/null @@ -1,112 +0,0 @@ -import { PEntity } from '../../../../warehouse/primary-ds/entity/PEntityService'; -import { PEntityId } from '../../../../warehouse/primary-ds/entity/PEntityService'; -/** - * Mock data related to entities - */ -export class EntityMock { - - // Naming 1 - - static readonly NAME_1: PEntity = { - entityType: 'teEn', - fkProject: 591, - pkEntity: 22, - fkClass: 365, - }; - static readonly NAME_1_ID: PEntityId = { - fkProject: EntityMock.NAME_1.fkProject, - pkEntity: EntityMock.NAME_1.pkEntity - }; - - // Naming 2 - - static readonly NAME_2: PEntity = { - entityType: 'teEn', - fkProject: 591, - pkEntity: 23, - fkClass: 365, - }; - static readonly NAME_2_ID: PEntityId = { - fkProject: EntityMock.NAME_2.fkProject, - pkEntity: EntityMock.NAME_2.pkEntity - }; - - // Naming 3 - static readonly NAME_3: PEntity = { - entityType: 'teEn', - fkProject: 591, - pkEntity: 24, - fkClass: 365, - } - static readonly NAME_3_ID: PEntityId = { - fkProject: EntityMock.NAME_3.fkProject, - pkEntity: EntityMock.NAME_3.pkEntity - } - - - // Person 1 - - static readonly PERS_1: PEntity = { - entityType: 'peIt', - fkProject: 591, - pkEntity: 33, - fkClass: 21, - }; - static readonly PERS_1_ID: PEntityId = { - fkProject: EntityMock.PERS_1.fkProject, - pkEntity: EntityMock.PERS_1.pkEntity - }; - - - // Person 2 - static readonly PERS_2: PEntity = { - entityType: 'peIt', - fkProject: 591, - pkEntity: 34, - fkClass: 21, - } - static readonly PERS_2_ID: PEntityId = { - fkProject: EntityMock.PERS_2.fkProject, - pkEntity: EntityMock.PERS_2.pkEntity - } - - - // Person 3 - static readonly PERS_3: PEntity = { - entityType: 'peIt', - fkProject: 591, - pkEntity: 35, - fkClass: 21, - } - static readonly PERS_3_ID: PEntityId = { - fkProject: EntityMock.PERS_3.fkProject, - pkEntity: EntityMock.PERS_3.pkEntity - } - - - // Birth 1 - static readonly BIRTH_1: PEntity = { - entityType: 'teEn', - fkProject: 591, - pkEntity: 41, - fkClass: 61, - } - static readonly BIRTH_1_ID: PEntityId = { - fkProject: EntityMock.BIRTH_1.fkProject, - pkEntity: EntityMock.BIRTH_1.pkEntity - } - - - // UNION 1 - static readonly UNION_1: PEntity = { - entityType: 'teEn', - fkProject: 591, - pkEntity: 51, - fkClass: 66, // ?? - } - static readonly UNION_1_ID: PEntityId = { - fkProject: EntityMock.UNION_1.fkProject, - pkEntity: EntityMock.UNION_1.pkEntity - } - -} diff --git a/server/src/__tests__/helpers/data/whDb/ProClassLabelMock.ts b/server/src/__tests__/helpers/data/whDb/ProClassLabelMock.ts deleted file mode 100644 index ff2908097..000000000 --- a/server/src/__tests__/helpers/data/whDb/ProClassLabelMock.ts +++ /dev/null @@ -1,36 +0,0 @@ -import {ProClassLabelId} from '../../../../warehouse/primary-ds/ProClassLabelService' -import {PK_DEFAULT_CONFIG_PROJECT} from '../../../../warehouse/Warehouse'; -import {ProjectMock} from './ProjectMock'; - -export class ProClassLabelMock { - - static readonly GV_NAMING_EN_ID: ProClassLabelId = { - fkProject: PK_DEFAULT_CONFIG_PROJECT, - fkLanguage: 18889, // en - fkClass: 365 - } - static readonly GV_NAMING_EN_LABEL = 'Naming'; - - static readonly GV_NAMING_DE_ID: ProClassLabelId = { - fkProject: PK_DEFAULT_CONFIG_PROJECT, - fkLanguage: 18605, // de - fkClass: 365 - } - static readonly GV_NAMING_DE_LABEL = 'Benennung'; - - - static readonly PROJECT_DE_NAMING_EN_ID: ProClassLabelId = { - fkProject: ProjectMock.PROJECT_DE_ID.pkProject, - fkLanguage: 18889, // en - fkClass: 365 - } - static readonly PROJECT_DE_NAMING_EN_LABEL = 'Name giving (so do I call it)'; - - static readonly PROJECT_DE_NAMING_DE_ID: ProClassLabelId = { - fkProject: ProjectMock.PROJECT_DE_ID.pkProject, - fkLanguage: 18605, // de - fkClass: 365 - } - static readonly PROJECT_DE_NAMING_DE_LABEL = 'Namensgebung'; - -} diff --git a/server/src/__tests__/helpers/data/whDb/ProjectMock.ts b/server/src/__tests__/helpers/data/whDb/ProjectMock.ts deleted file mode 100644 index 5db6db8c5..000000000 --- a/server/src/__tests__/helpers/data/whDb/ProjectMock.ts +++ /dev/null @@ -1,12 +0,0 @@ -import {ProjectId, ProjectVal} from '../../../../warehouse/primary-ds/ProProjectService' - -export class ProjectMock { - - static readonly PROJECT_DE_ID: ProjectId = { - pkProject: 591 - } - static readonly PROJECT_DE_VAL: ProjectVal = { - fkLanguage: 18605 - }; - -} diff --git a/server/src/__tests__/helpers/warehouse-helpers.ts b/server/src/__tests__/helpers/warehouse-helpers.ts deleted file mode 100644 index 193232bdc..000000000 --- a/server/src/__tests__/helpers/warehouse-helpers.ts +++ /dev/null @@ -1,232 +0,0 @@ -/* eslint-disable @typescript-eslint/camelcase */ -import {Where} from '@loopback/filter'; -import 'reflect-metadata'; -import {BehaviorSubject, merge, Observable} from 'rxjs'; -import {filter, first, switchMap, startWith} from 'rxjs/operators'; -import {WarClassPreview, WarEntityPreviewWithFulltext} from '../../models'; -import {createWarehouse, WarehouseStubs} from '../../warehouse/createWarehouse'; -import {getWarehouseConfig} from '../../warehouse/startScripts'; -import {Warehouse} from '../../warehouse/Warehouse'; -import {createWarClassPreviewRepo} from './atomic/war-class-preview.helper'; -import {createWarEntityPreviewRepo} from './atomic/war-entity-preview.helper'; - - - - -export async function setupCleanAndStartWarehouse(stubs?: WarehouseStubs) { - const config = getWarehouseConfig() - const injector = createWarehouse(config, stubs) - const wh: Warehouse = injector.get(Warehouse) - await wh.whPgPool.query(`drop schema if exists ${wh.schemaName} cascade;`) - await wh.start() - await wh.gvPgListener.query('LISTEN entity_previews_updated;') - await wh.gvPgListener.query('LISTEN modified_war_class_preview;') - return injector; -} - - -export async function truncateWarehouseTables(wh: Warehouse) { - await wh.whPgPool.query(` - CREATE OR REPLACE FUNCTION truncate_tables(_schemaname text) - RETURNS void AS - $func$ - BEGIN - -- RAISE NOTICE '%', - EXECUTE -- dangerous, test before you execute! - (SELECT 'TRUNCATE TABLE ' - || string_agg(quote_ident(schemaname) || '.' || quote_ident(tablename), ', ') - || ' CASCADE' - FROM pg_tables - WHERE schemaname = _schemaname - ); - END - $func$ LANGUAGE plpgsql; - - select truncate_tables('${wh.schemaName}') - `) -} - -export async function stopWarehouse(wh: Warehouse) { - // await wait(1000) - await wh.whPgPool.query('VACUUM ANALYZE') - await wh.stop() -} -/** - * Returns a Promise that resolves after given miliseconds - * @param ms - */ -export async function wait(ms: number):Promise { - return new Promise(res => {setTimeout(() => res(), ms)}) -} - -/** - * Takes an observable and returns a Promis that resolves as soon as - * the observable nexts the first time. - * @param observable$ - */ -export async function waitUntilNext(observable$: Observable) { - return new Promise((res) => {observable$.pipe(first()).subscribe((i) => {res(i)})}) -} - -/** - * Returns a Promis that resolves as soon as the database emits a entity preview - * that machtches the given whereFilter. - * @param observable$ - */ -export function waitForEntityPreview(wh: Warehouse, whereFilter: Where[]) { - return new Promise((res, rej) => { - const sub = wh.gvPgNotifications$.subscribe((msg) => { - if (msg.channel === 'entity_previews_updated') { - - createWarEntityPreviewRepo().find({ - where: { - and: [ - {tmsp_last_modification: {gte: msg.payload}}, - ...whereFilter - ] - } - }) - .then((result) => { - if (result?.length === 1) { - // console.log('OK! ', msg.payload) - res(result[0]) - sub.unsubscribe() - } else if (result.length > 1) { - rej('found too many entity peviews') - } - }) - .catch(e => rej(e)) - } - }) - }) -} - - -/** - * Returns a Promise that resolves as soon as the database emits a entity preview - * that machtches the given whereFilter. - * @param observable$ - */ -export function waitForEntityPreviewUntil(wh: Warehouse, compare: (item: WarEntityPreviewWithFulltext) => boolean) { - return new Promise((res, rej) => { - const sub = wh.gvPgNotifications$.subscribe((msg) => { - if (msg.channel === 'entity_previews_updated') { - - - createWarEntityPreviewRepo().find({ - where: { - and: [ - {tmsp_last_modification: {gte: msg.payload}} - ] - } - }) - .then((result) => { - - if (result?.length) { - result.forEach(i => { - if (compare(i)) { - res(result[0]) - sub.unsubscribe() - } - }) - } - }) - .catch(e => rej(e)) - } - }) - }) -} - -/** - * Returns a Promis that resolves as soon as it finds a entity preview - * matching the given filter. It begins to search emmediately and - * repeats the search every time the gvDB emits a entity preview - * @param observable$ - */ -export function searchForEntityPreview(wh: Warehouse, whereFilter: Where[]) { - return new Promise((res, rej) => { - const sub = wh.gvPgNotifications$ - .pipe(startWith({channel: 'entity_previews_updated'})) - .subscribe((msg) => { - if (msg.channel === 'entity_previews_updated') { - - createWarEntityPreviewRepo().find({ - where: { - and: [ - ...whereFilter - ] - } - }) - .then((result) => { - if (result?.length === 1) { - // console.log('OK! ', msg.payload) - res(result[0]) - sub.unsubscribe() - } else if (result.length > 1) { - rej('found too many entity peviews') - } - }) - .catch(e => rej(e)) - } - }) - }) -} - - -/** - * Returns a Promis that resolves as soon as the database emits a entity preview - * that machtches the given whereFilter. - * @param observable$ - */ -export async function waitForClassPreview(wh: Warehouse, whereFilter: Where[]) { - return new Promise((res, rej) => { - const sub = wh.gvPgNotifications$.subscribe((msg) => { - if (msg.channel === 'modified_war_class_preview') { - createWarClassPreviewRepo().find({ - where: { - and: [ - {tmsp_last_modification: {eq: msg.payload}}, - ...whereFilter - ] - } - }) - .then((result) => { - if (result?.length === 1) { - res(result[0]) - sub.unsubscribe() - } else if (result.length > 1) { - rej('found too many entity peviews') - } - }) - .catch(e => rej(e)) - } - }) - }) -} - - -export function waitUntilSatisfy(obs$: Observable, compare: (item: M) => boolean) { - return new Promise((res, rej) => { - const sub = obs$.pipe(filter(item => compare(item))).subscribe((x) => { - res(x) - sub.unsubscribe() - }) - }) -} - -export async function searchUntilSatisfy(options: { - notifier$: Observable, - getFn: () => Promise, - compare: (val?: M) => boolean -}) { - const x$ = new BehaviorSubject(undefined); - const item$ = merge(x$, options.notifier$) - .pipe( - switchMap(_ => options.getFn()), - ); - const result = await waitUntilSatisfy(item$, options.compare); - return result; -} - - - diff --git a/server/src/__tests__/unit/warehouse/base/DepencencyIndex.test.ts b/server/src/__tests__/unit/warehouse/base/DepencencyIndex.test.ts deleted file mode 100644 index afd61551f..000000000 --- a/server/src/__tests__/unit/warehouse/base/DepencencyIndex.test.ts +++ /dev/null @@ -1,89 +0,0 @@ -// import assert from 'assert'; -// import pgkDir from 'pkg-dir'; -// import {EntityLabelVal} from '../../../../warehouse/aggregator-ds/entity-label/entity-label.commons'; -// import {PEntityLabelService} from '../../../../warehouse/aggregator-ds/entity-label/p-entity-label/PEntityLabelService'; -// import {DependencyIndex} from '../../../../warehouse/base/classes/DependencyIndex'; -// import {pEntityIdToString, stringToPEntityId} from '../../../../warehouse/base/functions'; -// import {PEntityId} from '../../../../warehouse/primary-ds/entity/PEntityService'; -// import {Warehouse, WarehouseConfig} from '../../../../warehouse/Warehouse'; - -// const config: WarehouseConfig = { -// backups: undefined -// } - -// describe('DependencyIndex', function () { -// let idx: DependencyIndex -// let wh: Warehouse -// let stub1: PEntityLabelService -// let stub2: PEntityLabelService -// let receiver1: PEntityId -// let provider1: PEntityId -// let provider2: PEntityId -// let receiver2: PEntityId -// before(async () => { - -// wh = new Warehouse(config) -// stub1 = new PEntityLabelService(wh) -// stub2 = new PEntityLabelService(wh) - - -// receiver1 = {fkProject: 1, pkEntity: 100} -// provider1 = {fkProject: 1, pkEntity: 500} -// provider2 = {fkProject: 1, pkEntity: 501} - -// receiver2 = {fkProject: 1, pkEntity: 101} -// idx = new DependencyIndex( -// wh, -// stub1, -// stub2, -// pEntityIdToString, -// stringToPEntityId, -// pEntityIdToString, -// stringToPEntityId, -// ) - -// }) -// beforeEach(async function () { -// await idx.clearAll() -// await idx.addProvider(receiver1, provider1) -// await idx.addProvider(receiver1, provider2) -// await idx.addProvider(receiver2, provider2) -// }) -// it('should getProviders', async () => { -// const result = await idx.getProviders(receiver1) -// assert.deepStrictEqual(result, [provider1, provider2]) -// }) - -// it('should getProviderMap', async () => { -// const result = await idx.getProviderMap(receiver1) -// assert.deepStrictEqual(result, { -// [pEntityIdToString(provider1)]: true, -// [pEntityIdToString(provider2)]: true, -// }) -// }) -// it('should getReceivers', async () => { -// let result = await idx.getReceivers(provider1) -// assert.deepStrictEqual(result, [receiver1]) -// result = await idx.getReceivers(provider2) -// assert.deepStrictEqual(result, [receiver1, receiver2]) -// }) - -// it('should removeProviderByString', async () => { -// await idx.removeProviderByString(pEntityIdToString(receiver1), pEntityIdToString(provider1)) -// const result = await idx.getProviders(receiver1) -// assert.deepStrictEqual(result, [provider2]) -// }) - -// it('should removeProvider', async () => { -// await idx.removeProvider(receiver1, provider1) -// const result = await idx.getProviders(receiver1) -// assert.deepStrictEqual(result, [provider2]) -// }) - -// it('should removeAllProviders', async () => { -// await idx.removeAllProviders(receiver1) -// const result = await idx.getProviders(receiver1) -// assert.deepStrictEqual(result, []) -// }) - -// }) diff --git a/server/src/__tests__/unit/warehouse/base/IndexDB.test.ts b/server/src/__tests__/unit/warehouse/base/IndexDB.test.ts deleted file mode 100644 index 5f8fff32c..000000000 --- a/server/src/__tests__/unit/warehouse/base/IndexDB.test.ts +++ /dev/null @@ -1,162 +0,0 @@ -/* eslint-disable @typescript-eslint/no-invalid-this */ -import assert from 'assert'; -import {omit} from 'ramda'; -import {IndexDB} from '../../../../warehouse/base/classes/IndexDB'; -import {DfhOutgoingPropertyService} from '../../../../warehouse/primary-ds/DfhOutgoingPropertyService'; -import {Warehouse} from '../../../../warehouse/Warehouse'; -import {setupCleanAndStartWarehouse, waitUntilNext} from '../../../helpers/warehouse-helpers'; - - -// eslint-disable-next-line @typescript-eslint/no-explicit-any -export class TestIdx extends IndexDB { - keyToString(key: string) {return key;} - stringToKey(key: string) {return key;} -} - - -describe('IndexDB', function () { - let idx: TestIdx - let wh: Warehouse - - before(async function() { - this.timeout(50000); // A very long environment setup. - - const injector = await setupCleanAndStartWarehouse({ - primaryDataServices: [DfhOutgoingPropertyService], - aggDataServices: [] - }) - wh = injector.get(Warehouse) - idx = new TestIdx('test_idx', wh) - wh.createSchema$.next(); - wh.gvPgListenerConnected$.next(wh.gvPgListener) - await waitUntilNext(idx.ready$) - - }) - beforeEach(async function () { - await idx.clearIdx() - }) - it('should add and get key-boolean pair', async () => { - await idx.addToIdx('aaa', true) - const result = await idx.getFromIdx('aaa') - assert.deepStrictEqual(result, true) - }) - it('should add and get key-string pair', async () => { - const val = 'Hello World' - await idx.addToIdx('aaa', val) - const result = await idx.getFromIdx('aaa') - assert.deepStrictEqual(result, val) - }) - it('should add and get key-number pair', async () => { - const val = 123123123 - await idx.addToIdx('aaa', val) - const result = await idx.getFromIdx('aaa') - assert.deepStrictEqual(result, val) - }) - it('should add and get key-undefined pair', async () => { - const val = undefined - await idx.addToIdx('aaa', val) - const result = await idx.getFromIdx('aaa') - assert.deepStrictEqual(result, null) - }) - it('should add and get key-object pair', async () => { - const val = {foo: 123, bar: true, baz: undefined, x: null, y: `Hanse's ="" \n asdb { - const val = ['foo', 'foo2', 'foo3', "asdä!'a>dasdasd"] - await idx.addToIdx('aaa', val) - const result = await idx.getFromIdx('aaa') - assert.deepStrictEqual(result, val) - }) - - it('should add and remove key', async () => { - const val = 'Hello World' - await idx.addToIdx('aaa', val) - let result = await idx.getFromIdx('aaa') - assert.deepStrictEqual(result, val) - await idx.removeFromIdx('aaa') - result = await idx.getFromIdx('aaa') - assert.deepStrictEqual(result, undefined) - }) - - - it('should add and overwrite key-val pair', async () => { - let val = 'Hello World' - await idx.addToIdx('aaa', val) - let result = await idx.getFromIdx('aaa') - assert.deepStrictEqual(result, val) - val = 'Hallo Welt' - await idx.addToIdx('aaa', val) - result = await idx.getFromIdx('aaa') - assert.deepStrictEqual(result, val) - }) - - it('should iterate over keys', async () => { - - await idx.addToIdx('aaa', true) - await idx.addToIdx('aav', true) - const keys: string[] = [] - await idx.forEachKey(async (k) => { - keys.push(k) - return '' - }) - assert.deepStrictEqual(keys, ['aaa', 'aav']) - }) - - - it('should iterate over keys starting with', async () => { - - await idx.addToIdx('aaa', true) - await idx.addToIdx('xyz', true) - await idx.addToIdx('aav', true) - const keys: string[] = [] - await idx.forEachKeyStartingWith('aa', async (k) => { - keys.push(k) - return '' - }) - assert.deepStrictEqual(keys, ['aaa', 'aav']) - }) - - - it('should iterate over items starting with', async () => { - - await idx.addToIdx('aaa', 'Apple') - await idx.addToIdx('xyz', 'Pomme') - await idx.addToIdx('aav', 'Apfel') - const items: { - key: string; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - value: any; - }[] = [] - await idx.forEachItemStartingWith('aa', async (k) => { - items.push(k) - return '' - }) - assert.deepStrictEqual(items, [{key: 'aaa', value: 'Apple'}, {key: 'aav', value: 'Apfel'}]) - }) - - it('should say key does not exist', async () => { - - const result = await idx.keyExists('a') - assert.deepStrictEqual(result, false) - }) - - it('should say key does exist', async () => { - await idx.addToIdx('a', 'Apfel') - const result = await idx.keyExists('a') - assert.deepStrictEqual(result, true) - }) - - - it('should get length of keys', async () => { - await idx.addToIdx('a', 'Apfel') - await idx.addToIdx('ab', 'Pomme') - const result = await idx.getLength() - assert.deepStrictEqual(result, 2) - }) - - -}) diff --git a/server/src/__tests__/unit/warehouse/base/PgDataReplicator.test.ts b/server/src/__tests__/unit/warehouse/base/PgDataReplicator.test.ts deleted file mode 100644 index aa0d4d76c..000000000 --- a/server/src/__tests__/unit/warehouse/base/PgDataReplicator.test.ts +++ /dev/null @@ -1,178 +0,0 @@ -/* eslint-disable @typescript-eslint/no-invalid-this */ -import {Pool, PoolClient} from 'pg'; -import {createPoolConfig, getGvDatabaseUrl, getWhDatabaseUrl} from '../../../../utils/databaseUrl'; -import {expect} from '@loopback/testlab'; -import {PgDataReplicator} from '../../../../warehouse/base/classes/PgDataReplicator'; -import {sum} from 'ramda'; - - -describe('PgDataReplicator', function () { - let pool1: PoolClient - let pool2: PoolClient - - beforeEach(async function () { - pool1 = await new Pool(createPoolConfig(getGvDatabaseUrl())).connect() - pool2 = await new Pool(createPoolConfig(getWhDatabaseUrl())).connect() - - const beginSql = `BEGIN;` - await pool1.query(beginSql) - await pool2.query(beginSql) - - const createTestTable = `CREATE TEMP TABLE test_table ( - id integer, - name text - ) ON COMMIT DROP;` - await pool1.query(createTestTable) - await pool2.query(createTestTable) - - - }) - - afterEach(async function () { - await Promise.all([pool1.release(), pool2.release()]) - }) - - it('should have records in first table', async () => { - - const rows = 100 - await createData(pool1, rows); - const res = await pool1.query<{count: number}>('SELECT count(id)::int FROM test_table;') - - expect(res?.rows?.[0].count).to.equal(rows) - }) - - describe('replicate 100', function () { - const count = 100 - beforeEach(async function () { - await createData(pool1, count) - }) - it('should replicate table from db1 to db2', async () => { - - await new PgDataReplicator( - {client: pool1, table: 'test_table'}, - {client: pool2, table: 'test_table'} - ).replicateTable() - - const res = await pool2.query<{count: number}>('SELECT count(id)::int FROM test_table;') - expect(res?.rows?.[0].count).to.equal(count) - }) - }) - - - describe('replicate 100000', function () { - this.timeout(10000) - const count = 100000 - beforeEach(async function () { - await createData(pool1, count) - }) - it('should replicate table from db1 to db2 in 10 batches', async () => { - - const stats = await new PgDataReplicator( - {client: pool1, table: 'test_table'}, - {client: pool2, table: 'test_table'} - ).replicateTable() - - const res = await pool2.query<{count: number}>('SELECT count(id)::int FROM test_table;') - expect(res?.rows?.[0].count).to.equal(count) - expect(stats.length).to.equal(10) - - }) - it('should replicate table from db1 to db2 in 1 batch', async () => { - - const stats = await new PgDataReplicator( - {client: pool1, table: 'test_table'}, - {client: pool2, table: 'test_table'} - ).replicateTable(100000) - - const res = await pool2.query<{count: number}>('SELECT count(id)::int FROM test_table;') - expect(res?.rows?.[0].count).to.equal(count) - expect(stats.length).to.equal(1) - - }) - it('should replicate table from db1 to db2 in 100 batches', async () => { - - const stats = await new PgDataReplicator( - {client: pool1, table: 'test_table'}, - {client: pool2, table: 'test_table'} - ).replicateTable(1000) - - const res = await pool2.query<{count: number}>('SELECT count(id)::int FROM test_table;') - expect(res?.rows?.[0].count).to.equal(count) - expect(stats.length).to.equal(100) - - }) - }) - - describe('replicate 500000', function () { - const count = 500000 - beforeEach(async function () { - await createData(pool1, count) - }) - it('should replicate table from db1 to db2 in 50 batches', async () => { - this.timeout(10000) - - const stats = await new PgDataReplicator( - {client: pool1, table: 'test_table'}, - {client: pool2, table: 'test_table'} - ).replicateTable() - - const res = await pool2.query<{count: number}>('SELECT count(id)::int FROM test_table;') - expect(res?.rows?.[0].count).to.equal(count) - expect(stats.length).to.equal(50) - }) - }) - - describe('replicate with upsert statement counting upserts', function () { - const count1 = 1000 - const count2 = 1200 - beforeEach(async function () { - await createData(pool1, count1) - await createData(pool1, count2) - }) - it('should replicate table from db1 to db2 having correct nr of upserts', async () => { - await pool2.query(` - ALTER TABLE test_table ADD UNIQUE (id); - `) - const batchSize = 100 - const stats = await new PgDataReplicator<{count: number}>( - {client: pool1, table: 'test_table'}, - {client: pool2, table: 'test_table'}, - ['id', 'name'], - (insertStmt, fromStmt) => ` - WITH tw1 AS ( - ${insertStmt} - ${fromStmt} - ON CONFLICT (id) DO UPDATE - SET name = EXCLUDED.name - WHERE test_table.name <> EXCLUDED.name - RETURNING * - ) - SELECT count(*)::int FROM tw1 - ` - ).replicateTable(batchSize) - - expect(stats.length).to.equal((count1 + count2) / batchSize) - expect(sum(stats.map(s => s.rows?.[0].count))).to.equal(count2) - }) - }) - - - -}) - - - - -async function createData(pool1: PoolClient, rows = 100) { - const column1: number[] = []; - const column2: string[] = []; - for (let i = 0; i < rows; i++) { - column1.push(i); - column2.push('test string'); - } - await pool1.query(`INSERT INTO test_table (id, name) - SELECT * FROM UNNEST ($1::int[], $2::text[]) AS t`, [column1, column2]); - - await pool1.query(`ANALYZE test_table`); -} - diff --git a/server/src/__tests__/unit/warehouse/base/skipWhileFirst.test.ts b/server/src/__tests__/unit/warehouse/base/skipWhileFirst.test.ts deleted file mode 100644 index 8c3f86dea..000000000 --- a/server/src/__tests__/unit/warehouse/base/skipWhileFirst.test.ts +++ /dev/null @@ -1,37 +0,0 @@ -import {marbles} from "rxjs-marbles/mocha"; -import {skipWhileFirst} from '../../../../warehouse/base/functions'; - -describe('skipWhileFirst', function () { - - - it('should fullfill marble test with hot', marbles(m => { - const source = m.hot("--^-a-b-cd----e-f-g-h---------ijj--------|"); - const subs = " ^--------------------------------------!"; - const expected = " ----------a---------e---------------i--|"; - - const destination = source.pipe( - // tap(_ => console.log('t1', new Date().toISOString())), - skipWhileFirst(8), - // tap(_ => console.log('t2', new Date().toISOString())), - - ); - m.expect(destination).toBeObservable(expected); - m.expect(source).toHaveSubscriptions(subs); - })) - - // it('should fullfill marble test with cold', marbles(m => { - // const source = m.cold("---a-b-cd----e-f-g-h---------ijj--------|"); - // const expected = " ----------a---------e---------------i--|"; - - // const destination = source.pipe( - // // tap(_ => console.log('t1', new Date().toISOString())), - // skipWhileFirst(8), - // // tap(_ => console.log('t2', new Date().toISOString())), - - // ); - // m.expect(destination).toBeObservable(expected); - // })) - - -}) - diff --git a/server/src/__tests__/unit/warehouse/base/startScript.test.ts b/server/src/__tests__/unit/warehouse/base/startScript.test.ts deleted file mode 100644 index 37bf46bcf..000000000 --- a/server/src/__tests__/unit/warehouse/base/startScript.test.ts +++ /dev/null @@ -1,59 +0,0 @@ -/* eslint-disable @typescript-eslint/camelcase */ -import 'reflect-metadata'; -import {expect} from '@loopback/testlab'; -import {writeFileSync} from 'fs'; -import path from 'path'; -import {start} from '../../../../warehouse/startScripts'; - - - -describe('Warehouse Start Script', function () { - - - it('should create schema war_cache_commit08', async () => { - - createCompatList(['commit10', 'commit09', 'commit08']) - - const warehouse = await start(); - const res = await warehouse.whPgPool.query<{schema: string}>(` - SELECT schema_name AS schema - FROM information_schema.schemata - `) - expect(res.rows.some(x => x.schema === 'war_cache_commit08')).to.be.true() - expect(res.rows.filter(x => x.schema.startsWith('war_cache_')).length).to.equal(1) - }) - - it('should create schema war_cache_commit08 then war_cache_commit09 and delete war_cache_commit08', async () => { - - createCompatList(['commit10', 'commit09', 'commit08']) - - let warehouse = await start(); - let res = await warehouse.whPgPool.query<{schema: string}>(` - SELECT schema_name AS schema - FROM information_schema.schemata - `) - expect(res.rows.some(x => x.schema === 'war_cache_commit08')).to.be.true() - expect(res.rows.filter(x => x.schema.startsWith('war_cache_')).length).to.equal(1) - - await warehouse.stop() - - createCompatList(['commit11', 'commit10', 'commit09']) - - warehouse = await start(); - res = await warehouse.whPgPool.query<{schema: string}>(` - SELECT schema_name AS schema - FROM information_schema.schemata - `) - expect(res.rows.some(x => x.schema === 'war_cache_commit09')).to.be.true() - expect(res.rows.filter(x => x.schema.startsWith('war_cache_')).length).to.equal(1) - - }) - - -}) - - -export function createCompatList(commits: string[]) { - const directory = path.join(__dirname, '../../../../../../deployment/warehouse-compat-list.txt') - writeFileSync(directory, commits.join('\n')) -} diff --git a/server/src/index.dev.ts b/server/src/index.dev.ts deleted file mode 100644 index d815f15c8..000000000 --- a/server/src/index.dev.ts +++ /dev/null @@ -1,35 +0,0 @@ -import {ApplicationConfig} from '@loopback/core'; -import {GeovistoryServer} from './server'; -import {createHeavyFactoids} from './__tests__/helpers/graphs/heavy-factoids.helper'; -import {cleanDb} from './__tests__/helpers/meta/clean-db.helper'; -import {cleanAndStartDev} from './warehouse/startScripts'; -import {getGvDatabaseUrl} from './utils/databaseUrl'; - -/** - * This function starts the geovistory application and fills the database - * with mock data - * @param options - */ -export async function serveWithMockData(options: ApplicationConfig = {}) { - process.stdout.write('Cleaning database ... '); - await cleanDb(); - console.log('Done'); - - console.log('Seeding test database with mock data:') - // await forFeatureX(); - await createHeavyFactoids(); - console.log(`Test database is ready!\n`); - - console.log(`Cleaning and starting the Warehouse...`); - await cleanAndStartDev() - - console.log(`Starting server using database: ${getGvDatabaseUrl()}`) - console.log(`Starting server at ${options.rest.host}:${options.rest.port} ...`); - const server = new GeovistoryServer(options); - await server.boot(); - await server.start(); - - - console.log(`Server is up and running!`); - -} diff --git a/server/src/warehouse.ts b/server/src/warehouse.ts deleted file mode 100644 index a1fe15d8a..000000000 --- a/server/src/warehouse.ts +++ /dev/null @@ -1,10 +0,0 @@ -import {start} from './warehouse/startScripts'; - -start() - .catch(err => { - console.log(`***************************************`); - console.error('Warehouse was up and running, when this error occured:'); - console.error(err); - console.log(`***************************************`); - process.exit(1); - }); diff --git a/server/src/warehouse/Warehouse.ts b/server/src/warehouse/Warehouse.ts deleted file mode 100644 index 6a6f82b7e..000000000 --- a/server/src/warehouse/Warehouse.ts +++ /dev/null @@ -1,451 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import {forwardRef, Inject, Injectable, InjectionToken, Injector, Type} from 'injection-js'; -import {Notification, Pool, PoolClient, PoolConfig} from 'pg'; -import {parse} from 'pg-connection-string'; -import {values} from 'ramda'; -import {combineLatest, ReplaySubject, Subject} from 'rxjs'; -import {first} from 'rxjs/operators'; -import {createPoolConfig, getPgSslForPg8} from '../utils/databaseUrl'; -import {AggregatedDataService2} from './base/classes/AggregatedDataService2'; -import {IndexDBGeneric} from './base/classes/IndexDBGeneric'; -import {Logger} from './base/classes/Logger'; -import {PrimaryDataService} from './base/classes/PrimaryDataService'; - -export const PK_DEFAULT_CONFIG_PROJECT = 375669; -export const PK_ENGLISH = 18889; -export const APP_CONFIG = new InjectionToken('app.config'); -export const PRIMARY_DS = new InjectionToken[]>('primaryDs'); -export const AGG_DS = new InjectionToken[]>('aggDs'); -export const LAST_UPDATE_DONE_SUFFIX = '__last_update_done' -export const CHANGES_CONSIDERED_UNTIL_SUFFIX = '__changes_considered_until' - -interface NotificationHandler { - channel: string - listeners: { - [listenerName: string]: Subject, - } -} - -export interface WarehouseConfig { - warehouseDatabase: string, - warehouseDatabaseMaxConnections: number, - geovistoryDatabase: string, - geovistoryDatabaseMaxConnections: number, - warehouseSchema: string, -} -// used for consideredUpdatesUntil and leftDSupdateDone -export interface LeftDSDates {[DsName: string]: string} - -@Injectable() -export class Warehouse { - - // Geovistory postgres - gvPgPool: Pool; - gvPgListener: PoolClient; - gvPgListenerConnected$ = new ReplaySubject() - gvPgNotifications$ = new Subject() - - // Warehouse postgres - whPgPool: Pool; - createSchema$ = new Subject() - schemaName: string; - metaTimestamps: IndexDBGeneric; - aggregationTimestamps: IndexDBGeneric; - - // Warehosue inner logic - notificationHandlers: {[key: string]: NotificationHandler} = {} - // if true, changes on dependencies are not propagated to aggregators - preventPropagation = false - status: 'stopped' | 'initializing' | 'starting' | 'running' | 'backuping' - - constructor( - @Inject(APP_CONFIG) config: WarehouseConfig, - @Inject(PRIMARY_DS) private primaryDs: Type[], - @Inject(AGG_DS) private aggDs: Type[], - @Inject(forwardRef(() => Injector)) private injector: Injector - ) { - this.schemaName = config.warehouseSchema; - - const whPgConfig = createPoolConfig(config.warehouseDatabase, config.warehouseDatabaseMaxConnections) - this.whPgPool = new Pool(whPgConfig); - Logger.msg(this.constructor.name, `warehouse DB: ${config.warehouseDatabase.split('@')[1]}`) - Logger.msg(this.constructor.name, `warehouse DB max connections: ${config.warehouseDatabaseMaxConnections}`) - - - const gvPgUrl = config.geovistoryDatabase; - Logger.msg(this.constructor.name, `geovistory DB: ${gvPgUrl.split('@')[1]}`) - const gvPgConfig = parse(config.geovistoryDatabase) as PoolConfig - gvPgConfig.max = config.geovistoryDatabaseMaxConnections - gvPgConfig.ssl = getPgSslForPg8(gvPgUrl) - this.gvPgPool = new Pool(gvPgConfig); - Logger.msg(this.constructor.name, `geovistory DB max connections: ${config.geovistoryDatabaseMaxConnections}`) - - } - - /** - * start warehouse - */ - async start() { - - this.initDataServices(); - - - await this.dbSetup(); - const isInitialized = await this.primInitialized() - - if (!isInitialized) { - await this.createWhData(); - } - - await this.listen() - } - - /** - * sets the databases up: - * - connects to pg - * - initializes leveldb - */ - async dbSetup() { - - await this.connectPgListener(); - - await this.initWhDbSchema() - } - - /** - * Initalize warehouse service - */ - async createWhData() { - this.status = 'initializing'; - - let maxMemoryUsage = 0 - - const interval1 = setInterval(() => { - const m = process.memoryUsage().heapUsed - if (maxMemoryUsage < m) { - maxMemoryUsage = m - } - this.gvPgNotify('warehouse_initializing', 'true').catch(e => { }) - }, 1000) - - - - - const t0 = Logger.start(this.constructor.name, 'Create Warehouse data', 0) - - this.preventPropagation = true - await this.createPrimaryData(); - await this.createAggregatedData() - this.preventPropagation = false - - await this.createAggregatedData() - - clearInterval(interval1) - - await this.gvPgNotify('warehouse_initializing', 'false') - - Logger.msg(this.constructor.name, `************ Warehouse Index Created ***************`, 0) - Logger.msg(this.constructor.name, `The max memory usage was: ${Math.round(maxMemoryUsage / 1024 / 1024 * 100) / 100} MB of memory`, 0) - Logger.itTook(this.constructor.name, t0, 'to create warehouse data', 0) - Logger.msg(this.constructor.name, `****************************************************`, 0) - - }; - - - - /** - * Starts listening - */ - async listen() { - this.initDataServices() - - const t0 = Logger.start(this.constructor.name, 'Start listening Warehouse', 0) - - this.status = 'starting'; - - this.startListening() - - await this.catchUp(); - - this.status = 'running'; - - Logger.itTook(this.constructor.name, t0, 'to start listening. Warehouse is up and running', 0) - } - - private initDataServices() { - this.getPrimaryDs(); - this.getAggregatedDs(); - - } - - /** - * returns true if all primary data services have a valid - * date of a successfull update-done - */ - async primInitialized(): Promise { - const prim = this.getPrimaryDs() - const doneDates = await Promise.all(prim.map(ds => ds.getLastUpdateBegin())) - if (doneDates.includes(undefined)) return false - return true - } - - async primInitAll() { - const prim = this.getPrimaryDs() - for (const ds of prim) { - await ds.initIdx() - } - } - /** - * Deletes local and remote leveldb - */ - async hardReset(errorMsg: string) { - // delete the warehouse schema - await this.whPgPool.query(`DROP SCHEMA IF EXISTS ${this.schemaName}`) - // terminate process - throw new Error(errorMsg) - } - - /** - * Call this function to make the warehouse catching up with latest changes. - * - * Detects changes made on GvDB in the time from this.backupCatchUpDate - * until now. - */ - private async catchUp() { - const prim = this.getPrimaryDs() - for (const primDS of prim) { - await primDS.catchUp(); - } - - const agg = this.getAggregatedDs() - for (const aggDS of agg) { - await aggDS.startUpdate(); - } - - } - - /** - * gets earliest date of latest valid udate cycle of - * the primary data services - */ - private async getCatchUpDate() { - const prim = this.getPrimaryDs() - const dates: Date[] = [] - - for (const p of prim) { - const d = await p.getLastUpdateBegin() - if (d) dates.push(d) - } - dates.sort(); - return dates[dates.length - 1]; - } - - /** - * Initializes the 'database schema' and returns a Promise that resolves as - * soon as all 'tables' (= indexes) are ready to be used - */ - private async initWhDbSchema(): Promise { - await this.whPgPool.query(`CREATE SCHEMA IF NOT EXISTS ${this.schemaName}`) - await this.whPgPool.query(` - CREATE OR REPLACE FUNCTION ${this.schemaName}.tmsp_last_modification() - RETURNS trigger - LANGUAGE 'plpgsql' - COST 100 - VOLATILE NOT LEAKPROOF - AS $BODY$ - BEGIN NEW.tmsp_last_modification = clock_timestamp(); - RETURN NEW; - END; - $BODY$; - `) - - // creates a toplevel metadata table - // this can be used e.g. for storing timestamps of dataservices - this.metaTimestamps = new IndexDBGeneric( - (key: string) => key, - (str: string) => str, - this.constructor.name + '_timestamps', - this - ) - this.aggregationTimestamps = new IndexDBGeneric( - (key: string) => key, - (str: string) => str, - this.constructor.name + '_aggregation_timestamps', - this - ) - - const primaryDsPromises = this.getPrimaryDs().map(ds => ds.index.ready$.pipe(first()).toPromise()) - this.createSchema$.next() - - await this.metaTimestamps.ready$.pipe(first()).toPromise() - await this.aggregationTimestamps.ready$.pipe(first()).toPromise() - await Promise.all(primaryDsPromises) - } - - /** - * Initialize the indexes of the primary data services which will potentially - * trigger aggregated data services to update themselfs. - * In short: The function (re-)creates all indexes - */ - private async createPrimaryData() { - - const t1 = Logger.start(this.constructor.name, 'Initialize Primary Data Services', 0) - - await this.primInitAll() - - Logger.itTook(this.constructor.name, t1, 'to initialize Primary Data Services', 0) - } - /** - * Initialize the indexes of the secondary data services - */ - private async createAggregatedData() { - const t1 = Logger.start(this.constructor.name, 'Initialize Aggregated Data Services', 0) - for (const ds of this.getAggregatedDs()) { - await ds.startUpdate() - } - Logger.itTook(this.constructor.name, t1, 'to Initialize Aggregated Data Services', 0) - } - - private getAggregatedDs(): AggregatedDataService2[] { - return this.aggDs.map(klass => this.injector.get(klass)); - } - private getPrimaryDs(): PrimaryDataService[] { - return this.primaryDs.map(klass => this.injector.get(klass)); - } - - // /** - // * checks out a new postgres client from the pool and nexts the pgConnected$ - // * observable that allows classes aware of this warehouse to wait for the - // * connection before executing postgres queries. - // */ - public async connectPgListener() { - - this.gvPgListener = await this.gvPgPool.connect(); - this.gvPgListenerConnected$.next(this.gvPgListener) - this.gvPgListener.on('notification', (msg) => { - this.gvPgNotifications$.next(msg) - }) - } - - - private async getInitBackupDate(): Promise { - const dbNow = await this.whPgPool.query('SELECT now() as now'); - const tmsp: string = dbNow.rows?.[0]?.now; - return new Date(tmsp) - } - - - - /** - * returns now() tmsp from wh postgres as toISOString - */ - async whPgNow() { - const date = await this.whPgNowDate() - return date.toISOString() - } - /** - * returns now() tmsp from wh postgres as Date - */ - async whPgNowDate() { - const res = await this.whPgPool.query<{now: Date}>('select now()') - return res.rows[0].now - } - - - /** - * returns now() tmsp from gv postgres as toISOString - */ - async gvPgNow() { - const date = await this.gvPgNowDate() - return date.toISOString() - } - /** - * returns now() tmsp from gv postgres as Date - */ - async gvPgNowDate() { - const res = await this.gvPgPool.query<{now: Date}>('select now()') - return res.rows[0].now - } - - - - /** - * Registers a postgres db listener and add a notification handler - * @param channel - * @param callback - * @param name for debugging - */ - async registerDbListener(channel: string, emitter: Subject, listenerName: string) { - await this.gvPgListener.query(`LISTEN ${channel}`) - this.notificationHandlers[channel] = { - channel, - listeners: { - ...(this.notificationHandlers?.[channel]?.listeners ?? {}), - [listenerName]: emitter - } - } - } - - /** - * starts listening to notifications from postgres - * and calls callback of notification handler, if available for the channel - */ - startListening() { - - this.gvPgNotifications$.subscribe((msg) => { - const handler = this.notificationHandlers[msg.channel]; - - if (typeof msg.payload === 'string') { - const date = new Date(msg.payload); - if (isNaN(date.getTime())) console.error(`Invalid Timestamp provided by pg_notify channel ${msg.channel}`, msg.payload) - else if (handler) { - - values(handler.listeners).map(emitter => { - emitter.next(date) - }) - - } - } else { - console.error('payload of notification must be a string convertable to date') - } - }); - } - - - - /** - * Returns a promise that resolves as soon as all sync processes - * of the primary data services are done (and leveldb is actively used) - */ - async waitUntilSyncingDone(): Promise { - return new Promise((res, rej) => { - const prim = this.getPrimaryDs() - const syncStatuses$ = prim.map(p => p.syncing$) - combineLatest(syncStatuses$) - // wait until no sync status is true - .pipe(first(syncStatuses => syncStatuses.includes(true) === false)) - // resolve the promise - .subscribe(() => res()) - }) - } - - - - - gvPgNotify(channel: string, value: string) { - return this.gvPgPool.query(`SELECT pg_notify($1, $2)`, [channel, value]) - } - - - /** - * stops the warehouse - */ - async stop() { - this.status = 'stopped'; - this.notificationHandlers = {} - this.gvPgListener.release(); - await this.whPgPool.end(); - await this.gvPgPool.end(); - } -} - diff --git a/server/src/warehouse/aggregator-ds/class-field-label/p-class-field-label/PClassFieldLabelAggregator.ts b/server/src/warehouse/aggregator-ds/class-field-label/p-class-field-label/PClassFieldLabelAggregator.ts deleted file mode 100644 index a1ce3a9f4..000000000 --- a/server/src/warehouse/aggregator-ds/class-field-label/p-class-field-label/PClassFieldLabelAggregator.ts +++ /dev/null @@ -1,317 +0,0 @@ -// import {AbstractAggregator} from '../../../base/classes/AbstractAggregator'; -// import {PK_DEFAULT_CONFIG_PROJECT, PK_ENGLISH} from '../../../Warehouse'; -// import {PClassFieldLabelProviders} from './PClassFieldLabelProviders'; -// import {PClassFieldLabelId, PClassFieldLabelVal} from './PClassFieldLabelService'; - -// export class PClassFieldLabelAggregator extends AbstractAggregator { - - - - -// constructor( -// public providers: PClassFieldLabelProviders, -// public id: PClassFieldLabelId -// ) { -// super() -// } - -// async create() { - -// const project = await this.providers.project.get({pkProject: this.id.fkProject}) -// if (project) { - -// // default language (en) -// const defaultLang = PK_ENGLISH; - -// // project language -// const proLang = project.fkLanguage - -// // property label -// let label: string | undefined; - -// /** -// * Try to get label in project language -// */ - -// // from project -// label = (await this.providers.proPropertyLabel.get({ -// fkProject: this.id.fkProject, -// fkClass: this.id.fkClass, -// fkProperty: this.id.fkProperty, -// isOutgoing: this.id.isOutgoing, -// fkLanguage: proLang, -// }))?.label - -// if (label) return this.finalize(label); - -// // from geovistory -// label = (await this.providers.proPropertyLabel.get({ -// fkProject: PK_DEFAULT_CONFIG_PROJECT, -// fkClass: this.id.fkClass, -// fkProperty: this.id.fkProperty, -// isOutgoing: this.id.isOutgoing, -// fkLanguage: proLang, -// }))?.label - -// if (label) return this.finalize(label); - -// // from ontome -// const iso6391ProLang = pkLanguageIso6391Map[proLang]; -// if (iso6391ProLang) { -// label = (await this.providers.dfhPropertyLabel.get({ -// pkProperty: this.id.fkProperty, -// language: iso6391ProLang -// }))?.label -// } - -// if (label) { -// label = this.completeReverseLabels(label); -// return this.finalize(label); -// } - -// /** -// * Try to get label in english -// */ - -// // from project -// label = (await this.providers.proPropertyLabel.get({ -// fkProject: this.id.fkProject, -// fkClass: this.id.fkClass, -// fkProperty: this.id.fkProperty, -// isOutgoing: this.id.isOutgoing, -// fkLanguage: defaultLang, -// }))?.label - -// if (label) return this.finalize(label); - -// // from geovistory -// label = (await this.providers.proPropertyLabel.get({ -// fkProject: PK_DEFAULT_CONFIG_PROJECT, -// fkClass: this.id.fkClass, -// fkProperty: this.id.fkProperty, -// isOutgoing: this.id.isOutgoing, -// fkLanguage: defaultLang, -// }))?.label - -// if (label) return this.finalize(label); - -// // from ontome -// label = (await this.providers.dfhPropertyLabel.get({ -// pkProperty: this.id.fkProperty, -// language: 'en' - -// }))?.label -// if (label) { -// label = this.completeReverseLabels(label); -// return this.finalize(label); -// } - -// } - -// return this.finalize() -// } - -// finalize(label?: string): PClassFieldLabelVal { -// return {label}; -// } - -// /** -// * Completes labels of incoming properties, when no incoming label was found -// * @param propertyLabel -// */ -// private completeReverseLabels(propertyLabel: string) { -// if (!this.id.isOutgoing) -// propertyLabel = '[reverse of: ' + propertyLabel + ']'; -// return propertyLabel; -// } - -// } - - - -// export const pkLanguageIso6391Map: {[key: number]: string | undefined} = { -// 17082: 'aa', -// 17099: 'ab', -// 17184: 'af', -// 17260: 'ak', -// 17314: 'am', -// 17413: 'ar', -// 17418: 'an', -// 17448: 'as', -// 17508: 'av', -// 17511: 'ae', -// 17558: 'ay', -// 17572: 'az', -// 17588: 'ba', -// 17590: 'bm', -// 17689: 'be', -// 17691: 'bn', -// 17793: 'bi', -// 17925: 'bo', -// 17940: 'bs', -// 18000: 'br', -// 18080: 'bg', -// 18235: 'ca', -// 18290: 'cs', -// 18300: 'ch', -// 18304: 'ce', -// 18318: 'cu', -// 18319: 'cv', -// 18419: 'kw', -// 18420: 'co', -// 18443: 'cr', -// 18529: 'cy', -// 18547: 'da', -// 18605: 'de', -// 18660: 'dv', -// 18826: 'dz', -// 18865: 'el', -// 18889: 'en', -// 18903: 'eo', -// 18925: 'et', -// 18939: 'eu', -// 18943: 'ee', -// 18962: 'fo', -// 18965: 'fa', -// 18979: 'fj', -// 18981: 'fi', -// 19008: 'fr', -// 19019: 'fy', -// 19031: 'ff', -// 19192: 'gd', -// 19195: 'ga', -// 19196: 'gl', -// 19205: 'gv', -// 19282: 'gn', -// 19314: 'gu', -// 19393: 'ht', -// 19394: 'ha', -// 19404: 'sh', -// 19412: 'he', -// 19418: 'hz', -// 19434: 'hi', -// 19465: 'ho', -// 19516: 'hr', -// 19542: 'hu', -// 19564: 'hy', -// 19576: 'ig', -// 19590: 'io', -// 19616: 'ii', -// 19632: 'iu', -// 19639: 'ie', -// 19657: 'ia', -// 19659: 'id', -// 19675: 'ik', -// 19696: 'is', -// 19703: 'it', -// 19752: 'jv', -// 19839: 'ja', -// 19883: 'kl', -// 19885: 'kn', -// 19889: 'ks', -// 19890: 'ka', -// 19891: 'kr', -// 19896: 'kk', -// 20056: 'km', -// 20080: 'ki', -// 20083: 'rw', -// 20087: 'ky', -// 20234: 'kv', -// 20235: 'kg', -// 20239: 'ko', -// 20372: 'kj', -// 20389: 'ku', -// 20535: 'lo', -// 20540: 'la', -// 20542: 'lv', -// 20656: 'li', -// 20657: 'ln', -// 20663: 'lt', -// 20817: 'lb', -// 20819: 'lu', -// 20824: 'lg', -// 20869: 'mh', -// 20873: 'ml', -// 20877: 'mr', -// 21112: 'mk', -// 21139: 'mg', -// 21152: 'mt', -// 21217: 'mn', -// 21288: 'mi', -// 21306: 'ms', -// 21453: 'my', -// 21518: 'na', -// 21519: 'nv', -// 21534: 'nr', -// 21573: 'nd', -// 21583: 'ng', -// 21609: 'ne', -// 21740: 'nl', -// 21795: 'nn', -// 21807: 'nb', -// 21822: 'no', -// 21957: 'ny', -// 22005: 'oc', -// 22028: 'oj', -// 22107: 'or', -// 22108: 'om', -// 22125: 'os', -// 22170: 'pa', -// 22315: 'pi', -// 22383: 'pl', -// 22389: 'pt', -// 22479: 'ps', -// 22507: 'qu', -// 22670: 'rm', -// 22673: 'ro', -// 22701: 'rn', -// 22705: 'ru', -// 22727: 'sg', -// 22732: 'sa', -// 22893: 'si', -// 22953: 'sk', -// 22963: 'sl', -// 22972: 'se', -// 22981: 'sm', -// 22993: 'sn', -// 22996: 'sd', -// 23029: 'so', -// 23035: 'st', -// 23042: 'es', -// 23065: 'sq', -// 23078: 'sc', -// 23089: 'sr', -// 23121: 'ss', -// 23156: 'su', -// 23174: 'sw', -// 23177: 'sv', -// 23243: 'ty', -// 23247: 'ta', -// 23254: 'tt', -// 23341: 'te', -// 23369: 'tg', -// 23370: 'tl', -// 23384: 'th', -// 23418: 'ti', -// 23537: 'to', -// 23619: 'tn', -// 23620: 'ts', -// 23667: 'tk', -// 23673: 'tr', -// 23701: 'tw', -// 23782: 'ug', -// 23792: 'uk', -// 23838: 'ur', -// 23878: 'uz', -// 23904: 've', -// 23912: 'vi', -// 23958: 'vo', -// 24081: 'wa', -// 24127: 'wo', -// 24278: 'xh', -// 24572: 'yi', -// 24651: 'yo', -// 24776: 'za', -// 24781: 'zh', -// 24901: 'zu', -// } diff --git a/server/src/warehouse/aggregator-ds/class-field-label/p-class-field-label/PClassFieldLabelDependencies.ts b/server/src/warehouse/aggregator-ds/class-field-label/p-class-field-label/PClassFieldLabelDependencies.ts deleted file mode 100644 index 6f2efa6bd..000000000 --- a/server/src/warehouse/aggregator-ds/class-field-label/p-class-field-label/PClassFieldLabelDependencies.ts +++ /dev/null @@ -1,41 +0,0 @@ -// import {Dependencies} from '../../../base/classes/Dependencies' -// import {DependencyIndex} from '../../../base/classes/DependencyIndex' -// import {DfhPropertyLabelId, DfhPropertyLabelVal} from '../../../primary-ds/DfhPropertyLabelService' -// import {ProjectId, ProjectVal} from '../../../primary-ds/ProProjectService' -// import {ProPropertyLabelId, ProPropertyLabelVal} from '../../../primary-ds/ProPropertyLabelService' -// import {Warehouse} from '../../../Warehouse' -// import {PClassFieldLabelId, PClassFieldLabelVal} from './PClassFieldLabelService' -// import {Injectable, Inject, forwardRef} from 'injection-js'; - -// @Injectable() -// export class PClassFieldLabelDependencies extends Dependencies { -// project: DependencyIndex -// dfhPropertyLabel: DependencyIndex -// proPropertyLabel: DependencyIndex - -// // entityFulltextPropertyLabelDep: DependencyIndex; -// constructor( -// @Inject(forwardRef(() => Warehouse) -// ) private wh: Warehouse) { -// super() -// this.project = this.registerDepIdx(new DependencyIndex( -// wh, -// wh.agg.pClassFieldLabel, -// wh.prim.proProject, -// )) - -// this.dfhPropertyLabel = this.registerDepIdx(new DependencyIndex( -// wh, -// wh.agg.pClassFieldLabel, -// wh.prim.dfhPropertyLabel, -// )) - -// this.proPropertyLabel = this.registerDepIdx(new DependencyIndex( -// wh, -// wh.agg.pClassFieldLabel, -// wh.prim.proPropertyLabel, -// )) -// } - - -// } diff --git a/server/src/warehouse/aggregator-ds/class-field-label/p-class-field-label/PClassFieldLabelProviders.ts b/server/src/warehouse/aggregator-ds/class-field-label/p-class-field-label/PClassFieldLabelProviders.ts deleted file mode 100644 index a35d7183c..000000000 --- a/server/src/warehouse/aggregator-ds/class-field-label/p-class-field-label/PClassFieldLabelProviders.ts +++ /dev/null @@ -1,24 +0,0 @@ -// import {Provider} from '../../../base/classes/Provider'; -// import {Providers} from '../../../base/interfaces/Providers'; -// import {DfhPropertyLabelId, DfhPropertyLabelVal} from '../../../primary-ds/DfhPropertyLabelService'; -// import {ProjectId, ProjectVal} from '../../../primary-ds/ProProjectService'; -// import {ProPropertyLabelId, ProPropertyLabelVal} from '../../../primary-ds/ProPropertyLabelService'; -// import {PClassFieldLabelId, PClassFieldLabelService, PClassFieldLabelVal} from './PClassFieldLabelService'; - - -// export class PClassFieldLabelProviders extends Providers { -// project: Provider -// dfhPropertyLabel: Provider -// proPropertyLabel: Provider - -// constructor( -// dep: PClassFieldLabelService, -// protected receiverKey: PClassFieldLabelId -// ) { -// super() -// this.project = this.registerProvider(dep.depProProject, receiverKey); -// this.dfhPropertyLabel = this.registerProvider(dep.depDfhPropertyLabel, receiverKey) -// this.proPropertyLabel = this.registerProvider(dep.depProPropertyLabel, receiverKey); -// } -// } - diff --git a/server/src/warehouse/aggregator-ds/class-field-label/p-class-field-label/PClassFieldLabelService.ts b/server/src/warehouse/aggregator-ds/class-field-label/p-class-field-label/PClassFieldLabelService.ts deleted file mode 100644 index bb28c4a53..000000000 --- a/server/src/warehouse/aggregator-ds/class-field-label/p-class-field-label/PClassFieldLabelService.ts +++ /dev/null @@ -1,229 +0,0 @@ -import {forwardRef, Inject, Injectable} from 'injection-js'; -import {PoolClient} from 'pg'; -import {AggregatedDataService2} from '../../../base/classes/AggregatedDataService2'; -import {AggregatorSqlBuilder, CustomValSql} from '../../../base/classes/AggregatorSqlBuilder'; -import {DependencyIndex} from '../../../base/classes/DependencyIndex'; -import {KeyDefinition} from '../../../base/interfaces/KeyDefinition'; -import {DfhPropertyLabelId, DfhPropertyLabelService, DfhPropertyLabelVal} from '../../../primary-ds/DfhPropertyLabelService'; -import {PPropertyService} from '../../../primary-ds/property/PPropertyService'; -import {ProjectId, ProjectVal, ProProjectService} from '../../../primary-ds/ProProjectService'; -import {ProPropertyLabelId, ProPropertyLabelService, ProPropertyLabelVal} from '../../../primary-ds/ProPropertyLabelService'; -import {PK_DEFAULT_CONFIG_PROJECT, PK_ENGLISH, Warehouse} from '../../../Warehouse'; - -export interface PClassFieldLabelId { - fkProject: number, - fkClass: number - fkProperty: number - isOutgoing: boolean -} -export interface PClassFieldLabelVal { - label?: string -} -export const pClassFieldKeyDef: KeyDefinition[] = [ - {name: 'fkProject', type: 'integer'}, - {name: 'fkClass', type: 'integer'}, - {name: 'fkProperty', type: 'integer'}, - {name: 'isOutgoing', type: 'boolean'}, -] -@Injectable() -export class PClassFieldLabelService extends AggregatedDataService2{ - // aggregator = PClassFieldLabelAggregator; - // providers = PClassFieldLabelProviders; - depProProject: DependencyIndex - depDfhPropertyLabel: DependencyIndex - depProPropertyLabel: DependencyIndex - batchSize = 100000; - - constructor( - @Inject(forwardRef(() => Warehouse)) wh: Warehouse, - @Inject(forwardRef(() => PPropertyService)) pProperty: PPropertyService, - @Inject(forwardRef(() => ProProjectService)) proProject: ProProjectService, - @Inject(forwardRef(() => DfhPropertyLabelService)) dfhPropertyLabel: DfhPropertyLabelService, - @Inject(forwardRef(() => ProPropertyLabelService)) proPropertyLabel: ProPropertyLabelService, - ) { - super( - wh, - pClassFieldKeyDef - ) - - this.registerCreatorDS({ - dataService: pProperty, - customSql: [ - { - select: `"fkProject" as "fkProject", "fkDomain" as "fkClass", "pkProperty" as "fkProperty", true as "isOutgoing"`, - }, - { - select: `"fkProject" as "fkProject", "fkRange" as "fkClass", "pkProperty" as "fkProperty", false as "isOutgoing"`, - } - ] - }) - this.depProProject = this.addDepencency(proProject) - this.depDfhPropertyLabel = this.addDepencency(dfhPropertyLabel) - this.depProPropertyLabel = this.addDepencency(proPropertyLabel) - } - - async aggregateBatch(client: PoolClient, client2: PoolClient, limit: number, offset: number, currentTimestamp: string): Promise { - const builder = new AggregatorSqlBuilder(this, client, currentTimestamp, limit, offset) - - const projectLang = await builder.joinProviderThroughDepIdx({ - leftTable: builder.batchTmpTable.tableDef, - joinWithDepIdx: this.depProProject, - joinOnKeys: { - pkProject: {leftCol: 'fkProject'} - }, - conditionTrueIf: { - providerVal: {fkLanguage: 'IS NOT NULL'} - }, - createCustomObject: (() => `jsonb_build_object( - 'fkLanguage', t2.val->'fkLanguage' - )`) as CustomValSql<{fkLanguage: number}>, - createAggregationVal: { - sql: (provider) => `jsonb_build_object()`, - upsert: {whereCondition: '= false'} - } - }) - - /** - * Try to get label in project language - */ - - // from project - const proLangFromProject = await builder.joinProviderThroughDepIdx({ - leftTable: projectLang.aggregation.tableDef, - joinWithDepIdx: this.depProPropertyLabel, - joinWhereLeftTableCondition: '= true', - joinOnKeys: { - fkClass: {leftCol: 'fkClass'}, - fkProject: {leftCol: 'fkProject'}, - fkProperty: {leftCol: 'fkProperty'}, - isOutgoing: {leftCol: 'isOutgoing'}, - fkLanguage: {leftCustom: {name: 'fkLanguage', type: 'int'}}, - }, - conditionTrueIf: { - providerVal: {label: 'IS NOT NULL'} - }, - createAggregationVal: { - sql: (provider) => `jsonb_build_object('label',${provider}.val->>'label')`, - upsert: {whereCondition: '= true'} - }, - createCustomObject: (() => `t1.custom`) as CustomValSql<{fkLanguage: number}>, - }) - - // from geovistory - const proLangFromDefaultProject = await builder.joinProviderThroughDepIdx({ - leftTable: proLangFromProject.aggregation.tableDef, - joinWithDepIdx: this.depProPropertyLabel, - joinWhereLeftTableCondition: '= false', - joinOnKeys: { - fkClass: {leftCol: 'fkClass'}, - fkProject: {value: PK_DEFAULT_CONFIG_PROJECT}, - fkProperty: {leftCol: 'fkProperty'}, - isOutgoing: {leftCol: 'isOutgoing'}, - fkLanguage: {leftCustom: {name: 'fkLanguage', type: 'int'}}, - }, - conditionTrueIf: { - providerVal: {label: 'IS NOT NULL'} - }, - createAggregationVal: { - sql: (provider) => `jsonb_build_object('label',${provider}.val->>'label')`, - upsert: {whereCondition: '= true'} - }, - createCustomObject: (() => `t1.custom`) as CustomValSql<{fkLanguage: number}>, - }) - // from ontome - const proLangOntoMe = await builder.joinProviderThroughDepIdx({ - leftTable: proLangFromDefaultProject.aggregation.tableDef, - joinWhereLeftTableCondition: '= false', - joinWithDepIdx: this.depDfhPropertyLabel, - joinOnKeys: { - pkProperty: {leftCol: 'fkProperty'}, - language: {leftCustom: {name: 'fkLanguage', type: 'int'}}, - }, - conditionTrueIf: { - providerVal: {label: 'IS NOT NULL'} - }, - createAggregationVal: { - sql: this.completeReverseLabels(), - upsert: {whereCondition: '= true'} - } - }) - - /** - * Try to get label in english - */ - - // from project - const enFromProject = await builder.joinProviderThroughDepIdx({ - leftTable: proLangOntoMe.aggregation.tableDef, - joinWithDepIdx: this.depProPropertyLabel, - joinWhereLeftTableCondition: '= false', - joinOnKeys: { - fkClass: {leftCol: 'fkClass'}, - fkProject: {leftCol: 'fkProject'}, - fkProperty: {leftCol: 'fkProperty'}, - isOutgoing: {leftCol: 'isOutgoing'}, - fkLanguage: {value: PK_ENGLISH}, - }, - conditionTrueIf: { - providerVal: {label: 'IS NOT NULL'} - }, - createAggregationVal: { - sql: (provider) => `jsonb_build_object('label',${provider}.val->>'label')`, - upsert: {whereCondition: '= true'} - }, - createCustomObject: (() => `t1.custom`) as CustomValSql<{fkLanguage: number}>, - }) - - // from geovistory - const enFromDefaultProject = await builder.joinProviderThroughDepIdx({ - leftTable: enFromProject.aggregation.tableDef, - joinWithDepIdx: this.depProPropertyLabel, - joinWhereLeftTableCondition: '= false', - joinOnKeys: { - fkClass: {leftCol: 'fkClass'}, - fkProject: {value: PK_DEFAULT_CONFIG_PROJECT}, - fkProperty: {leftCol: 'fkProperty'}, - isOutgoing: {leftCol: 'isOutgoing'}, - fkLanguage: {value: PK_ENGLISH}, - }, - conditionTrueIf: { - providerVal: {label: 'IS NOT NULL'} - }, - createAggregationVal: { - sql: (provider) => `jsonb_build_object('label',${provider}.val->>'label')`, - upsert: {whereCondition: '= true'} - }, - createCustomObject: (() => `t1.custom`) as CustomValSql<{fkLanguage: number}>, - }) - // from ontome - await builder.joinProviderThroughDepIdx({ - leftTable: enFromDefaultProject.aggregation.tableDef, - joinWhereLeftTableCondition: '= false', - joinWithDepIdx: this.depDfhPropertyLabel, - joinOnKeys: { - pkProperty: {leftCol: 'fkProperty'}, - language: {value: PK_ENGLISH}, - }, - conditionTrueIf: { - providerVal: {label: 'IS NOT NULL'} - }, - createAggregationVal: { - sql: this.completeReverseLabels(), - upsert: {whereCondition: '= true'} - } - }) - - // await builder.printQueries() - const count = await builder.executeQueries() - return count - } - - - private completeReverseLabels(): (provTable: string) => string { - return (provider) => `jsonb_build_object('label', - CASE WHEN t1."r_isOutgoing" = true THEN ${provider}.val->>'label' - ELSE ${provider}.val->>'inverseLabel' - END - )`; - } -} diff --git a/server/src/warehouse/aggregator-ds/class-field-label/r-class-field-label/RClassFieldLabelAggregator.ts b/server/src/warehouse/aggregator-ds/class-field-label/r-class-field-label/RClassFieldLabelAggregator.ts deleted file mode 100644 index 5ac6dd8ad..000000000 --- a/server/src/warehouse/aggregator-ds/class-field-label/r-class-field-label/RClassFieldLabelAggregator.ts +++ /dev/null @@ -1,74 +0,0 @@ -// import {AbstractAggregator} from '../../../base/classes/AbstractAggregator'; -// import {PK_DEFAULT_CONFIG_PROJECT, PK_ENGLISH} from '../../../Warehouse'; -// import {RClassFieldLabelProviders} from './RClassFieldLabelProviders'; -// import {RClassFieldId, RClassFieldVal} from './RClassFieldLabelService'; -// import {ProPropertyLabelVal} from '../../../primary-ds/ProPropertyLabelService'; - -// export class RClassFieldLabelAggregator extends AbstractAggregator { - -// constructor( -// public providers: RClassFieldLabelProviders, -// public id: RClassFieldId -// ) { -// super() -// } - -// async create() { - - - -// // default language (en) -// const defaultLang = PK_ENGLISH; - - -// // property label -// let val: ProPropertyLabelVal | undefined; - - - -// /** -// * Try to get label in english -// */ -// // from geovistory -// val = await this.providers.proPropertyLabel.get({ -// fkProject: PK_DEFAULT_CONFIG_PROJECT, -// fkClass: this.id.fkClass, -// fkProperty: this.id.fkProperty, -// isOutgoing: this.id.isOutgoing, -// fkLanguage: defaultLang, -// }) - -// if (val?.label) return this.finalize(val.label); - -// // from ontome -// val = await this.providers.dfhPropertyLabel.get({ -// pkProperty: this.id.fkProperty, -// language: 'en' - -// }) -// if (val?.label) { -// const label = this.completeReverseLabels(val.label); -// return this.finalize(label); -// } - - - -// return this.finalize() -// } - -// finalize(label?: string): RClassFieldVal { -// return {label}; -// } - -// /** -// * Completes labels of incoming properties, when no incoming label was found -// * @param propertyLabel -// */ -// private completeReverseLabels(propertyLabel: string) { -// if (!this.id.isOutgoing) -// propertyLabel = '[reverse of: ' + propertyLabel + ']'; -// return propertyLabel; -// } - -// } - diff --git a/server/src/warehouse/aggregator-ds/class-field-label/r-class-field-label/RClassFieldLabelDependencies.ts b/server/src/warehouse/aggregator-ds/class-field-label/r-class-field-label/RClassFieldLabelDependencies.ts deleted file mode 100644 index c8268cf04..000000000 --- a/server/src/warehouse/aggregator-ds/class-field-label/r-class-field-label/RClassFieldLabelDependencies.ts +++ /dev/null @@ -1,33 +0,0 @@ -// import {Dependencies} from '../../../base/classes/Dependencies' -// import {DependencyIndex} from '../../../base/classes/DependencyIndex' -// import {DfhPropertyLabelId, DfhPropertyLabelVal} from '../../../primary-ds/DfhPropertyLabelService' -// import {ProPropertyLabelId, ProPropertyLabelVal} from '../../../primary-ds/ProPropertyLabelService' -// import {Warehouse} from '../../../Warehouse' -// import {RClassFieldId, RClassFieldVal} from './RClassFieldLabelService' -// import {Injectable, Inject, forwardRef} from 'injection-js'; - -// @Injectable() -// export class RClassFieldLabelDependencies extends Dependencies { -// dfhPropertyLabel: DependencyIndex -// proPropertyLabel: DependencyIndex - -// // entityFulltextPropertyLabelDep: DependencyIndex; -// constructor(@Inject(forwardRef(() => Warehouse)) private wh: Warehouse) { - -// super() - -// this.dfhPropertyLabel = this.registerDepIdx(new DependencyIndex( -// wh, -// wh.agg.rClassFieldLabel, -// wh.prim.dfhPropertyLabel, -// )) - -// this.proPropertyLabel = this.registerDepIdx(new DependencyIndex( -// wh, -// wh.agg.rClassFieldLabel, -// wh.prim.proPropertyLabel, -// )) -// } - -// } - diff --git a/server/src/warehouse/aggregator-ds/class-field-label/r-class-field-label/RClassFieldLabelProviders.ts b/server/src/warehouse/aggregator-ds/class-field-label/r-class-field-label/RClassFieldLabelProviders.ts deleted file mode 100644 index f1ff0f916..000000000 --- a/server/src/warehouse/aggregator-ds/class-field-label/r-class-field-label/RClassFieldLabelProviders.ts +++ /dev/null @@ -1,21 +0,0 @@ -// import {Provider} from '../../../base/classes/Provider'; -// import {Providers} from '../../../base/interfaces/Providers'; -// import {DfhPropertyLabelId, DfhPropertyLabelVal} from '../../../primary-ds/DfhPropertyLabelService'; -// import {ProPropertyLabelId, ProPropertyLabelVal} from '../../../primary-ds/ProPropertyLabelService'; -// import {RClassFieldId, RClassFieldLabelService, RClassFieldVal} from './RClassFieldLabelService'; - - -// export class RClassFieldLabelProviders extends Providers { -// dfhPropertyLabel: Provider -// proPropertyLabel: Provider - -// constructor( -// dep: RClassFieldLabelService, -// protected receiverKey: RClassFieldId -// ) { -// super() -// this.dfhPropertyLabel = this.registerProvider(dep.depDfhPropertyLabel, receiverKey) -// this.proPropertyLabel = this.registerProvider(dep.depProPropertyLabel, receiverKey); -// } -// } - diff --git a/server/src/warehouse/aggregator-ds/class-field-label/r-class-field-label/RClassFieldLabelService.ts b/server/src/warehouse/aggregator-ds/class-field-label/r-class-field-label/RClassFieldLabelService.ts deleted file mode 100644 index 7364b13e4..000000000 --- a/server/src/warehouse/aggregator-ds/class-field-label/r-class-field-label/RClassFieldLabelService.ts +++ /dev/null @@ -1,114 +0,0 @@ -import {forwardRef, Inject, Injectable} from 'injection-js'; -import {PoolClient} from 'pg'; -import {AggregatedDataService2} from '../../../base/classes/AggregatedDataService2'; -import {AggregatorSqlBuilder} from '../../../base/classes/AggregatorSqlBuilder'; -import {DependencyIndex} from '../../../base/classes/DependencyIndex'; -import {KeyDefinition} from '../../../base/interfaces/KeyDefinition'; -import {DfhPropertyLabelId, DfhPropertyLabelService, DfhPropertyLabelVal} from '../../../primary-ds/DfhPropertyLabelService'; -import {RPropertyService} from '../../../primary-ds/property/RPropertyService'; -import {ProPropertyLabelId, ProPropertyLabelService, ProPropertyLabelVal} from '../../../primary-ds/ProPropertyLabelService'; -import {PK_DEFAULT_CONFIG_PROJECT, PK_ENGLISH, Warehouse} from '../../../Warehouse'; - -export interface RClassFieldId { - fkClass: number - fkProperty: number - isOutgoing: boolean -} -export interface RClassFieldVal { - label?: string -} -export const rClassFieldKeyDef: KeyDefinition[] = [ - {name: 'fkClass', type: 'integer'}, - {name: 'fkProperty', type: 'integer'}, - {name: 'isOutgoing', type: 'boolean'}, -] -@Injectable() -export class RClassFieldLabelService extends AggregatedDataService2{ - // aggregator = RClassFieldLabelAggregator; - // providers = RClassFieldLabelProviders; - depDfhPropertyLabel: DependencyIndex - depProPropertyLabel: DependencyIndex - batchSize = 100000; - constructor( - @Inject(forwardRef(() => Warehouse)) wh: Warehouse, - @Inject(forwardRef(() => RPropertyService)) rProperty: RPropertyService, - @Inject(forwardRef(() => DfhPropertyLabelService)) dfhPropertyLabel: DfhPropertyLabelService, - @Inject(forwardRef(() => ProPropertyLabelService)) proPropertyLabel: ProPropertyLabelService - - ) { - super( - wh, - rClassFieldKeyDef - ) - - this.registerCreatorDS({ - dataService: rProperty, - customSql: [ - { - select: `"fkDomain" as "fkClass", "pkProperty" as "fkProperty", true as "isOutgoing"`, - }, - { - select: `"fkRange" as "fkClass", "pkProperty" as "fkProperty", false as "isOutgoing"`, - } - ] - }) - this.depDfhPropertyLabel = this.addDepencency(dfhPropertyLabel) - this.depProPropertyLabel = this.addDepencency(proPropertyLabel) - - } - async aggregateBatch(client: PoolClient, client2: PoolClient, limit: number, offset: number, currentTimestamp: string): Promise { - const builder = new AggregatorSqlBuilder(this, client, currentTimestamp, limit, offset) - - - // from geovistory - const enFromDefaultProject = await builder.joinProviderThroughDepIdx({ - leftTable: builder.batchTmpTable.tableDef, - joinWithDepIdx: this.depProPropertyLabel, - joinOnKeys: { - fkClass: {leftCol: 'fkClass'}, - fkProject: {value: PK_DEFAULT_CONFIG_PROJECT}, - fkProperty: {leftCol: 'fkProperty'}, - isOutgoing: {leftCol: 'isOutgoing'}, - fkLanguage: {value: PK_ENGLISH}, - }, - conditionTrueIf: { - providerVal: {label: 'IS NOT NULL'} - }, - createAggregationVal: { - sql: (provider) => `jsonb_build_object('label',${provider}.val->>'label')`, - upsert: {whereCondition: '= true'} - }, - }) - - // from ontome - await builder.joinProviderThroughDepIdx({ - leftTable: enFromDefaultProject.aggregation.tableDef, - joinWhereLeftTableCondition: '= false', - joinWithDepIdx: this.depDfhPropertyLabel, - joinOnKeys: { - pkProperty: {leftCol: 'fkProperty'}, - language: {value: PK_ENGLISH}, - }, - conditionTrueIf: { - providerVal: {label: 'IS NOT NULL'} - }, - createAggregationVal: { - sql: this.completeReverseLabels(), - upsert: {whereCondition: '= true'} - } - }) - - // await builder.printQueries() - const count = await builder.executeQueries() - return count - } - - - private completeReverseLabels(): (provTable: string) => string { - return (provider) => `jsonb_build_object('label', - CASE WHEN t1."r_isOutgoing" = true THEN ${provider}.val->>'label' - ELSE ${provider}.val->>'inverseLabel' - END - )`; - } -} diff --git a/server/src/warehouse/aggregator-ds/class-label/class-label.commons.ts b/server/src/warehouse/aggregator-ds/class-label/class-label.commons.ts deleted file mode 100644 index ff2673723..000000000 --- a/server/src/warehouse/aggregator-ds/class-label/class-label.commons.ts +++ /dev/null @@ -1,189 +0,0 @@ - - - -export const pkLanguageIso6391Map: {[key: number]: string | undefined;} = { - 17082: 'aa', - 17099: 'ab', - 17184: 'af', - 17260: 'ak', - 17314: 'am', - 17413: 'ar', - 17418: 'an', - 17448: 'as', - 17508: 'av', - 17511: 'ae', - 17558: 'ay', - 17572: 'az', - 17588: 'ba', - 17590: 'bm', - 17689: 'be', - 17691: 'bn', - 17793: 'bi', - 17925: 'bo', - 17940: 'bs', - 18000: 'br', - 18080: 'bg', - 18235: 'ca', - 18290: 'cs', - 18300: 'ch', - 18304: 'ce', - 18318: 'cu', - 18319: 'cv', - 18419: 'kw', - 18420: 'co', - 18443: 'cr', - 18529: 'cy', - 18547: 'da', - 18605: 'de', - 18660: 'dv', - 18826: 'dz', - 18865: 'el', - 18889: 'en', - 18903: 'eo', - 18925: 'et', - 18939: 'eu', - 18943: 'ee', - 18962: 'fo', - 18965: 'fa', - 18979: 'fj', - 18981: 'fi', - 19008: 'fr', - 19019: 'fy', - 19031: 'ff', - 19192: 'gd', - 19195: 'ga', - 19196: 'gl', - 19205: 'gv', - 19282: 'gn', - 19314: 'gu', - 19393: 'ht', - 19394: 'ha', - 19404: 'sh', - 19412: 'he', - 19418: 'hz', - 19434: 'hi', - 19465: 'ho', - 19516: 'hr', - 19542: 'hu', - 19564: 'hy', - 19576: 'ig', - 19590: 'io', - 19616: 'ii', - 19632: 'iu', - 19639: 'ie', - 19657: 'ia', - 19659: 'id', - 19675: 'ik', - 19696: 'is', - 19703: 'it', - 19752: 'jv', - 19839: 'ja', - 19883: 'kl', - 19885: 'kn', - 19889: 'ks', - 19890: 'ka', - 19891: 'kr', - 19896: 'kk', - 20056: 'km', - 20080: 'ki', - 20083: 'rw', - 20087: 'ky', - 20234: 'kv', - 20235: 'kg', - 20239: 'ko', - 20372: 'kj', - 20389: 'ku', - 20535: 'lo', - 20540: 'la', - 20542: 'lv', - 20656: 'li', - 20657: 'ln', - 20663: 'lt', - 20817: 'lb', - 20819: 'lu', - 20824: 'lg', - 20869: 'mh', - 20873: 'ml', - 20877: 'mr', - 21112: 'mk', - 21139: 'mg', - 21152: 'mt', - 21217: 'mn', - 21288: 'mi', - 21306: 'ms', - 21453: 'my', - 21518: 'na', - 21519: 'nv', - 21534: 'nr', - 21573: 'nd', - 21583: 'ng', - 21609: 'ne', - 21740: 'nl', - 21795: 'nn', - 21807: 'nb', - 21822: 'no', - 21957: 'ny', - 22005: 'oc', - 22028: 'oj', - 22107: 'or', - 22108: 'om', - 22125: 'os', - 22170: 'pa', - 22315: 'pi', - 22383: 'pl', - 22389: 'pt', - 22479: 'ps', - 22507: 'qu', - 22670: 'rm', - 22673: 'ro', - 22701: 'rn', - 22705: 'ru', - 22727: 'sg', - 22732: 'sa', - 22893: 'si', - 22953: 'sk', - 22963: 'sl', - 22972: 'se', - 22981: 'sm', - 22993: 'sn', - 22996: 'sd', - 23029: 'so', - 23035: 'st', - 23042: 'es', - 23065: 'sq', - 23078: 'sc', - 23089: 'sr', - 23121: 'ss', - 23156: 'su', - 23174: 'sw', - 23177: 'sv', - 23243: 'ty', - 23247: 'ta', - 23254: 'tt', - 23341: 'te', - 23369: 'tg', - 23370: 'tl', - 23384: 'th', - 23418: 'ti', - 23537: 'to', - 23619: 'tn', - 23620: 'ts', - 23667: 'tk', - 23673: 'tr', - 23701: 'tw', - 23782: 'ug', - 23792: 'uk', - 23838: 'ur', - 23878: 'uz', - 23904: 've', - 23912: 'vi', - 23958: 'vo', - 24081: 'wa', - 24127: 'wo', - 24278: 'xh', - 24572: 'yi', - 24651: 'yo', - 24776: 'za', - 24781: 'zh', - 24901: 'zu', -}; diff --git a/server/src/warehouse/aggregator-ds/class-label/p-class-label/PClassLabelAggregator.ts b/server/src/warehouse/aggregator-ds/class-label/p-class-label/PClassLabelAggregator.ts deleted file mode 100644 index a246eb678..000000000 --- a/server/src/warehouse/aggregator-ds/class-label/p-class-label/PClassLabelAggregator.ts +++ /dev/null @@ -1,110 +0,0 @@ -import {AbstractAggregator} from '../../../base/classes/AbstractAggregator'; -import {PClassId} from '../../../primary-ds/ProClassFieldsConfigService'; -import {PK_DEFAULT_CONFIG_PROJECT, PK_ENGLISH} from '../../../Warehouse'; -import {pkLanguageIso6391Map} from '../class-label.commons'; -import {PClassLabelProviders} from './PClassLabelProviders'; -import {PClassLabelVal} from './PClassLabelService'; - -export class PClassLabelAggregator extends AbstractAggregator { - - - - constructor( - public providers: PClassLabelProviders, - public id: PClassId - ) { - super() - } - - async create() { - - // const project = await this.providers.proProject.get({pkProject: this.id.fkProject}) - // if (project) { - - // // default language (en) - // const defaultLang = PK_ENGLISH; - - // // project language - // const proLang = project.fkLanguage - - // // class label - // let classLabel: string | undefined; - - // /** - // * Try to get label in project language - // */ - - // // from project - // classLabel = (await this.providers.proClassLabel.get({ - // fkClass: this.id.pkClass, - // fkLanguage: proLang, - // fkProject: this.id.fkProject - // }))?.label - - // if (classLabel) return this.finalize(classLabel); - - // // from geovistory - // classLabel = (await this.providers.proClassLabel.get({ - // fkClass: this.id.pkClass, - // fkLanguage: proLang, - // fkProject: PK_DEFAULT_CONFIG_PROJECT - // }))?.label - - // if (classLabel) return this.finalize(classLabel); - - // // from ontome - // const iso6391ProLang = pkLanguageIso6391Map[proLang]; - // if (iso6391ProLang) { - // classLabel =( await this.providers.dfhClassLabel.get({ - // pkClass: this.id.pkClass, - // language: iso6391ProLang - - // }))?.label - // } - - // if (classLabel) return this.finalize(classLabel); - - // /** - // * Try to get label in english - // */ - - // // from project - // classLabel = (await this.providers.proClassLabel.get({ - // fkClass: this.id.pkClass, - // fkLanguage: defaultLang, - // fkProject: this.id.fkProject - // }))?.label - - // if (classLabel) return this.finalize(classLabel); - - // // from geovistory - // classLabel = (await this.providers.proClassLabel.get({ - // fkClass: this.id.pkClass, - // fkLanguage: defaultLang, - // fkProject: PK_DEFAULT_CONFIG_PROJECT - // }))?.label - - // if (classLabel) return this.finalize(classLabel); - - // // from ontome - // classLabel = (await this.providers.dfhClassLabel.get({ - // pkClass: this.id.pkClass, - // language: 'en' - - // }))?.label - - // if (classLabel) return this.finalize(classLabel); - - // } - - return this.finalize() - } - - finalize(label?: string) { - return {label}; - } -} - - - - diff --git a/server/src/warehouse/aggregator-ds/class-label/p-class-label/PClassLabelDependencies.ts b/server/src/warehouse/aggregator-ds/class-label/p-class-label/PClassLabelDependencies.ts deleted file mode 100644 index 4d73f53ed..000000000 --- a/server/src/warehouse/aggregator-ds/class-label/p-class-label/PClassLabelDependencies.ts +++ /dev/null @@ -1,38 +0,0 @@ -// import {Dependencies} from '../../../base/classes/Dependencies' -// import {DependencyIndex} from '../../../base/classes/DependencyIndex' -// import {DfhClassLabelId, DfhClassLabelVal} from '../../../primary-ds/DfhClassLabelService' -// import {PClassId} from '../../../primary-ds/ProClassFieldsConfigService' -// import {ProClassLabelId, ProClassLabelVal} from '../../../primary-ds/ProClassLabelService' -// import {ProjectId, ProjectVal} from '../../../primary-ds/ProProjectService' -// import {Warehouse} from '../../../Warehouse' -// import {PClassLabelVal} from './PClassLabelService' -// import {Injectable, Inject, forwardRef} from 'injection-js' - -// @Injectable() -// export class PClassLabelDependencies extends Dependencies { -// proProject: DependencyIndex -// dfhClassLabel: DependencyIndex -// proClassLabel: DependencyIndex - -// // entityFulltextClassLabelDep: DependencyIndex; -// constructor(@Inject(forwardRef(() => Warehouse)) private wh: Warehouse) { -// super() -// this.proProject = this.registerDepIdx(new DependencyIndex( -// wh, -// wh.agg.pClassLabel, -// wh.prim.proProject, -// )) - -// this.dfhClassLabel = this.registerDepIdx(new DependencyIndex( -// wh, -// wh.agg.pClassLabel, -// wh.prim.dfhClassLabel, -// )) - -// this.proClassLabel = this.registerDepIdx(new DependencyIndex( -// wh, -// wh.agg.pClassLabel, -// wh.prim.proClassLabel, -// )) -// } -// } diff --git a/server/src/warehouse/aggregator-ds/class-label/p-class-label/PClassLabelProviders.ts b/server/src/warehouse/aggregator-ds/class-label/p-class-label/PClassLabelProviders.ts deleted file mode 100644 index b859caefb..000000000 --- a/server/src/warehouse/aggregator-ds/class-label/p-class-label/PClassLabelProviders.ts +++ /dev/null @@ -1,24 +0,0 @@ -import {Provider} from '../../../base/classes/Provider'; -import {Providers} from '../../../base/interfaces/Providers'; -import {DfhClassLabelId, DfhClassLabelVal} from '../../../primary-ds/DfhClassLabelService'; -import {PClassId} from '../../../primary-ds/ProClassFieldsConfigService'; -import {ProClassLabelId, ProClassLabelVal} from '../../../primary-ds/ProClassLabelService'; -import {ProjectId, ProjectVal} from '../../../primary-ds/ProProjectService'; -import {PClassLabelService, PClassLabelVal} from './PClassLabelService'; - -export class PClassLabelProviders extends Providers { - proProject: Provider - dfhClassLabel: Provider - proClassLabel: Provider - - constructor( - dep: PClassLabelService, - protected receiverKey: PClassId - ) { - super() - this.proProject = this.registerProvider(dep.depProProject, receiverKey); - this.dfhClassLabel = this.registerProvider(dep.depDfhClassLabel, receiverKey) - this.proClassLabel = this.registerProvider(dep.depProClassLabel, receiverKey); - } -} - diff --git a/server/src/warehouse/aggregator-ds/class-label/p-class-label/PClassLabelService.ts b/server/src/warehouse/aggregator-ds/class-label/p-class-label/PClassLabelService.ts deleted file mode 100644 index c30d91c39..000000000 --- a/server/src/warehouse/aggregator-ds/class-label/p-class-label/PClassLabelService.ts +++ /dev/null @@ -1,210 +0,0 @@ -import {forwardRef, Inject, Injectable} from 'injection-js'; -import {AggregatedDataService2} from '../../../base/classes/AggregatedDataService2'; -import {DependencyIndex} from '../../../base/classes/DependencyIndex'; -import {PClassService} from '../../../primary-ds/class/PClassService'; -import {DfhClassLabelId, DfhClassLabelService, DfhClassLabelVal} from '../../../primary-ds/DfhClassLabelService'; -import {PClassId, pClassIdKeyDef} from '../../../primary-ds/ProClassFieldsConfigService'; -import {ProClassLabelId, ProClassLabelService, ProClassLabelVal} from '../../../primary-ds/ProClassLabelService'; -import {ProjectId, ProjectVal, ProProjectService} from '../../../primary-ds/ProProjectService'; -import {Warehouse, PK_DEFAULT_CONFIG_PROJECT, PK_ENGLISH} from '../../../Warehouse'; -import {PClassLabelAggregator} from './PClassLabelAggregator'; -import {PClassLabelProviders} from './PClassLabelProviders'; -import {PoolClient} from 'pg'; -import {AggregatorSqlBuilder, CustomValSql} from '../../../base/classes/AggregatorSqlBuilder'; - -export interface PClassLabelVal {label?: string} -@Injectable() -export class PClassLabelService extends AggregatedDataService2{ - aggregator = PClassLabelAggregator; - providers = PClassLabelProviders; - depProProject: DependencyIndex - depDfhClassLabel: DependencyIndex - depProClassLabel: DependencyIndex - batchSize = 100000; - constructor( - @Inject(forwardRef(() => Warehouse)) wh: Warehouse, - @Inject(forwardRef(() => PClassService)) pClass: PClassService, - @Inject(forwardRef(() => ProProjectService)) proProject: ProProjectService, - @Inject(forwardRef(() => DfhClassLabelService)) dfhClassLabel: DfhClassLabelService, - @Inject(forwardRef(() => ProClassLabelService)) proClassLabel: ProClassLabelService) { - super( - wh, - pClassIdKeyDef - ) - this.registerCreatorDS({dataService: pClass}); - this.depProProject = this.addDepencency(proProject) - this.depDfhClassLabel = this.addDepencency(dfhClassLabel) - this.depProClassLabel = this.addDepencency(proClassLabel) - - } - getDependencies() { - return this - }; - onUpsertSql(tableAlias: string) { - return ` - INSERT INTO war.class_preview (fk_class, fk_project, label) - SELECT DISTINCT ON ("pkClass") "pkClass", "fkProject", val->>'label' - FROM ${tableAlias} - ON CONFLICT (fk_class, fk_project) DO UPDATE - SET label = EXCLUDED.label - WHERE EXCLUDED.label IS DISTINCT FROM war.class_preview.label` - } - async aggregateBatch(client: PoolClient, client2: PoolClient, limit: number, offset: number, currentTimestamp: string): Promise { - const builder = new AggregatorSqlBuilder(this, client, currentTimestamp, limit, offset) - - const projectLang = await builder.joinProviderThroughDepIdx({ - leftTable: builder.batchTmpTable.tableDef, - joinWithDepIdx: this.depProProject, - joinOnKeys: { - pkProject: {leftCol: 'fkProject'} - }, - conditionTrueIf: { - providerVal: {fkLanguage: 'IS NOT NULL'} - }, - createCustomObject: (() => `jsonb_build_object( - 'fkLanguage', t2.val->'fkLanguage' - )`) as CustomValSql<{fkLanguage: number}>, - createAggregationVal: { - sql: (provider) => `jsonb_build_object()`, - upsert: {whereCondition: '= false'} - } - }) - - /** - * Try to get label in project language - */ - - // from project - const proLangFromProject = await builder.joinProviderThroughDepIdx({ - leftTable: projectLang.aggregation.tableDef, - joinWithDepIdx: this.depProClassLabel, - joinWhereLeftTableCondition: '= true', - joinOnKeys: { - fkClass: {leftCol: 'pkClass'}, - fkProject: {leftCol: 'fkProject'}, - fkLanguage: {leftCustom: {name: 'fkLanguage', type: 'int'}}, - }, - conditionTrueIf: { - providerVal: {label: 'IS NOT NULL'} - }, - createAggregationVal: { - sql: (provider) => `jsonb_build_object('label',${provider}.val->>'label')`, - upsert: {whereCondition: '= true'} - }, - createCustomObject: (() => `t1.custom`) as CustomValSql<{fkLanguage: number}>, - }) - - // from geovistory - const proLangFromDefaultProject = await builder.joinProviderThroughDepIdx({ - leftTable: proLangFromProject.aggregation.tableDef, - joinWithDepIdx: this.depProClassLabel, - joinWhereLeftTableCondition: '= false', - joinOnKeys: { - fkClass: {leftCol: 'pkClass'}, - fkProject: {value: PK_DEFAULT_CONFIG_PROJECT}, - fkLanguage: {leftCustom: {name: 'fkLanguage', type: 'int'}}, - }, - conditionTrueIf: { - providerVal: {label: 'IS NOT NULL'} - }, - createAggregationVal: { - sql: (provider) => `jsonb_build_object('label',${provider}.val->>'label')`, - upsert: {whereCondition: '= true'} - }, - createCustomObject: (() => `t1.custom`) as CustomValSql<{fkLanguage: number}>, - }) - // from ontome - const proLangOntoMe = await builder.joinProviderThroughDepIdx({ - leftTable: proLangFromDefaultProject.aggregation.tableDef, - joinWhereLeftTableCondition: '= false', - joinWithDepIdx: this.depDfhClassLabel, - joinOnKeys: { - pkClass: {leftCol: 'pkClass'}, - language: {leftCustom: {name: 'fkLanguage', type: 'int'}}, - }, - conditionTrueIf: { - providerVal: {label: 'IS NOT NULL'} - }, - createAggregationVal: { - sql: (provider) => `jsonb_build_object('label',${provider}.val->>'label')`, - upsert: {whereCondition: '= true'} - } - }) - - /** - * Try to get label in english - */ - - // from project - const enFromProject = await builder.joinProviderThroughDepIdx({ - leftTable: proLangOntoMe.aggregation.tableDef, - joinWithDepIdx: this.depProClassLabel, - joinWhereLeftTableCondition: '= false', - joinOnKeys: { - fkClass: {leftCol: 'pkClass'}, - fkProject: {leftCol: 'fkProject'}, - fkLanguage: {value: PK_ENGLISH}, - }, - conditionTrueIf: { - providerVal: {label: 'IS NOT NULL'} - }, - createAggregationVal: { - sql: (provider) => `jsonb_build_object('label',${provider}.val->>'label')`, - upsert: {whereCondition: '= true'} - }, - createCustomObject: (() => `t1.custom`) as CustomValSql<{fkLanguage: number}>, - }) - - // from geovistory - const enFromDefaultProject = await builder.joinProviderThroughDepIdx({ - leftTable: enFromProject.aggregation.tableDef, - joinWithDepIdx: this.depProClassLabel, - joinWhereLeftTableCondition: '= false', - joinOnKeys: { - fkClass: {leftCol: 'pkClass'}, - fkProject: {value: PK_DEFAULT_CONFIG_PROJECT}, - fkLanguage: {value: PK_ENGLISH}, - }, - conditionTrueIf: { - providerVal: {label: 'IS NOT NULL'} - }, - createAggregationVal: { - sql: (provider) => `jsonb_build_object('label',${provider}.val->>'label')`, - upsert: {whereCondition: '= true'} - }, - createCustomObject: (() => `t1.custom`) as CustomValSql<{fkLanguage: number}>, - }) - // from ontome - await builder.joinProviderThroughDepIdx({ - leftTable: enFromDefaultProject.aggregation.tableDef, - joinWhereLeftTableCondition: '= false', - joinWithDepIdx: this.depDfhClassLabel, - joinOnKeys: { - pkClass: {leftCol: 'pkClass'}, - language: {value: PK_ENGLISH}, - }, - conditionTrueIf: { - providerVal: {label: 'IS NOT NULL'} - }, - createAggregationVal: { - sql: (provider) => `jsonb_build_object('label',${provider}.val->>'label')`, - upsert: {whereCondition: '= true'} - } - }) - - builder.registerUpsertHook( - {client: client2, table: 'war.class_preview'}, - (insertClause, fromClause) => ` - INSERT INTO war.class_preview (fk_class, fk_project, label) - SELECT DISTINCT ON ("pkClass") "pkClass", "fkProject", val->>'label' - FROM ${fromClause} t1 - ON CONFLICT (fk_class, fk_project) DO UPDATE - SET label = EXCLUDED.label - WHERE EXCLUDED.label IS DISTINCT FROM war.class_preview.label` - ) - // await builder.printQueries() - const count = await builder.executeQueries() - return count - } - -} diff --git a/server/src/warehouse/aggregator-ds/class-label/r-class-label/RClassLabelAggregator.ts b/server/src/warehouse/aggregator-ds/class-label/r-class-label/RClassLabelAggregator.ts deleted file mode 100644 index e37f1d5bd..000000000 --- a/server/src/warehouse/aggregator-ds/class-label/r-class-label/RClassLabelAggregator.ts +++ /dev/null @@ -1,60 +0,0 @@ -import {AbstractAggregator} from '../../../base/classes/AbstractAggregator'; -import {RClassId} from '../../../primary-ds/DfhClassHasTypePropertyService'; -import {RClassLabelProviders} from './RClassLabelProviders'; -import {RClassLabelValue} from './RClassLabelService'; - -export class RClassLabelAggregator extends AbstractAggregator { - - - // the resulting label - classLabel = ''; - - // For testing / debugging - labelMissing = true; - - - - constructor( - public providers: RClassLabelProviders, - public id: RClassId - ) { - super() - } - - async create() { - - // // default language (en) - // const defaultLang = PK_ENGLISH; - - // class label - let classLabel: string | undefined; - - // /** - // * Try to get label in english - // */ - - // // from geovistory - // classLabel = (await this.providers.proClassLabel.get({ - // fkClass: this.id.pkClass, - // fkLanguage: defaultLang, - // fkProject: PK_DEFAULT_CONFIG_PROJECT - // }))?.label - - // if (classLabel) return this.finalize(classLabel); - - // // from ontome - // classLabel = (await this.providers.dfhClassLabel.get({ - // pkClass: this.id.pkClass, - // language: 'en' - // }))?.label - - return this.finalize(classLabel); - - } - - finalize(label?: string) { - return {label}; - } -} - - diff --git a/server/src/warehouse/aggregator-ds/class-label/r-class-label/RClassLabelDependencies.ts b/server/src/warehouse/aggregator-ds/class-label/r-class-label/RClassLabelDependencies.ts deleted file mode 100644 index ce01a2d8b..000000000 --- a/server/src/warehouse/aggregator-ds/class-label/r-class-label/RClassLabelDependencies.ts +++ /dev/null @@ -1,34 +0,0 @@ -// import {Dependencies} from '../../../base/classes/Dependencies' -// import {DependencyIndex} from '../../../base/classes/DependencyIndex' -// import {RClassId} from '../../../primary-ds/DfhClassHasTypePropertyService' -// import {DfhClassLabelId, DfhClassLabelVal} from '../../../primary-ds/DfhClassLabelService' -// import {ProClassLabelId, ProClassLabelVal} from '../../../primary-ds/ProClassLabelService' -// import {Warehouse} from '../../../Warehouse' -// import {RClassLabelValue} from './RClassLabelService' -// import {Injectable, Inject, forwardRef} from 'injection-js'; - -// @Injectable() -// export class RClassLabelDependencies extends Dependencies { -// dfhClassLabel: DependencyIndex -// proClassLabel: DependencyIndex - -// constructor(@Inject(forwardRef(() => Warehouse)) private wh: Warehouse) { -// super() - - -// this.dfhClassLabel = this.registerDepIdx(new DependencyIndex( -// wh, -// wh.agg.rClassLabel, -// wh.prim.dfhClassLabel, -// )) - -// this.proClassLabel = this.registerDepIdx(new DependencyIndex( -// wh, -// wh.agg.rClassLabel, -// wh.prim.proClassLabel, -// )) -// } - - - -// } diff --git a/server/src/warehouse/aggregator-ds/class-label/r-class-label/RClassLabelProviders.ts b/server/src/warehouse/aggregator-ds/class-label/r-class-label/RClassLabelProviders.ts deleted file mode 100644 index dd13f04eb..000000000 --- a/server/src/warehouse/aggregator-ds/class-label/r-class-label/RClassLabelProviders.ts +++ /dev/null @@ -1,22 +0,0 @@ -import {Provider} from '../../../base/classes/Provider'; -import {Providers} from '../../../base/interfaces/Providers'; -import {RClassId} from '../../../primary-ds/DfhClassHasTypePropertyService'; -import {DfhClassLabelId, DfhClassLabelVal} from '../../../primary-ds/DfhClassLabelService'; -import {ProClassLabelId, ProClassLabelVal} from '../../../primary-ds/ProClassLabelService'; -// import {RClassLabelDependencies} from './RClassLabelDependencies'; -import {RClassLabelValue} from './RClassLabelService'; - -export class RClassLabelProviders extends Providers { - dfhClassLabel: Provider - proClassLabel: Provider - - constructor( - // dep: RClassLabelDependencies, - protected receiverKey: RClassId - ) { - super() - // this.dfhClassLabel = this.registerProvider(dep.dfhClassLabel, receiverKey) - // this.proClassLabel = this.registerProvider(dep.proClassLabel, receiverKey); - } -} - diff --git a/server/src/warehouse/aggregator-ds/class-label/r-class-label/RClassLabelService.ts b/server/src/warehouse/aggregator-ds/class-label/r-class-label/RClassLabelService.ts deleted file mode 100644 index ab94891ca..000000000 --- a/server/src/warehouse/aggregator-ds/class-label/r-class-label/RClassLabelService.ts +++ /dev/null @@ -1,139 +0,0 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -import {forwardRef, Inject, Injectable} from 'injection-js'; -import {PoolClient} from 'pg'; -import {AggregatedDataService2} from '../../../base/classes/AggregatedDataService2'; -import {AggregatorSqlBuilder} from '../../../base/classes/AggregatorSqlBuilder'; -import {DependencyIndex} from '../../../base/classes/DependencyIndex'; -import {RClassService} from '../../../primary-ds/class/RClassService'; -import {RClassId, rClassIdKeyDefs} from '../../../primary-ds/DfhClassHasTypePropertyService'; -import {DfhClassLabelId, DfhClassLabelService, DfhClassLabelVal} from '../../../primary-ds/DfhClassLabelService'; -import {ProClassLabelId, ProClassLabelService, ProClassLabelVal} from '../../../primary-ds/ProClassLabelService'; -import {PK_DEFAULT_CONFIG_PROJECT, PK_ENGLISH, Warehouse} from '../../../Warehouse'; -import {RClassLabelAggregator} from './RClassLabelAggregator'; -import {RClassLabelProviders} from './RClassLabelProviders'; - -export type RClassLabelValue = {label?: string} - -@Injectable() -export class RClassLabelService extends AggregatedDataService2{ - aggregator = RClassLabelAggregator; - providers = RClassLabelProviders; - - dfhClassLabel: DependencyIndex - proClassLabel: DependencyIndex - batchSize = 100000; - constructor( - @Inject(forwardRef(() => Warehouse)) wh: Warehouse, - @Inject(forwardRef(() => RClassService)) rClass: RClassService, - @Inject(forwardRef(() => DfhClassLabelService)) dfhClassLabel: DfhClassLabelService, - @Inject(forwardRef(() => ProClassLabelService)) proClassLabel: ProClassLabelService - ) { - super( - wh, - rClassIdKeyDefs - ) - this.registerCreatorDS({dataService: rClass}); - - this.dfhClassLabel = this.addDepencency(dfhClassLabel) - this.proClassLabel = this.addDepencency(proClassLabel) - } - - async aggregateBatch(client: PoolClient, client2: PoolClient, limit: number, offset: number, currentTimestamp: string): Promise { - const builder = new AggregatorSqlBuilder(this, client, currentTimestamp, limit, offset) - - const geovistoryLabelsTw = await builder.joinProviderThroughDepIdx({ - leftTable: builder.batchTmpTable.tableDef, - joinWithDepIdx: this.proClassLabel, - joinOnKeys: { - fkClass: {leftCol: 'pkClass'}, - fkProject: {value: PK_DEFAULT_CONFIG_PROJECT}, - fkLanguage: {value: PK_ENGLISH}, - }, - conditionTrueIf: { - providerVal: {label: 'IS NOT NULL'} - }, - createAggregationVal: { - sql: (provider) => `jsonb_build_object('label',${provider}.val->>'label')`, - upsert: {whereCondition: '= true'} - } - }) - - if (!geovistoryLabelsTw.aggregation) throw new Error("aggUpsertTableDef missing"); - await builder.joinProviderThroughDepIdx({ - leftTable: geovistoryLabelsTw.aggregation.tableDef, - joinWhereLeftTableCondition: '= false', - joinWithDepIdx: this.dfhClassLabel, - joinOnKeys: { - pkClass: {leftCol: 'pkClass'}, - language: {value: PK_ENGLISH}, - }, - conditionTrueIf: { - providerVal: {label: 'IS NOT NULL'} - }, - createAggregationVal: { - sql: (provider) => `jsonb_build_object('label',${provider}.val->>'label')`, - upsert: {whereCondition: '= true'} - } - }) - - builder.registerUpsertHook( - {client: client2, table: 'war.class_preview'}, - (insertClause, fromClause) => ` - INSERT INTO war.class_preview (fk_class, fk_project, label) - SELECT DISTINCT ON ("pkClass") "pkClass", 0, val->>'label' - FROM ${fromClause} t1 - ON CONFLICT (fk_class, fk_project) DO UPDATE - SET label = EXCLUDED.label - WHERE EXCLUDED.label IS DISTINCT FROM war.class_preview.label` - ) - const count = await builder.executeQueries() - // const count = await builder.printQueries() - return count - } - - - -} - - - - - - - - - -// import {AggregatedDataService} from '../../../base/classes/AggregatedDataService'; -// import {RClassService} from '../../../primary-ds/class/RClassService'; -// import {RClassId, rClassIdKeyDefs} from '../../../primary-ds/DfhClassHasTypePropertyService'; -// import {Warehouse} from '../../../Warehouse'; -// import {RClassLabelAggregator} from './RClassLabelAggregator'; -// import {RClassLabelProviders} from './RClassLabelProviders'; - -// export type RClassLabelValue = {label?: string} -// @Injectable() -// export class RClassLabelService extends AggregatedDataService{ -// creatorDS: RClassService -// aggregator = RClassLabelAggregator; -// providers = RClassLabelProviders; -// constructor(@Inject(forwardRef(() => Warehouse)) wh: Warehouse) { -// super( -// wh, -// rClassIdKeyDefs -// ) -// this.registerCreatorDS(wh.prim.rClass); - -// } -// getDependencies() { -// return this.wh.dep.rClassLabel -// }; -// onUpsertSql(tableAlias: string) { -// return ` -// INSERT INTO war.class_preview (fk_class, fk_project, label) -// SELECT "pkClass", 0, val->>'label' -// FROM ${tableAlias} -// ON CONFLICT (fk_class, fk_project) DO UPDATE -// SET label = EXCLUDED.label -// WHERE EXCLUDED.label IS DISTINCT FROM war.class_preview.label` -// } -// } diff --git a/server/src/warehouse/aggregator-ds/entity-class-label/p-entity-class-label/PEntityClassLabelAggregator.ts b/server/src/warehouse/aggregator-ds/entity-class-label/p-entity-class-label/PEntityClassLabelAggregator.ts deleted file mode 100644 index c0b053091..000000000 --- a/server/src/warehouse/aggregator-ds/entity-class-label/p-entity-class-label/PEntityClassLabelAggregator.ts +++ /dev/null @@ -1,61 +0,0 @@ -import {AbstractAggregator} from '../../../base/classes/AbstractAggregator'; -import {PEntityId} from '../../../primary-ds/entity/PEntityService'; -import {PEntityClassLabelProviders} from './PEntityClassLabelPoviders'; -import {PEntityClassLabelVal} from './PEntityClassLabelService'; - -export class PEntityClassLabelAggregator extends AbstractAggregator { - - // the resulting entityClassLabel - entityClassLabel = '(no label)'; - - // For testing / debugging - labelMissing = true; - - constructor( - public providers: PEntityClassLabelProviders, - public id: PEntityId - ) { - super() - } - - - /************************************************************************ - * Methods for creating entity label - ************************************************************************/ - - /** - * Create entity label - * - * Gets values from Indexes and chaches dependencies in itself. - */ - async create() { - - const entity = await this.providers.pEntity.get(this.id); - - if (entity) { - - - const classId = { - fkProject: entity.fkProject, - pkClass: entity.fkClass - } - - const classLabel = await this.providers.pClassLabels.get(classId) - - if (classLabel?.label) { - this.labelMissing = false - this.entityClassLabel = classLabel.label; - } - - } - return this.finalize() - } - - - finalize(): PEntityClassLabelVal { - return { - entityClassLabel: this.entityClassLabel - } - } - -} diff --git a/server/src/warehouse/aggregator-ds/entity-class-label/p-entity-class-label/PEntityClassLabelDependencies.ts b/server/src/warehouse/aggregator-ds/entity-class-label/p-entity-class-label/PEntityClassLabelDependencies.ts deleted file mode 100644 index 6e1c07ab4..000000000 --- a/server/src/warehouse/aggregator-ds/entity-class-label/p-entity-class-label/PEntityClassLabelDependencies.ts +++ /dev/null @@ -1,35 +0,0 @@ -// import {Dependencies} from '../../../base/classes/Dependencies' -// import {DependencyIndex} from '../../../base/classes/DependencyIndex' -// import {PEntity, PEntityId} from '../../../primary-ds/entity/PEntityService' -// import {PClassId} from '../../../primary-ds/ProClassFieldsConfigService' -// import {Warehouse} from '../../../Warehouse' -// import {PClassLabelVal} from '../../class-label/p-class-label/PClassLabelService' -// import {PEntityClassLabelVal} from './PEntityClassLabelService' -// import {Injectable, Inject, forwardRef} from 'injection-js'; - -// @Injectable() -// export class PEntityClassLabelDependencies extends Dependencies { -// entity: DependencyIndex -// pClassLabel: DependencyIndex - -// constructor(@Inject(forwardRef(() => Warehouse)) private wh: Warehouse) { -// super() -// // stores the dependency of entityLabel (receiver) on entity (provider) -// this.entity = this.registerDepIdx(new DependencyIndex( -// this.wh, -// this.wh.agg.pEntityClassLabel, -// this.wh.prim.pEntity, -// )) -// // stores the dependency of pEntityLabel (receiver) on pClassLabel (provider) -// this.pClassLabel = this.registerDepIdx(new DependencyIndex( -// this.wh, -// this.wh.agg.pEntityClassLabel, -// this.wh.agg.pClassLabel, -// )) - - -// } - - - -// } diff --git a/server/src/warehouse/aggregator-ds/entity-class-label/p-entity-class-label/PEntityClassLabelPoviders.ts b/server/src/warehouse/aggregator-ds/entity-class-label/p-entity-class-label/PEntityClassLabelPoviders.ts deleted file mode 100644 index 0cfc9ad85..000000000 --- a/server/src/warehouse/aggregator-ds/entity-class-label/p-entity-class-label/PEntityClassLabelPoviders.ts +++ /dev/null @@ -1,21 +0,0 @@ -import {Provider} from '../../../base/classes/Provider'; -import {Providers} from "../../../base/interfaces/Providers"; -import {PEntity, PEntityId} from '../../../primary-ds/entity/PEntityService'; -import {PClassId} from '../../../primary-ds/ProClassFieldsConfigService'; -import {PClassLabelVal} from '../../class-label/p-class-label/PClassLabelService'; -import {PEntityClassLabelService, PEntityClassLabelVal} from './PEntityClassLabelService'; - -export class PEntityClassLabelProviders extends Providers { - pEntity: Provider; - pClassLabels: Provider; - constructor( - dep: PEntityClassLabelService, - protected receiverKey: PEntityId - ) { - super() - this.pEntity = this.registerProvider(dep.depPEntity, receiverKey) - this.pClassLabels = this.registerProvider(dep.depPClassLabel, receiverKey) - } - -} - diff --git a/server/src/warehouse/aggregator-ds/entity-class-label/p-entity-class-label/PEntityClassLabelService.ts b/server/src/warehouse/aggregator-ds/entity-class-label/p-entity-class-label/PEntityClassLabelService.ts deleted file mode 100644 index 51c7bdcf1..000000000 --- a/server/src/warehouse/aggregator-ds/entity-class-label/p-entity-class-label/PEntityClassLabelService.ts +++ /dev/null @@ -1,93 +0,0 @@ -import {forwardRef, Inject, Injectable} from 'injection-js'; -import {PoolClient} from 'pg'; -import {AggregatedDataService2} from '../../../base/classes/AggregatedDataService2'; -import {AggregatorSqlBuilder, CustomValSql} from '../../../base/classes/AggregatorSqlBuilder'; -import {DependencyIndex} from '../../../base/classes/DependencyIndex'; -import {PEntity, PEntityId, pEntityKeyDefs, PEntityService} from '../../../primary-ds/entity/PEntityService'; -import {PClassId} from '../../../primary-ds/ProClassFieldsConfigService'; -import {Warehouse} from '../../../Warehouse'; -import {PClassLabelService, PClassLabelVal} from '../../class-label/p-class-label/PClassLabelService'; -import {PEntityClassLabelAggregator} from './PEntityClassLabelAggregator'; -import {PEntityClassLabelProviders} from './PEntityClassLabelPoviders'; - -export interface PEntityClassLabelVal {entityClassLabel: string} -@Injectable() -export class PEntityClassLabelService extends AggregatedDataService2{ - aggregator = PEntityClassLabelAggregator; - providers = PEntityClassLabelProviders; - depPEntity: DependencyIndex - depPClassLabel: DependencyIndex - batchSize = 100000; - constructor( - @Inject(forwardRef(() => Warehouse)) wh: Warehouse, - @Inject(forwardRef(() => PEntityService)) pEntity: PEntityService, - @Inject(forwardRef(() => PClassLabelService)) pClassLabel: PClassLabelService - ) { - super( - wh, - pEntityKeyDefs - ) - this.registerCreatorDS({dataService: pEntity}) - this.depPEntity = this.addDepencency(pEntity) - this.depPClassLabel = this.addDepencency(pClassLabel) - } - - getDependencies() { - return this - }; - // onUpsertSql(tableAlias: string) { - // return ` - // UPDATE war.entity_preview - // SET class_label = val->>'entityClassLabel' - // FROM ${tableAlias} - // WHERE pk_entity = "pkEntity" - // AND project = "fkProject" - // AND class_label IS DISTINCT FROM val->>'entityClassLabel'` - // } - - - async aggregateBatch(client: PoolClient, client2: PoolClient, limit: number, offset: number, currentTimestamp: string): Promise { - const builder = new AggregatorSqlBuilder(this, client, currentTimestamp, limit, offset) - /** - * join entity - */ - const pentity = await builder.joinProviderThroughDepIdx({ - leftTable: builder.batchTmpTable.tableDef, - joinWithDepIdx: this.depPEntity, - joinOnKeys: { - pkEntity: {leftCol: 'pkEntity'}, - fkProject: {leftCol: 'fkProject'} - }, - conditionTrueIf: { - providerKey: {pkEntity: 'IS NOT NULL'} - }, - createCustomObject: (() => `jsonb_build_object('fkClass', (t2.val->>'fkClass')::int)`) as CustomValSql<{fkClass: number}>, - }) - - /** - * join pClassLabel, create value and upsert it - */ - await builder.joinProviderThroughDepIdx({ - leftTable: pentity.aggregation.tableDef, - joinWithDepIdx: this.depPClassLabel, - joinOnKeys: { - fkProject: {leftCol: 'fkProject'}, - pkClass: {leftCustom: {name: 'fkClass', type: 'int'}}, - }, - conditionTrueIf: { - providerKey: {pkClass: 'IS NOT NULL'} - }, - createAggregationVal: { - sql: () => `jsonb_build_object('entityClassLabel', t2.val->>'label')`, - upsert: true - } - - }) - - // await builder.printQueries() - const count = await builder.executeQueries() - return count - } - -} - diff --git a/server/src/warehouse/aggregator-ds/entity-class-label/r-entity-class-label/REntityClassLabelAggregator.ts b/server/src/warehouse/aggregator-ds/entity-class-label/r-entity-class-label/REntityClassLabelAggregator.ts deleted file mode 100644 index f80c192aa..000000000 --- a/server/src/warehouse/aggregator-ds/entity-class-label/r-entity-class-label/REntityClassLabelAggregator.ts +++ /dev/null @@ -1,56 +0,0 @@ -import {AbstractAggregator} from '../../../base/classes/AbstractAggregator'; -import {REntityId} from '../../../primary-ds/entity/REntityService'; -import {REntityClassLabelProviders} from './REntityClassLabelPoviders'; -import {RClassId} from '../../../primary-ds/DfhClassHasTypePropertyService'; -import {REntityClassLabelVal} from './REntityClassLabelService'; - -export class REntityClassLabelAggregator extends AbstractAggregator { - - // the resulting entityClassLabel - entityClassLabel?: string = '(no label)'; - - constructor( - public providers: REntityClassLabelProviders, - public id: REntityId - ) { - super() - } - - - /************************************************************************ - * Methods for creating entity label - ************************************************************************/ - - /** - * Create entity label - * - * Gets values from Indexes and chaches dependencies in itself. - */ - async create() { - - const entity = await this.providers.entity.get(this.id); - - if (entity) { - - const classId: RClassId = { - pkClass: entity.fkClass - } - - const classLabel = await this.providers.rClassLabels.get(classId) - - if (classLabel?.label) { - this.entityClassLabel = classLabel.label; - } - - } - return this.finalize() - } - - finalize(): REntityClassLabelVal { - return { - entityClassLabel: this.entityClassLabel - } - } - - -} diff --git a/server/src/warehouse/aggregator-ds/entity-class-label/r-entity-class-label/REntityClassLabelDependencies.ts b/server/src/warehouse/aggregator-ds/entity-class-label/r-entity-class-label/REntityClassLabelDependencies.ts deleted file mode 100644 index 793b13b44..000000000 --- a/server/src/warehouse/aggregator-ds/entity-class-label/r-entity-class-label/REntityClassLabelDependencies.ts +++ /dev/null @@ -1,33 +0,0 @@ -// import {Dependencies} from '../../../base/classes/Dependencies' -// import {DependencyIndex} from '../../../base/classes/DependencyIndex' -// import {RClassId} from '../../../primary-ds/DfhClassHasTypePropertyService' -// import {REntity, REntityId} from '../../../primary-ds/entity/REntityService' -// import {Warehouse} from '../../../Warehouse' -// import {RClassLabelValue} from '../../class-label/r-class-label/RClassLabelService' -// import {REntityClassLabelVal} from './REntityClassLabelService' -// import {Injectable, Inject, forwardRef} from 'injection-js'; - -// @Injectable() -// export class REntityClassLabelDependencies extends Dependencies { -// rEntity: DependencyIndex -// rClassLabel: DependencyIndex - -// constructor(@Inject(forwardRef(() => Warehouse)) private wh: Warehouse) { -// super() -// // stores the dependency of entityLabel (receiver) on entity (provider) -// this.rEntity = this.registerDepIdx(new DependencyIndex( -// this.wh, -// this.wh.agg.rEntityClassLabel, -// this.wh.prim.rEntity, -// )) -// // stores the dependency of rEntityLabel (receiver) on rClassLabel (provider) -// this.rClassLabel = this.registerDepIdx(new DependencyIndex( -// this.wh, -// this.wh.agg.rEntityClassLabel, -// this.wh.agg.rClassLabel, -// )); - - -// } - -// } diff --git a/server/src/warehouse/aggregator-ds/entity-class-label/r-entity-class-label/REntityClassLabelPoviders.ts b/server/src/warehouse/aggregator-ds/entity-class-label/r-entity-class-label/REntityClassLabelPoviders.ts deleted file mode 100644 index 7e4fe17eb..000000000 --- a/server/src/warehouse/aggregator-ds/entity-class-label/r-entity-class-label/REntityClassLabelPoviders.ts +++ /dev/null @@ -1,21 +0,0 @@ -import {Provider} from '../../../base/classes/Provider'; -import {Providers} from "../../../base/interfaces/Providers"; -import {RClassId} from '../../../primary-ds/DfhClassHasTypePropertyService'; -import {REntity, REntityId} from '../../../primary-ds/entity/REntityService'; -import {RClassLabelValue} from '../../class-label/r-class-label/RClassLabelService'; -import {REntityClassLabelService, REntityClassLabelVal} from './REntityClassLabelService'; - -export class REntityClassLabelProviders extends Providers { - entity: Provider; - rClassLabels: Provider; - constructor( - dep: REntityClassLabelService, - protected receiverKey: REntityId - ) { - super() - this.entity = this.registerProvider(dep.depREntity, receiverKey) - this.rClassLabels = this.registerProvider(dep.depRClassLabel, receiverKey) - } - -} - diff --git a/server/src/warehouse/aggregator-ds/entity-class-label/r-entity-class-label/REntityClassLabelService.ts b/server/src/warehouse/aggregator-ds/entity-class-label/r-entity-class-label/REntityClassLabelService.ts deleted file mode 100644 index e9b4bdd88..000000000 --- a/server/src/warehouse/aggregator-ds/entity-class-label/r-entity-class-label/REntityClassLabelService.ts +++ /dev/null @@ -1,90 +0,0 @@ -import {forwardRef, Inject, Injectable} from 'injection-js'; -import {AggregatedDataService2} from '../../../base/classes/AggregatedDataService2'; -import {DependencyIndex} from '../../../base/classes/DependencyIndex'; -import {RClassId} from '../../../primary-ds/DfhClassHasTypePropertyService'; -import {REntity, REntityId, rEntityKeyDefs, REntityService} from '../../../primary-ds/entity/REntityService'; -import {Warehouse} from '../../../Warehouse'; -import {RClassLabelService, RClassLabelValue} from '../../class-label/r-class-label/RClassLabelService'; -import {REntityClassLabelAggregator} from './REntityClassLabelAggregator'; -import {REntityClassLabelProviders} from './REntityClassLabelPoviders'; -import {PoolClient} from 'pg'; -import {AggregatorSqlBuilder, CustomValSql} from '../../../base/classes/AggregatorSqlBuilder'; - - -export interface REntityClassLabelVal {entityClassLabel?: string} -@Injectable() -export class REntityClassLabelService extends AggregatedDataService2{ - aggregator = REntityClassLabelAggregator; - providers = REntityClassLabelProviders; - depREntity: DependencyIndex - depRClassLabel: DependencyIndex - batchSize = 100000; - constructor( - @Inject(forwardRef(() => Warehouse)) wh: Warehouse, - @Inject(forwardRef(() => REntityService)) rEntity: REntityService, - @Inject(forwardRef(() => RClassLabelService)) rClassLabel: RClassLabelService - ) { - super( - wh, - rEntityKeyDefs - ) - this.registerCreatorDS({dataService: rEntity}) - this.depREntity = this.addDepencency(rEntity) - this.depRClassLabel = this.addDepencency(rClassLabel) - } - getDependencies() { - return this - }; - // onUpsertSql(tableAlias: string) { - // return ` - // UPDATE war.entity_preview - // SET class_label = val->>'entityClassLabel' - // FROM ${tableAlias} - // WHERE pk_entity = "pkEntity" - // AND project = 0 - // AND class_label IS DISTINCT FROM val->>'entityClassLabel'` - // } - - - async aggregateBatch(client: PoolClient, client2: PoolClient, limit: number, offset: number, currentTimestamp: string): Promise { - const builder = new AggregatorSqlBuilder(this, client, currentTimestamp, limit, offset) - /** - * join entity - */ - const pentity = await builder.joinProviderThroughDepIdx({ - leftTable: builder.batchTmpTable.tableDef, - joinWithDepIdx: this.depREntity, - joinOnKeys: { - pkEntity: {leftCol: 'pkEntity'}, - }, - conditionTrueIf: { - providerKey: {pkEntity: 'IS NOT NULL'} - }, - createCustomObject: (() => `jsonb_build_object('fkClass', (t2.val->>'fkClass')::int)`) as CustomValSql<{fkClass: number}>, - }) - - /** - * join pClassLabel, create value and upsert it - */ - await builder.joinProviderThroughDepIdx({ - leftTable: pentity.aggregation.tableDef, - joinWithDepIdx: this.depRClassLabel, - joinOnKeys: { - pkClass: {leftCustom: {name: 'fkClass', type: 'int'}}, - }, - conditionTrueIf: { - providerKey: {pkClass: 'IS NOT NULL'} - }, - createAggregationVal: { - sql: () => `jsonb_build_object('entityClassLabel', t2.val->>'label')`, - upsert: true - } - - }) - - // await builder.printQueries() - const count = await builder.executeQueries() - return count - } -} - diff --git a/server/src/warehouse/aggregator-ds/entity-full-text/entity-full-text.commons.ts b/server/src/warehouse/aggregator-ds/entity-full-text/entity-full-text.commons.ts deleted file mode 100644 index 63962856e..000000000 --- a/server/src/warehouse/aggregator-ds/entity-full-text/entity-full-text.commons.ts +++ /dev/null @@ -1,25 +0,0 @@ - -// Outgoing Properties not included in full text -const hiddenOut: {[key: string]: true} = { - 71: true, 72: true, 150: true, 151: true, 152: true, 153: true, // time span to time primitive - 1016: true, // F4 Manifestation Singleton → R42 is representative manifestation singleton for (has representative manifestation singleton) → F2 Expression - 1218: true, // E73 Information Object → P2 mentions → E1 CRM Entity (Quantifiers 0,n:0,n) - 1334: true, // C1 [Geovistory] Digital → P9 refers to (is referred to by) → E1 CRM Entity -} - -// Ingoing Properties not included in full text -const hiddenIn: {[key: string]: true} = { - 1316: true, // F2 Expression → P5 carrier provided by → F5 Item - 979: true, // F2 Expression → R4 carriers provided by (comprises carriers of) → F3 Manifestation Product Type - 1305: true, // F2 Expression → P4 is server response to request → C4 Web Request - 1218: true, // E73 Information Object → P2 mentions → E1 CRM Entity (Quantifiers 0,n:0,n) - 1334: true, // C1 [Geovistory] Digital → P9 refers to (is referred to by) → E1 CRM Entity -} - -export function isHiddenOutgoingProperty(pkProperty: string): boolean { - return hiddenOut[pkProperty] === true -} - -export function isHiddenIngoingProperty(pkProperty: string): boolean { - return hiddenIn[pkProperty] === true -} diff --git a/server/src/warehouse/aggregator-ds/entity-full-text/p-entity-full-text/PEntityFullTextAggregator.ts b/server/src/warehouse/aggregator-ds/entity-full-text/p-entity-full-text/PEntityFullTextAggregator.ts deleted file mode 100644 index 08f7d4c09..000000000 --- a/server/src/warehouse/aggregator-ds/entity-full-text/p-entity-full-text/PEntityFullTextAggregator.ts +++ /dev/null @@ -1,223 +0,0 @@ -import {AbstractAggregator} from '../../../base/classes/AbstractAggregator'; -import {Edge} from "../../../primary-ds/edge/edge.commons"; -import {PEntityId} from '../../../primary-ds/entity/PEntityService'; -import {PClassFieldLabelId} from '../../class-field-label/p-class-field-label/PClassFieldLabelService'; -import {PEntityFullTextProviders} from './PEntityFullTextPoviders'; -import {PEntityFullTextVal} from './PEntityFullTextService'; - -export interface ClassLabelConfig { - fkProperty: number, - isOutgoing: boolean, - ordNum: number - nrOfStatementsInLabel?: number -} - - -export class PEntityFullTextAggregator extends AbstractAggregator { - - // Defines the maximum number of statements per field - // taken into consideration for the fulltext - // Prevents from having enormous fulltext e.g. of Groups with thousands - // of memeberships. - readonly MAX_STMTS_PER_FIELD = 10; - - - fullTextArr: string[] = []; - fullText?: string; - - constructor( - public providers: PEntityFullTextProviders, - public id: PEntityId - ) { - super() - } - - - /************************************************************************ - * Methods for creating entity label - ************************************************************************/ - - /** - * Create entity label - * - * Gets values from Indexes and chaches dependencies in itself. - */ - async create() { - - // const entity = await this.providers.pEntity.get(this.id); - // if (!entity) return this.finalize(); - - // // get entity fields of that entity - // const edges = await this.providers.pEdges.get(this.id) - // if (!edges) return this.finalize(); - // // if no edges, return - - // // get fields of that class - // const pClassId: PClassId = {pkClass: entity.fkClass, fkProject: entity.fkProject} - // // first look for config of this project - // let classFields = await this.providers.pClassFields.get(pClassId) - - // if (!classFields) { - // // second look for config of default config project - // classFields = await this.providers.pClassFields.get({pkClass: pClassId.pkClass, fkProject: PK_DEFAULT_CONFIG_PROJECT}) - // } - - // // create fulltext - // const fullText = await this.loopOverFields(edges, classFields, entity.fkProject, entity.fkClass) - - // // get class label - // const res = await this.providers.pClassLabel.get(pClassId) - - // const classLabel = res?.label ?? `[${entity.fkClass}]`; - // this.fullText = `${classLabel} – ${fullText}`; - return this.finalize() - } - - finalize(): PEntityFullTextVal { - return { - fullText: this.fullText - } - } - /** - * This function loops over all fields. Per field: - * 1. it adds string elements to the full text of the entity - * 2. it adds all the edges (statements) to the entity - * - * The order of the iteration is defined so: - * - it first loops over fields of the class fields configuration in the given order of these fields - * - if edges are left, it loops over the remaining edges, grouped by field, in an arbitrary order - * - * @param entityFields - */ - // async loopOverFields(entityFields: EntityFields, classFields: ProClassFieldVal = [], fkProject: number, fkClass: number) { - - // const loopedCache: {[key: string]: boolean;} = {}; - - // // loop over the fields of this class config first - // const promises: Promise[] = [] - - // for (const cF of classFields) { - // const k = fieldKey(cF.fkProperty, cF.isOutgoing); - - // if (!loopedCache[k]) { - // // get the edges of that field - // const edges = entityFields?.[cF.isOutgoing ? 'outgoing' : 'incoming']?.[cF.fkProperty]; - - // // add edges to resulting entity - // promises.push(this.loopOverEdges(edges, fkProject, fkClass, cF.fkProperty, cF.isOutgoing)); - // } - - // // mark field as covered - // loopedCache[k] = true; - // } - - // // loop over the remaining fields of the entity (not covered by class config) - // const isOutgoing = true; - // for (const fkProperty in entityFields.outgoing) { - // // hidden outgoing - // if (!loopedCache[fieldKey(fkProperty, isOutgoing)] && !isHiddenOutgoingProperty(fkProperty)) { - // const edges = entityFields.outgoing[fkProperty]; - // promises.push(this.loopOverEdges(edges, fkProject, fkClass, parseInt(fkProperty, 10), isOutgoing)); - // } - - // } - // const isIncoming = false; - // for (const fkProperty in entityFields.incoming) { - - // if (!loopedCache[fieldKey(fkProperty, isIncoming)] && !isHiddenIngoingProperty(fkProperty)) { - // const edges = entityFields.incoming[fkProperty]; - // promises.push(this.loopOverEdges(edges, fkProject, fkClass, parseInt(fkProperty, 10), isIncoming)); - // } - // } - - // const res = await Promise.all(promises) - // return flatten(res).join(', '); - // } - - - /** - * Creates strings array for given field-edges (edges must be of same field, i.e. property + direction) - * - * example result: ["fieldLabel:", "edgeTargetLabel1", "edgeTargetLabel2", ...] - * - * @param edges Array of edges - * @param config ` - * fkProject: project - * fkClass: class of the entity - * fkProperty: property of edges/field (needed to get fieldLabel) - * isOutgoing: direction of edges/field (needed to get fieldLabel) - * addFieldLabel: if true, fieldLabel is added as first element of array - * singleQuotesForEdgeLabels: if true creates "'edgeTargetLabel1'" else "edgeTargetLabel1" - * maxLabelNr max number of edgeTargetLabels - * ` - */ - private async loopOverEdges( - edges: Edge[], - fkProject: number, - fkClass: number, - fkProperty: number, - isOutgoing: boolean - ) { - const result: string[] = []; - - - const fieldId: PClassFieldLabelId = {fkProject, fkClass, fkProperty, isOutgoing} - - // are there any edges? - if (edges?.length) { - - // stop iteration at smaller number of limit or length of edges - const stop = Math.min(this.MAX_STMTS_PER_FIELD, edges.length); - - for (let i = 0; i < stop; i++) { - const e = edges[i]; - let fieldLabel = ''; - - if (i === 0) { - // Get the fieldLabel (the label for property in correct direction) - const res = await this.providers.pClassFieldLabel.get(fieldId) - const l = res?.label ?? `[c${fkClass},p${fkProperty},${isOutgoing ? 'out' : 'in'},proj${fkProject}]` - fieldLabel = (`${l}: `); - } - - let targetLabel; - if (e.targetIsEntity) { - // Fetch project variant of related entity label - const pEntLabel = await this.providers.pEntityLabel.get({pkEntity: e.fkTarget, fkProject: fkProject}) - targetLabel = pEntLabel?.entityLabel - - // If needed, fetch repo variant of related entity label - if (!pEntLabel) { - const rEntLabel = await this.providers.rEntityLabel.get({pkEntity: e.fkTarget}) - targetLabel = rEntLabel?.entityLabel - } - - targetLabel = targetLabel ?? `[e${e.fkTarget}]` - // const targetEntityId: PEntityId = {fkProject, pkEntity: e.fkTarget} - // const fkTargetClass = (await this.providers.pEntity.get(targetEntityId))?.fkClass - - // const targetClassLabel = (fkTargetClass ? - // await this.providers.pClassLabel.get({fkProject, pkClass: fkTargetClass}) : - // false) ?? `[c${fkTargetClass}]`; - - // targetLabel = `${targetClassLabel} – ${targetLabel}` - - - } else { - targetLabel = e.targetLabel; - } - - result.push(`${fieldLabel}'${targetLabel}'`); - - } - } - return result; - } - - -} - -// function fieldKey(fkProperty: number | string, isOutgoing: boolean) { -// return fkProperty + '_' + isOutgoing; -// } - diff --git a/server/src/warehouse/aggregator-ds/entity-full-text/p-entity-full-text/PEntityFullTextDependencies.ts b/server/src/warehouse/aggregator-ds/entity-full-text/p-entity-full-text/PEntityFullTextDependencies.ts deleted file mode 100644 index b053f6659..000000000 --- a/server/src/warehouse/aggregator-ds/entity-full-text/p-entity-full-text/PEntityFullTextDependencies.ts +++ /dev/null @@ -1,75 +0,0 @@ -// import {Dependencies} from '../../../base/classes/Dependencies' -// import {DependencyIndex} from '../../../base/classes/DependencyIndex' -// import {EntityFields} from "../../../primary-ds/edge/edge.commons" -// import {PEntity, PEntityId} from '../../../primary-ds/entity/PEntityService' -// import {REntityId} from '../../../primary-ds/entity/REntityService' -// import {PClassId, ProClassFieldVal} from '../../../primary-ds/ProClassFieldsConfigService' -// import {Warehouse} from '../../../Warehouse' -// import {PClassFieldLabelId, PClassFieldLabelVal} from '../../class-field-label/p-class-field-label/PClassFieldLabelService' -// import {PClassLabelVal} from '../../class-label/p-class-label/PClassLabelService' -// import {EntityLabelVal} from '../../entity-label/entity-label.commons' -// import {PEntityFullTextVal} from './PEntityFullTextService' -// import {Injectable, Inject, forwardRef} from 'injection-js'; - -// @Injectable() -// export class PEntityFullTextDependencies extends Dependencies { -// pEntity: DependencyIndex -// pEntityLabel: DependencyIndex -// rEntityLabel: DependencyIndex -// pEdge: DependencyIndex -// pClassLabel: DependencyIndex -// pClassFields: DependencyIndex -// pClassFieldLabel: DependencyIndex - -// constructor(@Inject(forwardRef(() => Warehouse)) private wh: Warehouse) { -// super() -// // stores the dependency of entityFullText (receiver) on pEntity (provider) -// this.pEntity = this.registerDepIdx(new DependencyIndex( -// this.wh, -// this.wh.agg.pEntityFullText, -// this.wh.prim.pEntity, -// )) - -// // stores the dependency of entityFullText (receiver) on pEntityLabel (provider) -// this.pEntityLabel = this.registerDepIdx(new DependencyIndex( -// this.wh, -// this.wh.agg.pEntityFullText, -// this.wh.agg.pEntityLabel, -// )); - -// // stores the dependency of entityFullText (receiver) on rEntityLabel (provider) -// this.rEntityLabel = this.registerDepIdx(new DependencyIndex( -// this.wh, -// this.wh.agg.pEntityFullText, -// this.wh.agg.rEntityLabel, -// )); - -// // stores the dependency of entityFullText (receiver) on pEdge (provider) -// this.pEdge = this.registerDepIdx(new DependencyIndex( -// this.wh, -// this.wh.agg.pEntityFullText, -// this.wh.prim.pEdge, -// )); - -// // stores the dependency of entityFullText (receiver) on dfhClassHasTypeProperty -// this.pClassFields = this.registerDepIdx(new DependencyIndex( -// this.wh, -// this.wh.agg.pEntityFullText, -// this.wh.prim.pClassFieldsConfig, -// )); - -// // stores the dependency of entityFullText (receiver) on project class labels -// this.pClassLabel = this.registerDepIdx(new DependencyIndex( -// this.wh, -// this.wh.agg.pEntityFullText, -// this.wh.agg.pClassLabel, -// )); - -// this.pClassFieldLabel = this.registerDepIdx(new DependencyIndex( -// this.wh, -// this.wh.agg.pEntityFullText, -// this.wh.agg.pClassFieldLabel, -// )); -// } - -// } diff --git a/server/src/warehouse/aggregator-ds/entity-full-text/p-entity-full-text/PEntityFullTextPoviders.ts b/server/src/warehouse/aggregator-ds/entity-full-text/p-entity-full-text/PEntityFullTextPoviders.ts deleted file mode 100644 index 0c068860a..000000000 --- a/server/src/warehouse/aggregator-ds/entity-full-text/p-entity-full-text/PEntityFullTextPoviders.ts +++ /dev/null @@ -1,36 +0,0 @@ -import {Provider} from '../../../base/classes/Provider'; -import {Providers} from "../../../base/interfaces/Providers"; -import {EntityFields} from "../../../primary-ds/edge/edge.commons"; -import {PEntity, PEntityId} from '../../../primary-ds/entity/PEntityService'; -import {REntityId} from '../../../primary-ds/entity/REntityService'; -import {PClassId, ProClassFieldVal} from '../../../primary-ds/ProClassFieldsConfigService'; -import {PClassFieldLabelId, PClassFieldLabelVal} from '../../class-field-label/p-class-field-label/PClassFieldLabelService'; -import {PClassLabelVal} from '../../class-label/p-class-label/PClassLabelService'; -import {EntityLabelVal} from '../../entity-label/entity-label.commons'; -import {PEntityFullTextService, PEntityFullTextVal} from './PEntityFullTextService'; - -export class PEntityFullTextProviders extends Providers { - pEntity: Provider; - pEdges: Provider; - pEntityLabel: Provider; - rEntityLabel: Provider; - pClassLabel: Provider; - pClassFields: Provider; - pClassFieldLabel: Provider; - - constructor( - dep: PEntityFullTextService, - protected receiverKey: PEntityId - ) { - super() - this.pEntity = this.registerProvider(dep.depPEntity, receiverKey) - this.pEntityLabel = this.registerProvider(dep.depPEntityLabel, receiverKey); - this.rEntityLabel = this.registerProvider(dep.depREntityLabel, receiverKey); - this.pEdges = this.registerProvider(dep.depPEdge, receiverKey) - this.pClassLabel = this.registerProvider(dep.depPClassLabel, receiverKey) - this.pClassFields = this.registerProvider(dep.depPClassFields, receiverKey) - this.pClassFieldLabel = this.registerProvider(dep.depPClassFieldLabel, receiverKey) - } - -} - diff --git a/server/src/warehouse/aggregator-ds/entity-full-text/p-entity-full-text/PEntityFullTextService.ts b/server/src/warehouse/aggregator-ds/entity-full-text/p-entity-full-text/PEntityFullTextService.ts deleted file mode 100644 index 7f6167bf8..000000000 --- a/server/src/warehouse/aggregator-ds/entity-full-text/p-entity-full-text/PEntityFullTextService.ts +++ /dev/null @@ -1,519 +0,0 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -import {forwardRef, Inject, Injectable} from 'injection-js'; -import {AggregatedDataService2} from '../../../base/classes/AggregatedDataService2'; -import {DependencyIndex} from '../../../base/classes/DependencyIndex'; -import {EntityFields} from '../../../primary-ds/edge/edge.commons'; -import {PEdgeService} from '../../../primary-ds/edge/PEdgeService'; -import {PEntity, PEntityId, pEntityKeyDefs, PEntityService} from '../../../primary-ds/entity/PEntityService'; -import {REntityId} from '../../../primary-ds/entity/REntityService'; -import {PClassId, ProClassFieldsConfigService, ProClassFieldVal} from '../../../primary-ds/ProClassFieldsConfigService'; -import {Warehouse, PK_DEFAULT_CONFIG_PROJECT} from '../../../Warehouse'; -import {PClassFieldLabelId, PClassFieldLabelService, PClassFieldLabelVal} from '../../class-field-label/p-class-field-label/PClassFieldLabelService'; -import {PClassLabelService, PClassLabelVal} from '../../class-label/p-class-label/PClassLabelService'; -import {EntityLabelVal} from '../../entity-label/entity-label.commons'; -import {PEntityLabelService} from '../../entity-label/p-entity-label/PEntityLabelService'; -import {REntityLabelService} from '../../entity-label/r-entity-label/REntityLabelService'; -import {PEntityFullTextAggregator} from './PEntityFullTextAggregator'; -import {PEntityFullTextProviders} from './PEntityFullTextPoviders'; -import {PoolClient} from 'pg'; -import {AggregatorSqlBuilder, CustomValSql} from '../../../base/classes/AggregatorSqlBuilder'; - -export interface PEntityFullTextVal {fullText?: string}; - -/** - * This Data Service manages the key-value store containing - * as a key the PEntityId (pkEntity and fkProject) - * and as value the PEntityFullTextVal (fkType, typeLabel) - * - * One example key-value pair in the this.index is: - * Key for the Project Entity Geo. Place 'Madrid' with pkEntity = 2002 in fkProject = 3001 - * - '2002_3001' - * - * Val for the Geo. Place Type 'City' with pkEntity = 2003 in fkProject = 3001 - * - fkType: 2003 - * - typeLabel: 'Citiy' - * - * - * - * -> The Val is the result of the PEntityFullTextAggregator - * - */ -@Injectable() -export class PEntityFullTextService extends AggregatedDataService2{ - aggregator = PEntityFullTextAggregator; - providers = PEntityFullTextProviders; - depPEntity: DependencyIndex - depPEntityLabel: DependencyIndex - depREntityLabel: DependencyIndex - depPEdge: DependencyIndex - depPClassLabel: DependencyIndex - depPClassFields: DependencyIndex - depPClassFieldLabel: DependencyIndex - batchSize = 100000; - constructor( - @Inject(forwardRef(() => Warehouse)) wh: Warehouse, - @Inject(forwardRef(() => PEntityService)) pEntity: PEntityService, - @Inject(forwardRef(() => PEntityLabelService)) pEntityLabel: PEntityLabelService, - @Inject(forwardRef(() => REntityLabelService)) rEntityLabel: REntityLabelService, - @Inject(forwardRef(() => PEdgeService)) pEdge: PEdgeService, - @Inject(forwardRef(() => PClassLabelService)) pClassLabel: PClassLabelService, - @Inject(forwardRef(() => ProClassFieldsConfigService)) pClassFields: ProClassFieldsConfigService, - @Inject(forwardRef(() => PClassFieldLabelService)) pClassFieldLabel: PClassFieldLabelService, - ) { - super( - wh, - pEntityKeyDefs - ) - this.registerCreatorDS({dataService: pEntity}) - - this.depPEntity = this.addDepencency(pEntity); - this.depPEntityLabel = this.addDepencency(pEntityLabel); - this.depREntityLabel = this.addDepencency(rEntityLabel); - this.depPEdge = this.addDepencency(pEdge); - this.depPClassLabel = this.addDepencency(pClassLabel); - this.depPClassFields = this.addDepencency(pClassFields); - this.depPClassFieldLabel = this.addDepencency(pClassFieldLabel); - } - - - getDependencies() { - return this - }; - // onUpsertSql(tableAlias: string) { - // return ` - // UPDATE war.entity_preview - // SET full_text = val->>'fullText' - // FROM ${tableAlias} - // WHERE pk_entity = "pkEntity" - // AND project = "fkProject" - // AND full_text IS DISTINCT FROM val->>'fullText'` - // } - async aggregateBatch(client: PoolClient, client2: PoolClient, limit: number, offset: number, currentTimestamp: string): Promise { - const builder = new AggregatorSqlBuilder(this, client, currentTimestamp, limit, offset) - const pEntity = await builder.joinProviderThroughDepIdx({ - leftTable: builder.batchTmpTable.tableDef, - joinWithDepIdx: this.depPEntity, - joinOnKeys: { - fkProject: {leftCol: 'fkProject'}, - pkEntity: {leftCol: 'pkEntity'} - }, - conditionTrueIf: { - providerKey: {pkEntity: 'IS NOT NULL'} - }, - createCustomObject: (() => `jsonb_build_object('fkClass', (t2.val->>'fkClass')::int)`) as CustomValSql<{fkClass: number}>, - }) - - const pEdges = await builder.joinProviderThroughDepIdx({ - leftTable: pEntity.aggregation.tableDef, - joinWithDepIdx: this.depPEdge, - joinWhereLeftTableCondition: '= true', - joinOnKeys: { - fkProject: {leftCol: 'fkProject'}, - pkEntity: {leftCol: 'pkEntity'} - }, - conditionTrueIf: { - or: [ - {providerVal: {outgoing: 'is not {}', }}, - {providerVal: {incoming: 'is not {}', }}, - ] - }, - createCustomObject: ((provider) => `jsonb_build_object( - 'fkClass', (t1.custom->>'fkClass')::int, - 'edges', t2.val - ) - `) as CustomValSql<{ - fkClass: number, - edges: EntityFields - }>, - }) - - // first look for config of this project - const projectClassFields = await builder.joinProviderThroughDepIdx({ - leftTable: pEntity.aggregation.tableDef, - joinWithDepIdx: this.depPClassFields, - joinWhereLeftTableCondition: '= true', - joinOnKeys: { - fkProject: {leftCol: 'fkProject'}, - pkClass: {leftCustom: {name: 'fkClass', type: 'int'}} - }, - conditionTrueIf: { - providerKey: {pkClass: 'IS NOT NULL'} - }, - createCustomObject: ((provider) => `jsonb_build_object( - 'fkClass', (t1.custom->>'fkClass')::int, - 'classFields', t2.val - ) - `) as CustomValSql<{ - fkClass: number, - edges: EntityFields, - classFields: ProClassFieldVal - }>, - }) - - // second look for config of default config project - const defaultClassFields = await builder.joinProviderThroughDepIdx({ - leftTable: projectClassFields.aggregation.tableDef, - joinWithDepIdx: this.depPClassFields, - joinWhereLeftTableCondition: '= false', - joinOnKeys: { - fkProject: {value: PK_DEFAULT_CONFIG_PROJECT}, - pkClass: {leftCustom: {name: 'fkClass', type: 'int'}} - }, - conditionTrueIf: { - providerKey: {pkClass: 'IS NOT NULL'} - }, - createCustomObject: ((provider) => `jsonb_build_object( - 'fkClass', (t1.custom->>'fkClass')::int, - 'classFields', t2.val - ) - `) as CustomValSql<{ - fkClass: number, - edges: EntityFields, - classFields: ProClassFieldVal - }>, - }) - - /** - * expand entity fields (statements grouped by property and direction) - */ - const entityFieldsTbl = builder.createTableName(); - const createEntityFieldsTbl = builder.createTableStmt(entityFieldsTbl) - const entityFieldsSql = ` - -- expand entity fields (statements grouped by property and direction) - ${createEntityFieldsTbl} ( - SELECT - t1."r_pkEntity", - t1."r_fkProject", - jsonb_build_object ( - 'fk_class', t1.custom->>'fkClass', - 'fk_property', jsonb_object_keys(t1.custom->'edges'->'incoming'), - 'direction', 'incoming', - 'is_outgoing', false - ) custom - FROM ${pEdges.aggregation.tableDef.tableName} t1 - UNION ALL - SELECT - t1."r_pkEntity", - t1."r_fkProject", - jsonb_build_object ( - 'fk_class', t1.custom->>'fkClass', - 'fk_property', jsonb_object_keys(t1.custom->'edges'->'outgoing'), - 'direction', 'outgoing', - 'is_outgoing', true - ) custom - FROM ${pEdges.aggregation.tableDef.tableName} t1 - )` - const entityFields = builder.registerTmpTable<{ - pkEntity: number, - fkProject: number, - }, { - fk_class: number, - fk_property: number, - direction: 'outgoing' | 'incoming', - is_outgoing: boolean - }, never>(entityFieldsSql, [], entityFieldsTbl) - - - const classFieldLabelCustom: CustomValSql<{ - pkEntity: number - fkProject: number - fkClass: number - fkProperty: number - isOutgoing: boolean - }> = () => { - return `jsonb_build_object( - 'pkEntity', t1."r_pkEntity", - 'fkProject', t1."r_fkProject", - 'fkClass', t1.custom->>'fk_class', - 'fkProperty', t1.custom->>'fk_property', - 'isOutgoing', t1.custom->>'is_outgoing', - 'direction', t1.custom->>'direction', - 'classFieldLabel', t2.val->>'label' - )` - } - - /** - * join class field labels throug dep index - */ - const classFieldLabel = await builder.joinProviderThroughDepIdx({ - leftTable: entityFields.tableDef, - joinWithDepIdx: this.depPClassFieldLabel, - joinOnKeys: { - fkProject: {leftCol: 'fkProject'}, - fkClass: {leftCustom: {name: 'fk_class', type: 'int'}}, - fkProperty: {leftCustom: {name: 'fk_property', type: 'int'}}, - isOutgoing: {leftCustom: {name: 'is_outgoing', type: 'bool'}}, - }, - conditionTrueIf: { - providerVal: {label: 'IS NOT NULL'} - }, - createCustomObject: classFieldLabelCustom - }) - - /** - * expand edges (max. 10 per entity field) and join with field order - */ - const edgesTbl = builder.createTableName(); - const createEdgesTbl = builder.createTableStmt(edgesTbl) - const edgesSql = ` - -- expand entity fields (statements grouped by property and direction) - ${createEdgesTbl} ( - WITH tw1 AS ( - -- left join class fields (to get ord number) - SELECT - t1. *, - t1.custom ->> 'fkClass' fk_class, - CASE - WHEN t2.custom ->> 'classFields' IS NOT NULL THEN t2.custom -> 'classFields' - ELSE t3.custom -> 'classFields' - END class_fields - FROM - ${pEdges.aggregation.tableDef.tableName} t1 - LEFT JOIN ${projectClassFields.aggregation.tableDef.tableName} t2 ON t1. "r_pkEntity" = t2. "r_pkEntity" - AND t1. "r_fkProject" = t2. "r_fkProject" - LEFT JOIN ${defaultClassFields.aggregation.tableDef.tableName} t3 ON t1. "r_pkEntity" = t3. "r_pkEntity" - AND t1. "r_fkProject" = t3. "r_fkProject" - ) - SELECT - -- statements and ord number of field - t1. "r_pkEntity", - t1. "r_fkProject", - jsonb_build_object( - 'fkClass', t1.fk_class, - 'fkProperty', t2.custom ->> 'fkProperty', - 'isOutgoing', t2.custom ->> 'isOutgoing', - 'classFieldLabel', t2.custom->>'classFieldLabel', - 'fieldOrder', t1.class_fields -> (t2.custom ->> 'direction') -> (t2.custom ->> 'fkProperty') -> 'ordNum', - 'stmtOrder', x.stmt_order, - 'edge', x.edge, - 'fkTarget', x.edge->>'fkTarget' - ) custom, - (x.edge->>'targetIsEntity')::bool condition - FROM - tw1 t1 - JOIN ${classFieldLabel.aggregation.tableDef.tableName} t2 ON t1. "r_pkEntity" = (t2.custom ->> 'pkEntity') :: int - AND t1. "r_fkProject" = (t2.custom ->> 'fkProject') :: int - cross join lateral ( - select - * - from - jsonb_array_elements( - t1.custom -> 'edges' -> (t2.custom ->> 'direction') -> (t2.custom ->> 'fkProperty') - ) with ordinality as x(edge, stmt_order) - order by - x.stmt_order - limit - 10 -- maximally 10 - ) x - ) - ` - const edges = builder.registerTmpTable<{ - pkEntity: number, - fkProject: number, - }, { - fkClass: number, - fkProperty: number, - isOutgoing: boolean, - classFieldLabel?: string, - fieldOrder?: number, - stmtOrder?: number, - fkTarget?: number - }, never>(edgesSql, [], edgesTbl) - - - - /** - * join pEntityLabel - */ - const pEntityLabel = await builder.joinProviderThroughDepIdx({ - leftTable: edges.tableDef, - joinWithDepIdx: this.depPEntityLabel, - joinWhereLeftTableCondition: '= true', - joinOnKeys: { - fkProject: {leftCol: 'fkProject'}, - pkEntity: {leftCustom: {name: 'fkTarget', type: 'int'}} - }, - conditionTrueIf: { - providerKey: {pkEntity: 'IS NOT NULL'} - }, - createCustomObject: ((provider) => `jsonb_insert(t1.custom, '{targetEntityLabel}', t2.val->'entityLabel') - `) as CustomValSql<{ - fkClass: number, - fkProperty: number, - isOutgoing: boolean, - classFieldLabel?: string, - fieldOrder?: number, - stmtOrder?: number, - fkTarget?: number, - targetEntityLabel?: string - }>, - }) - /** - * join rEntityLabel where no pEntityLabel - */ - const rEntityLabel = await builder.joinProviderThroughDepIdx({ - leftTable: pEntityLabel.aggregation.tableDef, - joinWithDepIdx: this.depREntityLabel, - joinWhereLeftTableCondition: '= false', - joinOnKeys: { - pkEntity: {leftCustom: {name: 'fkTarget', type: 'int'}} - }, - conditionTrueIf: { - providerKey: {pkEntity: 'IS NOT NULL'} - }, - createCustomObject: ((provider) => `jsonb_insert(t1.custom, '{targetEntityLabel}', t2.val->'entityLabel') - `) as CustomValSql<{ - fkClass: number, - fkProperty: number, - isOutgoing: boolean, - classFieldLabel?: string, - fieldOrder?: number, - stmtOrder?: number, - fkTarget?: number, - targetEntityLabel?: string - }> - }) - /** - * aggregate texts provided by edges (directed statements) - */ - const textsTbl = builder.createTableName(); - const createTextsTbl = builder.createTableStmt(textsTbl) - const textsSql = ` - -- aggregate texts provided by edges (directed statements) - ${createTextsTbl} ( - WITH tw1 AS( - SELECT - t2.custom->>'classFieldLabel' ||': '|| string_agg( - '''' || - coalesce( - t2.custom->'edge'->>'targetLabel', - t3.custom->>'targetEntityLabel', - t4.custom->>'targetEntityLabel' - ) - || '''' - , ', ' - ORDER BY - (t2.custom->'edge'->>'fieldOrder')::int - ) as field_text, - t1."r_pkEntity", - t1."r_fkProject", - t2.custom->'fkProperty', - t2.custom->'isOutgoing', - t5.custom->>'fkClass' fk_class, - (t2.custom ->> 'fieldOrder') :: int fieldOrder - FROM ${builder.batchTmpTable.tableDef.tableName} t1 - LEFT JOIN ${edges.tableDef.tableName} t2 - ON t1."r_pkEntity" = t2."r_pkEntity" - AND t1."r_fkProject" = t2."r_fkProject" - LEFT JOIN ${pEntityLabel.aggregation.tableDef.tableName} t3 - ON (t2.custom->'edge'->>'fkStatement')::int = (t3.custom->'edge'->>'fkStatement')::int - AND (t2.custom->'edge'->>'fkTarget')::int = (t3.custom->'edge'->>'fkTarget')::int - AND t2."r_fkProject" = t3."r_fkProject" - LEFT JOIN ${rEntityLabel.aggregation.tableDef.tableName} t4 - ON (t2.custom->'edge'->>'fkStatement')::int = (t4.custom->'edge'->>'fkStatement')::int - AND (t2.custom->'edge'->>'fkTarget')::int = (t4.custom->'edge'->>'fkTarget')::int - AND t2."r_fkProject" = t3."r_fkProject" - LEFT JOIN ${pEntity.aggregation.tableDef.tableName} t5 - ON t1. "r_pkEntity" = t5. "r_pkEntity" - AND t1. "r_fkProject" = t5. "r_fkProject" - GROUP BY - t1."r_pkEntity", - t1."r_fkProject", - t5.custom->>'fkClass', - t2.custom->'fkProperty', - t2.custom->'isOutgoing', - t2.custom->>'classFieldLabel', - t2.custom ->> 'fieldOrder' - ) - -- group by entity - - SELECT - t2."r_pkEntity", - t2."r_fkProject", - jsonb_build_object( - 'fkClass', t2.fk_class, - 'texts', string_agg(field_text, ', ' ORDER BY t2.fieldOrder :: int ASC) - ) custom, - t2.fk_class IS NOT NULL condition - FROM tw1 t2 - GROUP BY - t2."r_pkEntity", - t2."r_fkProject", - t2.fk_class - ) - ` - const texts = builder.registerTmpTable<{ - pkEntity: number, - fkProject: number, - }, - never, - { - texts: string, - fkClass: string - } - >(textsSql, [], textsTbl) - - /** - * join pClassLabel - */ - const classLabel = await builder.joinProviderThroughDepIdx({ - leftTable: texts.tableDef, - joinWithDepIdx: this.depPClassLabel, - joinOnKeys: { - fkProject: {leftCol: 'fkProject'}, - pkClass: {leftCustom: {name: 'fkClass', type: 'int'}}, - }, - conditionTrueIf: { - providerVal: {label: 'IS NOT NULL'} - }, - createCustomObject: ((provider) => `jsonb_insert(t1.custom, '{classLabel}', t2.val->'label') - `) as CustomValSql<{ - fkClass: number, - fkProperty: number, - isOutgoing: boolean, - classFieldLabel?: string, - fieldOrder?: number, - stmtOrder?: number, - fkTarget?: number, - targetEntityLabel?: string - classLabel?: string - }> - }) - /** - * create final table concatenating class label and texts from edges - */ - const finalTbl = builder.createTableName(); - const createfinalTbl = builder.createTableStmt(finalTbl) - const finalSql = ` - -- create final table - ${createfinalTbl} ( - SELECT - "r_pkEntity", - "r_fkProject", - jsonb_build_object( - 'fullText', (custom->>'classLabel' )::text ||' – ' || (custom->>'texts' )::text - ) val - FROM ${classLabel.aggregation.tableDef.tableName} - ) - ` - const final = builder.registerTmpTable<{ - pkEntity: number, - fkProject: number, - }, - PEntityFullTextVal, - never - >(finalSql, [], finalTbl) - - - - - await builder.tmpTableUpsertAggregations(this.index, final.tableDef.tableName) - - // await builder.printQueries() - const count = builder.executeQueries() - - return count - - } -} - diff --git a/server/src/warehouse/aggregator-ds/entity-full-text/r-entity-full-text/REntityFullTextAggregator.ts b/server/src/warehouse/aggregator-ds/entity-full-text/r-entity-full-text/REntityFullTextAggregator.ts deleted file mode 100644 index 62c2f9566..000000000 --- a/server/src/warehouse/aggregator-ds/entity-full-text/r-entity-full-text/REntityFullTextAggregator.ts +++ /dev/null @@ -1,212 +0,0 @@ -import {AbstractAggregator} from '../../../base/classes/AbstractAggregator'; -import {Edge} from "../../../primary-ds/edge/edge.commons"; -import {REntityId} from '../../../primary-ds/entity/REntityService'; -import {RClassFieldId} from '../../class-field-label/r-class-field-label/RClassFieldLabelService'; -import {REntityFullTextProviders} from './REntityFullTextPoviders'; -import {REntityFullTextVal} from './REntityFullTextService'; - -export interface ClassLabelConfig { - fkProperty: number, - isOutgoing: boolean, - ordNum: number - nrOfStatementsInLabel?: number -} - - -export class REntityFullTextAggregator extends AbstractAggregator { - - // Defines the maximum number of statements per field - // taken into consideration for the fulltext - // Prevents from having enormous fulltext e.g. of Groups with thousands - // of memeberships. - readonly MAX_STMTS_PER_FIELD = 10; - - - fullTextArr: string[] = []; - fullText?: string; - - constructor( - public providers: REntityFullTextProviders, - public id: REntityId - ) { - super() - } - - - /************************************************************************ - * Methods for creating entity label - ************************************************************************/ - - /** - * Create entity label - * - * Gets values from Indexes and chaches dependencies in itself. - */ - async create() { - - const entity = await this.providers.rEntity.get(this.id); - if (!entity) return this.finalize(); - - // get entity fields of that entity - const edges = await this.providers.rEdges.get(this.id) - if (!edges) return this.finalize(); - // if no edges, return - - // // get fields of that class - // const rClassId: RClassId = {pkClass: entity.fkClass} - // // look for config of efault config project - // const classFields = await this.providers.pClassFields.get({pkClass: rClassId.pkClass, fkProject: PK_DEFAULT_CONFIG_PROJECT}) - - // // create fulltext - // const fullText = await this.loopOverFields(edges, classFields, entity.fkClass) - - // // get class label - // const res = await this.providers.rClassLabel.get(rClassId) - - // const classLabel = res?.label ?? `[${entity.fkClass}]`; - // this.fullText = `${classLabel} – ${fullText}`; - return this.finalize() - } - - finalize(): REntityFullTextVal { - return { - fullText: this.fullText - } - } - - /** - * This function loops over all fields. Per field: - * 1. it adds string elements to the full text of the entity - * 2. it adds all the edges (statements) to the entity - * - * The order of the iteration is defined so: - * - it first loops over fields of the class fields configuration in the given order of these fields - * - if edges are left, it loops over the remaining edges, grouped by field, in an arbitrary order - * - * @param entityFields - */ - // async loopOverFields(entityFields: EntityFields, classFields: ProClassFieldVal = [], fkClass: number) { - - // const loopedCache: {[key: string]: boolean;} = {}; - - // // loop over the fields of this class config first - // const promises: Promise[] = [] - - // for (const cF of classFields) { - // const k = fieldKey(cF.fkProperty, cF.isOutgoing); - - // if (!loopedCache[k]) { - // // get the edges of that field - // const edges = entityFields?.[cF.isOutgoing ? 'outgoing' : 'incoming']?.[cF.fkProperty]; - - // // add edges to resulting entity - // promises.push(this.loopOverEdges(edges, fkClass, cF.fkProperty, cF.isOutgoing)); - // } - - // // mark field as covered - // loopedCache[k] = true; - // } - - // // loop over the remaining fields of the entity (not covered by class config) - // const isOutgoing = true; - // for (const fkProperty in entityFields.outgoing) { - - // if (!loopedCache[fieldKey(fkProperty, isOutgoing)] && !isHiddenOutgoingProperty(fkProperty)) { - // const edges = entityFields.outgoing[fkProperty]; - // promises.push(this.loopOverEdges(edges, fkClass, parseInt(fkProperty, 10), isOutgoing)); - // } - - // } - // const isIncoming = false; - // for (const fkProperty in entityFields.incoming) { - - // if (!loopedCache[fieldKey(fkProperty, isIncoming)] && !isHiddenIngoingProperty(fkProperty)) { - // const edges = entityFields.incoming[fkProperty]; - // promises.push(this.loopOverEdges(edges, fkClass, parseInt(fkProperty, 10), isIncoming)); - // } - // } - - // const res = await Promise.all(promises) - // return flatten(res).join(', '); - // } - - - - /** - * Creates strings array for given field-edges (edges must be of same field, i.e. property + direction) - * - * example result: ["fieldLabel:", "edgeTargetLabel1", "edgeTargetLabel2", ...] - * - * @param edges Array of edges - * @param config ` - * fkProject: project - * fkClass: class of the entity - * fkProperty: property of edges/field (needed to get fieldLabel) - * isOutgoing: direction of edges/field (needed to get fieldLabel) - * addFieldLabel: if true, fieldLabel is added as first element of array - * singleQuotesForEdgeLabels: if true creates "'edgeTargetLabel1'" else "edgeTargetLabel1" - * maxLabelNr max number of edgeTargetLabels - * ` - */ - private async loopOverEdges( - edges: Edge[], - fkClass: number, - fkProperty: number, - isOutgoing: boolean - ) { - const result: string[] = []; - - - const fieldId: RClassFieldId = {fkClass, fkProperty, isOutgoing} - - // are there any edges? - if (edges?.length) { - - // stop iteration at smaller number of limit or length of edges - const stop = Math.min(this.MAX_STMTS_PER_FIELD, edges.length); - - for (let i = 0; i < stop; i++) { - const e = edges[i]; - let fieldLabel = ''; - - if (i === 0) { - // Get the fieldLabel (the label for property in correct direction) - const res = await this.providers.rClassFieldLabel.get(fieldId) - const l = res?.label ?? `[c${fkClass},p${fkProperty},${isOutgoing ? 'out' : 'in'},repo]` - fieldLabel = (`${l}: `); - } - - let targetLabel; - if (e.targetIsEntity) { - // Fetch repo variant of related entity label - const rEntLabel = await this.providers.rEntityLabel.get({pkEntity: e.fkTarget}) - targetLabel = rEntLabel?.entityLabel ?? `[e${e.fkTarget}]` - - // const targetEntityId: REntityId = {fkProject, pkEntity: e.fkTarget} - // const fkTargetClass = (await this.providers.rEntity.get(targetEntityId))?.fkClass - - // const targetClassLabel = (fkTargetClass ? - // await this.providers.pClassLabel.get({fkProject, pkClass: fkTargetClass}) : - // false) ?? `[c${fkTargetClass}]`; - - // targetLabel = `${targetClassLabel} – ${targetLabel}` - - - } else { - targetLabel = e.targetLabel; - } - - result.push(`${fieldLabel}'${targetLabel}'`); - - } - } - return result; - } - - -} - -// function fieldKey(fkProperty: number | string, isOutgoing: boolean) { -// return fkProperty + '_' + isOutgoing; -// } - diff --git a/server/src/warehouse/aggregator-ds/entity-full-text/r-entity-full-text/REntityFullTextDependencies.ts b/server/src/warehouse/aggregator-ds/entity-full-text/r-entity-full-text/REntityFullTextDependencies.ts deleted file mode 100644 index b602a87bd..000000000 --- a/server/src/warehouse/aggregator-ds/entity-full-text/r-entity-full-text/REntityFullTextDependencies.ts +++ /dev/null @@ -1,67 +0,0 @@ -// import {Dependencies} from '../../../base/classes/Dependencies' -// import {DependencyIndex} from '../../../base/classes/DependencyIndex' -// import {RClassId} from '../../../primary-ds/DfhClassHasTypePropertyService' -// import {EntityFields} from "../../../primary-ds/edge/edge.commons" -// import {REntity, REntityId} from '../../../primary-ds/entity/REntityService' -// import {PClassId, ProClassFieldVal} from '../../../primary-ds/ProClassFieldsConfigService' -// import {Warehouse} from '../../../Warehouse' -// import {RClassFieldId, RClassFieldVal} from '../../class-field-label/r-class-field-label/RClassFieldLabelService' -// import {RClassLabelValue} from '../../class-label/r-class-label/RClassLabelService' -// import {EntityLabelVal} from '../../entity-label/entity-label.commons' -// import {REntityFullTextVal} from './REntityFullTextService' -// import {Injectable, Inject, forwardRef} from 'injection-js'; - -// @Injectable() -// export class REntityFullTextDependencies extends Dependencies { -// rEntity: DependencyIndex -// rEntityLabel: DependencyIndex -// rEdge: DependencyIndex -// rClassLabel: DependencyIndex -// rClassFieldLabel: DependencyIndex - -// pClassFields: DependencyIndex - -// constructor(@Inject(forwardRef(() => Warehouse)) private wh: Warehouse) { -// super() -// // stores the dependency of entityFullText (receiver) on rEntity (provider) -// this.rEntity = this.registerDepIdx(new DependencyIndex( -// this.wh, -// this.wh.agg.rEntityFullText, -// this.wh.prim.rEntity, -// )) - -// // stores the dependency of entityFullText (receiver) on rEntityLabel (provider) -// this.rEntityLabel = this.registerDepIdx(new DependencyIndex( -// this.wh, -// this.wh.agg.rEntityFullText, -// this.wh.agg.rEntityLabel, -// )); - -// // stores the dependency of entityFullText (receiver) on rEdge (provider) -// this.rEdge = this.registerDepIdx(new DependencyIndex( -// this.wh, -// this.wh.agg.rEntityFullText, -// this.wh.prim.rEdge, -// )); - -// // stores the dependency of entityFullText (receiver) on dfhClassHasTypeProperty -// this.pClassFields = this.registerDepIdx(new DependencyIndex( -// this.wh, -// this.wh.agg.rEntityFullText, -// this.wh.prim.pClassFieldsConfig, -// )); - -// // stores the dependency of entityFullText (receiver) on project class labels -// this.rClassLabel = this.registerDepIdx(new DependencyIndex( -// this.wh, -// this.wh.agg.rEntityFullText, -// this.wh.agg.rClassLabel, -// )); - -// this.rClassFieldLabel = this.registerDepIdx(new DependencyIndex( -// this.wh, -// this.wh.agg.rEntityFullText, -// this.wh.agg.rClassFieldLabel, -// )); -// } -// } diff --git a/server/src/warehouse/aggregator-ds/entity-full-text/r-entity-full-text/REntityFullTextPoviders.ts b/server/src/warehouse/aggregator-ds/entity-full-text/r-entity-full-text/REntityFullTextPoviders.ts deleted file mode 100644 index 6636bda98..000000000 --- a/server/src/warehouse/aggregator-ds/entity-full-text/r-entity-full-text/REntityFullTextPoviders.ts +++ /dev/null @@ -1,36 +0,0 @@ -import {Provider} from '../../../base/classes/Provider'; -import {Providers} from "../../../base/interfaces/Providers"; -import {RClassId} from '../../../primary-ds/DfhClassHasTypePropertyService'; -import {EntityFields} from "../../../primary-ds/edge/edge.commons"; -import {REntity, REntityId} from '../../../primary-ds/entity/REntityService'; -import {PClassId, ProClassFieldVal} from '../../../primary-ds/ProClassFieldsConfigService'; -import {RClassFieldId, RClassFieldVal} from '../../class-field-label/r-class-field-label/RClassFieldLabelService'; -import {RClassLabelValue} from '../../class-label/r-class-label/RClassLabelService'; -import {EntityLabelVal} from '../../entity-label/entity-label.commons'; -import {REntityFullTextVal, REntityFullTextService} from './REntityFullTextService'; - -export class REntityFullTextProviders extends Providers { - rEntity: Provider; - rEdges: Provider; - rEntityLabel: Provider; - rClassLabel: Provider; - rClassFieldLabel: Provider; - - pClassFields: Provider; - - constructor( - dep: REntityFullTextService, - protected receiverKey: REntityId - ) { - super() - this.rEntity = this.registerProvider(dep.depREntity, receiverKey) - this.rEntityLabel = this.registerProvider(dep.depREntityLabel, receiverKey); - this.rEdges = this.registerProvider(dep.depREdge, receiverKey) - this.rClassLabel = this.registerProvider(dep.depRClassLabel, receiverKey) - this.rClassFieldLabel = this.registerProvider(dep.depRClassFieldLabel, receiverKey) - - this.pClassFields = this.registerProvider(dep.depPClassFields, receiverKey) - } - -} - diff --git a/server/src/warehouse/aggregator-ds/entity-full-text/r-entity-full-text/REntityFullTextService.ts b/server/src/warehouse/aggregator-ds/entity-full-text/r-entity-full-text/REntityFullTextService.ts deleted file mode 100644 index 0e544fb22..000000000 --- a/server/src/warehouse/aggregator-ds/entity-full-text/r-entity-full-text/REntityFullTextService.ts +++ /dev/null @@ -1,441 +0,0 @@ -import {forwardRef, Inject, Injectable} from 'injection-js'; -import {PoolClient} from 'pg'; -import {AggregatedDataService2} from '../../../base/classes/AggregatedDataService2'; -import {AggregatorSqlBuilder, CustomValSql} from '../../../base/classes/AggregatorSqlBuilder'; -import {DependencyIndex} from '../../../base/classes/DependencyIndex'; -import {RClassId} from '../../../primary-ds/DfhClassHasTypePropertyService'; -import {EntityFields} from '../../../primary-ds/edge/edge.commons'; -import {REdgeService} from '../../../primary-ds/edge/REdgeService'; -import {REntity, REntityId, rEntityKeyDefs, REntityService} from '../../../primary-ds/entity/REntityService'; -import {PClassId, ProClassFieldsConfigService, ProClassFieldVal} from '../../../primary-ds/ProClassFieldsConfigService'; -import {PK_DEFAULT_CONFIG_PROJECT, Warehouse} from '../../../Warehouse'; -import {RClassFieldId, RClassFieldLabelService, RClassFieldVal} from '../../class-field-label/r-class-field-label/RClassFieldLabelService'; -import {RClassLabelService, RClassLabelValue} from '../../class-label/r-class-label/RClassLabelService'; -import {EntityLabelVal} from '../../entity-label/entity-label.commons'; -import {REntityLabelService} from '../../entity-label/r-entity-label/REntityLabelService'; -import {REntityFullTextAggregator} from './REntityFullTextAggregator'; -import {REntityFullTextProviders} from './REntityFullTextPoviders'; - -export interface REntityFullTextVal {fullText?: string}; - -/** - * This Data Service manages the key-value store containing - * as a key the REntityId (pkEntity and fkProject) - * and as value the REntityFullTextVal (fkType, typeLabel) - * - * One example key-value pair in the this.index is: - * Key for the Project Entity Geo. Place 'Madrid' with pkEntity = 2002 in fkProject = 3001 - * - '2002_3001' - * - * Val for the Geo. Place Type 'City' with pkEntity = 2003 in fkProject = 3001 - * - fkType: 2003 - * - typeLabel: 'Citiy' - * - * - * - * -> The Val is the result of the REntityFullTextAggregator - * - */ -@Injectable() -export class REntityFullTextService extends AggregatedDataService2{ - // creatorDS: REntityService - aggregator = REntityFullTextAggregator; - providers = REntityFullTextProviders; - depREntity: DependencyIndex - depREntityLabel: DependencyIndex - depREdge: DependencyIndex - depRClassLabel: DependencyIndex - depRClassFieldLabel: DependencyIndex - depPClassFields: DependencyIndex - batchSize = 100000; - - constructor( - @Inject(forwardRef(() => Warehouse)) wh: Warehouse, - @Inject(forwardRef(() => REntityService)) rEntity: REntityService, - @Inject(forwardRef(() => REntityLabelService)) rEntityLabel: REntityLabelService, - @Inject(forwardRef(() => REdgeService)) rEdge: REdgeService, - @Inject(forwardRef(() => RClassLabelService)) rClassLabel: RClassLabelService, - @Inject(forwardRef(() => RClassFieldLabelService)) rClassFieldLabel: RClassFieldLabelService, - @Inject(forwardRef(() => ProClassFieldsConfigService)) pClassFields: ProClassFieldsConfigService, - ) { - super( - wh, - rEntityKeyDefs - ) - this.registerCreatorDS({dataService: rEntity}) - - this.depREntity = this.addDepencency(rEntity); - this.depREntityLabel = this.addDepencency(rEntityLabel); - this.depREdge = this.addDepencency(rEdge); - this.depRClassLabel = this.addDepencency(rClassLabel); - this.depRClassFieldLabel = this.addDepencency(rClassFieldLabel); - this.depPClassFields = this.addDepencency(pClassFields); - } - - - getDependencies() { - return this - }; - // onUpsertSql(tableAlias: string) { - // return ` - // UPDATE war.entity_preview - // SET full_text = val->>'fullText' - // FROM ${tableAlias} - // WHERE pk_entity = "pkEntity" - // AND project = 0 - // AND full_text IS DISTINCT FROM val->>'fullText'` - // } - async aggregateBatch(client: PoolClient, client2: PoolClient, limit: number, offset: number, currentTimestamp: string): Promise { - const builder = new AggregatorSqlBuilder(this, client, currentTimestamp, limit, offset) - const rEntity = await builder.joinProviderThroughDepIdx({ - leftTable: builder.batchTmpTable.tableDef, - joinWithDepIdx: this.depREntity, - joinOnKeys: { - pkEntity: {leftCol: 'pkEntity'} - }, - conditionTrueIf: { - providerKey: {pkEntity: 'IS NOT NULL'} - }, - createCustomObject: (() => `jsonb_build_object('fkClass', (t2.val->>'fkClass')::int)`) as CustomValSql<{fkClass: number}>, - }) - - const rEdges = await builder.joinProviderThroughDepIdx({ - leftTable: rEntity.aggregation.tableDef, - joinWithDepIdx: this.depREdge, - joinWhereLeftTableCondition: '= true', - joinOnKeys: { - pkEntity: {leftCol: 'pkEntity'} - }, - conditionTrueIf: { - or: [ - {providerVal: {outgoing: 'is not {}', }}, - {providerVal: {incoming: 'is not {}', }}, - ] - }, - createCustomObject: ((provider) => `jsonb_build_object( - 'fkClass', (t1.custom->>'fkClass')::int, - 'edges', t2.val - ) - `) as CustomValSql<{ - fkClass: number, - edges: EntityFields - }>, - }) - - // first look for config of this project - const classFields = await builder.joinProviderThroughDepIdx({ - leftTable: rEntity.aggregation.tableDef, - joinWithDepIdx: this.depPClassFields, - joinWhereLeftTableCondition: '= true', - joinOnKeys: { - fkProject: {value: PK_DEFAULT_CONFIG_PROJECT}, - pkClass: {leftCustom: {name: 'fkClass', type: 'int'}} - }, - conditionTrueIf: { - providerKey: {pkClass: 'IS NOT NULL'} - }, - createCustomObject: ((provider) => `jsonb_build_object( - 'fkClass', (t1.custom->>'fkClass')::int, - 'classFields', t2.val - ) - `) as CustomValSql<{ - fkClass: number, - edges: EntityFields, - classFields: ProClassFieldVal - }>, - }) - - - /** - * expand entity fields (statements grouped by property and direction) - */ - const entityFieldsTbl = builder.createTableName(); - const createEntityFieldsTbl = builder.createTableStmt(entityFieldsTbl) - const entityFieldsSql = ` - -- expand entity fields (statements grouped by property and direction) - ${createEntityFieldsTbl} ( - SELECT - t1."r_pkEntity", - jsonb_build_object ( - 'fk_class', t1.custom->>'fkClass', - 'fk_property', jsonb_object_keys(t1.custom->'edges'->'incoming'), - 'direction', 'incoming', - 'is_outgoing', false - ) custom - FROM ${rEdges.aggregation.tableDef.tableName} t1 - UNION ALL - SELECT - t1."r_pkEntity", - jsonb_build_object ( - 'fk_class', t1.custom->>'fkClass', - 'fk_property', jsonb_object_keys(t1.custom->'edges'->'outgoing'), - 'direction', 'outgoing', - 'is_outgoing', true - ) custom - FROM ${rEdges.aggregation.tableDef.tableName} t1 - )` - const entityFields = builder.registerTmpTable<{ - pkEntity: number, - fkProject: number, - }, { - fk_class: number, - fk_property: number, - direction: 'outgoing' | 'incoming', - is_outgoing: boolean - }, never>(entityFieldsSql, [], entityFieldsTbl) - - - const classFieldLabelCustom: CustomValSql<{ - pkEntity: number - fkProject: number - fkClass: number - fkProperty: number - isOutgoing: boolean - }> = () => { - return `jsonb_build_object( - 'pkEntity', t1."r_pkEntity", - 'fkClass', t1.custom->>'fk_class', - 'fkProperty', t1.custom->>'fk_property', - 'isOutgoing', t1.custom->>'is_outgoing', - 'direction', t1.custom->>'direction', - 'classFieldLabel', t2.val->>'label' - )` - } - - /** - * join class field labels throug dep index - */ - const classFieldLabel = await builder.joinProviderThroughDepIdx({ - leftTable: entityFields.tableDef, - joinWithDepIdx: this.depRClassFieldLabel, - joinOnKeys: { - fkClass: {leftCustom: {name: 'fk_class', type: 'int'}}, - fkProperty: {leftCustom: {name: 'fk_property', type: 'int'}}, - isOutgoing: {leftCustom: {name: 'is_outgoing', type: 'bool'}}, - }, - conditionTrueIf: { - providerVal: {label: 'IS NOT NULL'} - }, - createCustomObject: classFieldLabelCustom - }) - - /** - * expand edges (max. 10 per entity field) and join with field order - */ - const edgesTbl = builder.createTableName(); - const createEdgesTbl = builder.createTableStmt(edgesTbl) - const edgesSql = ` - -- expand entity fields (statements grouped by property and direction) - ${createEdgesTbl} ( - WITH tw1 AS ( - -- left join class fields (to get ord number) - SELECT - t1. *, - t1.custom ->> 'fkClass' fk_class, - t2.custom -> 'classFields' class_fields - FROM - ${rEdges.aggregation.tableDef.tableName} t1 - LEFT JOIN ${classFields.aggregation.tableDef.tableName} t2 - ON t1. "r_pkEntity" = t2. "r_pkEntity" - - ) - SELECT - -- statements and ord number of field - t1. "r_pkEntity", - jsonb_build_object( - 'fkClass', t1.fk_class, - 'fkProperty', t2.custom ->> 'fkProperty', - 'isOutgoing', t2.custom ->> 'isOutgoing', - 'classFieldLabel', t2.custom->>'classFieldLabel', - 'fieldOrder', t1.class_fields -> (t2.custom ->> 'direction') -> (t2.custom ->> 'fkProperty') -> 'ordNum', - 'stmtOrder', x.stmt_order, - 'edge', x.edge, - 'fkTarget', x.edge->>'fkTarget' - ) custom, - (x.edge->>'targetIsEntity')::bool condition - FROM - tw1 t1 - JOIN ${classFieldLabel.aggregation.tableDef.tableName} t2 ON t1. "r_pkEntity" = (t2.custom ->> 'pkEntity') :: int - cross join lateral ( - select - * - from - jsonb_array_elements( - t1.custom -> 'edges' -> (t2.custom ->> 'direction') -> (t2.custom ->> 'fkProperty') - ) with ordinality as x(edge, stmt_order) - order by - x.stmt_order - limit - 10 -- maximally 10 - ) x - ) - ` - const edges = builder.registerTmpTable<{ - pkEntity: number, - fkProject: number, - }, { - fkClass: number, - fkProperty: number, - isOutgoing: boolean, - classFieldLabel?: string, - fieldOrder?: number, - stmtOrder?: number, - fkTarget?: number - }, never>(edgesSql, [], edgesTbl) - - - - /** - * join rEntityLabel - */ - const rEntityLabel = await builder.joinProviderThroughDepIdx({ - leftTable: edges.tableDef, - joinWithDepIdx: this.depREntityLabel, - joinWhereLeftTableCondition: '= true', - joinOnKeys: { - pkEntity: {leftCustom: {name: 'fkTarget', type: 'int'}} - }, - conditionTrueIf: { - providerKey: {pkEntity: 'IS NOT NULL'} - }, - createCustomObject: ((provider) => `jsonb_insert(t1.custom, '{targetEntityLabel}', t2.val->'entityLabel') - `) as CustomValSql<{ - fkClass: number, - fkProperty: number, - isOutgoing: boolean, - classFieldLabel?: string, - fieldOrder?: number, - stmtOrder?: number, - fkTarget?: number, - targetEntityLabel?: string - }>, - }) - - /** - * aggregate texts provided by edges (directed statements) - */ - const textsTbl = builder.createTableName(); - const createTextsTbl = builder.createTableStmt(textsTbl) - const textsSql = ` - -- aggregate texts provided by edges (directed statements) - ${createTextsTbl} ( - WITH tw1 AS( - SELECT - t2.custom->>'classFieldLabel' ||': '|| string_agg( - '''' || - coalesce( - t2.custom->'edge'->>'targetLabel', - t3.custom->>'targetEntityLabel' - ) - || '''' - , ', ' - ORDER BY - (t2.custom->'edge'->>'fieldOrder')::int - ) as field_text, - t1."r_pkEntity", - t2.custom->'fkProperty', - t2.custom->'isOutgoing', - t5.custom->>'fkClass' fk_class, - (t2.custom ->> 'fieldOrder') :: int fieldOrder - FROM ${builder.batchTmpTable.tableDef.tableName} t1 - LEFT JOIN ${edges.tableDef.tableName} t2 - ON t1."r_pkEntity" = t2."r_pkEntity" - LEFT JOIN ${rEntityLabel.aggregation.tableDef.tableName} t3 - ON (t2.custom->'edge'->>'fkStatement')::int = (t3.custom->'edge'->>'fkStatement')::int - AND (t2.custom->'edge'->>'fkTarget')::int = (t3.custom->'edge'->>'fkTarget')::int - LEFT JOIN ${rEntity.aggregation.tableDef.tableName} t5 - ON t1. "r_pkEntity" = t5. "r_pkEntity" - GROUP BY - t1."r_pkEntity", - t5.custom->>'fkClass', - t2.custom->'fkProperty', - t2.custom->'isOutgoing', - t2.custom->>'classFieldLabel', - t2.custom ->> 'fieldOrder' - ) - -- group by entity - - SELECT - t2."r_pkEntity", - jsonb_build_object( - 'fkClass', t2.fk_class, - 'texts', string_agg(field_text, ', ' ORDER BY t2.fieldOrder :: int ASC) - ) custom, - t2.fk_class IS NOT NULL condition - FROM tw1 t2 - GROUP BY - t2."r_pkEntity", - t2.fk_class - ) - ` - const texts = builder.registerTmpTable<{ - pkEntity: number, - fkProject: number, - }, - never, - { - texts: string, - fkClass: string - } - >(textsSql, [], textsTbl) - - /** - * join pClassLabel - */ - const classLabel = await builder.joinProviderThroughDepIdx({ - leftTable: texts.tableDef, - joinWithDepIdx: this.depRClassLabel, - joinOnKeys: { - pkClass: {leftCustom: {name: 'fkClass', type: 'int'}}, - }, - conditionTrueIf: { - providerVal: {label: 'IS NOT NULL'} - }, - createCustomObject: ((provider) => `jsonb_insert(t1.custom, '{classLabel}', t2.val->'label') - `) as CustomValSql<{ - fkClass: number, - fkProperty: number, - isOutgoing: boolean, - classFieldLabel?: string, - fieldOrder?: number, - stmtOrder?: number, - fkTarget?: number, - targetEntityLabel?: string - classLabel?: string - }> - }) - /** - * create final table concatenating class label and texts from edges - */ - const finalTbl = builder.createTableName(); - const createfinalTbl = builder.createTableStmt(finalTbl) - const finalSql = ` - -- create final table - ${createfinalTbl} ( - SELECT - "r_pkEntity", - jsonb_build_object( - 'fullText', (custom->>'classLabel' )::text ||' – ' || (custom->>'texts' )::text - ) val - FROM ${classLabel.aggregation.tableDef.tableName} - ) - ` - const final = builder.registerTmpTable< - REntityId, - REntityFullTextVal, - never - >(finalSql, [], finalTbl) - - - - - await builder.tmpTableUpsertAggregations(this.index, final.tableDef.tableName) - - // await builder.printQueries() - const count = builder.executeQueries() - - return count - - } -} - diff --git a/server/src/warehouse/aggregator-ds/entity-label/entity-label.commons.ts b/server/src/warehouse/aggregator-ds/entity-label/entity-label.commons.ts deleted file mode 100644 index 4bf49a1aa..000000000 --- a/server/src/warehouse/aggregator-ds/entity-label/entity-label.commons.ts +++ /dev/null @@ -1,74 +0,0 @@ -import {LabelPart} from '../../../models'; -import {createInfResource} from '../../../__tests__/helpers/atomic/inf-resource.helper'; -import {InfResourceMock} from '../../../__tests__/helpers/data/gvDB/InfResourceMock'; -import {createProInfoProjRel} from '../../../__tests__/helpers/atomic/pro-info-proj-rel.helper'; -import {ProInfoProjRelMock} from '../../../__tests__/helpers/data/gvDB/ProInfoProjRelMock'; -import {createInfStatement} from '../../../__tests__/helpers/atomic/inf-statement.helper'; -import {InfStatementMock} from '../../../__tests__/helpers/data/gvDB/InfStatementMock'; -import {createProEntityLabelConfig} from '../../../__tests__/helpers/atomic/pro-entity-label-config.helper'; -import {ProEntityLabelConfigMock} from '../../../__tests__/helpers/data/gvDB/ProEntityLabelConfigMock'; -import {createDfhApiClass} from '../../../__tests__/helpers/atomic/dfh-api-class.helper'; -import {createDfhApiProperty} from '../../../__tests__/helpers/atomic/dfh-api-property.helper'; -import {DfhApiClassMock} from '../../../__tests__/helpers/data/gvDB/DfhApiClassMock'; -import {DfhApiPropertyMock} from '../../../__tests__/helpers/data/gvDB/DfhApiPropertyMock'; - -export interface EntityLabelVal { - entityLabel?: string; - labelMissing: boolean; -} -export interface LabelPartKeys { - pkEntity: number, - property: number - direction: string - fielOrdNum: number - stmtOrdNum: number - condition: boolean - fkEntity: number -} -export interface LabelPartCustom { - targetLabel: string, - fkTarget: number, -} - -export const labelPartsForAppeInLang365: LabelPart[] = [ - { - ordNum: 1, - field: { - fkProperty: 1113, - isOutgoing: true, - nrOfStatementsInLabel: 1 - } - } -] -export const labelPartsForNormalEntities: LabelPart[] = [ - { - ordNum: 1, - field: { - fkProperty: 1111, - isOutgoing: false, - nrOfStatementsInLabel: 1 - } - } -] - -export const ENTITY_LABEL_MAX_LENGTH = 311; - - -export namespace EntityLabel { - export async function createInfinitLabel() { - await createDfhApiClass(DfhApiClassMock.EN_61_BIRTH); - await createDfhApiProperty(DfhApiPropertyMock.EN_1435_STEMS_FROM); - // TeEn - const birth = await createInfResource(InfResourceMock.BIRTH_1); - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_BIRTH); - - // Stmts - await createInfStatement(InfStatementMock.BIRTH_1_BROUGHT_INTO_LIFE_PERSON_1); - await createProInfoProjRel(ProInfoProjRelMock.PROJ_1_STMT_BIRTH_1_BROUGHT_INTO_LIFE_PERON_1); - - await createProEntityLabelConfig(ProEntityLabelConfigMock.C61_BIRTH_PROJECT_DEFAULT) - await createProEntityLabelConfig(ProEntityLabelConfigMock.C21_PERSON_INFINIT_LABEL_PROJECT_DEFAULT) - return {birth}; - } - -} diff --git a/server/src/warehouse/aggregator-ds/entity-label/p-entity-label/PEntityLabelAggregator.ts b/server/src/warehouse/aggregator-ds/entity-label/p-entity-label/PEntityLabelAggregator.ts deleted file mode 100644 index 659d16c24..000000000 --- a/server/src/warehouse/aggregator-ds/entity-label/p-entity-label/PEntityLabelAggregator.ts +++ /dev/null @@ -1,253 +0,0 @@ -// import {AbstractAggregator} from '../../../base/classes/AbstractAggregator'; -// import {EntityLabelConfigVal, LabelPart} from '../../../primary-ds/ProEntityLabelConfigService'; -// import {PClassId} from '../../../primary-ds/ProClassFieldsConfigService'; -// import {Edge, EntityFields} from "../../../primary-ds/edge/edge.commons"; -// import {PEntityId} from '../../../primary-ds/entity/PEntityService'; -// import {PK_DEFAULT_CONFIG_PROJECT} from '../../../Warehouse'; -// import {PEntityLabelProviders} from './PEntityLabelPoviders'; -// import {keys} from 'lodash'; -// import {EntityLabelVal} from '../entity-label.commons'; - -// export interface ClassLabelConfig { -// fkProperty: number, -// isOutgoing: boolean, -// ordNum: number -// nrOfStatementsInLabel?: number -// } - - -// export class PEntityLabelAggregator extends AbstractAggregator { - -// // array of strings to create label -// labelArr: string[] = []; - -// // the resulting entityLabel -// entityLabel = '(no label)'; - -// // For testing / debugging -// labelMissing = true; - -// constructor( -// public providers: PEntityLabelProviders, -// public id: PEntityId -// ) { -// super() -// } - - -// /************************************************************************ -// * Methods for creating entity label -// ************************************************************************/ - -// /** -// * Create entity label -// * -// * Gets values from Indexes and chaches dependencies in itself. -// */ -// async create() { - -// const entity = await this.providers.entity.get(this.id); - -// if (entity) { - -// const fieldsWithEdges = await this.providers.edges.get(this.id) - -// if (keys(fieldsWithEdges).length === 0) return this.finalize() - -// const classId = { -// fkProject: entity.fkProject, -// pkClass: entity.fkClass -// } - -// await this.createLabel(classId, fieldsWithEdges) -// } -// return this.finalize() -// } - -// finalize() { -// return { -// entityLabel: this.entityLabel, -// labelMissing: this.labelMissing, -// } -// } - -// async createLabel(classId: PClassId, entityFields?: EntityFields) { -// const labelParts = await this.getLabelParts(classId); - -// const entityLabel = await this.loopOverLabelParts(classId.fkProject, labelParts, entityFields) - -// if (entityLabel !== '') { -// this.labelMissing = false -// this.entityLabel = entityLabel; -// } -// return entityLabel -// } - - - -// /** -// * gets the label parts for given project class. -// * -// * Logic: -// * 1. if there is a entity label config for that class, return it -// * 2. else if there are identity defining props for that class, create -// * a label config on the fly and return it -// * 3. return empty array -> will lead to no label of the entity -// * -// * @param pClassId -// */ -// private async getLabelParts(pClassId: PClassId) { -// // in case of Appellation in a langage use this config hard coded -// // to prevent infinit loops -// if (pClassId.pkClass === 365) { -// return [ -// { -// ordNum: 1, -// field: { -// fkProperty: 1113, -// isOutgoing: true, -// nrOfStatementsInLabel: 1 -// } -// } -// ] -// } - - -// const entityLabelConfig = await this.getEntityLabelConfig(pClassId); - -// if (entityLabelConfig?.labelParts) { -// return entityLabelConfig.labelParts; -// } - -// const identifyingProps = await this.providers.identifyingProperty.get({pkClass: pClassId.pkClass}) - -// if (identifyingProps?.length) { -// return identifyingProps.map((item, i) => { -// const labelPart: LabelPart = { -// field: { -// fkProperty: item.fkProperty, -// isOutgoing: true, -// nrOfStatementsInLabel: 2 -// }, -// ordNum: i -// } -// return labelPart -// }) -// } - -// return [ -// { -// ordNum: 1, -// field: { -// fkProperty: 1111, -// isOutgoing: false, -// nrOfStatementsInLabel: 1 -// } -// } -// ] -// } - -// private async loopOverLabelParts(fkProject: number, labelParts: LabelPart[], entityFields?: EntityFields) { -// const promises: Promise[] = [] - -// // iterate over label parts -// for (const labelPart of labelParts) { -// if (labelPart.field) { -// const edges = entityFields?.[labelPart.field.isOutgoing ? 'outgoing' : 'incoming']?.[labelPart.field.fkProperty]; -// promises.push(this.loopOverEdges(fkProject, edges, labelPart?.field?.nrOfStatementsInLabel)) -// } -// } - -// const strings = await Promise.all(promises) -// return strings.join(', ') -// } - -// /** -// * Creates strings array for given field-edges (edges must be of same field, i.e. property + direction) -// * -// * example result: ["fieldLabel:", "edgeTargetLabel1", "edgeTargetLabel2", ...] -// * -// * @param edges Array of edges -// * @param config ` -// * fkProject: project -// * fkProperty: property of edges/field (needed to get fieldLabel) -// * isOutgoing: direction of edges/field (needed to get fieldLabel) -// * maxLabelNr max number of edgeTargetLabels -// * ` -// */ -// private async loopOverEdges( -// fkProject: number, -// edges?: Edge[], -// maxLabelNr = 1 -// ) { -// const result: string[] = [] - - -// // are there any edges? -// if (edges?.length) { - -// // stop iteration at smaller number of limit or length of edges -// const stop = Math.min(maxLabelNr, edges.length); - -// for (let i = 0; i < stop; i++) { -// const e = edges[i]; - -// let string; -// if (e.targetIsEntity) { -// const label = await this.providers.entityLabels.get({pkEntity: e.fkTarget, fkProject}) -// string = label?.entityLabel; -// } else { -// string = e.targetLabel; -// } - -// if (string) result.push(string); - -// } -// } -// return result.join(', '); -// } - - -// /************************************************************************ -// * Methods for registering values of this in other indexes -// ************************************************************************/ - - -// // /** -// // * Register the aggregated result and cleanup its dependencies (providers) -// // */ -// // async register() { - -// // /** -// // * update the result index `EntityPreviewIdx`: -// // * this adds update requests if needed -// // */ -// // await this.main.agg.entityLabel.add(this.id, this.entityLabel) - - -// // /** -// // * cleanup dependency indexes: -// // * remove providers from last cycle that are not needed anymore -// // */ -// // await this.providers.removeProvidersFromIndexes() - -// // return this -// // } - - - - -// private async getEntityLabelConfig(classId: PClassId): Promise { - - -// let res = await this.providers.entityLabelConfig.get(classId) - -// if (res) return res; - -// res = await this.providers.entityLabelConfig.get({...classId, fkProject: PK_DEFAULT_CONFIG_PROJECT}) - -// return res; -// } - - -// } diff --git a/server/src/warehouse/aggregator-ds/entity-label/p-entity-label/PEntityLabelDependencies.ts b/server/src/warehouse/aggregator-ds/entity-label/p-entity-label/PEntityLabelDependencies.ts deleted file mode 100644 index 74dae9b9d..000000000 --- a/server/src/warehouse/aggregator-ds/entity-label/p-entity-label/PEntityLabelDependencies.ts +++ /dev/null @@ -1,66 +0,0 @@ -// import {Dependencies} from '../../../base/classes/Dependencies' -// import {DependencyIndex} from '../../../base/classes/DependencyIndex' -// import {RClassId} from '../../../primary-ds/DfhClassHasTypePropertyService' -// import {EntityFields} from "../../../primary-ds/edge/edge.commons" -// import {PEntity, PEntityId} from '../../../primary-ds/entity/PEntityService' -// import {PClassId} from '../../../primary-ds/ProClassFieldsConfigService' -// import {EntityLabelConfigVal} from '../../../primary-ds/ProEntityLabelConfigService' -// import {Warehouse} from '../../../Warehouse' -// import {IdentifyingPropertyVal} from '../../identifying-property/IdentifyingPropertyService' -// import {EntityLabelVal} from '../entity-label.commons' -// import {Injectable, Inject, forwardRef} from 'injection-js'; - -// @Injectable() -// export class PEntityLabelDependencies extends Dependencies { -// entity: DependencyIndex -// entityLabelConfig: DependencyIndex -// identifyingProperty: DependencyIndex -// entityLabel: DependencyIndex -// edge: DependencyIndex - -// // entityFulltextClassLabelDep: DependencyIndex; -// constructor(@Inject(forwardRef(() => Warehouse)) private wh: Warehouse) { -// super() -// // stores the dependency of entityLabel (receiver) on entity (provider) -// this.entity = this.registerDepIdx(new DependencyIndex( -// this.wh, -// this.wh.agg.pEntityLabel, -// this.wh.prim.pEntity, -// )) - -// // stores the dependency of entityLabel (receiver) on entityLabelConfig (provider) -// this.entityLabelConfig = this.registerDepIdx(new DependencyIndex( -// this.wh, -// this.wh.agg.pEntityLabel, -// this.wh.prim.proEntityLabelConfig, -// )) -// // stores the dependency of entityLabel (receiver) on entityLabelConfig (provider) -// this.identifyingProperty = this.registerDepIdx(new DependencyIndex( -// this.wh, -// this.wh.agg.pEntityLabel, -// this.wh.agg.identifyingProperty, -// )) - - -// // stores the dependency of entityLabel (receiver) on entityLabel (provider) -// this.entityLabel = this.registerDepIdx(new DependencyIndex( -// this.wh, -// this.wh.agg.pEntityLabel, -// this.wh.agg.pEntityLabel, -// )) - -// // stores the dependency of entityLabel (receiver) on edge (provider) -// this.edge = this.registerDepIdx(new DependencyIndex( -// this.wh, -// this.wh.agg.pEntityLabel, -// this.wh.prim.pEdge, -// )) - -// // stores the dependency of entityFullText (receiver) on the classLabel (provider) -// // entityClassLabelDep = new EntityClassDependencyIndex(); - -// } - - - -// } diff --git a/server/src/warehouse/aggregator-ds/entity-label/p-entity-label/PEntityLabelPoviders.ts b/server/src/warehouse/aggregator-ds/entity-label/p-entity-label/PEntityLabelPoviders.ts deleted file mode 100644 index f51f9e451..000000000 --- a/server/src/warehouse/aggregator-ds/entity-label/p-entity-label/PEntityLabelPoviders.ts +++ /dev/null @@ -1,31 +0,0 @@ -// import {Provider} from '../../../base/classes/Provider'; -// import {Providers} from "../../../base/interfaces/Providers"; -// import {RClassId} from '../../../primary-ds/DfhClassHasTypePropertyService'; -// import {EntityFields} from "../../../primary-ds/edge/edge.commons"; -// import {PEntity, PEntityId} from '../../../primary-ds/entity/PEntityService'; -// import {PClassId} from '../../../primary-ds/ProClassFieldsConfigService'; -// import {EntityLabelConfigVal} from '../../../primary-ds/ProEntityLabelConfigService'; -// import {IdentifyingPropertyVal} from '../../identifying-property/IdentifyingPropertyService'; -// import {EntityLabelVal} from '../entity-label.commons'; -// import {PEntityLabelService} from './PEntityLabelService'; - -// export class PEntityLabelProviders extends Providers { -// entity: Provider; -// entityLabels: Provider; -// entityLabelConfig: Provider; -// identifyingProperty: Provider; -// edges: Provider; -// constructor( -// dep: PEntityLabelService, -// protected receiverKey: PEntityId -// ) { -// super() -// this.entity = this.registerProvider(dep.depPEntity, receiverKey) -// this.entityLabels = this.registerProvider(dep.depPEntityLabel, receiverKey); -// this.entityLabelConfig = this.registerProvider(dep.depProEntityLabelConfig, receiverKey); -// this.identifyingProperty = this.registerProvider(dep.depIdentifyingProperty, receiverKey); -// this.edges = this.registerProvider(dep.depPEdge, receiverKey) -// } - -// } - diff --git a/server/src/warehouse/aggregator-ds/entity-label/p-entity-label/PEntityLabelService.ts b/server/src/warehouse/aggregator-ds/entity-label/p-entity-label/PEntityLabelService.ts deleted file mode 100644 index 85eaf7775..000000000 --- a/server/src/warehouse/aggregator-ds/entity-label/p-entity-label/PEntityLabelService.ts +++ /dev/null @@ -1,324 +0,0 @@ -import {forwardRef, Inject, Injectable} from 'injection-js'; -import {PoolClient} from 'pg'; -import {AggregatedDataService2} from '../../../base/classes/AggregatedDataService2'; -import {AggregatorSqlBuilder, CustomValSql} from '../../../base/classes/AggregatorSqlBuilder'; -import {DependencyIndex} from '../../../base/classes/DependencyIndex'; -import {EntityFields} from '../../../primary-ds/edge/edge.commons'; -import {PEdgeService} from '../../../primary-ds/edge/PEdgeService'; -import {PEntity, PEntityId, pEntityKeyDefs, PEntityService} from '../../../primary-ds/entity/PEntityService'; -import {PClassId} from '../../../primary-ds/ProClassFieldsConfigService'; -import {EntityLabelConfigVal, ProEntityLabelConfigService} from '../../../primary-ds/ProEntityLabelConfigService'; -import {PK_DEFAULT_CONFIG_PROJECT, Warehouse} from '../../../Warehouse'; -import {EntityLabelVal, LabelPartCustom, LabelPartKeys, labelPartsForAppeInLang365, labelPartsForNormalEntities, ENTITY_LABEL_MAX_LENGTH} from '../entity-label.commons'; - -@Injectable() -export class PEntityLabelService extends AggregatedDataService2{ - // aggregator = PEntityLabelAggregator; - // providers = PEntityLabelProviders; - - depPEntity: DependencyIndex - depProEntityLabelConfig: DependencyIndex - // depIdentifyingProperty: DependencyIndex - depPEntityLabel: DependencyIndex - depPEdge: DependencyIndex - - batchSize = 100000; - constructor( - @Inject(forwardRef(() => Warehouse)) wh: Warehouse, - @Inject(forwardRef(() => PEntityService)) pEntity: PEntityService, - @Inject(forwardRef(() => ProEntityLabelConfigService)) entityLabelConfig: ProEntityLabelConfigService, - // @Inject(forwardRef(() => IdentifyingPropertyService)) identifyingProperty: IdentifyingPropertyService, - @Inject(forwardRef(() => PEdgeService)) pEdge: PEdgeService, - ) { - super( - wh, - pEntityKeyDefs - ) - this.registerCreatorDS({dataService: pEntity}) - this.depPEntity = this.addDepencency(pEntity); - this.depProEntityLabelConfig = this.addDepencency(entityLabelConfig); - // this.depIdentifyingProperty = this.addDepencency(identifyingProperty); - this.depPEntityLabel = this.addDepencency(this); - this.depPEdge = this.addDepencency(pEdge); - } - getDependencies() { - return this - }; - // onUpsertSql(tableAlias: string) { - // return ` - // UPDATE war.entity_preview - // SET entity_label = val->>'entityLabel' - // FROM ${tableAlias} - // WHERE pk_entity = "pkEntity" - // AND project = "fkProject" - // AND entity_label IS DISTINCT FROM val->>'entityLabel'` - // } - - async aggregateBatch(client: PoolClient, client2: PoolClient, limit: number, offset: number, currentTimestamp: string): Promise { - const builder = new AggregatorSqlBuilder(this, client, currentTimestamp, limit, offset) - - const pentity = await builder.joinProviderThroughDepIdx({ - leftTable: builder.batchTmpTable.tableDef, - joinWithDepIdx: this.depPEntity, - joinOnKeys: { - pkEntity: {leftCol: 'pkEntity'}, - fkProject: {leftCol: 'fkProject'} - }, - conditionTrueIf: { - providerKey: {pkEntity: 'IS NOT NULL'} - }, - createCustomObject: (() => `jsonb_build_object('fkClass', (t2.val->>'fkClass')::int)`) as CustomValSql<{fkClass: number}>, - }) - const pEdges = await builder.joinProviderThroughDepIdx({ - leftTable: pentity.aggregation.tableDef, - joinWithDepIdx: this.depPEdge, - joinWhereLeftTableCondition: '= true', - joinOnKeys: { - pkEntity: {leftCol: 'pkEntity'}, - fkProject: {leftCol: 'fkProject'} - }, - conditionTrueIf: { - or: [ - {providerVal: {outgoing: 'is not {}', }}, - {providerVal: {incoming: 'is not {}', }}, - ] - }, - createCustomObject: ((provider) => `t1.custom`) as CustomValSql<{fkClass: number}>, - }) - - const customVal: CustomValSql<{ - fkClass: number, - entityLabelConfig: EntityLabelConfigVal - }> = () => `jsonb_build_object( - 'fkClass', t1.custom->>'fkClass', - 'entityLabelConfig', t2.val - )` - const pEntityLabelConfig = await builder.joinProviderThroughDepIdx({ - leftTable: pentity.aggregation.tableDef, - joinWithDepIdx: this.depProEntityLabelConfig, - joinWhereLeftTableCondition: '= true', - joinOnKeys: { - pkClass: {leftCustom: {name: 'fkClass', type: 'int'}}, - fkProject: {leftCol: 'fkProject'} - }, - conditionTrueIf: { - providerKey: {pkClass: 'IS NOT NULL'} - }, - createCustomObject: customVal, - }) - const defaultEntityLabelConfig = await builder.joinProviderThroughDepIdx({ - leftTable: pEntityLabelConfig.aggregation.tableDef, - joinWithDepIdx: this.depProEntityLabelConfig, - joinWhereLeftTableCondition: '= false', - joinOnKeys: { - pkClass: {leftCustom: {name: 'fkClass', type: 'int'}}, - fkProject: {value: PK_DEFAULT_CONFIG_PROJECT} - }, - conditionTrueIf: {}, - createCustomObject: customVal - }) - - - - /** - * Expand Label parts for each entity - */ - const labelPartsTbl = builder.createTableName(); - const createlabelPartsTbl = builder.createTableStmt(labelPartsTbl) - - const labelPartsSql = ` - -- expands all labelParts for each entity - ${createlabelPartsTbl} ( - WITH tw1 AS ( - SELECT - t1. "r_pkEntity", - t1. "r_fkProject", - t1. "p_pkClass", - CASE WHEN (t1.custom->>'fkClass')::int = 365 THEN - '${JSON.stringify(labelPartsForAppeInLang365)}'::jsonb - WHEN t1.custom ->'entityLabelConfig'->> 'labelParts' IS NOT NULL THEN - t1.custom ->'entityLabelConfig'-> 'labelParts' - WHEN t2.custom ->'entityLabelConfig'->> 'labelParts' IS NOT NULL THEN - t2.custom ->'entityLabelConfig'-> 'labelParts' - ELSE - '${JSON.stringify(labelPartsForNormalEntities)}'::jsonb - END label_parts - FROM ${pEntityLabelConfig.aggregation.tableDef.tableName} t1 - LEFT JOIN ${defaultEntityLabelConfig.aggregation.tableDef.tableName} t2 - ON t1. "r_pkEntity" = t2. "r_pkEntity" - AND t1. "r_fkProject" = t2. "r_fkProject" - ) - SELECT t1.*, jsonb_array_elements(label_parts) label_part - FROM tw1 t1 - )` - builder.registerTmpTable(labelPartsSql, [], labelPartsTbl) - - - /** - * expands all 'slots' per labelPart according to nrOfStatementsInLabel - */ - const slotsTbl = builder.createTableName(); - const createSlotsTbl = builder.createTableStmt(slotsTbl) - const slotsSql = ` - -- expands all 'slots' per labelPart according to nrOfStatementsInLabel - ${createSlotsTbl} ( - SELECT - t1."r_pkEntity", - t1."r_fkProject", - t1."p_pkClass", - label_part->'field'->>'fkProperty' property, - label_part->'field'->>'nrOfStatementsInLabel', - CASE WHEN (label_part->'field'->>'isOutgoing')::bool = true THEN 'outgoing' ELSE 'incoming' END direction, - label_part->>'ordNum' fielOrdNum, - generate_series(1,(label_part->'field'->>'nrOfStatementsInLabel')::int) stmtOrdNum - FROM ${labelPartsTbl} t1 - )` - builder.registerTmpTable(slotsSql, [], slotsTbl) - - /** - * expands all 'slots' per labelPart according to nrOfStatementsInLabel - */ - const stmtsTbl = builder.createTableName(); - const createStmtsTbl = builder.createTableStmt(stmtsTbl) - const stmtsSql = ` - -- joins slots with statements - ${createStmtsTbl} ( - SELECT - t1."r_pkEntity", t1."r_fkProject", t2.property, t2.direction, t2.fielOrdNum, t2.stmtOrdNum, t1.p_val->t2.direction->t2.property->(t2.stmtOrdNum-1) stmt - FROM - ${pEdges.aggregation.tableDef.tableName} t1, - ${slotsTbl} t2 - WHERE - t1."r_pkEntity"=t2."r_pkEntity" - AND - t1."r_fkProject"=t2."r_fkProject" - - )` - builder.registerTmpTable(stmtsSql, [], stmtsTbl) - - - /** - * stmts relevant for entity label - */ - const finalTbl = builder.createTableName(); - const createFinalTbl = builder.createTableStmt(finalTbl) - const finalSql = ` - -- stmts relevant for entity label - ${createFinalTbl} ( - SELECT - t1."r_pkEntity", - t1."r_fkProject", - t1.property, - t1.direction, - t1.fielOrdNum, - t1.stmtOrdNum, - (t1.stmt->>'targetIsEntity')::bool condition, - jsonb_build_object( - 'fkTarget', (t1.stmt->>'fkTarget')::int, - 'targetLabel', t1.stmt->>'targetLabel' - ) custom - FROM ${stmtsTbl} t1 - )` - const labelParts = builder.registerTmpTable(finalSql, [], finalTbl) - - const remoteEntityLabelCustom: CustomValSql<{ - string: string - pkEntity: number - property: number - direction: string - fielOrdNum: number - stmtOrdNum: number - condition: boolean - }> = () => { - return `jsonb_build_object( - 'string', t2.val->>'entityLabel', - 'r_pkEntity', t1."r_pkEntity", - 'property', t1.property, - 'direction', t1.direction, - 'fielOrdNum', t1.fielOrdNum, - 'stmtOrdNum', t1.stmtOrdNum - )` - } - const remoteEntityLabel = await builder.joinProviderThroughDepIdx({ - leftTable: labelParts.tableDef, - joinWithDepIdx: this.depPEntityLabel, - joinWhereLeftTableCondition: '= true', - joinOnKeys: { - pkEntity: {leftCustom: {name: 'fkTarget', type: 'int'}}, - fkProject: {leftCol: 'fkProject'} - }, - conditionTrueIf: {}, - createCustomObject: remoteEntityLabelCustom - }) - - // const hook = builder.twOnUpsertHook() - - /** - * aggregate labels - */ - const labelPartsTable = labelParts.tableDef.tableName - const remoteEntityLabelTable = remoteEntityLabel.aggregation.tableDef.tableName - const aggLabelsTbl = builder.createTableName(); - const createAggLabelsTbl = builder.createTableStmt(aggLabelsTbl) - const aggLabelsSql = ` - -- aggregate entity labels - ${createAggLabelsTbl} ( - SELECT - t1."r_pkEntity", - t1."r_fkProject", - jsonb_build_object( - 'entityLabel', - substring( - coalesce( - string_agg( - coalesce( - t2.custom->>'string', - t1.custom->>'targetLabel' - -- if we ever implement a placeholder for not available stmt, put val here - ), - ', ' - ORDER BY t1.fielOrdNum ASC,t1.stmtOrdNum ASC - ), - '(no label)' - ) FROM 1 FOR ${ENTITY_LABEL_MAX_LENGTH} - ), - 'labelMissing', string_agg(coalesce( t2.custom->>'string', t1.custom->>'targetLabel' ),'') IS NULL - ) val, - true::boolean condition - FROM ${labelPartsTable} t1 - LEFT JOIN ${remoteEntityLabelTable} t2 - ON t1."r_pkEntity"=t2."r_pkEntity" - AND t1."r_fkProject"=t2."r_fkProject" - AND t1.property=t2.custom->>'property' - AND t1.direction=t2.custom->>'direction' - AND t1.fielOrdNum=t2.custom->>'fielOrdNum' - AND t1.stmtOrdNum=(t2.custom->>'stmtOrdNum')::int - GROUP BY t1."r_pkEntity", t1."r_fkProject" - )` - const aggregateLabels = builder.registerTmpTable(aggLabelsSql, [], aggLabelsTbl) - - - - await builder.tmpTableUpsertAggregations(this.index, aggregateLabels.tableDef.tableName, '= true') - - builder.registerUpsertHook( - {client: client2, table: 'war.entity_preview'}, - (insertStmt, fromStmt) => ` - UPDATE war.entity_preview - SET entity_label = val->>'entityLabel' - FROM ${fromStmt} t1 - WHERE pk_entity = "pkEntity" - AND project = "fkProject" - AND entity_label IS DISTINCT FROM val->>'entityLabel' - ` - ) - - // await builder.printQueries() - const count = builder.executeQueries() - - return count - } - - -} - diff --git a/server/src/warehouse/aggregator-ds/entity-label/r-entity-label/REntityLabelAggregator.ts b/server/src/warehouse/aggregator-ds/entity-label/r-entity-label/REntityLabelAggregator.ts deleted file mode 100644 index 4f81e1def..000000000 --- a/server/src/warehouse/aggregator-ds/entity-label/r-entity-label/REntityLabelAggregator.ts +++ /dev/null @@ -1,218 +0,0 @@ -// import {isEmpty} from 'ramda'; -// import {AbstractAggregator} from '../../../base/classes/AbstractAggregator'; -// import {RClassId} from '../../../primary-ds/DfhClassHasTypePropertyService'; -// import {Edge, EntityFields} from "../../../primary-ds/edge/edge.commons"; -// import {REntityId} from '../../../primary-ds/entity/REntityService'; -// import {EntityLabelConfigVal, LabelPart} from '../../../primary-ds/ProEntityLabelConfigService'; -// import {PK_DEFAULT_CONFIG_PROJECT} from '../../../Warehouse'; -// import {REntityLabelProviders} from './REntityLabelPoviders'; -// import {EntityLabelVal} from '../entity-label.commons'; - -// export interface ClassLabelConfig { -// fkProperty: number, -// isOutgoing: boolean, -// ordNum: number -// nrOfStatementsInLabel?: number -// } - - -// export class REntityLabelAggregator extends AbstractAggregator { - -// // array of strings to create label -// labelArr: string[] = []; - -// // the resulting entityLabel -// entityLabel = '(no label)'; - -// // For testing / debugging -// labelMissing = true; - -// constructor( -// public providers: REntityLabelProviders, -// public id: REntityId -// ) { -// super() -// } - - -// /************************************************************************ -// * Methods for creating entity label -// ************************************************************************/ - -// /** -// * Create entity label -// * -// * Gets values from Indexes and chaches dependencies in itself. -// */ -// async create() { - -// const entity = await this.providers.rEntity.get(this.id); - -// if (entity) { - -// const fieldsWithEdges = await this.providers.rEdges.get(this.id) - -// if (isEmpty(fieldsWithEdges) || (isEmpty(fieldsWithEdges?.outgoing) && isEmpty(fieldsWithEdges?.incoming))) { -// return this.finalize(); -// } - -// const classId: RClassId = { -// pkClass: entity.fkClass -// } - -// await this.createLabel(classId, fieldsWithEdges) -// } -// return this.finalize() -// } - -// finalize(): EntityLabelVal { -// return { -// labelMissing: this.labelMissing, -// entityLabel: this.entityLabel -// } -// } - -// async createLabel(classId: RClassId, entityFields?: EntityFields) { -// const labelParts = await this.getLabelParts(classId); - -// const entityLabel = await this.loopOverLabelParts(labelParts, entityFields) - -// if (entityLabel !== '') { -// this.labelMissing = false -// this.entityLabel = entityLabel; -// } -// return entityLabel -// } - - - -// /** -// * gets the label parts for given project class. -// * -// * Logic: -// * 1. if there is a entity label config for that class, return it -// * 2. else if there are identity defining props for that class, create -// * a label config on the fly and return it -// * 3. return empty array -> will lead to no label of the entity -// * -// * @param pClassId -// */ -// private async getLabelParts(pClassId: RClassId) { -// // in case of Appellation in a langage use this config hard coded -// // to prevent infinit loops -// if (pClassId.pkClass === 365) { -// return [ -// { -// ordNum: 1, -// field: { -// fkProperty: 1113, -// isOutgoing: true, -// nrOfStatementsInLabel: 1 -// } -// } -// ] -// } - - -// const entityLabelConfig = await this.getEntityLabelConfig(pClassId); - -// if (entityLabelConfig?.labelParts) { -// return entityLabelConfig.labelParts; -// } - -// const identifyingProps = await this.providers.identifyingProperty.get({pkClass: pClassId.pkClass}) - -// if (identifyingProps?.length) { -// return identifyingProps.map((item, i) => { -// const labelPart: LabelPart = { -// field: { -// fkProperty: item.fkProperty, -// isOutgoing: true, -// nrOfStatementsInLabel: 2 -// }, -// ordNum: i -// } -// return labelPart -// }) -// } - -// return [ -// { -// ordNum: 1, -// field: { -// fkProperty: 1111, -// isOutgoing: false, -// nrOfStatementsInLabel: 1 -// } -// } -// ] -// } - -// private async loopOverLabelParts(labelParts: LabelPart[], entityFields?: EntityFields) { -// const promises: Promise[] = [] - -// // iterate over label parts -// for (const labelPart of labelParts) { -// if (labelPart.field) { -// const edges = entityFields?.[labelPart.field.isOutgoing ? 'outgoing' : 'incoming']?.[labelPart.field.fkProperty]; -// promises.push(this.loopOverEdges(edges, labelPart?.field?.nrOfStatementsInLabel)) -// } -// } - -// const strings = await Promise.all(promises) -// return strings.join(', ') -// } - -// /** -// * Creates strings array for given field-edges (edges must be of same field, i.e. property + direction) -// * -// * example result: ["fieldLabel:", "edgeTargetLabel1", "edgeTargetLabel2", ...] -// * -// * @param edges Array of edges -// * @param config ` -// * fkProperty: property of edges/field (needed to get fieldLabel) -// * isOutgoing: direction of edges/field (needed to get fieldLabel) -// * maxLabelNr max number of edgeTargetLabels -// * ` -// */ -// private async loopOverEdges( -// edges?: Edge[], -// maxLabelNr = 1 -// ) { -// const result: string[] = [] - - -// // are there any edges? -// if (edges?.length) { - -// // stop iteration at smaller number of limit or length of edges -// const stop = Math.min(maxLabelNr, edges.length); - -// for (let i = 0; i < stop; i++) { -// const e = edges[i]; - -// let string; -// if (e.targetIsEntity) { -// const label = await this.providers.rEntityLabels.get({pkEntity: e.fkTarget}) -// string = label?.entityLabel; -// } else { -// string = e.targetLabel; -// } - -// if (string) result.push(string); - -// } -// } -// return result.join(', '); -// } - - -// private async getEntityLabelConfig(classId: RClassId): Promise { - -// const res = await this.providers.entityLabelConfig.get({pkClass: classId.pkClass, fkProject: PK_DEFAULT_CONFIG_PROJECT}) - -// return res; -// } - - -// } diff --git a/server/src/warehouse/aggregator-ds/entity-label/r-entity-label/REntityLabelDependencies.ts b/server/src/warehouse/aggregator-ds/entity-label/r-entity-label/REntityLabelDependencies.ts deleted file mode 100644 index b90ea851c..000000000 --- a/server/src/warehouse/aggregator-ds/entity-label/r-entity-label/REntityLabelDependencies.ts +++ /dev/null @@ -1,64 +0,0 @@ -// import {Dependencies} from '../../../base/classes/Dependencies' -// import {DependencyIndex} from '../../../base/classes/DependencyIndex' -// import {RClassId} from '../../../primary-ds/DfhClassHasTypePropertyService' -// import {EntityFields} from "../../../primary-ds/edge/edge.commons" -// import {REntity, REntityId} from '../../../primary-ds/entity/REntityService' -// import {PClassId} from '../../../primary-ds/ProClassFieldsConfigService' -// import {EntityLabelConfigVal} from '../../../primary-ds/ProEntityLabelConfigService' -// import {Warehouse} from '../../../Warehouse' -// import {IdentifyingPropertyVal} from '../../identifying-property/IdentifyingPropertyService' -// import {EntityLabelVal} from '../entity-label.commons' -// import {Injectable, Inject, forwardRef} from 'injection-js'; - -// @Injectable() -// export class REntityLabelDependencies extends Dependencies { -// entity: DependencyIndex -// entityLabelConfig: DependencyIndex -// identifyingProperty: DependencyIndex -// entityLabel: DependencyIndex -// edge: DependencyIndex - -// // entityFulltextClassLabelDep: DependencyIndex; -// constructor(@Inject(forwardRef(() => Warehouse)) private wh: Warehouse) { -// super() -// // stores the dependency of entityLabel (receiver) on entity (provider) -// this.entity = this.registerDepIdx(new DependencyIndex( -// this.wh, -// this.wh.agg.rEntityLabel, -// this.wh.prim.rEntity, -// )) - -// // stores the dependency of entityLabel (receiver) on entityLabelConfig (provider) -// this.entityLabelConfig = this.registerDepIdx(new DependencyIndex( -// this.wh, -// this.wh.agg.rEntityLabel, -// this.wh.prim.proEntityLabelConfig, -// )) -// // stores the dependency of entityLabel (receiver) on entityLabelConfig (provider) -// this.identifyingProperty = this.registerDepIdx(new DependencyIndex( -// this.wh, -// this.wh.agg.rEntityLabel, -// this.wh.agg.identifyingProperty, -// )) - - -// // stores the dependency of entityLabel (receiver) on entityLabel (provider) -// this.entityLabel = this.registerDepIdx(new DependencyIndex( -// this.wh, -// this.wh.agg.rEntityLabel, -// this.wh.agg.rEntityLabel, -// )); - -// // stores the dependency of entityLabel (receiver) on edge (provider) -// this.edge = this.registerDepIdx(new DependencyIndex( -// this.wh, -// this.wh.agg.rEntityLabel, -// this.wh.prim.rEdge, -// )); - -// // stores the dependency of entityFullText (receiver) on the classLabel (provider) -// // entityClassLabelDep = new EntityClassDependencyIndex(); - -// } - -// } diff --git a/server/src/warehouse/aggregator-ds/entity-label/r-entity-label/REntityLabelPoviders.ts b/server/src/warehouse/aggregator-ds/entity-label/r-entity-label/REntityLabelPoviders.ts deleted file mode 100644 index a0bfbe9ac..000000000 --- a/server/src/warehouse/aggregator-ds/entity-label/r-entity-label/REntityLabelPoviders.ts +++ /dev/null @@ -1,31 +0,0 @@ -// import {Provider} from '../../../base/classes/Provider'; -// import {Providers} from "../../../base/interfaces/Providers"; -// import {PClassId} from '../../../primary-ds/ProClassFieldsConfigService'; -// import {EntityFields} from "../../../primary-ds/edge/edge.commons"; -// import {EntityLabelConfigVal} from '../../../primary-ds/ProEntityLabelConfigService'; -// import {REntity, REntityId} from '../../../primary-ds/entity/REntityService'; -// // import {REntityLabelDependencies} from './REntityLabelDependencies'; -// import {IdentifyingPropertyVal} from '../../identifying-property/IdentifyingPropertyService'; -// import {RClassId} from '../../../primary-ds/DfhClassHasTypePropertyService'; -// import {EntityLabelVal} from '../entity-label.commons'; - -// export class REntityLabelProviders extends Providers { -// rEntity: Provider; -// rEntityLabels: Provider; -// entityLabelConfig: Provider; -// identifyingProperty: Provider; -// rEdges: Provider; -// constructor( -// // dep: REntityLabelDependencies, -// protected receiverKey: REntityId -// ) { -// super() -// // this.rEntity = this.registerProvider(dep.entity, receiverKey) -// // this.rEntityLabels = this.registerProvider(dep.entityLabel, receiverKey); -// // this.entityLabelConfig = this.registerProvider(dep.entityLabelConfig, receiverKey); -// // this.identifyingProperty = this.registerProvider(dep.identifyingProperty, receiverKey); -// // this.rEdges = this.registerProvider(dep.edge, receiverKey) -// } - -// } - diff --git a/server/src/warehouse/aggregator-ds/entity-label/r-entity-label/REntityLabelService.ts b/server/src/warehouse/aggregator-ds/entity-label/r-entity-label/REntityLabelService.ts deleted file mode 100644 index 3832add27..000000000 --- a/server/src/warehouse/aggregator-ds/entity-label/r-entity-label/REntityLabelService.ts +++ /dev/null @@ -1,325 +0,0 @@ - - -import {forwardRef, Inject, Injectable} from 'injection-js'; -import {PoolClient} from 'pg'; -import {AggregatedDataService2} from '../../../base/classes/AggregatedDataService2'; -import {AggregatorSqlBuilder, CustomValSql} from '../../../base/classes/AggregatorSqlBuilder'; -import {DependencyIndex} from '../../../base/classes/DependencyIndex'; -import {EntityFields} from '../../../primary-ds/edge/edge.commons'; -import {REdgeService} from '../../../primary-ds/edge/REdgeService'; -import {REntity, REntityId, rEntityKeyDefs, REntityService} from '../../../primary-ds/entity/REntityService'; -import {PClassId} from '../../../primary-ds/ProClassFieldsConfigService'; -import {EntityLabelConfigVal, ProEntityLabelConfigService} from '../../../primary-ds/ProEntityLabelConfigService'; -import {PK_DEFAULT_CONFIG_PROJECT, Warehouse} from '../../../Warehouse'; -import {EntityLabelVal, LabelPartCustom, LabelPartKeys, labelPartsForAppeInLang365, labelPartsForNormalEntities, ENTITY_LABEL_MAX_LENGTH} from '../entity-label.commons'; - -@Injectable() -export class REntityLabelService extends AggregatedDataService2{ - - depREntity: DependencyIndex - depProEntityLabelConfig: DependencyIndex - // depIdentifyingProperty: DependencyIndex - depREntityLabel: DependencyIndex - depREdge: DependencyIndex - - batchSize = 100000 - constructor( - @Inject(forwardRef(() => Warehouse)) wh: Warehouse, - @Inject(forwardRef(() => ProEntityLabelConfigService)) private proEntityLabelConfig: ProEntityLabelConfigService, - // @Inject(forwardRef(() => IdentifyingPropertyService)) private identifyingProperty: IdentifyingPropertyService, - @Inject(forwardRef(() => REdgeService)) private rEdge: REdgeService, - @Inject(forwardRef(() => REntityService)) private rEntity: REntityService, - ) { - super( - wh, - rEntityKeyDefs - ) - - this.registerCreatorDS({dataService: this.rEntity}) - - this.depREntity = this.addDepencency(this.rEntity) - this.depProEntityLabelConfig = this.addDepencency(this.proEntityLabelConfig) - // this.depIdentifyingProperty = this.addDepencency(this.identifyingProperty) - this.depREntityLabel = this.addDepencency(this) - this.depREdge = this.addDepencency(this.rEdge) - - } - - - // onUpsertSql(tableAlias: string) { - // return ` - // UPDATE war.entity_preview - // SET entity_label = val->>'entityLabel' - // FROM ${tableAlias} - // WHERE pk_entity = "pkEntity" - // AND project = 0 - // AND entity_label IS DISTINCT FROM val->>'entityLabel'` - // } - - - async aggregateBatch(client: PoolClient, client2: PoolClient, limit: number, offset: number, currentTimestamp: string): Promise { - const builder = new AggregatorSqlBuilder(this, client, currentTimestamp, limit, offset) - - const rentity = await builder.joinProviderThroughDepIdx({ - leftTable: builder.batchTmpTable.tableDef, - joinWithDepIdx: this.depREntity, - joinOnKeys: { - pkEntity: {leftCol: 'pkEntity'} - }, - conditionTrueIf: { - providerKey: {pkEntity: 'IS NOT NULL'} - }, - createCustomObject: (() => `jsonb_build_object('fkClass', (t2.val->>'fkClass')::int)`) as CustomValSql<{fkClass: number}>, - }) - const rEdges = await builder.joinProviderThroughDepIdx({ - leftTable: rentity.aggregation.tableDef, - joinWithDepIdx: this.depREdge, - joinWhereLeftTableCondition: '= true', - joinOnKeys: { - pkEntity: {leftCol: 'pkEntity'} - }, - conditionTrueIf: { - or: [ - {providerVal: {outgoing: 'is not {}', }}, - {providerVal: {incoming: 'is not {}', }}, - ] - }, - createCustomObject: ((provider) => `t1.custom`) as CustomValSql<{fkClass: number}>, - }) - const customVal: CustomValSql<{ - fkClass: number, - entityLabelConfig: EntityLabelConfigVal - }> = () => `jsonb_build_object( - 'fkClass', t1.custom->>'fkClass', - 'entityLabelConfig', t2.val - )` - - const entityLabelConfig = await builder.joinProviderThroughDepIdx({ - leftTable: rentity.aggregation.tableDef, - joinWithDepIdx: this.depProEntityLabelConfig, - joinWhereLeftTableCondition: '= true', - joinOnKeys: { - pkClass: {leftCustom: {name: 'fkClass', type: 'int'}}, - fkProject: {value: PK_DEFAULT_CONFIG_PROJECT} - }, - conditionTrueIf: {}, - createCustomObject: customVal - }) - - /** - * Expand Label parts for each entity - */ - const labelPartsTbl = builder.createTableName(); - const createlabelPartsTbl = builder.createTableStmt(labelPartsTbl) - const labelPartsSql = ` - -- expands all labelParts for each entity - ${createlabelPartsTbl} ( - WITH tw1 AS ( - SELECT - t1. "r_pkEntity", - t1. "p_pkClass", - CASE WHEN (t1.custom->>'fkClass')::int = 365 THEN - '${JSON.stringify(labelPartsForAppeInLang365)}'::jsonb - WHEN t1.custom ->'entityLabelConfig'->> 'labelParts' IS NOT NULL THEN - t1.custom ->'entityLabelConfig'-> 'labelParts' - ELSE - '${JSON.stringify(labelPartsForNormalEntities)}'::jsonb - END label_parts - FROM ${entityLabelConfig.aggregation.tableDef.tableName} t1 - ) - SELECT t1.*, jsonb_array_elements(label_parts) label_part - FROM tw1 t1 - )` - builder.registerTmpTable(labelPartsSql, [], labelPartsTbl) - - /** - * expands all 'slots' per labelPart according to nrOfStatementsInLabel - */ - const slotsTbl = builder.createTableName(); - const createSlotsTbl = builder.createTableStmt(slotsTbl) - const slotsSql = ` - -- expands all 'slots' per labelPart according to nrOfStatementsInLabel - ${createSlotsTbl} ( - SELECT - t1."r_pkEntity", - t1."p_pkClass", - label_part->'field'->>'fkProperty' property, - label_part->'field'->>'nrOfStatementsInLabel', - CASE WHEN (label_part->'field'->>'isOutgoing')::bool = true THEN 'outgoing' ELSE 'incoming' END direction, - label_part->>'ordNum' fielOrdNum, - generate_series(1,(label_part->'field'->>'nrOfStatementsInLabel')::int) stmtOrdNum - FROM ${labelPartsTbl} t1 - )` - builder.registerTmpTable(slotsSql, [], slotsTbl) - - /** - * expands all 'slots' per labelPart according to nrOfStatementsInLabel - */ - const stmtsTbl = builder.createTableName(); - const createStmtsTbl = builder.createTableStmt(stmtsTbl) - const stmtsSql = ` - -- joins slots with statements - ${createStmtsTbl} ( - SELECT - t1."r_pkEntity", t2.property, t2.direction, t2.fielOrdNum, t2.stmtOrdNum, t1.p_val->t2.direction->t2.property->(t2.stmtOrdNum-1) stmt - FROM - ${rEdges.aggregation.tableDef.tableName} t1, - ${slotsTbl} t2 - WHERE - t1."r_pkEntity"=t2."r_pkEntity" - )` - builder.registerTmpTable(stmtsSql, [], stmtsTbl) - - - /** - * stmts relevant for entity label - */ - const finalTbl = builder.createTableName(); - const createFinalTbl = builder.createTableStmt(finalTbl) - const finalSql = ` - -- stmts relevant for entity label - ${createFinalTbl} ( - SELECT - t1."r_pkEntity", - t1.property, - t1.direction, - t1.fielOrdNum, - t1.stmtOrdNum, - (t1.stmt->>'targetIsEntity')::bool condition, - jsonb_build_object( - 'fkTarget', (t1.stmt->>'fkTarget')::int, - 'targetLabel', t1.stmt->>'targetLabel' - ) custom - FROM ${stmtsTbl} t1 - )` - const labelParts = builder.registerTmpTable(finalSql, [], finalTbl) - - const remoteEntityLabelCustom: CustomValSql<{ - string: string - pkEntity: number - property: number - direction: string - fielOrdNum: number - stmtOrdNum: number - condition: boolean - }> = () => { - return `jsonb_build_object( - 'string', t2.val->>'entityLabel', - 'r_pkEntity', t1."r_pkEntity", - 'property', t1.property, - 'direction', t1.direction, - 'fielOrdNum', t1.fielOrdNum, - 'stmtOrdNum', t1.stmtOrdNum - )` - } - const remoteEntityLabel = await builder.joinProviderThroughDepIdx({ - leftTable: labelParts.tableDef, - joinWithDepIdx: this.depREntityLabel, - joinWhereLeftTableCondition: '= true', - joinOnKeys: { - pkEntity: {leftCustom: {name: 'fkTarget', type: 'int'}}, - }, - conditionTrueIf: {}, - createCustomObject: remoteEntityLabelCustom - }) - - // const hook = builder.twOnUpsertHook() - - /** - * aggregate labels - */ - const labelPartsTable = labelParts.tableDef.tableName - const remoteEntityLabelTable = remoteEntityLabel.aggregation.tableDef.tableName - const aggLabelsTbl = builder.createTableName(); - const createAggLabelsTbl = builder.createTableStmt(aggLabelsTbl) - const aggLabelsSql = ` - -- aggregate entity labels - ${createAggLabelsTbl} ( - SELECT - t1."r_pkEntity", - jsonb_build_object( - 'entityLabel', - substring( - coalesce( - string_agg( - coalesce( - t2.custom->>'string', - t1.custom->>'targetLabel' - -- if we ever implement a placeholder for not available stmt, put val here - ), - ', ' - ORDER BY t1.fielOrdNum ASC,t1.stmtOrdNum ASC - ), - '(no label)' - ) FROM 1 FOR ${ENTITY_LABEL_MAX_LENGTH} - ), - 'labelMissing', string_agg(coalesce( t2.custom->>'string', t1.custom->>'targetLabel' ),'') IS NULL - ) val, - true::boolean condition - FROM ${labelPartsTable} t1 - LEFT JOIN ${remoteEntityLabelTable} t2 - ON t1."r_pkEntity"=t2."r_pkEntity" - AND t1.property=t2.custom->>'property' - AND t1.direction=t2.custom->>'direction' - AND t1.fielOrdNum=t2.custom->>'fielOrdNum' - AND t1.stmtOrdNum=(t2.custom->>'stmtOrdNum')::int - GROUP BY t1."r_pkEntity" - )` - const aggregateLabels = builder.registerTmpTable(aggLabelsSql, [], aggLabelsTbl) - - - - await builder.tmpTableUpsertAggregations(this.index, aggregateLabels.tableDef.tableName, '= true') - // await builder.printQueries() - const count = await builder.executeQueries() - - return count - } - - - -} - - -/***** - * OLD AGGREGATOR - * - */ - - -// import {AggregatedDataService} from '../../../base/classes/AggregatedDataService'; -// import {REntityId, rEntityKeyDefs, REntityService} from '../../../primary-ds/entity/REntityService'; -// import {Warehouse} from '../../../Warehouse'; -// import {EntityLabelVal} from '../entity-label.commons'; -// import {REntityLabelAggregator} from './REntityLabelAggregator'; -// import {REntityLabelProviders} from './REntityLabelPoviders'; - -// @Injectable() -// export class REntityLabelService extends AggregatedDataService{ -// creatorDS: REntityService -// aggregator = REntityLabelAggregator; -// providers = REntityLabelProviders; -// constructor(@Inject(forwardRef(() => Warehouse)) wh: Warehouse) { -// super( -// wh, -// rEntityKeyDefs -// ) - -// this.registerCreatorDS(this.wh.prim.rEntity) - -// } - -// getDependencies() { -// return this.wh.dep.rEntityLabel -// }; -// onUpsertSql(tableAlias: string) { -// return ` -// UPDATE war.entity_preview -// SET entity_label = ${tableAlias}.val->>'entityLabel' -// FROM ${tableAlias} -// WHERE pk_entity = ${tableAlias}."pkEntity" -// AND project = 0 -// AND entity_label IS DISTINCT FROM ${tableAlias}.val->>'entityLabel'` -// } -// } - diff --git a/server/src/warehouse/aggregator-ds/entity-paths/PEntityPathService.ts b/server/src/warehouse/aggregator-ds/entity-paths/PEntityPathService.ts deleted file mode 100644 index 08c401d60..000000000 --- a/server/src/warehouse/aggregator-ds/entity-paths/PEntityPathService.ts +++ /dev/null @@ -1,121 +0,0 @@ -// import {forwardRef, Inject, Injectable} from 'injection-js'; -// import {PoolClient} from 'pg'; -// import {AggregatedDataService2} from '../../base/classes/AggregatedDataService2'; -// import {AggregatorSqlBuilder, CustomValSql} from '../../base/classes/AggregatorSqlBuilder'; -// import {DependencyIndex} from '../../base/classes/DependencyIndex'; -// import {RClassId} from '../../primary-ds/DfhClassHasTypePropertyService'; -// import {EntityFields} from '../../primary-ds/edge/edge.commons'; -// import {PEdgeService} from '../../primary-ds/edge/PEdgeService'; -// import {PEntity, PEntityId, pEntityKeyDefs, PEntityService} from '../../primary-ds/entity/PEntityService'; -// import {EntityPathConfigService, EntityPathConfigVal} from '../../primary-ds/EntityPathConfigService'; -// import {Warehouse} from '../../Warehouse'; -// interface EntityPathEntity { -// pkClass: number, -// pkEntity: number, -// outgoing: EntityPathProperty[] -// incoming: EntityPathProperty[] -// // label: 'Zitrusfrüchte' - -// } - -// interface EntityPathProperty { -// pkProperty: number, -// entities: [] -// // label: 'has broader term', -// } - -// interface EntityPathVal { -// [pathName: string]: EntityPathEntity -// } - -// @Injectable() -// export class PEntityPathService extends AggregatedDataService2{ - -// depPEntity: DependencyIndex -// depEntityPathConfig: DependencyIndex -// // depPEntityLabel: DependencyIndex -// depPEdge: DependencyIndex - -// batchSize = 100000; -// constructor( -// @Inject(forwardRef(() => Warehouse)) wh: Warehouse, -// @Inject(forwardRef(() => EntityPathConfigService)) entityPathConfig: EntityPathConfigService, -// @Inject(forwardRef(() => PEntityService)) pEntity: PEntityService, -// @Inject(forwardRef(() => PEdgeService)) pEdge: PEdgeService, -// ) { -// super( -// wh, -// pEntityKeyDefs -// ) -// this.registerCreatorDS({dataService: pEntity}) -// this.depEntityPathConfig = this.addDepencency(entityPathConfig); -// this.depPEntity = this.addDepencency(pEntity); -// // this.depPEntityLabel = this.addDepencency(this); -// this.depPEdge = this.addDepencency(pEdge); -// } -// getDependencies() { -// return this -// }; - - -// async aggregateBatch(client: PoolClient, client2: PoolClient, limit: number, offset: number, currentTimestamp: string): Promise { -// const builder = new AggregatorSqlBuilder(this, client, currentTimestamp, limit, offset) - -// const pentity = await builder.joinProviderThroughDepIdx({ -// leftTable: builder.batchTmpTable.tableDef, -// joinWithDepIdx: this.depPEntity, -// joinOnKeys: { -// pkEntity: {leftCol: 'pkEntity'}, -// fkProject: {leftCol: 'fkProject'} -// }, -// conditionTrueIf: { -// providerKey: {pkEntity: 'IS NOT NULL'} -// }, -// createCustomObject: (() => `jsonb_build_object('fkClass', (t2.val->>'fkClass')::int)`) as CustomValSql<{fkClass: number}>, -// }) -// const pEdges = await builder.joinProviderThroughDepIdx({ -// leftTable: pentity.aggregation.tableDef, -// joinWithDepIdx: this.depPEdge, -// joinWhereLeftTableCondition: '= true', -// joinOnKeys: { -// pkEntity: {leftCol: 'pkEntity'}, -// fkProject: {leftCol: 'fkProject'} -// }, -// conditionTrueIf: { -// or: [ -// {providerVal: {outgoing: 'is not {}', }}, -// {providerVal: {incoming: 'is not {}', }}, -// ] -// }, -// createCustomObject: ((provider) => `t1.custom`) as CustomValSql<{fkClass: number}>, -// }) - -// const entityPathConfig = await builder.joinProviderThroughDepIdx({ -// leftTable: pentity.aggregation.tableDef, -// joinWithDepIdx: this.depEntityPathConfig, -// joinWhereLeftTableCondition: '= true', -// joinOnKeys: { -// pkClass: { -// leftVal: {name: 'fkClass', type: 'int'} -// } -// }, -// conditionTrueIf: { -// providerKey: {pkClass: 'IS NOT NULL'} -// }, -// createCustomObject: ((provider) => `jsonb_build_object( -// 'fkClass', t1.val-à>>'fkClass')::int, -// 'pathConfigs', t2.val->'pathConfigs' - -// )`) as CustomValSql<{fkClass: number}>, -// }) - - -// await builder.printQueries() -// const count = builder.executeQueries() - -// return count -// } - - -// } - diff --git a/server/src/warehouse/aggregator-ds/entity-preview/EntityPreviewService.ts b/server/src/warehouse/aggregator-ds/entity-preview/EntityPreviewService.ts deleted file mode 100644 index 7238ed094..000000000 --- a/server/src/warehouse/aggregator-ds/entity-preview/EntityPreviewService.ts +++ /dev/null @@ -1,371 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -/* eslint-disable @typescript-eslint/naming-convention */ -import {forwardRef, Inject, Injectable, Optional} from 'injection-js'; -import {PoolClient} from 'pg'; -import {AggregatedDataService2} from '../../base/classes/AggregatedDataService2'; -import {AggregatorSqlBuilder, JoinedProvider} from '../../base/classes/AggregatorSqlBuilder'; -import {DependencyIndex} from '../../base/classes/DependencyIndex'; -import {PEntityId, pEntityKeyDefs, PEntityService} from '../../primary-ds/entity/PEntityService'; -import {REntityId, REntityService} from '../../primary-ds/entity/REntityService'; -import {Warehouse} from '../../Warehouse'; -import {EntityLabelVal} from '../entity-label/entity-label.commons'; -import {PEntityLabelService} from '../entity-label/p-entity-label/PEntityLabelService'; -import {REntityLabelService} from '../entity-label/r-entity-label/REntityLabelService'; -import {PEntityClassLabelService, PEntityClassLabelVal} from '../entity-class-label/p-entity-class-label/PEntityClassLabelService'; -import {REntityClassLabelService, REntityClassLabelVal} from '../entity-class-label/r-entity-class-label/REntityClassLabelService'; -import {PEntityFullTextService, PEntityFullTextVal} from '../entity-full-text/p-entity-full-text/PEntityFullTextService'; -import {REntityFullTextService, REntityFullTextVal} from '../entity-full-text/r-entity-full-text/REntityFullTextService'; -import {PEntityTimeSpanVal, PEntityTimeSpanService} from '../entity-time-span/p-entity-time-span/PEntityTimeSpanService'; -import {REntityTimeSpanVal, REntityTimeSpanService, REntityTimeSpan} from '../entity-time-span/r-entity-time-span/REntityTimeSpanService'; -import {PEntityTypeVal, PEntityTypeService} from '../entity-type/p-entity-type/PEntityTypeService'; -import {REntityTypeVal, REntityTypeService} from '../entity-type/r-entity-type/REntityTypeService'; - -export type EntityPreviewVal = { - entityLabel: string, - classLabel: string, - fullText: string - timeSpan?: REntityTimeSpan; - firstSecond?: number - lastSecond?: number - fkType: number - typeLabel: string -} -type JoinedProviders = (JoinedProvider | undefined)[]; - -interface PropDef { - r: string; - p?: string | { - n: string; - t: 'int' | 'text' | 'json'; - }; -} - -interface JoinOptions { - p: JoinedProviders; - builder: AggregatorSqlBuilder; - properties: PropDef[]; - depIdx?: DependencyIndex; -} - -@Injectable() -export class EntityPreviewService extends AggregatedDataService2{ - - - depPEntityLabel: DependencyIndex - depREntityLabel: DependencyIndex - - depPEntityClassLabel: DependencyIndex - depREntityClassLabel: DependencyIndex - - depPEntityFullText: DependencyIndex - depREntityFullText: DependencyIndex - - depPEntityTimeSpan: DependencyIndex - depREntityTimeSpan: DependencyIndex - - depPEntityType: DependencyIndex - depREntityType: DependencyIndex - - - - batchSize = 100000; - - constructor( - @Inject(forwardRef(() => Warehouse)) wh: Warehouse, - @Optional() @Inject(forwardRef(() => PEntityService)) pEntity?: PEntityService, - @Optional() @Inject(forwardRef(() => REntityService)) rEntity?: REntityService, - @Optional() @Inject(forwardRef(() => PEntityLabelService)) pEntityLabel?: PEntityLabelService, - @Optional() @Inject(forwardRef(() => REntityLabelService)) rEntityLabel?: REntityLabelService, - @Optional() @Inject(forwardRef(() => PEntityClassLabelService)) pEntityClassLabel?: PEntityClassLabelService, - @Optional() @Inject(forwardRef(() => REntityClassLabelService)) rEntityClassLabel?: REntityClassLabelService, - @Optional() @Inject(forwardRef(() => PEntityFullTextService)) pEntityFullText?: PEntityFullTextService, - @Optional() @Inject(forwardRef(() => REntityFullTextService)) rEntityFullText?: REntityFullTextService, - @Optional() @Inject(forwardRef(() => PEntityTimeSpanService)) pEntityTimeSpan?: PEntityTimeSpanService, - @Optional() @Inject(forwardRef(() => REntityTimeSpanService)) rEntityTimeSpan?: REntityTimeSpanService, - @Optional() @Inject(forwardRef(() => PEntityTypeService)) pEntityType?: PEntityTypeService, - @Optional() @Inject(forwardRef(() => REntityTypeService)) rEntityType?: REntityTypeService, - - ) { - super( - wh, - pEntityKeyDefs - ) - if (pEntity) this.registerCreatorDS({dataService: pEntity}); - if (rEntity) this.registerCreatorDS({ - dataService: rEntity, - customSql: [{ - select: `"pkEntity", 0::int as "fkProject"`, - }] - }); - - // Entity Label - if (pEntityLabel) this.depPEntityLabel = this.addDepencency(pEntityLabel) - if (rEntityLabel) this.depREntityLabel = this.addDepencency(rEntityLabel) - - // Entity Class Label - if (pEntityClassLabel) this.depPEntityClassLabel = this.addDepencency(pEntityClassLabel) - if (rEntityClassLabel) this.depREntityClassLabel = this.addDepencency(rEntityClassLabel) - - // Entity Full Text - if (pEntityFullText) this.depPEntityFullText = this.addDepencency(pEntityFullText) - if (rEntityFullText) this.depREntityFullText = this.addDepencency(rEntityFullText) - - // Entity Time Span - if (pEntityTimeSpan) this.depPEntityTimeSpan = this.addDepencency(pEntityTimeSpan) - if (rEntityTimeSpan) this.depREntityTimeSpan = this.addDepencency(rEntityTimeSpan) - - // Entity Type - if (pEntityType) this.depPEntityType = this.addDepencency(pEntityType) - if (rEntityType) this.depREntityType = this.addDepencency(rEntityType) - } - - onUpsertSql(tableAlias: string) { - return ` - UPDATE war.entity_preview - SET - class_label = val->>'classLabel', - entity_label = val->>'entityLabel', - full_text = val->>'fullText', - type_label = val->>'typeLabel', - fk_type = (val->>'fkType')::int, - time_span = val->'timeSpan', - first_second = (val->>'firstSecond')::bigint, - last_second = (val->>'lastSecond')::bigint - FROM ${tableAlias} - WHERE pk_entity = "pkEntity" - AND project = "fkProject" - AND ( - entity_label, - class_label, - entity_label, - full_text, - type_label, - fk_type, - time_span, - first_second, - last_second - ) - IS DISTINCT FROM - ( - val->>'entityLabel', - val->>'classLabel', - val->>'entityLabel', - val->>'fullText', - val->>'typeLabel', - (val->>'fkType')::int, - val->'timeSpan', - (val->>'firstSecond')::bigint, - (val->>'lastSecond')::bigint - ) - ` - - } - async aggregateBatch(client: PoolClient, client2: PoolClient, limit: number, offset: number, currentTimestamp: string): Promise { - const builder = new AggregatorSqlBuilder(this, client, currentTimestamp, limit, offset) - - const p: JoinedProviders = [] - // EntityLabel - await this.joinProjectColumnTable({p, builder, properties: [{r: 'entityLabel', p: 'entityLabel'}], depIdx: this.depPEntityLabel}) - await this.joinRepoColumnTable({p, builder, properties: [{r: 'entityLabel', p: 'entityLabel'}], depIdx: this.depREntityLabel}) - - // EntityClassLabel - await this.joinProjectColumnTable({p, builder, properties: [{r: 'classLabel', p: 'entityClassLabel'}], depIdx: this.depPEntityClassLabel}) - await this.joinRepoColumnTable({p, builder, properties: [{r: 'classLabel', p: 'entityClassLabel'}], depIdx: this.depREntityClassLabel}) - - // EntityFullText - await this.joinProjectColumnTable({p, builder, properties: [{r: 'fullText', p: 'fullText'}], depIdx: this.depPEntityFullText}) - await this.joinRepoColumnTable({p, builder, properties: [{r: 'fullText', p: 'fullText'}], depIdx: this.depREntityFullText}) - - // EntityTimeSpan - await this.joinProjectColumnTable({ - p, builder, properties: [ - {r: 'timeSpan', p: {n: 'timeSpan', t: 'json'}}, - {r: 'firstSecond', p: 'firstSecond'}, - {r: 'lastSecond', p: 'lastSecond'} - ], depIdx: this.depPEntityTimeSpan - }) - await this.joinRepoColumnTable({ - p, builder, properties: [ - {r: 'timeSpan', p: {n: 'timeSpan', t: 'json'}}, - {r: 'firstSecond', p: 'firstSecond'}, - {r: 'lastSecond', p: 'lastSecond'} - ], depIdx: this.depREntityTimeSpan - }) - - // EntityTimeSpan - await this.joinProjectColumnTable({p, builder, properties: [{r: 'fkType', p: {n: 'fkType', t: 'int'}}, {r: 'typeLabel', p: 'typeLabel'}], depIdx: this.depPEntityType}) - await this.joinRepoColumnTable({p, builder, properties: [{r: 'fkType', p: {n: 'fkType', t: 'int'}}, {r: 'typeLabel', p: 'typeLabel'}], depIdx: this.depREntityType}) - - const providers = p.filter(item => !!item) as JoinedProvider< - PEntityId, - EntityPreviewVal, - Partial - >[] - /** - * join columns of entity preview - */ - const joinTbl = builder.createTableName(); - const createJoinTbl = builder.createTableStmt(joinTbl) - if (!createJoinTbl.length) throw new Error("at least one join table needed"); - - const joinSql = ` - -- join columns of entity preview - ${createJoinTbl} ( - SELECT - t1."r_pkEntity", - t1."r_fkProject", - ${providers.map((x, i) => `t${i + 2}.custom`).join(' || ')} val - FROM ${builder.batchTmpTable.tableDef.tableName} t1 - ${providers.map((x, i) => ` - LEFT JOIN ${x.aggregation.tableDef.tableName} t${i + 2} - ON t1."r_pkEntity"=t${i + 2}."r_pkEntity" - AND t1."r_fkProject"=t${i + 2}."r_fkProject" - `).join('\n')} - - )` - const joinedColumns = builder.registerTmpTable(joinSql, [], joinTbl) - - - await builder.tmpTableUpsertAggregations(this.index, joinedColumns.tableDef.tableName) - builder.registerUpsertHook( - {client: client2, table: 'war.entity_preview'}, - (insertClause, fromClause) => ` - UPDATE war.entity_preview - SET - class_label = val->>'classLabel', - entity_label = val->>'entityLabel', - full_text = val->>'fullText', - type_label = val->>'typeLabel', - fk_type = (val->>'fkType')::int, - time_span = val->'timeSpan', - first_second = (val->>'firstSecond')::bigint, - last_second = (val->>'lastSecond')::bigint - FROM ${fromClause} t1 - WHERE pk_entity = "pkEntity" - AND project = "fkProject" - AND ( - entity_label, - class_label, - entity_label, - full_text, - type_label, - fk_type, - time_span, - first_second, - last_second - ) - IS DISTINCT FROM - ( - val->>'entityLabel', - val->>'classLabel', - val->>'entityLabel', - val->>'fullText', - val->>'typeLabel', - (val->>'fkType')::int, - val->'timeSpan', - (val->>'firstSecond')::bigint, - (val->>'lastSecond')::bigint - ) - ` - ) - - // await builder.printQueries() - const count = await builder.executeQueries() - return count - } - - - async joinProjectColumnTable({p, builder, properties, depIdx}: JoinOptions) { - if (depIdx) { - const x = await builder.joinProviderThroughDepIdx({ - leftTable: builder.batchTmpTable.tableDef, - joinWithDepIdx: depIdx, - joinOnKeys: { - pkEntity: {leftCol: 'pkEntity'}, - fkProject: {leftCol: 'fkProject'} - }, - conditionTrueIf: {}, - createCustomObject: () => this.createProviderCustom(properties), - }) - p.push(x) - } - } - async joinRepoColumnTable({p, builder, properties, depIdx}: JoinOptions) { - if (depIdx) { - const x = await builder.joinProviderThroughDepIdx({ - leftTable: builder.batchTmpTable.tableDef, - joinWithDepIdx: depIdx, - joinOnKeys: { - pkEntity: {leftCol: 'pkEntity'}, - }, - joinOnCustom: ['"r_fkProject" = 0'], - conditionTrueIf: {}, - createCustomObject: () => this.createProviderCustom(properties), - }) - p.push(x) - } - } - - - private createProviderCustom(properties: PropDef[]): string { - return `jsonb_strip_nulls(jsonb_build_object( - ${properties.map(prop => { - if (!prop.p) - return `'${prop.r}', t2.val`; - else if (typeof prop.p === 'string') { - return `'${prop.r}', t2.val->>'${prop.p}'`; - } - else { - if (prop.p.t === 'json') { - return `'${prop.r}', t2.val->'${prop.p.n}'`; - } else { - return `'${prop.r}', (t2.val->>'${prop.p.n}')::${prop.p.t}`; - } - } - }).join(',\n')} - ))`; - } -} - - - - - - - - - -// import {AggregatedDataService} from '../../../base/classes/AggregatedDataService'; -// import {RClassService} from '../../../primary-ds/class/RClassService'; -// import {RClassId, rClassIdKeyDefs} from '../../../primary-ds/DfhClassHasTypePropertyService'; -// import {Warehouse} from '../../../Warehouse'; -// import {RClassLabelAggregator} from './RClassLabelAggregator'; -// import {RClassLabelProviders} from './RClassLabelProviders'; - -// export type RClassLabelValue = {label?: string} -// @Injectable() -// export class EntityPreview extends AggregatedDataService{ -// creatorDS: RClassService -// aggregator = RClassLabelAggregator; -// providers = RClassLabelProviders; -// constructor(@Inject(forwardRef(() => Warehouse)) wh: Warehouse) { -// super( -// wh, -// rClassIdKeyDefs -// ) -// this.registerCreatorDS(wh.prim.rClass); - -// } -// getDependencies() { -// return this.wh.dep.rClassLabel -// }; -// onUpsertSql(tableAlias: string) { -// return ` -// INSERT INTO war.class_preview (fk_class, fk_project, label) -// SELECT "pkClass", 0, val->>'label' -// FROM ${tableAlias} -// ON CONFLICT (fk_class, fk_project) DO UPDATE -// SET label = EXCLUDED.label -// WHERE EXCLUDED.label IS DISTINCT FROM war.class_preview.label` -// } -// } diff --git a/server/src/warehouse/aggregator-ds/entity-time-span/p-entity-time-span/PEntityTimeSpanAggregator.ts b/server/src/warehouse/aggregator-ds/entity-time-span/p-entity-time-span/PEntityTimeSpanAggregator.ts deleted file mode 100644 index 138cb8993..000000000 --- a/server/src/warehouse/aggregator-ds/entity-time-span/p-entity-time-span/PEntityTimeSpanAggregator.ts +++ /dev/null @@ -1,92 +0,0 @@ -// import {keys} from 'ramda'; -// import {AbstractAggregator} from '../../../base/classes/AbstractAggregator'; -// import {PEntityId} from '../../../primary-ds/entity/PEntityService'; -// import {PEntityTimeSpanProviders} from './PEntityTimeSpanPoviders'; -// import {PEntityTimeSpan, TimeSpanKeys, PEntityTimeSpanVal} from './PEntityTimeSpanService'; -// type KeyMap = { -// [key in TimeSpanKeys]: number -// } -// const keyMap: KeyMap = { -// p81: 71, -// p82: 72, -// p81a: 150, -// p81b: 151, -// p82a: 152, -// p82b: 153, -// } -// const ks = keys(keyMap) -// export class PEntityTimeSpanAggregator extends AbstractAggregator { - -// // the resulting entityTimeSpan -// entityTimeSpan?: PEntityTimeSpan; -// // the first second of all time primitives of this time span -// firstSecond?: number; -// // the last second of all time primitives of this time span -// lastSecond?: number; - - -// constructor( -// public providers: PEntityTimeSpanProviders, -// public id: PEntityId -// ) { -// super() -// } - - -// /************************************************************************ -// * Methods for creating entity time span -// ************************************************************************/ - -// /** -// * Create entity time span -// * -// * Gets values from Indexes and caches dependencies in itself. -// */ -// async create() { - -// const entity = await this.providers.pEntity.get(this.id); - -// if (entity) { - -// const timeSpan: PEntityTimeSpan = {} -// let first = Number.POSITIVE_INFINITY; -// let last = Number.NEGATIVE_INFINITY; -// const fieldsWithEdges = await this.providers.pEdges.get(this.id); -// if (fieldsWithEdges) { -// for (const k of ks) { -// const fkProperty = keyMap[k]; -// const tpWithBounaries = fieldsWithEdges.outgoing?.[fkProperty]?.[0].targetValue?.timePrimitive ?? undefined -// if (tpWithBounaries) { -// timeSpan[k] = { -// julianDay: tpWithBounaries?.julianDay, -// duration: tpWithBounaries?.duration, -// calendar: tpWithBounaries?.calendar, -// } -// // retrieve the earliest second of time span -// if (tpWithBounaries.firstSecond && tpWithBounaries.firstSecond < first) { -// first = tpWithBounaries.firstSecond; -// } -// // retrieve the latest second of time span -// if (tpWithBounaries.lastSecond && tpWithBounaries.lastSecond > last) { -// last = tpWithBounaries.lastSecond; -// } -// } - -// } -// } - -// if (first !== Number.POSITIVE_INFINITY) this.firstSecond = first; -// if (last !== Number.NEGATIVE_INFINITY) this.lastSecond = last; -// if (Object.keys(timeSpan).length) this.entityTimeSpan = timeSpan - -// } -// return this.finalize() -// } -// finalize(): PEntityTimeSpanVal { -// return { -// firstSecond: this.firstSecond, -// lastSecond: this.lastSecond, -// timeSpan: this.entityTimeSpan -// } -// } -// } diff --git a/server/src/warehouse/aggregator-ds/entity-time-span/p-entity-time-span/PEntityTimeSpanDependencies.ts b/server/src/warehouse/aggregator-ds/entity-time-span/p-entity-time-span/PEntityTimeSpanDependencies.ts deleted file mode 100644 index b45418a29..000000000 --- a/server/src/warehouse/aggregator-ds/entity-time-span/p-entity-time-span/PEntityTimeSpanDependencies.ts +++ /dev/null @@ -1,55 +0,0 @@ -// import {Dependencies} from '../../../base/classes/Dependencies' -// import {DependencyIndex} from '../../../base/classes/DependencyIndex' -// import {EntityFields} from "../../../primary-ds/edge/edge.commons" -// import {PEntity, PEntityId} from '../../../primary-ds/entity/PEntityService' -// import {Warehouse} from '../../../Warehouse' -// import {PEntityTimeSpanVal} from './PEntityTimeSpanService' -// import {Injectable, Inject, forwardRef} from 'injection-js'; - -// @Injectable() -// export class PEntityTimeSpanDependencies extends Dependencies { - -// //the timespan depends on: -// //duration: information.time_primitive column duration -// //julian day: information.time_primitive column julian_day -// //calendar: project.info_proj_rel column calendar -// //property of timespan (71, 72, 150, 151, 152, 153): information.statement - -// /** -// * [DONE] To have the duration/julian_day we need to first know if the entity is a PeIt or a teEn -// * If it is a teEn, we need to find the statement($1) associating this teEn with one of the timespan properties (71, 72, 150, 151, 152, 153) with a timeprimitive -// * When we have found it we have 3/4 of the informations. -// * Now we still need to get the calendar type: find the calendar type of the relation between the statement $1 and the project -// * -// * -// * -// * So to get the EntityTimeSpan we need: -// * - pEntity : to know if it is a peIt or teEn -// * - pEdge : to access the outgoing properties (71, 72, 150, 151, 152, 153) -// * - (xxx): in the pEdge : the timeprimitive to have duration/julian_day -// * - (yyy): in the pEdge : the project to access the calendar -// * -// */ - -// pEntity: DependencyIndex -// pEdge: DependencyIndex - -// constructor(@Inject(forwardRef(() => Warehouse)) private wh: Warehouse) { -// super() -// // stores the dependency of entityType (receiver) on entity (provider) -// this.pEntity = this.registerDepIdx(new DependencyIndex( -// this.wh, -// this.wh.agg.pEntityTimeSpan, -// this.wh.prim.pEntity, -// )) - -// // stores the dependency of entityType (receiver) on edge (provider) -// this.pEdge = this.registerDepIdx(new DependencyIndex( -// this.wh, -// this.wh.agg.pEntityTimeSpan, -// this.wh.prim.pEdge, -// )); - -// } - -// } diff --git a/server/src/warehouse/aggregator-ds/entity-time-span/p-entity-time-span/PEntityTimeSpanPoviders.ts b/server/src/warehouse/aggregator-ds/entity-time-span/p-entity-time-span/PEntityTimeSpanPoviders.ts deleted file mode 100644 index e5536ef46..000000000 --- a/server/src/warehouse/aggregator-ds/entity-time-span/p-entity-time-span/PEntityTimeSpanPoviders.ts +++ /dev/null @@ -1,23 +0,0 @@ -// import {Provider} from '../../../base/classes/Provider'; -// import {Providers} from "../../../base/interfaces/Providers"; -// import {DfhClassHasTypePropVal, RClassId} from '../../../primary-ds/DfhClassHasTypePropertyService'; -// import {EntityFields} from "../../../primary-ds/edge/edge.commons"; -// import {PEntity, PEntityId} from '../../../primary-ds/entity/PEntityService'; -// import {PEntityTimeSpanService, PEntityTimeSpanVal} from './PEntityTimeSpanService'; - -// export class PEntityTimeSpanProviders extends Providers { -// pEntity: Provider; -// pEdges: Provider; -// pEntityLabel: Provider; -// dfhClassHasTypeProp: Provider; -// constructor( -// dep: PEntityTimeSpanService, -// protected receiverKey: PEntityId -// ) { -// super() -// this.pEntity = this.registerProvider(dep.depPEntity, receiverKey) -// this.pEdges = this.registerProvider(dep.depPEdge, receiverKey) -// } - -// } - diff --git a/server/src/warehouse/aggregator-ds/entity-time-span/p-entity-time-span/PEntityTimeSpanService.ts b/server/src/warehouse/aggregator-ds/entity-time-span/p-entity-time-span/PEntityTimeSpanService.ts deleted file mode 100644 index e1d364811..000000000 --- a/server/src/warehouse/aggregator-ds/entity-time-span/p-entity-time-span/PEntityTimeSpanService.ts +++ /dev/null @@ -1,187 +0,0 @@ -import {forwardRef, Inject, Injectable} from 'injection-js'; -import {PoolClient} from 'pg'; -import {WarEntityPreviewTimeSpan} from '../../../../models/entity-preview/WarEntityPreviewTimeSpan'; -import {AggregatedDataService2} from '../../../base/classes/AggregatedDataService2'; -import {AggregatorSqlBuilder, CustomValSql} from '../../../base/classes/AggregatorSqlBuilder'; -import {DependencyIndex} from '../../../base/classes/DependencyIndex'; -import {EntityFields, EntityTimePrimitive} from "../../../primary-ds/edge/edge.commons"; -import {PEdgeService} from '../../../primary-ds/edge/PEdgeService'; -import {PEntity, PEntityId, pEntityKeyDefs, PEntityService} from '../../../primary-ds/entity/PEntityService'; -import {Warehouse} from '../../../Warehouse'; - -export type TimeSpanKeys = - 'p82' // At some time within | outer bounds | not before – not after - | 'p81' // Ongoing throughout | inner bounds | surely from – surely to - | 'p81a' // end of the begin | left inner bound | surely from - | 'p82a' // begin of the begin | left outer bound | not before - | 'p81b' // begin of the end | right inner bound | surely to - | 'p82b' // end of the end | right outer bound | not after -export type PEntityTimeSpanVal = { - timeSpan?: WarEntityPreviewTimeSpan//PEntityTimeSpan; - firstSecond?: number - lastSecond?: number -} -export type PEntityTimeSpan = { - [key in TimeSpanKeys]?: EntityTimePrimitive; -} - -/** - * This Data Service manages the key-value store containing - * as a key the PEntityId (pkEntity and fkProject) - * and as value the PEntityTimeSpanVal (fkType, typeLabel) - * - * One example key-value pair in the this.index is: - * Key for the Project Entity Geo. Place 'Madrid' with pkEntity = 2002 in fkProject = 3001 - * - '2002_3001' - * - * Val for the Geo. Place Type 'City' with pkEntity = 2003 in fkProject = 3001 - * - fkType: 2003 - * - typeLabel: 'Citiy' - * - * - * - * -> The Val is the result of the PEntityTimeSpanAggregator - * - */ -@Injectable() -export class PEntityTimeSpanService extends AggregatedDataService2{ - - depPEntity: DependencyIndex - depPEdge: DependencyIndex - batchSize = 100000; - constructor( - @Inject(forwardRef(() => Warehouse)) wh: Warehouse, - @Inject(forwardRef(() => PEntityService)) pEntity: PEntityService, - @Inject(forwardRef(() => PEdgeService)) pEdge: PEdgeService - ) { - super( - wh, - pEntityKeyDefs - ) - - this.registerCreatorDS({ - dataService: pEntity, - customSql: [ - { - where: `val->>'entityType' = 'teEn'`, - } - ] - }) - this.depPEntity = this.addDepencency(pEntity); - this.depPEdge = this.addDepencency(pEdge); - - } - - async aggregateBatch(client: PoolClient, client2: PoolClient, limit: number, offset: number, currentTimestamp: string): Promise { - const builder = new AggregatorSqlBuilder(this, client, currentTimestamp, limit, offset) - - const pentity = await builder.joinProviderThroughDepIdx({ - leftTable: builder.batchTmpTable.tableDef, - joinWithDepIdx: this.depPEntity, - joinOnKeys: { - pkEntity: {leftCol: 'pkEntity'}, - fkProject: {leftCol: 'fkProject'} - }, - conditionTrueIf: { - providerKey: {pkEntity: 'IS NOT NULL'} - }, - createCustomObject: (() => `jsonb_build_object('fkClass', (t2.val->>'fkClass')::int)`) as CustomValSql<{fkClass: number}>, - }) - await builder.joinProviderThroughDepIdx({ - leftTable: pentity.aggregation.tableDef, - joinWithDepIdx: this.depPEdge, - joinWhereLeftTableCondition: '= true', - joinOnKeys: { - pkEntity: {leftCol: 'pkEntity'}, - fkProject: {leftCol: 'fkProject'} - }, - conditionTrueIf: {}, - createAggregationVal: { - upsert: true, - sql: () => ` - jsonb_strip_nulls(jsonb_build_object( - 'timeSpan', json_strip_nulls(json_build_object( - 'p81', CASE WHEN t2.val->'outgoing'->'71' IS NOT NULL THEN json_build_object( - 'julianDay', t2.val->'outgoing'->'71'->0->'targetValue'->'timePrimitive'->'julianDay', - 'duration', t2.val->'outgoing'->'71'->0->'targetValue'->'timePrimitive'->'duration', - 'calendar', t2.val->'outgoing'->'71'->0->'targetValue'->'timePrimitive'->'calendar' - ) END, - 'p82', CASE WHEN t2.val->'outgoing'->'72' IS NOT NULL THEN json_build_object( - 'julianDay', t2.val->'outgoing'->'72'->0->'targetValue'->'timePrimitive'->'julianDay', - 'duration', t2.val->'outgoing'->'72'->0->'targetValue'->'timePrimitive'->'duration', - 'calendar', t2.val->'outgoing'->'72'->0->'targetValue'->'timePrimitive'->'calendar' - ) END, - 'p81a', CASE WHEN t2.val->'outgoing'->'150' IS NOT NULL THEN json_build_object( - 'julianDay', t2.val->'outgoing'->'150'->0->'targetValue'->'timePrimitive'->'julianDay', - 'duration', t2.val->'outgoing'->'150'->0->'targetValue'->'timePrimitive'->'duration', - 'calendar', t2.val->'outgoing'->'150'->0->'targetValue'->'timePrimitive'->'calendar' - ) END, - 'p81b', CASE WHEN t2.val->'outgoing'->'151' IS NOT NULL THEN json_build_object( - 'julianDay', t2.val->'outgoing'->'151'->0->'targetValue'->'timePrimitive'->'julianDay', - 'duration', t2.val->'outgoing'->'151'->0->'targetValue'->'timePrimitive'->'duration', - 'calendar', t2.val->'outgoing'->'151'->0->'targetValue'->'timePrimitive'->'calendar' - ) END, - 'p82a', CASE WHEN t2.val->'outgoing'->'152' IS NOT NULL THEN json_build_object( - 'julianDay', t2.val->'outgoing'->'152'->0->'targetValue'->'timePrimitive'->'julianDay', - 'duration', t2.val->'outgoing'->'152'->0->'targetValue'->'timePrimitive'->'duration', - 'calendar', t2.val->'outgoing'->'152'->0->'targetValue'->'timePrimitive'->'calendar' - ) END, - 'p82b', CASE WHEN t2.val->'outgoing'->'153' IS NOT NULL THEN json_build_object( - 'julianDay', t2.val->'outgoing'->'153'->0->'targetValue'->'timePrimitive'->'julianDay', - 'duration', t2.val->'outgoing'->'153'->0->'targetValue'->'timePrimitive'->'duration', - 'calendar', t2.val->'outgoing'->'153'->0->'targetValue'->'timePrimitive'->'calendar' - ) END - )), - 'firstSecond', (SELECT min(unnest) FROM unnest(ARRAY[ - (t2.val->'outgoing'->'71'->0->'targetValue'->'timePrimitive'->>'firstSecond')::bigint, - (t2.val->'outgoing'->'72'->0->'targetValue'->'timePrimitive'->>'firstSecond')::bigint, - (t2.val->'outgoing'->'150'->0->'targetValue'->'timePrimitive'->>'firstSecond')::bigint, - (t2.val->'outgoing'->'151'->0->'targetValue'->'timePrimitive'->>'firstSecond')::bigint, - (t2.val->'outgoing'->'152'->0->'targetValue'->'timePrimitive'->>'firstSecond')::bigint, - (t2.val->'outgoing'->'153'->0->'targetValue'->'timePrimitive'->>'firstSecond')::bigint - ])), - 'lastSecond', (SELECT max(unnest) FROM unnest(ARRAY[ - (t2.val->'outgoing'->'71'->0->'targetValue'->'timePrimitive'->>'lastSecond')::bigint, - (t2.val->'outgoing'->'72'->0->'targetValue'->'timePrimitive'->>'lastSecond')::bigint, - (t2.val->'outgoing'->'150'->0->'targetValue'->'timePrimitive'->>'lastSecond')::bigint, - (t2.val->'outgoing'->'151'->0->'targetValue'->'timePrimitive'->>'lastSecond')::bigint, - (t2.val->'outgoing'->'152'->0->'targetValue'->'timePrimitive'->>'lastSecond')::bigint, - (t2.val->'outgoing'->'153'->0->'targetValue'->'timePrimitive'->>'lastSecond')::bigint - ])) - )) - ` - } - }) - - - - - // await builder.printQueries() - const count = builder.executeQueries() - - return count - } - - // onUpsertSql(tableAlias: string) { - // return ` - // UPDATE war.entity_preview - // SET time_span = (${tableAlias}.val->>'timeSpan')::jsonb, - // first_second = (${tableAlias}.val->>'firstSecond')::bigint, - // last_second = (${tableAlias}.val->>'lastSecond')::bigint - // FROM ${tableAlias} - // WHERE pk_entity = ${tableAlias}."pkEntity" - // AND project = ${tableAlias}."fkProject" - // AND ( - // time_span, - // first_second, - // last_second - // ) - // IS DISTINCT FROM - // ( - // (${tableAlias}.val->>'timeSpan')::jsonb, - // (${tableAlias}.val->>'firstSecond')::bigint, - // (${tableAlias}.val->>'lastSecond')::bigint - // )` - // } -} - diff --git a/server/src/warehouse/aggregator-ds/entity-time-span/r-entity-time-span/REntityTimeSpanAggregator.ts b/server/src/warehouse/aggregator-ds/entity-time-span/r-entity-time-span/REntityTimeSpanAggregator.ts deleted file mode 100644 index 0985c8821..000000000 --- a/server/src/warehouse/aggregator-ds/entity-time-span/r-entity-time-span/REntityTimeSpanAggregator.ts +++ /dev/null @@ -1,95 +0,0 @@ -import {keys} from 'ramda'; -import {AbstractAggregator} from '../../../base/classes/AbstractAggregator'; -import {REntityId} from '../../../primary-ds/entity/REntityService'; -import {REntityTimeSpanProviders} from './REntityTimeSpanPoviders'; -import {REntityTimeSpan, TimeSpanKeys, REntityTimeSpanVal} from './REntityTimeSpanService'; -type KeyMap = { - [key in TimeSpanKeys]: number -} -const keyMap: KeyMap = { - p81: 71, - p82: 72, - p81a: 150, - p81b: 151, - p82a: 152, - p82b: 153, -} -const ks = keys(keyMap) -export class REntityTimeSpanAggregator extends AbstractAggregator { - - // the resulting entityTimeSpan - entityTimeSpan?: REntityTimeSpan; - // the first second of all time primitives of this time span - firstSecond?: number; - // the last second of all time primitives of this time span - lastSecond?: number; - - - constructor( - public providers: REntityTimeSpanProviders, - public id: REntityId - ) { - super() - } - - - /************************************************************************ - * Methods for creating entity time span - ************************************************************************/ - - /** - * Create entity time span - * - * Gets values from Indexes and caches dependencies in itself. - */ - async create() { - - const entity = await this.providers.rEntity.get(this.id); - - if (entity) { - - const timeSpan: REntityTimeSpan = {} - let first = Number.POSITIVE_INFINITY; - let last = Number.NEGATIVE_INFINITY; - const fieldsWithEdges = await this.providers.rEdges.get(this.id); - if (fieldsWithEdges) { - for (const k of ks) { - const fkProperty = keyMap[k]; - - // TODO: get most often used statement - const tpWithBounaries = fieldsWithEdges.outgoing?.[fkProperty]?.[0].targetValue?.timePrimitive ?? undefined - if (tpWithBounaries) { - timeSpan[k] = { - julianDay: tpWithBounaries?.julianDay, - duration: tpWithBounaries?.duration, - calendar: tpWithBounaries?.calendar, - } - // retrieve the earliest second of time span - if (tpWithBounaries.firstSecond && tpWithBounaries.firstSecond < first) { - first = tpWithBounaries.firstSecond; - } - // retrieve the latest second of time span - if (tpWithBounaries.lastSecond && tpWithBounaries.lastSecond > last) { - last = tpWithBounaries.lastSecond; - } - } - - } - } - - if (first !== Number.POSITIVE_INFINITY) this.firstSecond = first; - if (last !== Number.NEGATIVE_INFINITY) this.lastSecond = last; - if (Object.keys(timeSpan).length) this.entityTimeSpan = timeSpan - - } - return this.finalize() - } - finalize(): REntityTimeSpanVal { - return { - firstSecond: this.firstSecond, - lastSecond: this.lastSecond, - timeSpan: this.entityTimeSpan - } - } - -} diff --git a/server/src/warehouse/aggregator-ds/entity-time-span/r-entity-time-span/REntityTimeSpanDependencies.ts b/server/src/warehouse/aggregator-ds/entity-time-span/r-entity-time-span/REntityTimeSpanDependencies.ts deleted file mode 100644 index 257e338ee..000000000 --- a/server/src/warehouse/aggregator-ds/entity-time-span/r-entity-time-span/REntityTimeSpanDependencies.ts +++ /dev/null @@ -1,55 +0,0 @@ -// import {Dependencies} from '../../../base/classes/Dependencies' -// import {DependencyIndex} from '../../../base/classes/DependencyIndex' -// import {EntityFields} from "../../../primary-ds/edge/edge.commons" -// import {REntity, REntityId} from '../../../primary-ds/entity/REntityService' -// import {Warehouse} from '../../../Warehouse' -// import {REntityTimeSpanVal} from './REntityTimeSpanService' -// import {Injectable, Inject, forwardRef} from 'injection-js'; - -// @Injectable() -// export class REntityTimeSpanDependencies extends Dependencies { - -// //the timespan depends on: -// //duration: information.time_primitive column duration -// //julian day: information.time_primitive column julian_day -// //calendar: project.info_proj_rel column calendar -// //property of timespan (71, 72, 150, 151, 152, 153): information.statement - -// /** -// * [DONE] To have the duration/julian_day we need to first know if the entity is a PeIt or a teEn -// * If it is a teEn, we need to find the statement($1) associating this teEn with one of the timespan properties (71, 72, 150, 151, 152, 153) with a timeprimitive -// * When we have found it we have 3/4 of the informations. -// * Now we still need to get the calendar type: find the calendar type of the relation between the statement $1 and the project -// * -// * -// * -// * So to get the EntityTimeSpan we need: -// * - rEntity : to know if it is a peIt or teEn -// * - pEdge : to access the outgoing properties (71, 72, 150, 151, 152, 153) -// * - (xxx): in the pEdge : the timeprimitive to have duration/julian_day -// * - (yyy): in the pEdge : the project to access the calendar -// * -// */ - -// rEntity: DependencyIndex -// rEdge: DependencyIndex - -// constructor(@Inject(forwardRef(() => Warehouse)) private wh: Warehouse) { -// super() -// // stores the dependency of entityType (receiver) on entity (provider) -// this.rEntity = this.registerDepIdx(new DependencyIndex( -// this.wh, -// this.wh.agg.rEntityTimeSpan, -// this.wh.prim.rEntity -// )) - -// // stores the dependency of entityType (receiver) on edge (provider) -// this.rEdge = this.registerDepIdx(new DependencyIndex( -// this.wh, -// this.wh.agg.rEntityTimeSpan, -// this.wh.prim.rEdge, -// )); - -// } - -// } diff --git a/server/src/warehouse/aggregator-ds/entity-time-span/r-entity-time-span/REntityTimeSpanPoviders.ts b/server/src/warehouse/aggregator-ds/entity-time-span/r-entity-time-span/REntityTimeSpanPoviders.ts deleted file mode 100644 index cd27e16c4..000000000 --- a/server/src/warehouse/aggregator-ds/entity-time-span/r-entity-time-span/REntityTimeSpanPoviders.ts +++ /dev/null @@ -1,22 +0,0 @@ -import {Provider} from '../../../base/classes/Provider'; -import {Providers} from "../../../base/interfaces/Providers"; -import {DfhClassHasTypePropVal, RClassId} from '../../../primary-ds/DfhClassHasTypePropertyService'; -import {EntityFields} from "../../../primary-ds/edge/edge.commons"; -import {REntity, REntityId} from '../../../primary-ds/entity/REntityService'; -import {REntityTimeSpanService, REntityTimeSpanVal} from './REntityTimeSpanService'; -export class REntityTimeSpanProviders extends Providers { - rEntity: Provider; - rEdges: Provider; - rEntityLabel: Provider; - dfhClassHasTypeProp: Provider; - constructor( - dep: REntityTimeSpanService, - protected receiverKey: REntityId - ) { - super() - this.rEntity = this.registerProvider(dep.depREntity, receiverKey) - this.rEdges = this.registerProvider(dep.depREdge, receiverKey) - } - -} - diff --git a/server/src/warehouse/aggregator-ds/entity-time-span/r-entity-time-span/REntityTimeSpanService.ts b/server/src/warehouse/aggregator-ds/entity-time-span/r-entity-time-span/REntityTimeSpanService.ts deleted file mode 100644 index 6f299845e..000000000 --- a/server/src/warehouse/aggregator-ds/entity-time-span/r-entity-time-span/REntityTimeSpanService.ts +++ /dev/null @@ -1,183 +0,0 @@ -import {forwardRef, Inject, Injectable} from 'injection-js'; -import {PoolClient} from 'pg'; -import {AggregatedDataService2} from '../../../base/classes/AggregatedDataService2'; -import {AggregatorSqlBuilder, CustomValSql} from '../../../base/classes/AggregatorSqlBuilder'; -import {DependencyIndex} from '../../../base/classes/DependencyIndex'; -import {EntityFields, EntityTimePrimitive} from "../../../primary-ds/edge/edge.commons"; -import {REdgeService} from '../../../primary-ds/edge/REdgeService'; -import {REntity, REntityId, rEntityKeyDefs, REntityService} from '../../../primary-ds/entity/REntityService'; -import {Warehouse} from '../../../Warehouse'; -import {REntityTimeSpanAggregator} from './REntityTimeSpanAggregator'; -import {REntityTimeSpanProviders} from './REntityTimeSpanPoviders'; - -export type TimeSpanKeys = - 'p82' // At some time within | outer bounds | not before – not after - | 'p81' // Ongoing throughout | inner bounds | surely from – surely to - | 'p81a' // end of the begin | left inner bound | surely from - | 'p82a' // begin of the begin | left outer bound | not before - | 'p81b' // begin of the end | right inner bound | surely to - | 'p82b' // end of the end | right outer bound | not after -export type REntityTimeSpanVal = { - timeSpan?: REntityTimeSpan; - firstSecond?: number - lastSecond?: number -} -export type REntityTimeSpan = { - [key in TimeSpanKeys]?: EntityTimePrimitive; -} - -/** - * This Data Service manages the key-value store containing - * as a key the REntityId (pkEntity and fkProject) - * and as value the REntityTimeSpanVal (fkType, typeLabel) - * - * One example key-value pair in the this.index is: - * Key for the Project Entity Geo. Place 'Madrid' with pkEntity = 2002 in fkProject = 3001 - * - '2002_3001' - * - * Val for the Geo. Place Type 'City' with pkEntity = 2003 in fkProject = 3001 - * - fkType: 2003 - * - typeLabel: 'Citiy' - * - * - * - * -> The Val is the result of the REntityTimeSpanAggregator - * - */ -@Injectable() -export class REntityTimeSpanService extends AggregatedDataService2{ - aggregator = REntityTimeSpanAggregator; - providers = REntityTimeSpanProviders; - - depREntity: DependencyIndex - depREdge: DependencyIndex - batchSize = 100000; - constructor( - @Inject(forwardRef(() => Warehouse)) wh: Warehouse, - @Inject(forwardRef(() => REntityService)) rEntity: REntityService, - @Inject(forwardRef(() => REdgeService)) rEdge: REdgeService - ) { - super( - wh, - rEntityKeyDefs - ) - this.registerCreatorDS({ - dataService: rEntity, - customSql: [ - { - where: `val->>'entityType' = 'teEn'`, - } - ] - }) - this.depREntity = this.addDepencency(rEntity); - this.depREdge = this.addDepencency(rEdge); - } - - async aggregateBatch(client: PoolClient, client2: PoolClient, limit: number, offset: number, currentTimestamp: string): Promise { - const builder = new AggregatorSqlBuilder(this, client, currentTimestamp, limit, offset) - - const pentity = await builder.joinProviderThroughDepIdx({ - leftTable: builder.batchTmpTable.tableDef, - joinWithDepIdx: this.depREntity, - joinOnKeys: { - pkEntity: {leftCol: 'pkEntity'}, - }, - conditionTrueIf: { - providerKey: {pkEntity: 'IS NOT NULL'} - }, - createCustomObject: (() => `jsonb_build_object('fkClass', (t2.val->>'fkClass')::int)`) as CustomValSql<{fkClass: number}>, - }) - await builder.joinProviderThroughDepIdx({ - leftTable: pentity.aggregation.tableDef, - joinWithDepIdx: this.depREdge, - joinWhereLeftTableCondition: '= true', - joinOnKeys: { - pkEntity: {leftCol: 'pkEntity'}, - }, - conditionTrueIf: {}, - createAggregationVal: { - upsert: true, - sql: () => ` - jsonb_strip_nulls(jsonb_build_object( - 'timeSpan', json_strip_nulls(json_build_object( - 'p81', CASE WHEN t2.val->'outgoing'->'71' IS NOT NULL THEN json_build_object( - 'julianDay', t2.val->'outgoing'->'71'->0->'targetValue'->'timePrimitive'->'julianDay', - 'duration', t2.val->'outgoing'->'71'->0->'targetValue'->'timePrimitive'->'duration', - 'calendar', t2.val->'outgoing'->'71'->0->'targetValue'->'timePrimitive'->'calendar' - ) END, - 'p82', CASE WHEN t2.val->'outgoing'->'72' IS NOT NULL THEN json_build_object( - 'julianDay', t2.val->'outgoing'->'72'->0->'targetValue'->'timePrimitive'->'julianDay', - 'duration', t2.val->'outgoing'->'72'->0->'targetValue'->'timePrimitive'->'duration', - 'calendar', t2.val->'outgoing'->'72'->0->'targetValue'->'timePrimitive'->'calendar' - ) END, - 'p81a', CASE WHEN t2.val->'outgoing'->'150' IS NOT NULL THEN json_build_object( - 'julianDay', t2.val->'outgoing'->'150'->0->'targetValue'->'timePrimitive'->'julianDay', - 'duration', t2.val->'outgoing'->'150'->0->'targetValue'->'timePrimitive'->'duration', - 'calendar', t2.val->'outgoing'->'150'->0->'targetValue'->'timePrimitive'->'calendar' - ) END, - 'p81b', CASE WHEN t2.val->'outgoing'->'151' IS NOT NULL THEN json_build_object( - 'julianDay', t2.val->'outgoing'->'151'->0->'targetValue'->'timePrimitive'->'julianDay', - 'duration', t2.val->'outgoing'->'151'->0->'targetValue'->'timePrimitive'->'duration', - 'calendar', t2.val->'outgoing'->'151'->0->'targetValue'->'timePrimitive'->'calendar' - ) END, - 'p82a', CASE WHEN t2.val->'outgoing'->'152' IS NOT NULL THEN json_build_object( - 'julianDay', t2.val->'outgoing'->'152'->0->'targetValue'->'timePrimitive'->'julianDay', - 'duration', t2.val->'outgoing'->'152'->0->'targetValue'->'timePrimitive'->'duration', - 'calendar', t2.val->'outgoing'->'152'->0->'targetValue'->'timePrimitive'->'calendar' - ) END, - 'p82b', CASE WHEN t2.val->'outgoing'->'153' IS NOT NULL THEN json_build_object( - 'julianDay', t2.val->'outgoing'->'153'->0->'targetValue'->'timePrimitive'->'julianDay', - 'duration', t2.val->'outgoing'->'153'->0->'targetValue'->'timePrimitive'->'duration', - 'calendar', t2.val->'outgoing'->'153'->0->'targetValue'->'timePrimitive'->'calendar' - ) END - )), - 'firstSecond', (SELECT min(unnest) FROM unnest(ARRAY[ - (t2.val->'outgoing'->'71'->0->'targetValue'->'timePrimitive'->>'firstSecond')::bigint, - (t2.val->'outgoing'->'72'->0->'targetValue'->'timePrimitive'->>'firstSecond')::bigint, - (t2.val->'outgoing'->'150'->0->'targetValue'->'timePrimitive'->>'firstSecond')::bigint, - (t2.val->'outgoing'->'151'->0->'targetValue'->'timePrimitive'->>'firstSecond')::bigint, - (t2.val->'outgoing'->'152'->0->'targetValue'->'timePrimitive'->>'firstSecond')::bigint, - (t2.val->'outgoing'->'153'->0->'targetValue'->'timePrimitive'->>'firstSecond')::bigint - ])), - 'lastSecond', (SELECT max(unnest) FROM unnest(ARRAY[ - (t2.val->'outgoing'->'71'->0->'targetValue'->'timePrimitive'->>'lastSecond')::bigint, - (t2.val->'outgoing'->'72'->0->'targetValue'->'timePrimitive'->>'lastSecond')::bigint, - (t2.val->'outgoing'->'150'->0->'targetValue'->'timePrimitive'->>'lastSecond')::bigint, - (t2.val->'outgoing'->'151'->0->'targetValue'->'timePrimitive'->>'lastSecond')::bigint, - (t2.val->'outgoing'->'152'->0->'targetValue'->'timePrimitive'->>'lastSecond')::bigint, - (t2.val->'outgoing'->'153'->0->'targetValue'->'timePrimitive'->>'lastSecond')::bigint - ])) - )) - ` - } - }) - - // await builder.printQueries() - const count = builder.executeQueries() - - return count - } - - // onUpsertSql(tableAlias: string) { - // return ` - // UPDATE war.entity_preview - // SET time_span = (${tableAlias}.val->>'timeSpan')::jsonb, - // first_second = (${tableAlias}.val->>'firstSecond')::bigint, - // last_second = (${tableAlias}.val->>'lastSecond')::bigint - // FROM ${tableAlias} - // WHERE pk_entity = ${tableAlias}."pkEntity" - // AND project = 0 - // AND ( - // time_span, - // first_second, - // last_second - // ) - // IS DISTINCT FROM - // ( - // (${tableAlias}.val->>'timeSpan')::jsonb, - // (${tableAlias}.val->>'firstSecond')::bigint, - // (${tableAlias}.val->>'lastSecond')::bigint - // )` - // } -} - diff --git a/server/src/warehouse/aggregator-ds/entity-type/p-entity-type/PEntityTypeAggregator.ts b/server/src/warehouse/aggregator-ds/entity-type/p-entity-type/PEntityTypeAggregator.ts deleted file mode 100644 index 84ead0d49..000000000 --- a/server/src/warehouse/aggregator-ds/entity-type/p-entity-type/PEntityTypeAggregator.ts +++ /dev/null @@ -1,93 +0,0 @@ -import {AbstractAggregator} from '../../../base/classes/AbstractAggregator'; -import {PEntityId} from '../../../primary-ds/entity/PEntityService'; -import {PEntityTypeProviders} from './PEntityTypePoviders'; -import {PEntityTypeVal} from './PEntityTypeService'; - -export interface ClassLabelConfig { - fkProperty: number, - isOutgoing: boolean, - ordNum: number - nrOfStatementsInLabel?: number -} - - -export class PEntityTypeAggregator extends AbstractAggregator { - - // the resulting entityTypeLabel - entityTypeLabel?: string; - fkEntityType?: number; - - // For testing / debugging - labelMissing = true; - - constructor( - public providers: PEntityTypeProviders, - public id: PEntityId - ) { - super() - } - - - /************************************************************************ - * Methods for creating entity type label - ************************************************************************/ - - /** - * Create entity type label - * - * Gets values from Indexes and chaches dependencies in itself. - */ - async create() { - const entity = await this.providers.pEntity.get(this.id); - if (entity) { - - if (!entity.fkClass) return this.finalize() - - const classId = { - pkClass: entity.fkClass - } - - // Find the dfh_pk_property of the 'has type'-subproperty going out of this class - const fkHasTypeSubproperty = await this.providers.dfhClassHasTypeProp.get(classId); - - if (fkHasTypeSubproperty) { - // Get the 'directed-statements' a.k.a. 'edges' of the entity - const fieldsWithEdges = await this.providers.pEdges.get(this.id) - - const hasTypeStmts = fieldsWithEdges?.outgoing?.[fkHasTypeSubproperty.fkProperty]; - - if (hasTypeStmts?.length) { - - // this gives the info for war.entity_preview (fk_type) - this.fkEntityType = hasTypeStmts[0].fkTarget // fk_object_info - - // this gives the info for war.entity_preview (type_label) - const typeEntityId: PEntityId = { - fkProject: entity.fkProject, - pkEntity: hasTypeStmts[0].fkTarget - } - // fetch project variant of the type's entityLabel - let l = await this.providers.pEntityLabel.get(typeEntityId) - // if project variant missing, try to fetch repo variant - if (!l) { - l = await this.providers.rEntityLabel.get({pkEntity: typeEntityId.pkEntity}) - } - - if (l) { - this.entityTypeLabel = l.entityLabel - this.labelMissing = false - } - - } - } - - } - return this.finalize() - } - finalize(): PEntityTypeVal { - return { - typeLabel: this.entityTypeLabel, - fkType: this.fkEntityType - } - } -} diff --git a/server/src/warehouse/aggregator-ds/entity-type/p-entity-type/PEntityTypeDependencies.ts b/server/src/warehouse/aggregator-ds/entity-type/p-entity-type/PEntityTypeDependencies.ts deleted file mode 100644 index 0c7cd0503..000000000 --- a/server/src/warehouse/aggregator-ds/entity-type/p-entity-type/PEntityTypeDependencies.ts +++ /dev/null @@ -1,62 +0,0 @@ -// import {Dependencies} from '../../../base/classes/Dependencies' -// import {DependencyIndex} from '../../../base/classes/DependencyIndex' -// import {DfhClassHasTypePropVal, RClassId} from '../../../primary-ds/DfhClassHasTypePropertyService' -// import {EntityFields} from "../../../primary-ds/edge/edge.commons" -// import {PEntity, PEntityId} from '../../../primary-ds/entity/PEntityService' -// import {REntityId} from '../../../primary-ds/entity/REntityService' -// import {Warehouse} from '../../../Warehouse' -// import {EntityLabelVal} from '../../entity-label/entity-label.commons' -// import {PEntityTypeVal} from './PEntityTypeService' -// import {Injectable, Inject, forwardRef} from 'injection-js'; - -// @Injectable() -// export class PEntityTypeDependencies extends Dependencies { -// pEntity: DependencyIndex -// pEntityLabel: DependencyIndex -// rEntityLabel: DependencyIndex -// pEdge: DependencyIndex -// dfhClassHasTypeProp: DependencyIndex - -// constructor(@Inject(forwardRef(() => Warehouse)) private wh: Warehouse) { -// super() -// // stores the dependency of pEntityType (receiver) on pEntity (provider) -// this.pEntity = this.registerDepIdx(new DependencyIndex( -// this.wh, -// this.wh.agg.pEntityType, -// this.wh.prim.pEntity, -// )) - -// // stores the dependency of pEntityType (receiver) on pEntityLabel (provider) -// this.pEntityLabel = this.registerDepIdx(new DependencyIndex( -// this.wh, -// this.wh.agg.pEntityType, -// this.wh.agg.pEntityLabel, -// )); - -// // stores the dependency of pEntityType (receiver) on rEntityLabel (provider) -// this.rEntityLabel = this.registerDepIdx(new DependencyIndex( -// this.wh, -// this.wh.agg.pEntityType, -// this.wh.agg.rEntityLabel -// )); - -// // stores the dependency of pEntityType (receiver) on pEdge (provider) -// this.pEdge = this.registerDepIdx(new DependencyIndex( -// this.wh, -// this.wh.agg.pEntityType, -// this.wh.prim.pEdge -// )); - -// // stores the dependency of pEntityType (receiver) on dfhClassHasTypeProperty -// this.dfhClassHasTypeProp = this.registerDepIdx(new DependencyIndex( -// this.wh, -// this.wh.agg.pEntityType, -// this.wh.prim.dfhClassHasTypeProperty -// )); - - -// } - - - -// } diff --git a/server/src/warehouse/aggregator-ds/entity-type/p-entity-type/PEntityTypePoviders.ts b/server/src/warehouse/aggregator-ds/entity-type/p-entity-type/PEntityTypePoviders.ts deleted file mode 100644 index da60f28c6..000000000 --- a/server/src/warehouse/aggregator-ds/entity-type/p-entity-type/PEntityTypePoviders.ts +++ /dev/null @@ -1,28 +0,0 @@ -import {Provider} from '../../../base/classes/Provider'; -import {Providers} from "../../../base/interfaces/Providers"; -import {DfhClassHasTypePropVal, RClassId} from '../../../primary-ds/DfhClassHasTypePropertyService'; -import {EntityFields} from "../../../primary-ds/edge/edge.commons"; -import {PEntity, PEntityId} from '../../../primary-ds/entity/PEntityService'; -import {REntityId} from '../../../primary-ds/entity/REntityService'; -import {EntityLabelVal} from '../../entity-label/entity-label.commons'; -import {PEntityTypeService, PEntityTypeVal} from './PEntityTypeService'; -export class PEntityTypeProviders extends Providers { - pEntity: Provider; - pEdges: Provider; - pEntityLabel: Provider; - rEntityLabel: Provider; - dfhClassHasTypeProp: Provider; - constructor( - dep: PEntityTypeService, - protected receiverKey: PEntityId - ) { - super() - this.pEntity = this.registerProvider(dep.depPEntity, receiverKey) - this.pEntityLabel = this.registerProvider(dep.depPEntityLabel, receiverKey); - this.rEntityLabel = this.registerProvider(dep.depREntityLabel, receiverKey); - this.pEdges = this.registerProvider(dep.depPEdge, receiverKey) - this.dfhClassHasTypeProp = this.registerProvider(dep.depDfhClassHasTypeProp, receiverKey) - } - -} - diff --git a/server/src/warehouse/aggregator-ds/entity-type/p-entity-type/PEntityTypeService.ts b/server/src/warehouse/aggregator-ds/entity-type/p-entity-type/PEntityTypeService.ts deleted file mode 100644 index 3c066bfe5..000000000 --- a/server/src/warehouse/aggregator-ds/entity-type/p-entity-type/PEntityTypeService.ts +++ /dev/null @@ -1,200 +0,0 @@ -import {forwardRef, Inject, Injectable} from 'injection-js'; -import {PoolClient} from 'pg'; -import {AggregatedDataService2} from '../../../base/classes/AggregatedDataService2'; -import {AggregatorSqlBuilder, CustomValSql} from '../../../base/classes/AggregatorSqlBuilder'; -import {DependencyIndex} from '../../../base/classes/DependencyIndex'; -import {DfhClassHasTypePropertyService, DfhClassHasTypePropVal, RClassId} from '../../../primary-ds/DfhClassHasTypePropertyService'; -import {EntityFields} from '../../../primary-ds/edge/edge.commons'; -import {PEdgeService} from '../../../primary-ds/edge/PEdgeService'; -import {PEntity, PEntityId, pEntityKeyDefs, PEntityService} from '../../../primary-ds/entity/PEntityService'; -import {REntityId} from '../../../primary-ds/entity/REntityService'; -import {Warehouse} from '../../../Warehouse'; -import {EntityLabelVal} from '../../entity-label/entity-label.commons'; -import {PEntityLabelService} from '../../entity-label/p-entity-label/PEntityLabelService'; -import {REntityLabelService} from '../../entity-label/r-entity-label/REntityLabelService'; -import {PEntityTypeAggregator} from './PEntityTypeAggregator'; -import {PEntityTypeProviders} from './PEntityTypePoviders'; - -export interface PEntityTypeVal { - fkType?: number, - typeLabel?: string -} - -/** - * This Data Service manages the key-value store containing - * as a key the PEntityId (pkEntity and fkProject) - * and as value the PEntityTypeVal (fkType, typeLabel) - * - * One example key-value pair in the this.index is: - * Key for the Project Entity Geo. Place 'Madrid' with pkEntity = 2002 in fkProject = 3001 - * - '2002_3001' - * - * Val for the Geo. Place Type 'City' with pkEntity = 2003 in fkProject = 3001 - * - fkType: 2003 - * - typeLabel: 'Citiy' - * - * - * - * -> The Val is the result of the PEntityTypeAggregator - * - */ -@Injectable() -export class PEntityTypeService extends AggregatedDataService2{ - aggregator = PEntityTypeAggregator; - providers = PEntityTypeProviders; - - depPEntity: DependencyIndex - depPEntityLabel: DependencyIndex - depREntityLabel: DependencyIndex - depPEdge: DependencyIndex - depDfhClassHasTypeProp: DependencyIndex - batchSize= 100000; - constructor( - @Inject(forwardRef(() => Warehouse)) wh: Warehouse, - @Inject(forwardRef(() => PEntityService)) pEntity: PEntityService, - @Inject(forwardRef(() => PEntityLabelService)) pEntityLabel: PEntityLabelService, - @Inject(forwardRef(() => REntityLabelService)) rEntityLabel: REntityLabelService, - @Inject(forwardRef(() => PEdgeService)) pEdge: PEdgeService, - @Inject(forwardRef(() => DfhClassHasTypePropertyService)) dfhClassHasTypeProp: DfhClassHasTypePropertyService, - ) { - super( - wh, - pEntityKeyDefs - ) - - this.registerCreatorDS({dataService: pEntity}) - this.depPEntity = this.addDepencency(pEntity) - this.depPEntityLabel = this.addDepencency(pEntityLabel) - this.depREntityLabel = this.addDepencency(rEntityLabel) - this.depPEdge = this.addDepencency(pEdge) - this.depDfhClassHasTypeProp = this.addDepencency(dfhClassHasTypeProp) - - } - - getDependencies() { - return this - }; - // onUpsertSql(tableAlias: string) { - // return ` - // UPDATE war.entity_preview - // SET type_label = val->>'typeLabel', - // fk_type = (val->>'fkType')::int - // FROM ${tableAlias} - // WHERE pk_entity = "pkEntity" - // AND project = "fkProject" - // AND ( - // type_label IS DISTINCT FROM val->>'typeLabel' - // OR - // fk_type IS DISTINCT FROM (val->>'fkType')::int - // )` - // } - async aggregateBatch(client: PoolClient, client2: PoolClient, limit: number, offset: number, currentTimestamp: string): Promise { - const builder = new AggregatorSqlBuilder(this, client, currentTimestamp, limit, offset) - - const pentity = await builder.joinProviderThroughDepIdx({ - leftTable: builder.batchTmpTable.tableDef, - joinWithDepIdx: this.depPEntity, - joinOnKeys: { - pkEntity: {leftCol: 'pkEntity'}, - fkProject: {leftCol: 'fkProject'} - }, - conditionTrueIf: { - providerKey: {pkEntity: 'IS NOT NULL'} - }, - createCustomObject: (() => `jsonb_build_object('fkClass', (t2.val->>'fkClass')::int)`) as CustomValSql<{fkClass: number}>, - }) - const dfhClassHasTypeProp = await builder.joinProviderThroughDepIdx({ - leftTable: pentity.aggregation.tableDef, - joinWithDepIdx: this.depDfhClassHasTypeProp, - joinWhereLeftTableCondition: '= true', - joinOnKeys: { - pkClass: {leftCustom: {name: 'fkClass', type: 'int'}}, - }, - conditionTrueIf: { - providerVal: {fkProperty: 'IS NOT NULL'} - }, - createCustomObject: ((provider) => `t2.val`) as CustomValSql, - // upsert no entity type where fk property is null - createAggregationVal: { - sql: () => `jsonb_build_object()`, - upsert: { - whereCondition: '= false' - } - } - }) - - const pEdges = await builder.joinProviderThroughDepIdx({ - leftTable: dfhClassHasTypeProp.aggregation.tableDef, - joinWithDepIdx: this.depPEdge, - // join only where fk property is is given - joinWhereLeftTableCondition: '= true', - joinOnKeys: { - pkEntity: {leftCol: 'pkEntity'}, - fkProject: {leftCol: 'fkProject'} - }, - conditionTrueIf: { - custom: `t2.val->'outgoing'->(t1.custom->>'fkProperty')->0->'fkTarget' IS NOT NULL` - }, - createCustomObject: ((provider) => - `jsonb_build_object('fkType', t2.val->'outgoing'->(t1.custom->>'fkProperty')->0->'fkTarget')`) as CustomValSql<{fkType?: number}>, - // upsert no entity type where no has type stmt with fkTarget found - createAggregationVal: { - sql: () => `jsonb_build_object()`, - upsert: { - whereCondition: '= false' - } - } - }) - - - const remotePEntityLabel = await builder.joinProviderThroughDepIdx({ - leftTable: pEdges.aggregation.tableDef, - joinWithDepIdx: this.depPEntityLabel, - joinWhereLeftTableCondition: '= true', - joinOnKeys: { - pkEntity: {leftCustom: {name: 'fkType', type: 'int'}}, - fkProject: {leftCol: 'fkProject'} - }, - conditionTrueIf: { - providerKey: {pkEntity: 'IS NOT NULL'} - }, - createCustomObject: (() => `t1.custom`) as CustomValSql<{fkType?: number}>, - createAggregationVal: { - sql: () => `jsonb_build_object( - 'typeLabel', t2.val->>'entityLabel', - 'fkType', t1.custom->>'fkType' - )`, - upsert: { - whereCondition: '= true' - } - } - }) - - - await builder.joinProviderThroughDepIdx({ - leftTable: remotePEntityLabel.aggregation.tableDef, - joinWithDepIdx: this.depREntityLabel, - joinWhereLeftTableCondition: '= false', - joinOnKeys: { - pkEntity: {leftCustom: {name: 'fkType', type: 'int'}} - }, - conditionTrueIf: {}, - createAggregationVal: { - sql: () => `jsonb_build_object( - 'typeLabel', t2.val->>'entityLabel', - 'fkType', t1.custom->>'fkType' - )`, - upsert: true - } - }) - - // await builder.printQueries() - const count = builder.executeQueries() - - return count - } - - -} - - diff --git a/server/src/warehouse/aggregator-ds/entity-type/r-entity-type/REntityTypeAggregator.ts b/server/src/warehouse/aggregator-ds/entity-type/r-entity-type/REntityTypeAggregator.ts deleted file mode 100644 index a386fbe40..000000000 --- a/server/src/warehouse/aggregator-ds/entity-type/r-entity-type/REntityTypeAggregator.ts +++ /dev/null @@ -1,88 +0,0 @@ -import {AbstractAggregator} from '../../../base/classes/AbstractAggregator'; -import {REntityId} from '../../../primary-ds/entity/REntityService'; -import {REntityTypeProviders} from './REntityTypePoviders'; -import {REntityTypeVal} from './REntityTypeService'; - -export interface ClassLabelConfig { - fkProperty: number, - isOutgoing: boolean, - ordNum: number - nrOfStatementsInLabel?: number -} - - -export class REntityTypeAggregator extends AbstractAggregator { - - // the resulting entityTypeLabel - entityTypeLabel?: string; - fkEntityType?: number; - - - constructor( - public providers: REntityTypeProviders, - public id: REntityId - ) { - super() - } - - - /************************************************************************ - * Methods for creating entity label - ************************************************************************/ - - /** - * Create entity label - * - * Gets values from Indexes and chaches dependencies in itself. - */ - async create() { - - const entity = await this.providers.rEntity.get(this.id); - if (entity) { - - if (!entity.fkClass) return this.finalize() - - const classId = { - pkClass: entity.fkClass - } - - // Find the dfh_pk_property of the 'has type'-subproperty going out of this class - const fkHasTypeSubproperty = await this.providers.dfhClassHasTypeProp.get(classId); - - if (fkHasTypeSubproperty?.fkProperty) { - // Get the 'directed-statements' a.k.a. 'edges' of the entity - const fieldsWithEdges = await this.providers.rEdges.get(this.id) - - const hasTypeStmts = fieldsWithEdges?.outgoing?.[fkHasTypeSubproperty.fkProperty]; - - if (hasTypeStmts?.length) { - - // this gives the info for war.entity_preview (fk_type) - this.fkEntityType = hasTypeStmts[0].fkTarget // fk_object_info - - // this gives the info for war.entity_preview (type_label) - const typeEntityId: REntityId = { - pkEntity: hasTypeStmts[0].fkTarget - } - - const l = await this.providers.rEntityLabel.get(typeEntityId) - - - if (l) { - this.entityTypeLabel = l.entityLabel - } - - } - } - - } - return this.finalize() - } - finalize(): REntityTypeVal { - return { - typeLabel: this.entityTypeLabel, - fkType: this.fkEntityType - } - } - -} diff --git a/server/src/warehouse/aggregator-ds/entity-type/r-entity-type/REntityTypeDependencies.ts b/server/src/warehouse/aggregator-ds/entity-type/r-entity-type/REntityTypeDependencies.ts deleted file mode 100644 index 27d2d2989..000000000 --- a/server/src/warehouse/aggregator-ds/entity-type/r-entity-type/REntityTypeDependencies.ts +++ /dev/null @@ -1,51 +0,0 @@ -// import {Dependencies} from '../../../base/classes/Dependencies' -// import {DependencyIndex} from '../../../base/classes/DependencyIndex' -// import {DfhClassHasTypePropVal, RClassId} from '../../../primary-ds/DfhClassHasTypePropertyService' -// import {EntityFields} from "../../../primary-ds/edge/edge.commons" -// import {REntity, REntityId} from '../../../primary-ds/entity/REntityService' -// import {Warehouse} from '../../../Warehouse' -// import {EntityLabelVal} from '../../entity-label/entity-label.commons' -// import {REntityTypeVal} from './REntityTypeService' -// import {Injectable, Inject, forwardRef} from 'injection-js'; - -// @Injectable() -// export class REntityTypeDependencies extends Dependencies { -// rEntity: DependencyIndex -// rEntityLabel: DependencyIndex -// rEdge: DependencyIndex -// dfhClassHasTypeProp: DependencyIndex - -// constructor(@Inject(forwardRef(() => Warehouse)) private wh: Warehouse) { -// super() -// // stores the dependency of entityType (receiver) on entity (provider) -// this.rEntity = this.registerDepIdx(new DependencyIndex( -// this.wh, -// this.wh.agg.rEntityType, -// this.wh.prim.rEntity, -// )) - -// // stores the dependency of entityType (receiver) on entityLabel (provider) -// this.rEntityLabel = this.registerDepIdx(new DependencyIndex( -// this.wh, -// this.wh.agg.rEntityType, -// this.wh.agg.rEntityLabel, -// )); - -// // stores the dependency of entityType (receiver) on edge (provider) -// this.rEdge = this.registerDepIdx(new DependencyIndex( -// this.wh, -// this.wh.agg.rEntityType, -// this.wh.prim.rEdge -// )); - -// // stores the dependency of entityType (receiver) on dfhClassHasTypeProperty -// this.dfhClassHasTypeProp = this.registerDepIdx(new DependencyIndex( -// this.wh, -// this.wh.agg.rEntityType, -// this.wh.prim.dfhClassHasTypeProperty, -// )); - - -// } - -// } diff --git a/server/src/warehouse/aggregator-ds/entity-type/r-entity-type/REntityTypePoviders.ts b/server/src/warehouse/aggregator-ds/entity-type/r-entity-type/REntityTypePoviders.ts deleted file mode 100644 index d3aed8e18..000000000 --- a/server/src/warehouse/aggregator-ds/entity-type/r-entity-type/REntityTypePoviders.ts +++ /dev/null @@ -1,25 +0,0 @@ -import {Provider} from '../../../base/classes/Provider'; -import {Providers} from "../../../base/interfaces/Providers"; -import {DfhClassHasTypePropVal, RClassId} from '../../../primary-ds/DfhClassHasTypePropertyService'; -import {EntityFields} from "../../../primary-ds/edge/edge.commons"; -import {REntity, REntityId} from '../../../primary-ds/entity/REntityService'; -import {EntityLabelVal} from '../../entity-label/entity-label.commons'; -import {REntityTypeService, REntityTypeVal} from './REntityTypeService'; -export class REntityTypeProviders extends Providers { - rEntity: Provider; - rEdges: Provider; - rEntityLabel: Provider; - dfhClassHasTypeProp: Provider; - constructor( - dep: REntityTypeService, - protected receiverKey: REntityId - ) { - super() - this.rEntity = this.registerProvider(dep.depREntity, receiverKey) - this.rEntityLabel = this.registerProvider(dep.depREntityLabel, receiverKey); - this.rEdges = this.registerProvider(dep.depREdge, receiverKey) - this.dfhClassHasTypeProp = this.registerProvider(dep.depDfhClassHasTypeProp, receiverKey) - } - -} - diff --git a/server/src/warehouse/aggregator-ds/entity-type/r-entity-type/REntityTypeService.ts b/server/src/warehouse/aggregator-ds/entity-type/r-entity-type/REntityTypeService.ts deleted file mode 100644 index d18cca108..000000000 --- a/server/src/warehouse/aggregator-ds/entity-type/r-entity-type/REntityTypeService.ts +++ /dev/null @@ -1,175 +0,0 @@ -import {forwardRef, Inject, Injectable} from 'injection-js'; -import {AggregatedDataService2} from '../../../base/classes/AggregatedDataService2'; -import {DependencyIndex} from '../../../base/classes/DependencyIndex'; -import {DfhClassHasTypePropertyService, DfhClassHasTypePropVal, RClassId} from '../../../primary-ds/DfhClassHasTypePropertyService'; -import {EntityFields} from '../../../primary-ds/edge/edge.commons'; -import {REdgeService} from '../../../primary-ds/edge/REdgeService'; -import {REntity, REntityId, rEntityKeyDefs, REntityService} from '../../../primary-ds/entity/REntityService'; -import {Warehouse} from '../../../Warehouse'; -import {EntityLabelVal} from '../../entity-label/entity-label.commons'; -import {REntityLabelService} from '../../entity-label/r-entity-label/REntityLabelService'; -import {REntityTypeAggregator} from './REntityTypeAggregator'; -import {REntityTypeProviders} from './REntityTypePoviders'; -import {PoolClient} from 'pg'; -import {AggregatorSqlBuilder, CustomValSql} from '../../../base/classes/AggregatorSqlBuilder'; - -export interface REntityTypeVal { - fkType?: number, - typeLabel?: string -} - -/** - * This Data Service manages the key-value store containing - * as a key the REntityId (pkEntity and fkProject) - * and as value the REntityTypeVal (fkType, typeLabel) - * - * One example key-value pair in the this.index is: - * Key for the Project Entity Geo. Place 'Madrid' with pkEntity = 2002 in fkProject = 3001 - * - '2002_3001' - * - * Val for the Geo. Place Type 'City' with pkEntity = 2003 in fkProject = 3001 - * - fkType: 2003 - * - typeLabel: 'Citiy' - * - * - * - * -> The Val is the result of the REntityTypeAggregator - * - */ -@Injectable() -export class REntityTypeService extends AggregatedDataService2{ - aggregator = REntityTypeAggregator; - providers = REntityTypeProviders; - - depREntity: DependencyIndex - depREntityLabel: DependencyIndex - depREdge: DependencyIndex - depDfhClassHasTypeProp: DependencyIndex - batchSize= 100000; - constructor( - @Inject(forwardRef(() => Warehouse)) wh: Warehouse, - @Inject(forwardRef(() => REntityService)) rEntity: REntityService, - @Inject(forwardRef(() => REntityLabelService)) rEntityLabel: REntityLabelService, - @Inject(forwardRef(() => REdgeService)) rEdge: REdgeService, - @Inject(forwardRef(() => DfhClassHasTypePropertyService)) dfhClassHasTypeProp: DfhClassHasTypePropertyService, - - ) { - super( - wh, - rEntityKeyDefs - ) - this.registerCreatorDS({dataService: rEntity}) - this.depREntity = this.addDepencency(rEntity); - this.depREntityLabel = this.addDepencency(rEntityLabel); - this.depREdge = this.addDepencency(rEdge); - this.depDfhClassHasTypeProp = this.addDepencency(dfhClassHasTypeProp); - } - - getDependencies() { - return this - }; - // onUpsertSql(tableAlias: string) { - // return ` - // UPDATE war.entity_preview - // SET type_label = val->>'typeLabel', - // fk_type = (val->>'fkType')::int - // FROM ${tableAlias} - // WHERE pk_entity = "pkEntity" - // AND project = 0 - // AND ( - // type_label IS DISTINCT FROM val->>'typeLabel' - // OR - // fk_type IS DISTINCT FROM (val->>'fkType')::int - // )` - // } - - async aggregateBatch(client: PoolClient, client2: PoolClient, limit: number, offset: number, currentTimestamp: string): Promise { - const builder = new AggregatorSqlBuilder(this, client, currentTimestamp, limit, offset) - - const pentity = await builder.joinProviderThroughDepIdx({ - leftTable: builder.batchTmpTable.tableDef, - joinWithDepIdx: this.depREntity, - joinOnKeys: { - pkEntity: {leftCol: 'pkEntity'}, - }, - conditionTrueIf: { - providerKey: {pkEntity: 'IS NOT NULL'} - }, - createCustomObject: (() => `jsonb_build_object('fkClass', (t2.val->>'fkClass')::int)`) as CustomValSql<{fkClass: number}>, - }) - const dfhClassHasTypeProp = await builder.joinProviderThroughDepIdx({ - leftTable: pentity.aggregation.tableDef, - joinWithDepIdx: this.depDfhClassHasTypeProp, - joinWhereLeftTableCondition: '= true', - joinOnKeys: { - pkClass: {leftCustom: {name: 'fkClass', type: 'int'}}, - }, - conditionTrueIf: { - providerVal: {fkProperty: 'IS NOT NULL'} - }, - createCustomObject: ((provider) => `t2.val`) as CustomValSql, - // upsert no entity type where fk property is null - createAggregationVal: { - sql: () => `jsonb_build_object()`, - upsert: { - whereCondition: '= false' - } - } - }) - - const pEdges = await builder.joinProviderThroughDepIdx({ - leftTable: dfhClassHasTypeProp.aggregation.tableDef, - joinWithDepIdx: this.depREdge, - // join only where fk property is is given - joinWhereLeftTableCondition: '= true', - joinOnKeys: { - pkEntity: {leftCol: 'pkEntity'}, - }, - conditionTrueIf: { - custom: `t2.val->'outgoing'->(t1.custom->>'fkProperty')->0->'fkTarget' IS NOT NULL` - }, - createCustomObject: ((provider) => - `jsonb_build_object('fkType', t2.val->'outgoing'->(t1.custom->>'fkProperty')->0->'fkTarget')`) as CustomValSql<{fkType?: number}>, - // upsert no entity type where no has type stmt with fkTarget found - createAggregationVal: { - sql: () => `jsonb_build_object()`, - upsert: { - whereCondition: '= false' - } - } - }) - - - await builder.joinProviderThroughDepIdx({ - leftTable: pEdges.aggregation.tableDef, - joinWithDepIdx: this.depREntityLabel, - joinWhereLeftTableCondition: '= true', - joinOnKeys: { - pkEntity: {leftCustom: {name: 'fkType', type: 'int'}}, - }, - conditionTrueIf: { - providerKey: {pkEntity: 'IS NOT NULL'} - }, - createCustomObject: (() => `t1.custom`) as CustomValSql<{fkType?: number}>, - createAggregationVal: { - sql: () => `jsonb_build_object( - 'typeLabel', t2.val->>'entityLabel', - 'fkType', t1.custom->>'fkType' - )`, - upsert: { - whereCondition: '= true' - } - } - }) - - - - - // await builder.printQueries() - const count = builder.executeQueries() - - return count - } -} - - diff --git a/server/src/warehouse/aggregator-ds/identifying-property/IdentifyingPropertyAggregator.ts b/server/src/warehouse/aggregator-ds/identifying-property/IdentifyingPropertyAggregator.ts deleted file mode 100644 index 900fc5b51..000000000 --- a/server/src/warehouse/aggregator-ds/identifying-property/IdentifyingPropertyAggregator.ts +++ /dev/null @@ -1,30 +0,0 @@ -import {AbstractAggregator} from '../../base/classes/AbstractAggregator'; -import {RClassId} from '../../primary-ds/DfhClassHasTypePropertyService'; -import {IdentifyingPropertyProviders} from './IdentifyingPropertyProviders'; -import {IdentifyingPropertyVal} from './IdentifyingPropertyService'; - -export class IdentifyingPropertyAggregator extends AbstractAggregator { - - - // the resulting label - identyfyingProperties: IdentifyingPropertyVal = []; - - constructor( - public providers: IdentifyingPropertyProviders, - public id: RClassId - ) { - super() - } - - async create() { - const outProps = await this.providers.outgoingProperty.getItemsWith({fkDomain: this.id.pkClass}); - for (const p of outProps) { - if (p.value.dfhIdentityDefining) { - this.identyfyingProperties.push(p.value) - } - - } - return this.identyfyingProperties - } - -} diff --git a/server/src/warehouse/aggregator-ds/identifying-property/IdentifyingPropertyProviders.ts b/server/src/warehouse/aggregator-ds/identifying-property/IdentifyingPropertyProviders.ts deleted file mode 100644 index f01d1055a..000000000 --- a/server/src/warehouse/aggregator-ds/identifying-property/IdentifyingPropertyProviders.ts +++ /dev/null @@ -1,19 +0,0 @@ -import {Provider} from '../../base/classes/Provider'; -import {Providers} from '../../base/interfaces/Providers'; -import {RClassId} from '../../primary-ds/DfhClassHasTypePropertyService'; -import {OutgoingPropertyVal, OutgoingProperyId} from '../../primary-ds/DfhOutgoingPropertyService'; -import {IdentifyingPropertyService, IdentifyingPropertyVal} from './IdentifyingPropertyService'; - - -export class IdentifyingPropertyProviders extends Providers { - outgoingProperty: Provider - - constructor( - dep: IdentifyingPropertyService, - protected receiverKey: RClassId - ) { - super() - this.outgoingProperty = this.registerProvider(dep.outgoingProperty, receiverKey); - } -} - diff --git a/server/src/warehouse/aggregator-ds/identifying-property/IdentifyingPropertyService.ts b/server/src/warehouse/aggregator-ds/identifying-property/IdentifyingPropertyService.ts deleted file mode 100644 index 8a077e4bb..000000000 --- a/server/src/warehouse/aggregator-ds/identifying-property/IdentifyingPropertyService.ts +++ /dev/null @@ -1,40 +0,0 @@ -import {forwardRef, Inject, Injectable} from 'injection-js'; -import {AggregatedDataService2} from '../../base/classes/AggregatedDataService2'; -import {DependencyIndex} from '../../base/classes/DependencyIndex'; -import {RClassId, rClassIdKeyDefs} from '../../primary-ds/DfhClassHasTypePropertyService'; -import {DfhOutgoingPropertyService, OutgoingPropertyVal, OutgoingProperyId} from '../../primary-ds/DfhOutgoingPropertyService'; -import {Warehouse} from '../../Warehouse'; -import {IdentifyingPropertyAggregator} from './IdentifyingPropertyAggregator'; -import {IdentifyingPropertyProviders} from './IdentifyingPropertyProviders'; - - -export type IdentifyingPropertyVal = OutgoingPropertyVal[] - -@Injectable() -export class IdentifyingPropertyService extends AggregatedDataService2{ - - - aggregator = IdentifyingPropertyAggregator; - providers = IdentifyingPropertyProviders; - - outgoingProperty: DependencyIndex - - constructor( - @Inject(forwardRef(() => Warehouse)) wh: Warehouse, - @Inject(forwardRef(() => DfhOutgoingPropertyService)) dfhOutgoingProperty: DfhOutgoingPropertyService, - ) { - super( - wh, - rClassIdKeyDefs - ) - this.registerCreatorDS({ - dataService: dfhOutgoingProperty, - customSql: [{select: `"fkDomain" as "pkClass"`}] - }) - this.outgoingProperty = this.addDepencency(dfhOutgoingProperty) - } - - getDependencies() { - return this - }; -} diff --git a/server/src/warehouse/base/bucketeer/Bucketeer.ts b/server/src/warehouse/base/bucketeer/Bucketeer.ts deleted file mode 100644 index 5b1159ea5..000000000 --- a/server/src/warehouse/base/bucketeer/Bucketeer.ts +++ /dev/null @@ -1,348 +0,0 @@ -import * as archiver from 'archiver'; -import async from 'async'; -import {S3} from 'aws-sdk'; -import {createReadStream, createWriteStream, existsSync, mkdirSync} from 'fs'; -import path from 'path'; -import readdir from 'recursive-readdir'; -import {Extract} from 'unzipper'; -import {Logger} from '../classes/Logger'; - -export class Bucketeer { - - private s3: S3 - private bucket: string; - constructor() { - this.verifyEnvVars(); - - this.s3 = new S3({ - accessKeyId: process.env.BUCKETEER_AWS_ACCESS_KEY_ID, - secretAccessKey: process.env.BUCKETEER_AWS_SECRET_ACCESS_KEY, - region: process.env.BUCKETEER_AWS_REGION, - // signatureVersion: 'v4' - }); - if (process.env.BUCKETEER_BUCKET_NAME) { - this.bucket = process.env.BUCKETEER_BUCKET_NAME - } - } - - /** - * Upload a folder to S3 - * @param rootPath the path to parent directory of folder to upload - * @param folderName the name of the directory to upload - * @param s3Key the name of the directory on S3 - */ - async uploadFolder(rootPath: string, folderName: string, s3Key: string) { - const inputFolderPath = path.resolve(rootPath, folderName) - const filesToUpload = await this.getFiles(inputFolderPath); - - return new Promise((resolve, reject) => { - async.eachOfLimit(filesToUpload, 10, async.asyncify(async (filePath: string) => { - const key = filePath.replace(`${inputFolderPath}/`, `${s3Key}/`); - return this.uploadFile(filePath, key) - }), (err) => { - if (err) { - return reject(err); - } - resolve({result: true}); - }); - }); - } - - async getFiles(dirPath: string) { - let files: string[] = [] - if (existsSync(dirPath)) files = await readdir(dirPath) - return files; - } - - - /** - * Upload a file to bucketeer - * @param filePath the path to the file to upload - * @param s3Key the key under which the file is stored in S3 - */ - public uploadFile(filePath: string, s3Key: string): Promise { - Logger.msg(this.constructor.name, `uploading: [${filePath}]`) - - return new Promise((res, rej) => { - // Configure the file stream and obtain the upload parameters - const fileStream = createReadStream(filePath); - fileStream.on('error', function (err) { - rej({msg: 'File Error', err}); - }); - const params: S3.PutObjectRequest = { - Key: s3Key, - Bucket: this.bucket, - Body: fileStream - }; - this.s3.putObject(params, function put(err, data) { - if (err) { - rej({msg: 'Upload Error', err}); - } else { - res(data); - } - }); - }) - } - - /** - * Upload a string to a file to bucketeer - * @param string the string to put in the file to upload - * @param s3Key the key under which the file is stored in S3 - */ - public uploadStringToFile(string: string, s3Key: string): Promise { - Logger.msg(this.constructor.name, `uploading string: [${string}] to file [${s3Key}]`) - - return new Promise((res, rej) => { - const params: S3.PutObjectRequest = { - Key: s3Key, - Bucket: this.bucket, - Body: new Buffer(string), - }; - this.s3.putObject(params, function put(err, data) { - if (err) { - rej({msg: 'Upload Error', err}); - } else { - res(data); - } - }); - }) - } - - /** - * Download a file from bucketeer and save it to the filesystem - */ - public downloadFile(filePath: string, s3Key: string): Promise { - Logger.msg(this.constructor.name, `downloading: [${s3Key}]`) - - return new Promise((res, rej) => { - // Configure the file stream - const file = createWriteStream(filePath); - - const params: S3.GetObjectRequest = { - Key: s3Key, - Bucket: this.bucket, - - }; - - this.s3.getObject(params).createReadStream() - .on('end', () => { - return res(params); - }) - .on('error', (err) => { - return rej(err); - }) - .pipe(file) - - }) - } - - - /** - * Read a file from bucketeer and return array of strings - */ - public readFile(s3Key: string): Promise { - Logger.msg(this.constructor.name, `reading file: [${s3Key}]`) - - return new Promise((res, rej) => { - // Configure the file stream - const params: S3.GetObjectRequest = { - Key: s3Key, - Bucket: this.bucket, - - }; - const lines: string[] = [] - this.s3.getObject(params).createReadStream() - .on('data', (line) => { - lines.push(line.toString()) - }) - .on('end', () => { - return res(lines); - }) - .on('error', (err) => { - return rej(err); - }) - - }) - } - - - /** - * Download a folder from S3 - */ - async downloadFolder(rootPath: string, localFolder: string, s3folder: string) { - const t = Logger.start(this.constructor.name, `download backup ${s3folder}`) - const outputFolderPath = path.resolve(rootPath, localFolder) - - const data = await this.listObjects(s3folder) - - if (data.Contents) { - const fileList = data.Contents - const dirCheckCache: {[dir: string]: true} = {} - return new Promise((resolve, reject) => { - async.eachOfLimit(fileList, 10, async.asyncify(async (file: S3.Object) => { - const key = file.Key - if (key) { - const destination = path.join(outputFolderPath, key.replace(s3folder, '')); - const dir = path.dirname(destination) - if (!dirCheckCache[dir] && !existsSync(dir)) { - mkdirSync(dir, {recursive: true}); - dirCheckCache[dir] = true - } - await this.downloadFile(destination, key) - } - }), (err) => { - if (err) { - return reject(err); - } - Logger.itTook(this.constructor.name, t, 'to download') - resolve({result: true}); - - }); - }); - - } - - } - - /** -* remove a folder from S3 (recursive) -*/ - async emptyS3Directory(dir: string) { - - - const listedObjects = await this.listObjects(dir); - - if (listedObjects?.Contents?.length) { - - const deleteParams: S3.DeleteObjectsRequest = { - Bucket: this.bucket, - Delete: {Objects: []} - }; - - listedObjects.Contents.forEach((object) => { - if (object.Key) deleteParams.Delete.Objects.push({Key: object.Key}); - }); - Logger.msg(this.constructor.name, `deleting ${listedObjects.Contents.length} from [${dir}]`) - - await this.s3.deleteObjects(deleteParams).promise(); - - if (listedObjects.IsTruncated) await this.emptyS3Directory(dir); - } - } - - - /** - * Call S3 to obtain a list of the objects in the bucket - */ - listObjects(prefix = '', delimiter?: string) { - return new Promise((res, rej) => { - - const params: S3.ListObjectsRequest = { - Bucket: this.bucket, - Prefix: prefix, - }; - if (delimiter) params.Delimiter = delimiter - - this.s3.listObjectsV2(params, function (err, data) { - if (err) { - console.log({msg: "Error", err}); - } else { - res(data); - } - }); - }) - } - - /** - * Verifies availability of all needed env vars to connect to S3 - */ - private verifyEnvVars() { - if (!process.env.BUCKETEER_AWS_ACCESS_KEY_ID) { - throw new Error('You must provide a BUCKETEER_AWS_ACCESS_KEY_ID env var.'); - } - if (!process.env.BUCKETEER_AWS_SECRET_ACCESS_KEY) { - throw new Error('You must provide a BUCKETEER_AWS_SECRET_ACCESS_KEY env var.'); - } - if (!process.env.BUCKETEER_BUCKET_NAME) { - throw new Error('You must provide a BUCKETEER_BUCKET_NAME env var.'); - } - if (!process.env.BUCKETEER_AWS_REGION) { - throw new Error('You must provide a BUCKETEER_AWS_REGION env var.'); - } - } - - zip(inputDirPath: string, outputZipPath: string) { - - // create a file to stream archive data to. - const output = createWriteStream(outputZipPath); - const archive = archiver.create('zip', { - zlib: {level: 9} // Sets the compression level. - }); - - // listen for all archive data to be written - // 'close' event is fired only when a file descriptor is involved - output.on('close', function () { - console.log(archive.pointer() + ' total bytes'); - console.log('archiver has been finalized and the output file descriptor has closed.'); - }); - - // This event is fired when the data source is drained no matter what was the data source. - // It is not part of this library but rather from the NodeJS Stream API. - // @see: https://nodejs.org/api/stream.html#stream_event_end - output.on('end', function () { - console.log('Data has been drained'); - }); - - // good practice to catch warnings (ie stat failures and other non-blocking errors) - archive.on('warning', function (err) { - if (err.code === 'ENOENT') { - // log warning - } else { - // throw error - throw err; - } - }); - - // good practice to catch this error explicitly - archive.on('error', function (err) { - throw err; - }); - - // pipe archive data to the file - archive.pipe(output); - - // append files from a sub-directory, putting its contents at the root of archive - archive.directory(inputDirPath, false); - - // finalize the archive (ie we are done appending files but streams have to finish yet) - // 'close', 'end' or 'finish' may be fired right after calling this method so register to them beforehand - return archive.finalize(); - - }; - - async unzip(inputZipPath: string, outputPath: string) { - - // const d = await Open.file(inputZipPath) - - // return d.extract({path: outputPath}) - return new Promise((res, rej) => { - - createReadStream(inputZipPath) - .pipe(Extract({path: outputPath})) - - .on('entry', entry => {entry.autodrain()}) - .promise() - .then(() => { - res(`unzipped: - ${inputZipPath} --> ${outputPath}`) - }) - .catch((e) => { - if (e?.message === 'FILE_ENDED') res(`unzipped: - ${inputZipPath} --> ${outputPath}`) - else rej(e) - }); - }) - - } -} - diff --git a/server/src/warehouse/base/bucketeer/S3LevelBackup.ts b/server/src/warehouse/base/bucketeer/S3LevelBackup.ts deleted file mode 100644 index b0c4a7746..000000000 --- a/server/src/warehouse/base/bucketeer/S3LevelBackup.ts +++ /dev/null @@ -1,157 +0,0 @@ -import {statSync} from 'fs'; -import {Logger} from '../classes/Logger'; -import {Bucketeer} from './Bucketeer'; -import * as path from 'path'; -import {equals} from 'ramda'; - -interface BackupId { - prefix: string - isoDate: string - gitCommit: string -} -const CURRENT = 'CURRENT' - -export class S3LevelBackup { - bucketeer: Bucketeer - - backupPrefix: string - delemiter = '___' - leveldbPath: string - constructor( - private rootFolder: string, - private leveldbFolder = 'leveldb' - ) { - this.bucketeer = new Bucketeer - this.backupPrefix = this.leveldbFolder; - this.leveldbPath = path.join(rootFolder, leveldbFolder) - } - - async createBackup(tmsp: Date, currentCommit: string) { - const backupId: BackupId = { - prefix: this.backupPrefix, - isoDate: tmsp.toISOString(), - gitCommit: currentCommit - } - // create s3 folder name - const backupname = this.backupIdToString(backupId) - - Logger.msg(this.constructor.name, `Creating backupId ${backupname}`) - - // get file names with sizes of folder - const folderBefore = await this.getFilesWithSizes(this.leveldbPath) - - // upload folder - await this.bucketeer.uploadFolder(this.rootFolder, this.leveldbFolder, backupname) - - // get file names with size of folder - const folderAfter = await this.getFilesWithSizes(this.leveldbPath) - - // Q: did file names and/or size change? - if (equals(folderBefore, folderAfter)) { - // A: Now -> backup successfull - - // set this backup as current backup - await this.bucketeer.uploadStringToFile(backupname, CURRENT); - - Logger.msg(this.constructor.name, `Backup created! Current backup: ${backupname}`) - - } - else { - // A: Yes -> backup potentially corrupt, we don't set it as current backup - Logger.msg(this.constructor.name, `WARN: Backup potentially corrupt, not marked as current: ${backupname}`) - - } - - } - - - /** - * returns array of backup-folder names ordered by latest first - */ - async listBackupsNewestFirst() { - let names: string[] = [] - // get backup folder names - const list = await this.bucketeer.listObjects(this.backupPrefix + this.delemiter, '/') - if (list?.CommonPrefixes?.length) { - names = list.CommonPrefixes.filter(o => !!o.Prefix).map(o => o.Prefix ?? '') - names.sort((a, b) => { - let aDate: Date, bDate: Date; - if (a) { - aDate = this.backupNameToDate(a); - } else return 0 - if (b) { - bDate = this.backupNameToDate(b); - } else return 0 - return aDate === bDate ? 0 : aDate > bDate ? -1 : 1; - }) - } - return names.map(name => name.replace('/', '')) - } - - private backupNameToDate(name: string) { - const backupId = this.stringToBackupId(name) - return new Date(backupId.isoDate); - } - - async downloadCurrentBackup(destinationFolder: string = this.leveldbFolder) { - const backupname = await this.getCurrentBackupName() - if (backupname) { - await this.bucketeer.downloadFolder(this.rootFolder, destinationFolder, backupname) - return this.stringToBackupId(backupname); - } - } - async getCurrentBackupName(): Promise { - try { - const currents = await this.bucketeer.readFile(CURRENT) - if (currents?.length === 1) { - return currents[0] - } - - } catch (error) { - Logger.err(this.constructor.name, error) - } - } - - async getCurrentBackupId(): Promise { - const backupname = await this.getCurrentBackupName() - if (backupname) return this.stringToBackupId(backupname) - } - - - async deleteUnusedBackups() { - const backupnames = await this.listBackupsNewestFirst() - const current = await this.getCurrentBackupName() - - if (backupnames?.length > 1) { - for (const backupname of backupnames) { - if (backupname !== current) { - await this.bucketeer.emptyS3Directory(backupname) - } - } - } - } - - - async deleteLinkToCurrent() { - await this.bucketeer.uploadStringToFile('', CURRENT); - } - - backupIdToString(id: BackupId): string { - return [id.prefix, id.isoDate, id.gitCommit].join(this.delemiter) - } - - stringToBackupId(str: string): BackupId { - const [prefix, isoDate, gitCommit] = str.replace('/', '').split(this.delemiter) - return {prefix, isoDate, gitCommit} - } - - async getFilesWithSizes(dirPath: string) { - const arrayOfFiles = await this.bucketeer.getFiles(dirPath) - const filesWithSizes: {[fileName: string]: number} = {} - arrayOfFiles.forEach(function (filePath) { - filesWithSizes[filePath] = statSync(filePath).size - }) - return filesWithSizes - } - -} diff --git a/server/src/warehouse/base/classes/AbstractAggregator.ts b/server/src/warehouse/base/classes/AbstractAggregator.ts deleted file mode 100644 index 8fc1b9782..000000000 --- a/server/src/warehouse/base/classes/AbstractAggregator.ts +++ /dev/null @@ -1,5 +0,0 @@ - -export abstract class AbstractAggregator { - constructor() { } - abstract create(): Promise; -} diff --git a/server/src/warehouse/base/classes/AggregatedDataService.ts b/server/src/warehouse/base/classes/AggregatedDataService.ts deleted file mode 100644 index d05178136..000000000 --- a/server/src/warehouse/base/classes/AggregatedDataService.ts +++ /dev/null @@ -1,722 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import {existsSync, mkdirSync, writeFileSync} from 'fs'; -import {PoolClient} from 'pg'; -import {ReplaySubject} from 'rxjs'; -import sqlFormatter from 'sql-formatter'; -import {brkOnErr, pgLogOnErr} from '../../../utils/helpers'; -import {CHANGES_CONSIDERED_UNTIL_SUFFIX, LAST_UPDATE_DONE_SUFFIX, LeftDSDates, Warehouse} from '../../Warehouse'; -import {KeyDefinition} from '../interfaces/KeyDefinition'; -import {Providers} from '../interfaces/Providers'; -import {AbstractAggregator} from './AbstractAggregator'; -import {DataIndexPostgres} from './DataIndexPostgres'; -import {DataService} from './DataService'; -import {Dependencies} from './Dependencies'; -import {DependencyIndex} from './DependencyIndex'; -import {Logger} from './Logger'; - -// eslint-disable-next-line @typescript-eslint/no-explicit-any -type Constructor = new (...args: any[]) => T; -interface CustomCreatorDsSql {select?: string, where?: string} - -interface CreatorDS { - dataService: DataService, - customSql?: CustomCreatorDsSql[] -} - - - -export abstract class AggregatedDataService extends DataService { - - // updater: UpdaterÚ - - index: DataIndexPostgres - - // eslint-disable-next-line @typescript-eslint/no-explicit-any - abstract getDependencies(): Dependencies | void - aggregator?: Constructor> - providers?: Constructor> - // array of dependency indexes where this data service is receiver - creatorDS: CreatorDS[] = [] - isReceiverOf: DependencyIndex[] = [] - - cycle = 0 //for debugging - - /** - * This is a 'hook' to allow this data service to instantly upsert - * into another table - * - * if this AggregatedDS inserts into /deletes from a table of the main - * GV-Database, this deleteSqlAlias defines how to insert (or update) them. - * - * Remark: in some occations, an AggregatedDS updates a table of the main - * GV-Database, but does not insert/delete it because the latter is done - * by the CreatorDS. - * - * @param updateSqlAlias name of table/WITH-clause containing new records - */ - onUpsertSql?(updateSqlAlias: string): string; - /** - * This is a 'hook' to allow this data service to instantly delete - * from another table - * - * if this AggregatedDS inserts into /deletes from a table of the main - * GV-Database, this deleteSqlAlias defines how to delete them. - * - * Remark: in some occations, an AggregatedDS updates a table of the main - * GV-Database, but does not insert/delete it because the latter is done - * by the CreatorDS. - * - * @param deleteSqlAlias name of table/WITH-clause containing delete records - */ - onDeleteSql?(deleteSqlAlias: string): string; - - /** - * If creatorDS and aggregatedDS have not the same KeyModel, - * their column names do not (necessarily) match. In this case, - * customCreatorDSSelect can provide the correct mapping by labeling the - * creatorDS columns with the name the corresponding columns have in this - * aggregated data service. - * - * example - * `"fkDomain" as "pkClass"` -> creatorDS.fkDomain = aggDS.pkClass - */ - customCreatorDSSql?: CustomCreatorDsSql[]; - - // true during running update cycle - updating = false; - // the running update cycle promise - updatingPromise: Promise - - shouldRestart = false; - - deleting = false; - shouldDelete = false; - - tempTable: string; - batchSize = 100000; - cleanupDependencyBatchSize = 10000; - - ready$ = new ReplaySubject() - - constructor( - public wh: Warehouse, - protected keyDefs: KeyDefinition[] - ) { - super(wh) - const tableName = 'agg_' + this.constructor.name.replace('Service', '') - this.index = new DataIndexPostgres( - this.keyDefs, - tableName, - wh - ) - this.tempTable = tableName + '_tmp' - } - emitReady() { - this.index.ready$.subscribe((b) => { - if (b) this.ready$.next(b) - }) - } - /** - * updates this aggregated data for time between last update and currentTimestamp - * to be exact, the changes considered are: - * changeTmsp > this.changesConsideredUntil AND <= currentTimestamp - * - * @param beginOfAggregation consider changes until (less or eq) to this tmsp - */ - private async update(beginOfAggregation: string) { - const t0 = Logger.start(this.constructor.name, `Run aggregation cycle ${this.cycle}`, 0) - - this.updating = true - let changes = 0; - - // get the 'changesConsideredUntil'-timestamp - // const changesConsideredUntil = await this.getChangesConsideredUntilTsmp() - const updatesConsideredUntil = await this.getUpdatesConsidered() - const providerUpdateTmsps = await this.getleftDSupdateDone() - - - // useful for debugging - // if (this.constructor.name === 'REntityLabelService') { - // console.log(`- ${this.constructor.name}.update(), cycle: ${this.cycle}), - // aggregatedUnitlTsmps: ${JSON.stringify(aggregatedUnitlTsmps)}, - // providerUpdateTmsps: ${JSON.stringify(providerUpdateTmsps)}, - // beginOfAggregation: ${beginOfAggregation}`) - // const x = await this.wh.pgPool.query(' select * from war_cache_1.agg_rentitylabel where "pkEntity" = 4002'); - // console.log(x.rows?.[0]) - // } - /** - * Handle deletes - */ - changes += await this.handleDeletes(updatesConsideredUntil, providerUpdateTmsps); - - /** - * Handle upserts - */ - // create client for the aggregation - - const client = await this.wh.whPgPool.connect() - const client2 = await this.wh.gvPgPool.connect() - Logger.msg(this.constructor.name, `pgPool connected (totalCount: ${this.wh.whPgPool.totalCount}, waitingCount: ${this.wh.whPgPool.waitingCount})`, 0) - - let hasError = false; - try { - await client.query('BEGIN') - await client2.query('BEGIN') - - // find what to aggregate (store it in temp table) - await this.findWhatToAggregate(client, updatesConsideredUntil, providerUpdateTmsps); - - // aggregate batchwise, reading from temp table - changes += await this.aggregateAll(client, client2, beginOfAggregation) - - // cleanup dependencies - await this.cleanupOldDependencies(client, beginOfAggregation); - - // // update 'changesConsideredUntil'-timestamp - // await this.setChangesConsideredUntilTsmp(beginOfAggregation) - const t1 = Logger.start(this.constructor.name, `commit aggregations`, 0) - await client.query('COMMIT') - await client2.query('COMMIT') - Logger.itTook(this.constructor.name, t1, `to commit aggregations `, 0) - } catch (e) { - hasError = true - await client.query('ROLLBACK') - await client2.query('ROLLBACK') - Logger.msg(this.constructor.name, `ERROR in aggregation: ${JSON.stringify(e)}`,) - } finally { - - client.release() - client2.release() - Logger.msg(this.constructor.name, `pgPool client released`) - - } - if (!hasError) { - await this.setUpdatesConsidered(providerUpdateTmsps) - } - - - // finalize - this.updating = false; - if (this.shouldRestart) { - // useful for debugging - // if (this.constructor.name === 'PEntityTypeService') { - // console.log('------------- restart startUpdate()') - // } - - Logger.itTook(this.constructor.name, t0, `for cycle ${this.cycle}, start over...`, 0) - // restart - beginOfAggregation = await this.wh.whPgNow(); - const nextChanges = await this.doUpdate(beginOfAggregation); - changes = changes + nextChanges; - } - - - // emit this.afterUpdate$ if anything has changed - if (changes > 0) { - const done = await this.wh.whPgNowDate(); - await this.setLastUpdateDone(done) - - // await this.setUpdatesConsidered(providerUpdateTmsps) - this.afterChange$.next() - } - // useful for debugging - // if (this.constructor.name === 'PEntityTypeService') { - // console.log(`------------- finalized cycle ${this.cycle} for time until ${currentTimestamp}`) - // } - - Logger.msg(this.constructor.name, `restart within cycle ${this.cycle}`, 0) - - return changes - } - async aggregate(id: KeyModel): Promise { - if (this.providers && this.aggregator) { - - const providers = new this.providers(this.getDependencies(), id) - return new this.aggregator(providers, id).create() - } - } - - public async startUpdate() { - const t0 = Logger.start(this.constructor.name, `Start update.`, 0) - const tmsp = await this.wh.whPgNow(); - const changes = await this.doUpdate(tmsp); - Logger.itTook(this.constructor.name, t0, `to start update`, 0) - return changes - } - - /** - * triggers update cycle if currently no update running, else put flag - * shouldUpdate to true, so that a new update cycle will be initialized - * as soon as the current one stops - * - * @param currentTimestamp consider changes until (less or eq) to this tmsp - */ - public doUpdate(currentTimestamp: string): Promise { - // useful for debugging - // if (this.constructor.name === 'EntityPreviewService') { - // console.log(`- EntityPreviewService.doUpdate() currentTimestamp: ${currentTimestamp}, cycle: ${this.cycle}, updating:${this.updating}`) - // } - - if (this.updating) { - this.shouldRestart = true - } - else { - this.shouldRestart = false - this.updatingPromise = this.update(currentTimestamp) - this.cycle++ - } - return this.updatingPromise - } - - - - - - /** find 'newDeletes' in creatorDS - * - items of the creatorDS - * that have a tmsp_deleted greater than 'changesConsideredUntil' and - * less or equal than 'currentTimestamp' - * (these items are the 'newDeletes') - * mark 'newDeletes' as deleted in AggregatedDS - * – items of this AggregatedDS - * that have the same keyModel as the 'newDeletes', setting tmsp_deleted='currentTimestamp' - * (don't delete them yet, because they may be providers of other aggregators) - * this operation happens directly on Postgres (within one query) - */ - private async handleDeletes(updatesConsidered: LeftDSDates, updatesDone: LeftDSDates) { - let changes = 0 - let twOnDelete = ''; - if (!this.creatorDS.length) throw new Error("At least one creator data service needed"); - const t0 = Logger.start(this.constructor.name, `handleDeletes`, 0) - - if (this.onDeleteSql) { - twOnDelete = `, - delete AS ( - ${this.onDeleteSql('tw1')} - )`; - } - const handleDeletes = ` - -- find new deletes in creatorDS - WITH tw1 AS ( - ${this.creatorDS.map(createDS => { - const ds = createDS.dataService - const customSqls = this.getCreatorCustomSql(createDS.customSql) - - const {after, until} = this.getAfterAndUntil(ds.constructor.name, updatesConsidered, updatesDone); - - return `${customSqls.map(part => ` - SELECT ${part.select ? part.select : this.index.keyCols} - FROM ${ds.index.schemaTable} - WHERE tmsp_deleted > '${after}' - AND tmsp_deleted <= '${until}' - ${part.where ? `AND ${part.where}` : ''} - `).join(' UNION ')}` - }).join(' UNION ')} - ) - ${twOnDelete}, - --mark them as deleted in aggregatedDS - tw2 AS( - UPDATE ${this.index.schemaTable} t1 - SET tmsp_deleted = clock_timestamp() - FROM tw1 t2 - WHERE ${this.index.keyDefs - .map(k => `t1."${k.name}"=t2."${k.name}"`) - .join(' AND ') - } - RETURNING t1.* - ) - - -- TODO Delete dependencies where items from tw1 are receivers! - - --count the rows marked as deleted - SELECT count(*):: int FROM tw2 - `; - // useful for debugging - // logSql(handleDeletes, []) - // if (this.constructor.name === 'PEntityLabelService') { - // console.log(`-- > ${this.cycle} handle deletes between ${changesConsideredUntil} and ${currentTimestamp} `) - // } - const res = await pgLogOnErr((s, p) => this.wh.whPgPool.query<{count: number;}>(s, p), handleDeletes, []) - changes += res?.rows?.[0].count ?? 0; - // useful for debugging - // if (this.constructor.name === 'PEntityLabelService') { - // console.log(`-- > ${this.cycle} handled ${res.rows[0].count} deletes`) - // if (res.rows[0].count) { - // console.log(``) - // } - // } - Logger.itTook(this.constructor.name, t0, `to handleDeletes `, 0) - - return changes; - } - - private getAfterAndUntil(name: string, updatesConsidered: LeftDSDates, updatesDone: LeftDSDates) { - const after = updatesConsidered[name] ?? new Date(0).toISOString(); - const until = updatesDone[name] ?? new Date().toISOString(); - return {after, until}; - } - - /** - * cleanup old dependencies - * delete dependencies where receivers have just been aggergated - * (=they are in temp table) and they have not anymore been used by the - * recent aggregation (=tmsp_last_aggregation less than 'currentTimestamp') - * @param currentTimestamp - */ - private async cleanupOldDependencies(client: PoolClient, currentTimestamp: string) { - const t0 = Logger.start(this.constructor.name, `cleanupOldDependencies`, 0) - const res = await brkOnErr(client.query<{count: number}>(`SELECT count(*):: integer From ${this.tempTable} `)) - const size = res.rows[0].count; - const limit = this.cleanupDependencyBatchSize; - // useful for debugging - // if (this.constructor.name === 'PEntityTypeService') { - // console.log(`-- > ${this.cycle} handle update of ${size} items`) - // } - for (let offset = 0; offset < size; offset += limit) { - const logString = `cleanupOldDependencies batch ${offset + limit > size ? size % limit : limit} (${(offset / limit) + 1} /${Math.floor(size / limit) + 1}) in cycle ${this.cycle} ` - const t2 = Logger.start(this.constructor.name, `${logString} `, 0) - await this.cleanUpDependenciesBatch(client, limit, offset, currentTimestamp); - Logger.itTook(this.constructor.name, t2, `to ${logString} `, 0) - - } - - // if (new Date().getTime() - t0 > 1000) { - // await this.printQuery(client, 'cleanup_old_deps', sql, []) - // } - Logger.itTook(this.constructor.name, t0, `to cleanupOldDependencies `, 0) - } - - async cleanUpDependenciesBatch(client: PoolClient, limit: number, offset: number, currentTimestamp: string) { - - // const promises: Promise[] = [] - const parts = this.isReceiverOf.map((depDS, i) => ` - tw${i} AS ( - DELETE - FROM ${depDS.schemaTable} t1 - WHERE (${depDS.receiverDS.index.keyDefs.map(k => `"r_${k.name}"`).join(',')}) IN ( - SELECT ${depDS.receiverDS.index.keyDefs.map(k => `"${k.name}"`).join(',')} - FROM ${this.tempTable} - LIMIT $1 OFFSET $2 - ) - AND t1.tmsp_last_aggregation < '${currentTimestamp}' - RETURNING * - ) - `).join(', ') - - const sum = ` - SELECT sum(count) - FROM ( - ${this.isReceiverOf.map((depDS, i) => `SELECT count(*) from tw${i}`).join(' UNION ')} - )x - ` - const sql = ` - WITH - ${parts} - ${sum} - ` - - - await client.query(sql, [limit, offset]) - } - - - async aggregateAll(client: PoolClient, client2: PoolClient, currentTimestamp: string) { - let changes = 0 - const res = await brkOnErr(client.query<{count: number}>(`SELECT count(*):: integer From ${this.tempTable} `)) - const size = res.rows[0].count; - const limit = this.batchSize; - // useful for debugging - // if (this.constructor.name === 'PEntityTypeService') { - // console.log(`-- > ${this.cycle} handle update of ${size} items`) - // } - for (let offset = 0; offset < size; offset += limit) { - const logString = `batch aggregate ${offset + limit > size ? size % limit : limit} (${(offset / limit) + 1} /${Math.floor(size / limit) + 1}) in cycle ${this.cycle} ` - const t0 = Logger.start(this.constructor.name, `${logString} `, 0) - changes += await this.aggregateBatch(client, client2, limit, offset, currentTimestamp); - Logger.itTook(this.constructor.name, t0, `to ${logString} `, 0) - - } - return changes - } - - async aggregateBatch(client: PoolClient, client2: PoolClient, limit: number, offset: number, currentTimestamp: string) { - let changes = 0 - - const toAggregate = await brkOnErr(client.query(` -SELECT * From ${this.tempTable} -LIMIT $1 OFFSET $2 - `, [limit, offset])); - - if (toAggregate.rows.length > 0) { - let valuesStr = ''; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const params: any[] = [], addParam = (val: any) => { - params.push(val); - return '$' + params.length; - }; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const addParams = (vals: any[]) => { - return vals.map((val) => addParam(val)).join(','); - }; - let i = 0; - for (const key of toAggregate.rows) { - i++; - const val = await brkOnErr(this.aggregate(key)); - const sql = `(${addParams([...this.index.getKeyModelValues(key), JSON.stringify(val)])}) ${i < toAggregate.rows.length ? ',' : ''} `; - valuesStr = valuesStr + sql; - } - // insert or update the results of the aggregation - const aggSql = ` -INSERT INTO ${this.index.schemaTable} (${this.index.keyCols}, val) -VALUES ${valuesStr} -ON CONFLICT(${this.index.keyCols}) -DO UPDATE -SET val = EXCLUDED.val -WHERE EXCLUDED.val IS DISTINCT FROM ${this.index.schemaTable}.val -RETURNING * - `; - const depSqls: string[] = []; - // get the dependency sqls - this.isReceiverOf.forEach(dep => { - const depSql = dep.getSqlForStoringCache(currentTimestamp, params); - if (depSql) - depSqls.push(depSql); - }); - // create the full query - const parts: string[] = [aggSql, ...depSqls]; - const tws = parts.map((part, j) => `tw${j} AS( - ${part} - )`).join(','); - let twOnUpsert = ''; - if (this.onUpsertSql) { - twOnUpsert = ` - , onUpsert AS( - ${this.onUpsertSql('tw0')} - )`; - } - const sql = ` -WITH -${tws} -${twOnUpsert} -SELECT count(*):: int changes FROM tw0; -`; - // logSql(sql, params) - const result = await brkOnErr(client.query<{changes: number;}>(sql, params)); - changes = result.rows?.[0].changes ?? 0; - } - return changes; - } - - /** - * Creates a table with the items that need to be (re-)aggregated - * - items of the creatorDS that have been modified since last update and not deleted until current timestamp - * that have a tmsp_last_modification greater than 'changesConsideredUntil' - * and less or equal than the 'currentTimestamp' - * and that have a tmsp_deleted that is null or greater than 'currentTimestamp' - * - * - items that have a provider (through dependency) that has been updated or deleted since last update - * that have a tmsp_last_modification greater than 'changesConsideredUntil' - * and less or equal than the 'currentTimestamp' - * or that have a tmsp_deleted greater than 'changesConsideredUntil' - * and less or equal than 'currentTimestamp' - * @param changesConsideredUntil - * @param currentTimestamp - */ - private async findWhatToAggregate(client: PoolClient, updatesConsidered: LeftDSDates, updatesDone: LeftDSDates) { - if (this.creatorDS.length === 0) throw new Error("At least one creator DS is needed"); - const t0 = Logger.start(this.constructor.name, `findWhatToAggregate`, 0) - - const sql0 = `CREATE TEMPORARY TABLE ${this.tempTable} ON COMMIT DROP AS` - const sql1 = this.creatorDS.map(createDS => { - const creatorIdx = createDS.dataService.index; - const creatorSqlParts = this.getCreatorCustomSql(createDS.customSql) - const {after, until} = this.getAfterAndUntil(createDS.dataService.constructor.name, updatesConsidered, updatesDone); - - return `${creatorSqlParts.map(part => ` - SELECT ${part.select ? part.select : this.index.keyCols} - FROM ${creatorIdx.schemaTable} - WHERE tmsp_last_modification > '${after}' - AND tmsp_last_modification <= '${until}' - AND (tmsp_deleted IS NULL OR tmsp_deleted > '${until}') - ${part.where ? `AND ${part.where}` : ''} - `).join(' UNION ') - }`; - }).join(' UNION ') - - const sql2 = this.isReceiverOf.map(depDS => { - const {after, until} = this.getAfterAndUntil(depDS.providerDS.constructor.name, updatesConsidered, updatesDone); - - return ` - UNION - SELECT ${depDS.receiverKeyDefs.map(k => `t1."${k.name}"`).join(',')} - FROM ${depDS.schema}.${depDS.table} t1, - ${depDS.providerDS.index.schema}.${depDS.providerDS.index.table} t2 - WHERE ${depDS.providerDS.index.keyDefs - .map(k => `t2."${k.name}" = t1."p_${k.name}"`).join(' AND ')} - AND ( - ( - t2.tmsp_last_modification > '${after}' - AND - t2.tmsp_last_modification <= '${until}' - ) - OR - ( - t2.tmsp_deleted > '${after}' - AND - t2.tmsp_deleted <= '${until}' - ) - ) - `; - }).join('\n'); - await client.query(sql0 + sql1 + sql2); - Logger.itTook(this.constructor.name, t0, `to findWhatToAggregate `, 0) - - // if (this.constructor.name === 'PEntityTypeService') { - // console.log('-------------') - // console.log(`cons: ${changesConsideredUntil}, curr: ${currentTimestamp} `) - // console.log('------------- agg_pentitylabel') - // const a = await this.index.pgPool.query('SELECT * FROM war_cache.agg_pentitylabel') - // console.log(JSON.stringify(a.rows, null, 2)) - // console.log('------------- agg_pentitytype__on__agg_pentitylabel') - // const b = await this.index.pgPool.query('SELECT * FROM war_cache.agg_pentitytype__on__agg_pentitylabel') - // console.log(JSON.stringify(b.rows, null, 2)) - // console.log(`------------- ${this.tempTable} ------------- `) - // console.log(`cons: ${changesConsideredUntil}, curr: ${currentTimestamp} `) - // const x = await client.query(`SELECT * FROM ${this.tempTable} `) - // console.log(JSON.stringify(x.rows, null, 2)) - // if(x.rows.length===2){ - // console.log(2) - - // } - // } - } - - - /** - * returns the select statement, i.e.: - * the columns to select from the creator data service. - * If creatorDS and aggregatedDS have not the same KeyModel, - * the column names do not (necessarily) match. In this case, - * this statement should provide the correct mapping by labeling the - * creatorDS columns with the name the corresponding columns have in this - * aggregated data service. - * - * example, where column names match (i.e. creator and aggegator have same KeyModel) - * `"pkEntity", "fkProject"` - * - * example where column - * `"fkDomain" as "pkClass"` -> creatorDS.fkDomain = aggDS.pkClass - */ - private getCreatorCustomSql(customSql: CustomCreatorDsSql[] | undefined) { - return customSql ?? [{select: this.index.keyCols}]; - } - - // /** - // * returns tmsp of last changes considered by this aggregator service - // * If not existing, returns a very early tmsp (year 1970) so that we - // * can assume that all changes after 1970 will be considered - // */ - // private async getChangesConsideredUntilTsmp() { - // const val = await this.wh.metaTimestamps.getFromIdx(this.constructor.name + '__changes_considered_until'); - // return val?.tmsp ?? new Date(0).toISOString() - // } - // private async setChangesConsideredUntilTsmp(tmsp: string) { - // await this.wh.metaTimestamps.addToIdx(this.constructor.name + '__changes_considered_until', {tmsp}); - // } - - - - /** - * registers the DataService that contains the items for each of them - * an aggregation should happen - */ - // eslint-disable-next-line @typescript-eslint/no-explicit-any - registerCreatorDS>(...creatorDS: CreatorDS[]) { - creatorDS.forEach((ds) => { - this.creatorDS.push(ds); - ds.dataService.registerIsCreatorOf(this) - }) - } - /** - * Adds dep to this.isReceiverOf with the effect that this DataService acts - * as the receiver of data for dep. - * @param dep - */ - registerReceiverOf(dep: DependencyIndex) { - this.isReceiverOf.push(dep) - } - - - - - async setUpdatesConsidered(leftDsDates: LeftDSDates) { - await this.wh.aggregationTimestamps.addToIdx(this.constructor.name + CHANGES_CONSIDERED_UNTIL_SUFFIX, leftDsDates); - } - async getUpdatesConsidered(): Promise { - const val = await this.wh.aggregationTimestamps.getFromIdx(this.constructor.name + CHANGES_CONSIDERED_UNTIL_SUFFIX); - return val ?? {} - } - - async getleftDSupdateDone(): Promise { - - - const leftDS = [ - ...this.isReceiverOf.map(depIdx => depIdx.providerDS), - ...this.creatorDS.map(item => item.dataService) - ] - const sql = ` - SELECT jsonb_object_agg(replace(key,'${LAST_UPDATE_DONE_SUFFIX}','') , val->>'tmsp') o - FROM ${this.wh.metaTimestamps.schemaTable} - WHERE key IN ( - ${leftDS.map(item => `'${item.constructor.name + LAST_UPDATE_DONE_SUFFIX}'`).join(',')} - ) - ` - const res = await this.wh.whPgPool.query<{o: LeftDSDates}>(sql) - - const returnval = res.rows?.[0]?.o ?? {} - return returnval - - } - - /** - * prints a query in a file with a query for the tmpTable for debugging - */ - private async printQuery(client: PoolClient, prefix: string, sql: string, params: any[]) { - const dir = './dev/agg-logs'; - if (!existsSync(dir)) { - mkdirSync(dir); - } - const filename = prefix + '-' + new Date().toISOString() - - const tmptable = await client.query(` - SELECT ${this.index.keyDefs.map(k => `"${k.name}"`)} - FROM ${this.tempTable} - `, []) - const tmpTableDebugSql = ` - CREATE TEMP TABLE ${this.tempTable} ON COMMIT DROP AS ( - SELECT ${this.index.keyDefs.map(k => `"${k.name}"`)} - FROM - (VALUES ${tmptable.rows.map(r => `(${this.index.keyDefs.map(k => r[k.name as keyof KeyModel]).join(', ')})`).join(', ')}) as x(${this.index.keyDefs.map(k => `"${k.name}"`).join(',')}) - ); - ` - - - params.forEach((param, j) => { - const replaceStr = new RegExp('\\$' + (j + 1) + '(?!\\d)', 'g') - sql = sql.replace(replaceStr, typeof param === 'string' ? "'" + param + "'" : param) - }) - sql = sqlFormatter.format(sql, {language: 'pl/sql'}); - - const log = `BEGIN; - ${tmpTableDebugSql} - - ${sql} - ` - writeFileSync(dir + '/' + filename, log, 'utf-8') - return 0 - } - -} - - - diff --git a/server/src/warehouse/base/classes/AggregatedDataService2.ts b/server/src/warehouse/base/classes/AggregatedDataService2.ts deleted file mode 100644 index 36f17f1f8..000000000 --- a/server/src/warehouse/base/classes/AggregatedDataService2.ts +++ /dev/null @@ -1,66 +0,0 @@ -import {combineLatest} from 'rxjs'; -import {filter} from 'rxjs/operators'; -import {Warehouse} from '../../Warehouse'; -import {KeyDefinition} from '../interfaces/KeyDefinition'; -import {AggregatedDataService} from './AggregatedDataService'; -import {DataService} from './DataService'; -import {DependencyIndex} from './DependencyIndex'; -import {QueryDef, TmpTableResult} from './AggregatorSqlBuilder'; -import {PoolClient} from 'pg'; -import {logSql} from '../../../utils/helpers'; - - -export abstract class AggregatedDataService2 extends AggregatedDataService { - - // eslint-disable-next-line @typescript-eslint/no-explicit-any - dependencyIndexes: DependencyIndex[] = []; - - constructor( - public wh: Warehouse, - protected keyDefs: KeyDefinition[] - ) { - super(wh, keyDefs) - } - - getDependencies() { } - - emitReady() { - combineLatest([this.index.ready$, ...this.dependencyIndexes.map(d => d.ready$)]).pipe( - filter(d => !d.some(item => item !== true)), - ).subscribe((data) => { - this.ready$.next(true) - }) - } - - async clearAll() { - await Promise.all(this.dependencyIndexes.map(x => x.clearAll())); - } - - // eslint-disable-next-line @typescript-eslint/no-explicit-any - private registerDepIdx>(dep: M) { - this.dependencyIndexes.push(dep); - return dep; - } - addDepencency(providerDS: DataService) { - return this.registerDepIdx(new DependencyIndex( - this.wh, - this, - providerDS, - )) - } - - - - - - // eslint-disable-next-line @typescript-eslint/no-explicit-any - async brkForDebugging(client: PoolClient, item: TmpTableResult | QueryDef) { - logSql(item.sql, item.params) - await client.query('commit;') - process.exit() - } - -} - - - diff --git a/server/src/warehouse/base/classes/AggregatedDataServicesBase.ts b/server/src/warehouse/base/classes/AggregatedDataServicesBase.ts deleted file mode 100644 index 86083d312..000000000 --- a/server/src/warehouse/base/classes/AggregatedDataServicesBase.ts +++ /dev/null @@ -1,29 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import {combineLatest, Observable} from 'rxjs'; -import {filter, mapTo} from 'rxjs/operators'; -import {AggregatedDataService2} from './AggregatedDataService2'; -import {DataServiceBundle} from './DataServiceBundle'; - -export abstract class AggregatedDataServicesBase extends DataServiceBundle> { - - - ready$: Observable - - constructor(...aggDs: AggregatedDataService2[]) { - super() - - this.registered = aggDs; - - this.ready$ = combineLatest( - this.registered.map(ds => ds.ready$.pipe(filter(r => r === true))), - ).pipe(mapTo(true)) - } - - - async startCycling() { - for (const ds of this.registered) { - await ds.startUpdate() - } - } - -} diff --git a/server/src/warehouse/base/classes/AggregatorSqlBuilder.ts b/server/src/warehouse/base/classes/AggregatorSqlBuilder.ts deleted file mode 100644 index b9b656840..000000000 --- a/server/src/warehouse/base/classes/AggregatorSqlBuilder.ts +++ /dev/null @@ -1,538 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import {PoolClient} from 'pg'; -import {pgLogOnErr} from '../../../utils/helpers'; -import {SqlBuilderBase} from '../../../utils/sql-builders/sql-builder-base'; -import {AggregatedDataService2} from './AggregatedDataService2'; -import {DataIndexPostgres} from './DataIndexPostgres'; -import {DependencyIndex} from './DependencyIndex'; -import {existsSync, mkdirSync, writeFileSync} from 'fs'; -import sqlFormatter from 'sql-formatter'; -import {PgDataReplicator, DataReplicatorSqlFn, PgTable} from './PgDataReplicator'; -export interface TableDef { - tableName: string -} -export type CustomValSql = (q: SqlBuilderBase) => string -export type ProviderKeyColMapping = { - [key in keyof PK]: { - leftCol?: keyof RK, - leftVal?: {name: keyof RV, type: 'int' | 'text' | 'bool'}, - leftCustom?: {name: keyof CustomObject, type: 'int' | 'text' | 'bool'}, - value?: string | number - } -} -type HasValOperator = 'IS NOT NULL' | 'is not {}' -export type HasValCondition = { - [key in keyof PV]?: HasValOperator -} -export type HasKeyCondition = { - [key in keyof PK]?: 'IS NOT NULL' -} -export type HasCondition = { - and?: HasCondition[], - or?: HasCondition[], - providerKey?: HasKeyCondition, - providerVal?: HasValCondition - custom?: string -} -export type AggregateWhereCondition = '= true' | '= false' -interface JoinProviderConfig { - leftTable: TableDef, - joinWithDepIdx: DependencyIndex, - joinOnKeys: ProviderKeyColMapping, - joinOnCustom?: string[], - joinWhereLeftTableCondition?: AggregateWhereCondition, - conditionTrueIf: HasCondition, - createAggregationVal?: { - sql: (provTable: string) => string - upsert: boolean | {whereCondition: '= true' | '= false'} - }, - createCustomObject?: CustomValSql -} -export interface TmpTableResult { - sql: string; - params: any[] - tableDef: TableDef; -} -export interface QueryDef { - sql: string; - params: any[] - // optional alternative sql used instead of sql by this.printQueries() - debugSql?: string; -} -export interface TmpTableQeryDef { - sql: string; - params: any[] -} -export interface JoinedProvider { - aggregation: TmpTableResult; - dependencyUpsert: TmpTableResult; - aggregationUpsert?: TmpTableResult; -} - -export class AggregatorSqlBuilder { - - // eslint-disable-next-line @typescript-eslint/no-explicit-any - params: any[] = [] - tws = 0; - aggUpsertTws: string[] = [] - debugTableNames: string[] = [] - - batchTmpTable: { - sql: string; - debugSql?: string; - params: any[]; - tableDef: TableDef; - } - // List of queries that can be executed by calling executeQueries() - queryRequests: QueryDef[] = []; - - // List of data replication requests that can be executed by calling executeQueries() - dataReplicationRequests: PgDataReplicator[] = [] - replicationSourceTable: string; - constructor( - private agg: AggregatedDataService2, - private client: PoolClient, - private currentTimestamp: string, - private limit: number, - private offset: number, - ) { - this.batchTmpTable = this.tmpTableBatch(limit, offset) - this.replicationSourceTable = this.createTableName() - } - - /** - * Function to join a providerDS through dependency index with the current - * AggregatorDS (= receiverDS). - * - * @param c - */ - public async joinProviderThroughDepIdx( - c: JoinProviderConfig - ): Promise> { - // tmpTable for aggregation - const aggregation = await this.tmpTableAggregate(c) - - // tmpTable for upsert deps - const dependencyUpsert = await this.tmpTableUpsertDependencies(c.joinWithDepIdx, aggregation.tableDef.tableName) - - if (c.createAggregationVal?.upsert) { - const aggDs = c.joinWithDepIdx.receiverDS.index - const tableName = aggregation.tableDef.tableName - const whereCondition = c.createAggregationVal.upsert === true ? undefined : - c.createAggregationVal.upsert.whereCondition - // tmpTable for upsert aggregation - const aggregationUpsert = await this.tmpTableUpsertAggregations(aggDs, tableName, whereCondition); - return {aggregation, dependencyUpsert, aggregationUpsert} - } - - return {aggregation, dependencyUpsert} - - } - - /********** - * Table creators - ***********/ - - private tmpTableBatch(limit: number, offset: number): TmpTableResult { - const q = new SqlBuilderBase() - const tableName = this.createTableName() - const createTableStmt = this.createTableStmt(tableName) - const sql = ` - ${createTableStmt} ( - SELECT ${this.agg.index.keyDefs.map(k => `"${k.name}" as "r_${k.name}"`)} - FROM ${this.agg.tempTable} - LIMIT ${q.addParam(limit)} OFFSET ${q.addParam(offset)} - )` - const debugSql = ` - SELECT - 'CREATE TEMP TABLE ${tableName} ON COMMIT DROP AS ( - SELECT ${this.agg.index.keyDefs.map(k => `"${k.name}" as "r_${k.name}"`)} - FROM - (VALUES ' || string_agg( - '(' || ${this.agg.index.keyDefs.map(k => ` t1."${k.name}"::text `).join(`||','||`)} ||')', - ',' - ) ||') as x(${this.agg.index.keyDefs.map(k => `"${k.name}"`).join(',')}) - )' as sql - FROM ( - SELECT ${this.agg.index.keyDefs.map(k => `"${k.name}"`)} - FROM ${this.agg.tempTable} - LIMIT ${limit} OFFSET ${offset} - ) t1 - ` - - return this.registerTmpTable(sql, q.params, tableName, debugSql); - - } - - - private async tmpTableAggregate( - c: JoinProviderConfig - ): Promise> { - const q = new SqlBuilderBase() - const tableName = this.createTableName() - - const provider = c.joinWithDepIdx.providerDS.index - const receiver = c.joinWithDepIdx.receiverDS.index - const {selectProviderKeys, joinOns} = this.aggSelectAndJoin(q, c.joinOnKeys, c.joinOnCustom); - const createConditionSql = this.aggCondition(c.conditionTrueIf); - const whereConditionSql = this.aggWhereCondition(c.joinWhereLeftTableCondition) - const receiverValSql = this.aggCreateReceiverVal(c.createAggregationVal?.sql) - const customObjectSql = this.aggCreateCustomObject(q, c.createCustomObject) - const createTableStmt = this.createTableStmt(tableName) - - const sql = ` - -- aggregate - ${createTableStmt} ( - SELECT - --receiver keys - ${receiver.keyDefs.map(k => `t1."r_${k.name}"`)}, - --provider keys - ${selectProviderKeys.join(',')}, - --provider val - t2.val p_val, - --val - ${receiverValSql} val, - --custom - ${customObjectSql} custom, - --condition - ${createConditionSql} condition - FROM ${c.leftTable.tableName} t1 - LEFT JOIN ${provider.schemaTable} t2 - ON ${joinOns.join(' AND ')} - AND tmsp_deleted IS NULL - ${whereConditionSql} - )`; - - return this.registerTmpTable(sql, q.params, tableName); - } - - - - private async tmpTableUpsertDependencies( - depIdx: DependencyIndex, - leftTableName: string - ): Promise> { - const q = new SqlBuilderBase() - const tableName = this.createTableName() - const createTableStmt = this.createTableStmt(tableName) - - const sql = ` - -- insert or update dependencies - ${createTableStmt} ( - WITH tw1 AS ( - INSERT INTO - ${depIdx.schemaTable} - (${depIdx.keyCols}, tmsp_last_aggregation) - SELECT DISTINCT ON ( - ${depIdx.providerKeyCols}, - ${depIdx.receiverKeyCols} - ) - --dep index keys - ${depIdx.providerKeyCols}, - ${depIdx.receiverKeyCols}, - --tmsp_last_aggregation - ${q.addParam(this.currentTimestamp)} - FROM ${leftTableName} - ON CONFLICT (${depIdx.keyCols}) - DO UPDATE - SET tmsp_last_aggregation = EXCLUDED.tmsp_last_aggregation - RETURNING * - ) - SELECT * FROM tw1 - )`; - return this.registerTmpTable(sql, q.params, tableName); - } - - - - async tmpTableUpsertAggregations( - aggregatorDS: DataIndexPostgres, - aggregationTableName: string, - whereCondition?: AggregateWhereCondition): - Promise> { - const tableName = this.addAggUpsertTw(); - const createTableStmt = this.createTableStmt(tableName) - - const sql = ` - -- insert or update aggregation results - ${createTableStmt} ( - WITH tw1 AS ( - INSERT INTO ${aggregatorDS.schemaTable} (${aggregatorDS.keyCols},val) - SELECT DISTINCT ON (${aggregatorDS.keyDefs.map(k => `"r_${k.name}"`)}) - --receiver keys - ${aggregatorDS.keyDefs.map(k => `"r_${k.name}"`)}, - --val - val - FROM ${aggregationTableName} - ${whereCondition ? ` - WHERE condition ${whereCondition}` : ''} - ON CONFLICT (${aggregatorDS.keyCols}) - DO UPDATE - SET val = EXCLUDED.val - WHERE - EXCLUDED.val IS DISTINCT FROM ${aggregatorDS.schemaTable}.val - RETURNING * - ) - SELECT * FROM tw1 - )`; - return this.registerTmpTable(sql, [], tableName); - - } - - - - - registerUpsertHook(target: PgTable, sqlFn: DataReplicatorSqlFn) { - - const replRequest = new PgDataReplicator<{count: number}>( - {client: this.client, table: this.replicationSourceTable}, - target, - [this.agg.index.keyCols, 'val'], - sqlFn - ) - this.dataReplicationRequests.push(replRequest) - } - - private createCountSql() { - let counts = 'SELECT 0 as count'; - if (this.aggUpsertTws.length) { - counts = `${this.aggUpsertTws.map(aggUpsertTw => `SELECT count(*)::int FROM ${aggUpsertTw}`).join(' UNION ALL ')}`; - } - const sql = ` - -- count nr of changes - WITH tw1 AS (${counts}) - SELECT sum(count)::int as changes - FROM tw1;`; - return sql; - } - - /** - * Function to count number of inserted or updated rows on the table - * of the AggregationDS referenced by this.agg - */ - private async executeCountSql(): Promise { - const sql = this.createCountSql(); - const result = await this.query<{changes: number}>(sql) - const changes = result?.rows?.[0].changes ?? 0; - - return changes - } - - - - - - /********** - * Medium helpers for specific table creators - ***********/ - private aggWhereCondition(aggregateWhereHasVal?: AggregateWhereCondition) { - return aggregateWhereHasVal ? `WHERE t1.condition ${aggregateWhereHasVal}` : ''; - } - - private aggCondition( - hasCondition: HasCondition): string { - if (Object.keys(hasCondition).length === 0) { - return `null::boolean` - } - else if (hasCondition.or) { - return `(${hasCondition.or.map(c => `${this.aggCondition(c)}`).join(' OR ')} )` - - } else if (hasCondition.and) { - return `(${hasCondition.and.map(c => `${this.aggCondition(c)}`).join(' AND ')} )` - } - else { - return this.createLeafCondition(hasCondition) - } - - } - private aggSelectAndJoin( - q: SqlBuilderBase, - keyMapping: ProviderKeyColMapping, - joinOnCustom: string[] = [] - ) { - const selectProviderKeys: string[] = []; - const joinOns: string[] = []; - for (const providerCol in keyMapping) { - if (Object.prototype.hasOwnProperty.call(keyMapping, providerCol)) { - const e = keyMapping[providerCol]; - let select: string; - let joinOn: string; - if (e.leftCol) { - select = `t1."r_${e.leftCol}" "p_${providerCol}"`; - joinOn = `t1."r_${e.leftCol}" = t2."${providerCol}"`; - } - else if (e.leftCustom) { - select = `(t1.custom->>'${e.leftCustom.name}')::${e.leftCustom.type} "p_${providerCol}"`; - joinOn = `(t1.custom->>'${e.leftCustom.name}')::${e.leftCustom.type} = t2."${providerCol}"`; - } - else if (e.leftVal) { - select = `(t1.val->>'${e.leftVal.name}')::${e.leftVal.type} "p_${providerCol}"`; - joinOn = `(t1.val->>'${e.leftVal.name}')::${e.leftVal.type} = t2."${providerCol}"`; - } - else if (e.value) { - const parse = typeof e.value === 'number' ? '::int' : ''; - joinOn = `${q.addParam(e.value)} = t2."${providerCol}"`; - - select = `${q.addParam(e.value)}${parse} "p_${providerCol}"`; - } - else { - throw new Error('Please provide how to map this key'); - } - selectProviderKeys.push(select); - joinOns.push(joinOn); - } - } - for (const joinCustom of joinOnCustom) { - joinOns.push(joinCustom) - } - return {selectProviderKeys, joinOns}; - } - - private aggCreateReceiverVal(createReceiverVal?: ((provTable: string) => string)) { - return !createReceiverVal ? `'{}'::jsonb` : - createReceiverVal('t2'); - } - - private aggCreateCustomObject(q: SqlBuilderBase, createCustomObject?: CustomValSql) { - return !createCustomObject ? `'{}'::jsonb` : - createCustomObject(q); - } - private createLeafCondition(hasCondition: HasCondition) { - const conditions: string[] = []; - const valConditions = hasCondition.providerVal; - const keyConditions = hasCondition.providerKey; - const custom = hasCondition.custom; - if (valConditions) { - for (const valProp in valConditions) { - if (Object.prototype.hasOwnProperty.call(valConditions, valProp)) { - const operator: HasValOperator | undefined = valConditions[valProp]; - if (operator === 'IS NOT NULL') { - conditions.push(`t2.val->>'${valProp}' IS NOT NULL`); - } else if (operator === 'is not {}') { - conditions.push(`t2.val->'${valProp}' != '{}'::jsonb`); - } - } - } - } - if (keyConditions) { - for (const keyCol in keyConditions) { - if (Object.prototype.hasOwnProperty.call(keyConditions, keyCol)) { - const operator = keyConditions[keyCol]; - conditions.push(`t2."${keyCol}" ${operator}`); - } - } - } - if (custom) { - conditions.push(custom); - } - return conditions.join(' AND '); - } - - /********** - * Small generic helpers - ***********/ - createTableName() { - this.tws++ - return `tw_${this.offset}_${this.tws}` - - } - addAggUpsertTw() { - const tw = this.createTableName() - this.aggUpsertTws.push(tw) - return tw - } - createTableStmt(tableName: string) { - return `CREATE TEMP TABLE ${tableName} ON COMMIT DROP AS`; - } - // eslint-disable-next-line @typescript-eslint/no-explicit-any - query(sql: string, params: any[] = []) { - return pgLogOnErr((s, p) => this.client.query(s, p), sql, params) - } - - registerTmpTable( - sql: string, - params: any[], - tableName: string, - debugSql?: string) { - this.registerQueryRequest(sql, params, debugSql); - const tableDef: TableDef = { - tableName - }; - return {sql, debugSql, params, tableDef}; - } - - registerQueryRequest(sql: string, params: any[], debugSql?: string) { - this.queryRequests.push({sql, params, debugSql}) - } - - /** - * execute the queries in queryRequests list - * and returns the result of count query - */ - async executeQueries() { - for (const q of this.queryRequests) { - await this.query(q.sql, q.params) - } - - await this.executeDataReplications(); - - return this.executeCountSql() - } - - private async executeDataReplications() { - if (this.dataReplicationRequests.length) { - - const createTableStmt = this.createTableStmt(this.replicationSourceTable); - const sql = ` - ${createTableStmt} - ${this.aggUpsertTws.map(aggUpsertTw => `SELECT * FROM ${aggUpsertTw}`).join(' UNION ALL ')}`; - await this.query(sql, []); - - for (const replReq of this.dataReplicationRequests) { - await replReq.replicateTable(); - } - - } - } - - /** - * prints the queries in queryRequests list including the count query - * for debugging - */ - async printQueries() { - const dir = './dev/agg-logs'; - if (!existsSync(dir)) { - mkdirSync(dir); - } - const filename = this.agg.tempTable + '-' + new Date().toISOString() - let log = `` - const countSql = this.createCountSql() - const allQueries = [...this.queryRequests, {sql: countSql, params: []}]; - for (let i = 0; i < allQueries.length; i++) { - const q = allQueries[i]; - const params = q.params - let sql = q.sql; - if (i === 0 && q.debugSql) { - const res = await this.query<{sql: string}>(q.debugSql, []) - sql = res.rows?.[0].sql ?? ''; - } - params.forEach((param, j) => { - const replaceStr = new RegExp('\\$' + (j + 1) + '(?!\\d)', 'g') - sql = sql.replace(replaceStr, typeof param === 'string' ? "'" + param + "'" : param) - }) - sql = sqlFormatter.format(sql, {language: 'pl/sql'}); - log = `${log}\n\n${sql};` - } - log = `BEGIN; - -- the subsequent blocks create tmp tables. see last line for how to select - -- their content for debugging - ${log} - - -- You can select from a tmp table: - -- SELECT * FROM tw2; - ` - writeFileSync(dir + '/' + filename, log, 'utf-8') - return 0 - } - -} diff --git a/server/src/warehouse/base/classes/ClearAll.ts b/server/src/warehouse/base/classes/ClearAll.ts deleted file mode 100644 index 639ffda26..000000000 --- a/server/src/warehouse/base/classes/ClearAll.ts +++ /dev/null @@ -1,8 +0,0 @@ -export interface ClearAll { - clearAll(): Promise; - // initIdx(): Promise; -} - -export interface InitAll { - initAll(): Promise; -} diff --git a/server/src/warehouse/base/classes/DataIndexPostgres.ts b/server/src/warehouse/base/classes/DataIndexPostgres.ts deleted file mode 100644 index 222fa6a3d..000000000 --- a/server/src/warehouse/base/classes/DataIndexPostgres.ts +++ /dev/null @@ -1,337 +0,0 @@ -import {Pool} from 'pg'; -import QueryStream from 'pg-query-stream'; -import {keys, values} from 'ramda'; -import {combineLatest, ReplaySubject} from 'rxjs'; -import {first} from 'rxjs/operators'; -import {Warehouse} from '../../Warehouse'; -import {KeyDefinition} from '../interfaces/KeyDefinition'; -import {handleAsyncStream} from './IndexLeveldb'; -import {SqlUpsertQueue} from './SqlUpsertQueue'; - -export class DataIndexPostgres { - - static indexes = 0; - pgPool: Pool - - schema: string; - table: string; - schemaTable: string; - ready$ = new ReplaySubject() - keyCols: string; // e.g. '"fkProject","pkEntity"' - - insertStmt: string; - keyJsonObjSql: string; - - upsertQueue: SqlUpsertQueue; - - constructor( - public keyDefs: KeyDefinition[], - name: string, - wh: Warehouse - ) { - if (!keyDefs || keyDefs?.length < 1) throw Error(`KeyDefs missing in ${name}`) - this.pgPool = wh.whPgPool; - this.schema = wh.schemaName; - this.table = name; - this.schemaTable = `${this.schema}.${this.table}` - this.keyCols = this.keyDefs.map(k => `"${k.name}"`).join(',') - this.insertStmt = this.createInsertStatement() - this.keyJsonObjSql = this.createKeyJsonObjSql() - combineLatest( - wh.gvPgListenerConnected$, - wh.createSchema$ - ).pipe(first()).subscribe(([client]) => { - this.setupTable() - .then(() => { - this.ready$.next(true) - }) - .catch((e) => this.err(e)) - }) - } - - - - private async setupTable() { - await this.pgPool.query(` - CREATE TABLE IF NOT EXISTS ${this.schema}.${this.table} ( - ${this.keyDefs.map(keyDef => `"${keyDef.name}" ${keyDef.type}`).join(',')}, - val jsonb, - tmsp_last_modification timestamp with time zone, - tmsp_deleted timestamp with time zone, - CONSTRAINT ${this.table}_keys_uniq UNIQUE (${this.keyCols}) - )`).catch((e) => { - console.log(`Error during CREATE TABLE: ${this.schema}.${this.table}:`, e) - }) - - await this.pgPool.query(` - DROP TRIGGER IF EXISTS last_modification_tmsp ON ${this.schema}.${this.table} - `).catch((e) => { - console.log(`Error during DROP TRIGGER last_modification_tmsp: ${this.schema}.${this.table}:`, e) - }) - await this.pgPool.query(` - CREATE TRIGGER last_modification_tmsp - BEFORE INSERT OR UPDATE - ON ${this.schema}.${this.table} - FOR EACH ROW - EXECUTE PROCEDURE ${this.schema}.tmsp_last_modification(); - `).catch((e) => { - console.log(`Error during CREATE TRIGGER last_modification_tmsp: ${this.schema}.${this.table}:`, e) - }) - const indexedCols = [ - ...this.keyDefs.map(k => k.name), - 'tmsp_last_modification', - 'tmsp_deleted' - ] - for (const indexCol of indexedCols) { - - await this.pgPool.query(` - CREATE INDEX IF NOT EXISTS ${this.schema}_${this.table}_${indexCol}_idx - ON ${this.schema}.${this.table}("${indexCol}"); - `).catch((e) => { - console.log(`Error during CREATE INDEX: ${this.schema}.${this.table}(${indexCol}):`, e) - }) - } - - } - - // forEachValue(cb: (val: ValueModel) => void): Promise { - // throw new Error('Method not implemented.'); - // } - // getValues(): Promise { - // throw new Error('Method not implemented.'); - // } - - // eslint-disable-next-line @typescript-eslint/no-explicit-any - err(e: any) { - console.log(`Error during setup of ${this.schema}.${this.table}:`, e) - } - - // async addToIdx(keyModel: KeyModel, val: ValueModel): Promise { - // const key: string = this.keyToString(keyModel); - - // return new Promise((res, rej) => { - // this.pgPool.query(this.insertStmt, [ - // ...this.getKeyModelValues(keyModel), - // key, - // Array.isArray(val) ? JSON.stringify(val) : val, - // ]).then(_ => res()) - // .catch(e => { - // console.log('error when inserting: ', {key, val}) - // console.log(e) - // }); - // }) - // } - - - getKeyModelValues(keyModel: KeyModel) { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - return this.keyDefs.map(k => (keyModel as any)[k.name]); - } - - // async removeFromIdx(keyModel: KeyModel): Promise { - // const key: string = this.keyToString(keyModel); - // await this.pgPool.query(` - // DELETE FROM ${this.schema}.${this.table} WHERE key = $1; - // `, [key]); - // } - - - async removeFromIdxWhereDeletedBefore(tmsp: string): Promise { - await this.pgPool.query(` - DELETE FROM ${this.schema}.${this.table} WHERE tmsp_deleted <= $1; - `, [tmsp]); - } - - /** - * Returns value of key. If key not exists, returns undefined. - * @param key - */ - - async getFromIdx(keyModel: KeyModel): Promise { - return new Promise((res, rej) => { - const sql = ` - SELECT val - FROM ${this.schema}.${this.table} - WHERE - ${this.keyDefs.map((k, i) => `"${k.name}"=$${i + 1}`).join(' AND ')} - ` - const params = this.getKeyModelValues(keyModel) - this.pgPool.query<{val: ValueModel}>(sql, params) - .then(results => res(results?.rows?.[0]?.val)) - .catch(e => rej(e)) - }) - } - - - async getFromIdxNoDeleted(keyModel: KeyModel): Promise { - return new Promise((res, rej) => { - const sql = ` - SELECT val - FROM ${this.schema}.${this.table} - WHERE - ${this.keyDefs.map((k, i) => `"${k.name}"=$${i + 1}`).join(' AND ')} - AND tmsp_deleted IS NULL -- Check if should not be (tmsp_deleted IS NULL OR tmsp_deleted > currentTsmp) - ` - const params = this.getKeyModelValues(keyModel) - this.pgPool.query<{val: ValueModel}>(sql, params) - .then(results => res(results?.rows?.[0]?.val)) - .catch(e => rej(e)) - }) - } - - async getFromIdxWithTmsps(keyModel: KeyModel): Promise<{ - val?: ValueModel, - lastModification?: Date, - deleted?: Date - } > { - return new Promise((res, rej) => { - const sql = ` - SELECT val, tmsp_last_modification, tmsp_deleted - FROM ${this.schema}.${this.table} - WHERE - ${this.keyDefs.map((k, i) => `"${k.name}"=$${i + 1}`).join(' AND ')} - ` - const params = this.getKeyModelValues(keyModel) - this.pgPool.query<{ - val: ValueModel, - tmsp_last_modification: string, - tmsp_deleted: string - }>(sql, params) - .then(results => { - const row = results?.rows?.[0] - if (!row) res({}); - res({ - val: row?.val, - lastModification: row.tmsp_last_modification ? new Date(row.tmsp_last_modification) : undefined, - deleted: row.tmsp_deleted ? new Date(row.tmsp_deleted) : undefined - }) - }) - .catch(e => rej(e)) - }) - } - - - // async forEachKey(cb: (key: KeyModel) => Promise) { - // const querystream = new QueryStream( - // `SELECT key FROM ${this.schema}.${this.table}` - // ) - // const stream = await this.manageQueryStream(querystream); - // return handleAsyncStream(stream, (item) => cb(this.stringToKey(item.key))); - - // } - - // async forEachKeyStartingWith(str: string, cb: (key: KeyModel) => Promise): Promise { - // const querystream = new QueryStream( - // `SELECT key FROM ${this.schema}.${this.table} WHERE key LIKE $1`, - // [str + '%'] - // ) - // const stream = await this.manageQueryStream(querystream); - // return handleAsyncStream(stream, (item) => cb(this.stringToKey(item.key))); - // } - - // async forEachItemWith(partialKey: Partial, cb: (item: {key: KeyModel, value: ValueModel}) => Promise): Promise { - // const cols = keys(partialKey); - // if (cols.length < 1) throw new Error("Partial key must contain at least one key"); - // const sql = ` - // SELECT ${this.keyJsonObjSql} as key, val as value - // FROM ${this.schemaTable} - // WHERE ${cols.map((k, i) => `"${k}"=$${i + 1}`).join(' AND ')}` - // const querystream = new QueryStream( - // sql, - // values(partialKey) - // ) - // const stream = await this.manageQueryStream(querystream); - // return handleAsyncStream(stream, (item) => cb({ - // key: item.key, - // value: item.value - // })); - // } - - async forEachItemWithNoDeleted(partialKey: Partial, cb: (item: {key: KeyModel, value: ValueModel}) => Promise): Promise { - const cols = keys(partialKey); - if (cols.length < 1) throw new Error("Partial key must contain at least one key"); - const sql = ` - SELECT ${this.keyJsonObjSql} as key, val as value - FROM ${this.schemaTable} - WHERE ${cols.map((k, i) => `"${k}"=$${i + 1}`).join(' AND ')} - AND tmsp_deleted IS NULL -- Check if should not be (tmsp_deleted IS NULL OR tmsp_deleted > currentTsmp) - ` - const querystream = new QueryStream( - sql, - values(partialKey) - ) - const stream = await this.manageQueryStream(querystream); - return handleAsyncStream(stream, (item) => cb({ - key: item.key, - value: item.value - })); - } - - - - // async clearIdx() { - // await this.pgPool.query(`TRUNCATE TABLE ${this.schema}.${this.table};`) - // } - - - // async keyExists(key: string): Promise { - // return new Promise((res, rej) => { - - // this.pgPool.query<{exists: boolean}>( - // `SELECT EXISTS (SELECT key FROM ${this.schema}.${this.table} WHERE key = $1);`, - // [key] - // ) - // .then(results => res(results?.rows?.[0].exists) - // ).catch(e => rej(e)) - - // }); - // } - - - // async getKeys(): Promise { - // return new Promise((res, rej) => { - // this.pgPool.query<{keys: KeyModel[]}>( - // `SELECT array_agg(key) AS keys FROM ${this.schema}.${this.table};` - // ).then(results => res(results?.rows?.[0].keys) - // ).catch(e => rej(e)) - // }) - // } - - - - // async getLength(): Promise { - // return new Promise((res, rej) => { - // this.pgPool.query<{count: string}>( - // `SELECT count(*) FROM ${this.schema}.${this.table}` - // ).then(results => res(parseInt(results?.rows?.[0].count, 10)) - // ).catch(e => rej(e)) - // }) - - // } - - /** - * Takes QueryStream, checks out a pg.Client and returns the QueryStream. - * It releases the checked out client when stream ends. - * @param querystream - */ - private async manageQueryStream(querystream: QueryStream) { - const client = await this.pgPool.connect(); - const stream = client.query(querystream); - stream.on('end', () => client.release()); - return stream; - } - - createInsertStatement() { - return `INSERT INTO ${this.schema}.${this.table} (${this.keyCols}, key, val) VALUES ( - $1, - $2, - ${this.keyDefs.map((k, i) => `$${i + 3}`)} - ) - ON CONFLICT (key) DO UPDATE SET val = EXCLUDED.val;` - } - - createKeyJsonObjSql() { - return `json_build_object(${this.keyDefs.map(k => `'${k.name}', "${k.name}"`).join(',')})` - } - -} diff --git a/server/src/warehouse/base/classes/DataService.ts b/server/src/warehouse/base/classes/DataService.ts deleted file mode 100644 index df3437708..000000000 --- a/server/src/warehouse/base/classes/DataService.ts +++ /dev/null @@ -1,115 +0,0 @@ -import {values} from 'ramda'; -import {Subject} from 'rxjs'; -import {LAST_UPDATE_DONE_SUFFIX, Warehouse} from '../../Warehouse'; -import {AggregatedDataService} from './AggregatedDataService'; -import {DataIndexPostgres} from './DataIndexPostgres'; -import {DependencyIndex} from './DependencyIndex'; - - - -export abstract class DataService{ - - abstract index: DataIndexPostgres - abstract clearAll(): Promise - - afterChange$ = new Subject() - - // array of dependency indexes where this data service is provider - // eslint-disable-next-line @typescript-eslint/no-explicit-any - isProviderOf: DependencyIndex[] = [] - - // eslint-disable-next-line @typescript-eslint/no-explicit-any - isCreatorOf: AggregatedDataService[] = [] - - - constructor(public wh: Warehouse) { - - this.afterChange$ - // .pipe(throttleTime(10)) TODO: Test if this helps! - .subscribe(_ => { - // do not propagate updates, if warehouse is initalizing primary data services - if (wh.preventPropagation === false) { - this.propagateUpdates().catch(e => console.log(e)); - } - }) - - } - - - private async propagateUpdates() { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const toInform: {[key: string]: AggregatedDataService;} = {}; - - // add receiver DataServices - for (const depIdx of this.isProviderOf) { - toInform[depIdx.receiverDS.constructor.name] = depIdx.receiverDS; - } - // add created DataServices - for (const createdDs of this.isCreatorOf) { - toInform[createdDs.constructor.name] = createdDs; - } - const ds = values(toInform) - if (ds.length) { - const wh = ds[0].wh - const currentTime = await wh.whPgNow() - - // useful for debugging - // for (const d of ds) { - // if (this.constructor.name === 'REntityService'){ - // if( d.constructor.name === 'REntityLabelService') { - // console.log(`- ${this.constructor.name}.propagateUpdates() --> ${d.constructor.name}, currentTime: ${currentTime}`) - // } - // } - // } - - - // propagate updates - const updates = ds.map(d => d.doUpdate(currentTime)); - - // wait until all aggregated DS have handled updates - await Promise.all(updates) - - // useful for debugging - // if (this.constructor.name === 'PEntityLabelService') { - // console.log(`---> cleanup items marked as deleted bevore or eq: ${currentTime}`) - // } - - // cleanup items marked as deleted - this.index.removeFromIdxWhereDeletedBefore(currentTime) - .catch(e => console.log(e)) - } - } - - /** - * Adds dep to this.isProviderOf with the effect that this DataService acts - * as the provider of data for dep. This is useful to inform the receiver - * of the registered dep to perform updates, when the index of this - * DataService is modified (put / del). - * @param dep - */ - registerProviderOf(dep: DependencyIndex) { - this.isProviderOf.push(dep) - } - - /** - * Adds AggregatedDataService to the array of dataservices that should - * have one aggregated item for each item in this DataService. - * @param createdDS - */ - // eslint-disable-next-line @typescript-eslint/no-explicit-any - registerIsCreatorOf(createdDS: AggregatedDataService) { - this.isCreatorOf.push(createdDS) - } - - - async setLastUpdateDone(date: Date) { - await this.wh.metaTimestamps.addToIdx(this.constructor.name + LAST_UPDATE_DONE_SUFFIX, {tmsp: date.toISOString()}); - } - async getLastUpdateBegin(): Promise { - const val = await this.wh.metaTimestamps.getFromIdx(this.constructor.name + LAST_UPDATE_DONE_SUFFIX); - const isoDate = val?.tmsp; - return isoDate ? new Date(isoDate) : undefined - } - - -} diff --git a/server/src/warehouse/base/classes/DataServiceBundle.ts b/server/src/warehouse/base/classes/DataServiceBundle.ts deleted file mode 100644 index 3cda78c7a..000000000 --- a/server/src/warehouse/base/classes/DataServiceBundle.ts +++ /dev/null @@ -1,9 +0,0 @@ -export abstract class DataServiceBundle { - registered: DS[] = []; - - - registerDataService(dep: M) { - this.registered.push(dep); - return dep; - } -} diff --git a/server/src/warehouse/base/classes/Dependencies.ts b/server/src/warehouse/base/classes/Dependencies.ts deleted file mode 100644 index 32e92625f..000000000 --- a/server/src/warehouse/base/classes/Dependencies.ts +++ /dev/null @@ -1,21 +0,0 @@ -import {ClearAll} from './ClearAll'; -import {DependencyIndex} from './DependencyIndex'; - -export abstract class Dependencies implements ClearAll { - - // eslint-disable-next-line @typescript-eslint/no-explicit-any - registered: DependencyIndex[] = []; - - async clearAll() { - await Promise.all(this.registered.map(x => x.clearAll())); - } - - // eslint-disable-next-line @typescript-eslint/no-explicit-any - registerDepIdx>(dep: M) { - this.registered.push(dep); - return dep; - } - - async initIdx(): Promise { }; - -} diff --git a/server/src/warehouse/base/classes/DependencyDataServicesBase.ts b/server/src/warehouse/base/classes/DependencyDataServicesBase.ts deleted file mode 100644 index dcd5567ee..000000000 --- a/server/src/warehouse/base/classes/DependencyDataServicesBase.ts +++ /dev/null @@ -1,24 +0,0 @@ -import {combineLatest, Observable} from 'rxjs'; -import {mapTo} from 'rxjs/operators'; -import {DataServiceBundle} from './DataServiceBundle'; -import {Dependencies} from './Dependencies'; - -export abstract class DependencyDataServicesBase extends DataServiceBundle { - - - ready$: Observable - - constructor(...aggDs: Dependencies[]) { - super() - - this.registered = aggDs; - const readies$ = [] - for (const reg1 of this.registered) { - for (const reg2 of reg1.registered) { - readies$.push(reg2.ready$) - } - } - this.ready$ = combineLatest(readies$).pipe(mapTo(true)) - } - -} diff --git a/server/src/warehouse/base/classes/DependencyIndex.ts b/server/src/warehouse/base/classes/DependencyIndex.ts deleted file mode 100644 index 1c50fce47..000000000 --- a/server/src/warehouse/base/classes/DependencyIndex.ts +++ /dev/null @@ -1,152 +0,0 @@ -import {Pool} from 'pg'; -import {values} from 'ramda'; -import {combineLatest, ReplaySubject} from 'rxjs'; -import {first} from 'rxjs/operators'; -import {Warehouse} from '../../Warehouse'; -import {KeyDefinition} from '../interfaces/KeyDefinition'; -import {AggregatedDataService} from './AggregatedDataService'; -import {ClearAll} from './ClearAll'; -import {DataService} from './DataService'; - -interface Cache { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - [key: string]: {receiverKeys: any[], providerKeys: any[]} -} - -export class DependencyIndex implements ClearAll { - - schema: string; - table: string; - schemaTable: string; - ready$ = new ReplaySubject() - providerKeyCols: string; // e.g. '"fkProject","pkEntity"' - receiverKeyCols: string; // e.g. '"fkProject","pkEntity"' - providerKeyDefs: KeyDefinition[]; - receiverKeyDefs: KeyDefinition[]; - keyCols: string; // e.g. '"r_fkProject","r_pkEntity", "p_fkProject","p_pkEntity"' - pgPool: Pool - - - private cache: Cache = {}; - - constructor( - public wh: Warehouse, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - public receiverDS: AggregatedDataService, - public providerDS: DataService - ) { - this.pgPool = wh.whPgPool; - providerDS.registerProviderOf(this) - receiverDS.registerReceiverOf(this) - this.schema = wh.schemaName; - this.table = receiverDS.index.table + '__on__' + providerDS.index.table; - if (this.table.length > 63) throw new Error(`pg table has ${this.table.length} chars (max: 63): ${this.schemaTable}`); - this.schemaTable = `${this.schema}.${this.table}` - this.providerKeyDefs = this.providerDS.index.keyDefs.map(k => ({...k, name: 'p_' + k.name})); - this.receiverKeyDefs = this.receiverDS.index.keyDefs.map(k => ({...k, name: 'r_' + k.name})); - this.providerKeyCols = this.providerKeyDefs.map(k => `"${k.name}"`).join(',') - this.receiverKeyCols = this.receiverKeyDefs.map(k => `"${k.name}"`).join(',') - this.keyCols = `${this.providerKeyCols},${this.receiverKeyCols}` - combineLatest( - wh.gvPgListenerConnected$, - wh.createSchema$ - ).pipe(first()).subscribe(([client]) => { - this.setupTable() - .then(() => { - this.ready$.next(true) - }) - .catch((e) => this.err(e)) - }) - } - - - private async setupTable() { - await this.pgPool.query(` - CREATE TABLE IF NOT EXISTS ${this.schemaTable} ( - -- provider key cols - ${this.providerKeyDefs.map(k => `"${k.name}" ${k.type}`).join(',')}, - -- receiver key cols - ${this.receiverKeyDefs.map(k => `"${k.name}" ${k.type}`).join(',')}, - val jsonb, - tmsp_last_aggregation timestamp with time zone, - CONSTRAINT ${this.table}_keys_uniq UNIQUE (${this.providerKeyCols},${this.receiverKeyCols}) - )`).catch((e) => { - console.log(`Error during CREATE TABLE: ${this.schemaTable}:`, e) - }) - - const indexedCols = [ - ...this.providerKeyDefs.map(k => k.name), - ...this.receiverKeyDefs.map(k => k.name), - 'tmsp_last_aggregation' - ] - for (const indexCol of indexedCols) { - const indexName= `${indexCol}_${this.table}` - - await this.pgPool.query(` - CREATE INDEX IF NOT EXISTS ${indexName} - ON ${this.schemaTable}("${indexCol}"); - `).catch((e) => { - console.log(`Error during CREATE INDEX: ${this.schemaTable}(${indexCol}):`, e) - }) - } - - } - - - cacheNewDependencies(receiverKey: ReceiverKeyModel, providerKey: ProviderKeyModel) { - const prov = this.providerDS.index.getKeyModelValues(providerKey) - const rec = this.receiverDS.index.getKeyModelValues(receiverKey) - this.cache[JSON.stringify([...prov, ...rec])] = ({receiverKeys: rec, providerKeys: prov}) - } - // eslint-disable-next-line @typescript-eslint/no-explicit-any - getSqlForStoringCache(tmsp: string, params: any[]): string | undefined { - const cacheVals = values(this.cache) - if (cacheVals.length === 0) return - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const addParam = (val: any) => { - params.push(val) - return '$' + params.length - } - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const addParams = (vals: any[]) => { - return vals.map((val) => addParam(val)).join(','); - } - const sql = ` - INSERT INTO - ${this.schemaTable} - (${this.keyCols}, tmsp_last_aggregation) - VALUES - ${cacheVals.map(v => `( - ${addParams(v.providerKeys)}, - ${addParams(v.receiverKeys)}, - ${addParam(tmsp)} - )`).join(',')} - ON CONFLICT (${this.keyCols}) - DO UPDATE - SET tmsp_last_aggregation = EXCLUDED.tmsp_last_aggregation - ` - this.clearCache() - return sql - } - - clearCache() { - this.cache = {} - } - - async clearAll(): Promise { - - await this.pgPool.query(` - DELETE FROM ${this.schemaTable}; - `).catch((e) => { - console.log(`Error during DELETE INDEX: ${this.schemaTable}:`, e) - }) - } - - - // eslint-disable-next-line @typescript-eslint/no-explicit-any - err(e: any) { - console.log(`Error during setup of ${this.schemaTable}:`, e) - } - -} - diff --git a/server/src/warehouse/base/classes/IndexDB.ts b/server/src/warehouse/base/classes/IndexDB.ts deleted file mode 100644 index f6bccedd8..000000000 --- a/server/src/warehouse/base/classes/IndexDB.ts +++ /dev/null @@ -1,28 +0,0 @@ - -/** - * Abstract base class for indexes - * - * Derived classes are responsible for the handling of an index - * - * Methods - * - add item - * - remove item - * - * - get item and save dependency of requester - * - * - add update request - */ - -import {Warehouse} from '../../Warehouse'; -import {IndexPostgres} from './IndexPostgres'; - -export abstract class IndexDB extends IndexPostgres { - constructor( - name = 'default', - wh: Warehouse - ) { - super(name, wh) - } -} - - diff --git a/server/src/warehouse/base/classes/IndexDBGeneric.ts b/server/src/warehouse/base/classes/IndexDBGeneric.ts deleted file mode 100644 index 00b0a9a00..000000000 --- a/server/src/warehouse/base/classes/IndexDBGeneric.ts +++ /dev/null @@ -1,13 +0,0 @@ -import {IndexDB} from './IndexDB'; -import {StringToKeyModel} from '../interfaces/StringToKeyModel'; -import {KeyModelToString} from '../interfaces/KeyModelToString'; -import {Warehouse} from '../../Warehouse'; - -export class IndexDBGeneric extends IndexDB { - constructor( - public keyToString: KeyModelToString, - public stringToKey: StringToKeyModel, - name: string, - wh: Warehouse - ) {super(name, wh);} -} diff --git a/server/src/warehouse/base/classes/IndexLeveldb.ts b/server/src/warehouse/base/classes/IndexLeveldb.ts deleted file mode 100644 index c593d95a9..000000000 --- a/server/src/warehouse/base/classes/IndexLeveldb.ts +++ /dev/null @@ -1,229 +0,0 @@ -import {LevelUp} from 'levelup'; -import {ReplaySubject} from 'rxjs'; -import subleveldown from 'subleveldown'; -import {Warehouse} from '../../Warehouse'; -import {Index} from '../interfaces/Index'; -import {Logger} from './Logger'; -// Temporary implementation as leveldb key-value store - -// export abstract class IndexLeveldb implements Index { - - -// static sublevels = 0; -// indexName = '/' + IndexLeveldb.sublevels++ -// private db: LevelUp; -// ready$ = new ReplaySubject() -// constructor( -// name = 'default', -// wh: Warehouse -// ) { -// this.db = subleveldown(wh.leveldb, this.indexName, {valueEncoding: 'json'}); -// this.ready$.next(true) -// // console.log(this.indexName, name) -// } - -// async addToIdx(keyModel: KeyModel, val: ValueModel) { -// const key = this.keyToString(keyModel); -// try { -// await this.db.put(key, val); -// } -// catch (error) { -// Logger.err(this.constructor.name, error); -// } -// // this.idx[this.keyToString(key)] = val -// } -// async removeFromIdx(keyModel: KeyModel) { -// const key = this.keyToString(keyModel); -// try { -// await this.db.del(key); -// } -// catch (error) { -// Logger.err(this.constructor.name, error); -// } -// } - -// /** -// * Returns value of key. If key not exists, returns undefined. -// * @param key -// */ - -// async getFromIdx(keyModel: KeyModel): Promise { - -// const key = this.keyToString(keyModel); -// let val: (ValueModel | undefined) = undefined; -// try { -// val = await this.db.get(key); -// return val; -// } -// catch (error) { -// if (!error.notFound) { -// Logger.err(this.constructor.name, error); -// } -// return val; -// } - -// } - - - - -// async forEachKey(cb: (key: KeyModel) => Promise) { -// const stream = this.db.createKeyStream(); -// return handleAsyncStream(stream, (item) => cb(this.stringToKey(item))); - -// } - - -// async forEachKeyStartingWith(str: string, cb: (key: KeyModel) => Promise): Promise { -// if (!this.db.isOpen()) return; - -// const stream = this.db.createKeyStream(whereKeyStartsWith(str)); -// return handleAsyncStream(stream, (item) => cb(this.stringToKey(item))); -// } - - -// async forEachItemStartingWith(str: string, cb: (item: {key: KeyModel, value: ValueModel}) => Promise): Promise { -// const stream = this.db.createReadStream(whereKeyStartsWith(str)); -// return handleAsyncStream(stream, (item) => cb({ -// key: this.stringToKey(item.key), -// value: item.value -// })); -// } - -// async forEachValue(cb: (val: ValueModel) => void) { -// return new Promise((res, rej) => { -// this.db.createValueStream() -// .on('data', (val) => { -// cb(val); -// }) -// .on('error', function (err) { -// rej(err); -// }) -// .on('close', function () { -// res(); -// }) -// .on('end', function () { -// res(); -// }); -// // return values(this.idx) -// }); -// } - - -// async clearIdx() { -// await this.db.clear(); -// } - - -// async keyExists(key: string): Promise { -// return new Promise((res, rej) => { - -// this.db.get(key, (err, value) => { -// if (!err) { -// res(true); -// } -// else { -// res(false); -// } -// }); -// }); -// } - - -// async getKeys(): Promise { -// const keys: KeyModel[] = []; -// await this.forEachKey(async (k) => keys.push(k)); -// return keys; -// } - - -// async getValues(): Promise { -// const vals: ValueModel[] = []; -// await this.forEachValue((v) => vals.push(v)); -// return vals; -// } - - -// async getLength(): Promise { -// let length = 0; -// return new Promise((res, rej) => { -// this.db.createKeyStream() -// .on('data', () => { -// length++; -// }) -// .on('error', function (err) { -// rej(err); -// }) -// .on('close', function () { -// res(length); -// }) -// .on('end', function () { -// res(length); -// }); -// }); - -// } - - -// abstract keyToString(key: KeyModel): string; -// abstract stringToKey(str: string): KeyModel; - -// } - - -export function handleAsyncStream(stream: NodeJS.ReadableStream, transform: (item: Item) => Promise) { - return new Promise((res, rej) => { - let streamFinished = false; - let awaitingCallback = false; - stream - .on('data', (item: Item) => { - - stream.pause(); - awaitingCallback = true; - transform(item). - then(() => { - awaitingCallback = false; - if (streamFinished) { - res(); - } - else { - stream.resume(); - } - }) - .catch(e => { - console.log(e); - }); - }) - .on('error', function (err: unknown) { - rej(err); - }) - .on('close', () => { - streamFinished = true; - if (!awaitingCallback) - res(); - }) - .on('end', function () { - streamFinished = true; - }); - }); -} - - -/** - * Returns an object for streaming keys that begin with the given string - * - * @param str beginning of the streamed keys - * - * Read more here: - * https://github.com/Level/levelup/issues/285#issuecomment-57205251 - * - * Read more about '!' and '~' here: - * https://medium.com/@kevinsimper/how-to-get-range-of-keys-in-leveldb-and-how-gt-and-lt-works-29a8f1e11782 - * @param str - */ -export function whereKeyStartsWith(str: string) { - return { - gte: str + '!', - lte: str + '~' - } -} diff --git a/server/src/warehouse/base/classes/IndexMemory.ts b/server/src/warehouse/base/classes/IndexMemory.ts deleted file mode 100644 index 9480f8dbf..000000000 --- a/server/src/warehouse/base/classes/IndexMemory.ts +++ /dev/null @@ -1,88 +0,0 @@ -import {values} from 'ramda'; -import {Index} from '../interfaces/Index'; -import {Warehouse} from '../../Warehouse'; -import {ReplaySubject} from 'rxjs'; -// Temporary implementation as in-memory key-value store - -export abstract class IndexMemory implements Index { - ready$ = new ReplaySubject() - - private idx: {[key: string]: ValueModel;} = {}; - constructor( - name = 'default', - wh: Warehouse - ) { - this.ready$.next(true) - - } - - async addToIdx(keyModel: KeyModel, val: ValueModel) { - this.idx[this.keyToString(keyModel)] = val; - } - async removeFromIdx(keyModel: KeyModel) { - delete this.idx[this.keyToString(keyModel)]; - } - - /** - * Returns value of key. If key not exists, returns undefined. - * @param key - */ - - async getFromIdx(keyModel: KeyModel): Promise { - const res = this.idx[this.keyToString(keyModel)]; - return res; - } - - - async forEachKey(cb: (keyModel: KeyModel) => Promise) { - for (const key of Object.keys(this.idx)) { - await cb(this.stringToKey(key)); - } - } - - async forEachKeyStartingWith(str: string, cb: (key: KeyModel) => Promise): Promise { - for (const key of Object.keys(this.idx)) { - if (key.startsWith(str)) await cb(this.stringToKey(key)); - } - } - - async forEachItemStartingWith(str: string, cb: (item: {key: KeyModel, value: ValueModel}) => Promise): Promise { - for (const key of Object.keys(this.idx)) { - if (key.startsWith(str)) await cb({key: this.stringToKey(key), value: this.idx[key]}); - } - } - async forEachValue(cb: (val: ValueModel) => void) { - return values(this.idx).forEach(val => { - cb(val); - }); - } - - - - async clearIdx() { - this.idx = {}; - } - - - async keyExists(key: string): Promise { - return !!this.idx[key]; - } - - - async getKeys(): Promise { - return Object.keys(this.idx).map(k => this.stringToKey(k)); - } - - - async getValues(): Promise { - return values(this.idx); - } - async getLength(): Promise { - return Object.keys(this.idx).length; - } - - - abstract keyToString(key: KeyModel): string; - abstract stringToKey(str: string): KeyModel; - -} diff --git a/server/src/warehouse/base/classes/IndexPostgres.ts b/server/src/warehouse/base/classes/IndexPostgres.ts deleted file mode 100644 index 9fafba1f9..000000000 --- a/server/src/warehouse/base/classes/IndexPostgres.ts +++ /dev/null @@ -1,218 +0,0 @@ -import {Pool} from 'pg'; -import QueryStream from 'pg-query-stream'; -import {combineLatest, ReplaySubject} from 'rxjs'; -import {first} from 'rxjs/operators'; -import {Warehouse} from '../../Warehouse'; -import {Index} from '../interfaces/Index'; -import {handleAsyncStream} from './IndexLeveldb'; -// Temporary implementation as leveldb key-value store - -export abstract class IndexPostgres implements Index { - - - static indexes = 0; - indexName = '/' + IndexPostgres.indexes++ - pgPool: Pool - - schema: string; - table: string; - schemaTable: string; - ready$ = new ReplaySubject() - - constructor( - name = 'default', - wh: Warehouse - ) { - this.schema = wh.schemaName; - this.table = name; - this.schemaTable = `${this.schema}.${this.table}` - this.pgPool = wh.whPgPool; - combineLatest( - wh.gvPgListenerConnected$, - wh.createSchema$ - ).pipe(first()).subscribe(([client]) => { - this.pgPool.query(`CREATE SCHEMA IF NOT EXISTS ${this.schema}`) - .then(_ => { - this.setupTable() - .then(() => { - this.ready$.next(true) - }) - .catch((e) => this.err(e)) - }).catch((e) => { - console.log(`Error during CREATE SCHEMA: ${this.indexName}:`, e) - }) - - }) - } - - private async setupTable() { - await this.pgPool.query(` - CREATE TABLE IF NOT EXISTS ${this.schema}.${this.table} ( - key varchar NOT NULL UNIQUE, - val json - )`).catch((e) => { - console.log(`Error during CREATE TABLE: ${this.indexName}:`, e) - }) - - await this.pgPool.query(` - CREATE INDEX IF NOT EXISTS ${this.schema}_${this.table}_key_idx - ON ${this.schema}.${this.table}(key); - `).catch((e) => { - console.log(`Error during REATE INDEX: ${this.indexName}:`, e) - }) - } - - forEachValue(cb: (val: ValueModel) => void): Promise { - throw new Error('Method not implemented.'); - } - getValues(): Promise { - throw new Error('Method not implemented.'); - } - - // eslint-disable-next-line @typescript-eslint/no-explicit-any - err(e: any) { - console.log(`Error during setup of ${this.indexName}:`, e) - } - - async addToIdx(keyModel: KeyModel, val: ValueModel): Promise { - const key = this.keyToString(keyModel); - - return new Promise((res, rej) => { - this.pgPool.query(` - INSERT INTO ${this.schema}.${this.table} (key, val) VALUES ( - $1, - ${typeof val === 'string' ? 'to_json($2::text)' : '$2'} - ) - ON CONFLICT (key) DO UPDATE SET val = EXCLUDED.val; - `, [ - key, - Array.isArray(val) ? JSON.stringify(val) : val - ]).then(_ => res()) - .catch(e => { - console.log('error when inserting: ', {key, val}) - console.log(e) - }); - }) - // await this.pgClient.query(` - // INSERT INTO ${this.schema}.${this.table} (key, val) VALUES ( - // $1, - // ${typeof val === 'string' ? 'to_json($2::text)' : '$2'} - // ) - // ON CONFLICT (key) DO UPDATE SET val = EXCLUDED.val; - // `, [key, val]); - - // this.idx[this.keyToString(key)] = val - } - async removeFromIdx(keyModel: KeyModel): Promise { - const key = this.keyToString(keyModel); - await this.pgPool.query(` - DELETE FROM ${this.schema}.${this.table} WHERE key = $1; - `, [key]); - } - - /** - * Returns value of key. If key not exists, returns undefined. - * @param key - */ - - async getFromIdx(keyModel: KeyModel): Promise { - const key = this.keyToString(keyModel); - - return new Promise((res, rej) => { - this.pgPool.query<{val: ValueModel}>( - `SELECT val FROM ${this.schema}.${this.table} WHERE key = $1;`, - [key] - ).then(results => res(results?.rows?.[0]?.val) - ).catch(e => rej(e)) - }) - } - - - async forEachKey(cb: (key: KeyModel) => Promise) { - const querystream = new QueryStream( - `SELECT key FROM ${this.schema}.${this.table}` - ) - const stream = await this.manageQueryStream(querystream); - return handleAsyncStream(stream, (item) => cb(this.stringToKey(item.key))); - - } - - async forEachKeyStartingWith(str: string, cb: (key: KeyModel) => Promise): Promise { - const querystream = new QueryStream( - `SELECT key FROM ${this.schema}.${this.table} WHERE key LIKE $1`, - [str + '%'] - ) - const stream = await this.manageQueryStream(querystream); - return handleAsyncStream(stream, (item) => cb(this.stringToKey(item.key))); - } - - async forEachItemStartingWith(str: string, cb: (item: {key: KeyModel, value: ValueModel}) => Promise): Promise { - const querystream = new QueryStream( - `SELECT key, val as value FROM ${this.schema}.${this.table} WHERE key LIKE $1`, - [str + '%'] - ) - const stream = await this.manageQueryStream(querystream); - return handleAsyncStream(stream, (item) => cb({ - key: this.stringToKey(item.key), - value: item.value - })); - } - - - async clearIdx() { - await this.pgPool.query(`TRUNCATE TABLE ${this.schema}.${this.table};`) - } - - - async keyExists(key: string): Promise { - return new Promise((res, rej) => { - - this.pgPool.query<{exists: boolean}>( - `SELECT EXISTS (SELECT key FROM ${this.schema}.${this.table} WHERE key = $1);`, - [key] - ) - .then(results => res(results?.rows?.[0].exists) - ).catch(e => rej(e)) - - }); - } - - - async getKeys(): Promise { - return new Promise((res, rej) => { - this.pgPool.query<{keys: KeyModel[]}>( - `SELECT array_agg(key) AS keys FROM ${this.schema}.${this.table};` - ).then(results => res(results?.rows?.[0].keys) - ).catch(e => rej(e)) - }) - } - - - - async getLength(): Promise { - return new Promise((res, rej) => { - this.pgPool.query<{count: string}>( - `SELECT count(*) FROM ${this.schema}.${this.table}` - ).then(results => res(parseInt(results?.rows?.[0].count, 10)) - ).catch(e => rej(e)) - }) - - } - - /** - * Takes QueryStream, checks out a pg.Client and returns the QueryStream. - * It releases the checked out client when stream ends. - * @param querystream - */ - private async manageQueryStream(querystream: QueryStream) { - const client = await this.pgPool.connect(); - const stream = client.query(querystream); - stream.on('end', () => client.release()); - return stream; - } - - - abstract keyToString(key: KeyModel): string; - abstract stringToKey(str: string): KeyModel; - -} diff --git a/server/src/warehouse/base/classes/Logger.ts b/server/src/warehouse/base/classes/Logger.ts deleted file mode 100644 index 2a62410f5..000000000 --- a/server/src/warehouse/base/classes/Logger.ts +++ /dev/null @@ -1,91 +0,0 @@ -import prettyMilliseconds from 'pretty-ms'; -import * as readline from 'readline' - -export class Logger { - static err(slug: string, msg: string, intendation = 1) { - // if (process.env.LOGS === 'OFF') return - process.stdout.write(Logger.slug(slug) + Logger.ind(intendation) + 'ERR: ' + msg); - Logger.endLine() - } - static msg(slug: string, msg: string, intendation = 1) { - if (process.env.LOGS === 'OFF') return - - Logger.log(Logger.slug(slug) + Logger.ind(intendation) + msg); - } - - static getTime() { - return new Date().getTime(); - } - static slug(msg: string, template = ' ') { - const length = template.length; - const m = msg.substring(0, length) - return `${(template + m).slice(-length)} | `; - } - /** - * print msg and return time - * @param msg - */ - static start(slug: string, msg: string, intendation = 1) { - if (process.env.LOGS === 'OFF') return new Date().getTime(); - - const ind = new Array(intendation + 1).join(' '); - Logger.log(`${Logger.slug(slug)}${ind}----> ${msg}`); - return new Date().getTime(); - } - static itTook(slug: string, beginMs: number, msg: string, intendation = 1, endLine = true) { - if (process.env.LOGS === 'OFF') return 0 - - const ind = new Array(intendation + 1).join(' '); - const ram = Math.round(process.memoryUsage().heapUsed / 1024 / 1024 * 100) / 100 - const diff = new Date().getTime() - beginMs; - Logger.log(`${Logger.slug(slug)}${ind} > it took \u{1b}[33m${Logger.passedMs(beginMs)}\u{1b}[0m ${msg} \u{1b}[32m (${ram} mb) \u{1b}[0m`, endLine); - return diff - } - static done(slug: string, beginMs: number, intendation = 1) { - if (process.env.LOGS === 'OFF') return - - const ind = new Array(intendation + 1).join(' '); - Logger.log(`${Logger.slug(slug)}${ind} > done in ${Logger.passedMs(beginMs)}`); - } - - static diffMs(ms: number) { - return new Date().getTime() - ms - } - static passedMs(ms: number) { - return prettyMilliseconds(Logger.diffMs(ms)) - } - - static ind(x: number) { - return new Array(x + 2).join(' '); - } - - static log(msg = '', endLine = true) { - if (process.env.LOGS === 'OFF') return - process.stdout.write(msg); - if (endLine) Logger.endLine() - } - static endLine() { - process.stdout.write('\n'); - } - - // static printMsg(msg: string, i^tendation = 1) { - // Logger.print(Logger.ind(intendation) + msg); - // } - // static printClearEntireLine() { - // clearLine(process.stdout, 1) - // cursorTo(process.stdout, 0) - // } - - // static print(msg: any) { - // if (process.env.LOGS === 'OFF') return - // process.stdout.write(msg); - - - // } - - static resetLine() { - readline.clearLine(process.stdout, 0) - readline.cursorTo(process.stdout, 0) - } - -} diff --git a/server/src/warehouse/base/classes/PgDataReplicator.ts b/server/src/warehouse/base/classes/PgDataReplicator.ts deleted file mode 100644 index e588ff60f..000000000 --- a/server/src/warehouse/base/classes/PgDataReplicator.ts +++ /dev/null @@ -1,246 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import {PoolClient, QueryArrayResult} from 'pg'; -import {Logger} from './Logger'; -import {brkOnErr, pgLogOnErr} from '../../../utils/helpers'; - -export interface PgTable { - // pool client connected to the database of the table - client: PoolClient, - - // provide full name. Pattern: 'schema.table', e.g. 'war.entity_preview' - table: string -} -export type DataReplicatorSqlFn = (insertClause: string, fromClause: string) => string - -export class PgDataReplicator { - - /** - * create a data replicator - * the getInsertState - * @param source PgTable object containing information source table - * @param target PgTable object containing information source table - * @param columns Optional array of col names of the source table to read from - * @param getInsertStatement Optional function that creates the replication sql - */ - constructor( - private source: PgTable, - private target: PgTable, - private columns?: string[], - private getInsertStatement: DataReplicatorSqlFn = (insertClause: string, fromClause: string) => `${insertClause} ${fromClause}` - - ) { } - /** - * Loops over source table for batches of given batchSize and - * executes statement given by this.getInsertStatement - * @param batchSize max number of rows considered by one batch. - * @returns statitistics about batches and time (miliseconds) used per batch - */ - async replicateTable(batchSize = 10000) { - const res = await brkOnErr(this.source.client.query<{count: number}>(`SELECT count(*):: integer From ${this.source.table} `)) - const size = res.rows[0].count; - const limit = batchSize; - const logPrefix = `${this.constructor.name} ${this.source.table}` - - const stats: {items: number, duration: number, rows: M[]}[] = [] - - for (let offset = 0; offset < size; offset += limit) { - const items = limit > size ? size % limit : limit; - const last = offset + items; - - const logString = `replicate ${offset} - ${last} of ${size} items (${(offset / limit) + 1} /${Math.floor(size / limit) + 1} batches)` - const t0 = Logger.start(logPrefix, `${logString} `, 0) - - const data = await this.readFromTable(limit, offset) - const {sql, params} = this.convertReadResults(data) - - const rows = (await this.writeToTable(sql, params)).rows - const duration = Logger.itTook(logPrefix, t0, `to ${logString} `, 0) - stats.push({items, duration, rows}) - } - return stats - } - - /** - * Loops over source table for batches of given batchSize and - * executes statement given by this.getInsertStatement - * @param batchSize max number of rows considered by one batch. - * @returns statitistics about batches and time (miliseconds) used per batch - */ - async replicateBatch( - date: Date, - countSql: string, - batchSql: (limit: number, offset: number) => string, - batchSize = 10000 - ) { - const res = await brkOnErr(this.source.client.query<{count: number}>(countSql, [date])) - const size = res.rows[0].count; - const limit = batchSize; - const logPrefix = `${this.constructor.name} ${this.source.table}` - - const stats: {items: number, duration: number, rows: M[]}[] = [] - - for (let offset = 0; offset < size; offset += limit) { - const items = limit > size ? size % limit : limit; - const last = offset + items; - - const logString = `replicate ${offset} - ${last} of ${size} items (${(offset / limit) + 1} /${Math.floor(size / limit) + 1} batches)` - const t0 = Logger.start(logPrefix, `${logString} `, 0) - - const data = await this.source.client.query( - { - text: batchSql(limit, offset), - rowMode: 'array', - values: [date] - }, - ) - const {sql, params} = this.convertReadResults(data) - - const rows = (await this.writeToTable(sql, params)).rows - const duration = Logger.itTook(logPrefix, t0, `to ${logString} `, 0) - stats.push({items, duration, rows}) - } - return stats - } - - private async readFromTable(limit: number, offset: number) { - const cols = this.columns ? this.columns.join(',') : '*' - return this.source.client.query({ - text: `select ${cols} from ${this.source.table} LIMIT ${limit} OFFSET ${offset}`, - rowMode: 'array', - }) - } - - private convertReadResults(data: QueryArrayResult): {sql: string, params: any[][]} { - const params: any[][] = [] - const placeholders: string[] = []; - const colNames: string[] = [] - data.fields.forEach((field, i) => { - const placeholder = `$${i + 1}::${this.getDataTypeName(field.dataTypeID)}[]`; - placeholders.push(placeholder) - const colName = `"${field.name}"`; - colNames.push(colName) - params.push([]) - for (const row of data.rows) { - params[i].push(row[i]) - } - }) - - const insertClause = `INSERT INTO ${this.target.table} (${colNames.join(',')})` - const fromClause = `(SELECT * FROM UNNEST (${placeholders.join(',')}) t(${colNames.join(',')}))` - const sql = this.getInsertStatement(insertClause, fromClause) - return {sql, params} - } - - private async writeToTable(sql: string, params: any[]) { - return pgLogOnErr((s, p) => this.target.client.query(s, p), sql, params) - } - - private getDataTypeName(id: number): string { - const name = dataTypeIdLookup[id]; - if (!name) throw new Error(`Datatype with ${id} not found.`); - return name - } - - - -} - - -const dataTypeIdLookup: {[id: number]: string} = { - 16: 'bool', - 17: 'bytea', - 18: 'char', - 19: 'name', - 20: 'int8', - 21: 'int2', - 22: 'int2vector', - 23: 'int4', - 24: 'regproc', - 25: 'text', - 26: 'oid', - 27: 'tid', - 28: 'xid', - 29: 'cid', - 30: 'oidvector', - 71: 'pg_type', - 75: 'pg_attribute', - 81: 'pg_proc', - 83: 'pg_class', - 114: 'json', - 142: 'xml', - 194: 'pg_node_tree', - 3361: 'pg_ndistinct', - 3402: 'pg_dependencies', - 5017: 'pg_mcv_list', - 32: 'pg_ddl_command', - 600: 'point', - 601: 'lseg', - 602: 'path', - 603: 'box', - 604: 'polygon', - 628: 'line', - 700: 'float4', - 701: 'float8', - 705: 'unknown', - 718: 'circle', - 790: 'money', - 829: 'macaddr', - 869: 'inet', - 650: 'cidr', - 774: 'macaddr8', - 1033: 'aclitem', - 1042: 'bpchar', - 1043: 'varchar', - 1082: 'date', - 1083: 'time', - 1114: 'timestamp', - 1184: 'timestamptz', - 1186: 'interval', - 1266: 'timetz', - 1560: 'bit', - 1562: 'varbit', - 1700: 'numeric', - 1790: 'refcursor', - 2202: 'regprocedure', - 2203: 'regoper', - 2204: 'regoperator', - 2205: 'regclass', - 2206: 'regtype', - 4096: 'regrole', - 4089: 'regnamespace', - 2950: 'uuid', - 3220: 'pg_lsn', - 3614: 'tsvector', - 3642: 'gtsvector', - 3615: 'tsquery', - 3734: 'regconfig', - 3769: 'regdictionary', - 3802: 'jsonb', - 4072: 'jsonpath', - 2970: 'txid_snapshot', - 3904: 'int4range', - 3906: 'numrange', - 3908: 'tsrange', - 3910: 'tstzrange', - 3912: 'daterange', - 3926: 'int8range', - 2249: 'record', - 2287: '_record', - 2275: 'cstring', - 2276: 'any', - 2277: 'anyarray', - 2278: 'void', - 2279: 'trigger', - 3838: 'event_trigger', - 2280: 'language_handler', - 2281: 'internal', - 2282: 'opaque', - 2283: 'anyelement', - 2776: 'anynonarray', - 3500: 'anyenum', - 3115: 'fdw_handler', - 325: 'index_am_handler', - 3310: 'tsm_handler', - 269: 'table_am_handler', - 3831: 'anyrange', -} diff --git a/server/src/warehouse/base/classes/PrimaryDataService.ts b/server/src/warehouse/base/classes/PrimaryDataService.ts deleted file mode 100644 index f86c5a785..000000000 --- a/server/src/warehouse/base/classes/PrimaryDataService.ts +++ /dev/null @@ -1,326 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import { PoolClient } from 'pg'; -import { sum } from 'ramda'; -import { BehaviorSubject, Subject } from 'rxjs'; -import { CHANGES_CONSIDERED_UNTIL_SUFFIX, Warehouse } from '../../Warehouse'; -import { KeyDefinition } from '../interfaces/KeyDefinition'; -import { ClearAll } from './ClearAll'; -import { DataIndexPostgres } from './DataIndexPostgres'; -import { DataService } from './DataService'; -import { Logger } from './Logger'; -import { DataReplicatorSqlFn, PgDataReplicator } from './PgDataReplicator'; - -export abstract class PrimaryDataService extends DataService implements ClearAll { - - // number of iterations before measurin time an memory - abstract measure: number; - - // True if sync() is running - syncing$ = new BehaviorSubject(false); - - // True if running sync() should restart right after finishing - restartSyncing = false; - - index: DataIndexPostgres - - // List of data replication requests that can be executed by calling executeQueries() - updateReplications: { - targetTable: string, - sqlFn: DataReplicatorSqlFn - }[] = [] - - replicationBatchSizeUpdates = 10000; // batch size of the replication of updates or inserts - replicationBatchSizeDeletes = 10000; // batch size of the replication of deletes - - constructor( - public wh: Warehouse, - private listenTo: string[], - private keyDefs: KeyDefinition[] - ) { - super(wh) - this.index = new DataIndexPostgres( - this.keyDefs, - 'prim_' + this.constructor.name.replace('Service', ''), - wh - ) - - } - - /** - * init idx - */ - async initIdx() { - await this.clearAll() - - await this.addPgListeners() - - const dbNow = await this.wh.gvPgPool.query('SELECT now() as now'); - - await this.sync(new Date(dbNow.rows?.[0]?.now)) - - } - - /** - * start listening - * @param date Will look for changes since date. - */ - async startAndSyncSince(lastUpdateBegin: Date) { - await this.addPgListeners() - - await this.setChangesConsideredUntil(lastUpdateBegin) - - await this.sync(lastUpdateBegin) - } - - - async catchUp() { - - const lastUpdateBegin = await this.getChangesConsideredUntil() - if (lastUpdateBegin) { - Logger.msg(this.constructor.name, `Catch up changes since ${lastUpdateBegin}`); - await this.startAndSyncSince(lastUpdateBegin) - } - else { - Logger.msg(this.constructor.name, `WARNING: no lastUpdateBegin date found for catchUp()!`) - await this.initIdx() - } - } - - async clearAll() { - // await Promise.all([this.index.clearIdx(), this.meta.clearIdx()]) - } - - /** - * Adds the postgres listener - * @param listenTo - */ - private async addPgListeners() { - const sync$ = new Subject() - sync$.pipe( - // skipWhileFirst(80) - ).subscribe(tmsp => { - this.sync(tmsp).catch(e => console.log(e)) - }) - for (const eventName of this.listenTo) { - await this.wh.registerDbListener( - eventName, - sync$, - this.constructor.name - ) - } - } - - /** - * - * @param tmsp the timestamp sent by the notification of the original table - */ - async sync(tmsp: Date): Promise { - let changes = 0 - - // If syncing is true, it sets restartSyncing to true and stops the function here - if (this.syncing$.value) { - this.restartSyncing = true - return changes; - } - - // Else sets syncing to true and restartSyncing to false - this.syncing$.next(true); - this.restartSyncing = false; - - // throttle sync process for 10 ms - // await new Promise((res, rej) => { setTimeout(() => { res() }, 10) }) - const changesConsideredUntil = await this.getChangesConsideredUntil() - const t1 = Logger.start(this.constructor.name, `manageUpdatesSince ${changesConsideredUntil}`, 1); - - const c1 = await this.wh.gvPgPool.connect() - const c2 = await this.wh.whPgPool.connect() - let hasError = false; - let updates = 0; - let deletes = 0; - - try { - - await c1.query('BEGIN') - await c2.query('BEGIN') - - // Look for updates since the date of lastUpdateBegin or 1970 - updates += await this.manageUpdatesSince(c1, c2, changesConsideredUntil) - - // Look for deletes if - // - there is a deleteSql and - // - this data service has ever been synced - if (changesConsideredUntil) { - const deleteSql = this.getDeletesSql(changesConsideredUntil) - if (deleteSql !== '') deletes += await this.manageDeletesSince(c1, c2, changesConsideredUntil, deleteSql) - } - await c1.query('COMMIT') - await c2.query('COMMIT') - - } catch (e) { - hasError = true - await c1.query('ROLLBACK') - await c2.query('ROLLBACK') - Logger.err(this.constructor.name, `ERROR in primary data service: ${e}`) - } finally { - c1.release() - c2.release() - Logger.msg(this.constructor.name, `pgPool client released`) - } - - - // set lastUpdateBegin to timestamp that was used to run this sync - // process. - if (!hasError) { - await this.setChangesConsideredUntil(tmsp) - } - - // await the calls produced above - - changes += updates + deletes; - - Logger.itTook(this.constructor.name, t1, `to manage ${updates} updates and ${deletes} deletes by ${this.constructor.name}`, 1); - - this.syncing$.next(false) - if (this.restartSyncing) { - changes += await this.sync(tmsp); - } - const now = await this.wh.whPgNowDate() - await this.setLastUpdateDone(now) - if (changes > 0) { - this.afterChange$.next() - } - - return changes - - } - - /** - * Executes SQL query to GvDB to get all items that have been inserted or updated since date. - * - * The type of a row in the query result must be InitItem. - * - * Since query result can be huge, results are streamed (using pg-query-stream). - * - * On each streamed item, the function calls putInitItem(item) - * - * @param date - */ - async manageUpdatesSince(pool1: PoolClient, pool2: PoolClient, date: Date = new Date(0)) { - - const t2 = Logger.start(this.constructor.name, `Execute update query ...`, 2); - - const tmpTable = `${this.constructor.name}_update_tmp` - const updateSql = this.getUpdatesSql(date) - - await pool1.query(`CREATE TEMP TABLE ${tmpTable} ON COMMIT DROP AS ${updateSql}`, [date]) - const stats = await new PgDataReplicator<{ count: number }>( - { client: pool1, table: tmpTable }, - { client: pool2, table: this.index.schemaTable }, - [this.index.keyCols, 'val'], - (insertClause, fromClause) => ` - WITH tw1 AS ( - ${insertClause} - ${fromClause} - ON CONFLICT (${this.index.keyCols}) DO UPDATE - SET val = EXCLUDED.val - WHERE ${this.index.schemaTable}.val <> EXCLUDED.val - RETURNING * - ) - SELECT count(*)::int FROM tw1 - ` - ).replicateTable(this.replicationBatchSizeUpdates) - const upserted = sum(stats.map(s => s.rows?.[0].count)) - - Logger.itTook(this.constructor.name, t2, `to update Primary Data Service with ${upserted} new lines`, 2); - - if (this.updateReplications.length > 0) { - const pool1b = await this.wh.gvPgPool.connect() - const replicationRequest = this.updateReplications.map(repl => { - return new PgDataReplicator<{ count: number }>( - { client: pool1, table: tmpTable }, - { client: pool1b, table: repl.targetTable }, - [this.index.keyCols, 'val'], - repl.sqlFn - ).replicateTable() - }) - await Promise.all(replicationRequest) - pool1b.release() - } - - return upserted - - } - - /** - * Executes SQL query to GvDB to get all items that have been deleted since date. - * - * The type of a row in the query result must be KeyModel. - * - * Since query result can be huge, results are streamed (using pg-query-stream). - * - * On each streamed key, the function calls delItem(key) - * @param date - */ - private async manageDeletesSince(pool1: PoolClient, pool2: PoolClient, date: Date, deleteSql: string) { - const t2 = Logger.start(this.constructor.name, `Start deletes query ...`, 2); - - const tmpTable = `${this.constructor.name}_delete_tmp` - - await pool1.query(`CREATE TEMP TABLE ${tmpTable} ON COMMIT DROP AS ${deleteSql}`, [date]) - const stats = await new PgDataReplicator<{ count: number }>( - { client: pool1, table: tmpTable }, - { client: pool2, table: this.index.schemaTable }, - [this.index.keyCols], - (insertClause, fromClause) => ` - WITH tw2 AS ( - UPDATE ${this.index.schemaTable} t1 - SET tmsp_deleted = now() - FROM (${fromClause}) t2 - WHERE - ${this.index.keyDefs.map(k => `t1."${k.name}" = t2."${k.name}"`).join(' AND ')} - RETURNING t1.* - ) - SELECT count(*)::int FROM tw2 - ` - ).replicateTable(this.replicationBatchSizeDeletes) - const deleted = sum(stats.map(s => s.rows?.[0].count)) - - - Logger.itTook(this.constructor.name, t2, `To mark items as deleted ...`, 2); - return deleted - - } - - // sql statement used to query updates for the index - abstract getUpdatesSql(tmsp: Date): string; - - // sql statement used to do anything with the update sql - get2ndUpdatesSql?(updateSqlAlias: string, tmsp: Date): string; - - // sql statement used to query deletes for the index - // if the warehouse does not need to consider deletets, set this to null - abstract getDeletesSql(tmsp: Date): string; - - // sql statement used to do anything with the delete sql - get2ndDeleteSql?(deleteSqlAlias: string, tmsp: Date): string; - - // The following 4 function are for: - // Storing date and time of the last time the function sync() was called, - // marking the begin of the sync() process, which can be considerably earlier - // than its end. It is null until sync() is called the first time. - - async setChangesConsideredUntil(date: Date) { - await this.wh.metaTimestamps.addToIdx(this.constructor.name + CHANGES_CONSIDERED_UNTIL_SUFFIX, { tmsp: date.toISOString() }); - } - async getChangesConsideredUntil(): Promise { - const val = await this.wh.metaTimestamps.getFromIdx(this.constructor.name + CHANGES_CONSIDERED_UNTIL_SUFFIX); - const isoDate = val?.tmsp; - return isoDate ? new Date(isoDate) : undefined - } - - - registerUpdateReplication(targetTable: string, sqlFn: DataReplicatorSqlFn) { - this.updateReplications.push({ targetTable, sqlFn }) - } - -} diff --git a/server/src/warehouse/base/classes/PrimaryDataServicesBase.ts b/server/src/warehouse/base/classes/PrimaryDataServicesBase.ts deleted file mode 100644 index 7b5d3b536..000000000 --- a/server/src/warehouse/base/classes/PrimaryDataServicesBase.ts +++ /dev/null @@ -1,34 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import {combineLatest, Observable} from 'rxjs'; -import {filter, mapTo} from 'rxjs/operators'; -import {DataServiceBundle} from './DataServiceBundle'; -import {PrimaryDataService} from './PrimaryDataService'; - -export abstract class PrimaryDataServicesBase extends DataServiceBundle> { - - - ready$: Observable - - constructor(...aggDs: PrimaryDataService[] - ) { - super() - this.registered = aggDs; - - this.ready$ = combineLatest( - this.registered.map(ds => ds.index.ready$.pipe(filter(r => r === true))) - ).pipe(mapTo(true)) - - } - - // async initAllIndexes() { - // for (const ds of this.registered) { - // await ds.initIdx() - // } - // } - - // async everythingInitialized(): Promise { - // const doneDates = await Promise.all(this.registered.map(ds => ds.getLastUpdateDone())) - // if (doneDates.includes(undefined)) return false - // return true - // } -} diff --git a/server/src/warehouse/base/classes/Provider.ts b/server/src/warehouse/base/classes/Provider.ts deleted file mode 100644 index a3a8357f1..000000000 --- a/server/src/warehouse/base/classes/Provider.ts +++ /dev/null @@ -1,35 +0,0 @@ -import {DependencyIndex} from './DependencyIndex'; - -export class Provider { - - - constructor( - private dependencyIndex: DependencyIndex, - private receiverKey: ReceiverKeyModel - ) { } - - /** - * Gets array of items where the some of the provider key colums match the given - * partialKey. - * - * @param partialKey - */ - async getItemsWith(partialKey: Partial): Promise<{key: ProviderKeyModel, value: ProviderValModel}[]> { - const batch: {key: ProviderKeyModel, value: ProviderValModel}[] = [] - await this.dependencyIndex.providerDS.index.forEachItemWithNoDeleted(partialKey, async (item) => { - this.dependencyIndex.cacheNewDependencies(this.receiverKey, item.key); - batch.push(item) - }) - return batch - } - /** - * gets value from providing DataService - * adds the provider to the dependency index - */ - public async get(providerKey: ProviderKeyModel) { - this.dependencyIndex.cacheNewDependencies(this.receiverKey, providerKey); - return this.dependencyIndex.providerDS.index.getFromIdxNoDeleted(providerKey) - - } - -} diff --git a/server/src/warehouse/base/classes/SqlUpsertQueue.ts b/server/src/warehouse/base/classes/SqlUpsertQueue.ts deleted file mode 100644 index f9c73ddfa..000000000 --- a/server/src/warehouse/base/classes/SqlUpsertQueue.ts +++ /dev/null @@ -1,114 +0,0 @@ -import {omit, values} from 'ramda'; -import {BehaviorSubject, race, timer} from 'rxjs'; -import {filter, first} from 'rxjs/operators'; -import {Warehouse} from '../../Warehouse'; -import {Logger} from './Logger'; - -/** - * - */ -export class SqlUpsertQueue { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - queue: {[uniq: string]: any[];} = {}; - queueLength = 0; - queueLength$ = new BehaviorSubject(0); - queueing = false; - - constructor( - private wh: Warehouse, - private queueName: string, - private getSql: (valuesStr: string) => string, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - private itemToParams: (item: {key: KeyModel; val: ValueModel;}) => any[], - private keyModelToString: (key: KeyModel) => string, - private maxQueueLength = 1000, - private maxQueueTime = 20) { - } - - /** - * add item to queue - */ - public add(item: {key: KeyModel; val: ValueModel;}) { - if (!this.queueing) this.activateRace(); - - const params = this.itemToParams(item); - this.queue[this.keyModelToString(item.key)] = params; - this.queueLength++; - this.queueLength$.next(this.queueLength); - } - - /** - * remove item from queue - */ - public remove(key: KeyModel) { - const str = this.keyModelToString(key); - if (this.queue[str]) { - this.queue = omit([str], this.queue) - this.queueLength--; - this.queueLength$.next(this.queueLength); - } - } - - - /** - * Activates a new race of the timer (emitting after maxQueueTime) - * versus the queueLength (emitting when maxQueueLength reached). - * Once the race is won by eighter timer or queueLength, the - * sql is fired. - */ - private activateRace() { - this.queueing = true; - race( - this.queueLength$.pipe(filter(length => length >= this.maxQueueLength)), - timer(this.maxQueueTime) - ).pipe(first()).subscribe(() => { - this.queueing = false; - this.fire() - }); - } - - - - - /** - * creates and fires sql. - * resets queue - */ - private fire() { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const params: any[] = []; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const addParam = (val: any) => { - params.push(val); - return '$' + params.length; - }; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const addParams = (vals: any[]) => { - return vals.map(val => addParam(val)).join(','); - }; - - const length = this.queueLength; - const placeholder = values(this.queue).map(row => `(${addParams(row)})`).join(','); - const q = this.getSql(placeholder); - if (this.wh.status !== 'stopped') { - this.wh.whPgPool.query(q, params) - .then(() => { - Logger.msg(this.constructor.name, `\u{1b}[34m(async) Upserted ${length} ${this.queueName} \u{1b}[0m`); - }) - .catch(e => { - console.log(`Error on upserting items in ${this.queueName}: - PARAMS: ${params} - SQL - ${q}`, e); - }); - } - // console.log(this.queueName,this.queue) - - this.queue = {}; - this.queueLength = 0; - this.queueLength$.next(this.queueLength); - - - return length; - } -} diff --git a/server/src/warehouse/base/classes/UniqIdx.ts b/server/src/warehouse/base/classes/UniqIdx.ts deleted file mode 100644 index 99e002c62..000000000 --- a/server/src/warehouse/base/classes/UniqIdx.ts +++ /dev/null @@ -1,5 +0,0 @@ -import {IndexDB} from './IndexDB'; -export class UniqIdx extends IndexDB { - keyToString(key: string) {return key;} - stringToKey(key: string) {return key;} -} diff --git a/server/src/warehouse/base/classes/UniqIndexGeneric.ts b/server/src/warehouse/base/classes/UniqIndexGeneric.ts deleted file mode 100644 index 04fcf3d6b..000000000 --- a/server/src/warehouse/base/classes/UniqIndexGeneric.ts +++ /dev/null @@ -1,15 +0,0 @@ -import {IndexMemory} from './IndexMemory'; -import {KeyModelToString} from "../interfaces/KeyModelToString"; -import {StringToKeyModel} from "../interfaces/StringToKeyModel"; -import {Warehouse} from '../../Warehouse'; - -export class UniqIndexGeneric extends IndexMemory { - constructor( - name = 'default', - wh: Warehouse, - public keyToString: KeyModelToString, - public stringToKey: StringToKeyModel - ) { - super(name, wh); - } -} diff --git a/server/src/warehouse/base/classes/Updater.ts b/server/src/warehouse/base/classes/Updater.ts deleted file mode 100644 index b0d71d067..000000000 --- a/server/src/warehouse/base/classes/Updater.ts +++ /dev/null @@ -1,188 +0,0 @@ -// import {Logger} from './Logger'; -// import {AbstractAggregator} from './AbstractAggregator'; -// import {UniqIndexGeneric} from './UniqIndexGeneric'; -// import {StringToKeyModel} from '../interfaces/StringToKeyModel'; -// import {KeyModelToString} from '../interfaces/KeyModelToString'; -// import prettyms from 'pretty-ms'; -// import {Warehouse} from '../../Warehouse'; - -// export class Updater> { -// cycleNr = 1; - -// hot: 'A' | 'B' = 'A' - -// _queueA: UniqIndexGeneric -// _queueB: UniqIndexGeneric - -// // returns the index / db that is currently storing new pending requests -// get growingQueue() { -// return this.hot === 'A' ? this._queueA : this._queueB -// } - -// // returns the index / db that is used by the cycle -// get shrinkingQueue() { -// return this.hot === 'A' ? this._queueB : this._queueA -// } - -// // true during running update cycle -// updating = false; - -// constructor( -// private wh: Warehouse, -// public name: string, -// private aggregatorFactory: (modelId: KeyModel) => Promise, -// private register: (result: Aggregator) => Promise, -// keyToString: KeyModelToString, -// stringToKey: StringToKeyModel, -// ) { - -// this._queueA = new UniqIndexGeneric(name + '_queueA', wh, keyToString, stringToKey) -// this._queueB = new UniqIndexGeneric(name + '_queueB', wh, keyToString, stringToKey) -// } - - -// async addItemToQueue(id: KeyModel) { -// await this.growingQueue.addToIdx(id, true) - -// // initialize new recursive update cycle -// if (this.updating === false && this.wh.initializingIndexes === false -// ) { -// this.updating = true; -// setTimeout(() => { -// this.startCylcling().catch(e => {Logger.err(this.constructor.name, e)}) -// }, 0) -// } -// } - -// async addItemsToQueue(ids: KeyModel[]) { -// for (const entityId of ids) { -// await this.addItemToQueue(entityId) -// } -// } - -// async removeItems(ids: KeyModel[]) { -// for (const entityId of ids) { -// await this.removeItem(entityId) -// } -// } - -// private async removeItem(id: KeyModel) { -// await this.growingQueue.removeFromIdx(id) -// } - - -// /** -// * Start looping over the update queue. -// * May run multiple cycles. -// */ -// async startCylcling() { -// this.updating = true - -// const t0 = Logger.start(this.constructor.name, `${this.name} > Run recursive cycle ${this.cycleNr}`, 0) -// await this.runOneCycle(); -// Logger.itTook(this.constructor.name, t0, `to run recursive cycle`, 0) -// // this.updating = true - - -// const t1 = Logger.start(this.constructor.name, `get length of growingQueue`) -// const hotReqsLenght = await this.growingQueue.getLength() -// Logger.msg(this.constructor.name, `${hotReqsLenght} pending request remaining`) -// Logger.itTook(this.constructor.name, t1, `get length`) - -// // - restart cicle, if pending - -// if (hotReqsLenght > 0) { -// await this.startCylcling() -// } - -// this.updating = false - -// } - - -// // run one update cycle -// // TODO: implement this using streams instead of storing everythin in memory -// private async runOneCycle() { -// // this.updating = true -// const length = await this.growingQueue.getLength() -// const results: Aggregator[] = []; -// let i = 0 -// const measure = 1000; -// let t = Logger.getTime() -// const t2 = Logger.getTime() - -// let minMeasure: number | null = null, maxMeasure: number | null = null; - -// // - switch the active index recieving new update requests during the cycle -// this.switchActivePendingIdx() - -// // create aggregator for each item in shrinking queue -// await this.shrinkingQueue.forEachKey(async (entityId) => { - -// const created = await this.aggregatorFactory(entityId) -// results.push(created); - -// i++ -// if (i % measure === 0) { -// // Logger.resetLine() -// const time = Logger.itTook(this.constructor.name, t, `to aggregate ${measure} items by ${this.name} – item ${i}/${length}`, 1) -// t = Logger.getTime() -// if (!minMeasure || minMeasure > time) minMeasure = time; -// if (!maxMeasure || maxMeasure < time) maxMeasure = time; -// } - -// return; -// }) - -// // Logger.resetLine() -// if (minMeasure && maxMeasure) { -// const average = prettyms(Logger.diffMs(t2) / i) -// Logger.itTook(this.constructor.name, t2, `to aggregate ${i} items by ${this.constructor.name} – fastest: \u{1b}[33m${prettyms(minMeasure)}\u{1b}[0m , average: \u{1b}[33m${average}\u{1b}[0m , slowest: \u{1b}[33m${prettyms(maxMeasure)}\u{1b}[0m`, 1); -// } else { -// Logger.itTook(this.constructor.name, t2, `to aggregate ${i} items by ${this.constructor.name}`); -// } - - -// minMeasure = maxMeasure = null; - -// // Register EntityPreview for each created preview -// i = 0; -// for (const result of results) { -// await this.register(result) - -// i++ -// if (i % measure === 0) { -// // Logger.resetLine() -// Logger.itTook(this.constructor.name, t, `to register ${measure} items by ${this.name} – item ${i}/${results.length}`, 1) -// t = Logger.getTime() -// } -// // await result.register(); -// } -// // Logger.resetLine() -// if (minMeasure && maxMeasure) { -// const average = prettyms(Logger.diffMs(t2) / i) -// Logger.itTook(this.constructor.name, t2, `to register ${i} items by ${this.constructor.name} – fastest: \u{1b}[33m${prettyms(minMeasure)}\u{1b}[0m , average: \u{1b}[33m${average}\u{1b}[0m , slowest: \u{1b}[33m${prettyms(maxMeasure)}\u{1b}[0m`, 1); -// } else { -// Logger.itTook(this.constructor.name, t2, `to register ${i} items by ${this.constructor.name}`); -// } - - -// // clear the now consumed pending index -// await this.shrinkingQueue.clearIdx() - -// this.cycleNr++ -// // this.updating = false -// } - -// private switchActivePendingIdx() { -// this.hot = this.hot === 'A' ? 'B' : 'A'; -// } - -// async clearIdx() { -// this.cycleNr = 1; -// return Promise.all([ -// this.shrinkingQueue.clearIdx(), -// this.growingQueue.clearIdx() -// ]) -// } -// } diff --git a/server/src/warehouse/base/database.ts b/server/src/warehouse/base/database.ts deleted file mode 100644 index ae092345e..000000000 --- a/server/src/warehouse/base/database.ts +++ /dev/null @@ -1,36 +0,0 @@ - -// import leveldown from 'leveldown' -// import levelup from 'levelup' - -/** - * This creates the level database that can be imported - * throughout the app in order to use the database - */ -// export const leveldbpath = './leveldb'; -// export const leveldb = levelup(leveldown('./leveldb')) - -/** - * Other db implementations - */ - -// import sqlite3 from 'sqlite3' -// import r from 'rethinkdb' -// import { LevelUp } from 'levelup' -// const levelRocksdb = require('level-rocksdb') - - -// export const rocksdb: LevelUp = levelRocksdb('./rocksdb') - - -// const v = sqlite3.verbose() -// export const sqlite3db = new v.Database('sqlite') -// sqlite3db.serialize() - - -// export let rethinkdbConn: r.Connection | null = null -// export async function createRethinkdbConnection() { -// const conn = await r.connect({ host: 'localhost', port: 28015 }) -// await r.dbCreate('war').run(conn) -// rethinkdbConn = conn; -// return conn; -// } diff --git a/server/src/warehouse/base/functions.ts b/server/src/warehouse/base/functions.ts deleted file mode 100644 index 3d9cdfcdf..000000000 --- a/server/src/warehouse/base/functions.ts +++ /dev/null @@ -1,221 +0,0 @@ -// import {PClassId} from '../primary-ds/ProClassFieldsConfigService'; -// import {DfhClassLabelId} from '../primary-ds/DfhClassLabelService'; -// import {PEntityId} from '../primary-ds/entity/PEntityService'; -// import {ProClassLabelId} from '../primary-ds/ProClassLabelService'; -// import {ProjectId} from '../primary-ds/ProProjectService'; -// import {RClassId} from '../primary-ds/DfhClassHasTypePropertyService'; -// import {PPropertyId} from '../primary-ds/property/PPropertyService'; -// import {DfhPropertyLabelId} from '../primary-ds/DfhPropertyLabelService'; -// import {ProPropertyLabelId} from '../primary-ds/ProPropertyLabelService'; -// import {PClassFieldLabelId} from '../aggregator-ds/class-field-label/p-class-field-label/PClassFieldLabelService'; -// import {OutgoingProperyId} from '../primary-ds/DfhOutgoingPropertyService'; -// import {REntityId} from '../primary-ds/entity/REntityService'; -// import {RClassFieldId} from '../aggregator-ds/class-field-label/r-class-field-label/RClassFieldLabelService'; -// import {RPropertyId} from '../primary-ds/property/RPropertyService'; -import {switchMap, filter, map} from 'rxjs/operators'; -import {timer, pipe} from 'rxjs'; - -/** - * RxJS operator function - * When it recieves a value, waits for specified miliseconds - * until it emits the value. All values it receives during the - * miliseconds are skipped. After emitting the value, it starts over. - */ -export function skipWhileFirst(miliseconds?: number) { - let blocked = false; - return pipe( - filter((a: T) => !blocked), - switchMap((value: T) => { - blocked = true - return timer(miliseconds).pipe( - map(_ => { - blocked = false - return value - }) - ) - }) - ) -} - -// export function pEntityIdToString(key: PEntityId): string { -// return key.fkProject + '_' + key.pkEntity; -// } -// export function stringToPEntityId(str: string): PEntityId { -// const [fkProject, pkEntity] = str.split('_'); -// return {fkProject: parseInt(fkProject, 10), pkEntity: parseInt(pkEntity, 10)}; -// } - -// // project class id to string -// export function pClassIdToString(key: PClassId): string { -// return key.fkProject + '_' + key.pkClass; -// } -// // string to project class -// export function stringToPClassId(str: string): PClassId { -// const [fkProject, pkClass] = str.split('_'); -// return {fkProject: parseInt(fkProject, 10), pkClass: parseInt(pkClass, 10)}; -// } - -// // project property id to string -// export function pPropertyIdToString(key: PPropertyId): string { -// return key.fkProject + '_' + key.pkProperty + '_' + key.fkDomain + '_' + key.fkRange; -// } -// // string to project property -// export function stringToPPropertyId(str: string): PPropertyId { -// const [fkProject, pkProperty, fkDomain,fkRange] = str.split('_'); -// return { -// fkProject: parseInt(fkProject, 10), -// pkProperty: parseInt(pkProperty, 10), -// fkDomain: parseInt(fkDomain, 10), -// fkRange: parseInt(fkRange, 10), -// }; -// } - -// // repo property id to string -// export function rPropertyIdToString(key: RPropertyId): string { -// return '' + key.pkProperty+ '_' + key.fkDomain + '_' + key.fkRange; -// } -// // string to repo property -// export function stringToRPropertyId(str: string): RPropertyId { -// const [ pkProperty, fkDomain,fkRange] = str.split('_'); -// return { -// pkProperty: parseInt(pkProperty, 10), -// fkDomain: parseInt(fkDomain, 10), -// fkRange: parseInt(fkRange, 10), -// }; -// } - - -// // project field id to string -// export function pClassFieldIdToString(key: PClassFieldLabelId): string { -// return key.fkProject + '_' + key.fkClass + '_' + key.fkProperty + '_' + key.isOutgoing;; -// } -// // string to project field -// export function stringToPClassFieldId(str: string): PClassFieldLabelId { -// const [fkProject, fkClass, fkProperty, isOutgoing] = str.split('_'); -// return { -// fkProject: parseInt(fkProject, 10), -// fkClass: parseInt(fkClass, 10), -// fkProperty: parseInt(fkProperty, 10), -// isOutgoing: isOutgoing === 'true' -// }; -// } - - -// // repo field id to string -// export function rClassFieldIdToString(key: RClassFieldId): string { -// return key.fkClass + '_' + key.fkProperty + '_' + key.isOutgoing;; -// } -// // string to repo field -// export function stringToRClassFieldId(str: string): RClassFieldId { -// const [fkClass, fkProperty, isOutgoing] = str.split('_'); -// return { -// fkClass: parseInt(fkClass, 10), -// fkProperty: parseInt(fkProperty, 10), -// isOutgoing: isOutgoing === 'true' -// }; -// } - -// export function rEntityIdToString(key: REntityId): string { -// return '' + key.pkEntity; -// } -// export function stringToREntityId(str: string): REntityId { -// return {pkEntity: parseInt(str, 10)}; -// } - - -// // class to string -// export function rClassIdToString(key: RClassId): string { -// return key.pkClass.toString(); -// } -// // string to class -// export function stringToRClassId(str: string): RClassId { -// return {pkClass: parseInt(str, 10)}; -// } - -// export function projectIdToString(key: ProjectId): string { -// return key.pkProject.toString() -// } - -// export function stringToProjectId(str: string): ProjectId { -// return {pkProject: parseInt(str, 10)}; -// } - -// export function dfhClassIdToString(key: DfhClassLabelId): string { -// return key.pkClass + '_' + key.language - -// } -// export function stringToDfhClassId(str: string): DfhClassLabelId { -// const [pkClass, language] = str.split('_') -// return {pkClass: parseInt(pkClass, 10), language}; -// } - - -// export function dfhPropertyIdToString(key: DfhPropertyLabelId): string { -// return key.pkProperty + '_' + key.language - -// } -// export function stringToDfhPropertyId(str: string): DfhPropertyLabelId { -// const [pkProperty, language] = str.split('_') -// return {pkProperty: parseInt(pkProperty, 10), language}; -// } - - - - -// export function proClassIdToString(key: ProClassLabelId): string { -// return key.fkProject + '_' + key.fkClass + '_' + key.fkLanguage -// } -// export function stringToProClassId(str: string): ProClassLabelId { -// const [fkProject, pkClass, fkLanguage] = str.split('_') -// return {fkProject: parseInt(fkProject, 10), fkClass: parseInt(pkClass, 10), fkLanguage: parseInt(fkLanguage, 10)}; -// } - - -// export function proPropertyIdToString(key: ProPropertyLabelId): string { -// return key.fkProject + '_' + key.fkClass + '_' + key.fkProperty + '_' + key.isOutgoing + '_' + key.fkLanguage -// } -// export function stringToProPropertyId(str: string): ProPropertyLabelId { -// const [fkProject, fkClass, fkProperty, isOutgoing, fkLanguage] = str.split('_') -// return { -// fkProject: parseInt(fkProject, 10), -// fkClass: parseInt(fkClass, 10), -// fkProperty: parseInt(fkProperty, 10), -// isOutgoing: isOutgoing === 'true', -// fkLanguage: parseInt(fkLanguage, 10) -// }; -// } - -// // // this sql is useful for update statements of all entity preview parts -// // // contributing to the TSearch Vector of an entity preview: -// // // entity_label, type_label, class_label, full_text -// // export const sqlForTsVector = `ts_vector = ( -// // SELECT -// // setweight(to_tsvector(coalesce(entity_label, '')), 'A') || -// // setweight(to_tsvector(coalesce(type_label, class_label, '')), 'B') || -// // setweight(to_tsvector(coalesce(full_text, '')), 'C') -// // )` - - -// export function outgoingProperyIdToString(key: OutgoingProperyId): string { -// return key.fkDomain + '_' + key.fkProperty -// } -// export function stringToOutgoingProperyId(str: string): OutgoingProperyId { -// const [fkDomain, fkProperty] = str.split('_') -// return { -// fkDomain: parseInt(fkDomain, 10), -// fkProperty: parseInt(fkProperty, 10) -// }; -// } - - - - - -// /** -// * returns memory usage in MB -// */ -// export function getMemoryUsage(): number { -// const used = process.memoryUsage().heapUsed / 1024 / 1024; -// return Math.round(used * 100) / 100 -// } - diff --git a/server/src/warehouse/base/interfaces/Index.ts b/server/src/warehouse/base/interfaces/Index.ts deleted file mode 100644 index 932051230..000000000 --- a/server/src/warehouse/base/interfaces/Index.ts +++ /dev/null @@ -1,43 +0,0 @@ -import {ReplaySubject} from 'rxjs'; - -export interface Index { - - ready$: ReplaySubject - - addToIdx(keyModel: KeyModel, val: ValueModel): Promise - - removeFromIdx(keyModel: KeyModel): Promise - - - /** - * Returns value of key. If key not exists, returns undefined. - * @param key - */ - getFromIdx(keyModel: KeyModel): Promise - - - forEachKey(cb: (key: KeyModel) => Promise): Promise - - forEachKeyStartingWith(str: string, cb: (key: KeyModel) => Promise): Promise - - forEachItemStartingWith(str: string, cb: (item: {key: KeyModel, value: ValueModel}) => Promise): Promise - - forEachValue(cb: (val: ValueModel) => void): Promise - - clearIdx(): Promise - - keyExists(key: string): Promise - - - getKeys(): Promise - - getValues(): Promise - - - getLength(): Promise - - - keyToString(key: KeyModel): string; - stringToKey(str: string): KeyModel; - -} diff --git a/server/src/warehouse/base/interfaces/KeyDefinition.ts b/server/src/warehouse/base/interfaces/KeyDefinition.ts deleted file mode 100644 index 95266e27f..000000000 --- a/server/src/warehouse/base/interfaces/KeyDefinition.ts +++ /dev/null @@ -1,5 +0,0 @@ - -export interface KeyDefinition { - name: string; - type: 'integer' | 'text' | 'boolean'; -} diff --git a/server/src/warehouse/base/interfaces/KeyModelToString.ts b/server/src/warehouse/base/interfaces/KeyModelToString.ts deleted file mode 100644 index 9ec56f88f..000000000 --- a/server/src/warehouse/base/interfaces/KeyModelToString.ts +++ /dev/null @@ -1,2 +0,0 @@ - -export type KeyModelToString = (key: KeyModel) => string; diff --git a/server/src/warehouse/base/interfaces/Providers.ts b/server/src/warehouse/base/interfaces/Providers.ts deleted file mode 100644 index 55f277f8a..000000000 --- a/server/src/warehouse/base/interfaces/Providers.ts +++ /dev/null @@ -1,20 +0,0 @@ -import {DependencyIndex} from '../classes/DependencyIndex'; -import {Provider} from '../classes/Provider'; - -export abstract class Providers { - - protected abstract receiverKey: ReceiverKeyModel; - - // eslint-disable-next-line @typescript-eslint/no-explicit-any - private providers: Provider[] = [] - - registerProvider( - dependencyIndex: DependencyIndex, - receiverKey: ReceiverKeyModel) { - const p = new Provider(dependencyIndex, receiverKey) - this.providers.push(p) - return p - } - - -} diff --git a/server/src/warehouse/base/interfaces/StringToKeyModel.ts b/server/src/warehouse/base/interfaces/StringToKeyModel.ts deleted file mode 100644 index 0edd429bd..000000000 --- a/server/src/warehouse/base/interfaces/StringToKeyModel.ts +++ /dev/null @@ -1 +0,0 @@ -export type StringToKeyModel = (str: string) => KeyModel; diff --git a/server/src/warehouse/createWarehouse.ts b/server/src/warehouse/createWarehouse.ts deleted file mode 100644 index dabb7b5a5..000000000 --- a/server/src/warehouse/createWarehouse.ts +++ /dev/null @@ -1,130 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import {Provider, ReflectiveInjector} from 'injection-js'; -import 'reflect-metadata'; -// import {PClassFieldLabelDependencies} from './aggregator-ds/class-field-label/p-class-field-label/PClassFieldLabelDependencies'; -import {PClassFieldLabelService} from './aggregator-ds/class-field-label/p-class-field-label/PClassFieldLabelService'; -import {RClassFieldLabelService} from './aggregator-ds/class-field-label/r-class-field-label/RClassFieldLabelService'; -import {PClassLabelService} from './aggregator-ds/class-label/p-class-label/PClassLabelService'; -import {RClassLabelService} from './aggregator-ds/class-label/r-class-label/RClassLabelService'; -import {PEntityClassLabelService} from './aggregator-ds/entity-class-label/p-entity-class-label/PEntityClassLabelService'; -import {REntityClassLabelService} from './aggregator-ds/entity-class-label/r-entity-class-label/REntityClassLabelService'; -import {PEntityFullTextService} from './aggregator-ds/entity-full-text/p-entity-full-text/PEntityFullTextService'; -import {REntityFullTextService} from './aggregator-ds/entity-full-text/r-entity-full-text/REntityFullTextService'; -import {PEntityLabelService} from './aggregator-ds/entity-label/p-entity-label/PEntityLabelService'; -import {REntityLabelService} from './aggregator-ds/entity-label/r-entity-label/REntityLabelService'; -import {EntityPreviewService} from './aggregator-ds/entity-preview/EntityPreviewService'; -import {PEntityTimeSpanService} from './aggregator-ds/entity-time-span/p-entity-time-span/PEntityTimeSpanService'; -import {REntityTimeSpanService} from './aggregator-ds/entity-time-span/r-entity-time-span/REntityTimeSpanService'; -import {PEntityTypeService} from './aggregator-ds/entity-type/p-entity-type/PEntityTypeService'; -import {REntityTypeService} from './aggregator-ds/entity-type/r-entity-type/REntityTypeService'; -import {PClassService} from './primary-ds/class/PClassService'; -import {RClassService} from './primary-ds/class/RClassService'; -import {DfhClassHasTypePropertyService} from './primary-ds/DfhClassHasTypePropertyService'; -import {DfhClassLabelService} from './primary-ds/DfhClassLabelService'; -import {DfhOutgoingPropertyService} from './primary-ds/DfhOutgoingPropertyService'; -import {DfhPropertyLabelService} from './primary-ds/DfhPropertyLabelService'; -import {PEdgeService} from './primary-ds/edge/PEdgeService'; -import {REdgeService} from './primary-ds/edge/REdgeService'; -import {PEntityService} from './primary-ds/entity/PEntityService'; -import {REntityService} from './primary-ds/entity/REntityService'; -import {PFieldChangeService} from './primary-ds/PFieldChangeService'; -import {ProClassFieldsConfigService} from './primary-ds/ProClassFieldsConfigService'; -import {ProClassLabelService} from './primary-ds/ProClassLabelService'; -import {ProEntityLabelConfigService} from './primary-ds/ProEntityLabelConfigService'; -import {PPropertyService} from './primary-ds/property/PPropertyService'; -import {RPropertyService} from './primary-ds/property/RPropertyService'; -import {ProProjectService} from './primary-ds/ProProjectService'; -import {ProPropertyLabelService} from './primary-ds/ProPropertyLabelService'; -import {RFieldChangeService} from './primary-ds/RFieldChangeService'; -import {PStatementService} from './primary-ds/statement/PStatementService'; -import {AGG_DS, APP_CONFIG, PRIMARY_DS, Warehouse, WarehouseConfig} from './Warehouse'; - -export interface WarehouseStubs { - primaryDataServices: Provider[]; - aggDataServices: Provider[]; -} - - -/** - * - * @param config configuration of the warehouse - * @param stubs stubs allow to substitute dependencies for testing purposes - */ -export function createWarehouse( - config: WarehouseConfig, - stubs?: WarehouseStubs -): ReflectiveInjector { - - // prepare injection of PrimaryDataService classes - const primaryDataServices = stubs?.primaryDataServices ?? defaultPrimaryDataServices - - // prepare injection of AggregatedDataService classes - const aggDataServices = stubs?.aggDataServices ?? defaultAggregatedDataServices - - const injector = ReflectiveInjector.resolveAndCreate([ - primaryDataServices, - aggDataServices, - {provide: APP_CONFIG, useValue: config}, - {provide: PRIMARY_DS, useValue: primaryDataServices}, - {provide: AGG_DS, useValue: aggDataServices}, - Warehouse, - - ]); - return injector -} - -const defaultPrimaryDataServices = [ - DfhClassLabelService, - DfhPropertyLabelService, - DfhClassHasTypePropertyService, - DfhOutgoingPropertyService, - - ProProjectService, - ProClassLabelService, - ProPropertyLabelService, - ProEntityLabelConfigService, - - PClassService, - PPropertyService, - - ProClassFieldsConfigService, - - PEdgeService, - PEntityService, - PStatementService, - PFieldChangeService, - RFieldChangeService, - - RClassService, - RPropertyService, - - REntityService, - REdgeService, - -]; - -const defaultAggregatedDataServices = [ - // IdentifyingPropertyService, - - // Project aggegators - PClassLabelService, - PClassFieldLabelService, - PEntityLabelService, - PEntityTypeService, - PEntityClassLabelService, - PEntityFullTextService, - PEntityTimeSpanService, - - // Repo aggregators - RClassLabelService, - RClassFieldLabelService, - REntityLabelService, - REntityTypeService, - REntityClassLabelService, - REntityFullTextService, - REntityTimeSpanService, - - - // Resulting aggregators - EntityPreviewService -]; diff --git a/server/src/warehouse/ds-bundles/AggregatedDataServices.ts b/server/src/warehouse/ds-bundles/AggregatedDataServices.ts deleted file mode 100644 index 25b9d9fd7..000000000 --- a/server/src/warehouse/ds-bundles/AggregatedDataServices.ts +++ /dev/null @@ -1,151 +0,0 @@ -// // import { EntityPreviewService } from '../data-services/aggregated/EntityPreviewService'; -// import {Injectable, Inject, forwardRef} from 'injection-js'; -// import {PClassFieldLabelService} from '../aggregator-ds/class-field-label/p-class-field-label/PClassFieldLabelService'; -// import {RClassFieldLabelService} from '../aggregator-ds/class-field-label/r-class-field-label/RClassFieldLabelService'; -// import {PClassLabelService} from '../aggregator-ds/class-label/p-class-label/PClassLabelService'; -// import {RClassLabelService} from '../aggregator-ds/class-label/r-class-label/RClassLabelService'; -// import {PEntityClassLabelService} from '../aggregator-ds/entity-class-label/p-entity-class-label/PEntityClassLabelService'; -// import {REntityClassLabelService} from '../aggregator-ds/entity-class-label/r-entity-class-label/REntityClassLabelService'; -// import {PEntityFullTextService} from '../aggregator-ds/entity-full-text/p-entity-full-text/PEntityFullTextService'; -// import {REntityFullTextService} from '../aggregator-ds/entity-full-text/r-entity-full-text/REntityFullTextService'; -// import {PEntityLabelService} from '../aggregator-ds/entity-label/p-entity-label/PEntityLabelService'; -// import {REntityLabelService} from '../aggregator-ds/entity-label/r-entity-label/REntityLabelService'; -// import {PEntityTimeSpanService} from '../aggregator-ds/entity-time-span/p-entity-time-span/PEntityTimeSpanService'; -// import {REntityTimeSpanService} from '../aggregator-ds/entity-time-span/r-entity-time-span/REntityTimeSpanService'; -// import {PEntityTypeService} from '../aggregator-ds/entity-type/p-entity-type/PEntityTypeService'; -// import {REntityTypeService} from '../aggregator-ds/entity-type/r-entity-type/REntityTypeService'; -// import {IdentifyingPropertyService} from '../aggregator-ds/identifying-property/IdentifyingPropertyService'; -// import {AggregatedDataServicesBase} from '../base/classes/AggregatedDataServicesBase'; - -// @Injectable() -// // eslint-disable-next-line @typescript-eslint/no-explicit-any -// export class AggregatedDataServices extends AggregatedDataServicesBase { -// constructor( -// // Model aggregators -// @Inject(forwardRef(() => IdentifyingPropertyService)) public identifyingProperty: IdentifyingPropertyService, - -// // Project aggegators -// public pClassLabel: PClassLabelService, -// public pClassFieldLabel: PClassFieldLabelService, -// public pEntityClassLabel: PEntityClassLabelService, -// public pEntityLabel: PEntityLabelService, -// public pEntityType: PEntityTypeService, -// public pEntityFullText: PEntityFullTextService, -// public pEntityTimeSpan: PEntityTimeSpanService, - -// // Repo aggregators -// public rClassLabel: RClassLabelService, -// public rClassFieldLabel: RClassFieldLabelService, -// public rEntityClassLabel: REntityClassLabelService, -// public rEntityLabel: REntityLabelService, -// public rEntityType: REntityTypeService, -// public rEntityFullText: REntityFullTextService, -// public rEntityTimeSpan: REntityTimeSpanService, -// ) { -// super( -// identifyingProperty, -// pClassLabel, -// pClassFieldLabel, -// pEntityClassLabel, -// pEntityLabel, -// pEntityType, -// pEntityFullText, -// pEntityTimeSpan, -// rClassLabel, -// rClassFieldLabel, -// rEntityClassLabel, -// rEntityLabel, -// rEntityType, -// rEntityFullText, -// rEntityTimeSpan) -// } -// } - - - - -// // @Injectable() -// // // eslint-disable-next-line @typescript-eslint/no-explicit-any -// // export class AggregatedDataServices extends DataServiceBundle> { - - -// // ready$: Observable - -// // constructor( -// // // Model aggregators -// // public identifyingProperty: IdentifyingPropertyService, - -// // // Project aggegators -// // public pClassLabel: PClassLabelService, -// // public pClassFieldLabel: PClassFieldLabelService, -// // public pEntityClassLabel: PEntityClassLabelService, -// // public pEntityLabel: PEntityLabelService, -// // public pEntityType: PEntityTypeService, -// // public pEntityFullText: PEntityFullTextService, -// // public pEntityTimeSpan: PEntityTimeSpanService, - -// // // Repo aggregators -// // public rClassLabel: RClassLabelService, -// // public rClassFieldLabel: RClassFieldLabelService, -// // public rEntityClassLabel: REntityClassLabelService, -// // public rEntityLabel: REntityLabelService, -// // public rEntityType: REntityTypeService, -// // public rEntityFullText: REntityFullTextService, -// // public rEntityTimeSpan: REntityTimeSpanService, -// // ) { -// // super() -// // // // Model aggregators - - - - -// // this.registerDataService(this.identifyingProperty); // rEntityLabel - -// // // // Project aggegators -// // // this.registerDataService(this.pClassLabel) -// // // this.registerDataService(this.pClassFieldLabel) -// // // this.registerDataService(this.pEntityLabel); -// // // this.registerDataService(this.pEntityType); -// // // this.registerDataService(this.pEntityClassLabel) -// // // this.registerDataService(this.pEntityFullText) -// // // this.registerDataService(this.pEntityTimeSpan) - -// // // // // Repo aggregators -// // // this.registerDataService(this.rClassLabel) // rClassLabel -// // // this.registerDataService(this.rClassFieldLabel) -// // this.registerDataService(this.rEntityLabel); // rEntityLabel -// // // this.registerDataService(this.rEntityType); -// // // this.registerDataService(this.rEntityClassLabel) -// // // this.registerDataService(this.rEntityFullText) -// // // this.registerDataService(this.rEntityTimeSpan) - -// // this.ready$ = combineLatest( -// // this.registered.map(ds => ds.ready$.pipe(filter(r => r === true))), -// // ).pipe(mapTo(true)) -// // } - - -// // async startCycling() { -// // // Model aggregators -// // await this.identifyingProperty.startUpdate() // rEntityLabel - -// // // Project aggegators -// // // await this.pClassLabel.startUpdate() -// // // await this.pClassFieldLabel.startUpdate() -// // // await this.pEntityClassLabel.startUpdate() -// // // await this.pEntityLabel.startUpdate() -// // // await this.pEntityType.startUpdate() -// // // await this.pEntityFullText.startUpdate() -// // // await this.pEntityTimeSpan.startUpdate() - -// // // Repo aggregators -// // // await this.rClassLabel.startUpdate() // rClassLabel -// // // await this.rClassFieldLabel.startUpdate() -// // // await this.rEntityClassLabel.startUpdate() -// // await this.rEntityLabel.startUpdate() // rEntityLabel -// // // await this.rEntityType.startUpdate() -// // // await this.rEntityFullText.startUpdate() -// // // await this.rEntityTimeSpan.startUpdate() -// // } - -// // } diff --git a/server/src/warehouse/ds-bundles/DependencyDataServices.ts b/server/src/warehouse/ds-bundles/DependencyDataServices.ts deleted file mode 100644 index ff6287221..000000000 --- a/server/src/warehouse/ds-bundles/DependencyDataServices.ts +++ /dev/null @@ -1,50 +0,0 @@ -// import {forwardRef, Inject, Injectable} from 'injection-js'; -// import {Observable} from 'rxjs'; -// // import {PClassFieldLabelDependencies} from '../aggregator-ds/class-field-label/p-class-field-label/PClassFieldLabelDependencies'; - -// import {DependencyDataServicesBase} from '../base/classes/DependencyDataServicesBase'; - - -// @Injectable() -// export class DependencyDataServices extends DependencyDataServicesBase { - - -// ready$: Observable - -// constructor( -// // @Inject(forwardRef(()=>IdentifyingPropertyDependencies)) public identifyingProperty: IdentifyingPropertyDependencies, - -// // @Inject(forwardRef(()=>PClassLabelDependencies)) public pClassLabel: PClassLabelDependencies, -// // // @Inject(forwardRef(()=>PClassFieldLabelDependencies)) public pClassFieldLabel: PClassFieldLabelDependencies, -// // @Inject(forwardRef(()=>PEntityLabelDependencies)) public pEntityLabel: PEntityLabelDependencies, -// // @Inject(forwardRef(()=>PEntityTypeDependencies)) public pEntityType: PEntityTypeDependencies, -// // @Inject(forwardRef(()=>PEntityClassLabelDependencies)) public pEntityClassLabel: PEntityClassLabelDependencies, -// // @Inject(forwardRef(()=>PEntityFullTextDependencies)) public pEntityFullText: PEntityFullTextDependencies, -// // @Inject(forwardRef(()=>PEntityTimeSpanDependencies)) public pEntityTimeSpan: PEntityTimeSpanDependencies, - -// // // @Inject(forwardRef(()=>RClassLabelDependencies)) public rClassLabel: RClassLabelDependencies, -// // @Inject(forwardRef(()=>RClassFieldLabelDependencies)) public rClassFieldLabel: RClassFieldLabelDependencies, -// // // @Inject(forwardRef(()=>REntityLabelDependencies)) public rEntityLabel: REntityLabelDependencies, -// // @Inject(forwardRef(()=>REntityTypeDependencies)) public rEntityType: REntityTypeDependencies, - -// // @Inject(forwardRef(()=>REntityClassLabelDependencies)) public rEntityClassLabel: REntityClassLabelDependencies, -// // @Inject(forwardRef(()=>REntityFullTextDependencies)) public rEntityFullText: REntityFullTextDependencies, -// // @Inject(forwardRef(()=>REntityTimeSpanDependencies)) public rEntityTimeSpan: REntityTimeSpanDependencies, -// ) { -// super( -// // identifyingProperty, -// pClassLabel, -// // pClassFieldLabel,yy -// pEntityLabel, -// pEntityType, -// pEntityClassLabel, -// pEntityFullText, -// pEntityTimeSpan, -// rClassFieldLabel, -// rEntityType, -// rEntityClassLabel, -// rEntityFullText, -// rEntityTimeSpan) -// } - -// } diff --git a/server/src/warehouse/ds-bundles/PrimaryDataServices.ts b/server/src/warehouse/ds-bundles/PrimaryDataServices.ts deleted file mode 100644 index 622284343..000000000 --- a/server/src/warehouse/ds-bundles/PrimaryDataServices.ts +++ /dev/null @@ -1,74 +0,0 @@ -// import {forwardRef, Inject, Injectable} from 'injection-js'; -// import {Observable} from 'rxjs'; -// import {PrimaryDataServicesBase} from '../base/classes/PrimaryDataServicesBase'; -// import {PClassService} from '../primary-ds/class/PClassService'; -// import {RClassService} from '../primary-ds/class/RClassService'; -// import {DfhClassHasTypePropertyService} from '../primary-ds/DfhClassHasTypePropertyService'; -// import {DfhClassLabelService} from '../primary-ds/DfhClassLabelService'; -// import {DfhOutgoingPropertyService} from '../primary-ds/DfhOutgoingPropertyService'; -// import {DfhPropertyLabelService} from '../primary-ds/DfhPropertyLabelService'; -// import {PEdgeService} from '../primary-ds/edge/PEdgeService'; -// import {REdgeService} from '../primary-ds/edge/REdgeService'; -// import {PEntityService} from '../primary-ds/entity/PEntityService'; -// import {REntityService} from '../primary-ds/entity/REntityService'; -// import {ProClassFieldsConfigService} from '../primary-ds/ProClassFieldsConfigService'; -// import {ProClassLabelService} from '../primary-ds/ProClassLabelService'; -// import {ProEntityLabelConfigService} from '../primary-ds/ProEntityLabelConfigService'; -// import {PPropertyService} from '../primary-ds/property/PPropertyService'; -// import {RPropertyService} from '../primary-ds/property/RPropertyService'; -// import {ProProjectService} from '../primary-ds/ProProjectService'; -// import {ProPropertyLabelService} from '../primary-ds/ProPropertyLabelService'; - -// @Injectable() -// // eslint-disable-next-line @typescript-eslint/no-explicit-any -// export class PrimaryDataServices extends PrimaryDataServicesBase { - - -// ready$: Observable - -// constructor( -// @Inject(forwardRef(() => DfhClassLabelService)) public dfhClassLabel: DfhClassLabelService, -// @Inject(forwardRef(() => DfhPropertyLabelService)) public dfhPropertyLabel: DfhPropertyLabelService, -// @Inject(forwardRef(() => DfhClassHasTypePropertyService)) public dfhClassHasTypeProperty: DfhClassHasTypePropertyService, -// @Inject(forwardRef(() => DfhOutgoingPropertyService)) public dfhOutgoingProperty: DfhOutgoingPropertyService, - -// @Inject(forwardRef(() => ProProjectService)) public proProject: ProProjectService, -// @Inject(forwardRef(() => ProClassLabelService)) public proClassLabel: ProClassLabelService, -// @Inject(forwardRef(() => ProPropertyLabelService)) public proPropertyLabel: ProPropertyLabelService, -// @Inject(forwardRef(() => ProEntityLabelConfigService)) public proEntityLabelConfig: ProEntityLabelConfigService, - -// @Inject(forwardRef(() => PClassService)) public pClass: PClassService, -// @Inject(forwardRef(() => PPropertyService)) public pProperty: PPropertyService, - -// @Inject(forwardRef(() => ProClassFieldsConfigService)) public pClassFieldsConfig: ProClassFieldsConfigService, - -// @Inject(forwardRef(() => PEdgeService)) public pEdge: PEdgeService, -// @Inject(forwardRef(() => PEntityService)) public pEntity: PEntityService, - -// @Inject(forwardRef(() => RClassService)) public rClass: RClassService, -// @Inject(forwardRef(() => RPropertyService)) public rProperty: RPropertyService, - -// @Inject(forwardRef(() => REntityService)) public rEntity: REntityService, -// @Inject(forwardRef(() => REdgeService)) public rEdge: REdgeService, -// ) { -// super( -// dfhClassLabel, -// dfhPropertyLabel, -// dfhClassHasTypeProperty, -// dfhOutgoingProperty, -// proProject, -// proClassLabel, -// proPropertyLabel, -// proEntityLabelConfig, -// pClass, -// pProperty, -// pClassFieldsConfig, -// pEdge, -// pEntity, -// rClass, -// rProperty, -// rEntity, -// rEdge) -// } - -// } diff --git a/server/src/warehouse/primary-ds/DfhClassHasTypePropertyService.ts b/server/src/warehouse/primary-ds/DfhClassHasTypePropertyService.ts deleted file mode 100644 index 4d9154fa0..000000000 --- a/server/src/warehouse/primary-ds/DfhClassHasTypePropertyService.ts +++ /dev/null @@ -1,98 +0,0 @@ -import {forwardRef, Inject, Injectable} from 'injection-js'; -import {PrimaryDataService} from '../base/classes/PrimaryDataService'; -import {KeyDefinition} from '../base/interfaces/KeyDefinition'; -import {Warehouse} from '../Warehouse'; -export interface RClassId { - pkClass: number -} -export const rClassIdKeyDefs: KeyDefinition[] = [ - {name: 'pkClass', type: 'integer'} -] - -export interface DfhClassHasTypePropVal {fkProperty: number}; - -/** - * This PrimaryDataService creates an index of all properties that are - * 'has-type'-Subproperty, indexed by the domain class. - * - * Example : - * { - * // key is the pk_class of 'Geographical Place – C13' - * // value is the pk_property of 'P20 has geographical place type' - * '363': 1110 - * } - */ -@Injectable() -export class DfhClassHasTypePropertyService extends PrimaryDataService{ - measure = 1000; - constructor(@Inject(forwardRef(() => Warehouse)) wh: Warehouse) { - super( - wh, - ['modified_data_for_history_api_property'], - rClassIdKeyDefs - ) - } - getUpdatesSql(tmsp: Date) { - return updateSql - } - getDeletesSql(tmsp: Date) { - return deleteSql - } -} - -interface DbItem { - fkClass: number - fkProperty: number -} - -const updateSql = ` - SELECT DISTINCT ON (dfh_property_domain) - dfh_property_domain "pkClass", - jsonb_build_object('fkProperty',dfh_pk_property) val - FROM - data_for_history.api_property - WHERE - tmsp_last_modification >= $1 - AND - dfh_is_has_type_subproperty = true -` - - -const deleteSql = ` - --- give me all records where latest item in version history --- dfh_is_has_type_subproperty = true ... -WITH tw1 AS ( - SELECT - dfh_property_domain, - dfh_pk_property, - dfh_is_has_type_subproperty - FROM - data_for_history.api_property_vt - WHERE - tmsp_last_modification >= $1 - ORDER BY entity_version desc - LIMIT 1 -) -SELECT - dfh_property_domain "pkClass", - dfh_pk_property "fkProperty" -FROM tw1 -WHERE - dfh_is_has_type_subproperty = true - - - EXCEPT --- ... excluding the items, where in current version it is still: --- dfh_is_has_type_subproperty = true - -SELECT DISTINCT - dfh_property_domain "pkClass", - dfh_pk_property "fkProperty" -FROM - data_for_history.api_property -WHERE - dfh_is_has_type_subproperty = true - --- ... as a result we get only the items that have been set to false -` diff --git a/server/src/warehouse/primary-ds/DfhClassLabelService.ts b/server/src/warehouse/primary-ds/DfhClassLabelService.ts deleted file mode 100644 index 86681836a..000000000 --- a/server/src/warehouse/primary-ds/DfhClassLabelService.ts +++ /dev/null @@ -1,247 +0,0 @@ -import {PrimaryDataService} from '../base/classes/PrimaryDataService'; -import {KeyDefinition} from '../base/interfaces/KeyDefinition'; -import {Warehouse} from '../Warehouse'; -import {Injectable, Inject, forwardRef} from 'injection-js'; - -export interface DfhClassLabelId { - pkClass: number - language: number -} -const keyDefs: KeyDefinition[] = [ - { - name: 'pkClass', - type: 'integer' - }, - { - name: 'language', - type: 'integer' - } -] -export interface DfhClassLabelVal {label: string}; - -@Injectable() -export class DfhClassLabelService extends PrimaryDataService{ - measure = 1000; - constructor(@Inject(forwardRef(() => Warehouse)) wh: Warehouse) { - super( - wh, - ['modified_data_for_history_api_class'], - keyDefs - ) - } - getUpdatesSql(tmsp: Date) { - return updateSql - } - getDeletesSql(tmsp: Date) {return ''}; - -} - -interface DbItem { - pkClass: number - language: string - label: string -} - -const updateSql = ` - - WITH tw1 AS ( - SELECT * - FROM ( - VALUES - (17082, 'aa'), - (17099, 'ab'), - (17184, 'af'), - (17260, 'ak'), - (17314, 'am'), - (17413, 'ar'), - (17418, 'an'), - (17448, 'as'), - (17508, 'av'), - (17511, 'ae'), - (17558, 'ay'), - (17572, 'az'), - (17588, 'ba'), - (17590, 'bm'), - (17689, 'be'), - (17691, 'bn'), - (17793, 'bi'), - (17925, 'bo'), - (17940, 'bs'), - (18000, 'br'), - (18080, 'bg'), - (18235, 'ca'), - (18290, 'cs'), - (18300, 'ch'), - (18304, 'ce'), - (18318, 'cu'), - (18319, 'cv'), - (18419, 'kw'), - (18420, 'co'), - (18443, 'cr'), - (18529, 'cy'), - (18547, 'da'), - (18605, 'de'), - (18660, 'dv'), - (18826, 'dz'), - (18865, 'el'), - (18889, 'en'), - (18903, 'eo'), - (18925, 'et'), - (18939, 'eu'), - (18943, 'ee'), - (18962, 'fo'), - (18965, 'fa'), - (18979, 'fj'), - (18981, 'fi'), - (19008, 'fr'), - (19019, 'fy'), - (19031, 'ff'), - (19192, 'gd'), - (19195, 'ga'), - (19196, 'gl'), - (19205, 'gv'), - (19282, 'gn'), - (19314, 'gu'), - (19393, 'ht'), - (19394, 'ha'), - (19404, 'sh'), - (19412, 'he'), - (19418, 'hz'), - (19434, 'hi'), - (19465, 'ho'), - (19516, 'hr'), - (19542, 'hu'), - (19564, 'hy'), - (19576, 'ig'), - (19590, 'io'), - (19616, 'ii'), - (19632, 'iu'), - (19639, 'ie'), - (19657, 'ia'), - (19659, 'id'), - (19675, 'ik'), - (19696, 'is'), - (19703, 'it'), - (19752, 'jv'), - (19839, 'ja'), - (19883, 'kl'), - (19885, 'kn'), - (19889, 'ks'), - (19890, 'ka'), - (19891, 'kr'), - (19896, 'kk'), - (20056, 'km'), - (20080, 'ki'), - (20083, 'rw'), - (20087, 'ky'), - (20234, 'kv'), - (20235, 'kg'), - (20239, 'ko'), - (20372, 'kj'), - (20389, 'ku'), - (20535, 'lo'), - (20540, 'la'), - (20542, 'lv'), - (20656, 'li'), - (20657, 'ln'), - (20663, 'lt'), - (20817, 'lb'), - (20819, 'lu'), - (20824, 'lg'), - (20869, 'mh'), - (20873, 'ml'), - (20877, 'mr'), - (21112, 'mk'), - (21139, 'mg'), - (21152, 'mt'), - (21217, 'mn'), - (21288, 'mi'), - (21306, 'ms'), - (21453, 'my'), - (21518, 'na'), - (21519, 'nv'), - (21534, 'nr'), - (21573, 'nd'), - (21583, 'ng'), - (21609, 'ne'), - (21740, 'nl'), - (21795, 'nn'), - (21807, 'nb'), - (21822, 'no'), - (21957, 'ny'), - (22005, 'oc'), - (22028, 'oj'), - (22107, 'or'), - (22108, 'om'), - (22125, 'os'), - (22170, 'pa'), - (22315, 'pi'), - (22383, 'pl'), - (22389, 'pt'), - (22479, 'ps'), - (22507, 'qu'), - (22670, 'rm'), - (22673, 'ro'), - (22701, 'rn'), - (22705, 'ru'), - (22727, 'sg'), - (22732, 'sa'), - (22893, 'si'), - (22953, 'sk'), - (22963, 'sl'), - (22972, 'se'), - (22981, 'sm'), - (22993, 'sn'), - (22996, 'sd'), - (23029, 'so'), - (23035, 'st'), - (23042, 'es'), - (23065, 'sq'), - (23078, 'sc'), - (23089, 'sr'), - (23121, 'ss'), - (23156, 'su'), - (23174, 'sw'), - (23177, 'sv'), - (23243, 'ty'), - (23247, 'ta'), - (23254, 'tt'), - (23341, 'te'), - (23369, 'tg'), - (23370, 'tl'), - (23384, 'th'), - (23418, 'ti'), - (23537, 'to'), - (23619, 'tn'), - (23620, 'ts'), - (23667, 'tk'), - (23673, 'tr'), - (23701, 'tw'), - (23782, 'ug'), - (23792, 'uk'), - (23838, 'ur'), - (23878, 'uz'), - (23904, 've'), - (23912, 'vi'), - (23958, 'vo'), - (24081, 'wa'), - (24127, 'wo'), - (24278, 'xh'), - (24572, 'yi'), - (24651, 'yo'), - (24776, 'za'), - (24781, 'zh'), - (24901, 'zu') - ) x(fk_entity, iso_6391 ) - ) - SELECT DISTINCT ON (dfh_pk_class,t2.fk_entity) - dfh_pk_class "pkClass", - t2.fk_entity "language", - jsonb_build_object('label',t1.dfh_class_label) val - FROM - data_for_history.api_class t1, - tw1 t2 - WHERE - t1.tmsp_last_modification > $1 - AND t1.dfh_class_label_language = t2.iso_6391 -` diff --git a/server/src/warehouse/primary-ds/DfhOutgoingPropertyService.ts b/server/src/warehouse/primary-ds/DfhOutgoingPropertyService.ts deleted file mode 100644 index af30765ff..000000000 --- a/server/src/warehouse/primary-ds/DfhOutgoingPropertyService.ts +++ /dev/null @@ -1,80 +0,0 @@ -import {PrimaryDataService} from '../base/classes/PrimaryDataService'; -import {KeyDefinition} from '../base/interfaces/KeyDefinition'; -import {Warehouse} from '../Warehouse'; -import {Injectable, Inject, forwardRef} from 'injection-js'; -export interface OutgoingProperyId { - fkDomain: number - fkProperty: number -} -export type OutgoingPropertyVal = { - fkDomain: number, - fkProperty: number, - dfhIdentityDefining: boolean, - dfhIsHasTypeSubproperty: boolean, - dfhRangeInstancesMaxQuantifier: number, - dfhRangeInstancesMinQuantifier: number, - dfhDomainInstancesMaxQuantifier: number, - dfhDomainInstancesMinQuantifier: number, -}; - -/** - * This PrimaryDataService creates an index of all properties, - * storing the not-language dependent aspects like - * - quantifiers - * - has type subproperty - * - is identity defining - * - * Example : - * { - * // key is the pk_class of 'Geographical Place – C13' - * // value is the pk_property of 'P20 has geographical place type' - * '363': 1110 - * } - */ -const outgoingPropertykeyDefs: KeyDefinition[] = [ - { - name: 'fkDomain', - type: 'integer' - }, - { - name: 'fkProperty', - type: 'integer' - }, -] -@Injectable() -export class DfhOutgoingPropertyService extends PrimaryDataService{ - measure = 1000; - constructor(@Inject(forwardRef(() => Warehouse)) wh: Warehouse) { - super(wh, - ['modified_data_for_history_api_property'], - outgoingPropertykeyDefs - ) - } - - getUpdatesSql(tmsp: Date) { - return updateSql - } - getDeletesSql(tmsp: Date) {return ''}; -} - -const updateSql = ` - SELECT DISTINCT ON (dfh_property_domain, dfh_pk_property) - dfh_property_domain "fkDomain", - dfh_pk_property "fkProperty", - jsonb_build_object( - 'fkDomain', dfh_property_domain, - 'fkProperty', dfh_pk_property, - 'dfhIdentityDefining', dfh_identity_defining, - 'dfhIsHasTypeSubproperty', dfh_is_has_type_subproperty, - 'dfhRangeInstancesMaxQuantifier', dfh_range_instances_max_quantifier, - 'dfhRangeInstancesMinQuantifier', dfh_range_instances_min_quantifier, - 'dfhDomainInstancesMaxQuantifier', dfh_domain_instances_max_quantifier, - 'dfhDomainInstancesMinQuantifier', dfh_domain_instances_min_quantifier - ) val - FROM - data_for_history.api_property - WHERE - tmsp_last_modification >= $1 -` - - diff --git a/server/src/warehouse/primary-ds/DfhPropertyLabelService.ts b/server/src/warehouse/primary-ds/DfhPropertyLabelService.ts deleted file mode 100644 index fe2000dad..000000000 --- a/server/src/warehouse/primary-ds/DfhPropertyLabelService.ts +++ /dev/null @@ -1,235 +0,0 @@ -import {forwardRef, Inject, Injectable} from 'injection-js'; -import {PrimaryDataService} from '../base/classes/PrimaryDataService'; -import {KeyDefinition} from '../base/interfaces/KeyDefinition'; -import {Warehouse} from '../Warehouse'; -export interface DfhPropertyLabelId { - pkProperty: number - language: number -} -export interface DfhPropertyLabelVal {label: string, inverseLabel: string}; -export const dfhPropertyLabelIdKeyDef: KeyDefinition[] = [ - {name: 'pkProperty', type: 'integer'}, - {name: 'language', type: 'integer'} -] -@Injectable() -export class DfhPropertyLabelService extends PrimaryDataService{ - measure = 1000; - constructor(@Inject(forwardRef(() => Warehouse)) wh: Warehouse) { - super( - wh, - ['modified_data_for_history_api_property'], - dfhPropertyLabelIdKeyDef - ) - } - getUpdatesSql(tmsp: Date) { - return updateSql - } - getDeletesSql(tmsp: Date) {return ''}; -} - - -const updateSql = ` - WITH tw1 AS ( - SELECT * - FROM ( - VALUES - (17082, 'aa'), - (17099, 'ab'), - (17184, 'af'), - (17260, 'ak'), - (17314, 'am'), - (17413, 'ar'), - (17418, 'an'), - (17448, 'as'), - (17508, 'av'), - (17511, 'ae'), - (17558, 'ay'), - (17572, 'az'), - (17588, 'ba'), - (17590, 'bm'), - (17689, 'be'), - (17691, 'bn'), - (17793, 'bi'), - (17925, 'bo'), - (17940, 'bs'), - (18000, 'br'), - (18080, 'bg'), - (18235, 'ca'), - (18290, 'cs'), - (18300, 'ch'), - (18304, 'ce'), - (18318, 'cu'), - (18319, 'cv'), - (18419, 'kw'), - (18420, 'co'), - (18443, 'cr'), - (18529, 'cy'), - (18547, 'da'), - (18605, 'de'), - (18660, 'dv'), - (18826, 'dz'), - (18865, 'el'), - (18889, 'en'), - (18903, 'eo'), - (18925, 'et'), - (18939, 'eu'), - (18943, 'ee'), - (18962, 'fo'), - (18965, 'fa'), - (18979, 'fj'), - (18981, 'fi'), - (19008, 'fr'), - (19019, 'fy'), - (19031, 'ff'), - (19192, 'gd'), - (19195, 'ga'), - (19196, 'gl'), - (19205, 'gv'), - (19282, 'gn'), - (19314, 'gu'), - (19393, 'ht'), - (19394, 'ha'), - (19404, 'sh'), - (19412, 'he'), - (19418, 'hz'), - (19434, 'hi'), - (19465, 'ho'), - (19516, 'hr'), - (19542, 'hu'), - (19564, 'hy'), - (19576, 'ig'), - (19590, 'io'), - (19616, 'ii'), - (19632, 'iu'), - (19639, 'ie'), - (19657, 'ia'), - (19659, 'id'), - (19675, 'ik'), - (19696, 'is'), - (19703, 'it'), - (19752, 'jv'), - (19839, 'ja'), - (19883, 'kl'), - (19885, 'kn'), - (19889, 'ks'), - (19890, 'ka'), - (19891, 'kr'), - (19896, 'kk'), - (20056, 'km'), - (20080, 'ki'), - (20083, 'rw'), - (20087, 'ky'), - (20234, 'kv'), - (20235, 'kg'), - (20239, 'ko'), - (20372, 'kj'), - (20389, 'ku'), - (20535, 'lo'), - (20540, 'la'), - (20542, 'lv'), - (20656, 'li'), - (20657, 'ln'), - (20663, 'lt'), - (20817, 'lb'), - (20819, 'lu'), - (20824, 'lg'), - (20869, 'mh'), - (20873, 'ml'), - (20877, 'mr'), - (21112, 'mk'), - (21139, 'mg'), - (21152, 'mt'), - (21217, 'mn'), - (21288, 'mi'), - (21306, 'ms'), - (21453, 'my'), - (21518, 'na'), - (21519, 'nv'), - (21534, 'nr'), - (21573, 'nd'), - (21583, 'ng'), - (21609, 'ne'), - (21740, 'nl'), - (21795, 'nn'), - (21807, 'nb'), - (21822, 'no'), - (21957, 'ny'), - (22005, 'oc'), - (22028, 'oj'), - (22107, 'or'), - (22108, 'om'), - (22125, 'os'), - (22170, 'pa'), - (22315, 'pi'), - (22383, 'pl'), - (22389, 'pt'), - (22479, 'ps'), - (22507, 'qu'), - (22670, 'rm'), - (22673, 'ro'), - (22701, 'rn'), - (22705, 'ru'), - (22727, 'sg'), - (22732, 'sa'), - (22893, 'si'), - (22953, 'sk'), - (22963, 'sl'), - (22972, 'se'), - (22981, 'sm'), - (22993, 'sn'), - (22996, 'sd'), - (23029, 'so'), - (23035, 'st'), - (23042, 'es'), - (23065, 'sq'), - (23078, 'sc'), - (23089, 'sr'), - (23121, 'ss'), - (23156, 'su'), - (23174, 'sw'), - (23177, 'sv'), - (23243, 'ty'), - (23247, 'ta'), - (23254, 'tt'), - (23341, 'te'), - (23369, 'tg'), - (23370, 'tl'), - (23384, 'th'), - (23418, 'ti'), - (23537, 'to'), - (23619, 'tn'), - (23620, 'ts'), - (23667, 'tk'), - (23673, 'tr'), - (23701, 'tw'), - (23782, 'ug'), - (23792, 'uk'), - (23838, 'ur'), - (23878, 'uz'), - (23904, 've'), - (23912, 'vi'), - (23958, 'vo'), - (24081, 'wa'), - (24127, 'wo'), - (24278, 'xh'), - (24572, 'yi'), - (24651, 'yo'), - (24776, 'za'), - (24781, 'zh'), - (24901, 'zu') - ) x(fk_entity, iso_6391 ) - ) - SELECT DISTINCT ON (dfh_pk_property, t2.fk_entity) - t1.dfh_pk_property "pkProperty", - t2.fk_entity "language", - jsonb_build_object( - 'label', t1.dfh_property_label, - 'inverseLabel', t1.dfh_property_inverse_label - ) val - FROM - data_for_history.api_property t1, - tw1 t2 - WHERE - t1.tmsp_last_modification > $1 - AND t1.dfh_property_label_language = t2.iso_6391 -` diff --git a/server/src/warehouse/primary-ds/EntityPathConfigService.ts b/server/src/warehouse/primary-ds/EntityPathConfigService.ts deleted file mode 100644 index 206029b60..000000000 --- a/server/src/warehouse/primary-ds/EntityPathConfigService.ts +++ /dev/null @@ -1,78 +0,0 @@ -// import {forwardRef, Inject, Injectable} from 'injection-js'; -// import {PrimaryDataService} from '../base/classes/PrimaryDataService'; -// import {Warehouse} from '../Warehouse'; -// import {RClassId, rClassIdKeyDefs} from './DfhClassHasTypePropertyService'; - - -// export interface EntityPathConfigKey { -// whereSourceClass: number -// } -// interface EntityPathConfig { -// pathName: string -// isOutgoing: boolean -// whereSourceClass: number -// whereProperty: number -// whereTargetClass: number -// } -// export interface EntityPathConfigVal { -// pathConfigs: EntityPathConfig[] -// } - - -// @Injectable() -// export class EntityPathConfigService extends PrimaryDataService{ - -// measure = 1000; - - -// constructor(@Inject(forwardRef(() => Warehouse)) wh: Warehouse) { -// super( -// wh, -// ['modified_entity_path_config'], -// rClassIdKeyDefs -// ) -// } - -// getUpdatesSql(tmsp: Date) { -// return updateSql -// } -// getDeletesSql(tmsp: Date) {return ''}; -// } - -// const items: EntityPathConfig[] = [ -// { -// pathName: 'sourcesPath', -// isOutgoing: true, -// whereSourceClass: 503, // Expression portion -// whereProperty: 1317, // is part of -// whereTargetClass: 503 // Expression portion -// }, -// { -// pathName: 'sourcesPath', -// isOutgoing: true, -// whereSourceClass: 503, // Expression portion -// whereProperty: 1317, // is part of -// whereTargetClass: 218 // Expression -// }, -// { -// pathName: 'sourcesPath', -// isOutgoing: true, -// whereSourceClass: 218, // Expression -// whereProperty: 979, // carriers provided by -// whereTargetClass: 219 // Manifestation Product Type -// } -// ] - -// const updateSql = ` -// WITH tw1 as ( -// SELECT json_array_elements('${JSON.stringify(items)}'::json) as val -// WHERE $1::timestamp with time zone IS NOT NULL -// ) -// select -// (val->>'whereSourceClass')::int as "pkClass", -// json_build_object('pathConfigs',json_agg(val)) val -// FROM tw1 -// GROUP BY val->>'whereSourceClass' -// ` - - diff --git a/server/src/warehouse/primary-ds/PFieldChangeService.ts b/server/src/warehouse/primary-ds/PFieldChangeService.ts deleted file mode 100644 index 0993e6fb7..000000000 --- a/server/src/warehouse/primary-ds/PFieldChangeService.ts +++ /dev/null @@ -1,151 +0,0 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -import {forwardRef, Inject, Injectable} from 'injection-js'; -import {PrimaryDataService} from '../base/classes/PrimaryDataService'; -import {KeyDefinition} from '../base/interfaces/KeyDefinition'; -import {Warehouse} from '../Warehouse'; -export interface PFieldChangeId { - fkProject: number; - - fkSourceInfo: number; - fkSourceTablesCell: number; - - fkProperty: number; - fkPropertyOfProperty: number; - - isOutgoing: boolean; - - -} -export interface PFieldChangeVal { - tmspLastModification: string -} -export const pStatementKeyDefs: KeyDefinition[] = [ - {name: 'fkProject', type: 'integer'}, - - {name: 'fkSourceInfo', type: 'integer'}, - {name: 'fkSourceTablesCell', type: 'integer'}, - - {name: 'fkProperty', type: 'integer'}, - {name: 'fkPropertyOfProperty', type: 'integer'}, - - {name: 'isOutgoing', type: 'boolean'}, -] - -@Injectable() -export class PFieldChangeService extends PrimaryDataService{ - - measure = 10000; - - - constructor(@Inject(forwardRef(() => Warehouse)) wh: Warehouse) { - super( - wh, - [ - 'modified_projects_info_proj_rel', - ], - pStatementKeyDefs - ) - this.registerUpdateReplication( - 'war.field_change', - (insertClause: string, fromClause: string) => ` - INSERT INTO war.field_change ( - fk_project, - fk_source_info, - fk_source_tables_cell, - fk_property, - fk_property_of_property, - is_outgoing, - tmsp_last_modification - ) - SELECT - t1."fkProject", - t1."fkSourceInfo", - t1."fkSourceTablesCell", - t1."fkProperty", - t1."fkPropertyOfProperty", - t1."isOutgoing", - (t1.val->>'tmspLastModification')::timestamp with time zone - FROM ${fromClause} t1 - ON CONFLICT ( - fk_project, - fk_source_info, - fk_source_tables_cell, - fk_property, - fk_property_of_property, - is_outgoing - ) DO UPDATE - SET - fk_project = EXCLUDED.fk_project, - fk_source_info = EXCLUDED.fk_source_info, - fk_source_tables_cell = EXCLUDED.fk_source_tables_cell, - fk_property = EXCLUDED.fk_property, - fk_property_of_property = EXCLUDED.fk_property_of_property, - is_outgoing = EXCLUDED.is_outgoing, - tmsp_last_modification = EXCLUDED.tmsp_last_modification - ` - ) - } - - - getUpdatesSql(tmsp: Date) { - return updateSql - } - getDeletesSql(tmsp: Date) {return ''}; - -} - - -const updateSql = ` - -- select last modification tmsp of incoming fields - SELECT - json_build_object('tmspLastModification',max(t1.tmsp_last_modification)) val, - t1.fk_project "fkProject", - t2.fk_object_info "fkSourceInfo", - t2.fk_object_tables_cell "fkSourceTablesCell", - t2.fk_property "fkProperty", - t2.fk_property_of_property "fkPropertyOfProperty", - false "isOutgoing" - FROM - projects.info_proj_rel t1, - information."statement" t2 - WHERE - t1.fk_entity = t2.pk_entity - AND - (t2.fk_object_info != 0 OR t2.fk_object_tables_cell != 0) - AND - t1.tmsp_last_modification > $1 - GROUP BY - t1.fk_project, - t2.fk_object_info, - t2.fk_object_tables_cell, - t2.fk_property, - t2.fk_property_of_property - - UNION ALL - - -- select last modification tmsp of outgoing fields - SELECT - json_build_object('tmspLastModification',max(t1.tmsp_last_modification)) val, - t1.fk_project "fkProject", - t2.fk_subject_info "fkSourceInfo", - t2.fk_subject_tables_cell "fkSourceTablesCell", - t2.fk_property "fkProperty", - t2.fk_property_of_property "fkPropertyOfProperty", - true "isOutgoing" - FROM - projects.info_proj_rel t1, - information."statement" t2 - WHERE - t1.fk_entity = t2.pk_entity - AND - (t2.fk_subject_info != 0 OR t2.fk_subject_tables_cell != 0) - AND - t1.tmsp_last_modification > $1 - GROUP BY - t1.fk_project, - t2.fk_subject_info, - t2.fk_subject_tables_cell, - t2.fk_property, - t2.fk_property_of_property; -` - diff --git a/server/src/warehouse/primary-ds/ProClassFieldsConfigService.ts b/server/src/warehouse/primary-ds/ProClassFieldsConfigService.ts deleted file mode 100644 index 0ac2c6efc..000000000 --- a/server/src/warehouse/primary-ds/ProClassFieldsConfigService.ts +++ /dev/null @@ -1,117 +0,0 @@ -import {PrimaryDataService} from '../base/classes/PrimaryDataService'; -import {KeyDefinition} from '../base/interfaces/KeyDefinition'; -import {Warehouse} from '../Warehouse'; -import {Injectable, Inject, forwardRef} from 'injection-js'; - -export interface PClassId { - fkProject: number, - pkClass: number -} -export interface ClassField { - fkProperty: number, - isOutgoing: boolean, - ordNum: number -} -export const pClassIdKeyDef: KeyDefinition[] = [ - { - name: 'fkProject', - type: 'integer' - }, - { - name: 'pkClass', - type: 'integer' - } -] -export interface ProClassFieldVal { - outgoing: { - [fkProperty: number]: {ordNum: number} - } - incoming: { - [fkProperty: number]: {ordNum: number} - } -} // = ClassField[] -@Injectable() -export class ProClassFieldsConfigService extends PrimaryDataService{ - measure = 1000; - constructor(@Inject(forwardRef(() => Warehouse)) wh: Warehouse) { - super( - wh, - ['modified_projects_class_field_config'], - pClassIdKeyDef - ) - } - - getUpdatesSql(tmsp: Date) { - return updateSql - } - getDeletesSql(tmsp: Date) {return ''}; - - -} - -export const updateSql = ` -WITH tw1 AS( - SELECT - fk_project, - fk_domain_class fk_source_class, - json_object_agg(fk_property, json_build_object( - 'ordNum', ord_num - ) ORDER BY ord_num ) outgoing, - max(tmsp_last_modification) - FROM projects.class_field_config - WHERE fk_domain_class IS NOT NULL - GROUP BY fk_project, fk_source_class -), -tw2 AS ( - SELECT - fk_project, - fk_range_class fk_source_class, - json_object_agg(fk_property, json_build_object( - 'ordNum', ord_num - ) ORDER BY ord_num ) incoming, - max(tmsp_last_modification) - FROM projects.class_field_config - WHERE fk_range_class IS NOT NULL - GROUP BY fk_project, fk_source_class -) -SELECT -coalesce(t1.fk_project,t2.fk_project) "fkProject", -coalesce(t1.fk_source_class,t2.fk_source_class) "pkClass", -json_build_object( - 'outgoing', COALESCE(outgoing,'{}'::json), - 'incoming', COALESCE(incoming,'{}'::json) - ) val -FROM tw1 t1 -FULL OUTER JOIN tw2 t2 ON t1.fk_project=t2.fk_project -AND t1.fk_source_class=t2.fk_source_class -WHERE t1.max >= $1 OR t2.max >= $1 - - ---WITH tw1 AS ( --- SELECT --- fk_project, --- CASE WHEN --- fk_domain_class IS NOT NULL --- THEN true --- ELSE false --- END --- property_is_outgoing, --- coalesce(fk_domain_class, fk_range_class) fk_source_class, --- fk_property, --- ord_num --- FROM projects.class_field_config --- WHERE tmsp_last_modification >= $1 --- ORDER BY ord_num ---) --- SELECT --- fk_project "fkProject", --- fk_source_class "pkClass", --- json_agg(json_build_object( --- 'fkProperty', fk_property, --- 'isOutgoing', property_is_outgoing, --- 'ordNum', ord_num --- )) val --- FROM tw1 --- GROUP BY fk_project, fk_source_class -` - diff --git a/server/src/warehouse/primary-ds/ProClassLabelService.ts b/server/src/warehouse/primary-ds/ProClassLabelService.ts deleted file mode 100644 index f1f0db69c..000000000 --- a/server/src/warehouse/primary-ds/ProClassLabelService.ts +++ /dev/null @@ -1,90 +0,0 @@ -import {PrimaryDataService} from '../base/classes/PrimaryDataService'; -import {KeyDefinition} from '../base/interfaces/KeyDefinition'; -import {Warehouse} from '../Warehouse'; -import {Injectable, Inject, forwardRef} from 'injection-js'; -export interface ProClassLabelId { - fkProject: number - fkClass: number - fkLanguage: number -} -const keyDefs: KeyDefinition[] = [ - { - name: 'fkClass', - type: 'integer' - }, - { - name: 'fkProject', - type: 'integer' - }, - { - name: 'fkLanguage', - type: 'integer' - } -] -export interface ProClassLabelVal {label: string} - -@Injectable() -export class ProClassLabelService extends PrimaryDataService< ProClassLabelId, ProClassLabelVal>{ - measure = 1000; - constructor(@Inject(forwardRef(() => Warehouse)) wh: Warehouse) { - super( - wh, - ['modified_projects_text_property'], - keyDefs) - } - - dbItemToKeyVal = undefined - getUpdatesSql(tmsp: Date) { - return updateSql - } - getDeletesSql(tmsp: Date) { - return deleteSql - }; - -} - - -interface DbItem { - fkProject: number, - fkClass: number, - fkLanguage: number, - label: string -} - - -export const updateSql = ` -SELECT - fk_project "fkProject", - fk_dfh_class "fkClass", - fk_language "fkLanguage", - jsonb_build_object( 'label', string ) val -FROM - projects.text_property -WHERE - fk_dfh_class IS NOT NULL -AND - fk_system_type = 639 -AND - tmsp_last_modification > $1 -`; - - -const deleteSql = ` - SELECT - fk_project "fkProject", - fk_dfh_class "fkClass", - fk_language "fkLanguage" - FROM - projects.text_property_vt - WHERE - upper(sys_period) > $1 - - EXCEPT - - SELECT - fk_project "fkProject", - fk_dfh_class "fkClass", - fk_language "fkLanguage" - FROM - projects.text_property -` diff --git a/server/src/warehouse/primary-ds/ProEntityLabelConfigService.ts b/server/src/warehouse/primary-ds/ProEntityLabelConfigService.ts deleted file mode 100644 index 12e7d0f06..000000000 --- a/server/src/warehouse/primary-ds/ProEntityLabelConfigService.ts +++ /dev/null @@ -1,89 +0,0 @@ -import {PrimaryDataService} from '../base/classes/PrimaryDataService'; -import {PK_DEFAULT_CONFIG_PROJECT, Warehouse} from '../Warehouse'; -import {PClassId, pClassIdKeyDef} from './ProClassFieldsConfigService'; -import {Injectable, Inject, forwardRef} from 'injection-js'; - - -export interface LabelPart { - field?: { - fkProperty: number, - isOutgoing: boolean, - nrOfStatementsInLabel?: number - } - ordNum: number -} -export interface EntityLabelConfigVal { - labelParts?: LabelPart[] -} - - -@Injectable() -export class ProEntityLabelConfigService extends PrimaryDataService{ - - measure = 1000; - - - constructor(@Inject(forwardRef(() => Warehouse)) wh: Warehouse) { - super( - wh, - ['modified_projects_entity_label_config'], - pClassIdKeyDef - ) - } - - - /** - * returns entity label config of requested project, else of default config project - * @param classId - */ - async getEntityLabelConfig(classId: PClassId) { - let x = await this.index.getFromIdx(classId) - if (x) return x - x = await this.index.getFromIdx({ - fkProject: PK_DEFAULT_CONFIG_PROJECT, - pkClass: classId.pkClass - }) - return x - } - getUpdatesSql(tmsp: Date) { - return updateSql - } - getDeletesSql(tmsp: Date) { - return deleteSql - }; - -} - - -interface InitItem { - pkClass: number - fkProject: number - config: EntityLabelConfigVal -} - -const updateSql = ` - SELECT - fk_project "fkProject", - fk_class "pkClass", - config val - FROM - projects.entity_label_config - WHERE - tmsp_last_modification >= $1` - -const deleteSql = ` - SELECT - fk_project "fkProject", - fk_class "pkClass" - FROM - projects.entity_label_config_vt - WHERE - upper(sys_period) >= $1 - - EXCEPT - - SELECT - fk_project "fkProject", - fk_class "pkClass" - FROM - projects.entity_label_config` diff --git a/server/src/warehouse/primary-ds/ProProjectService.ts b/server/src/warehouse/primary-ds/ProProjectService.ts deleted file mode 100644 index 4cbd8fe3f..000000000 --- a/server/src/warehouse/primary-ds/ProProjectService.ts +++ /dev/null @@ -1,57 +0,0 @@ -import {PrimaryDataService} from '../base/classes/PrimaryDataService'; -import {KeyDefinition} from '../base/interfaces/KeyDefinition'; -import {Warehouse} from '../Warehouse'; -import {Injectable, Inject, forwardRef} from 'injection-js'; -export interface ProjectId { - pkProject: number -} -export interface ProjectVal { - fkLanguage: number -} -export const pProjectKeyDef: KeyDefinition[] = [ - {name: 'pkProject', type: 'integer'} -] -@Injectable() -export class ProProjectService extends PrimaryDataService{ - measure = 1000; - - constructor(@Inject(forwardRef(() => Warehouse)) wh: Warehouse) { - super( - wh, - ['modified_projects_project'], - pProjectKeyDef - ) - } - - getUpdatesSql(tmsp: Date) { - return updateSql - } - getDeletesSql(tmsp: Date) { - return deleteSql - }; -} - -const updateSql = ` - SELECT - pk_entity "pkProject", - jsonb_build_object('fkLanguage', fk_language) val - FROM - projects.project - WHERE - tmsp_last_modification >= $1 -` -const deleteSql = ` - SELECT - pk_entity "pkProject" - FROM - projects.project_vt - WHERE - upper(sys_period) >= $1 - - EXCEPT - - SELECT - pk_entity "pkProject" - FROM - projects.project -` diff --git a/server/src/warehouse/primary-ds/ProPropertyLabelService.ts b/server/src/warehouse/primary-ds/ProPropertyLabelService.ts deleted file mode 100644 index 72fc2f803..000000000 --- a/server/src/warehouse/primary-ds/ProPropertyLabelService.ts +++ /dev/null @@ -1,93 +0,0 @@ -import {Injectable, Inject, forwardRef} from 'injection-js'; -import {PrimaryDataService} from '../base/classes/PrimaryDataService'; -import {KeyDefinition} from '../base/interfaces/KeyDefinition'; -import {Warehouse} from '../Warehouse'; -export interface ProPropertyLabelId { - fkProject: number - fkProperty: number - fkClass: number - isOutgoing: boolean - fkLanguage: number -} -export interface ProPropertyLabelVal {label: string} -export const proPropertyLabelKeyDef: KeyDefinition[] = [ - {name: 'fkProject', type: 'integer'}, - {name: 'fkProperty', type: 'integer'}, - {name: 'fkClass', type: 'integer'}, - {name: 'isOutgoing', type: 'boolean'}, - {name: 'fkLanguage', type: 'integer'}, -] -@Injectable() -export class ProPropertyLabelService extends PrimaryDataService{ - measure = 1000; - - constructor(@Inject(forwardRef(() => Warehouse)) wh: Warehouse) { - super( - wh, - ['modified_projects_text_property'], - proPropertyLabelKeyDef - ) - } - getUpdatesSql(tmsp: Date) { - return updateSql - } - getDeletesSql(tmsp: Date) { - return deleteSql - }; -} - - -export const updateSql = ` -SELECT - fk_project "fkProject", - fk_dfh_property "fkProperty", - COALESCE(fk_dfh_property_domain, fk_dfh_property_range) "fkClass", - CASE WHEN fk_dfh_property_domain IS NOT NULL - THEN true - ELSE false - END "isOutgoing", - fk_language "fkLanguage", - jsonb_build_object('label', string) val -FROM - projects.text_property -WHERE - fk_dfh_property IS NOT NULL -AND - fk_system_type = 639 -AND - tmsp_last_modification >= $1 -`; - - -const deleteSql = ` -WITH tw1 AS ( - SELECT - fk_project "fkProject", - fk_dfh_property "fkProperty", - fk_dfh_property_domain "fkPropertyDomain", - fk_dfh_property_range "fkPropertyRange", - fk_language "fkLanguage" - FROM - projects.text_property_vt - WHERE - upper(sys_period) >= $1 - EXCEPT - - SELECT - fk_project "fkProject", - fk_dfh_property "fkProperty", - fk_dfh_property_domain "fkPropertyDomain", - fk_dfh_property_range "fkPropertyRange", - fk_language "fkLanguage" - FROM - projects.text_property - ) - SELECT - "fkProject", - "fkProperty", - COALESCE("fkPropertyDomain", "fkPropertyRange") "fkClass", - CASE WHEN "fkPropertyDomain" IS NOT NULL THEN true ELSE false END "isOutgoing", - "fkLanguage" - FROM - tw1 -` diff --git a/server/src/warehouse/primary-ds/RFieldChangeService.ts b/server/src/warehouse/primary-ds/RFieldChangeService.ts deleted file mode 100644 index 2b68d9bd3..000000000 --- a/server/src/warehouse/primary-ds/RFieldChangeService.ts +++ /dev/null @@ -1,146 +0,0 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -import {forwardRef, Inject, Injectable} from 'injection-js'; -import {PrimaryDataService} from '../base/classes/PrimaryDataService'; -import {KeyDefinition} from '../base/interfaces/KeyDefinition'; -import {Warehouse} from '../Warehouse'; -export interface PFieldChangeId { - fkProject: number; - - fkSourceInfo: number; - fkSourceTablesCell: number; - - fkProperty: number; - fkPropertyOfProperty: number; - - isOutgoing: boolean; - - -} -export interface PFieldChangeVal { - tmspLastModification: string -} -export const rStatementKeyDefs: KeyDefinition[] = [ - - {name: 'fkSourceInfo', type: 'integer'}, - {name: 'fkSourceTablesCell', type: 'integer'}, - - {name: 'fkProperty', type: 'integer'}, - {name: 'fkPropertyOfProperty', type: 'integer'}, - - {name: 'isOutgoing', type: 'boolean'}, -] - -@Injectable() -export class RFieldChangeService extends PrimaryDataService{ - - measure = 10000; - - - constructor(@Inject(forwardRef(() => Warehouse)) wh: Warehouse) { - super( - wh, - [ - 'modified_projects_info_proj_rel', - ], - rStatementKeyDefs - ) - this.registerUpdateReplication( - 'war.field_change', - (insertClause: string, fromClause: string) => ` - INSERT INTO war.field_change ( - fk_project, - fk_source_info, - fk_source_tables_cell, - fk_property, - fk_property_of_property, - is_outgoing, - tmsp_last_modification - ) - SELECT - 0, - t1."fkSourceInfo", - t1."fkSourceTablesCell", - t1."fkProperty", - t1."fkPropertyOfProperty", - t1."isOutgoing", - (t1.val->>'tmspLastModification')::timestamp with time zone - FROM ${fromClause} t1 - ON CONFLICT ( - fk_project, - fk_source_info, - fk_source_tables_cell, - fk_property, - fk_property_of_property, - is_outgoing - ) DO UPDATE - SET - fk_project = EXCLUDED.fk_project, - fk_source_info = EXCLUDED.fk_source_info, - fk_source_tables_cell = EXCLUDED.fk_source_tables_cell, - fk_property = EXCLUDED.fk_property, - fk_property_of_property = EXCLUDED.fk_property_of_property, - is_outgoing = EXCLUDED.is_outgoing, - tmsp_last_modification = EXCLUDED.tmsp_last_modification - ` - ) - } - - - getUpdatesSql(tmsp: Date) { - return updateSql - } - getDeletesSql(tmsp: Date) {return ''}; - -} - - -const updateSql = ` - -- select last modification tmsp of incoming fields - SELECT - json_build_object('tmspLastModification',max(t1.tmsp_last_modification)) val, - t2.fk_object_info "fkSourceInfo", - t2.fk_object_tables_cell "fkSourceTablesCell", - t2.fk_property "fkProperty", - t2.fk_property_of_property "fkPropertyOfProperty", - false "isOutgoing" - FROM - projects.info_proj_rel t1, - information."statement" t2 - WHERE - t1.fk_entity = t2.pk_entity - AND - (t2.fk_object_info != 0 OR t2.fk_object_tables_cell != 0) - AND - t1.tmsp_last_modification > $1 - GROUP BY - t2.fk_object_info, - t2.fk_object_tables_cell, - t2.fk_property, - t2.fk_property_of_property - - UNION ALL - - -- select last modification tmsp of outgoing fields - SELECT - json_build_object('tmspLastModification',max(t1.tmsp_last_modification)) val, - t2.fk_subject_info "fkSourceInfo", - t2.fk_subject_tables_cell "fkSourceTablesCell", - t2.fk_property "fkProperty", - t2.fk_property_of_property "fkPropertyOfProperty", - true "isOutgoing" - FROM - projects.info_proj_rel t1, - information."statement" t2 - WHERE - t1.fk_entity = t2.pk_entity - AND - (t2.fk_subject_info != 0 OR t2.fk_subject_tables_cell != 0) - AND - t1.tmsp_last_modification > $1 - GROUP BY - t2.fk_subject_info, - t2.fk_subject_tables_cell, - t2.fk_property, - t2.fk_property_of_property; -` - diff --git a/server/src/warehouse/primary-ds/class/PClassService.ts b/server/src/warehouse/primary-ds/class/PClassService.ts deleted file mode 100644 index 610e526ac..000000000 --- a/server/src/warehouse/primary-ds/class/PClassService.ts +++ /dev/null @@ -1,112 +0,0 @@ -import {forwardRef, Inject, Injectable} from 'injection-js'; -import {PrimaryDataService} from '../../base/classes/PrimaryDataService'; -import {Warehouse} from '../../Warehouse'; -import {pClassIdKeyDef} from '../ProClassFieldsConfigService'; -export interface PClassId {fkProject: number, pkClass: number} -export interface PClassVal { - fkClass: number - fkProject: number - basicType: number -} - -@Injectable() -export class PClassService extends PrimaryDataService{ - - measure = 1000; - - constructor(@Inject(forwardRef(() => Warehouse)) wh: Warehouse) { - super( - wh, - [ - 'modified_projects_project', - 'modified_projects_dfh_profile_proj_rel', - 'modified_data_for_history_api_class', - 'modified_system_config' - ], - pClassIdKeyDef - ) - - /** - * Add actions after a new ProjectClass is put/updated into index - */ - // this.afterPut$.subscribe(item => { - // // Add update requests on aggregaters based on project class - // wh.agg.pClassLabel.updater.addItemToQueue(item.key).catch(e => console.log(e)) - - - // // Generate incoming class field for 'has appellation' property 1111 - // if ([8, 9, 30].includes(item.val.basicType)) { - // const incomingField: PClassFieldId = { - // fkProject: item.key.fkProject, - // fkClass: item.val.fkClass, - // fkProperty: 1111, - // isOutgoing: false - // } - // wh.agg.pClassFieldLabel.updater.addItemToQueue(incomingField).catch(e => console.log(e)) - // } - // }) - - - - } - - - getUpdatesSql(tmsp: Date) { - return updateSql - } - getDeletesSql(tmsp: Date) { - return deleteSql - }; - -} - - - -const updateSql = ` - WITH tw1 AS ( - SELECT fk_profile, fk_project, enabled, tmsp_last_modification - FROM projects.dfh_profile_proj_rel - WHERE enabled = true - UNION ALL - SELECT fk_profile, pk_entity as fk_project, true, null - FROM projects.project, - ( - SELECT jsonb_array_elements_text(config->'ontome'->'requiredOntomeProfiles')::int fk_profile - FROM system.config - ) as requiredProfiles - ) - SELECT DISTINCT ON (dfh_pk_class, fk_project) - dfh_pk_class "pkClass", - fk_project "fkProject", - jsonb_build_object( - 'fkClass', dfh_pk_class, - 'fkProject', fk_project, - 'basicType', dfh_basic_type - ) val - FROM - tw1 t1, - data_for_history.api_class t2 - WHERE t1.fk_profile = t2.dfh_fk_profile - AND ( - t1.tmsp_last_modification >= $1 - OR - t2.tmsp_last_modification >= $1 - ) -` -export const deleteSql = ` - WITH tw1 AS ( - SELECT fk_profile, fk_project, enabled, tmsp_last_modification - FROM projects.dfh_profile_proj_rel - WHERE enabled = false - AND tmsp_last_modification >= $1 - ) - SELECT DISTINCT - dfh_pk_class "pkClass", - fk_project "fkProject" - FROM - tw1 t1, - data_for_history.api_class t2 - WHERE t1.fk_profile = t2.dfh_fk_profile -` - - diff --git a/server/src/warehouse/primary-ds/class/RClassService.ts b/server/src/warehouse/primary-ds/class/RClassService.ts deleted file mode 100644 index 392ccb2ad..000000000 --- a/server/src/warehouse/primary-ds/class/RClassService.ts +++ /dev/null @@ -1,75 +0,0 @@ -import {PrimaryDataService} from '../../base/classes/PrimaryDataService'; -import {KeyDefinition} from '../../base/interfaces/KeyDefinition'; -import {Warehouse} from '../../Warehouse'; -import {RClassId} from '../DfhClassHasTypePropertyService'; -import {Injectable, Inject, forwardRef} from 'injection-js'; - - -const keyDefs: KeyDefinition[] = [ - { - name: 'pkClass', - type: 'integer' - } -] -@Injectable() -export class RClassService extends PrimaryDataService{ - - measure = 1000; - - constructor(@Inject(forwardRef(() => Warehouse)) wh: Warehouse) { - super( - wh, - [ - 'modified_data_for_history_api_class' - ], - keyDefs - ) - - /** - * Add actions after a new RClass is put/updated into index - */ - // this.afterPut$.subscribe(item => { - // // Add update requests on aggregaters based on project class - // // wh.agg.rClassLabel.updater.addItemToQueue(item.key).catch(e => console.log(e)) - // // Generate incoming class field for 'has appellation' property 1111 - // if ([8, 9, 30].includes(item.val.basicType)) { - // const incomingField: RClassFieldId = { - // fkClass: item.val.fkClass, - // fkProperty: 1111, - // isOutgoing: false - // } - // wh.agg.rClassFieldLabel.updater.addItemToQueue(incomingField).catch(e => console.log(e)) - // } - // }) - - - } - - getUpdatesSql(tmsp: Date) { - return updateSql - } - getDeletesSql(tmsp: Date) {return ''} - -} - - -const updateSql = ` - SELECT DISTINCT ON (dfh_pk_class) - dfh_pk_class "pkClass", - jsonb_build_object( - 'fkClass', dfh_pk_class, - 'basicType', dfh_basic_type - ) val - FROM - data_for_history.api_class t1 - WHERE - t1.tmsp_last_modification >= $1 -` - -export interface RClass { - fkClass: number - basicType: number -} - - - diff --git a/server/src/warehouse/primary-ds/edge/PEdgeService.ts b/server/src/warehouse/primary-ds/edge/PEdgeService.ts deleted file mode 100644 index ee7573a01..000000000 --- a/server/src/warehouse/primary-ds/edge/PEdgeService.ts +++ /dev/null @@ -1,237 +0,0 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -import {forwardRef, Inject, Injectable} from 'injection-js'; -import {sum} from 'lodash'; -import {PoolClient} from 'pg'; -import {Logger} from '../../base/classes/Logger'; -import {PgDataReplicator} from '../../base/classes/PgDataReplicator'; -import {PrimaryDataService} from '../../base/classes/PrimaryDataService'; -import {Warehouse} from '../../Warehouse'; -import {PEntityId, pEntityKeyDefs} from '../entity/PEntityService'; -import {buildIncomingEdges, buildOutgoingEdges, EntityFields} from './edge.commons'; - -@Injectable() -export class PEdgeService extends PrimaryDataService{ - - measure = 10000; - - - constructor(@Inject(forwardRef(() => Warehouse)) wh: Warehouse) { - super( - wh, - [ - 'modified_projects_info_proj_rel', - ], - pEntityKeyDefs - ) - } - - - getUpdatesSql(tmsp: Date) {return ''} - getDeletesSql(tmsp: Date) {return ''}; - - async manageUpdatesSince(pool1: PoolClient, pool2: PoolClient, date: Date = new Date(0)) { - - const t2 = Logger.start(this.constructor.name, `Execute update query ...`, 2); - - const tmpTable = `${this.constructor.name}_update_tmp` - - const stats = await new PgDataReplicator<{count: number}>( - {client: pool1, table: tmpTable}, - {client: pool2, table: this.index.schemaTable}, - [this.index.keyCols, 'val'], - (insertClause, fromClause) => ` - WITH tw1 AS ( - ${insertClause} - ${fromClause} - ON CONFLICT (${this.index.keyCols}) DO UPDATE - SET val = EXCLUDED.val - WHERE ${this.index.schemaTable}.val <> EXCLUDED.val - RETURNING * - ) - SELECT count(*)::int FROM tw1 - ` - ).replicateBatch( - date, - countSql, - updateBatchSql, - ) - const upserted = sum(stats.map(s => s.rows?.[0].count)) - - Logger.itTook(this.constructor.name, t2, `to update Primary Data Service with ${upserted} new lines`, 2); - - if (this.updateReplications.length > 0) { - const replicationRequest = this.updateReplications.map(repl => { - return new PgDataReplicator<{count: number}>( - {client: pool1, table: tmpTable}, - {client: pool1, table: repl.targetTable}, - [this.index.keyCols, 'val'], - repl.sqlFn - ).replicateTable() - }) - await Promise.all(replicationRequest) - } - - return upserted - - } - -} - - -const twIds = ` - WITH tw AS ( - -- select affected entities - SELECT - t2.fk_subject_info pk_entity, - t1.fk_project - FROM - projects.info_proj_rel t1 - JOIN - information."statement" t2 ON t1.fk_entity = t2.pk_entity - JOIN - information.resource t3 ON t2.fk_subject_info = t3.pk_entity - WHERE - t1.tmsp_last_modification > $1 - UNION ALL - SELECT - t2.fk_object_info pk_entity, - t1.fk_project - FROM - projects.info_proj_rel t1 - JOIN - information."statement" t2 ON t1.fk_entity = t2.pk_entity - JOIN - information.resource t3 ON t2.fk_object_info = t3.pk_entity - WHERE - t1.tmsp_last_modification > $1 - UNION ALL - SELECT - t2.pk_entity, - t1.fk_project - FROM - projects.info_proj_rel t1 - JOIN - information.resource t2 ON t1.fk_entity = t2.pk_entity - WHERE - t1.tmsp_last_modification > $1 - - ) -` -const countSql = ` - ${twIds}, - tw1 AS ( - SELECT fk_project, pk_entity - FROM tw - GROUP BY fk_project, pk_entity - ) - select count(*):: integer - from tw1 -` -const updateBatchSql = (limit: number, offset: number) => ` - ${twIds}, -tw0 AS ( - SELECT fk_project, pk_entity - FROM tw - GROUP BY fk_project, pk_entity - LIMIT ${limit} OFFSET ${offset} -), -tw1 AS ( - SELECT DISTINCT - t1.fk_project, - t1.ord_num_of_domain, - t1.ord_num_of_range, - t2.pk_entity as pk_statement, - t2.fk_property, - t2.fk_subject_info, - t8.table_name subject_table, - t2.fk_object_info, - t9.table_name object_table, - t3.string as appellation, - t4.notes as language, - t7.string as lang_string, - t6.julian_day, - t6.duration, - t6.calendar - FROM - tw0 t0 - JOIN projects.info_proj_rel t1 ON t1.fk_project = t0.fk_project - JOIN information."statement" t2 ON t1.fk_entity = t2.pk_entity - JOIN information.entity t8 ON t8.pk_entity = t2.fk_subject_info - JOIN information.entity t9 ON t9.pk_entity = t2.fk_object_info - LEFT JOIN information.appellation t3 ON t3.pk_entity = t2.fk_object_info - LEFT JOIN information.language t4 ON t4.pk_entity = t2.fk_object_info - LEFT JOIN information.place t5 ON t5.pk_entity = t2.fk_object_info - LEFT JOIN information.time_primitive t6 ON t6.pk_entity = t2.fk_object_info - LEFT JOIN information.lang_string t7 ON t7.pk_entity = t2.fk_object_info - WHERE - ( - t2.fk_subject_info = t0.pk_entity - OR - t2.fk_object_info = t0.pk_entity - ) AND - t1.is_in_project=true - AND t2.fk_object_info IS NOT NULL - - -- TODO: Remove these where clauses as soon as the left joined value object - -- table is implemented (below in json) - AND t4.pk_entity IS NULL - AND t5.pk_entity IS NULL -), --- outgoing -tw2 AS ( - SELECT - fk_project, - fk_property, - fk_subject_info pk_entity, - json_agg( - ${buildOutgoingEdges} - ORDER BY t1.ord_num_of_range ASC - ) outgoing - FROM tw1 t1 - WHERE t1.subject_table IN ('resource') - GROUP BY fk_project, fk_property, fk_subject_info - ORDER BY fk_project, fk_property, fk_subject_info -), --- incoming -tw3 AS ( - SELECT - fk_project, - fk_property, - fk_object_info pk_entity, - json_agg( - ${buildIncomingEdges} - ORDER BY t1.ord_num_of_domain ASC - ) incoming - FROM tw1 t1 - WHERE t1.object_table IN ('resource') - GROUP BY fk_project, fk_property, fk_object_info - ORDER BY fk_project, fk_property, fk_object_info -), -tw4 AS ( - SELECT fk_project, fk_property, pk_entity, outgoing, NULL::json incoming - FROM tw2 - UNION ALL - SELECT fk_project, fk_property, pk_entity, NULL::json outgoing, incoming - FROM tw3 -), -tw5 AS ( - SELECT - fk_project, - pk_entity, - json_build_object( - 'outgoing', json_strip_nulls(json_object_agg(fk_property, outgoing)), - 'incoming', json_strip_nulls(json_object_agg(fk_property, incoming)) - ) fields - FROM tw4 - - GROUP BY - fk_project, - pk_entity -) -SELECT -t1.fk_project "fkProject", -t1.pk_entity "pkEntity", -COALESCE(t2.fields, '{}'::json) val -FROM tw0 t1 -LEFT JOIN tw5 t2 ON t1.pk_entity = t2.pk_entity AND t1.fk_project = t2.fk_project -` diff --git a/server/src/warehouse/primary-ds/edge/REdgeService.ts b/server/src/warehouse/primary-ds/edge/REdgeService.ts deleted file mode 100644 index 5cc15fa59..000000000 --- a/server/src/warehouse/primary-ds/edge/REdgeService.ts +++ /dev/null @@ -1,296 +0,0 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -import {forwardRef, Inject, Injectable} from 'injection-js'; -import {sum} from 'lodash'; -import {PoolClient} from 'pg'; -import {Logger} from '../../base/classes/Logger'; -import {PgDataReplicator} from '../../base/classes/PgDataReplicator'; -import {PrimaryDataService} from '../../base/classes/PrimaryDataService'; -import {Warehouse} from '../../Warehouse'; -import {REntityId, rEntityKeyDefs} from '../entity/REntityService'; -import {EntityFields} from "./edge.commons"; - - -@Injectable() -export class REdgeService extends PrimaryDataService{ - - measure = 10000; - replicationBatchSizeUpdates = 1000; - replicationBatchSizeDeletes = 1000; - - constructor(@Inject(forwardRef(() => Warehouse)) wh: Warehouse) { - super( - wh, - [ - 'modified_projects_info_proj_rel', - ], - rEntityKeyDefs - ) - } - - - getUpdatesSql(tmsp: Date) {return ''} - getDeletesSql(tmsp: Date) {return ''}; - - async manageUpdatesSince(pool1: PoolClient, pool2: PoolClient, date: Date = new Date(0)) { - - const t2 = Logger.start(this.constructor.name, `Execute update query ...`, 2); - - const tmpTable = `${this.constructor.name}_update_tmp` - - const stats = await new PgDataReplicator<{count: number}>( - {client: pool1, table: tmpTable}, - {client: pool2, table: this.index.schemaTable}, - [this.index.keyCols, 'val'], - (insertClause, fromClause) => ` - WITH tw1 AS ( - ${insertClause} - ${fromClause} - ON CONFLICT (${this.index.keyCols}) DO UPDATE - SET val = EXCLUDED.val - WHERE ${this.index.schemaTable}.val <> EXCLUDED.val - RETURNING * - ) - SELECT count(*)::int FROM tw1 - ` - ).replicateBatch( - date, - countSql, - updateBatchSql, - ) - const upserted = sum(stats.map(s => s.rows?.[0].count)) - - Logger.itTook(this.constructor.name, t2, `to update Primary Data Service with ${upserted} new lines`, 2); - - if (this.updateReplications.length > 0) { - const replicationRequest = this.updateReplications.map(repl => { - return new PgDataReplicator<{count: number}>( - {client: pool1, table: tmpTable}, - {client: pool1, table: repl.targetTable}, - [this.index.keyCols, 'val'], - repl.sqlFn - ).replicateTable() - }) - await Promise.all(replicationRequest) - } - - return upserted - - } - -} - -const twIds = ` - WITH tw AS ( - -- select affected entities - SELECT - DISTINCT t2.fk_subject_info pk_entity - FROM - projects.info_proj_rel t1 - JOIN information. "statement" t2 ON t1.fk_entity = t2.pk_entity - JOIN information.resource t3 ON t2.fk_subject_info = t3.pk_entity - WHERE - t1.tmsp_last_modification >= $1 - UNION - SELECT - DISTINCT t2.fk_object_info pk_entity - FROM - projects.info_proj_rel t1 - JOIN information. "statement" t2 ON t1.fk_entity = t2.pk_entity - JOIN information.resource t3 ON t2.fk_object_info = t3.pk_entity - WHERE - t1.tmsp_last_modification >= $1 - UNION - SELECT - DISTINCT t2.pk_entity - FROM - projects.info_proj_rel t1 - JOIN information.resource t2 ON t1.fk_entity = t2.pk_entity - WHERE - t1.tmsp_last_modification >= $1 - ) -` - -const countSql = ` - ${twIds} - select count(*):: integer - from tw -` - -const updateBatchSql = (limit: number, offset: number) => ` - ${twIds}, -tw0 AS ( - SELECT pk_entity - FROM tw - LIMIT ${limit} OFFSET ${offset} -), -tw1 AS ( - SELECT - t2.pk_entity as pk_statement, - t2.fk_property, - t2.fk_subject_info, - t2.fk_object_info, - count(t1.fk_project) is_in_project_count, - AVG(t1.ord_num_of_range) :: numeric(10, 2) ord_num_of_range, - (t10.pk_entity IS NOT NULL) target_is_entity, - COALESCE(t12.string, t14.notes, t15.string) target_label, - t12.string appellation_str, - t14.notes language_str, - t13.julian_day, - t13.duration, - t13.calendar - FROM - tw0 t0 - JOIN information."statement" t2 ON t2.fk_subject_info = t0.pk_entity - - LEFT JOIN information.resource t8 ON t8.pk_entity = t2.fk_subject_info - - LEFT JOIN information.resource t10 ON t10.pk_entity = t2.fk_object_info - LEFT JOIN information.appellation t12 ON t12.pk_entity = t2.fk_object_info - LEFT JOIN information.time_primitive t13 ON t13.pk_entity = t2.fk_object_info - LEFT JOIN information.language t14 ON t14.pk_entity = t2.fk_object_info - LEFT JOIN information.lang_string t15 ON t15.pk_entity = t2.fk_object_info - - JOIN projects.info_proj_rel t1 ON t1.fk_entity = t2.pk_entity - AND t1.is_in_project = true - - WHERE - -- subject is not null - t8.pk_entity IS NOT NULL - AND - -- object is not null - ( - t10.pk_entity IS NOT NULL - OR t12.pk_entity IS NOT NULL - OR t13.pk_entity IS NOT NULL - OR t14.pk_entity IS NOT NULL - OR t15.pk_entity IS NOT NULL - ) - GROUP BY - t2.pk_entity, - t2.fk_property, - t2.fk_subject_info, - t2.fk_object_info, - t10.pk_entity, - t12.string, - t14.notes, - t15.string, - t13.julian_day, - t13.duration, - t13.calendar -), -tw2 AS ( - - SELECT - t1.fk_subject_info pk_entity, - t1.fk_property, - json_agg( - json_build_object( - 'fkProperty', t1.fk_property, - 'isOutgoing', true, - 'fkStatement', t1.pk_statement, - 'fkSource', t1.fk_subject_info, - 'fkTarget', t1.fk_object_info, - 'ordNumWithinField', t1.ord_num_of_range, - 'targetIsEntity', t1.target_is_entity, - 'targetLabel', t1.target_label, - 'targetValue', - json_strip_nulls( - json_build_object( - 'appellation', t1.appellation_str, - 'language', t1.language_str, - 'timePrimitive', - CASE - WHEN t1.julian_day IS NOT NULL THEN json_strip_nulls( - json_build_object( - 'julianDay', t1.julian_day, - 'duration', t1.duration, - 'calendar', t1.calendar, - 'firstSecond', commons.time_primitive__get_first_second(t1.julian_day), - 'lastSecond', commons.time_primitive__get_last_second(t1.julian_day, t1.duration, t1.calendar) - ) - ) - ELSE null - END - ) - ) - ) - ORDER BY - t1.ord_num_of_range ASC, - t1.is_in_project_count DESC - ) outgoing - from tw1 t1 - GROUP BY - t1.fk_property, - t1.fk_subject_info -), -tw3 AS ( - SELECT - t2.pk_entity as pk_statement, - t2.fk_property, - t2.fk_subject_info, - t2.fk_object_info, - count(t1.fk_project) is_in_project_count, - AVG(t1.ord_num_of_domain) :: numeric(10, 2) ord_num_of_domain, - (t10.pk_entity IS NOT NULL) target_is_entity - FROM - tw0 t0 - JOIN information."statement" t2 ON t2.fk_object_info = t0.pk_entity - - LEFT JOIN information.resource t8 ON t8.pk_entity = t2.fk_object_info - - LEFT JOIN information.resource t10 ON t10.pk_entity = t2.fk_subject_info - - JOIN projects.info_proj_rel t1 ON t1.fk_entity = t2.pk_entity - AND t1.is_in_project = true - - WHERE - t8.pk_entity IS NOT NULL - AND - t10.pk_entity IS NOT NULL - GROUP BY - t2.pk_entity, - t2.fk_property, - t2.fk_subject_info, - t2.fk_object_info, - t10.pk_entity -), -tw4 AS ( - - SELECT - t1.fk_object_info pk_entity, - t1.fk_property, - json_agg( - json_build_object( - 'fkProperty', t1.fk_property, - 'isOutgoing', false, - 'fkStatement', t1.pk_statement, - 'fkSource', t1.fk_object_info, - 'fkTarget', t1.fk_subject_info, - 'ordNumWithinField', t1.ord_num_of_domain, - 'targetIsEntity', t1.target_is_entity - ) - ORDER BY - t1.ord_num_of_domain ASC, - t1.is_in_project_count DESC - ) incoming - from tw3 t1 - GROUP BY - t1.fk_property, - t1.fk_object_info -) - SELECT - t1.pk_entity "pkEntity", - json_build_object( - 'outgoing', - json_strip_nulls( COALESCE(json_object_agg(t2.fk_property, t2.outgoing) FILTER (WHERE t2.fk_property IS NOT NULL), '{}') ), - 'incoming', - json_strip_nulls( COALESCE(json_object_agg(t3.fk_property, t3.incoming) FILTER (WHERE t3.fk_property IS NOT NULL), '{}') ) - ) val - FROM - tw0 t1 - LEFT JOIN tw2 t2 ON t1.pk_entity = t2.pk_entity - LEFT JOIN tw4 t3 ON t1.pk_entity = t3.pk_entity - GROUP BY - t1.pk_entity - ` - diff --git a/server/src/warehouse/primary-ds/edge/edge.commons.ts b/server/src/warehouse/primary-ds/edge/edge.commons.ts deleted file mode 100644 index 54eab2901..000000000 --- a/server/src/warehouse/primary-ds/edge/edge.commons.ts +++ /dev/null @@ -1,122 +0,0 @@ -interface DirectedFields { - [pkPropery: string]: Edge[]; -} -export interface EntityFields { - outgoing: DirectedFields; - incoming: DirectedFields; -} -// This interface is to ease creation of Mock data -export interface StatementItemToIndexate { - fk_project: number; - ord_num_of_domain: number; - ord_num_of_range: number; - pk_statement: number; - fk_property: number; - fk_subject_info: number; - subject_table: string; - fk_object_info: number; - object_table: string; - appellation: string | null; - language: string | null; - lang_string: string | null; -} - -export interface EdgeInitItem { - fkProject: number; - pkEntity: number; - fields: EntityFields; -} - -export interface Edge { - - // model information - // fkSubject: number - fkProperty: number; - // fkObject: number - isOutgoing: boolean; - - // instance level reference - fkStatement: number; - - // instance level values - fkSource: number; - fkTarget: number; - ordNumWithinField?: number; - - // true, if target is another entity (TeEn/PeIt) - targetIsEntity: boolean; - - // if targetValue is set, the target is a value object (literal like) - targetLabel?: string; - targetValue?: { - appellation?: string | null; - dimension?: any; - place?: any; - timePrimitive?: EntityTimePrimitiveWithBoundaries | null; - language?: any; - langString?: any; - }; -} -export type CalendarType = 'gregorian' | 'julian'; -export type Granularity = - '1 century' | - '1 decade' | - '1 year' | - '1 month' | - '1 day' | - '1 hour' | - '1 minute' | - '1 second'; -export interface EntityTimePrimitive { - julianDay?: number; - duration?: Granularity; - calendar?: CalendarType; -} -export interface EntityTimePrimitiveWithBoundaries extends EntityTimePrimitive { - firstSecond?: number; - lastSecond?: number; -} - - -export const edgeSqlTargetLabel = `'targetLabel', COALESCE( t1.appellation, t1.language, t1.lang_string)` - -export const edgeSqlTargetValue = `'targetValue', json_strip_nulls(json_build_object( - 'appellation', t1.appellation, - 'language', t1.language, - 'langString', t1.lang_string, - 'timePrimitive', CASE WHEN t1.julian_day IS NOT NULL THEN - json_strip_nulls(json_build_object( - 'julianDay', t1.julian_day, - 'duration', t1.duration, - 'calendar', t1.calendar, - 'firstSecond', commons.time_primitive__get_first_second(t1.julian_day), - 'lastSecond', commons.time_primitive__get_last_second(t1.julian_day,t1.duration,t1.calendar) - )) - ELSE - null - END -))` - -export const buildOutgoingEdges = `json_build_object( - 'fkProperty', t1.fk_property, - 'isOutgoing', true, - 'fkStatement', t1.pk_statement, - 'fkSource', t1.fk_subject_info, - 'fkTarget', t1.fk_object_info, - 'ordNumWithinField', t1.ord_num_of_range, - 'targetIsEntity', t1.object_table IN ('resource'), - ${edgeSqlTargetLabel}, - ${edgeSqlTargetValue} -)` - -export const buildIncomingEdges = `json_build_object( - 'fkProperty', t1.fk_property, - 'isOutgoing', false, - 'fkStatement', t1.pk_statement, - 'fkSource', t1.fk_object_info, - 'fkTarget', t1.fk_subject_info, - 'ordNumWithinField', t1.ord_num_of_domain, - 'targetIsEntity', t1.subject_table IN ('resource'), - ${edgeSqlTargetLabel}, - ${edgeSqlTargetValue} -)` diff --git a/server/src/warehouse/primary-ds/entity/PEntityService.ts b/server/src/warehouse/primary-ds/entity/PEntityService.ts deleted file mode 100644 index fde5726b1..000000000 --- a/server/src/warehouse/primary-ds/entity/PEntityService.ts +++ /dev/null @@ -1,158 +0,0 @@ -import {PrimaryDataService} from '../../base/classes/PrimaryDataService'; -import {KeyDefinition} from '../../base/interfaces/KeyDefinition'; -import {Warehouse} from '../../Warehouse'; -import {Injectable, Inject, forwardRef} from 'injection-js'; - -export interface PEntityId {fkProject: number, pkEntity: number} -export const pEntityKeyDefs: KeyDefinition[] = [ - { - name: 'pkEntity', - type: 'integer' - }, - { - name: 'fkProject', - type: 'integer' - } -] -@Injectable() -export class PEntityService extends PrimaryDataService< PEntityId, PEntity>{ - - measure = 1000; - - constructor(@Inject(forwardRef(() => Warehouse)) wh: Warehouse) { - super(wh, [ - 'modified_projects_info_proj_rel', - 'modified_information_resource' - ], - pEntityKeyDefs - ) - - - /** - * Remove entity preview from db - */ - // this.afterDel$.subscribe(item => { - // wh.agg.pEntityLabel.del(item).catch(e => console.log(e)) - // wh.agg.pEntityClassLabel.del(item).catch(e => console.log(e)) - // wh.agg.pEntityType.del(item).catch(e => console.log(e)) - // wh.agg.pEntityTimeSpan.del(item).catch(e => console.log(e)) - // wh.agg.pEntityFullText.del(item).catch(e => console.log(e)) - - // this.deleteEntityPreview(item).catch(e => console.log(e)) - // }) - - - } - - getUpdatesSql(tmsp: Date) { - return updateSql - } - getDeletesSql(tmsp: Date) { - return deleteSql - }; - get2ndUpdatesSql(tableAlias: string) { - return ` - INSERT INTO war.entity_preview (pk_entity, fk_project, project, fk_class, entity_type) - SELECT "pkEntity", "fkProject", "fkProject", "fkClass", "entityType" - FROM ${tableAlias} - ON CONFLICT (pk_entity, project) DO UPDATE - SET fk_class = EXCLUDED.fk_class, entity_type = EXCLUDED.entity_type - WHERE EXCLUDED.fk_class IS DISTINCT FROM war.entity_preview.fk_class - OR EXCLUDED.entity_type IS DISTINCT FROM war.entity_preview.entity_type - ` - } - get2ndDeleteSql(tableAlias: string) { - return ` - DELETE FROM war.entity_preview - USING ${tableAlias} - WHERE pk_entity = ${tableAlias}."pkEntity" - AND fk_project = ${tableAlias}."fkProject" - ` - } - dbItemToKeyVal = undefined -} - - - -export const updateSql = ` -WITH tw1 AS ( - - SELECT - t1.fk_project, - t2.pk_entity, - t2.fk_class, - CASE WHEN t3.basic_type IN (8,30) THEN 'peIt' - ELSE 'teEn' END as "entity_type" - FROM - projects.info_proj_rel t1 - JOIN information.resource t2 ON t1.fk_entity = t2.pk_entity - JOIN data_for_history.v_class t3 ON t2.fk_class = t3.pk_class - WHERE t1.is_in_project=true - AND t1.tmsp_last_modification >= $1 - - UNION - - SELECT - t1.fk_project, - t2.pk_entity, - t2.fk_class, - CASE WHEN t3.basic_type IN (8,30) THEN 'peIt' - ELSE 'teEn' END as "entity_type" - FROM - projects.info_proj_rel t1 - JOIN information.resource t2 ON t1.fk_entity = t2.pk_entity - JOIN data_for_history.v_class t3 ON t2.fk_class = t3.pk_class - WHERE t1.is_in_project=true - AND t2.tmsp_last_modification >= $1 - -), -tw2 AS ( - INSERT INTO war.entity_preview (pk_entity, fk_project, project, fk_class, entity_type) - SELECT pk_entity, fk_project, fk_project, fk_class, entity_type - FROM tw1 - ON CONFLICT (pk_entity, project) DO UPDATE - SET fk_class = EXCLUDED.fk_class, entity_type = EXCLUDED.entity_type - WHERE EXCLUDED.fk_class IS DISTINCT FROM war.entity_preview.fk_class - OR EXCLUDED.entity_type IS DISTINCT FROM war.entity_preview.entity_type -) -SELECT - tw1.fk_project "fkProject", - tw1.pk_entity "pkEntity", - tw1.fk_class "fkClass", - tw1.entity_type "entityType", - jsonb_build_object( - 'pkEntity', tw1.pk_entity, - 'fkProject', tw1.fk_project, - 'fkClass', tw1.fk_class, - 'entityType', tw1.entity_type - ) val -FROM tw1 - ` -export const deleteSql = ` -WITH tw1 AS ( - SELECT - t1.fk_project "fkProject", - t2.pk_entity "pkEntity" - FROM - projects.info_proj_rel t1 - JOIN information.resource t2 ON t1.fk_entity = t2.pk_entity - WHERE t1.is_in_project=false - AND t1.tmsp_last_modification >= $1 -), -tw2 AS ( - DELETE FROM war.entity_preview - USING tw1 - WHERE pk_entity = tw1."pkEntity" - AND fk_project = tw1."fkProject" -) -SELECT * FROM tw1 -` -export interface PEntity { - pkEntity: number - fkClass: number - fkProject: number - entityType: 'peIt' | 'teEn' -} - - - diff --git a/server/src/warehouse/primary-ds/entity/REntityService.ts b/server/src/warehouse/primary-ds/entity/REntityService.ts deleted file mode 100644 index 675e3c556..000000000 --- a/server/src/warehouse/primary-ds/entity/REntityService.ts +++ /dev/null @@ -1,144 +0,0 @@ -import {forwardRef, Inject, Injectable} from 'injection-js'; -import {PrimaryDataService} from '../../base/classes/PrimaryDataService'; -import {KeyDefinition} from '../../base/interfaces/KeyDefinition'; -import {Warehouse} from '../../Warehouse'; - -export interface REntity { - pkEntity: number - fkClass: number - entityType: 'peIt' | 'teEn' - isInProjectCount: number -} -export interface REntityId {pkEntity: number} -export const rEntityKeyDefs: KeyDefinition[] = [ - {name: 'pkEntity', type: 'integer'} -] -@Injectable() -export class REntityService extends PrimaryDataService{ - - measure = 1000; - - constructor(@Inject(forwardRef(() => Warehouse)) wh: Warehouse) { - super( - wh, - [ - 'modified_projects_info_proj_rel', - 'modified_information_resource' - ], - rEntityKeyDefs - ) - - - /** - * Add actions after a new ProjectEntity is put/updated into index - */ - // this.afterPut$.subscribe(item => { - // // exclude dirty data - // if (item.val.fkClass) { - - // // Add update requests on aggregaters based on project entity - // // wh.agg.rEntityLabel.updater.addItemToQueue(item.key).catch(e => console.log(e)) - // // wh.agg.rEntityClassLabel.updater.addItemToQueue(item.key).catch(e => console.log(e)) - // // wh.agg.rEntityType.updater.addItemToQueue(item.key).catch(e => console.log(e)) - // // if (item.val.entityType === 'teEn') wh.agg.rEntityTimeSpan.updater.addItemToQueue(item.key).catch(e => console.log(e)) - // // wh.agg.rEntityFullText.updater.addItemToQueue(item.key).catch(e => console.log(e)) - - // // Add item to queue to upsert it into db - // this.upsertQueue.add(item) - // } - // }) - - } - - getUpdatesSql(tmsp: Date) { - return updateSql - } - getDeletesSql(tmsp: Date) {return deleteSql}; - - get2ndUpdatesSql(tableAlias: string) { - return ` - INSERT INTO war.entity_preview (pk_entity, fk_project, project, fk_class, entity_type) - SELECT "pkEntity", null, 0, fk_class, entity_type - FROM ${tableAlias} - ON CONFLICT (pk_entity, project) DO UPDATE - SET fk_class = EXCLUDED.fk_class, entity_type = EXCLUDED.entity_type - WHERE EXCLUDED.fk_class IS DISTINCT FROM war.entity_preview.fk_class OR EXCLUDED.entity_type IS DISTINCT FROM war.entity_preview.entity_type` - } - - -} - - - -export const updateSql = ` -WITH tw1 AS ( - SELECT - t2.pk_entity, - t2.fk_class, - CASE WHEN t3.basic_type IN (8,30) THEN 'peIt' - ELSE 'teEn' END as "entity_type" - FROM - projects.info_proj_rel t1 - JOIN information.resource t2 ON t1.fk_entity = t2.pk_entity - AND (t2.community_visibility->>'toolbox')::boolean=true - JOIN data_for_history.v_class t3 ON t2.fk_class = t3.pk_class - AND t1.tmsp_last_modification >= $1 - - UNION - - SELECT - t2.pk_entity, - t2.fk_class, - CASE WHEN t3.basic_type IN (8,30) THEN 'peIt' - ELSE 'teEn' END as "entity_type" - FROM information.resource t2 - JOIN data_for_history.v_class t3 ON t2.fk_class = t3.pk_class - WHERE t2.tmsp_last_modification >= $1 - AND (t2.community_visibility->>'toolbox')::boolean=true - ), - tw2 AS ( - INSERT INTO war.entity_preview (pk_entity, fk_project, project, fk_class, entity_type) - SELECT pk_entity, null, 0, fk_class, entity_type - FROM tw1 - ON CONFLICT (pk_entity, project) DO UPDATE - SET fk_class = EXCLUDED.fk_class, entity_type = EXCLUDED.entity_type - WHERE EXCLUDED.fk_class IS DISTINCT FROM war.entity_preview.fk_class OR EXCLUDED.entity_type IS DISTINCT FROM war.entity_preview.entity_type - - ) - SELECT - tw1.pk_entity "pkEntity", - tw1.fk_class, - tw1.entity_type, - jsonb_build_object( - 'pkEntity', tw1.pk_entity, - 'fkClass', tw1.fk_class, - 'entityType', tw1.entity_type, - 'isInProjectCount', count(t2.pk_entity) - ) val - FROM tw1 - LEFT JOIN projects.info_proj_rel t2 - ON tw1.pk_entity = t2.fk_entity AND t2.is_in_project= true - GROUP BY - tw1.pk_entity, - tw1.fk_class, - tw1.entity_type - - - - ` - -export const deleteSql = ` - WITH tw1 AS ( - SELECT t2.pk_entity "pkEntity" - FROM information.resource t2 - WHERE t2.tmsp_last_modification >= $1 - AND (t2.community_visibility->>'toolbox')::boolean IS DISTINCT FROM true - ), - tw2 AS ( - DELETE FROM war.entity_preview - USING tw1 - WHERE pk_entity = tw1."pkEntity" - AND fk_project IS NULL - ) - SELECT * FROM tw1 - ` diff --git a/server/src/warehouse/primary-ds/property/PPropertyService.ts b/server/src/warehouse/primary-ds/property/PPropertyService.ts deleted file mode 100644 index 8a930da6c..000000000 --- a/server/src/warehouse/primary-ds/property/PPropertyService.ts +++ /dev/null @@ -1,127 +0,0 @@ -import {forwardRef, Inject, Injectable} from 'injection-js'; -import {PrimaryDataService} from '../../base/classes/PrimaryDataService'; -import {KeyDefinition} from '../../base/interfaces/KeyDefinition'; -import {Warehouse} from '../../Warehouse'; -export interface PPropertyId { - fkProject: number, - pkProperty: number, - fkDomain: number - fkRange: number -} -export interface PPropertyVal { - fkProperty: number - fkProject: number, - fkDomain: number - fkRange: number -} -export const pPropertyKeyDef: KeyDefinition[] = [ - {name: 'fkProject', type: 'integer'}, - {name: 'pkProperty', type: 'integer'}, - {name: 'fkDomain', type: 'integer'}, - {name: 'fkRange', type: 'integer'} -] -@Injectable() -export class PPropertyService extends PrimaryDataService{ - - measure = 1000; - constructor(@Inject(forwardRef(() => Warehouse)) wh: Warehouse) { - super( - wh, - [ - 'modified_projects_project', - 'modified_projects_dfh_profile_proj_rel', - 'modified_data_for_history_api_property', - 'modified_data_for_history_api_class' - ], - pPropertyKeyDef - ) - - } - - getUpdatesSql(tmsp: Date) { - return updateSql - } - getDeletesSql(tmsp: Date) { - return deleteSql - }; -} - -const updateSql = ` - WITH tw1 AS ( - SELECT fk_profile, fk_project, enabled, tmsp_last_modification - FROM projects.dfh_profile_proj_rel - WHERE enabled = true - UNION ALL - SELECT fk_profile, pk_entity as fk_project, true, null - FROM projects.project, - ( - SELECT jsonb_array_elements_text(config->'ontome'->'requiredOntomeProfiles')::int fk_profile - FROM system.config - ) as requiredProfiles - ) - SELECT DISTINCT - fk_project "fkProject", - dfh_pk_property "pkProperty", - dfh_property_domain "fkDomain", - dfh_property_range "fkRange", - jsonb_build_object( - 'fkProject', fk_project, - 'fkProperty', dfh_pk_property, - 'fkDomain', dfh_property_domain, - 'fkRange', dfh_property_range - ) val - FROM - tw1 t1, - data_for_history.api_property t2 - WHERE t1.fk_profile = t2.dfh_fk_profile - AND ( - t1.tmsp_last_modification >= $1 - OR - t2.tmsp_last_modification >= $1 - ) - --- add generic incoming 1111 for all classes of basic type 8/9/30 - UNION - SELECT DISTINCT - fk_project "fkProject", - 1111 "pkProperty", - 365 "fkDomain", - t2.dfh_pk_class "fkRange", - jsonb_build_object( - 'fkProject', fk_project, - 'fkProperty', 1111, - 'fkDomain', 365, - 'fkRange', t2.dfh_pk_class - ) val - FROM - tw1 t1, - data_for_history.api_class t2 - WHERE t1.fk_profile = t2.dfh_fk_profile - AND t2.dfh_pk_class <> 365 - AND ( - t2.tmsp_last_modification >= $1 - OR - t2.tmsp_last_modification >= $1 - ) - AND t2.dfh_basic_type IN (8, 9, 30) - GROUP BY t2.dfh_pk_class, t1.fk_project -` -export const deleteSql = ` - WITH tw1 AS ( - SELECT fk_profile, fk_project, enabled, tmsp_last_modification - FROM projects.dfh_profile_proj_rel - WHERE enabled = false - AND tmsp_last_modification >= $1 - ) - SELECT DISTINCT - dfh_pk_property "pkProperty", - fk_project "fkProject", - dfh_property_domain "fkDomain", - dfh_property_range "fkRange" - FROM - tw1 t1, - data_for_history.api_property t2 - WHERE t1.fk_profile = t2.dfh_fk_profile -` - - - diff --git a/server/src/warehouse/primary-ds/property/RPropertyService.ts b/server/src/warehouse/primary-ds/property/RPropertyService.ts deleted file mode 100644 index ce4d48875..000000000 --- a/server/src/warehouse/primary-ds/property/RPropertyService.ts +++ /dev/null @@ -1,116 +0,0 @@ -import {PrimaryDataService} from '../../base/classes/PrimaryDataService'; -import {KeyDefinition} from '../../base/interfaces/KeyDefinition'; -import {Warehouse} from '../../Warehouse'; -import {Injectable, Inject, forwardRef} from 'injection-js'; -export interface RPropertyId { - pkProperty: number, - fkDomain: number - fkRange: number -} - -export interface RPropertyVal { - fkProperty: number - fkDomain: number - fkRange: number -} -export const rPropertyIdKeyConfig: KeyDefinition[] = [ - { - name: 'pkProperty', - type: 'integer' - }, - { - name: 'fkDomain', - type: 'integer' - }, - { - name: 'fkRange', - type: 'integer' - } -] -@Injectable() -export class RPropertyService extends PrimaryDataService{ - - measure = 1000; - - - - constructor(@Inject(forwardRef(() => Warehouse)) wh: Warehouse) { - super( - wh, - [ - 'modified_data_for_history_api_property', - 'modified_data_for_history_api_class' - ], - rPropertyIdKeyConfig - ) - - /** - * Add actions after a new RProperty is put/updated into index - */ - // this.afterPut$.subscribe(item => { - // // Add update requests on aggregaters based on project property - // const outgoingField: RClassFieldId = { - // fkClass: item.val.fkDomain, - // fkProperty: item.val.fkProperty, - // isOutgoing: true - // } - // wh.agg.rClassFieldLabel.updater.addItemToQueue(outgoingField).catch(e => console.log(e)) - - // const incomingField: RClassFieldId = { - // fkClass: item.val.fkRange, - // fkProperty: item.val.fkProperty, - // isOutgoing: false - // } - // wh.agg.rClassFieldLabel.updater.addItemToQueue(incomingField).catch(e => console.log(e)) - // }) - } - - getUpdatesSql(tmsp: Date) { - return updateSql - } - getDeletesSql(tmsp: Date) {return ''} -} - - -interface InitItem { - fkProperty: number, - fkDomain: number - fkRange: number -} - -const updateSql = ` - SELECT DISTINCT - dfh_pk_property "pkProperty", - dfh_property_domain "fkDomain", - dfh_property_range "fkRange", - jsonb_build_object( - 'fkProperty', dfh_pk_property, - 'fkDomain', dfh_property_domain, - 'fkRange', dfh_property_range - ) val - FROM - data_for_history.api_property t1 - WHERE - t1.tmsp_last_modification >= $1 - - --- add generic incoming 1111 for all classes of basic type 8/9/30 - UNION - SELECT DISTINCT - 1111 "pkProperty", - 365 "fkDomain", - t1.dfh_pk_class "fkRange", - jsonb_build_object( - 'fkProperty', 1111, - 'fkDomain', 365, - 'fkRange', t1.dfh_pk_class - ) val - FROM data_for_history.api_class t1 - WHERE t1.dfh_pk_class <> 365 - AND t1.tmsp_last_modification >= $1 - AND t1.dfh_basic_type IN (8, 9, 30) - GROUP BY t1.dfh_pk_class -` - - - - diff --git a/server/src/warehouse/primary-ds/statement/PStatementService.ts b/server/src/warehouse/primary-ds/statement/PStatementService.ts deleted file mode 100644 index 0d87df8b8..000000000 --- a/server/src/warehouse/primary-ds/statement/PStatementService.ts +++ /dev/null @@ -1,256 +0,0 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -import {forwardRef, Inject, Injectable} from 'injection-js'; -import {PrimaryDataService} from '../../base/classes/PrimaryDataService'; -import {KeyDefinition} from '../../base/interfaces/KeyDefinition'; -import {Warehouse} from '../../Warehouse'; -export interface PStatementId { - pkEntity: number; - fkProject: number; - -} -export interface PStatementVal { - fkProperty: number; - fkObjectInfo: number; - fkSubjectInfo: number; - ordNumOfDomain: number; - ordNumOfRange: number; - isInProjectCount: number; -} -export const pStatementKeyDefs: KeyDefinition[] = [ - { - name: 'pkEntity', - type: 'integer' - }, - { - name: 'fkProject', - type: 'integer' - } -] - -@Injectable() -export class PStatementService extends PrimaryDataService{ - - measure = 10000; - - - constructor(@Inject(forwardRef(() => Warehouse)) wh: Warehouse) { - super( - wh, - [ - 'modified_projects_info_proj_rel', - ], - pStatementKeyDefs - ) - this.registerUpdateReplication( - 'war.statement', - (insertClause: string, fromClause: string) => ` - INSERT INTO war.statement - (pk_entity, - fk_project, - project, - fk_property, - fk_object_info, - fk_subject_info, - ord_num_of_domain, - ord_num_of_range, - is_in_project_count, - object_info_value) - SELECT - t1."pkEntity", - t1."fkProject", - t1."fkProject", - (t1.val->>'fkProperty')::int, - (t1.val->>'fkObjectInfo')::int, - (t1.val->>'fkSubjectInfo')::int, - (t1.val->>'ordNumOfDomain')::int, - (t1.val->>'ordNumOfRange')::int, - (t1.val->>'isInProjectCount')::int, - t1.val->'objectInfoValue' - FROM ${fromClause} t1 - ON CONFLICT (pk_entity, project) DO UPDATE - SET pk_entity = EXCLUDED.pk_entity, - fk_project = EXCLUDED.fk_project, - project = EXCLUDED.project, - fk_property = EXCLUDED.fk_property, - fk_object_info = EXCLUDED.fk_object_info, - fk_subject_info = EXCLUDED.fk_subject_info, - ord_num_of_domain = EXCLUDED.ord_num_of_domain, - ord_num_of_range = EXCLUDED.ord_num_of_range, - is_in_project_count = EXCLUDED.is_in_project_count, - object_info_value = EXCLUDED.object_info_value - ` - ) - } - - - getUpdatesSql(tmsp: Date) { - return updateSql - } - getDeletesSql(tmsp: Date) { - return deleteSql - }; - -} - - -const updateSql = ` - -WITH tw0 AS ( - -- select chanced info_proj_rel pointing to a statement - SELECT DISTINCT - t2.pk_entity, - t1.fk_project, - t1.ord_num_of_domain, - t1.ord_num_of_range - FROM - projects.info_proj_rel t1 - JOIN - information."statement" t2 ON t1.fk_entity = t2.pk_entity - WHERE - t1.tmsp_last_modification > $1 - AND - t1.is_in_project = true -), -tw1 AS ( -SELECT - t0.fk_project, - t0.pk_entity, - t0.ord_num_of_domain, - t0.ord_num_of_range, - t1.fk_property, - t1.fk_object_info, - t1.fk_subject_info, - t2.is_in_project_count, - - -- appellation - CASE WHEN t3.pk_entity IS NOT NULL THEN jsonb_build_object( - 'pkEntity', t3.pk_entity, - 'fkClass', t3.fk_class, - 'string', t3.string - ) ELSE NULL::jsonb END appellation, - - -- language - CASE WHEN t4.pk_entity IS NOT NULL THEN jsonb_build_object( - 'pkEntity', t4.pk_entity, - 'fkClass', t4.fk_class, - 'iso6391', t4.iso6391, - 'iso6392b', t4.iso6392b, - 'iso6392t', t4.iso6392t, - 'label', t4.notes - ) ELSE NULL::jsonb END "language", - - -- place - CASE WHEN t5.pk_entity IS NOT NULL THEN jsonb_build_object( - 'pkEntity', t5.pk_entity, - 'fkClass', t5.fk_class, - 'geoJSON', ST_AsGeoJSON(t5.geo_point::geometry)::json - ) ELSE NULL::jsonb END place, - - -- time_primitive - CASE WHEN t6.pk_entity IS NOT NULL THEN - commons.time_primitive__pretty_json(t6) - ELSE NULL::jsonb END time_primitive, - - -- lang_string - CASE WHEN t7.pk_entity IS NOT NULL THEN jsonb_build_object( - 'pkEntity', t7.pk_entity, - 'fkClass', t7.fk_class, - 'string', t7.string, - 'fkLanguage', t7.fk_language - ) - ELSE NULL::jsonb END lang_string, - - -- dimension - CASE WHEN t8.pk_entity IS NOT NULL THEN jsonb_build_object( - 'pkEntity', t8.pk_entity, - 'fkClass', t8.fk_class, - 'fkMeasurementUnit', t8.fk_measurement_unit, - 'numericValue', t8.numeric_value - ) - ELSE NULL::jsonb END dimension, - - -- cell - CASE WHEN t9.pk_cell IS NOT NULL THEN jsonb_build_object( - 'pkCell', t9.pk_cell, - 'fkClass', t9.fk_class, - 'fkColumn', t9.fk_column, - 'fkRow', t9.fk_row, - 'stringValue', t9.string_value, - 'numericValue', t9.numeric_value - ) - ELSE NULL::jsonb END cell - - - FROM tw0 t0 - JOIN information.statement t1 ON t1.pk_entity = t0.pk_entity - LEFT JOIN LATERAL ( - SELECT count(info_proj_rel.pk_entity)::integer AS is_in_project_count - FROM projects.info_proj_rel - WHERE info_proj_rel.fk_entity = t1.pk_entity - AND info_proj_rel.is_in_project = true - GROUP BY info_proj_rel.fk_entity - ) t2 ON true - - LEFT JOIN information.appellation t3 ON t3.pk_entity = t1.fk_object_info - LEFT JOIN information.language t4 ON t4.pk_entity = t1.fk_object_info - LEFT JOIN information.place t5 ON t5.pk_entity = t1.fk_object_info - LEFT JOIN information.time_primitive t6 ON t6.pk_entity = t1.fk_object_info - LEFT JOIN information.lang_string t7 ON t7.pk_entity = t1.fk_object_info - LEFT JOIN information.dimension t8 ON t8.pk_entity = t1.fk_object_info - LEFT JOIN tables.cell t9 ON t9.pk_cell = t1.fk_object_tables_cell - -) -SELECT -t1.fk_project "fkProject", -t1.pk_entity "pkEntity", -jsonb_build_object( -'fkProperty', fk_property, -'fkObjectInfo', fk_object_info, -'fkSubjectInfo', fk_subject_info, -'ordNumOfDomain', ord_num_of_domain, -'ordNumOfRange', ord_num_of_range, -'isInProjectCount', is_in_project_count, -'objectInfoValue', CASE WHEN - t1.appellation IS NULL - AND t1.place IS NULL - AND t1.language IS NULL - AND t1.time_primitive IS NULL - AND t1.lang_string IS NULL - AND t1.dimension IS NULL - AND t1.cell IS NULL - THEN NULL::jsonb - ELSE - jsonb_strip_nulls(jsonb_build_object( - 'string', t1.appellation, - 'geometry', t1.place, - 'language', t1.language, - 'timePrimitive', t1.time_primitive, - 'langString', t1.lang_string, - 'dimension', t1.dimension, - 'cell', t1.cell - )) - END -) val -FROM tw1 t1 -` - - -const deleteSql = ` -WITH tw1 AS ( - SELECT - t1.fk_project "fkProject", - t2.pk_entity "pkEntity" - FROM - projects.info_proj_rel t1 - JOIN information.statement t2 ON t1.fk_entity = t2.pk_entity - WHERE t1.is_in_project=false - AND t1.tmsp_last_modification >= $1 -), -tw2 AS ( - DELETE FROM war.statement - USING tw1 - WHERE pk_entity = tw1."pkEntity" - AND fk_project = tw1."fkProject" -) -SELECT * FROM tw1 -` diff --git a/server/src/warehouse/startScripts.ts b/server/src/warehouse/startScripts.ts deleted file mode 100644 index 2c86d15de..000000000 --- a/server/src/warehouse/startScripts.ts +++ /dev/null @@ -1,98 +0,0 @@ -import c from 'child_process'; -import fs from 'fs'; -import path from 'path'; -import { Pool } from 'pg'; -import pgkDir from 'pkg-dir'; -import { getGvDatabaseUrl, getWhDatabaseUrl } from '../utils/databaseUrl'; -import { createWarehouse } from './createWarehouse'; -import { Warehouse, WarehouseConfig } from './Warehouse'; - -const appRoot = pgkDir.sync() ?? '' -const schemaPrefix = 'war_cache_' - -// Use this function to start warehouse on heroku environments -// The warehouse-compat-list is required -// (it's not created by this function, because on heroku .git folder is missing) -export async function start() { - const config = getWarehouseConfig() - const warehouse: Warehouse = createWarehouse(config).get(Warehouse) - await warehouse.start(); - await removeWarehouseSchemasExcept(warehouse.whPgPool, schemaPrefix, warehouse.schemaName) - return warehouse -} - - -// Use this function to start warehouse on local environments -// The function will create the warehouse-compat-list -// (this requires git CLI and .git folder) -export async function startDev() { - c.execSync(`cd ${path.join(appRoot, '..')} && sh deployment/create-warehouse-compat-list.sh`); - await start(); -} - - -// Use this function to start warehouse on local environments -// Cleans the whDB and starts warehouse without backups -export async function cleanAndStartDev() { - c.execSync(`cd ${path.join(appRoot, '..')} && sh deployment/create-warehouse-compat-list.sh`); - const config: WarehouseConfig = getWarehouseConfig() - const warehouse: Warehouse = createWarehouse(config).get(Warehouse) - await warehouse.whPgPool.query(`drop schema if exists ${warehouse.schemaName} cascade;`) - await warehouse.start(); - await removeWarehouseSchemasExcept(warehouse.whPgPool, schemaPrefix, warehouse.schemaName) - return warehouse -} - - - -function getSchemaName() { - const file = 'warehouse-compat-list.txt' - const filePath = path.join(__dirname, '../../', file) - // reads warhouse compatible commits - const compatibleWithCommits = fs - .readFileSync(filePath) - .toString() - .split('\n') - .filter(str => !!str); - - if (!compatibleWithCommits.length) throw new Error('Can\'t find the latest commit from compatibleWithCommits') - - // gets commit sha of last change on warehouse directory - const warehouseCommit = compatibleWithCommits[compatibleWithCommits.length - 1] - - return schemaPrefix + warehouseCommit -} - -export function getWarehouseConfig() { - const gvDb = getGvDatabaseUrl(); - const whDb = getWhDatabaseUrl(); - - const config: WarehouseConfig = { - warehouseDatabase: whDb, - warehouseDatabaseMaxConnections: parseInt(process.env.WAREHOUSE_WH_DB_POOL_SIZE ?? '25', 10), - geovistoryDatabase: gvDb, - geovistoryDatabaseMaxConnections: parseInt(process.env.WAREHOUSE_GV_DB_POOL_SIZE ?? '25', 10), - warehouseSchema: getSchemaName() - } - - return config -} - -/** - * - * @param pgPool pg PoolClient - * @param warehouseSchemaPrefix prefix of warehouse schemas - * @param notRemovingSchema complete name of schema not to remove, even if it begins with warehouseSchemaPrefix - */ -async function removeWarehouseSchemasExcept(pgPool: Pool, warehouseSchemaPrefix: string, notRemovingSchema: string) { - const res = await pgPool.query<{ schema: string }>(` - SELECT schema_name AS schema - FROM information_schema.schemata - WHERE schema_name iLike $1 || '%' - AND schema_name != $2 - `, [warehouseSchemaPrefix, notRemovingSchema]) - const schemasToRemove = res.rows - for (const item of schemasToRemove) { - await pgPool.query(`DROP SCHEMA ${item.schema} CASCADE;`) - } -} diff --git a/server/warehouse-compat-list.txt b/server/warehouse-compat-list.txt deleted file mode 100644 index 02ffb33e8..000000000 --- a/server/warehouse-compat-list.txt +++ /dev/null @@ -1,11 +0,0 @@ -08b3f14 -7111750 -5e51495 -ecd0089 -cf16673 -638f443 -890d8bc -143b14e -9be1f4d -7d30914 -8f735c1 diff --git a/server/warehouse-docker-build-and-push.sh b/server/warehouse-docker-build-and-push.sh deleted file mode 100644 index 7a787a9de..000000000 --- a/server/warehouse-docker-build-and-push.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -if [ -z "$(git status --porcelain)" ]; then - # Working directory clean - echo 'GIT: working directory clean' -else - # Uncommitted changes - echo 'GIT: Uncommitted changes, please cleanup directory first' - exit 1 -fi - - -COMMIT=$(git rev-parse --verify HEAD) -docker image build -f "warehouse.dockerfile" . \ - --build-arg "app_name=warehouse" \ - -t "docker.pkg.github.com/kleiolab/geovistory/warehouse:latest" \ - -t "docker.pkg.github.com/kleiolab/geovistory/warehouse:${COMMIT}" - -docker push docker.pkg.github.com/kleiolab/geovistory/warehouse:${COMMIT} diff --git a/server/warehouse.dockerfile b/server/warehouse.dockerfile deleted file mode 100644 index cd2ac627f..000000000 --- a/server/warehouse.dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -FROM node:12.8.1 - -# Create app directory -WORKDIR /app - -# Install app dependencies -# A wildcard is used to ensure both package.json AND package-lock.json are copied -# where available (npm@5+) -COPY package*.json /app/ - -# “Ci” will install packages directly from the lock file. -RUN npm ci - -# Copy typescript (everything needed to compile) -COPY tsconfig.json /app -COPY src /app/src -# Copy compatibility list (should be generated here) -COPY warehouse-compat-list.txt /app - -# Copy migration files -COPY db-migrate /app/db-migrate - -# Copy entrypoint -COPY warehouse.start.sh /app - -# Compile TypeScript (in workdir) -RUN npm run tsc - -# If you are building your code for production -#RUN npm ci --only=production - -CMD [ "bash", "warehouse.start.sh"] diff --git a/server/warehouse.start.sh b/server/warehouse.start.sh deleted file mode 100644 index 2d6c2f947..000000000 --- a/server/warehouse.start.sh +++ /dev/null @@ -1,14 +0,0 @@ -set -e # exit when any command fails - -# bash ./db-migrate/up.sh # migrate up - -# Wait, if there are migrations to run. -CHECK_OUTPUT=$(bash ./db-migrate/up.sh --check) -while [[ $CHECK_OUTPUT =~ '[INFO] Migrations to run:' ]]; do - echo "Waiting for database to be migrated up..." - sleep 2 - CHECK_OUTPUT=$(bash ./db-migrate/up.sh --check) -done - -echo "Database is ready, starting warehouse server..." -node dist/warehouse.js # start server diff --git a/server/webserver-docker-build-and-push.sh b/server/webserver-docker-build-and-push.sh deleted file mode 100644 index 72cc93cd4..000000000 --- a/server/webserver-docker-build-and-push.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -if [ -z "$(git status --porcelain)" ]; then - # Working directory clean - echo 'GIT: working directory clean' -else - # Uncommitted changes - echo 'GIT: Uncommitted changes, please cleanup directory first' - exit 1 -fi - - -COMMIT=$(git rev-parse --verify HEAD) -docker image build -f "webserver.dockerfile" . \ - --build-arg "app_name=webserver" \ - -t "docker.pkg.github.com/kleiolab/geovistory/webserver:latest" \ - -t "docker.pkg.github.com/kleiolab/geovistory/webserver:${COMMIT}" - -docker push docker.pkg.github.com/kleiolab/geovistory/webserver:${COMMIT} diff --git a/web.bash b/web.bash deleted file mode 100644 index e19c81748..000000000 --- a/web.bash +++ /dev/null @@ -1 +0,0 @@ -./server/node_modules/.bin/forever ./server/dist/webserver.js \ No newline at end of file diff --git a/worker.bash b/worker.bash deleted file mode 100644 index d1eee0f4b..000000000 --- a/worker.bash +++ /dev/null @@ -1 +0,0 @@ -./server/node_modules/.bin/forever ./server/dist/warehouse.js \ No newline at end of file