create morpheus-data to handle all related with data

 - add building for morpheus-data
 - include morpheus-data wheels in morpheus-server build process
 - move all repositories to morpheus_data
 - add morpheus-data installation in ci-backend
 - add morpheus-data instructions in readme
 - move lora information in models and schemas to morpheus-data
 - add morpheus-data to monorepo cicd pipeline
 - create requierements and install it in morpheus-data to allow the use
   of alembic
 - include rust to fix build error for safetensors
 - update migration in cd-api to use morpheus-data image
 - update and add README

.github/workflows/cd-api.yaml

@@ -30,7 +30,7 @@ jobs:
       - name: Deploy
         run: |
           # Running migrations by a pod
-          kubectl run morpheus-migrations --pod-running-timeout=15m --rm -i --image monadicalsas/morpheus-api:${{ github.run_id }} --overrides="{
+          kubectl run morpheus-migrations --pod-running-timeout=15m --rm -i --image monadicalsas/morpheus-data:latest --overrides="{
             \"spec\": {
               \"imagePullSecrets\": [
                 {
@@ -40,7 +40,7 @@ jobs:
               \"containers\": [
                 {
                   \"name\": \"morpheus\",
-                  \"image\": \"monadicalsas/morpheus-api:${{ github.run_id }}\",
+                  \"image\": \"monadicalsas/morpheus-data:latest\",
                   \"imagePullPolicy\": \"Always\",
                   \"command\": [
                     \"alembic\", \"upgrade\", \"head\"

.github/workflows/cd-data.yaml

@@ -0,0 +1,14 @@
+name: CI Test and deploy Frontend
+on:
+  workflow_dispatch:
+  workflow_call:
+env:
+  MAX_LINE_LENGTH: 120
+  WORKING_DIR: morpheus-data
+jobs:
+  ci-data:
+    uses: ./.github/workflows/ci-data.yaml
+    with:
+      called: true
+    secrets: inherit
+

.github/workflows/ci-api.yaml

@@ -67,6 +67,8 @@ jobs:
           pip install pytest
       - name: Install dependencies
         run: |
+          pip install build
+          pip install -e ../morpheus-data
           pip install -r requirements.txt
           pip install torch --index-url https://download.pytorch.org/whl/cpu
       - name: Run tests

.github/workflows/ci-data.yaml

@@ -0,0 +1,70 @@
+name: CI data library
+on:
+  workflow_call:
+    inputs:
+      called:
+        required: false
+        type: string
+  workflow_dispatch:
+    inputs:
+      called:
+        required: false
+        type: string
+env:
+  MAX_LINE_LENGTH: 120
+  WORKING_DIR: morpheus-data
+jobs:
+  linters:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ${{ env.WORKING_DIR }}
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v4
+        id: python-cache
+        with:
+          python-version: '3.10'
+          cache: pip
+          cache-dependency-path: ./${{ env.WORKING_DIR }}/requirements.lint.txt
+      - name: Install packages
+        run: |
+          pip install -r requirements.lint.txt
+      - name: Black
+        run: |
+          black --line-length $MAX_LINE_LENGTH --exclude morpheus_data/migrations/ .
+      - name: Flake8
+        run: |
+          flake8 --max-line-length $MAX_LINE_LENGTH --exclude morpheus_data/migrations/ .
+  ci-data-library:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ${{ env.WORKING_DIR }}
+    steps:
+      - uses: actions/checkout@v2
+      -
+        name: Set up QEMU
+        if: ${{ inputs.called == 'true' }}
+        uses: docker/setup-qemu-action@v2
+      -
+        name: Set up Docker Buildx
+        if: ${{ inputs.called == 'true' }}
+        uses: docker/setup-buildx-action@v2
+      -
+        name: Login to DockerHub
+        if: ${{ inputs.called == 'true' }}
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_HUB_USER }}
+          password: ${{ secrets.DOCKER_HUB_TOKEN }}
+      -
+        name: Build and push
+        if: ${{ inputs.called == 'true' }}
+        uses: docker/build-push-action@v3
+        with:
+          context: "{{defaultContext}}:morpheus-data"
+          build-args: |
+            BUILD_ENV=copy
+          push: true
+          tags: monadicalsas/morpheus-data:latest

.github/workflows/ci.yaml

@@ -13,6 +13,7 @@ jobs:
       collaborative: ${{ steps.collaborative-step.outputs.collaborative }}
       frontend: ${{ steps.frontend-step.outputs.frontend }}
       workerray: ${{ steps.worker-ray-step.outputs.workerray }}
+      data: ${{ steps.data-step.outputs.data }}
     steps:
       - uses: actions/checkout@v2
         with:
@@ -46,12 +47,24 @@ jobs:
         run: |
           workerray=$(python ./monorepo/cicd.py -c "origin/main" --project morpheus-worker-ray)
           echo "workerray=$workerray" >> "$GITHUB_OUTPUT"
+      - id: data-step
+        name: Check change in data
+        run: |
+          data=$(python ./monorepo/cicd.py -c "origin/main" --project morpheus-data)
+          echo "data=$data" >> "$GITHUB_OUTPUT"
 
 
+  ci-data:
+    if: needs.ci.outputs.data == 'true'
+    needs:
+      - ci
+    uses: ./.github/workflows/ci-data.yaml
+    secrets: inherit
   ci-api:
-    if: needs.ci.outputs.api == 'true'
+    if: needs.ci.outputs.api == 'true' || needs.ci.outputs.data == 'true'
     needs:
       - ci
+      - ci-data
     uses: ./.github/workflows/ci-api.yaml
     secrets: inherit
   ci-collaborative:
@@ -72,7 +85,7 @@ jobs:
       - ci
     uses: ./.github/workflows/ci-worker-ray.yaml
     secrets: inherit
-  
+
 #############################################################################################################
 ### CD - jobs
 #############################################################################################################
@@ -84,6 +97,7 @@ jobs:
       collaborative: ${{ steps.collaborative-step.outputs.collaborative }}
       frontend: ${{ steps.frontend-step.outputs.frontend }}
       workerray: ${{ steps.worker-ray-step.outputs.workerray }}
+      data: ${{ steps.data-step.outputs.data }}
     steps:
       - uses: actions/checkout@v2
         with:
@@ -114,11 +128,24 @@ jobs:
         run: |
           workerray=$(python ./monorepo/cicd.py --project morpheus-worker-ray)
           echo "workerray=$workerray" >> "$GITHUB_OUTPUT"
+      - id: data-step
+        name: Check change in data
+        run: |
+          data=$(python ./monorepo/cicd.py --project morpheus-data)
+          echo "data=$data" >> "$GITHUB_OUTPUT"
+
+  cd-data:
+    if: needs.cd.outputs.data == 'true'
+    needs:
+      - cd
+    uses: ./.github/workflows/cd-data.yaml
+    secrets: inherit
 
   cd-api:
-    if: needs.cd.outputs.api == 'true'
+    if: needs.cd.outputs.api == 'true' || needs.cd.outputs.data == 'true'
     needs:
       - cd
+      - cd-data
     uses: ./.github/workflows/cd-api.yaml
     secrets: inherit
   cd-frontend:

README.md

@@ -261,13 +261,15 @@ docker compose run --rm api pytest tests/test_module.py::test_function
 ```
 
 ### Running the migrations
+To use morpheus-data image to run the migrations, you need to create a secrets.env file in the morpheus-server 
+directory. For more information, you can read the morpheus-data [README](./morpheus-data/README.md).
 
 ```shell
 # Create migration
-docker-compose run --rm api alembic revision --autogenerate -m "Initial migration"
+docker-compose run --rm datalib alembic revision --autogenerate -m "Initial migration"
 
 # Migrate / Update the head
-docker-compose run --rm api alembic upgrade head
+docker-compose run --rm datalib alembic upgrade head
 ```
 
 ### PG admin
@@ -282,9 +284,41 @@ [email protected]
 PGADMIN_DEFAULT_PASSWORD=password
 ```
 
+### Implement changes in morpheus-data
+
+`morpheus-data` works as a package to have a transverse Python library to manage all ORM-related operations in Morpheus.
+Other Morpheus microservices can import into them and use it.
+
+To run Morpheus locally using morpheus-data as a library, you need to do this:
+
+```bash
+
+# Building in separate steps
+#---------------------------------------------
+# build morpheus-data wheel
+docker compose build datalib
+
+# build morpheus-server api
+docker compose build api
+
+# Building alltogether
+#---------------------------------------------
+docker compose --profile <local|staging|manage> build
+
+# Run
+#---------------------------------------------
+docker compose --profile <local|staging> up
+```
+
+**Note**:  You need to build `morpheus-data` and `morpheus-server` API service (or any other microservice that uses it)
+every time you make a change inside `morpheus-data`. This build is necessary because you need to build the wheel 
+file again and install it in the `morpheus-server` API service or any other service that uses it
+For more information, you can read the morpheus-data [README](./morpheus-data/README.md).
+
+
 ### Adding a new dependency to the backend
 
-1. Add the new dependency directly to the respective requirements.txt file
+1. Add the new dependency directly to the respective `requirements.txt` file
 2. Update the docker image
    ```shell
     docker-compose build api

docker-compose.yaml

@@ -1,4 +1,4 @@
-version: "3.3"
+version: "3.8"
 services:
   postgres:
     image: postgres:15-alpine
@@ -45,6 +45,22 @@ services:
     depends_on:
       postgres:
         condition: service_healthy
+      datalib:
+        condition: service_completed_successfully
+
+  datalib:
+    build:
+      context: morpheus-data
+      dockerfile: ./Dockerfile
+      args:
+        DOCKER_BUILDKIT: 1
+    image: monadicalsas/morpheus-data:latest
+    volumes:
+      - ./wheels:/wheels
+      - ./morpheus-data:/app
+    depends_on:
+      postgres:
+        condition: service_healthy
 
   client:
     build:
@@ -248,6 +264,9 @@ services:
       - ./morpheus-server:/opt/api
     stdin_open: true
     tty: true
+    depends_on:
+      datalib:
+        condition: service_completed_successfully
 
 
 volumes:

monorepo/cicd.py

@@ -9,7 +9,8 @@
 LAST_COMMIT = "HEAD~1"
 CICD_REPO_PATH = os.getenv("CICD_REPO_PATH", "/home/runner/work/Morpheus/Morpheus")
 
-def get_status(repo, path, commit = LAST_COMMIT):
+
+def get_status(repo, path, commit=LAST_COMMIT):
     changed = [item.a_path for item in repo.index.diff(commit)]
     if path in repo.untracked_files:
         return "untracked"
@@ -18,6 +19,7 @@ def get_status(repo, path, commit = LAST_COMMIT):
     else:
         return "na"
 
+
 def search_meta(repo_path, path):
     meta_file = os.path.join(path, META_FILE_NAME)
     exist_meta = os.path.exists(meta_file)
@@ -28,12 +30,14 @@ def search_meta(repo_path, path):
             return None
         return search_meta(repo_path, os.path.dirname(path))
 
+
 def load_json(meta_file):
     f = open(meta_file)
     data = json.load(f)
     return data
 
-def search_in_updated_projects(project_name, repo_path, commit = LAST_COMMIT):
+
+def search_in_updated_projects(project_name, repo_path, commit=LAST_COMMIT):
     repo = Repo(repo_path)
     for item in repo.index.diff(commit):
         status = get_status(repo, item.a_path)
@@ -47,10 +51,12 @@ def search_in_updated_projects(project_name, repo_path, commit = LAST_COMMIT):
                     return "true"
     return "false"
 
+
 def get_current_branch(repo_path):
     repo = Repo(repo_path)
     return str(repo.active_branch)
 
+
 parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
 parser.add_argument("-c", "--commit", help="Commit or branch name", default=LAST_COMMIT)
 parser.add_argument("-p", "--project", help="Project")

morpheus-data/.dockerignore

@@ -0,0 +1,4 @@
+secrets.env
+build
+dist
+*.egg-info

morpheus-data/.gitignore

@@ -0,0 +1,4 @@
+build
+dist
+*.egg-info
+*.env

morpheus-data/Dockerfile

@@ -0,0 +1,28 @@
+FROM python:3.10 AS builder
+
+WORKDIR /app
+
+# Install dependencies
+RUN apt-get update && apt-get install -y curl build-essential && rm -rf /var/lib/apt/lists/*
+
+# Install dependencies
+RUN pip install --upgrade pip
+RUN pip install --upgrade build pip-tools
+
+# Copy over pyproject.toml
+COPY pyproject.toml ./
+
+# Generate requirements.txt from pyproject.toml
+RUN pip-compile --output-file=requirements.txt pyproject.toml
+
+# Install dependencies
+RUN pip install -r requirements.txt
+
+# Copy over the rest of your project files
+COPY . .
+
+# Build the wheel
+RUN python -m build --wheel && mkdir /wheels
+
+# Copy the built wheel to a volume
+RUN cp dist/*.whl /wheels/