diff --git a/.github/workflows/cd-api.yaml b/.github/workflows/cd-api.yaml index abcd04ee..6a77d428 100644 --- a/.github/workflows/cd-api.yaml +++ b/.github/workflows/cd-api.yaml @@ -30,7 +30,7 @@ jobs: - name: Deploy run: | # Running migrations by a pod - kubectl run morpheus-migrations --pod-running-timeout=15m --rm -i --image monadicalsas/morpheus-api:${{ github.run_id }} --overrides="{ + kubectl run morpheus-migrations --pod-running-timeout=15m --rm -i --image monadicalsas/morpheus-data:latest --overrides="{ \"spec\": { \"imagePullSecrets\": [ { @@ -40,7 +40,7 @@ jobs: \"containers\": [ { \"name\": \"morpheus\", - \"image\": \"monadicalsas/morpheus-api:${{ github.run_id }}\", + \"image\": \"monadicalsas/morpheus-data:latest\", \"imagePullPolicy\": \"Always\", \"command\": [ \"alembic\", \"upgrade\", \"head\" diff --git a/.github/workflows/cd-data.yaml b/.github/workflows/cd-data.yaml new file mode 100644 index 00000000..dc46ff42 --- /dev/null +++ b/.github/workflows/cd-data.yaml @@ -0,0 +1,14 @@ +name: CI Test and deploy Frontend +on: + workflow_dispatch: + workflow_call: +env: + MAX_LINE_LENGTH: 120 + WORKING_DIR: morpheus-data +jobs: + ci-data: + uses: ./.github/workflows/ci-data.yaml + with: + called: true + secrets: inherit + diff --git a/.github/workflows/ci-api.yaml b/.github/workflows/ci-api.yaml index d6dae1a9..97b92ada 100644 --- a/.github/workflows/ci-api.yaml +++ b/.github/workflows/ci-api.yaml @@ -67,6 +67,8 @@ jobs: pip install pytest - name: Install dependencies run: | + pip install build + pip install -e ../morpheus-data pip install -r requirements.txt pip install torch --index-url https://download.pytorch.org/whl/cpu - name: Run tests diff --git a/.github/workflows/ci-data.yaml b/.github/workflows/ci-data.yaml new file mode 100644 index 00000000..b37fbf40 --- /dev/null +++ b/.github/workflows/ci-data.yaml @@ -0,0 +1,70 @@ +name: CI data library +on: + workflow_call: + inputs: + called: + required: false + type: string + workflow_dispatch: + inputs: + called: + required: false + type: string +env: + MAX_LINE_LENGTH: 120 + WORKING_DIR: morpheus-data +jobs: + linters: + runs-on: ubuntu-latest + defaults: + run: + working-directory: ${{ env.WORKING_DIR }} + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v4 + id: python-cache + with: + python-version: '3.10' + cache: pip + cache-dependency-path: ./${{ env.WORKING_DIR }}/requirements.lint.txt + - name: Install packages + run: | + pip install -r requirements.lint.txt + - name: Black + run: | + black --line-length $MAX_LINE_LENGTH --exclude morpheus_data/migrations/ . + - name: Flake8 + run: | + flake8 --max-line-length $MAX_LINE_LENGTH --exclude morpheus_data/migrations/ . + ci-data-library: + runs-on: ubuntu-latest + defaults: + run: + working-directory: ${{ env.WORKING_DIR }} + steps: + - uses: actions/checkout@v2 + - + name: Set up QEMU + if: ${{ inputs.called == 'true' }} + uses: docker/setup-qemu-action@v2 + - + name: Set up Docker Buildx + if: ${{ inputs.called == 'true' }} + uses: docker/setup-buildx-action@v2 + - + name: Login to DockerHub + if: ${{ inputs.called == 'true' }} + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_HUB_USER }} + password: ${{ secrets.DOCKER_HUB_TOKEN }} + - + name: Build and push + if: ${{ inputs.called == 'true' }} + uses: docker/build-push-action@v3 + with: + context: "{{defaultContext}}:morpheus-data" + build-args: | + BUILD_ENV=copy + push: true + tags: monadicalsas/morpheus-data:latest diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 48ed625c..9fe2c5a7 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -13,6 +13,7 @@ jobs: collaborative: ${{ steps.collaborative-step.outputs.collaborative }} frontend: ${{ steps.frontend-step.outputs.frontend }} workerray: ${{ steps.worker-ray-step.outputs.workerray }} + data: ${{ steps.data-step.outputs.data }} steps: - uses: actions/checkout@v2 with: @@ -46,12 +47,24 @@ jobs: run: | workerray=$(python ./monorepo/cicd.py -c "origin/main" --project morpheus-worker-ray) echo "workerray=$workerray" >> "$GITHUB_OUTPUT" + - id: data-step + name: Check change in data + run: | + data=$(python ./monorepo/cicd.py -c "origin/main" --project morpheus-data) + echo "data=$data" >> "$GITHUB_OUTPUT" + ci-data: + if: needs.ci.outputs.data == 'true' + needs: + - ci + uses: ./.github/workflows/ci-data.yaml + secrets: inherit ci-api: - if: needs.ci.outputs.api == 'true' + if: needs.ci.outputs.api == 'true' || needs.ci.outputs.data == 'true' needs: - ci + - ci-data uses: ./.github/workflows/ci-api.yaml secrets: inherit ci-collaborative: @@ -72,7 +85,7 @@ jobs: - ci uses: ./.github/workflows/ci-worker-ray.yaml secrets: inherit - + ############################################################################################################# ### CD - jobs ############################################################################################################# @@ -84,6 +97,7 @@ jobs: collaborative: ${{ steps.collaborative-step.outputs.collaborative }} frontend: ${{ steps.frontend-step.outputs.frontend }} workerray: ${{ steps.worker-ray-step.outputs.workerray }} + data: ${{ steps.data-step.outputs.data }} steps: - uses: actions/checkout@v2 with: @@ -114,11 +128,24 @@ jobs: run: | workerray=$(python ./monorepo/cicd.py --project morpheus-worker-ray) echo "workerray=$workerray" >> "$GITHUB_OUTPUT" + - id: data-step + name: Check change in data + run: | + data=$(python ./monorepo/cicd.py --project morpheus-data) + echo "data=$data" >> "$GITHUB_OUTPUT" + + cd-data: + if: needs.cd.outputs.data == 'true' + needs: + - cd + uses: ./.github/workflows/cd-data.yaml + secrets: inherit cd-api: - if: needs.cd.outputs.api == 'true' + if: needs.cd.outputs.api == 'true' || needs.cd.outputs.data == 'true' needs: - cd + - cd-data uses: ./.github/workflows/cd-api.yaml secrets: inherit cd-frontend: diff --git a/README.md b/README.md index 4821779d..77702567 100644 --- a/README.md +++ b/README.md @@ -261,13 +261,15 @@ docker compose run --rm api pytest tests/test_module.py::test_function ``` ### Running the migrations +To use morpheus-data image to run the migrations, you need to create a secrets.env file in the morpheus-server +directory. For more information, you can read the morpheus-data [README](./morpheus-data/README.md). ```shell # Create migration -docker-compose run --rm api alembic revision --autogenerate -m "Initial migration" +docker-compose run --rm datalib alembic revision --autogenerate -m "Initial migration" # Migrate / Update the head -docker-compose run --rm api alembic upgrade head +docker-compose run --rm datalib alembic upgrade head ``` ### PG admin @@ -282,9 +284,41 @@ PGADMIN_DEFAULT_EMAIL=admin@admin.com PGADMIN_DEFAULT_PASSWORD=password ``` +### Implement changes in morpheus-data + +`morpheus-data` works as a package to have a transverse Python library to manage all ORM-related operations in Morpheus. +Other Morpheus microservices can import into them and use it. + +To run Morpheus locally using morpheus-data as a library, you need to do this: + +```bash + +# Building in separate steps +#--------------------------------------------- +# build morpheus-data wheel +docker compose build datalib + +# build morpheus-server api +docker compose build api + +# Building alltogether +#--------------------------------------------- +docker compose --profile build + +# Run +#--------------------------------------------- +docker compose --profile up +``` + +**Note**: You need to build `morpheus-data` and `morpheus-server` API service (or any other microservice that uses it) +every time you make a change inside `morpheus-data`. This build is necessary because you need to build the wheel +file again and install it in the `morpheus-server` API service or any other service that uses it +For more information, you can read the morpheus-data [README](./morpheus-data/README.md). + + ### Adding a new dependency to the backend -1. Add the new dependency directly to the respective requirements.txt file +1. Add the new dependency directly to the respective `requirements.txt` file 2. Update the docker image ```shell docker-compose build api diff --git a/docker-compose.yaml b/docker-compose.yaml index 5be11ff7..f5a467b7 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,4 +1,4 @@ -version: "3.3" +version: "3.8" services: postgres: image: postgres:15-alpine @@ -45,6 +45,22 @@ services: depends_on: postgres: condition: service_healthy + datalib: + condition: service_completed_successfully + + datalib: + build: + context: morpheus-data + dockerfile: ./Dockerfile + args: + DOCKER_BUILDKIT: 1 + image: monadicalsas/morpheus-data:latest + volumes: + - ./wheels:/wheels + - ./morpheus-data:/app + depends_on: + postgres: + condition: service_healthy client: build: @@ -248,6 +264,9 @@ services: - ./morpheus-server:/opt/api stdin_open: true tty: true + depends_on: + datalib: + condition: service_completed_successfully volumes: diff --git a/monorepo/cicd.py b/monorepo/cicd.py index d8c942ee..9d3b7fa3 100644 --- a/monorepo/cicd.py +++ b/monorepo/cicd.py @@ -9,7 +9,8 @@ LAST_COMMIT = "HEAD~1" CICD_REPO_PATH = os.getenv("CICD_REPO_PATH", "/home/runner/work/Morpheus/Morpheus") -def get_status(repo, path, commit = LAST_COMMIT): + +def get_status(repo, path, commit=LAST_COMMIT): changed = [item.a_path for item in repo.index.diff(commit)] if path in repo.untracked_files: return "untracked" @@ -18,6 +19,7 @@ def get_status(repo, path, commit = LAST_COMMIT): else: return "na" + def search_meta(repo_path, path): meta_file = os.path.join(path, META_FILE_NAME) exist_meta = os.path.exists(meta_file) @@ -28,12 +30,14 @@ def search_meta(repo_path, path): return None return search_meta(repo_path, os.path.dirname(path)) + def load_json(meta_file): f = open(meta_file) data = json.load(f) return data -def search_in_updated_projects(project_name, repo_path, commit = LAST_COMMIT): + +def search_in_updated_projects(project_name, repo_path, commit=LAST_COMMIT): repo = Repo(repo_path) for item in repo.index.diff(commit): status = get_status(repo, item.a_path) @@ -47,10 +51,12 @@ def search_in_updated_projects(project_name, repo_path, commit = LAST_COMMIT): return "true" return "false" + def get_current_branch(repo_path): repo = Repo(repo_path) return str(repo.active_branch) + parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter) parser.add_argument("-c", "--commit", help="Commit or branch name", default=LAST_COMMIT) parser.add_argument("-p", "--project", help="Project") diff --git a/morpheus-data/.dockerignore b/morpheus-data/.dockerignore new file mode 100644 index 00000000..1f578e89 --- /dev/null +++ b/morpheus-data/.dockerignore @@ -0,0 +1,4 @@ +secrets.env +build +dist +*.egg-info diff --git a/morpheus-data/.gitignore b/morpheus-data/.gitignore new file mode 100644 index 00000000..af5c3b91 --- /dev/null +++ b/morpheus-data/.gitignore @@ -0,0 +1,4 @@ +build +dist +*.egg-info +*.env \ No newline at end of file diff --git a/morpheus-data/Dockerfile b/morpheus-data/Dockerfile new file mode 100644 index 00000000..5f4f082a --- /dev/null +++ b/morpheus-data/Dockerfile @@ -0,0 +1,28 @@ +FROM python:3.10 AS builder + +WORKDIR /app + +# Install dependencies +RUN apt-get update && apt-get install -y curl build-essential && rm -rf /var/lib/apt/lists/* + +# Install dependencies +RUN pip install --upgrade pip +RUN pip install --upgrade build pip-tools + +# Copy over pyproject.toml +COPY pyproject.toml ./ + +# Generate requirements.txt from pyproject.toml +RUN pip-compile --output-file=requirements.txt pyproject.toml + +# Install dependencies +RUN pip install -r requirements.txt + +# Copy over the rest of your project files +COPY . . + +# Build the wheel +RUN python -m build --wheel && mkdir /wheels + +# Copy the built wheel to a volume +RUN cp dist/*.whl /wheels/ \ No newline at end of file diff --git a/morpheus-data/README.md b/morpheus-data/README.md new file mode 100644 index 00000000..dd6f7a12 --- /dev/null +++ b/morpheus-data/README.md @@ -0,0 +1,146 @@ +# Morpheus data + +This service enables you to seamlessly manage all aspects of data in Morpheus. Its purpose is +to separate business logic from data logic and provide easy access to data for other Morpheus services. + +To enable other services to utilize it, morpheus-data is designed as a python library that can +be easily installed in any of the other services through a wheel created in the morpheus-data dockerfile + +## What contains morpheus-data + +The morpheus-data library handles the following modules: + + - **models**. This module contains all database models and data schemas of morpheus. Database models + are defined using [SQLAlchemy](https://www.sqlalchemy.org/) while data schemas are defined using + [Pydantic](https://pydantic-docs.helpmanual.io/). + ``` + morpheus-data/morpheus_data/models/ + ├── __init__.py + ├── models.py + └── schemas.py + ``` + - **repository**. This module handles the interactions with the data sources: databases and storage + bucket provider (AWS S3 by default). + ``` + morpheus-data/morpheus_data/repository/ + ├── __init__.py + ├── artwork_repository.py + ├── collection_repository.py + ├── controlnet_repository.py + ├── files + │ ├── __init__.py + │ ├── files_interface.py + │ ├── s3_files_repository.py + │ └── s3_models_repository.py + ├── firebase_repository.py + ├── model_repository.py + ├── prompt_repository.py + └── user_repository.py + ``` + - **migrations**. This module contains the database migrations. It uses [Alembic](https://alembic.sqlalchemy.org/en/latest/) + to handle migrations. + + +## Working with morpheus-data + +### Add a new dependency +To include a new dependency in morpheus-data, you need to first add it to the `pyproject.toml` file. +After that, you should rebuild the morpheus-data wheel. To do this, execute the following command: +```shell +docker compose build datalib +``` + +### Use morpheus-data as a service +To use `morpheus-data` image to run some process as run a test or run a migration, you must include a +`secret.env` file in the root of the service. This file must contain the following variables: +```shell +POSTGRES_USER=morpheus +POSTGRES_DB=morpheus +POSTGRES_PASSWORD=password +POSTGRES_HOST=postgres +POSTGRES_PORT=5432 + +# PGAdmin credentials +PGADMIN_DEFAULT_EMAIL=admin@admin.com +PGADMIN_DEFAULT_PASSWORD=admin + +FIREBASE_PROJECT_ID= +FIREBASE_PRIVATE_KEY= +FIREBASE_CLIENT_EMAIL= +FIREBASE_WEB_API_KEY= + +# AWS credentials +AWS_ACCESS_KEY_ID= +AWS_SECRET_ACCESS_KEY= + +# S3 BUCKETS ID +#------------------------ +# Bucket name where images are stored +IMAGES_BUCKET= +# Bucket name where models are stored +MODELS_BUCKET= +# Bucket/Folder where temporal images are stored +IMAGES_TEMP_BUCKET= +``` + +You can use the `secrets.env` file used in the morpheus-server for ease of use. + +For instance, to run a migration, you can execute the following command: +```shell +docker compose run --rm datalib alembic revision --autogenerate -m "Migration description" +``` +and it needs to use the `secret.env` file to connect to the database. + + + + +### Add or modify a database model +To add or modify a database model, you must do the following: +- modify the `models.py` file +- create the migration + ```shell + docker compose run --rm datalib alembic revision --autogenerate -m "Migration description" + ``` +- apply the migration + ```shell + docker compose run --rm datalib alembic upgrade head + ``` +- build the morpheus-data wheel + ```shell + docker compose build datalib + ``` +- update and install the wheel in the service that uses it + ```shell + docker compose build + ``` + +### Add or modify a data schema +To add or modify a data schema, just modify the `schemas.py` file and build the wheel of +morpheus-data as well as update and install the wheel in the service that uses it. +```shell +docker compose build datalib +docker compose build +``` + +### Add or modify a repository +To add or modify a repository, you must modify the corresponding files in the `repository` +module or add a new file if necessary. Then you must build the morpheus-data wheel and +install the wheel on the service that uses it. +```shell +docker compose build datalib +docker compose build +``` + +## How to use morpheus-data +To use morpheus-data in another service, just import the library and use the modules you need, +since the library is installed when building the service. For example +```python +from morpheus_data.repository.user_repository import UserRepository +from morpheus_data.models import User +from morpheus_data.schemas import UserCreate +from morpheus_data.repository.firebase_repository import FirebaseRepository +from morpheus_data.repository.files.s3_files_repository import S3FilesRepository +``` + + + diff --git a/morpheus-server/alembic.ini b/morpheus-data/alembic.ini similarity index 98% rename from morpheus-server/alembic.ini rename to morpheus-data/alembic.ini index c0871191..c688df4d 100644 --- a/morpheus-server/alembic.ini +++ b/morpheus-data/alembic.ini @@ -2,7 +2,7 @@ [alembic] # path to migration scripts -script_location = app/migrations +script_location = morpheus_data/migrations # template used to generate migration file names; The default value is %%(rev)s_%%(slug)s # Uncomment the line below if you want the files to be prepended with date and time diff --git a/morpheus-data/meta.json b/morpheus-data/meta.json new file mode 100644 index 00000000..75d13b47 --- /dev/null +++ b/morpheus-data/meta.json @@ -0,0 +1,3 @@ +{ + "name": "morpheus-data" +} \ No newline at end of file diff --git a/morpheus-server/app/repository/files/__init__.py b/morpheus-data/morpheus_data/__init__.py similarity index 100% rename from morpheus-server/app/repository/files/__init__.py rename to morpheus-data/morpheus_data/__init__.py diff --git a/morpheus-data/morpheus_data/config.py b/morpheus-data/morpheus_data/config.py new file mode 100644 index 00000000..563dc352 --- /dev/null +++ b/morpheus-data/morpheus_data/config.py @@ -0,0 +1,56 @@ +from enum import Enum +from functools import lru_cache + +from pydantic import BaseSettings, PostgresDsn + + +class EnvironmentEnum(str, Enum): + local = "local" + local_mps = "local-mps" + dev = "dev" + stage = "stage" + prod = "prod" + + +class Settings(BaseSettings): + postgres_user: str = "postgres" + postgres_password: str = "password" + postgres_host: str = "postgres" + postgres_port: str = "5432" + postgres_db: str = "morpheus" + + firebase_project_id: str + firebase_private_key: str + firebase_client_email: str + firebase_web_api_key: str + + bucket_type: str = "S3" + images_bucket: str + images_temp_bucket: str + models_bucket: str + + aws_access_key_id: str = "" + aws_secret_access_key: str = "" + + model_default: str = "stabilityai/stable-diffusion-2" + sampler_default: str = "PNDMScheduler" + max_num_images: int = 4 + + class Config: + env_file = "secrets.env" + + def get_db_url(self) -> str: + return PostgresDsn.build( + scheme="postgresql", + user=self.postgres_user, + password=self.postgres_password, + host=self.postgres_host, + port=self.postgres_port, + path=f"/{self.postgres_db}", + ) + + +@lru_cache() +def get_settings() -> Settings: + settings = Settings() + return settings diff --git a/morpheus-server/app/database.py b/morpheus-data/morpheus_data/database.py similarity index 89% rename from morpheus-server/app/database.py rename to morpheus-data/morpheus_data/database.py index b6104333..d17c8c3b 100644 --- a/morpheus-server/app/database.py +++ b/morpheus-data/morpheus_data/database.py @@ -2,7 +2,7 @@ from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import sessionmaker -from app.config import get_settings +from morpheus_data.config import get_settings settings = get_settings() engine = create_engine(settings.get_db_url()) diff --git a/morpheus-server/app/migrations/README b/morpheus-data/morpheus_data/migrations/README similarity index 100% rename from morpheus-server/app/migrations/README rename to morpheus-data/morpheus_data/migrations/README diff --git a/morpheus-server/app/migrations/env.py b/morpheus-data/morpheus_data/migrations/env.py similarity index 93% rename from morpheus-server/app/migrations/env.py rename to morpheus-data/morpheus_data/migrations/env.py index bbaa5abc..734b654f 100644 --- a/morpheus-server/app/migrations/env.py +++ b/morpheus-data/morpheus_data/migrations/env.py @@ -2,7 +2,7 @@ from alembic import context -import app.models.models as models +import morpheus_data.models.models as models # this is the Alembic Config object, which provides # access to the values within the .ini file in use. @@ -38,7 +38,7 @@ def run_migrations_offline() -> None: script output. """ - from app.config import get_settings + from morpheus_data.config import get_settings settings = get_settings() url = settings.get_db_url() @@ -65,7 +65,7 @@ def run_migrations_online() -> None: # prefix="sqlalchemy.", # poolclass=pool.NullPool, # ) - from app.database import engine + from morpheus_data.database import engine connectable = engine with connectable.connect() as connection: diff --git a/morpheus-server/app/migrations/script.py.mako b/morpheus-data/morpheus_data/migrations/script.py.mako similarity index 100% rename from morpheus-server/app/migrations/script.py.mako rename to morpheus-data/morpheus_data/migrations/script.py.mako diff --git a/morpheus-server/app/migrations/versions/cd0b4780c076_initial_migration.py b/morpheus-data/morpheus_data/migrations/versions/cd0b4780c076_initial_migration.py similarity index 100% rename from morpheus-server/app/migrations/versions/cd0b4780c076_initial_migration.py rename to morpheus-data/morpheus_data/migrations/versions/cd0b4780c076_initial_migration.py diff --git a/morpheus-data/morpheus_data/models/__init__.py b/morpheus-data/morpheus_data/models/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/morpheus-server/app/models/models.py b/morpheus-data/morpheus_data/models/models.py similarity index 99% rename from morpheus-server/app/models/models.py rename to morpheus-data/morpheus_data/models/models.py index 318703fa..2d7b524b 100644 --- a/morpheus-server/app/models/models.py +++ b/morpheus-data/morpheus_data/models/models.py @@ -4,7 +4,7 @@ from sqlalchemy.dialects.postgresql import UUID from sqlalchemy.orm import relationship -from app.database import Base +from morpheus_data.database import Base class User(Base): diff --git a/morpheus-data/morpheus_data/models/schemas.py b/morpheus-data/morpheus_data/models/schemas.py new file mode 100644 index 00000000..302ee9a4 --- /dev/null +++ b/morpheus-data/morpheus_data/models/schemas.py @@ -0,0 +1,300 @@ +from enum import Enum +from typing import Optional +from uuid import UUID + +from morpheus_data.config import get_settings +from pydantic import BaseModel, Field, validator + +settings = get_settings() + + +class User(BaseModel): + email: str + name: str = None + bio: str = None + avatar: str = None + phone: str = None + + class Config: + orm_mode = True + schema_extra = { + "example": { + "email": "juan.david@monadical.com", + "name": "Juan Arias", + "bio": "Juan Arias biography", + "avatar": "https://upload.wikimedia.org/wikipedia/en/8/86/Avatar_Aang.png", # noqa + "phone": "+573003000000", + } + } + + +class CollectionCreate(BaseModel): + name: str + description: str = None + image: str = None + + class Config: + schema_extra = { + "example": { + "name": "Collection name", + "description": "Collection description", + "image": "https://upload.wikimedia.org/wikipedia/en/8/86/Avatar_Aang.png", # noqa + } + } + + +class Collection(BaseModel): + id: UUID + name: str + description: str = None + image: str = None + + class Config: + orm_mode = True + schema_extra = { + "example": { + "id": "c0a80121-7ac0-11eb-9439-0242ac130002", + "name": "Collection Name", + "description": "Collection description", + "image": "https://upload.wikimedia.org/wikipedia/en/8/86/Avatar_Aang.png", # noqa + } + } + + +class ControlNetType(str, Enum): + canny = "canny" + hed = "hed" + depth = "depth" + seg = "seg" + normalmap = "normalmap" + mlsd = "mlsd" + scribble = "scribble" + poses = "poses" + + +class Prompt(BaseModel): + prompt: str = Field(..., min_length=1) + model: str + sampler: str + negative_prompt: str + width: int = 512 + height: int = 512 + num_inference_steps: int = 50 + guidance_scale: int = 10 + num_images_per_prompt: int = 1 + generator: int = -1 + strength: Optional[float] = 0.75 + use_lora: Optional[bool] = False + lora_path: Optional[str] = "" + lora_scale: Optional[float] = 1.0 + use_embedding: Optional[bool] = False + embedding_path: Optional[str] = "" + + @validator("model") + def check_if_empty_model(cls, model): + return model or settings.model_default + + @validator("sampler") + def check_if_empty_sampler(cls, sampler): + return sampler or settings.sampler_default + + @validator("negative_prompt") + def check_if_empty_neg_prompt(cls, neg_prompt): + return neg_prompt or None + + @validator("num_images_per_prompt") + def check_if_valid_num_images_per_prompt(cls, num_images_per_prompt): + if num_images_per_prompt <= 0: + return 1 + elif num_images_per_prompt >= settings.max_num_images: + return settings.max_num_images + else: + return num_images_per_prompt + + class Config: + orm_mode = True + schema_extra = { + "example": { + "prompt": "Prompt text", + "model": "stabilityai/stable-diffusion-2", + "sampler": "Euler", + "negative_prompt": "Negative prompt text", + "width": 512, + "height": 512, + "num_inference_steps": 50, + "guidance_scale": 10, + "num_images_per_prompt": 1, + "generator": -1, + "strength": 0.75, + "use_lora": False, + "lora_path": "", + "lora_scale": 1.0, + "use_embedding": False, + "embedding_path": "", + } + } + + +class PromptControlNet(Prompt): + controlnet_model: str + controlnet_type: ControlNetType + + @validator("controlnet_model") + def check_if_empty_cnet_model(cls, cnet_model): + return cnet_model or None + + +class MagicPrompt(BaseModel): + prompt: str + + +class ArtWorkCreate(BaseModel): + title: str + image: str = None + prompt: Prompt = None + collection_id: UUID = None + + class Config: + schema_extra = { + "example": { + "title": "Artwork Title", + "image": "https://upload.wikimedia.org/wikipedia/en/8/86/Avatar_Aang.png", # noqa + "prompt": { + "prompt": "Prompt text", + "width": 512, + "height": 512, + "num_inference_steps": 50, + "guidance_scale": 10, + "num_images_per_prompt": 1, + "generator": -1, + }, + "collection_id": "c0a80121-7ac0-11eb-9439-0242ac130002", + } + } + + +class ArtWork(BaseModel): + id: UUID + title: str + image: str = None + prompt: Prompt = None + collection_id: UUID = None + + class Config: + orm_mode = True + schema_extra = { + "example": { + "id": "c0a80121-7ac0-11eb-9439-0242ac130002", + "title": "Artwork Title", + "image": "https://upload.wikimedia.org/wikipedia/en/8/86/Avatar_Aang.png", # noqa + "prompt": { + "prompt": "Prompt text", + "width": 512, + "height": 512, + "num_inference_steps": 50, + "guidance_scale": 10, + "num_images_per_prompt": 1, + "generator": -1, + }, + } + } + + +class StableDiffusionSchema(BaseModel): + prompt: str = Field(..., min_length=1) + + +class SDModelCreate(BaseModel): + name: str + source: str + description: str = None + is_active: bool = True + url_docs: str = None + text2img: bool | None = False + img2img: bool | None = False + inpainting: bool | None = False + controlnet: bool | None = False + pix2pix: bool | None = False + + class Config: + schema_extra = { + "example": { + "name": "Model Name", + "source": "https://modelurl.com", + "description": "Model description", + "is_active": True, + "url_docs": "https://modeldocs.com", + "text2img": False, + "img2img": False, + "inpainting": False, + "controlnet": False, + "pix2pix": False, + } + } + + +class SDModel(SDModelCreate): + id: UUID + + class Config: + orm_mode = True + schema_extra = { + "example": { + "id": "c0a80121-7ac0-11eb-9439-0242ac130002", + "name": "Model Name", + "source": "https://modelurl.com", + "description": "Model description", + "is_active": True, + "url_docs": "https://modeldocs.com", + "text2img": False, + "img2img": False, + "inpainting": False, + "controlnet": False, + "pix2pix": False, + } + } + + +class ControlNetModelCreate(BaseModel): + name: str + type: str + source: str + description: str = None + is_active: bool = True + url_docs: str = None + + class Config: + schema_extra = { + "example": { + "name": "Model Name", + "type": "mname", + "source": "https://modelurl.com", + "description": "Model description", + "is_active": True, + "url_docs": "https://modeldocs.com", + } + } + + +class ControlNetModel(ControlNetModelCreate): + id: UUID + + class Config: + orm_mode = True + schema_extra = { + "example": { + "id": "c0a80121-7ac0-11eb-9439-0242ac130002", + "name": "Model Name", + "type": "mname", + "source": "https://modelurl.com", + "description": "Model description", + "is_active": True, + "url_docs": "https://modeldocs.com", + } + } + + +class SamplerModel(BaseModel): + id: str + name: str + description: str diff --git a/morpheus-data/morpheus_data/repository/__init__.py b/morpheus-data/morpheus_data/repository/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/morpheus-server/app/repository/artwork_repository.py b/morpheus-data/morpheus_data/repository/artwork_repository.py similarity index 95% rename from morpheus-server/app/repository/artwork_repository.py rename to morpheus-data/morpheus_data/repository/artwork_repository.py index 91f6281a..0655411b 100644 --- a/morpheus-server/app/repository/artwork_repository.py +++ b/morpheus-data/morpheus_data/repository/artwork_repository.py @@ -3,8 +3,8 @@ from sqlalchemy.orm import Session -from app.models.models import ArtWork, Prompt -from app.models.schemas import ArtWorkCreate +from morpheus_data.models.models import ArtWork, Prompt +from morpheus_data.models.schemas import ArtWorkCreate class ArtWorkRepository: diff --git a/morpheus-server/app/repository/collection_repository.py b/morpheus-data/morpheus_data/repository/collection_repository.py similarity index 95% rename from morpheus-server/app/repository/collection_repository.py rename to morpheus-data/morpheus_data/repository/collection_repository.py index d5d3574b..271e30f4 100644 --- a/morpheus-server/app/repository/collection_repository.py +++ b/morpheus-data/morpheus_data/repository/collection_repository.py @@ -3,8 +3,8 @@ from sqlalchemy.orm import Session -from app.models.models import Collection, User -from app.models.schemas import CollectionCreate +from morpheus_data.models.models import Collection, User +from morpheus_data.models.schemas import CollectionCreate class CollectionRepository: diff --git a/morpheus-server/app/repository/controlnet_repository.py b/morpheus-data/morpheus_data/repository/controlnet_repository.py similarity index 92% rename from morpheus-server/app/repository/controlnet_repository.py rename to morpheus-data/morpheus_data/repository/controlnet_repository.py index bccc12c8..0ab66754 100644 --- a/morpheus-server/app/repository/controlnet_repository.py +++ b/morpheus-data/morpheus_data/repository/controlnet_repository.py @@ -3,8 +3,8 @@ from sqlalchemy.orm import Session -from app.models.models import SDControlNetModel as CNModel -from app.models.schemas import ControlNetModelCreate +from morpheus_data.models.models import SDControlNetModel as CNModel +from morpheus_data.models.schemas import ControlNetModelCreate class ControlNetModelRepository: diff --git a/morpheus-data/morpheus_data/repository/files/__init__.py b/morpheus-data/morpheus_data/repository/files/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/morpheus-server/app/repository/files/files_interface.py b/morpheus-data/morpheus_data/repository/files/files_interface.py similarity index 100% rename from morpheus-server/app/repository/files/files_interface.py rename to morpheus-data/morpheus_data/repository/files/files_interface.py diff --git a/morpheus-server/app/repository/files/s3_files_repository.py b/morpheus-data/morpheus_data/repository/files/s3_files_repository.py similarity index 96% rename from morpheus-server/app/repository/files/s3_files_repository.py rename to morpheus-data/morpheus_data/repository/files/s3_files_repository.py index bb3c9239..eb7ffb63 100644 --- a/morpheus-server/app/repository/files/s3_files_repository.py +++ b/morpheus-data/morpheus_data/repository/files/s3_files_repository.py @@ -8,10 +8,10 @@ from fastapi import UploadFile from rich import print -from app.config import get_settings -from app.repository.files.files_interface import FileRepositoryInterface -from app.utils.images import from_image_to_bytes -from app.utils.timer import get_timestamp +from morpheus_data.config import get_settings +from morpheus_data.repository.files.files_interface import FileRepositoryInterface +from morpheus_data.utils.images import from_image_to_bytes +from morpheus_data.utils.timer import get_timestamp settings = get_settings() logger = logging.getLogger(__name__) diff --git a/morpheus-server/app/repository/files/s3_models_repository.py b/morpheus-data/morpheus_data/repository/files/s3_models_repository.py similarity index 95% rename from morpheus-server/app/repository/files/s3_models_repository.py rename to morpheus-data/morpheus_data/repository/files/s3_models_repository.py index 76334f9b..2a8a653f 100644 --- a/morpheus-server/app/repository/files/s3_models_repository.py +++ b/morpheus-data/morpheus_data/repository/files/s3_models_repository.py @@ -5,8 +5,8 @@ import boto3 import tqdm -from app.config import get_settings -from app.repository.files.files_interface import ModelRepositoryInterface +from morpheus_data.config import get_settings +from morpheus_data.repository.files.files_interface import ModelRepositoryInterface settings = get_settings() logger = logging.getLogger(__name__) diff --git a/morpheus-server/app/repository/firebase_repository.py b/morpheus-data/morpheus_data/repository/firebase_repository.py similarity index 97% rename from morpheus-server/app/repository/firebase_repository.py rename to morpheus-data/morpheus_data/repository/firebase_repository.py index 536f23d4..60d57de5 100644 --- a/morpheus-server/app/repository/firebase_repository.py +++ b/morpheus-data/morpheus_data/repository/firebase_repository.py @@ -4,7 +4,7 @@ import requests from firebase_admin import auth -from app.config import get_settings +from morpheus_data.config import get_settings settings = get_settings() diff --git a/morpheus-server/app/repository/model_repository.py b/morpheus-data/morpheus_data/repository/model_repository.py similarity index 94% rename from morpheus-server/app/repository/model_repository.py rename to morpheus-data/morpheus_data/repository/model_repository.py index 2bb1c235..1e48fb80 100644 --- a/morpheus-server/app/repository/model_repository.py +++ b/morpheus-data/morpheus_data/repository/model_repository.py @@ -3,8 +3,8 @@ from sqlalchemy.orm import Session -from app.models.models import SDModel -from app.models.schemas import SDModelCreate +from morpheus_data.models.models import SDModel +from morpheus_data.models.schemas import SDModelCreate class ModelRepository: diff --git a/morpheus-server/app/repository/prompt_repository.py b/morpheus-data/morpheus_data/repository/prompt_repository.py similarity index 92% rename from morpheus-server/app/repository/prompt_repository.py rename to morpheus-data/morpheus_data/repository/prompt_repository.py index 675e8dfc..42488f66 100644 --- a/morpheus-server/app/repository/prompt_repository.py +++ b/morpheus-data/morpheus_data/repository/prompt_repository.py @@ -2,8 +2,8 @@ from sqlalchemy.orm import Session -from app.models.models import User, Prompt -from app.models.schemas import Prompt as PromptCreate +from morpheus_data.models.models import User, Prompt +from morpheus_data.models.schemas import Prompt as PromptCreate class PromptRepository: diff --git a/morpheus-server/app/repository/user_repository.py b/morpheus-data/morpheus_data/repository/user_repository.py similarity index 94% rename from morpheus-server/app/repository/user_repository.py rename to morpheus-data/morpheus_data/repository/user_repository.py index afe53efb..6212e48d 100644 --- a/morpheus-server/app/repository/user_repository.py +++ b/morpheus-data/morpheus_data/repository/user_repository.py @@ -2,8 +2,8 @@ from sqlalchemy.orm import Session -from app.models.models import User -from app.repository.files.s3_files_repository import IMAGES_BUCKET +from morpheus_data.models.models import User +from morpheus_data.repository.files.s3_files_repository import IMAGES_BUCKET class UserRepository: diff --git a/morpheus-data/morpheus_data/utils/__init__.py b/morpheus-data/morpheus_data/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/morpheus-server/app/utils/images.py b/morpheus-data/morpheus_data/utils/images.py similarity index 100% rename from morpheus-server/app/utils/images.py rename to morpheus-data/morpheus_data/utils/images.py diff --git a/morpheus-server/app/utils/timer.py b/morpheus-data/morpheus_data/utils/timer.py similarity index 100% rename from morpheus-server/app/utils/timer.py rename to morpheus-data/morpheus_data/utils/timer.py diff --git a/morpheus-data/pyproject.toml b/morpheus-data/pyproject.toml new file mode 100644 index 00000000..bf4bfb90 --- /dev/null +++ b/morpheus-data/pyproject.toml @@ -0,0 +1,24 @@ +[project] +name = "morpheus-data" +version = "0.1.0" +description = "" +authors = [{name = "Monadical", email = "devops@monadical.com"}] +readme = "README.md" +requires-python = ">=3.9" +dependencies = [ + "pydantic>=1.10.7,<2", + "sqlalchemy==1.4.43", + "alembic>=1.10.4", + "psycopg2>=2.9.6", + "python-dotenv>=1.0.0", + "firebase-admin>=6.1.0", + "boto3>=1.26.132", + "rich>=13.3.5", + "tqdm>=4.65.0", + "pillow>=9.5.0", + "fastapi>=0.95.1", +] + +[build-system] +requires = ["setuptools", "wheel", "setuptools-scm"] +build-backend = "setuptools.build_meta" \ No newline at end of file diff --git a/morpheus-data/requirements.lint.txt b/morpheus-data/requirements.lint.txt new file mode 100644 index 00000000..4e92b9de --- /dev/null +++ b/morpheus-data/requirements.lint.txt @@ -0,0 +1,2 @@ +black +flake8 diff --git a/morpheus-data/secrets.env.dist b/morpheus-data/secrets.env.dist new file mode 100644 index 00000000..f57a95e1 --- /dev/null +++ b/morpheus-data/secrets.env.dist @@ -0,0 +1,25 @@ +POSTGRES_USER=morpheus +POSTGRES_DB=morpheus +POSTGRES_PASSWORD=password +POSTGRES_HOST=postgres +POSTGRES_PORT=5432 + +# PGAdmin credentials +PGADMIN_DEFAULT_EMAIL=admin@admin.com +PGADMIN_DEFAULT_PASSWORD=admin + +FIREBASE_PROJECT_ID= +FIREBASE_PRIVATE_KEY= +FIREBASE_CLIENT_EMAIL= +FIREBASE_WEB_API_KEY= + + +# S3 BUCKETS ID +#------------------------ +# Bucket name where images are stored +IMAGES_BUCKET= +# Bucket name where models are stored +MODELS_BUCKET= +# Bucket/Folder where temporal images are stored +IMAGES_TEMP_BUCKET= + diff --git a/morpheus-server/Dockerfile b/morpheus-server/Dockerfile index 2b0c0985..0f53c305 100644 --- a/morpheus-server/Dockerfile +++ b/morpheus-server/Dockerfile @@ -1,4 +1,5 @@ ARG BUILD_ENV="no_copy" +FROM monadicalsas/morpheus-data:latest as lib FROM nvidia/cuda:11.7.1-base-ubuntu22.04 as base # Configuration defaults @@ -17,7 +18,7 @@ ENV PYTHONDONTWRITEBYTECODE 1 ENV PYTHONUNBUFFERED 1 # Install system requirements -RUN apt-get update && apt-get install -y jq build-essential python3-dev libpq-dev python3-pip python3-venv && rm -rf /var/lib/apt/lists/* +RUN apt-get update && apt-get install -y jq build-essential curl python3-dev libpq-dev python3-pip python3-venv && rm -rf /var/lib/apt/lists/* WORKDIR "$API_ROOT" @@ -25,10 +26,19 @@ RUN pip3 install virtualenv && \ virtualenv "/opt/$VENV_NAME" ENV PATH="/opt/$VENV_NAME/bin:${PATH}" +# Install rust for some wheel building processes +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +ENV PATH="/root/.cargo/bin:${PATH}" + # Install dependencies RUN pip install --upgrade pip COPY requirements.txt . -RUN pip install torch==1.13.1 && pip install -r requirements.txt +RUN pip install torch==1.13.1 && pip install -r requirements.txt --no-cache-dir + +# Find the latest wheel file of morpheus-data in the /wheels directory and install it +COPY --from=lib /wheels /wheels +RUN latest_wheel=$(ls -1 /wheels/*.whl | sort -V | tail -n 1) && \ + pip install $latest_wheel # Create api user RUN userdel "$API_USER" && addgroup --system --gid "$API_USER_ID" "$API_USER" && \ diff --git a/morpheus-server/app/api/artwork_api.py b/morpheus-server/app/api/artwork_api.py index d7b89340..2e1cc605 100644 --- a/morpheus-server/app/api/artwork_api.py +++ b/morpheus-server/app/api/artwork_api.py @@ -5,10 +5,12 @@ from fastapi import Depends from sqlalchemy.orm import Session +from morpheus_data.database import get_db +from morpheus_data.models.schemas import ArtWork, ArtWorkCreate + from app.config import get_file_handlers -from app.database import get_db from app.integrations.firebase import get_user -from app.models.schemas import ArtWork, ArtWorkCreate, Response +from app.models.schemas import Response from app.services.artwork_services import ArtWorkService router = APIRouter() diff --git a/morpheus-server/app/api/collections_api.py b/morpheus-server/app/api/collections_api.py index 8c2ce559..c7408270 100644 --- a/morpheus-server/app/api/collections_api.py +++ b/morpheus-server/app/api/collections_api.py @@ -5,10 +5,12 @@ from fastapi import Depends from sqlalchemy.orm import Session +from morpheus_data.database import get_db +from morpheus_data.models.schemas import Collection, CollectionCreate + from app.config import get_file_handlers -from app.database import get_db from app.integrations.firebase import get_user -from app.models.schemas import Collection, CollectionCreate, Response +from app.models.schemas import Response from app.services.collection_services import CollectionService router = APIRouter() diff --git a/morpheus-server/app/api/controlnet_model_api.py b/morpheus-server/app/api/controlnet_model_api.py index 80df1dbf..331619a3 100644 --- a/morpheus-server/app/api/controlnet_model_api.py +++ b/morpheus-server/app/api/controlnet_model_api.py @@ -1,12 +1,14 @@ -from typing import Union, List +from typing import List, Union from uuid import UUID from fastapi import APIRouter from fastapi import Depends from sqlalchemy.orm import Session -from app.database import get_db -from app.models.schemas import ControlNetModelCreate, ControlNetModel, Response +from morpheus_data.database import get_db +from morpheus_data.models.schemas import ControlNetModel, ControlNetModelCreate + +from app.models.schemas import Response from app.services.controlnet_services import ControlNetModelService router = APIRouter() diff --git a/morpheus-server/app/api/files_api.py b/morpheus-server/app/api/files_api.py index f678856f..9c6a1692 100644 --- a/morpheus-server/app/api/files_api.py +++ b/morpheus-server/app/api/files_api.py @@ -3,8 +3,9 @@ from fastapi import APIRouter, Depends, File, UploadFile from sqlalchemy.orm import Session +from morpheus_data.database import get_db + from app.config import get_file_handlers -from app.database import get_db from app.integrations.firebase import get_user from app.models.schemas import Response from app.services.files_services import FilesService diff --git a/morpheus-server/app/api/models_api.py b/morpheus-server/app/api/models_api.py index cc44c00b..0943f7f4 100644 --- a/morpheus-server/app/api/models_api.py +++ b/morpheus-server/app/api/models_api.py @@ -5,8 +5,10 @@ from fastapi import Depends from sqlalchemy.orm import Session -from app.database import get_db -from app.models.schemas import Response, SDModel, SDModelCreate +from morpheus_data.database import get_db +from morpheus_data.models.schemas import SDModel, SDModelCreate + +from app.models.schemas import Response from app.services.models_services import ModelService router = APIRouter() diff --git a/morpheus-server/app/api/samplers_api.py b/morpheus-server/app/api/samplers_api.py index 203bc095..ace1d008 100644 --- a/morpheus-server/app/api/samplers_api.py +++ b/morpheus-server/app/api/samplers_api.py @@ -1,10 +1,12 @@ -from typing import Union, List +from typing import List, Union from fastapi import APIRouter -from omegaconf import OmegaConf, DictConfig +from omegaconf import DictConfig, OmegaConf + +from morpheus_data.models.schemas import SamplerModel from app.config import samplers -from app.models.schemas import SamplerModel, Response +from app.models.schemas import Response router = APIRouter() diff --git a/morpheus-server/app/api/sdiffusion_api.py b/morpheus-server/app/api/sdiffusion_api.py index 020e598b..04bcca3f 100644 --- a/morpheus-server/app/api/sdiffusion_api.py +++ b/morpheus-server/app/api/sdiffusion_api.py @@ -1,13 +1,12 @@ from celery.result import AsyncResult from fastapi import APIRouter, Depends, File, UploadFile -from app.database import get_db +from morpheus_data.database import get_db +from morpheus_data.models.schemas import MagicPrompt, Prompt, PromptControlNet + from app.error.error import ImageNotProvidedError, ModelNotFoundError, UserNotFoundError from app.integrations.firebase import get_user from app.models.schemas import ( - MagicPrompt, - PromptControlNet, - Prompt, Response, TaskResponse, TaskStatus, diff --git a/morpheus-server/app/api/user_api.py b/morpheus-server/app/api/user_api.py index 17775d50..1c1a77ef 100644 --- a/morpheus-server/app/api/user_api.py +++ b/morpheus-server/app/api/user_api.py @@ -4,9 +4,11 @@ from fastapi import Depends from sqlalchemy.orm import Session -from app.database import get_db +from morpheus_data.database import get_db +from morpheus_data.models.schemas import User + from app.integrations.firebase import get_user -from app.models.schemas import User, Response +from app.models.schemas import Response from app.services.user_services import UserService router = APIRouter() diff --git a/morpheus-server/app/app.py b/morpheus-server/app/app.py index ad435dea..19ee0b60 100644 --- a/morpheus-server/app/app.py +++ b/morpheus-server/app/app.py @@ -6,6 +6,8 @@ from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware +from morpheus_data.database import engine, Base + from app.api.artwork_api import router as ArtworksRouter from app.api.auth_api import router as AuthRouter from app.api.collections_api import router as CollectionsRouter @@ -15,8 +17,6 @@ from app.api.samplers_api import router as SamplersRouter from app.api.sdiffusion_api import router as SDiffusionRouter from app.api.user_api import router as UserRouter -from app.database import engine -from app.models.models import Base from config.logger import InitLogger SENTRY_DSN = os.environ.get("SENTRY_DSN", "") diff --git a/morpheus-server/app/celery/mlmodels/magic_prompt.py b/morpheus-server/app/celery/mlmodels/magic_prompt.py index 4138d006..4568d1f8 100644 --- a/morpheus-server/app/celery/mlmodels/magic_prompt.py +++ b/morpheus-server/app/celery/mlmodels/magic_prompt.py @@ -6,8 +6,9 @@ from dynamicprompts.generators.magicprompt import MagicPromptGenerator from loguru import logger +from morpheus_data.models.schemas import MagicPrompt + from app.config import get_settings -from app.models.schemas import MagicPrompt settings = get_settings() diff --git a/morpheus-server/app/celery/mlmodels/stable_diffusion.py b/morpheus-server/app/celery/mlmodels/stable_diffusion.py index d2d6abac..f98487c3 100644 --- a/morpheus-server/app/celery/mlmodels/stable_diffusion.py +++ b/morpheus-server/app/celery/mlmodels/stable_diffusion.py @@ -10,10 +10,11 @@ ) from loguru import logger +from morpheus_data.models.schemas import Prompt, PromptControlNet + +import app.utils.lora_ti_utils as lora_ti_utils from app.celery.mlmodels.controlnet import preprocessing_image from app.config import get_settings -from app.models.schemas import Prompt, PromptControlNet -import app.utils.lora_ti_utils as lora_ti_utils settings = get_settings() diff --git a/morpheus-server/app/celery/tasks/magic_prompt.py b/morpheus-server/app/celery/tasks/magic_prompt.py index 1c26e91d..408567ce 100644 --- a/morpheus-server/app/celery/tasks/magic_prompt.py +++ b/morpheus-server/app/celery/tasks/magic_prompt.py @@ -4,10 +4,11 @@ from loguru import logger from torch.cuda import OutOfMemoryError +from morpheus_data.models.schemas import Prompt + from app.celery.workers.stable_diffusion_app import app from app.config import get_settings from app.error.error import ModelLoadError, OutOfMemoryGPUError -from app.models.schemas import Prompt settings = get_settings() diff --git a/morpheus-server/app/celery/tasks/stable_diffusion.py b/morpheus-server/app/celery/tasks/stable_diffusion.py index f342c6ba..82bb90e0 100644 --- a/morpheus-server/app/celery/tasks/stable_diffusion.py +++ b/morpheus-server/app/celery/tasks/stable_diffusion.py @@ -5,10 +5,11 @@ from loguru import logger from torch.cuda import OutOfMemoryError +from morpheus_data.models.schemas import Prompt, PromptControlNet + from app.celery.workers.stable_diffusion_app import app from app.config import get_settings, get_file_handlers from app.error.error import ModelLoadError, OutOfMemoryGPUError -from app.models.schemas import Prompt, PromptControlNet from app.services.files_services import FilesService from app.utils.decorators import ( check_environment, diff --git a/morpheus-server/app/config.py b/morpheus-server/app/config.py index bec6bbbd..c9b3f368 100644 --- a/morpheus-server/app/config.py +++ b/morpheus-server/app/config.py @@ -3,7 +3,9 @@ from functools import lru_cache from omegaconf import OmegaConf -from pydantic import BaseSettings, PostgresDsn +from pydantic import PostgresDsn + +from morpheus_data.config import Settings as SettingsData class EnvironmentEnum(str, Enum): @@ -14,27 +16,9 @@ class EnvironmentEnum(str, Enum): prod = "prod" -class Settings(BaseSettings): - postgres_user: str = "postgres" - postgres_password: str = "password" - postgres_host: str = "postgres" - postgres_port: str = "5432" - postgres_db: str = "morpheus" - - firebase_project_id: str - firebase_private_key: str - firebase_client_email: str - firebase_web_api_key: str - - bucket_type: str = "S3" - images_bucket: str - images_temp_bucket: str - models_bucket: str - - aws_access_key_id: str = "" - aws_secret_access_key: str = "" - +class Settings(SettingsData): environment: EnvironmentEnum = EnvironmentEnum.local + model_parent_path: str = "/mnt/" model_default: str = "stabilityai/stable-diffusion-2" controlnet_model_default = "lllyasviel/sd-controlnet-canny" @@ -66,11 +50,6 @@ def get_db_url(self) -> str: ) -class APISettings(BaseSettings): - sd_host: str - testing: bool - - @lru_cache() def get_settings() -> Settings: settings = Settings() @@ -84,7 +63,9 @@ def read_available_samplers(file: str): samplers = read_available_samplers("config/sd-schedulers.yaml") -file_handlers = {"S3": {"module": "app.repository.files.s3_files_repository", "handler": "S3ImagesRepository"}} +file_handlers = { + "S3": {"module": "morpheus_data.repository.files.s3_files_repository", "handler": "S3ImagesRepository"} +} @lru_cache() diff --git a/morpheus-server/app/models/schemas.py b/morpheus-server/app/models/schemas.py index a381db64..fe5e11bf 100644 --- a/morpheus-server/app/models/schemas.py +++ b/morpheus-server/app/models/schemas.py @@ -1,8 +1,7 @@ from enum import Enum -from typing import Any, Optional -from uuid import UUID +from typing import Any -from pydantic import BaseModel, Field, validator +from pydantic import BaseModel from app.config import get_settings @@ -26,295 +25,3 @@ class TaskResponse(BaseModel): status: TaskStatus message: str = "Operation completed successfully" data: Any = None - - -class User(BaseModel): - email: str - name: str = None - bio: str = None - avatar: str = None - phone: str = None - - class Config: - orm_mode = True - schema_extra = { - "example": { - "email": "juan.david@monadical.com", - "name": "Juan Arias", - "bio": "Juan Arias biography", - "avatar": "https://upload.wikimedia.org/wikipedia/en/8/86/Avatar_Aang.png", # noqa - "phone": "+573003000000", - } - } - - -class CollectionCreate(BaseModel): - name: str - description: str = None - image: str = None - - class Config: - schema_extra = { - "example": { - "name": "Collection name", - "description": "Collection description", - "image": "https://upload.wikimedia.org/wikipedia/en/8/86/Avatar_Aang.png", # noqa - } - } - - -class Collection(BaseModel): - id: UUID - name: str - description: str = None - image: str = None - - class Config: - orm_mode = True - schema_extra = { - "example": { - "id": "c0a80121-7ac0-11eb-9439-0242ac130002", - "name": "Collection Name", - "description": "Collection description", - "image": "https://upload.wikimedia.org/wikipedia/en/8/86/Avatar_Aang.png", # noqa - } - } - - -class ControlNetType(str, Enum): - canny = "canny" - hed = "hed" - depth = "depth" - seg = "seg" - normalmap = "normalmap" - mlsd = "mlsd" - scribble = "scribble" - poses = "poses" - - -class Prompt(BaseModel): - prompt: str = Field(..., min_length=1) - model: str - sampler: str - negative_prompt: str - width: int = 512 - height: int = 512 - num_inference_steps: int = 50 - guidance_scale: int = 10 - num_images_per_prompt: int = 1 - generator: int = -1 - strength: Optional[float] = 0.75 - use_lora: Optional[bool] = False - lora_path: Optional[str] = "" - lora_scale: Optional[float] = 1.0 - use_embedding: Optional[bool] = False - embedding_path: Optional[str] = "" - - @validator("model") - def check_if_empty_model(cls, model): - return model or settings.model_default - - @validator("sampler") - def check_if_empty_sampler(cls, sampler): - return sampler or settings.sampler_default - - @validator("negative_prompt") - def check_if_empty_neg_prompt(cls, neg_prompt): - return neg_prompt or None - - @validator("num_images_per_prompt") - def check_if_valid_num_images_per_prompt(cls, num_images_per_prompt): - if num_images_per_prompt <= 0: - return 1 - elif num_images_per_prompt >= settings.max_num_images: - return settings.max_num_images - else: - return num_images_per_prompt - - class Config: - orm_mode = True - schema_extra = { - "example": { - "prompt": "Prompt text", - "model": "stabilityai/stable-diffusion-2", - "sampler": "Euler", - "negative_prompt": "Negative prompt text", - "width": 512, - "height": 512, - "num_inference_steps": 50, - "guidance_scale": 10, - "num_images_per_prompt": 1, - "generator": -1, - "strength": 0.75, - "use_lora": False, - "lora_path": "", - "lora_scale": 1.0, - "use_embedding": False, - "embedding_path": "", - } - } - - -class PromptControlNet(Prompt): - controlnet_model: str - controlnet_type: ControlNetType - - @validator("controlnet_model") - def check_if_empty_cnet_model(cls, cnet_model): - return cnet_model or None - - -class MagicPrompt(BaseModel): - prompt: str - - -class ArtWorkCreate(BaseModel): - title: str - image: str = None - prompt: Prompt = None - collection_id: UUID = None - - class Config: - schema_extra = { - "example": { - "title": "Artwork Title", - "image": "https://upload.wikimedia.org/wikipedia/en/8/86/Avatar_Aang.png", # noqa - "prompt": { - "prompt": "Prompt text", - "width": 512, - "height": 512, - "num_inference_steps": 50, - "guidance_scale": 10, - "num_images_per_prompt": 1, - "generator": -1, - }, - "collection_id": "c0a80121-7ac0-11eb-9439-0242ac130002", - } - } - - -class ArtWork(BaseModel): - id: UUID - title: str - image: str = None - prompt: Prompt = None - collection_id: UUID = None - - class Config: - orm_mode = True - schema_extra = { - "example": { - "id": "c0a80121-7ac0-11eb-9439-0242ac130002", - "title": "Artwork Title", - "image": "https://upload.wikimedia.org/wikipedia/en/8/86/Avatar_Aang.png", # noqa - "prompt": { - "prompt": "Prompt text", - "width": 512, - "height": 512, - "num_inference_steps": 50, - "guidance_scale": 10, - "num_images_per_prompt": 1, - "generator": -1, - }, - } - } - - -class StableDiffusionSchema(BaseModel): - prompt: str = Field(..., min_length=1) - - -class SDModelCreate(BaseModel): - name: str - source: str - description: str = None - is_active: bool = True - url_docs: str = None - text2img: bool | None = False - img2img: bool | None = False - inpainting: bool | None = False - controlnet: bool | None = False - pix2pix: bool | None = False - - class Config: - schema_extra = { - "example": { - "name": "Model Name", - "source": "https://modelurl.com", - "description": "Model description", - "is_active": True, - "url_docs": "https://modeldocs.com", - "text2img": False, - "img2img": False, - "inpainting": False, - "controlnet": False, - "pix2pix": False, - } - } - - -class SDModel(SDModelCreate): - id: UUID - - class Config: - orm_mode = True - schema_extra = { - "example": { - "id": "c0a80121-7ac0-11eb-9439-0242ac130002", - "name": "Model Name", - "source": "https://modelurl.com", - "description": "Model description", - "is_active": True, - "url_docs": "https://modeldocs.com", - "text2img": False, - "img2img": False, - "inpainting": False, - "controlnet": False, - "pix2pix": False, - } - } - - -class ControlNetModelCreate(BaseModel): - name: str - type: str - source: str - description: str = None - is_active: bool = True - url_docs: str = None - - class Config: - schema_extra = { - "example": { - "name": "Model Name", - "type": "mname", - "source": "https://modelurl.com", - "description": "Model description", - "is_active": True, - "url_docs": "https://modeldocs.com", - } - } - - -class ControlNetModel(ControlNetModelCreate): - id: UUID - - class Config: - orm_mode = True - schema_extra = { - "example": { - "id": "c0a80121-7ac0-11eb-9439-0242ac130002", - "name": "Model Name", - "type": "mname", - "source": "https://modelurl.com", - "description": "Model description", - "is_active": True, - "url_docs": "https://modeldocs.com", - } - } - - -class SamplerModel(BaseModel): - id: str - name: str - description: str diff --git a/morpheus-server/app/repository/sdiffusion_repository.py b/morpheus-server/app/repository/sdiffusion_repository.py index 54d7cb40..76beda1a 100644 --- a/morpheus-server/app/repository/sdiffusion_repository.py +++ b/morpheus-server/app/repository/sdiffusion_repository.py @@ -1,6 +1,8 @@ from PIL import Image from loguru import logger +from morpheus_data.models.schemas import MagicPrompt, Prompt, PromptControlNet + from app.celery.tasks.magic_prompt import generate_stable_diffusion_magicprompt_output_task from app.celery.tasks.stable_diffusion import ( generate_stable_diffusion_controlnet_output_task, @@ -10,7 +12,6 @@ generate_stable_diffusion_text2img_output_task, generate_stable_diffusion_upscale_output_task, ) -from app.models.schemas import MagicPrompt, Prompt, PromptControlNet class StableDiffusionRepository: diff --git a/morpheus-server/app/services/artwork_services.py b/morpheus-server/app/services/artwork_services.py index 3cd84eed..0f2e3445 100644 --- a/morpheus-server/app/services/artwork_services.py +++ b/morpheus-server/app/services/artwork_services.py @@ -3,12 +3,12 @@ from sqlalchemy.orm import Session -from app.models.schemas import ArtWork, ArtWorkCreate -from app.repository.artwork_repository import ArtWorkRepository -from app.repository.collection_repository import CollectionRepository -from app.repository.files.files_interface import FileRepositoryInterface -from app.repository.prompt_repository import PromptRepository -from app.repository.user_repository import UserRepository +from morpheus_data.models.schemas import ArtWork, ArtWorkCreate +from morpheus_data.repository.artwork_repository import ArtWorkRepository +from morpheus_data.repository.collection_repository import CollectionRepository +from morpheus_data.repository.files.files_interface import FileRepositoryInterface +from morpheus_data.repository.prompt_repository import PromptRepository +from morpheus_data.repository.user_repository import UserRepository class ArtWorkService: diff --git a/morpheus-server/app/services/collection_services.py b/morpheus-server/app/services/collection_services.py index a7d0c5f6..f66dd363 100644 --- a/morpheus-server/app/services/collection_services.py +++ b/morpheus-server/app/services/collection_services.py @@ -3,10 +3,10 @@ from sqlalchemy.orm import Session -from app.models.schemas import Collection, CollectionCreate -from app.repository.collection_repository import CollectionRepository -from app.repository.files.files_interface import FileRepositoryInterface -from app.repository.user_repository import UserRepository +from morpheus_data.models.schemas import Collection, CollectionCreate +from morpheus_data.repository.collection_repository import CollectionRepository +from morpheus_data.repository.files.files_interface import FileRepositoryInterface +from morpheus_data.repository.user_repository import UserRepository class CollectionService: diff --git a/morpheus-server/app/services/controlnet_services.py b/morpheus-server/app/services/controlnet_services.py index 6acc87c0..876efbda 100644 --- a/morpheus-server/app/services/controlnet_services.py +++ b/morpheus-server/app/services/controlnet_services.py @@ -3,8 +3,8 @@ from sqlalchemy.orm import Session -from app.models.schemas import ControlNetModel, ControlNetModelCreate -from app.repository.controlnet_repository import ControlNetModelRepository +from morpheus_data.models.schemas import ControlNetModel, ControlNetModelCreate +from morpheus_data.repository.controlnet_repository import ControlNetModelRepository class ControlNetModelService: diff --git a/morpheus-server/app/services/files_services.py b/morpheus-server/app/services/files_services.py index d35a2ddb..fdea8ea7 100644 --- a/morpheus-server/app/services/files_services.py +++ b/morpheus-server/app/services/files_services.py @@ -4,8 +4,8 @@ from PIL import Image from sqlalchemy.orm import Session -from app.repository.files.files_interface import FileRepositoryInterface -from app.repository.user_repository import UserRepository +from morpheus_data.repository.files.files_interface import FileRepositoryInterface +from morpheus_data.repository.user_repository import UserRepository logger = logging.getLogger(__name__) diff --git a/morpheus-server/app/services/models_services.py b/morpheus-server/app/services/models_services.py index 4493ab25..04e5a9ac 100644 --- a/morpheus-server/app/services/models_services.py +++ b/morpheus-server/app/services/models_services.py @@ -3,8 +3,8 @@ from sqlalchemy.orm import Session -from app.models.schemas import SDModel, SDModelCreate -from app.repository.model_repository import ModelRepository +from morpheus_data.models.schemas import SDModel, SDModelCreate +from morpheus_data.repository.model_repository import ModelRepository class ModelService: diff --git a/morpheus-server/app/services/sdiffusion_services.py b/morpheus-server/app/services/sdiffusion_services.py index f47f2dcb..b576375f 100644 --- a/morpheus-server/app/services/sdiffusion_services.py +++ b/morpheus-server/app/services/sdiffusion_services.py @@ -3,13 +3,14 @@ from PIL import Image from sqlalchemy.orm import Session +from morpheus_data.models.schemas import MagicPrompt, Prompt, PromptControlNet +from morpheus_data.repository.model_repository import ModelRepository +from morpheus_data.repository.user_repository import UserRepository +from morpheus_data.utils.images import get_rgb_image_from_bytes, resize_image + from app.config import get_settings from app.error.error import ImageNotProvidedError, ModelNotFoundError, UserNotFoundError -from app.models.schemas import MagicPrompt, PromptControlNet, Prompt -from app.repository.model_repository import ModelRepository from app.repository.sdiffusion_repository import StableDiffusionRepository -from app.repository.user_repository import UserRepository -from app.utils.images import get_rgb_image_from_bytes, resize_image class StableDiffusionService: diff --git a/morpheus-server/app/services/user_services.py b/morpheus-server/app/services/user_services.py index bcff08f0..0c64d401 100644 --- a/morpheus-server/app/services/user_services.py +++ b/morpheus-server/app/services/user_services.py @@ -1,11 +1,11 @@ -from typing import Union, List +from typing import List, Union from sqlalchemy.orm import Session -from app.models.schemas import User -from app.repository.collection_repository import CollectionRepository -from app.repository.firebase_repository import FirebaseRepository -from app.repository.user_repository import UserRepository +from morpheus_data.models.schemas import User +from morpheus_data.repository.collection_repository import CollectionRepository +from morpheus_data.repository.firebase_repository import FirebaseRepository +from morpheus_data.repository.user_repository import UserRepository class UserService: diff --git a/morpheus-server/monitor/cloudwatch.py b/morpheus-server/monitor/cloudwatch.py index 02495205..da536e1e 100644 --- a/morpheus-server/monitor/cloudwatch.py +++ b/morpheus-server/monitor/cloudwatch.py @@ -57,7 +57,7 @@ def cloudwatch_metric(): print("Scheduled is None ...") except Exception as e: print("Exception in scheduled:" + str(e)) - + if num_workers != 0: metric = pending_tasks_count / num_workers cloudwatch = boto3.client("cloudwatch", region_name=CLOUDWATCH_REGION) @@ -68,6 +68,7 @@ def cloudwatch_metric(): return True return False + while True: try: result = cloudwatch_metric() diff --git a/morpheus-server/scripts/models/Dockerfile b/morpheus-server/scripts/models/Dockerfile index 8d0f0e86..dc503670 100644 --- a/morpheus-server/scripts/models/Dockerfile +++ b/morpheus-server/scripts/models/Dockerfile @@ -1,3 +1,4 @@ +FROM monadicalsas/morpheus-data:latest as lib FROM python:3.10-slim # Configuration defaults @@ -13,7 +14,7 @@ ENV PYTHONDONTWRITEBYTECODE 1 ENV PYTHONUNBUFFERED 1 # Install system requirements -RUN apt-get update && apt-get install -y jq build-essential python3-dev libpq-dev python3-pip python3-venv && rm -rf /var/lib/apt/lists/* +RUN apt-get update && apt-get install -y jq build-essential curl python3-dev libpq-dev python3-pip python3-venv && rm -rf /var/lib/apt/lists/* WORKDIR "$API_ROOT" @@ -22,9 +23,19 @@ RUN pip3 install virtualenv && \ ENV PATH="/opt/$VENV_NAME/bin:${PATH}" RUN dir -s +# Install rust for some wheel building processes +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +ENV PATH="/root/.cargo/bin:${PATH}" + # Install & use pipenv -RUN pip install --upgrade pip pipenv +RUN pip install --upgrade pip COPY requirements.txt . RUN pip install torch==1.13.1 && pip install -r requirements.txt + +# Find the latest wheel file of morpheus_data in the /wheels directory and install it +COPY --from=lib /wheels /wheels +RUN latest_wheel=$(ls -1 /wheels/*.whl | sort -V | tail -n 1) && \ + pip install $latest_wheel + COPY . . ENTRYPOINT ["python", "scripts/models/cli.py"] diff --git a/morpheus-server/scripts/models/config.py b/morpheus-server/scripts/models/config.py index 4dc98e19..b91ec45e 100644 --- a/morpheus-server/scripts/models/config.py +++ b/morpheus-server/scripts/models/config.py @@ -43,8 +43,8 @@ def get_settings() -> Settings: api_server_urls = { "local": "http://api:8001", - "staging": "http://216.153.52.83:8001", - 'production': "https://api-morpheus.monadical.io", + "staging": "http://216.153.50.111:8001", + "production": "https://api-morpheus.monadical.io", } # API service endpoints diff --git a/morpheus-server/scripts/models/s3.py b/morpheus-server/scripts/models/s3.py index 1a475e83..df854a7a 100644 --- a/morpheus-server/scripts/models/s3.py +++ b/morpheus-server/scripts/models/s3.py @@ -1,4 +1,3 @@ -import sys from glob import glob from pathlib import Path @@ -7,8 +6,7 @@ from config import Target, config_file, download_model from utils import load_config_from_file -sys.path.append(".") -from app.repository.files.s3_models_repository import S3ModelFileRepository # noqa: E402 +from morpheus_data.repository.files.s3_models_repository import S3ModelFileRepository # noqa: E402 app = typer.Typer(help="subcommand to manage models in S3: register/list/delete") diff --git a/morpheus-server/scripts/models/utils.py b/morpheus-server/scripts/models/utils.py index 29d30db1..a8d9a281 100644 --- a/morpheus-server/scripts/models/utils.py +++ b/morpheus-server/scripts/models/utils.py @@ -1,4 +1,3 @@ -import sys from pathlib import Path import torch @@ -7,11 +6,6 @@ from dynamicprompts.generators.magicprompt import MagicPromptGenerator from omegaconf import OmegaConf -sys.path.append(".") -from app.config import get_settings # noqa: E402 - -settings = get_settings() - def load_config_from_file(filename): return OmegaConf.load(filename) @@ -29,7 +23,6 @@ def download_model_from_huggingface(params): params.source, revision="fp16", torch_dtype=torch.float16, - use_auth_token=settings.hf_auth_token, ) except OSError as e: print("OSerror", e) @@ -37,7 +30,6 @@ def download_model_from_huggingface(params): params.source, # revision="fp16", torch_dtype=torch.float16, - use_auth_token=settings.hf_auth_token, ) model.save_pretrained(f"tmp/{output}") return output diff --git a/morpheus-server/tests/conftest.py b/morpheus-server/tests/conftest.py index ded3ea6f..8c6e49df 100644 --- a/morpheus-server/tests/conftest.py +++ b/morpheus-server/tests/conftest.py @@ -1,11 +1,12 @@ import pytest from httpx import AsyncClient +from morpheus_data.database import get_db +from morpheus_data.models.schemas import User +from morpheus_data.repository.firebase_repository import FirebaseRepository +from morpheus_data.repository.user_repository import UserRepository + from app.app import app -from app.database import get_db -from app.models.schemas import User -from app.repository.firebase_repository import FirebaseRepository -from app.repository.user_repository import UserRepository db = next(get_db()) diff --git a/morpheus-server/tests/test_boto3.py b/morpheus-server/tests/test_boto3.py index e033472b..be973e34 100644 --- a/morpheus-server/tests/test_boto3.py +++ b/morpheus-server/tests/test_boto3.py @@ -2,7 +2,7 @@ import pytest -from app.repository.files.s3_files_repository import S3ImagesRepository +from morpheus_data.repository.files.s3_files_repository import S3ImagesRepository COLLECTION_PATH = "collections" AVATAR_PATH = "avatars"