Merge pull request #192 from Tauffer-Consulting/dev

Dev

CHANGELOG.md

@@ -1,3 +1,14 @@
+# v0.8.2
+
+### Features
+- Add `container_resources` and `tags` to db and responses
+- Add default values for container resources in frontend forms
+- Import examples gallery from github json
+
+### Fixes
+- Fix k8s airflow xcom stream stdout
+- Fix `skip_envs` import in testing module
+
 # v0.8.1
 
 ### Features

frontend/src/@types/piece/piece.d.ts

@@ -37,11 +37,26 @@ export interface PieceSchema {
   $defs: Definitions;
 }
 
+interface ContainerResources {
+  limits: {
+    cpu: number;
+    memory: number;
+  };
+  requests: {
+    cpu: number;
+    memory: number;
+  };
+  use_gpu?: boolean;
+}
+
 export interface Piece {
   id: number;
   name: string;
   description: string;
 
+  container_resources: ContainerResources;
+  tags: string[];
+
   repository_id: number;
 
   input_schema: PieceSchema;

frontend/src/features/workflowEditor/api/workflowsExample/getWorkflowExamples.ts

@@ -0,0 +1,59 @@
+import axios from "axios";
+import { type IWorkflowPieceData } from "features/workflowEditor/context/types";
+import { type Edge, type Node } from "reactflow";
+import useSWR from "swr";
+
+const REPO_URL =
+  "https://raw.githubusercontent.com/Tauffer-Consulting/domino_pieces_gallery/main/workflows_gallery";
+
+const getWorkflowsExampleUrl = `${REPO_URL}/index.json`;
+
+type GithubReposContent = Array<{
+  title: string;
+  description: string;
+  jsonFile: string;
+  levelTag: "Advanced" | "Beginner" | "Intermediate";
+}>;
+
+interface JSONFile {
+  workflowPieces: Record<string, Piece>;
+  workflowPiecesData: IWorkflowPieceData;
+  workflowNodes: Node[];
+  workflowEdges: Edge[];
+}
+
+export type WorkflowsGalleryExamples = Array<{
+  title: string;
+  description: string;
+  jsonFile: JSONFile;
+  levelTag: "Advanced" | "Beginner" | "Intermediate";
+}>;
+
+const getWorkflowsExample: () => Promise<WorkflowsGalleryExamples> =
+  async () => {
+    const { data } = await axios.get<GithubReposContent>(
+      getWorkflowsExampleUrl,
+    );
+    const jsons: WorkflowsGalleryExamples = [];
+    for (const value of data) {
+      const { data: json } = await axios.get<JSONFile>(
+        `${REPO_URL}/${value.jsonFile}`,
+      );
+      jsons.push({ ...value, jsonFile: json });
+    }
+
+    return jsons;
+  };
+
+export const useAuthenticatedGetWorkflowsExamples = (fetch: boolean) => {
+  const fetcher = async () => await getWorkflowsExample().then((data) => data);
+
+  return useSWR(
+    fetch ? getWorkflowsExampleUrl : null,
+    async () => await fetcher(),
+    {
+      revalidateOnFocus: false,
+      revalidateOnReconnect: false,
+    },
+  );
+};

frontend/src/features/workflowEditor/api/workflowsExample/index.ts

@@ -0,0 +1 @@
+export * from "./getWorkflowExamples";

frontend/src/features/workflowEditor/components/SidebarForm/ContainerResourceForm.tsx

@@ -8,7 +8,7 @@ import * as yup from "yup";
 // TODO check if these values make sense
 const minAcceptedMemory = 128;
 const minAcceptedCpu = 100;
-const maxAcceptedMemory = 12800;
+const maxAcceptedMemory = 24000;
 const maxAcceptedCpu = 10000;
 
 export const defaultContainerResources: IContainerResourceFormData = {

frontend/src/features/workflowEditor/components/WorkflowEditor.tsx

@@ -25,8 +25,10 @@ import * as yup from "yup";
 
 import { type IWorkflowPieceData, storageAccessModes } from "../context/types";
 import { type GenerateWorkflowsParams } from "../context/workflowsEditor";
-import { containerResourcesSchema } from "../schemas/containerResourcesSchemas";
-import { extractDefaultInputValues, extractDefaultValues } from "../utils";
+import {
+  extractDefaultContainerResources,
+  extractDefaultInputValues,
+} from "../utils";
 import {
   type Differences,
   importJsonWorkflow,
@@ -352,8 +354,9 @@ export const WorkflowsEditorComponent: React.FC = () => {
       const defaultInputs = extractDefaultInputValues(
         piece as unknown as Piece,
       );
-      const defaultContainerResources = extractDefaultValues(
-        containerResourcesSchema as any,
+
+      const defaultContainerResources = extractDefaultContainerResources(
+        piece?.container_resources,
       );
 
       const currentWorkflowPieces = await getForageWorkflowPieces();

frontend/src/features/workflowEditor/components/WorkflowExamplesGalleryModal/index.tsx

@@ -8,8 +8,12 @@ import {
   Chip,
 } from "@mui/material";
 import { Modal, type ModalRef } from "components/Modal";
+import {
+  type WorkflowsGalleryExamples,
+  useAuthenticatedGetWorkflowsExamples,
+} from "features/workflowEditor/api/workflowsExample";
 import theme from "providers/theme.config";
-import { forwardRef, type ForwardedRef, useState } from "react";
+import { forwardRef, type ForwardedRef, useState, useMemo } from "react";
 
 import CloudSegmentationWorkflow from "../../utils/workflows/cloud_segmentation_workflow.json";
 import DimensionalityReductionWorkflow from "../../utils/workflows/dimensionality_reduction.json";
@@ -24,55 +28,67 @@ interface WorkflowGalleryModalProps {
   confirmFn: (json: any) => void;
 }
 
+const USE_LOCAL_CARDS = true;
+
+const localCardsContents = [
+  {
+    title: "Youtube Summarizer",
+    description:
+      "Sends the summary of the last BBCNews youtube channel video to an emails list. You must configure Secrets and Local storage to use it.",
+    jsonFile: YoutubeSummarizerWorkflow,
+    levelTag: "Advanced",
+  },
+  {
+    title: "Image Filter Workflow",
+    description: "A simple workflow that applies a filter to an image.",
+    jsonFile: ImageFilterWorkflow,
+    levelTag: "Beginner",
+  },
+  {
+    title: "NASA Image Workflow",
+    description: "A simple workflow that gets an image from NASA API.",
+    jsonFile: NasaImageWorkflow,
+    levelTag: "Beginner",
+  },
+  {
+    title: "Dimensionality Reduction",
+    description:
+      "A workflow that applies dimensionality reduction to a dataset. To use it, you must use Shared Storage.",
+    jsonFile: DimensionalityReductionWorkflow,
+    levelTag: "Intermediate",
+  },
+  {
+    title: "Random Forest Classifier",
+    description:
+      "A machine learning workflow to train a random forest and use it to predict a new data. To use it, you must use Shared Storage",
+    jsonFile: RandomForestClassifierWorkflow,
+    levelTag: "Intermediate",
+  },
+  {
+    title: "Cloud Segmentation Workflow",
+    description:
+      "A workflow that uses OpenCV to create a cloud segmentation over a NASA earth image. To use it, you must use Shared Storage",
+    jsonFile: CloudSegmentationWorkflow,
+    levelTag: "Intermediate",
+  },
+] as unknown as WorkflowsGalleryExamples;
+
 const WorkflowExamplesGalleryModal = forwardRef(
   (
     props: WorkflowGalleryModalProps,
     ref: ForwardedRef<WorkflowGalleryModalRef>,
   ) => {
     const [selected, setSelected] = useState<number | null>(null);
+    // only make requests if USE_LOCAL_CARDS=false
+    const { data } = useAuthenticatedGetWorkflowsExamples(!USE_LOCAL_CARDS);
 
-    const cardsContents = [
-      {
-        title: "Youtube Summarizer",
-        description:
-          "Sends the summary of the last BBCNews youtube channel video to an emails list. You must configure Secrets and Local storage to use it.",
-        jsonFile: YoutubeSummarizerWorkflow,
-        levelTag: "Advanced",
-      },
-      {
-        title: "Image Filter Workflow",
-        description: "A simple workflow that applies a filter to an image.",
-        jsonFile: ImageFilterWorkflow,
-        levelTag: "Beginner",
-      },
-      {
-        title: "NASA Image Workflow",
-        description: "A simple workflow that gets an image from NASA API.",
-        jsonFile: NasaImageWorkflow,
-        levelTag: "Beginner",
-      },
-      {
-        title: "Dimensionality Reduction",
-        description:
-          "A workflow that applies dimensionality reduction to a dataset. To use it, you must use Shared Storage.",
-        jsonFile: DimensionalityReductionWorkflow,
-        levelTag: "Intermediate",
-      },
-      {
-        title: "Random Forest Classifier",
-        description:
-          "A machine learning workflow to train a random forest and use it to predict a new data. To use it, you must use Shared Storage",
-        jsonFile: RandomForestClassifierWorkflow,
-        levelTag: "Intermediate",
-      },
-      {
-        title: "Cloud Segmentation Workflow",
-        description:
-          "A workflow that uses OpenCV to create a cloud segmentation over a NASA earth image. To use it, you must use Shared Storage",
-        jsonFile: CloudSegmentationWorkflow,
-        levelTag: "Intermediate",
-      },
-    ];
+    const cardsContents = useMemo<WorkflowsGalleryExamples>(() => {
+      if (USE_LOCAL_CARDS) {
+        return localCardsContents;
+      } else {
+        return data ?? [];
+      }
+    }, [data, localCardsContents, USE_LOCAL_CARDS]);
 
     const levelTagMap: any = {
       Beginner: {

frontend/src/features/workflowEditor/utils/jsonSchema.ts

@@ -1,6 +1,12 @@
 // Extract default values from Schema
 
-import { type IWorkflowPieceData } from "../context/types";
+import { isEmpty } from "utils";
+
+import { defaultContainerResources } from "../components/SidebarForm/ContainerResourceForm";
+import {
+  type IContainerResourceFormData,
+  type IWorkflowPieceData,
+} from "../context/types";
 
 import { getFromUpstream } from "./getFromUpstream";
 
@@ -85,7 +91,7 @@ export const extractDefaultValues = (
 ) => {
   output = output === null ? {} : output;
 
-  if (schema) {
+  if (!isEmpty(schema) && "properties" in schema) {
     const properties = schema.properties;
     for (const [key, value] of Object.entries(properties)) {
       if (value?.from_upstream === "always") {
@@ -104,3 +110,23 @@ export const extractDefaultValues = (
 
   return output;
 };
+
+export const extractDefaultContainerResources = (
+  cr?: Piece["container_resources"],
+): IContainerResourceFormData => {
+  if (cr && !isEmpty(cr) && "limits" in cr && "requests" in cr) {
+    return {
+      cpu: {
+        max: Number(cr.limits.cpu),
+        min: Number(cr.requests.cpu),
+      },
+      memory: {
+        max: Number(cr.limits.memory),
+        min: Number(cr.requests.memory),
+      },
+      useGpu: cr?.use_gpu ?? false,
+    };
+  } else {
+    return defaultContainerResources;
+  }
+};

rest/constants/schemas/__init__.py

@@ -0,0 +1 @@
+from .container_resources import ContainerResourcesModel

rest/constants/schemas/container_resources.py

@@ -0,0 +1,11 @@
+from pydantic import BaseModel, Field
+
+class SystemRequirementsModel(BaseModel):
+    cpu: int = Field(default=128)
+    memory: int = Field(memory=100)
+
+
+class ContainerResourcesModel(BaseModel):
+    requests: SystemRequirementsModel = Field(default=SystemRequirementsModel(cpu=100, memory=128))
+    limits: SystemRequirementsModel = Field(default=SystemRequirementsModel(cpu=100, memory=128))
+    use_gpu: bool = False

rest/database/alembic/versions/93da7356c3d7_.py

@@ -0,0 +1,30 @@
+"""empty message
+
+Revision ID: 93da7356c3d7
+Revises: f7214a10a4df
+Create Date: 2023-11-29 07:55:27.576939
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '93da7356c3d7'
+down_revision = 'f7214a10a4df'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('piece', sa.Column('tags', sa.ARRAY(sa.String()), server_default='{}', nullable=False))
+    op.add_column('piece', sa.Column('container_resources', sa.JSON(), server_default=sa.text("'{}'::jsonb"), nullable=False))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('piece', 'container_resources')
+    op.drop_column('piece', 'tags')
+    # ### end Alembic commands ###

rest/database/models/piece.py

@@ -1,6 +1,6 @@
 from database.models.base import Base, BaseDatabaseModel
 from sqlalchemy.orm import relationship
-from sqlalchemy import Column, String, Integer, JSON, ForeignKey, text
+from sqlalchemy import Column, String, Integer, JSON, ForeignKey, text, ARRAY
 
 class Piece(Base, BaseDatabaseModel):
     __tablename__ = "piece"
@@ -13,6 +13,8 @@ class Piece(Base, BaseDatabaseModel):
     input_schema = Column(JSON, nullable=False, server_default=text("'{}'::jsonb"))
     output_schema = Column(JSON, nullable=False, server_default=text("'{}'::jsonb")) # Using server default empty JSON object to avoid null value in database
     secrets_schema = Column(JSON, nullable=False, server_default=text("'{}'::jsonb"))
+    tags = Column(ARRAY(String), nullable=False, server_default="{}")
+    container_resources = Column(JSON, nullable=False, server_default=text("'{}'::jsonb"))
     style = Column(JSON, nullable=True)
     source_url = Column(String, nullable=True)
     repository_id = Column(Integer, ForeignKey('piece_repository.id', ondelete='cascade'), nullable=False)