Replace boto3 with numpy

.github/scripts/python_package_upload/package_upload.sh

@@ -1,33 +1,27 @@
-#!/usr/bin/env bash
+#!/bin/bash
 
 set -ex
 
-kfp_directory=kfp
-boto_directory=boto3
+# easy for troubleshooting
+tmp_dir=tmp
 
-mkdir -p "$kfp_directory"
-mkdir -p "$boto_directory"
-
-# Download kfp package
-pip download kfp==2.7.0 -d "$kfp_directory"
-
-# Download boto3 package
-pip download boto3 -d "$boto_directory"
+mkdir -p "$tmp_dir"
 
+# Download packages
+for python_version in "3.9" "3.10" "3.11" "3.12"; do
+  for package in "kfp" "numpy"; do
+    # If we don't set the --python-version it will use the one from the computer that may not be the one that the
+    # pipeline is running
+    pip download $package -d "$tmp_dir" --only-binary=:all: --python-version $python_version
+  done
+done
 
 # Print the pods in the namespace
 oc -n test-pypiserver get pods
 
 pod_name=$(oc -n test-pypiserver get pod | grep pypi | awk '{print $1}')
 
-# Copy kfp packages
-for kfp_entry in "$kfp_directory"/*; do
-    echo oc -n test-pypiserver cp "$kfp_entry" $pod_name:/opt/app-root/packages
-    oc -n test-pypiserver cp "$kfp_entry" $pod_name:/opt/app-root/packages
-done
-
-# Copy boto3 packages
-for boto_entry in "$boto_directory"/*; do
-    echo oc -n test-pypiserver cp "$boto_entry" $pod_name:/opt/app-root/packages
-    oc -n test-pypiserver cp "$boto_entry" $pod_name:/opt/app-root/packages
+# Copy packages
+for entry in "$tmp_dir"/*; do
+    oc -n test-pypiserver cp "$entry" $pod_name:/opt/app-root/packages
 done

.github/scripts/tests/tests.sh

@@ -192,6 +192,7 @@ remove_namespace_created_for_rhoai() {
 }
 
 setup_kind_requirements() {
+  upload_python_packages_to_pypi_server
   apply_crd
   build_image
   create_opendatahub_namespace
@@ -202,7 +203,6 @@ setup_kind_requirements() {
   deploy_pypi_server
   wait_for_dspo_dependencies
   wait_for_dependencies
-  upload_python_packages_to_pypi_server
   create_dspa_namespace
   create_namespace_dspa_external_connections
   apply_mariadb_minio_secrets_configmaps_external_namespace

.github/workflows/kind-integration.yml

@@ -43,6 +43,13 @@ jobs:
           go-version: '1.21.x'
         id: go
 
+      - name: pip version
+        working-directory: ${{ github.workspace }}/.github/scripts/tests
+        run: |
+          pip --version
+          pip install --upgrade pip
+          pip --version
+
       - name: Setup and start KinD cluster
         uses: ./.github/actions/kind
 

.gitignore

@@ -34,4 +34,4 @@ Dockerfile.cross
 __pycache__/
 *.py[cod]
 *$py.class
-.github/scripts/python_package_upload/
+.github/scripts/python_package_upload/tmp

tests/resources/test-pipeline-run.yaml

@@ -20,29 +20,29 @@ deploymentSpec:
     exec-say-hello:
       container:
         args:
-          - --executor_input
-          - '{{$}}'
-          - --function_to_execute
-          - say_hello
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - say_hello
         command:
-          - sh
-          - -c
-          - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
           \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
-          \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.3.0'\
+          \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.9.0'\
           \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\
           $0\" \"$@\"\n"
-          - sh
-          - -ec
-          - 'program_path=$(mktemp -d)
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
 
 
           printf "%s" "$0" > "$program_path/ephemeral_component.py"
 
           _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
 
           '
-          - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
           \ *\n\ndef say_hello(name: str) -> str:\n    hello_text = f'Hello, {name}!'\n\
           \    print(hello_text)\n    return hello_text\n\n"
         image: quay.io/opendatahub/ds-pipelines-ci-executor-image:v1.1
@@ -77,4 +77,4 @@ root:
       Output:
         parameterType: STRING
 schemaVersion: 2.1.0
-sdkVersion: kfp-2.3.0
+sdkVersion: kfp-2.9.0

tests/resources/test-pipeline-with-custom-pip-server-run.yaml

@@ -22,13 +22,14 @@ deploymentSpec:
         - sh
         - -c
         - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
-          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n
-          \ PIP_DISABLE_PIP_VERSION_CHECK=1\
-          \ python3 -m pip install --quiet --no-warn-script-location --index-url  https://nginx-service.test-pypiserver.svc.cluster.local:443/simple/\
-          \ 'kfp==2.7.0' '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"\
-          3.9\"'  &&  python3 -m pip install --quiet --no-warn-script-location --index-url\
-          \  https://nginx-service.test-pypiserver.svc.cluster.local:443/simple/ 'boto3' && \"\
-          $0\" \"$@\"\n"
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet --no-warn-script-location --index-url https://nginx-service.test-pypiserver.svc.cluster.local/simple/\
+          \ --trusted-host nginx-service.test-pypiserver.svc.cluster.local 'kfp==2.9.0'\
+          \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"'  &&\
+          \  python3 -m pip install --quiet --no-warn-script-location --index-url\
+          \ https://nginx-service.test-pypiserver.svc.cluster.local/simple/ --trusted-host\
+          \ nginx-service.test-pypiserver.svc.cluster.local 'numpy' && \"$0\" \"$@\"\
+          \n"
         - sh
         - -ec
         - 'program_path=$(mktemp -d)
@@ -40,8 +41,9 @@ deploymentSpec:
 
           '
         - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
-          \ *\n\ndef say_hello() -> str:\n    hello_text = f'Hello!'\n    print(hello_text)\n\
-          \    return hello_text\n\n"
+          \ *\n\ndef say_hello() -> str:\n    import numpy as np\n    hello_text =\
+          \ f'Numpy version: {np.__version__}'\n    print(hello_text)\n    return\
+          \ hello_text\n\n"
         image: quay.io/opendatahub/ds-pipelines-ci-executor-image:v1.0
 pipelineInfo:
   name: hello-pipeline
@@ -66,4 +68,4 @@ root:
       Output:
         parameterType: STRING
 schemaVersion: 2.1.0
-sdkVersion: kfp-2.7.0
+sdkVersion: kfp-2.9.0

tests/resources/test-pipeline-with-custom-pip-server.py

@@ -3,10 +3,12 @@
 # Edited the compiled version manually, to remove the --trusted-host flag
 # this is so we can test for tls certs validations when launcher installs packages
 @dsl.component(base_image="quay.io/opendatahub/ds-pipelines-ci-executor-image:v1.0",
-packages_to_install=['boto3'],
-pip_index_urls=['https://nginx-service.test-pypiserver.svc.cluster.local:443/simple/'])
+packages_to_install=['numpy'],
+pip_index_urls=['https://nginx-service.test-pypiserver.svc.cluster.local/simple/'],
+pip_trusted_hosts=['nginx-service.test-pypiserver.svc.cluster.local'])
 def say_hello() -> str:
-    hello_text = 'Hello!'
+    import numpy as np
+    hello_text = f'Numpy version: {np.__version__}'
     print(hello_text)
     return hello_text
 
@@ -18,4 +20,4 @@ def hello_pipeline() -> str:
 
 
 if __name__ == '__main__':
-    compiler.Compiler().compile(hello_pipeline, __file__ + '.yaml')
+    compiler.Compiler().compile(hello_pipeline, __file__.replace('.py', '-run.yaml'))

tests/resources/test-pipeline-src.py → tests/resources/test-pipeline.py

@@ -15,4 +15,4 @@ def hello_pipeline(recipient: str) -> str:
 
 
 if __name__ == '__main__':
-    compiler.Compiler().compile(hello_pipeline, __file__ + '.yaml')
+    compiler.Compiler().compile(hello_pipeline, __file__.replace('.py', '-run.yaml'))