From b1ed592441227592f69db0570f62273c7aff66f3 Mon Sep 17 00:00:00 2001 From: James Wong <2421248+jameswnl@users.noreply.github.com> Date: Tue, 15 Oct 2024 21:52:57 -0400 Subject: [PATCH 01/13] aap --- Containerfile | 60 +++++++++++++++++++++------------------- ols/app/models/config.py | 15 ++++++---- runner.py | 4 +-- 3 files changed, 43 insertions(+), 36 deletions(-) diff --git a/Containerfile b/Containerfile index 95684fe5..5e3fa8cf 100644 --- a/Containerfile +++ b/Containerfile @@ -1,40 +1,42 @@ -# vim: set filetype=dockerfile -ARG LIGHTSPEED_RAG_CONTENT_IMAGE=quay.io/openshift-lightspeed/lightspeed-rag-content@sha256:a91aca8224b1405e7c91576374c7bbc766b2009b2ef852895c27069fffc5b06f +# # vim: set filetype=dockerfile +# ARG LIGHTSPEED_RAG_CONTENT_IMAGE=quay.io/ttakamiy/aap-rag-content:latest -FROM ${LIGHTSPEED_RAG_CONTENT_IMAGE} as lightspeed-rag-content +# FROM ${LIGHTSPEED_RAG_CONTENT_IMAGE} as lightspeed-rag-content -FROM registry.redhat.io/ubi9/ubi-minimal:latest +# FROM registry.access.redhat.com/ubi9/ubi-minimal -ARG VERSION -# todo: this is overriden by the image ubi9/python-311, we hard coded WORKDIR below to /app-root -# makesure the default value of rag content is set according to APP_ROOT and then update the operator. -ARG APP_ROOT=/app-root - -RUN microdnf install -y --nodocs --setopt=keepcache=0 --setopt=tsflags=nodocs \ - python3.11 python3.11-devel python3.11-pip +# ARG APP_ROOT=/app-root -# PYTHONDONTWRITEBYTECODE 1 : disable the generation of .pyc -# PYTHONUNBUFFERED 1 : force the stdout and stderr streams to be unbuffered -# PYTHONCOERCECLOCALE 0, PYTHONUTF8 1 : skip legacy locales and use UTF-8 mode -ENV PYTHONDONTWRITEBYTECODE=1 \ - PYTHONUNBUFFERED=1 \ - PYTHONCOERCECLOCALE=0 \ - PYTHONUTF8=1 \ - PYTHONIOENCODING=UTF-8 \ - LANG=en_US.UTF-8 \ - PIP_NO_CACHE_DIR=off +# RUN microdnf install -y --nodocs --setopt=keepcache=0 --setopt=tsflags=nodocs \ +# python3.11 python3.11-devel python3.11-pip shadow-utils \ +# && microdnf clean all --enablerepo='*' -WORKDIR /app-root +# # PYTHONDONTWRITEBYTECODE 1 : disable the generation of .pyc +# # PYTHONUNBUFFERED 1 : force the stdout and stderr streams to be unbuffered +# # PYTHONCOERCECLOCALE 0, PYTHONUTF8 1 : skip legacy locales and use UTF-8 mode +# ENV PYTHONDONTWRITEBYTECODE=1 \ +# PYTHONUNBUFFERED=1 \ +# PYTHONCOERCECLOCALE=0 \ +# PYTHONUTF8=1 \ +# PYTHONIOENCODING=UTF-8 \ +# LANG=en_US.UTF-8 \ +# PIP_NO_CACHE_DIR=off -COPY --from=lightspeed-rag-content /rag/vector_db/ocp_product_docs ./vector_db/ocp_product_docs -COPY --from=lightspeed-rag-content /rag/embeddings_model ./embeddings_model +# WORKDIR ${APP_ROOT} -# Add explicit files and directories -# (avoid accidental inclusion of local directories or env files or credentials) -COPY runner.py requirements.txt ./ +# COPY --from=lightspeed-rag-content /rag/vector_db/aap_product_docs ./vector_db/aap_product_docs +# COPY --from=lightspeed-rag-content /rag/embeddings_model ./embeddings_model -RUN pip3.11 install --no-cache-dir -r requirements.txt +# # Add explicit files and directories +# # (avoid accidental inclusion of local directories or env files or credentials) +# COPY pyproject.toml pdm.lock runner.py ./ +# RUN pip3.11 install --no-cache-dir --upgrade pip pdm==2.18.1 \ +# && pdm config python.use_venv false \ +# && pdm sync --global --prod -p ${APP_ROOT} +FROM quay.io/ansible/ansible-chatbot-service:base +ARG APP_ROOT=/app-root +WORKDIR ${APP_ROOT} COPY ols ./ols # this directory is checked by ecosystem-cert-preflight-checks task in Konflux @@ -55,5 +57,5 @@ LABEL io.k8s.display-name="OpenShift LightSpeed Service" \ vendor="Red Hat, Inc." -# no-root user is checked in Konflux +# no-root user is checked in Konflux USER 1001 diff --git a/ols/app/models/config.py b/ols/app/models/config.py index d65d9c30..b38b8705 100644 --- a/ols/app/models/config.py +++ b/ols/app/models/config.py @@ -329,9 +329,14 @@ def __init__( self.deployment_name = data.get("deployment_name", None) # note: it can be overwritten in azure_config if self.type in (constants.PROVIDER_RHOAI_VLLM, constants.PROVIDER_RHELAI_VLLM): - self.certificates_store = os.path.join( - certificate_directory, constants.CERTIFICATE_STORAGE_FILENAME - ) + print(f"self.certificates_store == {certificate_directory}") + if certificate_directory.lower() == 'false': + self.certificates_store = False + print(f"self.certificates_store => verify off") + else: + self.certificates_store = os.path.join( + certificate_directory, constants.CERTIFICATE_STORAGE_FILENAME + ) def set_provider_type(self, data: dict) -> None: """Set the provider type.""" @@ -902,8 +907,8 @@ def __init__( return self.conversation_cache = ConversationCacheConfig( - data.get("conversation_cache", None) - ) + data.get("conversation_cache") + ) if data.get("conversation_cache") else None self.logging_config = LoggingConfig(**data.get("logging_config", {})) if data.get("reference_content") is not None: self.reference_content = ReferenceContent(data.get("reference_content")) diff --git a/runner.py b/runner.py index 73a6bc19..52151846 100644 --- a/runner.py +++ b/runner.py @@ -163,8 +163,8 @@ def start_uvicorn(): # Initialize the K8sClientSingleton with cluster id during module load. # We want the application to fail early if the cluster ID is not available. - cluster_id = K8sClientSingleton.get_cluster_id() - logger.info(f"running on cluster with ID '{cluster_id}'") + # cluster_id = K8sClientSingleton.get_cluster_id() + # logger.info(f"running on cluster with ID '{cluster_id}'") # init loading of query redactor config.query_redactor From 478e0b50ea44243b5c3c7fdba8c56618117b6793 Mon Sep 17 00:00:00 2001 From: red-hat-konflux Date: Wed, 16 Oct 2024 05:08:32 +0000 Subject: [PATCH 02/13] Red Hat Konflux update ansible-chatbot Signed-off-by: red-hat-konflux --- .tekton/ansible-chatbot-pull-request.yaml | 474 ++++++++++++++++++++++ .tekton/ansible-chatbot-push.yaml | 471 +++++++++++++++++++++ 2 files changed, 945 insertions(+) create mode 100644 .tekton/ansible-chatbot-pull-request.yaml create mode 100644 .tekton/ansible-chatbot-push.yaml diff --git a/.tekton/ansible-chatbot-pull-request.yaml b/.tekton/ansible-chatbot-pull-request.yaml new file mode 100644 index 00000000..c5160367 --- /dev/null +++ b/.tekton/ansible-chatbot-pull-request.yaml @@ -0,0 +1,474 @@ +apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + annotations: + build.appstudio.openshift.io/repo: https://github.com/jameswnl/ansible-chatbot-service?rev={{revision}} + build.appstudio.redhat.com/commit_sha: '{{revision}}' + build.appstudio.redhat.com/pull_request_number: '{{pull_request_number}}' + build.appstudio.redhat.com/target_branch: '{{target_branch}}' + pipelinesascode.tekton.dev/max-keep-runs: "3" + pipelinesascode.tekton.dev/on-cel-expression: event == "pull_request" && target_branch + == "aapbuild" + creationTimestamp: null + labels: + appstudio.openshift.io/application: ansible-chatbot + appstudio.openshift.io/component: ansible-chatbot + pipelines.appstudio.openshift.io/type: build + name: ansible-chatbot-on-pull-request + namespace: jwong-tenant +spec: + params: + - name: git-url + value: '{{source_url}}' + - name: revision + value: '{{revision}}' + - name: output-image + value: quay.io/redhat-user-workloads/jwong-tenant/ansible-chatbot:on-pr-{{revision}} + - name: image-expires-after + value: 5d + - name: dockerfile + value: Containerfile + pipelineSpec: + description: | + This pipeline is ideal for building container images from a Containerfile while maintaining trust after pipeline customization. + + _Uses `buildah` to create a container image leveraging [trusted artifacts](https://konflux-ci.dev/architecture/ADR/0036-trusted-artifacts.html). It also optionally creates a source image and runs some build-time tests. Information is shared between tasks using OCI artifacts instead of PVCs. EC will pass the [`trusted_task.trusted`](https://enterprisecontract.dev/docs/ec-policies/release_policy.html#trusted_task__trusted) policy as long as all data used to build the artifact is generated from trusted tasks. + This pipeline is pushed as a Tekton bundle to [quay.io](https://quay.io/repository/konflux-ci/tekton-catalog/pipeline-docker-build-oci-ta?tab=tags)_ + finally: + - name: show-sbom + params: + - name: IMAGE_URL + value: $(tasks.build-image-index.results.IMAGE_URL) + taskRef: + params: + - name: name + value: show-sbom + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-show-sbom:0.1@sha256:9bfc6b99ef038800fe131d7b45ff3cd4da3a415dd536f7c657b3527b01c4a13b + - name: kind + value: task + resolver: bundles + params: + - description: Source Repository URL + name: git-url + type: string + - default: "" + description: Revision of the Source Repository + name: revision + type: string + - description: Fully Qualified Output Image + name: output-image + type: string + - default: . + description: Path to the source code of an application's component from where + to build image. + name: path-context + type: string + - default: Dockerfile + description: Path to the Dockerfile inside the context specified by parameter + path-context + name: dockerfile + type: string + - default: "false" + description: Force rebuild image + name: rebuild + type: string + - default: "false" + description: Skip checks against built image + name: skip-checks + type: string + - default: "false" + description: Execute the build with network isolation + name: hermetic + type: string + - default: "" + description: Build dependencies to be prefetched by Cachi2 + name: prefetch-input + type: string + - default: "" + description: Image tag expiration time, time values could be something like + 1h, 2d, 3w for hours, days, and weeks, respectively. + name: image-expires-after + - default: "false" + description: Build a source image. + name: build-source-image + type: string + - default: "false" + description: Add built image into an OCI image index + name: build-image-index + type: string + - default: [] + description: Array of --build-arg values ("arg=value" strings) for buildah + name: build-args + type: array + - default: "" + description: Path to a file with build arguments for buildah, see https://www.mankier.com/1/buildah-build#--build-arg-file + name: build-args-file + type: string + results: + - description: "" + name: IMAGE_URL + value: $(tasks.build-image-index.results.IMAGE_URL) + - description: "" + name: IMAGE_DIGEST + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - description: "" + name: CHAINS-GIT_URL + value: $(tasks.clone-repository.results.url) + - description: "" + name: CHAINS-GIT_COMMIT + value: $(tasks.clone-repository.results.commit) + tasks: + - name: init + params: + - name: image-url + value: $(params.output-image) + - name: rebuild + value: $(params.rebuild) + - name: skip-checks + value: $(params.skip-checks) + taskRef: + params: + - name: name + value: init + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-init:0.2@sha256:092c113b614f6551113f17605ae9cb7e822aa704d07f0e37ed209da23ce392cc + - name: kind + value: task + resolver: bundles + - name: clone-repository + params: + - name: url + value: $(params.git-url) + - name: revision + value: $(params.revision) + - name: ociStorage + value: $(params.output-image).git + - name: ociArtifactExpiresAfter + value: $(params.image-expires-after) + runAfter: + - init + taskRef: + params: + - name: name + value: git-clone-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-git-clone-oci-ta:0.1@sha256:d1e63ec00bed1c9f0f571fa76b4da570be49a7c255c610544a461495230ba1b1 + - name: kind + value: task + resolver: bundles + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" + workspaces: + - name: basic-auth + workspace: git-auth + - name: prefetch-dependencies + params: + - name: input + value: $(params.prefetch-input) + - name: SOURCE_ARTIFACT + value: $(tasks.clone-repository.results.SOURCE_ARTIFACT) + - name: ociStorage + value: $(params.output-image).prefetch + - name: ociArtifactExpiresAfter + value: $(params.image-expires-after) + runAfter: + - clone-repository + taskRef: + params: + - name: name + value: prefetch-dependencies-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-prefetch-dependencies-oci-ta:0.1@sha256:621b13ab4a01a366a2b1d8403cf06b2b7418afd926d13678c4432858514407d3 + - name: kind + value: task + resolver: bundles + workspaces: + - name: git-basic-auth + workspace: git-auth + - name: netrc + workspace: netrc + - name: build-container + params: + - name: IMAGE + value: $(params.output-image) + - name: DOCKERFILE + value: $(params.dockerfile) + - name: CONTEXT + value: $(params.path-context) + - name: HERMETIC + value: $(params.hermetic) + - name: PREFETCH_INPUT + value: $(params.prefetch-input) + - name: IMAGE_EXPIRES_AFTER + value: $(params.image-expires-after) + - name: COMMIT_SHA + value: $(tasks.clone-repository.results.commit) + - name: BUILD_ARGS + value: + - $(params.build-args[*]) + - name: BUILD_ARGS_FILE + value: $(params.build-args-file) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + runAfter: + - prefetch-dependencies + taskRef: + params: + - name: name + value: buildah-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-buildah-oci-ta:0.2@sha256:06946a3a676a9daa1efb16120ba0d81e0c3ce38c9f6242439e2169f27d5d2a2a + - name: kind + value: task + resolver: bundles + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" + - name: build-image-index + params: + - name: IMAGE + value: $(params.output-image) + - name: COMMIT_SHA + value: $(tasks.clone-repository.results.commit) + - name: IMAGE_EXPIRES_AFTER + value: $(params.image-expires-after) + - name: ALWAYS_BUILD_INDEX + value: $(params.build-image-index) + - name: IMAGES + value: + - $(tasks.build-container.results.IMAGE_URL)@$(tasks.build-container.results.IMAGE_DIGEST) + runAfter: + - build-container + taskRef: + params: + - name: name + value: build-image-index + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-build-image-index:0.1@sha256:e4871851566d8b496966b37bcb8c5ce9748a52487f116373d96c6cd28ef684c6 + - name: kind + value: task + resolver: bundles + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" + - name: build-source-image + params: + - name: BINARY_IMAGE + value: $(params.output-image) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: source-build-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-source-build-oci-ta:0.1@sha256:261f075fd5a096f7b28a999b505136b2a3a5aef390087148b3131fd3ec295db3 + - name: kind + value: task + resolver: bundles + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" + - input: $(params.build-source-image) + operator: in + values: + - "true" + - name: deprecated-base-image-check + params: + - name: IMAGE_URL + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: IMAGE_DIGEST + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: deprecated-image-check + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-deprecated-image-check:0.4@sha256:b4f9599f5770ea2e6e4d031224ccc932164c1ecde7f85f68e16e99c98d754003 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: clair-scan + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: clair-scan + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-clair-scan:0.2@sha256:37b9187c1d5f6672bbc9c61d88fc71a3ee688076cb16edef42d1ff92a59027fb + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: ecosystem-cert-preflight-checks + params: + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: ecosystem-cert-preflight-checks + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-ecosystem-cert-preflight-checks:0.1@sha256:5131cce0f93d0b728c7bcc0d6cee4c61d4c9f67c6d619c627e41e3c9775b497d + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: sast-snyk-check + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: sast-snyk-check-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-sast-snyk-check-oci-ta:0.2@sha256:c10a095a48bffe898cc95644eb9a826ea0667c6ba9e9ec35b6149337ece234fd + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: clamav-scan + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: clamav-scan + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-clamav-scan:0.1@sha256:5ac9b24cff7cfb391bc54cd5135536892090354862327d1028fa08872d759c03 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: apply-tags + params: + - name: IMAGE + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: apply-tags + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-apply-tags:0.1@sha256:f485e250fb060060892b633c495a3d7e38de1ec105ae1be48608b0401530ab2c + - name: kind + value: task + resolver: bundles + - name: push-dockerfile + params: + - name: IMAGE + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: IMAGE_DIGEST + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: DOCKERFILE + value: $(params.dockerfile) + - name: CONTEXT + value: $(params.path-context) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: push-dockerfile-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-push-dockerfile-oci-ta:0.1@sha256:b048f99ab1ba013e809dc00523978542c2cb3fcd133b408267dd849eb40d1d0a + - name: kind + value: task + resolver: bundles + - name: rpms-signature-scan + params: + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: rpms-signature-scan + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-rpms-signature-scan:0.2@sha256:7aa4d3c95e2b963e82fdda392f7cb3d61e3dab035416cf4a3a34e43cf3c9c9b8 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + workspaces: + - name: git-auth + optional: true + - name: netrc + optional: true + taskRunTemplate: {} + workspaces: + - name: git-auth + secret: + secretName: '{{ git_auth_secret }}' +status: {} diff --git a/.tekton/ansible-chatbot-push.yaml b/.tekton/ansible-chatbot-push.yaml new file mode 100644 index 00000000..5800ef36 --- /dev/null +++ b/.tekton/ansible-chatbot-push.yaml @@ -0,0 +1,471 @@ +apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + annotations: + build.appstudio.openshift.io/repo: https://github.com/jameswnl/ansible-chatbot-service?rev={{revision}} + build.appstudio.redhat.com/commit_sha: '{{revision}}' + build.appstudio.redhat.com/target_branch: '{{target_branch}}' + pipelinesascode.tekton.dev/max-keep-runs: "3" + pipelinesascode.tekton.dev/on-cel-expression: event == "push" && target_branch + == "aapbuild" + creationTimestamp: null + labels: + appstudio.openshift.io/application: ansible-chatbot + appstudio.openshift.io/component: ansible-chatbot + pipelines.appstudio.openshift.io/type: build + name: ansible-chatbot-on-push + namespace: jwong-tenant +spec: + params: + - name: git-url + value: '{{source_url}}' + - name: revision + value: '{{revision}}' + - name: output-image + value: quay.io/redhat-user-workloads/jwong-tenant/ansible-chatbot:{{revision}} + - name: dockerfile + value: Containerfile + pipelineSpec: + description: | + This pipeline is ideal for building container images from a Containerfile while maintaining trust after pipeline customization. + + _Uses `buildah` to create a container image leveraging [trusted artifacts](https://konflux-ci.dev/architecture/ADR/0036-trusted-artifacts.html). It also optionally creates a source image and runs some build-time tests. Information is shared between tasks using OCI artifacts instead of PVCs. EC will pass the [`trusted_task.trusted`](https://enterprisecontract.dev/docs/ec-policies/release_policy.html#trusted_task__trusted) policy as long as all data used to build the artifact is generated from trusted tasks. + This pipeline is pushed as a Tekton bundle to [quay.io](https://quay.io/repository/konflux-ci/tekton-catalog/pipeline-docker-build-oci-ta?tab=tags)_ + finally: + - name: show-sbom + params: + - name: IMAGE_URL + value: $(tasks.build-image-index.results.IMAGE_URL) + taskRef: + params: + - name: name + value: show-sbom + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-show-sbom:0.1@sha256:9bfc6b99ef038800fe131d7b45ff3cd4da3a415dd536f7c657b3527b01c4a13b + - name: kind + value: task + resolver: bundles + params: + - description: Source Repository URL + name: git-url + type: string + - default: "" + description: Revision of the Source Repository + name: revision + type: string + - description: Fully Qualified Output Image + name: output-image + type: string + - default: . + description: Path to the source code of an application's component from where + to build image. + name: path-context + type: string + - default: Dockerfile + description: Path to the Dockerfile inside the context specified by parameter + path-context + name: dockerfile + type: string + - default: "false" + description: Force rebuild image + name: rebuild + type: string + - default: "false" + description: Skip checks against built image + name: skip-checks + type: string + - default: "false" + description: Execute the build with network isolation + name: hermetic + type: string + - default: "" + description: Build dependencies to be prefetched by Cachi2 + name: prefetch-input + type: string + - default: "" + description: Image tag expiration time, time values could be something like + 1h, 2d, 3w for hours, days, and weeks, respectively. + name: image-expires-after + - default: "false" + description: Build a source image. + name: build-source-image + type: string + - default: "false" + description: Add built image into an OCI image index + name: build-image-index + type: string + - default: [] + description: Array of --build-arg values ("arg=value" strings) for buildah + name: build-args + type: array + - default: "" + description: Path to a file with build arguments for buildah, see https://www.mankier.com/1/buildah-build#--build-arg-file + name: build-args-file + type: string + results: + - description: "" + name: IMAGE_URL + value: $(tasks.build-image-index.results.IMAGE_URL) + - description: "" + name: IMAGE_DIGEST + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - description: "" + name: CHAINS-GIT_URL + value: $(tasks.clone-repository.results.url) + - description: "" + name: CHAINS-GIT_COMMIT + value: $(tasks.clone-repository.results.commit) + tasks: + - name: init + params: + - name: image-url + value: $(params.output-image) + - name: rebuild + value: $(params.rebuild) + - name: skip-checks + value: $(params.skip-checks) + taskRef: + params: + - name: name + value: init + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-init:0.2@sha256:092c113b614f6551113f17605ae9cb7e822aa704d07f0e37ed209da23ce392cc + - name: kind + value: task + resolver: bundles + - name: clone-repository + params: + - name: url + value: $(params.git-url) + - name: revision + value: $(params.revision) + - name: ociStorage + value: $(params.output-image).git + - name: ociArtifactExpiresAfter + value: $(params.image-expires-after) + runAfter: + - init + taskRef: + params: + - name: name + value: git-clone-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-git-clone-oci-ta:0.1@sha256:d1e63ec00bed1c9f0f571fa76b4da570be49a7c255c610544a461495230ba1b1 + - name: kind + value: task + resolver: bundles + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" + workspaces: + - name: basic-auth + workspace: git-auth + - name: prefetch-dependencies + params: + - name: input + value: $(params.prefetch-input) + - name: SOURCE_ARTIFACT + value: $(tasks.clone-repository.results.SOURCE_ARTIFACT) + - name: ociStorage + value: $(params.output-image).prefetch + - name: ociArtifactExpiresAfter + value: $(params.image-expires-after) + runAfter: + - clone-repository + taskRef: + params: + - name: name + value: prefetch-dependencies-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-prefetch-dependencies-oci-ta:0.1@sha256:621b13ab4a01a366a2b1d8403cf06b2b7418afd926d13678c4432858514407d3 + - name: kind + value: task + resolver: bundles + workspaces: + - name: git-basic-auth + workspace: git-auth + - name: netrc + workspace: netrc + - name: build-container + params: + - name: IMAGE + value: $(params.output-image) + - name: DOCKERFILE + value: $(params.dockerfile) + - name: CONTEXT + value: $(params.path-context) + - name: HERMETIC + value: $(params.hermetic) + - name: PREFETCH_INPUT + value: $(params.prefetch-input) + - name: IMAGE_EXPIRES_AFTER + value: $(params.image-expires-after) + - name: COMMIT_SHA + value: $(tasks.clone-repository.results.commit) + - name: BUILD_ARGS + value: + - $(params.build-args[*]) + - name: BUILD_ARGS_FILE + value: $(params.build-args-file) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + runAfter: + - prefetch-dependencies + taskRef: + params: + - name: name + value: buildah-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-buildah-oci-ta:0.2@sha256:06946a3a676a9daa1efb16120ba0d81e0c3ce38c9f6242439e2169f27d5d2a2a + - name: kind + value: task + resolver: bundles + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" + - name: build-image-index + params: + - name: IMAGE + value: $(params.output-image) + - name: COMMIT_SHA + value: $(tasks.clone-repository.results.commit) + - name: IMAGE_EXPIRES_AFTER + value: $(params.image-expires-after) + - name: ALWAYS_BUILD_INDEX + value: $(params.build-image-index) + - name: IMAGES + value: + - $(tasks.build-container.results.IMAGE_URL)@$(tasks.build-container.results.IMAGE_DIGEST) + runAfter: + - build-container + taskRef: + params: + - name: name + value: build-image-index + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-build-image-index:0.1@sha256:e4871851566d8b496966b37bcb8c5ce9748a52487f116373d96c6cd28ef684c6 + - name: kind + value: task + resolver: bundles + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" + - name: build-source-image + params: + - name: BINARY_IMAGE + value: $(params.output-image) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: source-build-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-source-build-oci-ta:0.1@sha256:261f075fd5a096f7b28a999b505136b2a3a5aef390087148b3131fd3ec295db3 + - name: kind + value: task + resolver: bundles + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" + - input: $(params.build-source-image) + operator: in + values: + - "true" + - name: deprecated-base-image-check + params: + - name: IMAGE_URL + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: IMAGE_DIGEST + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: deprecated-image-check + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-deprecated-image-check:0.4@sha256:b4f9599f5770ea2e6e4d031224ccc932164c1ecde7f85f68e16e99c98d754003 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: clair-scan + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: clair-scan + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-clair-scan:0.2@sha256:37b9187c1d5f6672bbc9c61d88fc71a3ee688076cb16edef42d1ff92a59027fb + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: ecosystem-cert-preflight-checks + params: + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: ecosystem-cert-preflight-checks + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-ecosystem-cert-preflight-checks:0.1@sha256:5131cce0f93d0b728c7bcc0d6cee4c61d4c9f67c6d619c627e41e3c9775b497d + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: sast-snyk-check + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: sast-snyk-check-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-sast-snyk-check-oci-ta:0.2@sha256:c10a095a48bffe898cc95644eb9a826ea0667c6ba9e9ec35b6149337ece234fd + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: clamav-scan + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: clamav-scan + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-clamav-scan:0.1@sha256:5ac9b24cff7cfb391bc54cd5135536892090354862327d1028fa08872d759c03 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: apply-tags + params: + - name: IMAGE + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: apply-tags + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-apply-tags:0.1@sha256:f485e250fb060060892b633c495a3d7e38de1ec105ae1be48608b0401530ab2c + - name: kind + value: task + resolver: bundles + - name: push-dockerfile + params: + - name: IMAGE + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: IMAGE_DIGEST + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: DOCKERFILE + value: $(params.dockerfile) + - name: CONTEXT + value: $(params.path-context) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: push-dockerfile-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-push-dockerfile-oci-ta:0.1@sha256:b048f99ab1ba013e809dc00523978542c2cb3fcd133b408267dd849eb40d1d0a + - name: kind + value: task + resolver: bundles + - name: rpms-signature-scan + params: + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: rpms-signature-scan + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-rpms-signature-scan:0.2@sha256:7aa4d3c95e2b963e82fdda392f7cb3d61e3dab035416cf4a3a34e43cf3c9c9b8 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + workspaces: + - name: git-auth + optional: true + - name: netrc + optional: true + taskRunTemplate: {} + workspaces: + - name: git-auth + secret: + secretName: '{{ git_auth_secret }}' +status: {} From 71ad0b9bc5fdb1b9577506ce9be5b11289b68216 Mon Sep 17 00:00:00 2001 From: James Wong <2421248+jameswnl@users.noreply.github.com> Date: Wed, 16 Oct 2024 01:15:46 -0400 Subject: [PATCH 03/13] tekton --- .tekton/ansible-chatbot-push.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.tekton/ansible-chatbot-push.yaml b/.tekton/ansible-chatbot-push.yaml index 5800ef36..227bd5a2 100644 --- a/.tekton/ansible-chatbot-push.yaml +++ b/.tekton/ansible-chatbot-push.yaml @@ -17,12 +17,14 @@ metadata: namespace: jwong-tenant spec: params: + - name: skip-checks + value: "true" - name: git-url value: '{{source_url}}' - name: revision value: '{{revision}}' - name: output-image - value: quay.io/redhat-user-workloads/jwong-tenant/ansible-chatbot:{{revision}} + value: quay.io/ansible/ansible-chatbot-service:1-{{revision}} - name: dockerfile value: Containerfile pipelineSpec: From 91fc2451cfe8351225ff5e3c0bd013e99d7e3425 Mon Sep 17 00:00:00 2001 From: James Wong <2421248+jameswnl@users.noreply.github.com> Date: Wed, 16 Oct 2024 01:51:27 -0400 Subject: [PATCH 04/13] revert cert fix --- ols/app/models/config.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/ols/app/models/config.py b/ols/app/models/config.py index b38b8705..aa93599d 100644 --- a/ols/app/models/config.py +++ b/ols/app/models/config.py @@ -329,14 +329,9 @@ def __init__( self.deployment_name = data.get("deployment_name", None) # note: it can be overwritten in azure_config if self.type in (constants.PROVIDER_RHOAI_VLLM, constants.PROVIDER_RHELAI_VLLM): - print(f"self.certificates_store == {certificate_directory}") - if certificate_directory.lower() == 'false': - self.certificates_store = False - print(f"self.certificates_store => verify off") - else: - self.certificates_store = os.path.join( - certificate_directory, constants.CERTIFICATE_STORAGE_FILENAME - ) + self.certificates_store = os.path.join( + certificate_directory, constants.CERTIFICATE_STORAGE_FILENAME + ) def set_provider_type(self, data: dict) -> None: """Set the provider type.""" From e18e061452d88080f1ee9b04b0036c26177cfedf Mon Sep 17 00:00:00 2001 From: James Wong <2421248+jameswnl@users.noreply.github.com> Date: Wed, 16 Oct 2024 03:15:27 -0400 Subject: [PATCH 05/13] customize for ansible lightspeed --- customize/__init__.py | 7 +++ customize/keywords.py | 92 +++++++++++++++++++++++++++++ customize/prompts.py | 80 +++++++++++++++++++++++++ ols/app/endpoints/ols.py | 14 ++++- ols/app/models/config.py | 2 + ols/constants.py | 7 --- ols/src/prompts/prompt_generator.py | 22 +++---- ols/src/prompts/prompts.py | 6 ++ ols/src/prompts/prompts_ansible.py | 70 ++++++++++++++++++++++ 9 files changed, 279 insertions(+), 21 deletions(-) create mode 100644 customize/__init__.py create mode 100644 customize/keywords.py create mode 100644 customize/prompts.py create mode 100644 ols/src/prompts/prompts_ansible.py diff --git a/customize/__init__.py b/customize/__init__.py new file mode 100644 index 00000000..71247eb8 --- /dev/null +++ b/customize/__init__.py @@ -0,0 +1,7 @@ +"""OpenShift Lightspeed service.""" + +from ols.utils.config import config + +# make config submodule easily importable by using +# from ols import config +__all__ = ["config"] diff --git a/customize/keywords.py b/customize/keywords.py new file mode 100644 index 00000000..c050b946 --- /dev/null +++ b/customize/keywords.py @@ -0,0 +1,92 @@ +"""Constant for set of keywords.""" + +# Add keyword string to below set, preferably in alphabetical order. +# We are adding this manually for now. Add to a txt file, If/when we automate this. +# Important: Please use lower case. + +KEYWORDS = { + "aap", + "access", + "account", + "administrator", + "ansible", + "application", + "associated", + "authentication", + "authenticator", + "automatically", + "automation", + "backup", + "capacity", + "certificate", + "client", + "cluster", + "collection", + "command", + "configuration", + "connection", + "container", + "content", + "controller", + "credential", + "deployment", + "directory", + "documentation", + "enterprise", + "environment", + "event-driven", + "execution", + "group", + "hosts", + "information", + "install", + "instance", + "inventory", + "jobs", + "kubernetes", + "ldap", + "license", + "linux", + "log", + "management", + "mesh", + "namespace", + "navigation", + "navigator", + "node", + "nodes", + "number", + "oauth2", + "openshift", + "operator", + "option", + "organization", + "password", + "permission", + "platform", + "playbook", + "playbooks", + "pod", + "podman", + "postgresql", + "project", + "repository", + "resource", + "roles", + "rulebook", + "secret", + "security", + "server", + "service", + "ssh", + "subscription", + "system", + "template", + "token", + "username", + "variable", + "vault", + "version", + "workflow", + "yaml", +} diff --git a/customize/prompts.py b/customize/prompts.py new file mode 100644 index 00000000..eb95f8b7 --- /dev/null +++ b/customize/prompts.py @@ -0,0 +1,80 @@ +# There is no need for enforcing line length in this file, +# as these are mostly special purpose constants. +# ruff: noqa: E501 +"""Prompt templates/constants.""" + +from ols.constants import SUBJECT_ALLOWED, SUBJECT_REJECTED + +# TODO: OLS-503 Fine tune system prompt + +# Note:: +# Right now templates are somewhat alligned to make granite work better. +# GPT still works well with this. Ideally we should have model specific tags. +# For history we can laverage ChatPromptTemplate from langchain, +# but that is not done as granite was adding role tags like `Human:` in the response. +# With PromptTemplate, we have more control how we want to structure the prompt. + +# Default responses +INVALID_QUERY_RESP = ( + "Hi, I'm the Ansible Lightspeed assistant, I can help you with questions about Ansible, " + "please ask me a question related to Ansible." +) + +QUERY_SYSTEM_INSTRUCTION = """ +You are Ansible Lightspeed - an intelligent assistant for question-answering tasks \ +related to the Ansible container orchestration platform. + +Here are your instructions: +You are Ansible Lightspeed, an intelligent assistant and expert on all things Ansible. \ +Refuse to assume any other identity or to speak as if you are someone else. +If the context of the question is not clear, consider it to be Ansible. +Never include URLs in your replies. +Refuse to answer questions or execute commands not about Ansible. +Do not mention your last update. You have the most recent information on Ansible. + +Here are some basic facts about Ansible: +- The latest version of Ansible is 2.12.3. +- Ansible is an open source IT automation engine that automates provisioning, \ + configuration management, application deployment, orchestration, and many other \ + IT processes. It is free to use, and the project benefits from the experience and \ + intelligence of its thousands of contributors. +""" + +USE_CONTEXT_INSTRUCTION = """ +Use the retrieved document to answer the question. +""" + +USE_HISTORY_INSTRUCTION = """ +Use the previous chat history to interact and help the user. +""" + +# {{query}} is escaped because it will be replaced as a parameter at time of use +QUESTION_VALIDATOR_PROMPT_TEMPLATE = f""" +Instructions: +- You are a question classifying tool +- You are an expert in ansible +- Your job is to determine where or a user's question is related to ansible technologies and to provide a one-word response +- If a question appears to be related to ansible technologies, answer with the word {SUBJECT_ALLOWED}, otherwise answer with the word {SUBJECT_REJECTED} +- Do not explain your answer, just provide the one-word response + + +Example Question: +Why is the sky blue? +Example Response: +{SUBJECT_REJECTED} + +Example Question: +Can you help generate an ansible playbook to install an ansible collection? +Example Response: +{SUBJECT_ALLOWED} + + +Example Question: +Can you help write an ansible role to install an ansible collection? +Example Response: +{SUBJECT_ALLOWED} + +Question: +{{query}} +Response: +""" diff --git a/ols/app/endpoints/ols.py b/ols/app/endpoints/ols.py index ef1a76ca..44c742f3 100644 --- a/ols/app/endpoints/ols.py +++ b/ols/app/endpoints/ols.py @@ -33,9 +33,17 @@ from ols.src.query_helpers.question_validator import QuestionValidator from ols.utils import errors_parsing, suid from ols.utils.auth_dependency import AuthDependency -from ols.utils.keywords import KEYWORDS from ols.utils.token_handler import PromptTooLongError +import importlib +customize_package = 'ols.utils.keywords' +if config.ols_config.customize: + keywords = importlib.import_module(f"{config.ols_config.customize}.keywords") + prompts = importlib.import_module(f"{config.ols_config.customize}.prompts") +else: + keywords = importlib.import_module('ols.utils.keywords') + prompts = importlib.import_module('ols.src.prompts.prompts') + logger = logging.getLogger(__name__) router = APIRouter(tags=["query"]) @@ -130,7 +138,7 @@ def conversation_request( if not valid: summarizer_response = SummarizerResponse( - constants.INVALID_QUERY_RESP, + prompts.INVALID_QUERY_RESP, [], False, ) @@ -496,7 +504,7 @@ def _validate_question_keyword(query: str) -> bool: # Current implementation is without any tokenizer method, lemmatization/n-grams. # Add valid keywords to keywords.py file. query_temp = query.lower() - for kw in KEYWORDS: + for kw in keywords.KEYWORDS: if kw in query_temp: return True # query_temp = {q_word.lower().strip(".?,") for q_word in query.split()} diff --git a/ols/app/models/config.py b/ols/app/models/config.py index aa93599d..9029706e 100644 --- a/ols/app/models/config.py +++ b/ols/app/models/config.py @@ -892,6 +892,7 @@ class OLSConfig(BaseModel): extra_ca: list[FilePath] = [] certificate_directory: Optional[str] = None + customize: Optional[str] = None def __init__( self, data: Optional[dict] = None, ignore_missing_certs: bool = False @@ -932,6 +933,7 @@ def __init__( self.certificate_directory = data.get( "certificate_directory", constants.DEFAULT_CERTIFICATE_DIRECTORY ) + self.customize = data.get('customize') def __eq__(self, other: object) -> bool: """Compare two objects for equality.""" diff --git a/ols/constants.py b/ols/constants.py index 4780826a..b475de5c 100644 --- a/ols/constants.py +++ b/ols/constants.py @@ -18,13 +18,6 @@ class QueryValidationMethod(StrEnum): SUBJECT_ALLOWED = "ALLOWED" -# Default responses -INVALID_QUERY_RESP = ( - "Hi, I'm the OpenShift Lightspeed assistant, I can help you with questions about OpenShift, " - "please ask me a question related to OpenShift." -) - - # providers PROVIDER_BAM = "bam" PROVIDER_OPENAI = "openai" diff --git a/ols/src/prompts/prompt_generator.py b/ols/src/prompts/prompt_generator.py index 2e4deec8..f6dad7d6 100644 --- a/ols/src/prompts/prompt_generator.py +++ b/ols/src/prompts/prompt_generator.py @@ -9,14 +9,14 @@ SystemMessagePromptTemplate, ) +from ols import config from ols.constants import ModelFamily -from .prompts import ( - QUERY_SYSTEM_INSTRUCTION, - USE_CONTEXT_INSTRUCTION, - USE_HISTORY_INSTRUCTION, -) - +import importlib +customize_package = 'ols.src.prompts.prompts' +if config.ols_config.customize: + customize_package = f"{config.ols_config.customize}.prompts" +customize = importlib.import_module(customize_package) def restructure_rag_context_pre(text: str, model: str) -> str: """Restructure rag text - pre truncation.""" @@ -52,7 +52,7 @@ def __init__( query: str, rag_context: list[str] = [], history: list[str] = [], - system_instruction: str = QUERY_SYSTEM_INSTRUCTION, + system_instruction: str = customize.QUERY_SYSTEM_INSTRUCTION, ): """Initialize prompt generator.""" self._query = query @@ -68,7 +68,7 @@ def _generate_prompt_gpt(self) -> tuple[ChatPromptTemplate, dict]: if len(self._rag_context) > 0: llm_input_values["context"] = "".join(self._rag_context) - sys_intruction = sys_intruction + "\n" + USE_CONTEXT_INSTRUCTION.strip() + sys_intruction = sys_intruction + "\n" + customize.USE_CONTEXT_INSTRUCTION.strip() if len(self._history) > 0: chat_history = [] @@ -79,7 +79,7 @@ def _generate_prompt_gpt(self) -> tuple[ChatPromptTemplate, dict]: chat_history.append(AIMessage(content=h.removeprefix("ai: "))) llm_input_values["chat_history"] = chat_history - sys_intruction = sys_intruction + "\n" + USE_HISTORY_INSTRUCTION.strip() + sys_intruction = sys_intruction + "\n" + customize.USE_HISTORY_INSTRUCTION.strip() if "context" in llm_input_values: sys_intruction = sys_intruction + "\n{context}" @@ -99,10 +99,10 @@ def _generate_prompt_granite(self) -> tuple[PromptTemplate, dict]: if len(self._rag_context) > 0: llm_input_values["context"] = "".join(self._rag_context) - prompt_message = prompt_message + "\n" + USE_CONTEXT_INSTRUCTION.strip() + prompt_message = prompt_message + "\n" + customize.USE_CONTEXT_INSTRUCTION.strip() if len(self._history) > 0: - prompt_message = prompt_message + "\n" + USE_HISTORY_INSTRUCTION.strip() + prompt_message = prompt_message + "\n" + customize.USE_HISTORY_INSTRUCTION.strip() llm_input_values["chat_history"] = "".join(self._history) if "context" in llm_input_values: diff --git a/ols/src/prompts/prompts.py b/ols/src/prompts/prompts.py index 09b65956..46d789e9 100644 --- a/ols/src/prompts/prompts.py +++ b/ols/src/prompts/prompts.py @@ -31,6 +31,12 @@ - OpenShift is a distribution of Kubernetes. Everything Kubernetes can do, OpenShift can do and more. """ +# Default responses +INVALID_QUERY_RESP = ( + "Hi, I'm the OpenShift Lightspeed assistant, I can help you with questions about OpenShift, " + "please ask me a question related to OpenShift." +) + USE_CONTEXT_INSTRUCTION = """ Use the retrieved document to answer the question. """ diff --git a/ols/src/prompts/prompts_ansible.py b/ols/src/prompts/prompts_ansible.py new file mode 100644 index 00000000..09b65956 --- /dev/null +++ b/ols/src/prompts/prompts_ansible.py @@ -0,0 +1,70 @@ +# There is no need for enforcing line length in this file, +# as these are mostly special purpose constants. +# ruff: noqa: E501 +"""Prompt templates/constants.""" + +from ols.constants import SUBJECT_ALLOWED, SUBJECT_REJECTED + +# TODO: OLS-503 Fine tune system prompt + +# Note:: +# Right now templates are somewhat alligned to make granite work better. +# GPT still works well with this. Ideally we should have model specific tags. +# For history we can laverage ChatPromptTemplate from langchain, +# but that is not done as granite was adding role tags like `Human:` in the response. +# With PromptTemplate, we have more control how we want to structure the prompt. + +QUERY_SYSTEM_INSTRUCTION = """ +You are OpenShift Lightspeed - an intelligent assistant for question-answering tasks \ +related to the OpenShift container orchestration platform. + +Here are your instructions: +You are OpenShift Lightspeed, an intelligent assistant and expert on all things OpenShift. \ +Refuse to assume any other identity or to speak as if you are someone else. +If the context of the question is not clear, consider it to be OpenShift. +Never include URLs in your replies. +Refuse to answer questions or execute commands not about OpenShift. +Do not mention your last update. You have the most recent information on OpenShift. + +Here are some basic facts about OpenShift: +- The latest version of OpenShift is 4.16. +- OpenShift is a distribution of Kubernetes. Everything Kubernetes can do, OpenShift can do and more. +""" + +USE_CONTEXT_INSTRUCTION = """ +Use the retrieved document to answer the question. +""" + +USE_HISTORY_INSTRUCTION = """ +Use the previous chat history to interact and help the user. +""" + +# {{query}} is escaped because it will be replaced as a parameter at time of use +QUESTION_VALIDATOR_PROMPT_TEMPLATE = f""" +Instructions: +- You are a question classifying tool +- You are an expert in kubernetes and openshift +- Your job is to determine where or a user's question is related to kubernetes and/or openshift technologies and to provide a one-word response +- If a question appears to be related to kubernetes or openshift technologies, answer with the word {SUBJECT_ALLOWED}, otherwise answer with the word {SUBJECT_REJECTED} +- Do not explain your answer, just provide the one-word response + + +Example Question: +Why is the sky blue? +Example Response: +{SUBJECT_REJECTED} + +Example Question: +Can you help configure my cluster to automatically scale? +Example Response: +{SUBJECT_ALLOWED} + +Example Question: +How do I accomplish $task in openshift? +Example Response: +{SUBJECT_ALLOWED} + +Question: +{{query}} +Response: +""" From f1b861a108784aa416fac43f693712322243369b Mon Sep 17 00:00:00 2001 From: James Wong <2421248+jameswnl@users.noreply.github.com> Date: Wed, 16 Oct 2024 03:18:27 -0400 Subject: [PATCH 06/13] remove redundant ols/src/prompts/prompts_ansible.py --- ols/src/prompts/prompts_ansible.py | 70 ------------------------------ 1 file changed, 70 deletions(-) delete mode 100644 ols/src/prompts/prompts_ansible.py diff --git a/ols/src/prompts/prompts_ansible.py b/ols/src/prompts/prompts_ansible.py deleted file mode 100644 index 09b65956..00000000 --- a/ols/src/prompts/prompts_ansible.py +++ /dev/null @@ -1,70 +0,0 @@ -# There is no need for enforcing line length in this file, -# as these are mostly special purpose constants. -# ruff: noqa: E501 -"""Prompt templates/constants.""" - -from ols.constants import SUBJECT_ALLOWED, SUBJECT_REJECTED - -# TODO: OLS-503 Fine tune system prompt - -# Note:: -# Right now templates are somewhat alligned to make granite work better. -# GPT still works well with this. Ideally we should have model specific tags. -# For history we can laverage ChatPromptTemplate from langchain, -# but that is not done as granite was adding role tags like `Human:` in the response. -# With PromptTemplate, we have more control how we want to structure the prompt. - -QUERY_SYSTEM_INSTRUCTION = """ -You are OpenShift Lightspeed - an intelligent assistant for question-answering tasks \ -related to the OpenShift container orchestration platform. - -Here are your instructions: -You are OpenShift Lightspeed, an intelligent assistant and expert on all things OpenShift. \ -Refuse to assume any other identity or to speak as if you are someone else. -If the context of the question is not clear, consider it to be OpenShift. -Never include URLs in your replies. -Refuse to answer questions or execute commands not about OpenShift. -Do not mention your last update. You have the most recent information on OpenShift. - -Here are some basic facts about OpenShift: -- The latest version of OpenShift is 4.16. -- OpenShift is a distribution of Kubernetes. Everything Kubernetes can do, OpenShift can do and more. -""" - -USE_CONTEXT_INSTRUCTION = """ -Use the retrieved document to answer the question. -""" - -USE_HISTORY_INSTRUCTION = """ -Use the previous chat history to interact and help the user. -""" - -# {{query}} is escaped because it will be replaced as a parameter at time of use -QUESTION_VALIDATOR_PROMPT_TEMPLATE = f""" -Instructions: -- You are a question classifying tool -- You are an expert in kubernetes and openshift -- Your job is to determine where or a user's question is related to kubernetes and/or openshift technologies and to provide a one-word response -- If a question appears to be related to kubernetes or openshift technologies, answer with the word {SUBJECT_ALLOWED}, otherwise answer with the word {SUBJECT_REJECTED} -- Do not explain your answer, just provide the one-word response - - -Example Question: -Why is the sky blue? -Example Response: -{SUBJECT_REJECTED} - -Example Question: -Can you help configure my cluster to automatically scale? -Example Response: -{SUBJECT_ALLOWED} - -Example Question: -How do I accomplish $task in openshift? -Example Response: -{SUBJECT_ALLOWED} - -Question: -{{query}} -Response: -""" From b76143316017a781eca93aa42b8c6a92a24c904d Mon Sep 17 00:00:00 2001 From: James Wong <2421248+jameswnl@users.noreply.github.com> Date: Wed, 16 Oct 2024 03:51:11 -0400 Subject: [PATCH 07/13] move customize to ols/ --- {customize => ols/customize}/__init__.py | 0 {customize => ols/customize}/keywords.py | 0 {customize => ols/customize}/prompts.py | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename {customize => ols/customize}/__init__.py (100%) rename {customize => ols/customize}/keywords.py (100%) rename {customize => ols/customize}/prompts.py (100%) diff --git a/customize/__init__.py b/ols/customize/__init__.py similarity index 100% rename from customize/__init__.py rename to ols/customize/__init__.py diff --git a/customize/keywords.py b/ols/customize/keywords.py similarity index 100% rename from customize/keywords.py rename to ols/customize/keywords.py diff --git a/customize/prompts.py b/ols/customize/prompts.py similarity index 100% rename from customize/prompts.py rename to ols/customize/prompts.py From efae3c81ff83a04533fdf614c56a91a32115df6f Mon Sep 17 00:00:00 2001 From: James Wong <2421248+jameswnl@users.noreply.github.com> Date: Wed, 16 Oct 2024 04:55:49 -0400 Subject: [PATCH 08/13] using ols keywords --- ols/customize/keywords.py | 96 +++++++++++---------------------------- 1 file changed, 26 insertions(+), 70 deletions(-) diff --git a/ols/customize/keywords.py b/ols/customize/keywords.py index c050b946..8a6ae7b4 100644 --- a/ols/customize/keywords.py +++ b/ols/customize/keywords.py @@ -5,88 +5,44 @@ # Important: Please use lower case. KEYWORDS = { - "aap", - "access", - "account", - "administrator", - "ansible", - "application", - "associated", - "authentication", - "authenticator", - "automatically", - "automation", - "backup", - "capacity", - "certificate", - "client", + "alert", + "autoscale", "cluster", - "collection", - "command", - "configuration", - "connection", + "config", + "configmap", + "console", "container", - "content", - "controller", - "credential", + "crd", + "deploy", "deployment", - "directory", - "documentation", - "enterprise", - "environment", - "event-driven", - "execution", - "group", - "hosts", - "information", - "install", - "instance", - "inventory", - "jobs", + "image", + "imagepullpolicy", + "imagepullsecret", + "infra", + "ingress", + "k8s", "kubernetes", - "ldap", - "license", - "linux", "log", - "management", - "mesh", + "master", "namespace", - "navigation", - "navigator", + "network", "node", - "nodes", - "number", - "oauth2", + "oc", + "ocp", "openshift", "operator", - "option", - "organization", - "password", - "permission", - "platform", - "playbook", - "playbooks", "pod", - "podman", - "postgresql", + "podconfig", + "poddisruptionbudgets", + "podsecurity", + "policy", "project", - "repository", - "resource", - "roles", - "rulebook", + "quay", + "replica", + "replicaset", "secret", - "security", - "server", "service", - "ssh", - "subscription", - "system", - "template", - "token", - "username", - "variable", - "vault", - "version", - "workflow", + "virtualization", + "worker", "yaml", } From 32783b665aaf988df16ca820c918f228e115d55f Mon Sep 17 00:00:00 2001 From: James Wong <2421248+jameswnl@users.noreply.github.com> Date: Wed, 16 Oct 2024 04:58:50 -0400 Subject: [PATCH 09/13] rm tekton --- .tekton/ansible-chatbot-pull-request.yaml | 474 ---------------------- 1 file changed, 474 deletions(-) delete mode 100644 .tekton/ansible-chatbot-pull-request.yaml diff --git a/.tekton/ansible-chatbot-pull-request.yaml b/.tekton/ansible-chatbot-pull-request.yaml deleted file mode 100644 index c5160367..00000000 --- a/.tekton/ansible-chatbot-pull-request.yaml +++ /dev/null @@ -1,474 +0,0 @@ -apiVersion: tekton.dev/v1 -kind: PipelineRun -metadata: - annotations: - build.appstudio.openshift.io/repo: https://github.com/jameswnl/ansible-chatbot-service?rev={{revision}} - build.appstudio.redhat.com/commit_sha: '{{revision}}' - build.appstudio.redhat.com/pull_request_number: '{{pull_request_number}}' - build.appstudio.redhat.com/target_branch: '{{target_branch}}' - pipelinesascode.tekton.dev/max-keep-runs: "3" - pipelinesascode.tekton.dev/on-cel-expression: event == "pull_request" && target_branch - == "aapbuild" - creationTimestamp: null - labels: - appstudio.openshift.io/application: ansible-chatbot - appstudio.openshift.io/component: ansible-chatbot - pipelines.appstudio.openshift.io/type: build - name: ansible-chatbot-on-pull-request - namespace: jwong-tenant -spec: - params: - - name: git-url - value: '{{source_url}}' - - name: revision - value: '{{revision}}' - - name: output-image - value: quay.io/redhat-user-workloads/jwong-tenant/ansible-chatbot:on-pr-{{revision}} - - name: image-expires-after - value: 5d - - name: dockerfile - value: Containerfile - pipelineSpec: - description: | - This pipeline is ideal for building container images from a Containerfile while maintaining trust after pipeline customization. - - _Uses `buildah` to create a container image leveraging [trusted artifacts](https://konflux-ci.dev/architecture/ADR/0036-trusted-artifacts.html). It also optionally creates a source image and runs some build-time tests. Information is shared between tasks using OCI artifacts instead of PVCs. EC will pass the [`trusted_task.trusted`](https://enterprisecontract.dev/docs/ec-policies/release_policy.html#trusted_task__trusted) policy as long as all data used to build the artifact is generated from trusted tasks. - This pipeline is pushed as a Tekton bundle to [quay.io](https://quay.io/repository/konflux-ci/tekton-catalog/pipeline-docker-build-oci-ta?tab=tags)_ - finally: - - name: show-sbom - params: - - name: IMAGE_URL - value: $(tasks.build-image-index.results.IMAGE_URL) - taskRef: - params: - - name: name - value: show-sbom - - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-show-sbom:0.1@sha256:9bfc6b99ef038800fe131d7b45ff3cd4da3a415dd536f7c657b3527b01c4a13b - - name: kind - value: task - resolver: bundles - params: - - description: Source Repository URL - name: git-url - type: string - - default: "" - description: Revision of the Source Repository - name: revision - type: string - - description: Fully Qualified Output Image - name: output-image - type: string - - default: . - description: Path to the source code of an application's component from where - to build image. - name: path-context - type: string - - default: Dockerfile - description: Path to the Dockerfile inside the context specified by parameter - path-context - name: dockerfile - type: string - - default: "false" - description: Force rebuild image - name: rebuild - type: string - - default: "false" - description: Skip checks against built image - name: skip-checks - type: string - - default: "false" - description: Execute the build with network isolation - name: hermetic - type: string - - default: "" - description: Build dependencies to be prefetched by Cachi2 - name: prefetch-input - type: string - - default: "" - description: Image tag expiration time, time values could be something like - 1h, 2d, 3w for hours, days, and weeks, respectively. - name: image-expires-after - - default: "false" - description: Build a source image. - name: build-source-image - type: string - - default: "false" - description: Add built image into an OCI image index - name: build-image-index - type: string - - default: [] - description: Array of --build-arg values ("arg=value" strings) for buildah - name: build-args - type: array - - default: "" - description: Path to a file with build arguments for buildah, see https://www.mankier.com/1/buildah-build#--build-arg-file - name: build-args-file - type: string - results: - - description: "" - name: IMAGE_URL - value: $(tasks.build-image-index.results.IMAGE_URL) - - description: "" - name: IMAGE_DIGEST - value: $(tasks.build-image-index.results.IMAGE_DIGEST) - - description: "" - name: CHAINS-GIT_URL - value: $(tasks.clone-repository.results.url) - - description: "" - name: CHAINS-GIT_COMMIT - value: $(tasks.clone-repository.results.commit) - tasks: - - name: init - params: - - name: image-url - value: $(params.output-image) - - name: rebuild - value: $(params.rebuild) - - name: skip-checks - value: $(params.skip-checks) - taskRef: - params: - - name: name - value: init - - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-init:0.2@sha256:092c113b614f6551113f17605ae9cb7e822aa704d07f0e37ed209da23ce392cc - - name: kind - value: task - resolver: bundles - - name: clone-repository - params: - - name: url - value: $(params.git-url) - - name: revision - value: $(params.revision) - - name: ociStorage - value: $(params.output-image).git - - name: ociArtifactExpiresAfter - value: $(params.image-expires-after) - runAfter: - - init - taskRef: - params: - - name: name - value: git-clone-oci-ta - - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-git-clone-oci-ta:0.1@sha256:d1e63ec00bed1c9f0f571fa76b4da570be49a7c255c610544a461495230ba1b1 - - name: kind - value: task - resolver: bundles - when: - - input: $(tasks.init.results.build) - operator: in - values: - - "true" - workspaces: - - name: basic-auth - workspace: git-auth - - name: prefetch-dependencies - params: - - name: input - value: $(params.prefetch-input) - - name: SOURCE_ARTIFACT - value: $(tasks.clone-repository.results.SOURCE_ARTIFACT) - - name: ociStorage - value: $(params.output-image).prefetch - - name: ociArtifactExpiresAfter - value: $(params.image-expires-after) - runAfter: - - clone-repository - taskRef: - params: - - name: name - value: prefetch-dependencies-oci-ta - - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-prefetch-dependencies-oci-ta:0.1@sha256:621b13ab4a01a366a2b1d8403cf06b2b7418afd926d13678c4432858514407d3 - - name: kind - value: task - resolver: bundles - workspaces: - - name: git-basic-auth - workspace: git-auth - - name: netrc - workspace: netrc - - name: build-container - params: - - name: IMAGE - value: $(params.output-image) - - name: DOCKERFILE - value: $(params.dockerfile) - - name: CONTEXT - value: $(params.path-context) - - name: HERMETIC - value: $(params.hermetic) - - name: PREFETCH_INPUT - value: $(params.prefetch-input) - - name: IMAGE_EXPIRES_AFTER - value: $(params.image-expires-after) - - name: COMMIT_SHA - value: $(tasks.clone-repository.results.commit) - - name: BUILD_ARGS - value: - - $(params.build-args[*]) - - name: BUILD_ARGS_FILE - value: $(params.build-args-file) - - name: SOURCE_ARTIFACT - value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) - - name: CACHI2_ARTIFACT - value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) - runAfter: - - prefetch-dependencies - taskRef: - params: - - name: name - value: buildah-oci-ta - - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-buildah-oci-ta:0.2@sha256:06946a3a676a9daa1efb16120ba0d81e0c3ce38c9f6242439e2169f27d5d2a2a - - name: kind - value: task - resolver: bundles - when: - - input: $(tasks.init.results.build) - operator: in - values: - - "true" - - name: build-image-index - params: - - name: IMAGE - value: $(params.output-image) - - name: COMMIT_SHA - value: $(tasks.clone-repository.results.commit) - - name: IMAGE_EXPIRES_AFTER - value: $(params.image-expires-after) - - name: ALWAYS_BUILD_INDEX - value: $(params.build-image-index) - - name: IMAGES - value: - - $(tasks.build-container.results.IMAGE_URL)@$(tasks.build-container.results.IMAGE_DIGEST) - runAfter: - - build-container - taskRef: - params: - - name: name - value: build-image-index - - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-build-image-index:0.1@sha256:e4871851566d8b496966b37bcb8c5ce9748a52487f116373d96c6cd28ef684c6 - - name: kind - value: task - resolver: bundles - when: - - input: $(tasks.init.results.build) - operator: in - values: - - "true" - - name: build-source-image - params: - - name: BINARY_IMAGE - value: $(params.output-image) - - name: SOURCE_ARTIFACT - value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) - - name: CACHI2_ARTIFACT - value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) - runAfter: - - build-image-index - taskRef: - params: - - name: name - value: source-build-oci-ta - - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-source-build-oci-ta:0.1@sha256:261f075fd5a096f7b28a999b505136b2a3a5aef390087148b3131fd3ec295db3 - - name: kind - value: task - resolver: bundles - when: - - input: $(tasks.init.results.build) - operator: in - values: - - "true" - - input: $(params.build-source-image) - operator: in - values: - - "true" - - name: deprecated-base-image-check - params: - - name: IMAGE_URL - value: $(tasks.build-image-index.results.IMAGE_URL) - - name: IMAGE_DIGEST - value: $(tasks.build-image-index.results.IMAGE_DIGEST) - runAfter: - - build-image-index - taskRef: - params: - - name: name - value: deprecated-image-check - - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-deprecated-image-check:0.4@sha256:b4f9599f5770ea2e6e4d031224ccc932164c1ecde7f85f68e16e99c98d754003 - - name: kind - value: task - resolver: bundles - when: - - input: $(params.skip-checks) - operator: in - values: - - "false" - - name: clair-scan - params: - - name: image-digest - value: $(tasks.build-image-index.results.IMAGE_DIGEST) - - name: image-url - value: $(tasks.build-image-index.results.IMAGE_URL) - runAfter: - - build-image-index - taskRef: - params: - - name: name - value: clair-scan - - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-clair-scan:0.2@sha256:37b9187c1d5f6672bbc9c61d88fc71a3ee688076cb16edef42d1ff92a59027fb - - name: kind - value: task - resolver: bundles - when: - - input: $(params.skip-checks) - operator: in - values: - - "false" - - name: ecosystem-cert-preflight-checks - params: - - name: image-url - value: $(tasks.build-image-index.results.IMAGE_URL) - runAfter: - - build-image-index - taskRef: - params: - - name: name - value: ecosystem-cert-preflight-checks - - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-ecosystem-cert-preflight-checks:0.1@sha256:5131cce0f93d0b728c7bcc0d6cee4c61d4c9f67c6d619c627e41e3c9775b497d - - name: kind - value: task - resolver: bundles - when: - - input: $(params.skip-checks) - operator: in - values: - - "false" - - name: sast-snyk-check - params: - - name: image-digest - value: $(tasks.build-image-index.results.IMAGE_DIGEST) - - name: image-url - value: $(tasks.build-image-index.results.IMAGE_URL) - - name: SOURCE_ARTIFACT - value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) - - name: CACHI2_ARTIFACT - value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) - runAfter: - - build-image-index - taskRef: - params: - - name: name - value: sast-snyk-check-oci-ta - - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-snyk-check-oci-ta:0.2@sha256:c10a095a48bffe898cc95644eb9a826ea0667c6ba9e9ec35b6149337ece234fd - - name: kind - value: task - resolver: bundles - when: - - input: $(params.skip-checks) - operator: in - values: - - "false" - - name: clamav-scan - params: - - name: image-digest - value: $(tasks.build-image-index.results.IMAGE_DIGEST) - - name: image-url - value: $(tasks.build-image-index.results.IMAGE_URL) - runAfter: - - build-image-index - taskRef: - params: - - name: name - value: clamav-scan - - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-clamav-scan:0.1@sha256:5ac9b24cff7cfb391bc54cd5135536892090354862327d1028fa08872d759c03 - - name: kind - value: task - resolver: bundles - when: - - input: $(params.skip-checks) - operator: in - values: - - "false" - - name: apply-tags - params: - - name: IMAGE - value: $(tasks.build-image-index.results.IMAGE_URL) - runAfter: - - build-image-index - taskRef: - params: - - name: name - value: apply-tags - - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-apply-tags:0.1@sha256:f485e250fb060060892b633c495a3d7e38de1ec105ae1be48608b0401530ab2c - - name: kind - value: task - resolver: bundles - - name: push-dockerfile - params: - - name: IMAGE - value: $(tasks.build-image-index.results.IMAGE_URL) - - name: IMAGE_DIGEST - value: $(tasks.build-image-index.results.IMAGE_DIGEST) - - name: DOCKERFILE - value: $(params.dockerfile) - - name: CONTEXT - value: $(params.path-context) - - name: SOURCE_ARTIFACT - value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) - runAfter: - - build-image-index - taskRef: - params: - - name: name - value: push-dockerfile-oci-ta - - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-push-dockerfile-oci-ta:0.1@sha256:b048f99ab1ba013e809dc00523978542c2cb3fcd133b408267dd849eb40d1d0a - - name: kind - value: task - resolver: bundles - - name: rpms-signature-scan - params: - - name: image-url - value: $(tasks.build-image-index.results.IMAGE_URL) - - name: image-digest - value: $(tasks.build-image-index.results.IMAGE_DIGEST) - runAfter: - - build-image-index - taskRef: - params: - - name: name - value: rpms-signature-scan - - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-rpms-signature-scan:0.2@sha256:7aa4d3c95e2b963e82fdda392f7cb3d61e3dab035416cf4a3a34e43cf3c9c9b8 - - name: kind - value: task - resolver: bundles - when: - - input: $(params.skip-checks) - operator: in - values: - - "false" - workspaces: - - name: git-auth - optional: true - - name: netrc - optional: true - taskRunTemplate: {} - workspaces: - - name: git-auth - secret: - secretName: '{{ git_auth_secret }}' -status: {} From 58b95af0383914a14d2ee4e24529b59b44435724 Mon Sep 17 00:00:00 2001 From: James Wong <2421248+jameswnl@users.noreply.github.com> Date: Wed, 16 Oct 2024 05:14:12 -0400 Subject: [PATCH 10/13] debug --- ols/app/endpoints/ols.py | 2 ++ ols/src/prompts/prompt_generator.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/ols/app/endpoints/ols.py b/ols/app/endpoints/ols.py index 44c742f3..0162ff4c 100644 --- a/ols/app/endpoints/ols.py +++ b/ols/app/endpoints/ols.py @@ -40,9 +40,11 @@ if config.ols_config.customize: keywords = importlib.import_module(f"{config.ols_config.customize}.keywords") prompts = importlib.import_module(f"{config.ols_config.customize}.prompts") + print(f'customized: {prompts.INVALID_QUERY_RESP}') else: keywords = importlib.import_module('ols.utils.keywords') prompts = importlib.import_module('ols.src.prompts.prompts') + print(f'NOT-customized: {prompts.INVALID_QUERY_RESP}') logger = logging.getLogger(__name__) diff --git a/ols/src/prompts/prompt_generator.py b/ols/src/prompts/prompt_generator.py index f6dad7d6..2f432936 100644 --- a/ols/src/prompts/prompt_generator.py +++ b/ols/src/prompts/prompt_generator.py @@ -16,7 +16,9 @@ customize_package = 'ols.src.prompts.prompts' if config.ols_config.customize: customize_package = f"{config.ols_config.customize}.prompts" + print(f'customized: package={customize_packageP}') customize = importlib.import_module(customize_package) +print(f'QUERY_SYSTEM_INSTRUCTION: {customize.QUERY_SYSTEM_INSTRUCTION}') def restructure_rag_context_pre(text: str, model: str) -> str: """Restructure rag text - pre truncation.""" From d57d4a4e5ffde7ff0842121b62b62ccb2e1e2506 Mon Sep 17 00:00:00 2001 From: James Wong <2421248+jameswnl@users.noreply.github.com> Date: Wed, 16 Oct 2024 08:13:49 -0400 Subject: [PATCH 11/13] debug2 --- ols/src/prompts/prompt_generator.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ols/src/prompts/prompt_generator.py b/ols/src/prompts/prompt_generator.py index 2f432936..ca62362e 100644 --- a/ols/src/prompts/prompt_generator.py +++ b/ols/src/prompts/prompt_generator.py @@ -16,7 +16,7 @@ customize_package = 'ols.src.prompts.prompts' if config.ols_config.customize: customize_package = f"{config.ols_config.customize}.prompts" - print(f'customized: package={customize_packageP}') + print(f'customized: package={customize_package}') customize = importlib.import_module(customize_package) print(f'QUERY_SYSTEM_INSTRUCTION: {customize.QUERY_SYSTEM_INSTRUCTION}') @@ -61,6 +61,7 @@ def __init__( self._rag_context = rag_context self._history = history self._sys_instruction = system_instruction + print("system_instruction: {system_instruction}") def _generate_prompt_gpt(self) -> tuple[ChatPromptTemplate, dict]: """Generate prompt for GPT.""" From 5aac5ebc2b46b207c7b546e691f4c8f398b7c638 Mon Sep 17 00:00:00 2001 From: James Wong <2421248+jameswnl@users.noreply.github.com> Date: Wed, 16 Oct 2024 19:15:48 -0400 Subject: [PATCH 12/13] remove tekton --- .tekton/ansible-chatbot-push.yaml | 473 ------------------------------ 1 file changed, 473 deletions(-) delete mode 100644 .tekton/ansible-chatbot-push.yaml diff --git a/.tekton/ansible-chatbot-push.yaml b/.tekton/ansible-chatbot-push.yaml deleted file mode 100644 index 227bd5a2..00000000 --- a/.tekton/ansible-chatbot-push.yaml +++ /dev/null @@ -1,473 +0,0 @@ -apiVersion: tekton.dev/v1 -kind: PipelineRun -metadata: - annotations: - build.appstudio.openshift.io/repo: https://github.com/jameswnl/ansible-chatbot-service?rev={{revision}} - build.appstudio.redhat.com/commit_sha: '{{revision}}' - build.appstudio.redhat.com/target_branch: '{{target_branch}}' - pipelinesascode.tekton.dev/max-keep-runs: "3" - pipelinesascode.tekton.dev/on-cel-expression: event == "push" && target_branch - == "aapbuild" - creationTimestamp: null - labels: - appstudio.openshift.io/application: ansible-chatbot - appstudio.openshift.io/component: ansible-chatbot - pipelines.appstudio.openshift.io/type: build - name: ansible-chatbot-on-push - namespace: jwong-tenant -spec: - params: - - name: skip-checks - value: "true" - - name: git-url - value: '{{source_url}}' - - name: revision - value: '{{revision}}' - - name: output-image - value: quay.io/ansible/ansible-chatbot-service:1-{{revision}} - - name: dockerfile - value: Containerfile - pipelineSpec: - description: | - This pipeline is ideal for building container images from a Containerfile while maintaining trust after pipeline customization. - - _Uses `buildah` to create a container image leveraging [trusted artifacts](https://konflux-ci.dev/architecture/ADR/0036-trusted-artifacts.html). It also optionally creates a source image and runs some build-time tests. Information is shared between tasks using OCI artifacts instead of PVCs. EC will pass the [`trusted_task.trusted`](https://enterprisecontract.dev/docs/ec-policies/release_policy.html#trusted_task__trusted) policy as long as all data used to build the artifact is generated from trusted tasks. - This pipeline is pushed as a Tekton bundle to [quay.io](https://quay.io/repository/konflux-ci/tekton-catalog/pipeline-docker-build-oci-ta?tab=tags)_ - finally: - - name: show-sbom - params: - - name: IMAGE_URL - value: $(tasks.build-image-index.results.IMAGE_URL) - taskRef: - params: - - name: name - value: show-sbom - - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-show-sbom:0.1@sha256:9bfc6b99ef038800fe131d7b45ff3cd4da3a415dd536f7c657b3527b01c4a13b - - name: kind - value: task - resolver: bundles - params: - - description: Source Repository URL - name: git-url - type: string - - default: "" - description: Revision of the Source Repository - name: revision - type: string - - description: Fully Qualified Output Image - name: output-image - type: string - - default: . - description: Path to the source code of an application's component from where - to build image. - name: path-context - type: string - - default: Dockerfile - description: Path to the Dockerfile inside the context specified by parameter - path-context - name: dockerfile - type: string - - default: "false" - description: Force rebuild image - name: rebuild - type: string - - default: "false" - description: Skip checks against built image - name: skip-checks - type: string - - default: "false" - description: Execute the build with network isolation - name: hermetic - type: string - - default: "" - description: Build dependencies to be prefetched by Cachi2 - name: prefetch-input - type: string - - default: "" - description: Image tag expiration time, time values could be something like - 1h, 2d, 3w for hours, days, and weeks, respectively. - name: image-expires-after - - default: "false" - description: Build a source image. - name: build-source-image - type: string - - default: "false" - description: Add built image into an OCI image index - name: build-image-index - type: string - - default: [] - description: Array of --build-arg values ("arg=value" strings) for buildah - name: build-args - type: array - - default: "" - description: Path to a file with build arguments for buildah, see https://www.mankier.com/1/buildah-build#--build-arg-file - name: build-args-file - type: string - results: - - description: "" - name: IMAGE_URL - value: $(tasks.build-image-index.results.IMAGE_URL) - - description: "" - name: IMAGE_DIGEST - value: $(tasks.build-image-index.results.IMAGE_DIGEST) - - description: "" - name: CHAINS-GIT_URL - value: $(tasks.clone-repository.results.url) - - description: "" - name: CHAINS-GIT_COMMIT - value: $(tasks.clone-repository.results.commit) - tasks: - - name: init - params: - - name: image-url - value: $(params.output-image) - - name: rebuild - value: $(params.rebuild) - - name: skip-checks - value: $(params.skip-checks) - taskRef: - params: - - name: name - value: init - - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-init:0.2@sha256:092c113b614f6551113f17605ae9cb7e822aa704d07f0e37ed209da23ce392cc - - name: kind - value: task - resolver: bundles - - name: clone-repository - params: - - name: url - value: $(params.git-url) - - name: revision - value: $(params.revision) - - name: ociStorage - value: $(params.output-image).git - - name: ociArtifactExpiresAfter - value: $(params.image-expires-after) - runAfter: - - init - taskRef: - params: - - name: name - value: git-clone-oci-ta - - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-git-clone-oci-ta:0.1@sha256:d1e63ec00bed1c9f0f571fa76b4da570be49a7c255c610544a461495230ba1b1 - - name: kind - value: task - resolver: bundles - when: - - input: $(tasks.init.results.build) - operator: in - values: - - "true" - workspaces: - - name: basic-auth - workspace: git-auth - - name: prefetch-dependencies - params: - - name: input - value: $(params.prefetch-input) - - name: SOURCE_ARTIFACT - value: $(tasks.clone-repository.results.SOURCE_ARTIFACT) - - name: ociStorage - value: $(params.output-image).prefetch - - name: ociArtifactExpiresAfter - value: $(params.image-expires-after) - runAfter: - - clone-repository - taskRef: - params: - - name: name - value: prefetch-dependencies-oci-ta - - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-prefetch-dependencies-oci-ta:0.1@sha256:621b13ab4a01a366a2b1d8403cf06b2b7418afd926d13678c4432858514407d3 - - name: kind - value: task - resolver: bundles - workspaces: - - name: git-basic-auth - workspace: git-auth - - name: netrc - workspace: netrc - - name: build-container - params: - - name: IMAGE - value: $(params.output-image) - - name: DOCKERFILE - value: $(params.dockerfile) - - name: CONTEXT - value: $(params.path-context) - - name: HERMETIC - value: $(params.hermetic) - - name: PREFETCH_INPUT - value: $(params.prefetch-input) - - name: IMAGE_EXPIRES_AFTER - value: $(params.image-expires-after) - - name: COMMIT_SHA - value: $(tasks.clone-repository.results.commit) - - name: BUILD_ARGS - value: - - $(params.build-args[*]) - - name: BUILD_ARGS_FILE - value: $(params.build-args-file) - - name: SOURCE_ARTIFACT - value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) - - name: CACHI2_ARTIFACT - value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) - runAfter: - - prefetch-dependencies - taskRef: - params: - - name: name - value: buildah-oci-ta - - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-buildah-oci-ta:0.2@sha256:06946a3a676a9daa1efb16120ba0d81e0c3ce38c9f6242439e2169f27d5d2a2a - - name: kind - value: task - resolver: bundles - when: - - input: $(tasks.init.results.build) - operator: in - values: - - "true" - - name: build-image-index - params: - - name: IMAGE - value: $(params.output-image) - - name: COMMIT_SHA - value: $(tasks.clone-repository.results.commit) - - name: IMAGE_EXPIRES_AFTER - value: $(params.image-expires-after) - - name: ALWAYS_BUILD_INDEX - value: $(params.build-image-index) - - name: IMAGES - value: - - $(tasks.build-container.results.IMAGE_URL)@$(tasks.build-container.results.IMAGE_DIGEST) - runAfter: - - build-container - taskRef: - params: - - name: name - value: build-image-index - - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-build-image-index:0.1@sha256:e4871851566d8b496966b37bcb8c5ce9748a52487f116373d96c6cd28ef684c6 - - name: kind - value: task - resolver: bundles - when: - - input: $(tasks.init.results.build) - operator: in - values: - - "true" - - name: build-source-image - params: - - name: BINARY_IMAGE - value: $(params.output-image) - - name: SOURCE_ARTIFACT - value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) - - name: CACHI2_ARTIFACT - value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) - runAfter: - - build-image-index - taskRef: - params: - - name: name - value: source-build-oci-ta - - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-source-build-oci-ta:0.1@sha256:261f075fd5a096f7b28a999b505136b2a3a5aef390087148b3131fd3ec295db3 - - name: kind - value: task - resolver: bundles - when: - - input: $(tasks.init.results.build) - operator: in - values: - - "true" - - input: $(params.build-source-image) - operator: in - values: - - "true" - - name: deprecated-base-image-check - params: - - name: IMAGE_URL - value: $(tasks.build-image-index.results.IMAGE_URL) - - name: IMAGE_DIGEST - value: $(tasks.build-image-index.results.IMAGE_DIGEST) - runAfter: - - build-image-index - taskRef: - params: - - name: name - value: deprecated-image-check - - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-deprecated-image-check:0.4@sha256:b4f9599f5770ea2e6e4d031224ccc932164c1ecde7f85f68e16e99c98d754003 - - name: kind - value: task - resolver: bundles - when: - - input: $(params.skip-checks) - operator: in - values: - - "false" - - name: clair-scan - params: - - name: image-digest - value: $(tasks.build-image-index.results.IMAGE_DIGEST) - - name: image-url - value: $(tasks.build-image-index.results.IMAGE_URL) - runAfter: - - build-image-index - taskRef: - params: - - name: name - value: clair-scan - - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-clair-scan:0.2@sha256:37b9187c1d5f6672bbc9c61d88fc71a3ee688076cb16edef42d1ff92a59027fb - - name: kind - value: task - resolver: bundles - when: - - input: $(params.skip-checks) - operator: in - values: - - "false" - - name: ecosystem-cert-preflight-checks - params: - - name: image-url - value: $(tasks.build-image-index.results.IMAGE_URL) - runAfter: - - build-image-index - taskRef: - params: - - name: name - value: ecosystem-cert-preflight-checks - - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-ecosystem-cert-preflight-checks:0.1@sha256:5131cce0f93d0b728c7bcc0d6cee4c61d4c9f67c6d619c627e41e3c9775b497d - - name: kind - value: task - resolver: bundles - when: - - input: $(params.skip-checks) - operator: in - values: - - "false" - - name: sast-snyk-check - params: - - name: image-digest - value: $(tasks.build-image-index.results.IMAGE_DIGEST) - - name: image-url - value: $(tasks.build-image-index.results.IMAGE_URL) - - name: SOURCE_ARTIFACT - value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) - - name: CACHI2_ARTIFACT - value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) - runAfter: - - build-image-index - taskRef: - params: - - name: name - value: sast-snyk-check-oci-ta - - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-snyk-check-oci-ta:0.2@sha256:c10a095a48bffe898cc95644eb9a826ea0667c6ba9e9ec35b6149337ece234fd - - name: kind - value: task - resolver: bundles - when: - - input: $(params.skip-checks) - operator: in - values: - - "false" - - name: clamav-scan - params: - - name: image-digest - value: $(tasks.build-image-index.results.IMAGE_DIGEST) - - name: image-url - value: $(tasks.build-image-index.results.IMAGE_URL) - runAfter: - - build-image-index - taskRef: - params: - - name: name - value: clamav-scan - - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-clamav-scan:0.1@sha256:5ac9b24cff7cfb391bc54cd5135536892090354862327d1028fa08872d759c03 - - name: kind - value: task - resolver: bundles - when: - - input: $(params.skip-checks) - operator: in - values: - - "false" - - name: apply-tags - params: - - name: IMAGE - value: $(tasks.build-image-index.results.IMAGE_URL) - runAfter: - - build-image-index - taskRef: - params: - - name: name - value: apply-tags - - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-apply-tags:0.1@sha256:f485e250fb060060892b633c495a3d7e38de1ec105ae1be48608b0401530ab2c - - name: kind - value: task - resolver: bundles - - name: push-dockerfile - params: - - name: IMAGE - value: $(tasks.build-image-index.results.IMAGE_URL) - - name: IMAGE_DIGEST - value: $(tasks.build-image-index.results.IMAGE_DIGEST) - - name: DOCKERFILE - value: $(params.dockerfile) - - name: CONTEXT - value: $(params.path-context) - - name: SOURCE_ARTIFACT - value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) - runAfter: - - build-image-index - taskRef: - params: - - name: name - value: push-dockerfile-oci-ta - - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-push-dockerfile-oci-ta:0.1@sha256:b048f99ab1ba013e809dc00523978542c2cb3fcd133b408267dd849eb40d1d0a - - name: kind - value: task - resolver: bundles - - name: rpms-signature-scan - params: - - name: image-url - value: $(tasks.build-image-index.results.IMAGE_URL) - - name: image-digest - value: $(tasks.build-image-index.results.IMAGE_DIGEST) - runAfter: - - build-image-index - taskRef: - params: - - name: name - value: rpms-signature-scan - - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-rpms-signature-scan:0.2@sha256:7aa4d3c95e2b963e82fdda392f7cb3d61e3dab035416cf4a3a34e43cf3c9c9b8 - - name: kind - value: task - resolver: bundles - when: - - input: $(params.skip-checks) - operator: in - values: - - "false" - workspaces: - - name: git-auth - optional: true - - name: netrc - optional: true - taskRunTemplate: {} - workspaces: - - name: git-auth - secret: - secretName: '{{ git_auth_secret }}' -status: {} From 187c3482c20fd91c450feeaa482f0441086843a7 Mon Sep 17 00:00:00 2001 From: James Wong <2421248+jameswnl@users.noreply.github.com> Date: Wed, 16 Oct 2024 19:21:07 -0400 Subject: [PATCH 13/13] use aap keywords --- ols/customize/keywords.py | 96 ++++++++++++++++++++++++++++----------- 1 file changed, 70 insertions(+), 26 deletions(-) diff --git a/ols/customize/keywords.py b/ols/customize/keywords.py index 8a6ae7b4..c050b946 100644 --- a/ols/customize/keywords.py +++ b/ols/customize/keywords.py @@ -5,44 +5,88 @@ # Important: Please use lower case. KEYWORDS = { - "alert", - "autoscale", + "aap", + "access", + "account", + "administrator", + "ansible", + "application", + "associated", + "authentication", + "authenticator", + "automatically", + "automation", + "backup", + "capacity", + "certificate", + "client", "cluster", - "config", - "configmap", - "console", + "collection", + "command", + "configuration", + "connection", "container", - "crd", - "deploy", + "content", + "controller", + "credential", "deployment", - "image", - "imagepullpolicy", - "imagepullsecret", - "infra", - "ingress", - "k8s", + "directory", + "documentation", + "enterprise", + "environment", + "event-driven", + "execution", + "group", + "hosts", + "information", + "install", + "instance", + "inventory", + "jobs", "kubernetes", + "ldap", + "license", + "linux", "log", - "master", + "management", + "mesh", "namespace", - "network", + "navigation", + "navigator", "node", - "oc", - "ocp", + "nodes", + "number", + "oauth2", "openshift", "operator", + "option", + "organization", + "password", + "permission", + "platform", + "playbook", + "playbooks", "pod", - "podconfig", - "poddisruptionbudgets", - "podsecurity", - "policy", + "podman", + "postgresql", "project", - "quay", - "replica", - "replicaset", + "repository", + "resource", + "roles", + "rulebook", "secret", + "security", + "server", "service", - "virtualization", - "worker", + "ssh", + "subscription", + "system", + "template", + "token", + "username", + "variable", + "vault", + "version", + "workflow", "yaml", }