feat: federated learning use case implementation

.gitignore

@@ -14,6 +14,7 @@ monitoring-dashboard-samples/
 **/tfplan
 *.terraform/
 *.terraform-*/
+terraform.tfstate*
 
 # Test
 test/log/*.log

platforms/gke/base/use-cases/README.md

@@ -1 +1,3 @@
 # GKE Accelerated Platform Use Cases
+
+- [Federated learning](/docs/use-cases/federated-learning/README.md)

platforms/gke/base/use-cases/federated-learning/README.md

@@ -1 +1,17 @@
 # Federated learning on Google Cloud
+
+## Deploy the Federated learning reference architecture
+
+1. Provision the Federated Learning reference architecture:
+
+   ```sh
+   "${ACP_PLATFORM_BASE_DIR}/use-cases/federated-learning/deploy.sh"
+   ```
+
+## Teardown the Federated Learning reference architecture
+
+1. Teardown the Federated Learning reference architecture:
+
+   ```sh
+   "${ACP_PLATFORM_BASE_DIR}/use-cases/federated-learning/teardown.sh"
+   ```

platforms/gke/base/use-cases/federated-learning/common.sh

@@ -0,0 +1,98 @@
+#!/usr/bin/env bash
+#
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+ACP_PLATFORM_SHARED_CONFIG_DIR="${ACP_PLATFORM_BASE_DIR}/_shared_config"
+
+# shellcheck disable=SC2034 # Variable is used in other scripts
+ACP_PLATFORM_SHARED_CONFIG_CLUSTER_AUTO_VARS_FILE="${ACP_PLATFORM_SHARED_CONFIG_DIR}/cluster.auto.tfvars"
+
+# shellcheck disable=SC1091
+source "${ACP_PLATFORM_SHARED_CONFIG_DIR}/scripts/set_environment_variables.sh" "${ACP_PLATFORM_SHARED_CONFIG_DIR}"
+
+FEDERATED_LEARNING_USE_CASE_DIR="${ACP_PLATFORM_BASE_DIR}/use-cases/federated-learning"
+FEDERATED_LEARNING_USE_CASE_TERRAFORM_DIR="${FEDERATED_LEARNING_USE_CASE_DIR}/terraform"
+# shellcheck disable=SC2034 # Variable is used in other scripts
+FEDERATED_LEARNING_SHARED_CONFIG_DIR="${FEDERATED_LEARNING_USE_CASE_TERRAFORM_DIR}/_shared_config"
+
+# shellcheck disable=SC2034 # Variable is used in other scripts
+federated_learning_terraservices=(
+  "initialize"
+  "container_image_repository"
+  "private_google_access"
+)
+
+# shellcheck disable=SC2034 # Variable is used in other scripts
+TERRAFORM_CLUSTER_CONFIGURATION=(
+)
+
+apply_or_destroy_terraservice() {
+  local terraservice
+  terraservice="${1}"
+
+  local operation_mode
+  operation_mode="${2:-"not set"}"
+
+  cd "${FEDERATED_LEARNING_USE_CASE_TERRAFORM_DIR}/${terraservice}" &&
+    terraform init
+
+  echo "Current working directory: $(pwd)"
+
+  if [[ "${operation_mode}" == "apply" ]]; then
+    echo "Provisioning ${terraservice}"
+    terraform plan -input=false -out=tfplan &&
+      terraform apply -input=false tfplan
+    _terraform_result=$?
+  elif [[ "${operation_mode}" == "destroy" ]]; then
+    echo "Destroying ${terraservice}"
+    terraform destroy -auto-approve
+    _terraform_result=$?
+  else
+    echo "Error: operation mode not supported: ${operation_mode}"
+    _terraform_result=1
+  fi
+
+  rm -rf \
+    "${FEDERATED_LEARNING_USE_CASE_TERRAFORM_DIR}/${terraservice}/.terraform" \
+    "${FEDERATED_LEARNING_USE_CASE_TERRAFORM_DIR}/${terraservice}/tfplan"
+
+  if [[ ${_terraform_result} -ne 0 ]]; then
+    echo "Terraform ${operation_mode} command failed with code ${_terraform_result} for ${terraservice}"
+    exit ${_terraform_result}
+  fi
+}
+
+provision_terraservice() {
+  apply_or_destroy_terraservice "${1}" "apply"
+}
+
+destroy_terraservice() {
+  apply_or_destroy_terraservice "${1}" "destroy"
+}
+
+get_terraform_output() {
+  terraservice="${1}"
+  output_name="${2}"
+  local output
+  if ! output="$(cd "${FEDERATED_LEARNING_USE_CASE_TERRAFORM_DIR}/${terraservice}" && terraform output -raw "${output_name}")"; then
+    echo "Error while getting ${output_name} output"
+    return 1
+  fi
+  echo "${output}"
+}

platforms/gke/base/use-cases/federated-learning/deploy.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+#
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# shellcheck disable=SC1091
+source "${ACP_PLATFORM_BASE_DIR}/use-cases/federated-learning/common.sh"
+
+start_timestamp_federated_learning=$(date +%s)
+
+echo "Preparing core platform configuration files"
+for configuration_variable in "${TERRAFORM_CLUSTER_CONFIGURATION[@]}"; do
+  configuration_variable_name="$(echo "${configuration_variable}" | awk '{ print $1 }')"
+  echo "Checking if ${configuration_variable_name} is in ${ACP_PLATFORM_SHARED_CONFIG_CLUSTER_AUTO_VARS_FILE}"
+  grep -q "${configuration_variable_name}" "${ACP_PLATFORM_SHARED_CONFIG_CLUSTER_AUTO_VARS_FILE}" || echo "${configuration_variable}" >>"${ACP_PLATFORM_SHARED_CONFIG_CLUSTER_AUTO_VARS_FILE}"
+done
+terraform fmt "${ACP_PLATFORM_SHARED_CONFIG_CLUSTER_AUTO_VARS_FILE}"
+
+echo "Provisioning the core platform"
+"${ACP_PLATFORM_CORE_DIR}/deploy.sh"
+
+echo "Provisioning the use case resources"
+# shellcheck disable=SC2154 # variable defined in common.sh
+for terraservice in "${federated_learning_terraservices[@]}"; do
+  provision_terraservice "${terraservice}"
+done
+
+end_timestamp_federated_learning=$(date +%s)
+total_runtime_value_federated_learning=$((end_timestamp_federated_learning - start_timestamp_federated_learning))
+echo "Total runtime (Federated learning use case provisioning and configuration): $(date -d@${total_runtime_value_federated_learning} -u +%H:%M:%S)"

platforms/gke/base/use-cases/federated-learning/teardown.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+#
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# shellcheck disable=SC1091
+source "${ACP_PLATFORM_BASE_DIR}/use-cases/federated-learning/common.sh"
+
+start_timestamp_federated_learning=$(date +%s)
+
+# Iterate over the terraservices array so we destroy them in reverse order, keeping the
+# initialize terraservice last.
+# shellcheck disable=SC2154 # variable defined in common.sh
+for ((i = ${#federated_learning_terraservices[@]} - 1; i >= 0; i--)); do
+  terraservice=${federated_learning_terraservices[i]}
+  destroy_terraservice "${terraservice}"
+done
+
+echo "Destroying the core platform"
+"${ACP_PLATFORM_CORE_DIR}/teardown.sh"
+
+for configuration_variable in "${TERRAFORM_CLUSTER_CONFIGURATION[@]}"; do
+  configuration_variable_name="$(echo "${configuration_variable}" | awk ' { print $1 }'))"
+  sed -i "/${configuration_variable_name}/d" "${ACP_PLATFORM_SHARED_CONFIG_CLUSTER_AUTO_VARS_FILE}"
+done
+terraform fmt "${ACP_PLATFORM_SHARED_CONFIG_CLUSTER_AUTO_VARS_FILE}"
+
+end_timestamp_federated_learning=$(date +%s)
+total_runtime_value_federated_learning=$((end_timestamp_federated_learning - start_timestamp_federated_learning))
+echo "Total runtime (Federated learning use case provisioning and configuration): $(date -d@${total_runtime_value_federated_learning} -u +%H:%M:%S)"

platforms/gke/base/use-cases/federated-learning/terraform/_shared_config/uc_federated_learning.auto.tfvars

@@ -0,0 +1,13 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

platforms/gke/base/use-cases/federated-learning/terraform/_shared_config/uc_federated_learning_variables.tf

@@ -0,0 +1,13 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

platforms/gke/base/use-cases/federated-learning/terraform/container_image_repository/_cluster.auto.tfvars

@@ -0,0 +1 @@
+../../../../_shared_config/cluster.auto.tfvars

platforms/gke/base/use-cases/federated-learning/terraform/container_image_repository/_cluster_variables.tf

@@ -0,0 +1 @@
+../../../../_shared_config/cluster_variables.tf

platforms/gke/base/use-cases/federated-learning/terraform/container_image_repository/_platform.auto.tfvars

@@ -0,0 +1 @@
+../../../../_shared_config/platform.auto.tfvars

platforms/gke/base/use-cases/federated-learning/terraform/container_image_repository/_platform_variables.tf

@@ -0,0 +1 @@
+../../../../_shared_config/platform_variables.tf

platforms/gke/base/use-cases/federated-learning/terraform/container_image_repository/_uc_federated_learning.auto.tfvars

@@ -0,0 +1 @@
+../_shared_config/uc_federated_learning.auto.tfvars

platforms/gke/base/use-cases/federated-learning/terraform/container_image_repository/_uc_federated_learning_variables.tf

@@ -0,0 +1 @@
+../_shared_config/uc_federated_learning_variables.tf

platforms/gke/base/use-cases/federated-learning/terraform/container_image_repository/main.tf

@@ -0,0 +1,30 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+resource "google_artifact_registry_repository" "container_image_repository" {
+  location      = var.cluster_region
+  repository_id = "federated-learning-container-image-repository"
+  description   = "Federated Learning container image repository"
+  format        = "DOCKER"
+  project       = google_project_service.artifactregistry_googleapis_com.project
+
+  cleanup_policies {
+    action = "DELETE"
+    id     = "Delete untagged images"
+
+    condition {
+      tag_state = "UNTAGGED"
+    }
+  }
+}

platforms/gke/base/use-cases/federated-learning/terraform/container_image_repository/project.tf

@@ -0,0 +1,24 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+data "google_project" "default" {
+  project_id = var.cluster_project_id
+}
+
+resource "google_project_service" "artifactregistry_googleapis_com" {
+  disable_dependent_services = false
+  disable_on_destroy         = false
+  project                    = data.google_project.default.project_id
+  service                    = "artifactregistry.googleapis.com"
+}

platforms/gke/base/use-cases/federated-learning/terraform/container_image_repository/versions.tf

@@ -0,0 +1,28 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+terraform {
+  required_version = ">= 1.5.7"
+
+  required_providers {
+    google = {
+      source  = "hashicorp/google"
+      version = "6.12.0"
+    }
+  }
+
+  provider_meta "google" {
+    module_name = "cloud-solutions/acp_fl_container_image_repository_deploy-v1"
+  }
+}