99-clickhouseinstallation-max.yaml

# We may need to label nodes with clickhouse=allow label for this example to run
# See ./label_nodes.sh for this purpose

apiVersion: "clickhouse.altinity.com/v1"
kind: "ClickHouseInstallation"
metadata:
  name: "clickhouse-installation-max"
  labels:
    label1: label1_value
    label2: label2_value
  annotations:
    annotation1: annotation1_value
    annotation2: annotation2_value

spec:
  # Allows to define custom taskID for CHI update and watch status of this update execution.
  # Displayed in all .status.taskID* fields.
  # By default (if not filled) every update of CHI manifest will generate random taskID
  taskID: "qweqwe"

  # Allows to stop all ClickHouse clusters defined in a CHI.
  # Works as the following:
  #  - When `stop` is `1` operator sets `Replicas: 0` in each StatefulSet. Thie leads to having all `Pods` and `Service` deleted. All PVCs are kept intact.
  #  - When `stop` is `0` operator sets `Replicas: 1` and `Pod`s and `Service`s will created again and all retained PVCs will be attached to `Pod`s.
  stop: "no"

  # In case 'RollingUpdate' specified, the operator will always restart ClickHouse pods during reconcile.
  # This options is used in rare cases when force restart is required and is typically removed after the use in order to avoid unneeded restarts.
  restart: "RollingUpdate"

  # Allows to troubleshoot Pods during CrashLoopBack state.
  # This may happen when wrong configuration applied, in this case `clickhouse-server` wouldn't start.
  # Command within ClickHouse container is modified with `sleep` in order to avoid quick restarts
  # and give time to troubleshoot via CLI.
  # Liveness and Readiness probes are disabled as well.
  troubleshoot: "no"

  # Custom domain pattern which will be used for DNS names of `Service` or `Pod`.
  # Typical use scenario - custom cluster domain in Kubernetes cluster
  namespaceDomainPattern:  "%s.svc.my.test"

  # Optional, applicable inside ClickHouseInstallationTemplate only.
  # Defines current ClickHouseInstallationTemplate application policy to target ClickHouseInstallation(s)."
  templating:
    # When defined as `auto` inside ClickhouseInstallationTemplate, this ClickhouseInstallationTemplate
    # will be auto-added into ClickHouseInstallation, selectable by `chiSelector`.
    # Default value is `manual`, meaning ClickHouseInstallation should request this ClickhouseInstallationTemplate explicitly.
    # Possible values:
    #  - "auto"
    #  - "manual"
    policy: "manual"
    # Optional, defines selector for ClickHouseInstallation(s) to be templated with ClickhouseInstallationTemplate
    chiSelector:
      name1: value1
      name2: value2
    # TODO introduce one-time actions
    # chiTaskID: "qweqwe"
    # autoPurge: "yes"

  # Optional, allows tuning reconciling cycle for ClickhouseInstallation from clickhouse-operator side
  reconciling:
    # DISCUSSED TO BE DEPRECATED
    # Syntax sugar
    # Overrides all three 'reconcile.host.wait.{exclude, queries, include}' values from the operator's config
    # Possible values:
    #  - wait - should wait to exclude host, complete queries and include host back into the cluster
    #  - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster
    policy: "nowait"

    # Timeout in seconds for `clickhouse-operator` to wait for modified `ConfigMap` to propagate into the `Pod`
    # More details: https://kubernetes.io/docs/concepts/configuration/configmap/#mounted-configmaps-are-updated-automatically
    configMapPropagationTimeout: 90

    # Optional, defines behavior for cleanup Kubernetes resources during reconcile cycle
    cleanup:
      # Describes what clickhouse-operator should do with found Kubernetes resources which should be managed by clickhouse-operator,
      # but do not have `ownerReference` to any currently managed `ClickHouseInstallation` resource.
      # Default behavior is `Delete`"
      unknownObjects:
        # Behavior policy for unknown PVC, `Delete` by default
        statefulSet: Delete
        # Behavior policy for unknown PVC, `Delete` by default
        pvc: Delete
        # Behavior policy for unknown ConfigMap, `Delete` by default
        configMap: Delete
        # Behavior policy for unknown Service, `Delete` by default
        service: Delete
      # Describes what clickhouse-operator should do with Kubernetes resources which are failed during reconcile.
      # Default behavior is `Retain`"
      reconcileFailedObjects:
        # Behavior policy for failed StatefulSet, `Retain` by default
        statefulSet: Retain
        # Behavior policy for failed PVC, `Retain` by default
        pvc: Retain
        # Behavior policy for failed ConfigMap, `Retain` by default
        configMap: Retain
        # Behavior policy for failed Service, `Retain` by default
        service: Retain

  # List of templates used by a CHI
  useTemplates:
    - name: template1
      namespace: ns1
      useType: merge
    - name: template2
      # No namespace specified - use CHI namespace

  defaults:
    replicasUseFQDN: "no"
    distributedDDL:
      profile: default
    storageManagement:
      # Specify PVC provisioner.
      # 1. StatefulSet. PVC would be provisioned by the StatefulSet
      # 2. Operator. PVC would be provisioned by the operator
      provisioner: StatefulSet

      # Specify PVC reclaim policy.
      # 1. Retain. Keep PVC from being deleted
      #    Retaining PVC will also keep backing PV from deletion. This is useful in case we need to keep data intact.
      # 2. Delete
      reclaimPolicy: Retain

    templates:
      hostTemplate: host-template-custom-ports
      podTemplate: clickhouse-v23.8
      dataVolumeClaimTemplate: default-volume-claim
      logVolumeClaimTemplate: default-volume-claim
      serviceTemplate: chi-service-template
      clusterServiceTemplate: cluster-service-template
      shardServiceTemplate: shard-service-template
      replicaServiceTemplate: replica-service-template

  configuration:
    zookeeper:
      nodes:
        - host: zookeeper-0.zookeepers.zoo3ns.svc.cluster.local
          port: 2181
        - host: zookeeper-1.zookeepers.zoo3ns.svc.cluster.local
          port: 2181
        - host: zookeeper-2.zookeepers.zoo3ns.svc.cluster.local
          port: 2181
      session_timeout_ms: 30000
      operation_timeout_ms: 10000
      root: "/path/to/zookeeper/root/node"
      identity: "user:password"
    users:
      readonly/profile: readonly
      #     <users>
      #        <readonly>
      #          <profile>readonly</profile>
      #        </readonly>
      #     </users>
      test/networks/ip:
        - "127.0.0.1"
        - "::/0"
      #     <users>
      #        <test>
      #          <networks>
      #            <ip>127.0.0.1</ip>
      #            <ip>::/0</ip>
      #          </networks>
      #        </test>
      #     </users>
      test/profile: default
      test/quotas: default
    profiles:
      readonly/readonly: "1"
      #      <profiles>
      #        <readonly>
      #          <readonly>1</readonly>
      #        </readonly>
      #      </profiles>
      default/max_memory_usage: "1000000000"
    quotas:
      default/interval/duration: "3600"
      #     <quotas>
      #       <default>
      #         <interval>
      #           <duration>3600</duration>
      #         </interval>
      #       </default>
      #     </quotas>
    settings:
      compression/case/method: zstd
      #      <compression>
      #        <case>
      #          <method>zstd</method>
      #        </case>
      #      </compression>
      disable_internal_dns_cache: 1
      #      <disable_internal_dns_cache>1</disable_internal_dns_cache>
    files:
      dict1.xml: |
        <yandex>
            <!-- ref to file /etc/clickhouse-data/config.d/source1.csv -->
        </yandex>
      source1.csv: |
        a1,b1,c1,d1
        a2,b2,c2,d2

    clusters:

      - name: all-counts
        templates:
          podTemplate: clickhouse-v23.8
          dataVolumeClaimTemplate: default-volume-claim
          logVolumeClaimTemplate: default-volume-claim
        schemaPolicy:
          replica: All
          shard: All
        layout:
          shardsCount: 3
          replicasCount: 2

      - name: shards-only
        templates:
          podTemplate: clickhouse-v23.8
          dataVolumeClaimTemplate: default-volume-claim
          logVolumeClaimTemplate: default-volume-claim
        layout:
          shardsCount: 3
          # replicasCount not specified, assumed = 1, by default

      - name: replicas-only
        templates:
          podTemplate: clickhouse-v23.8
          dataVolumeClaimTemplate: default-volume-claim
          logVolumeClaimTemplate: default-volume-claim
        layout:
          # shardsCount not specified, assumed = 1, by default
          replicasCount: 3

      - name: customized
        templates:
          podTemplate: clickhouse-v23.8
          dataVolumeClaimTemplate: default-volume-claim
          logVolumeClaimTemplate: default-volume-claim
        schemaPolicy:
          replica: None
          shard: None
        layout:
          shards:
            - name: shard0
              replicasCount: 3
              weight: 1
              internalReplication: Disabled
              templates:
                podTemplate: clickhouse-v23.8
                dataVolumeClaimTemplate: default-volume-claim
                logVolumeClaimTemplate: default-volume-claim

            - name: shard1
              templates:
                podTemplate: clickhouse-v23.8
                dataVolumeClaimTemplate: default-volume-claim
                logVolumeClaimTemplate: default-volume-claim
              replicas:
                - name: replica0
                - name: replica1
                - name: replica2

            - name: shard2
              replicasCount: 3
              templates:
                podTemplate: clickhouse-v23.8
                dataVolumeClaimTemplate: default-volume-claim
                logVolumeClaimTemplate: default-volume-claim
                replicaServiceTemplate: replica-service-template
              replicas:
                - name: replica0
                  tcpPort: 9000
                  httpPort: 8123
                  interserverHTTPPort: 9009
                  templates:
                    podTemplate: clickhouse-v23.8
                    dataVolumeClaimTemplate: default-volume-claim
                    logVolumeClaimTemplate: default-volume-claim
                    replicaServiceTemplate: replica-service-template

      - name: with-secret
        # Insecure communication.
        # Opens/Closes insecure ports
        insecure: "yes"
        # Secure communication.
        # Opens/Closes secure ports
        # Translates into <secure>1</secure> ClickHouse setting for remote replicas
        secure: "yes"
        # Shared secret value to secure cluster communications
        secret:
          # Auto-generate shared secret value to secure cluster communications
          auto: "True"
          # Cluster shared secret value in plain text
          value: "plaintext secret"
          # Cluster shared secret source
          valueFrom:
            secretKeyRef:
              name: "SecretName"
              key: "Key"
        layout:
          shardsCount: 2

  templates:
    hostTemplates:
      - name: host-template-custom-ports
        portDistribution:
          - type: "ClusterScopeIndex"
        spec:
          # Insecure communication.
          # Opens/Closes insecure ports
          insecure: "yes"
          # Secure communication.
          # Opens/Closes secure ports
          # Translates into <secure>1</secure> ClickHouse setting for remote replicas
          secure: "no"
          tcpPort: 7000
          httpPort: 7001
          interserverHTTPPort: 7002
          #settings:
          #files:
          #templates:

    serviceTemplates:
      - name: chi-service-template
        # generateName understands different sets of macroses,
        # depending on the level of the object, for which Service is being created:
        #
        # For CHI-level Service:
        # 1. {chi} - ClickHouseInstallation name
        # 2. {chiID} - short hashed ClickHouseInstallation name (BEWARE, this is an experimental feature)
        #
        # For Cluster-level Service:
        # 1. {chi} - ClickHouseInstallation name
        # 2. {chiID} - short hashed ClickHouseInstallation name (BEWARE, this is an experimental feature)
        # 3. {cluster} - cluster name
        # 4. {clusterID} - short hashed cluster name (BEWARE, this is an experimental feature)
        # 5. {clusterIndex} - 0-based index of the cluster in the CHI (BEWARE, this is an experimental feature)
        #
        # For Shard-level Service:
        # 1. {chi} - ClickHouseInstallation name
        # 2. {chiID} - short hashed ClickHouseInstallation name (BEWARE, this is an experimental feature)
        # 3. {cluster} - cluster name
        # 4. {clusterID} - short hashed cluster name (BEWARE, this is an experimental feature)
        # 5. {clusterIndex} - 0-based index of the cluster in the CHI (BEWARE, this is an experimental feature)
        # 6. {shard} - shard name
        # 7. {shardID} - short hashed shard name (BEWARE, this is an experimental feature)
        # 8. {shardIndex} - 0-based index of the shard in the cluster (BEWARE, this is an experimental feature)
        #
        # For Replica-level Service:
        # 1. {chi} - ClickHouseInstallation name
        # 2. {chiID} - short hashed ClickHouseInstallation name (BEWARE, this is an experimental feature)
        # 3. {cluster} - cluster name
        # 4. {clusterID} - short hashed cluster name (BEWARE, this is an experimental feature)
        # 5. {clusterIndex} - 0-based index of the cluster in the CHI (BEWARE, this is an experimental feature)
        # 6. {shard} - shard name
        # 7. {shardID} - short hashed shard name (BEWARE, this is an experimental feature)
        # 8. {shardIndex} - 0-based index of the shard in the cluster (BEWARE, this is an experimental feature)
        # 9. {replica} - replica name
        # 10. {replicaID} - short hashed replica name (BEWARE, this is an experimental feature)
        # 11. {replicaIndex} - 0-based index of the replica in the shard (BEWARE, this is an experimental feature)
        # 12. {chiScopeIndex} - 0-based index of the host in the chi (BEWARE, this is an experimental feature)
        # 13. {chiScopeCycleIndex} - 0-based index of the host's cycle in the chi-scope (BEWARE, this is an experimental feature)
        # 14. {chiScopeCycleOffset} - 0-based offset of the host in the chi-scope cycle (BEWARE, this is an experimental feature)
        # 15. {clusterScopeIndex} - 0-based index of the host in the cluster (BEWARE, this is an experimental feature)
        # 16. {clusterScopeCycleIndex} - 0-based index of the host's cycle in the cluster-scope (BEWARE, this is an experimental feature)
        # 17. {clusterScopeCycleOffset} - 0-based offset of the host in the cluster-scope cycle (BEWARE, this is an experimental feature)
        # 18. {shardScopeIndex} - 0-based index of the host in the shard (BEWARE, this is an experimental feature)
        # 19. {replicaScopeIndex} - 0-based index of the host in the replica (BEWARE, this is an experimental feature)
        # 20. {clusterScopeCycleHeadPointsToPreviousCycleTail} - 0-based cluster-scope index of previous cycle tail
        generateName: "service-{chi}"
        # type ObjectMeta struct from k8s.io/meta/v1
        metadata:
          labels:
            custom.label: "custom.value"
          annotations:
            # For more details on Internal Load Balancer check
            # https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer
            cloud.google.com/load-balancer-type: "Internal"
            service.beta.kubernetes.io/aws-load-balancer-internal: "true"
            service.beta.kubernetes.io/azure-load-balancer-internal: "true"
            service.beta.kubernetes.io/openstack-internal-load-balancer: "true"
            service.beta.kubernetes.io/cce-load-balancer-internal-vpc: "true"

            # NLB Load Balancer
            service.beta.kubernetes.io/aws-load-balancer-type: "nlb"

        # type ServiceSpec struct from k8s.io/core/v1
        spec:
          ports:
            - name: http
              port: 8123
            - name: tcp
              port: 9000
          type: LoadBalancer

      - name: replica-service-template
        # type ServiceSpec struct from k8s.io/core/v1
        spec:
          ports:
            - name: http
              port: 8123
            - name: tcp
              port: 9000
            - name: interserver
              port: 9009
          type: ClusterIP
          clusterIP: None

      - name: preserve-client-source-ip
        # For more details on Preserving Client Source IP check
        # https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip
        spec:
          selector:
            app: example
          ports:
            - name: http
              port: 8123
            - name: tcp
              port: 9000
            - name: interserver
              port: 9009
          externalTrafficPolicy: Local
          type: LoadBalancer

    volumeClaimTemplates:
      - name: default-volume-claim
        # Specify PVC provisioner.
        # 1. StatefulSet. PVC would be provisioned by the StatefulSet
        # 2. Operator. PVC would be provisioned by the operator
        provisioner: StatefulSet

        # Specify PVC reclaim policy.
        # 1. Retain. Keep PVC from being deleted
        #    Retaining PVC will also keep backing PV from deletion. This is useful in case we need to keep data intact.
        # 2. Delete
        reclaimPolicy: Retain

        # type ObjectMeta struct {} from k8s.io/meta/v1
        metadata:
          labels:
            a: "b"
        # type PersistentVolumeClaimSpec struct from k8s.io/core/v1
        spec:
          # 1. If storageClassName is not specified, default StorageClass
          # (must be specified by cluster administrator) would be used for provisioning
          # 2. If storageClassName is set to an empty string (‘’), no storage class will be used
          # dynamic provisioning is disabled for this PVC. Existing, “Available”, PVs
          # (that do not have a specified storageClassName) will be considered for binding to the PVC
          #storageClassName: gold
          accessModes:
            - ReadWriteOnce
          resources:
            requests:
              storage: 1Gi

      - name: volume-claim-retain-pvc
        # Specify PVC provisioner.
        # 1. StatefulSet. PVC would be provisioned by the StatefulSet
        # 2. Operator. PVC would be provisioned by the operator
        provisioner: StatefulSet

        # Specify PVC reclaim policy.
        # 1. Retain. Keep PVC from being deleted
        #    Retaining PVC will also keep backing PV from deletion. This is useful in case we need to keep data intact.
        # 2. Delete
        reclaimPolicy: Retain

        # type ObjectMeta struct {} from k8s.io/meta/v1
        metadata:
          labels:
            a: "b"
        # type PersistentVolumeClaimSpec struct from k8s.io/core/v1
        spec:
          # 1. If storageClassName is not specified, default StorageClass
          # (must be specified by cluster administrator) would be used for provisioning
          # 2. If storageClassName is set to an empty string (‘’), no storage class will be used
          # dynamic provisioning is disabled for this PVC. Existing, “Available”, PVs
          # (that do not have a specified storageClassName) will be considered for binding to the PVC
          #storageClassName: gold
          accessModes:
            - ReadWriteOnce
          resources:
            requests:
              storage: 1Gi

    podTemplates:
      # multiple pod templates makes possible to update version smoothly
      # pod template for ClickHouse v18.16.1
      - name: clickhouse-v23.8
        # We may need to label nodes with clickhouse=allow label for this example to run
        # See ./label_nodes.sh for this purpose
        zone:
          key: "clickhouse"
          values:
            - "allow"
        # Shortcut version for AWS installations
        #zone:
        #  values:
        #    - "us-east-1a"

        # Possible values for podDistribution are:
        # Unspecified - empty value
        # ClickHouseAntiAffinity - AntiAffinity by ClickHouse instances.
        #   Pod pushes away other ClickHouse pods, which allows one ClickHouse instance per topologyKey-specified unit
        #   CH - (push away) - CH - (push away) - CH
        # ShardAntiAffinity - AntiAffinity by shard name.
        #   Pod pushes away other pods of the same shard (replicas of this shard),
        #   which allows one replica of a shard instance per topologyKey-specified unit.
        #   Other shards are allowed - it does not push all shards away, but CH-instances of the same shard only.
        #   Used for data loss avoidance - keeps all copies of the shard on different topologyKey-specified units.
        #   shard1,replica1 - (push away) - shard1,replica2 - (push away) - shard1,replica3
        # ReplicaAntiAffinity - AntiAffinity by replica name.
        #   Pod pushes away other pods of the same replica (shards of this replica),
        #   which allows one shard of a replica per topologyKey-specified unit.
        #   Other replicas are allowed - it does not push all replicas away, but CH-instances of the same replica only.
        #   Used to evenly distribute load from "full cluster scan" queries.
        #   shard1,replica1 - (push away) - shard2,replica1 - (push away) - shard3,replica1
        # AnotherNamespaceAntiAffinity - AntiAffinity by "another" namespace.
        #   Pod pushes away pods from another namespace, which allows same-namespace pods per topologyKey-specified unit.
        #   ns1 - (push away) - ns2 - (push away) - ns3
        # AnotherClickHouseInstallationAntiAffinity - AntiAffinity by "another" ClickHouseInstallation name.
        #   Pod pushes away pods from another ClickHouseInstallation,
        #   which allows same-ClickHouseInstallation pods per topologyKey-specified unit.
        #   CHI1 - (push away) - CHI2 - (push away) - CHI3
        # AnotherClusterAntiAffinity - AntiAffinity by "another" cluster name.
        #   Pod pushes away pods from another Cluster,
        #   which allows same-cluster pods per topologyKey-specified unit.
        #   cluster1 - (push away) - cluster2 - (push away) - cluster3
        # MaxNumberPerNode - AntiAffinity by cycle index.
        #   Pod pushes away pods from the same cycle,
        #   which allows to specify maximum number of ClickHouse instances per topologyKey-specified unit.
        #   Used to setup circular replication.
        # NamespaceAffinity - Affinity by namespace.
        #   Pod attracts pods from the same namespace,
        #   which allows pods from same namespace per topologyKey-specified unit.
        #   ns1 + (attracts) + ns1
        # ClickHouseInstallationAffinity - Affinity by ClickHouseInstallation name.
        #   Pod attracts pods from the same ClickHouseInstallation,
        #   which allows pods from the same CHI per topologyKey-specified unit.
        #   CHI1 + (attracts) + CHI1
        # ClusterAffinity - Affinity by cluster name.
        #   Pod attracts pods from the same cluster,
        #   which allows pods from the same Cluster per topologyKey-specified unit.
        #   cluster1 + (attracts) + cluster1
        # ShardAffinity - Affinity by shard name.
        #   Pod attracts pods from the same shard,
        #   which allows pods from the same Shard per topologyKey-specified unit.
        #   shard1 + (attracts) + shard1
        # ReplicaAffinity - Affinity by replica name.
        #   Pod attracts pods from the same replica,
        #   which allows pods from the same Replica per topologyKey-specified unit.
        #   replica1 + (attracts) + replica1
        # PreviousTailAffinity - Affinity to overlap cycles. Used to make cycle pod distribution
        #   cycle head + (attracts to) + previous cycle tail
        podDistribution:
          - type: ShardAntiAffinity
          - type: MaxNumberPerNode
            number: 2
            # Apply podDistribution on per-host basis
            topologyKey: "kubernetes.io/hostname"
            # Apply podDistribution on per-zone basis
            #topologyKey: "kubernetes.io/zone"

        # type ObjectMeta struct {} from k8s.io/meta/v1
        metadata:
          labels:
            a: "b"
        # type PodSpec struct {} from k8s.io/core/v1
        spec:
          containers:
            - name: clickhouse
              image: clickhouse/clickhouse-server:23.8
              volumeMounts:
                - name: default-volume-claim
                  mountPath: /var/lib/clickhouse
              resources:
                requests:
                  memory: "64Mi"
                  cpu: "100m"
                limits:
                  memory: "64Mi"
                  cpu: "100m"
            - name: clickhouse-log
              image: clickhouse/clickhouse-server:23.8
              command:
                - "/bin/sh"
                - "-c"
                - "--"
              args:
                - "while true; do sleep 30; done;"

      # pod template for ClickHouse v23.8
      - name: clickhouse-v23.8
        # type ObjectMeta struct {} from k8s.io/meta/v1
        metadata:
          labels:
            a: "b"
        # type PodSpec struct {} from k8s.io/core/v1
        spec:
          containers:
            - name: clickhouse
              image: clickhouse/clickhouse-server:23.8
              volumeMounts:
                - name: default-volume-claim
                  mountPath: /var/lib/clickhouse
              resources:
                requests:
                  memory: "64Mi"
                  cpu: "100m"
                limits:
                  memory: "64Mi"
                  cpu: "100m"
            - name: clickhouse-log
              image: clickhouse/clickhouse-server:23.8
              command:
                - "/bin/sh"
                - "-c"
                - "--"
              args:
                - "while true; do sleep 30; done;"