diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000..f6cd152 Binary files /dev/null and b/.DS_Store differ diff --git a/.github/workflows/master.yaml b/.github/workflows/master.yaml deleted file mode 100644 index 12b6ec5..0000000 --- a/.github/workflows/master.yaml +++ /dev/null @@ -1,25 +0,0 @@ -name: Deploy app -on: - push: - branches: - - main -env: - ENCRYPTION_KEY: ${{ secrets.ENCRYPTION_KEY }} - DOCKERCONFIG_JSON: ${{ secrets.DOCKERCONFIG_JSON }} - NATS_STAGING_PASSWORD: ${{ secrets.NATS_STAGING_PASSWORD }} -jobs: - deploy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Download kubectl - run: curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" - - name: chmod kubectl - run: chmod +x ./kubectl - - name: Setting up kubeconfig - run: echo "${{ secrets.KUBE_CONFIG }}" > kube.config - - name: Setting up secrets - working-directory: kustomization/bases/secrets/ - run: ./generate.sh - - name: Deploy - run: ./kubectl --kubeconfig ./kube.config apply -k ./kustomization/overlays/staging diff --git a/.github/workflows/production.yaml b/.github/workflows/production.yaml deleted file mode 100644 index 41f93bf..0000000 --- a/.github/workflows/production.yaml +++ /dev/null @@ -1,22 +0,0 @@ -name: Deploy app -on: workflow_dispatch -env: - ENCRYPTION_KEY: ${{ secrets.ENCRYPTION_KEY }} - DOCKERCONFIG_JSON: ${{ secrets.DOCKERCONFIG_JSON }} - NATS_PRODUCTION_PASSWORD: ${{ secrets.NATS_PRODUCTION_PASSWORD }} -jobs: - deploy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Download kubectl - run: curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" - - name: chmod kubectl - run: chmod +x ./kubectl - - name: Setting up kubeconfig - run: echo "${{ secrets.KUBE_CONFIG }}" > kube.config - - name: Setting up secrets - working-directory: kustomization/bases/secrets/ - run: ./generate.sh - - name: Deploy - run: ./kubectl --kubeconfig ./kube.config apply -k ./kustomization/overlays/production diff --git a/.gitignore b/.gitignore index 563b026..98c5478 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ **/*.env -kustomization/bases/charts \ No newline at end of file +kustomization/bases/charts +*.tar.gz \ No newline at end of file diff --git a/Dockerfile.PostgresS3 b/Dockerfile.PostgresS3 new file mode 100644 index 0000000..7522eb5 --- /dev/null +++ b/Dockerfile.PostgresS3 @@ -0,0 +1,12 @@ +# Use the official PostgreSQL image as the base image +FROM postgres:latest + +# Install rclone for S3 interactions +RUN apt-get update && apt-get install -y curl \ + && curl -O https://downloads.rclone.org/rclone-current-linux-amd64.deb \ + && dpkg -i rclone-current-linux-amd64.deb \ + && rm rclone-current-linux-amd64.deb \ + && rm -rf /var/lib/apt/lists/* + +# Set a default command or an entrypoint as needed +CMD ["echo", "command needed"] diff --git a/README.md b/README.md index 3cfb742..daf48fc 100644 --- a/README.md +++ b/README.md @@ -6,3 +6,5 @@ 2. Add base64 encoded secrets 3. `export $(cat .env | xargs)` 4. `cat secrets.yaml | envsubst | kubectl apply -f -` + +Caddyfile location /etc/caddy/Caddyfile diff --git a/caddy/Caddyfile b/caddy/Caddyfile new file mode 100644 index 0000000..3e71ee9 --- /dev/null +++ b/caddy/Caddyfile @@ -0,0 +1,11 @@ +:80 { + respond "OK" +} + +grafana.nocodelytics.com { + reverse_proxy :5478 +} + +prometheus.nocodelytics.com { + reverse_proxy :9090 +} diff --git a/caddy/upload_caddy_conf.sh b/caddy/upload_caddy_conf.sh new file mode 100755 index 0000000..0e96bb0 --- /dev/null +++ b/caddy/upload_caddy_conf.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +PRIVATE_KEY="${PRIVATE_KEY}" +SSH_PORT="${SSH_PORT}" +REMOTE_USER="${REMOTE_USER}" +SERVER_IP="${SERVER_IP}" +TMP_DIR_UPLOAD_FILE="/home/$REMOTE_USER/" + +scp -i $PRIVATE_KEY -P $SSH_PORT ./Caddyfile $REMOTE_USER@$SERVER_IP:$TMP_DIR_UPLOAD_FILE/Caddyfile +ssh -i $PRIVATE_KEY -p $SSH_PORT $REMOTE_USER@$SERVER_IP "sudo cp /etc/caddy/Caddyfile /etc/caddy/Caddyfile.bak" +ssh -i $PRIVATE_KEY -p $SSH_PORT $REMOTE_USER@$SERVER_IP "sudo cp $TMP_DIR_UPLOAD_FILE/Caddyfile /etc/caddy/Caddyfile" +ssh -i $PRIVATE_KEY -p $SSH_PORT $REMOTE_USER@$SERVER_IP "sudo systemctl restart caddy" \ No newline at end of file diff --git a/cadvisor.yaml b/cadvisor.yaml new file mode 100644 index 0000000..3a5a01b --- /dev/null +++ b/cadvisor.yaml @@ -0,0 +1,44 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: cadvisor + namespace: kube-system +spec: + selector: + matchLabels: + name: cadvisor + template: + metadata: + labels: + name: cadvisor + spec: + automountServiceAccountToken: false + containers: + - name: cadvisor + image: gcr.io/cadvisor/cadvisor:v0.47.2 + volumeMounts: + - name: rootfs + mountPath: /rootfs + readOnly: true + - name: var-run + mountPath: /var/run + readOnly: true + - name: sys + mountPath: /sys + readOnly: true + - name: docker + mountPath: /var/lib/docker + readOnly: true + volumes: + - name: rootfs + hostPath: + path: / + - name: var-run + hostPath: + path: /var/run + - name: sys + hostPath: + path: /sys + - name: docker + hostPath: + path: /var/lib/docker diff --git a/clickhouse-operator-install-bundle.yaml b/clickhouse-operator-install-bundle.yaml deleted file mode 100644 index 925bed9..0000000 --- a/clickhouse-operator-install-bundle.yaml +++ /dev/null @@ -1,3510 +0,0 @@ -# Template Parameters: -# -# KIND=ClickHouseInstallation -# SINGULAR=clickhouseinstallation -# PLURAL=clickhouseinstallations -# SHORT=chi -# -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: clickhouseinstallations.clickhouse.altinity.com - labels: - clickhouse.altinity.com/chop: 0.20.0 -spec: - group: clickhouse.altinity.com - scope: Namespaced - names: - kind: ClickHouseInstallation - singular: clickhouseinstallation - plural: clickhouseinstallations - shortNames: - - chi - versions: - - name: v1 - served: true - storage: true - additionalPrinterColumns: - - name: version - type: string - description: Operator version - priority: 1 # show in wide view - jsonPath: .status.chop-version - - name: clusters - type: integer - description: Clusters count - priority: 0 # show in standard view - jsonPath: .status.clusters - - name: shards - type: integer - description: Shards count - priority: 1 # show in wide view - jsonPath: .status.shards - - name: hosts - type: integer - description: Hosts count - priority: 0 # show in standard view - jsonPath: .status.hosts - - name: taskID - type: string - description: TaskID - priority: 1 # show in wide view - jsonPath: .status.taskID - - name: status - type: string - description: CHI status - priority: 0 # show in standard view - jsonPath: .status.status - - name: updated - type: integer - description: Updated hosts count - priority: 1 # show in wide view - jsonPath: .status.updated - - name: added - type: integer - description: Added hosts count - priority: 1 # show in wide view - jsonPath: .status.added - - name: deleted - type: integer - description: Hosts deleted count - priority: 1 # show in wide view - jsonPath: .status.deleted - - name: delete - type: integer - description: Hosts to be deleted count - priority: 1 # show in wide view - jsonPath: .status.delete - - name: endpoint - type: string - description: Client access endpoint - priority: 1 # show in wide view - jsonPath: .status.endpoint - - name: age - type: date - description: Age of the resource - # Displayed in all priorities - jsonPath: .metadata.creationTimestamp - subresources: - status: {} - schema: - openAPIV3Schema: - description: "define a set of Kubernetes resources (StatefulSet, PVC, Service, ConfigMap) which describe behavior one or more ClickHouse clusters" - type: object - required: - - spec - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - status: - type: object - description: "Current ClickHouseInstallation manifest status, contains many fields like a normalized configuration, clickhouse-operator version, current action and all applied action list, current taskID and all applied taskIDs and other" - properties: - chop-version: - type: string - description: "ClickHouse operator version" - chop-commit: - type: string - description: "ClickHouse operator git commit SHA" - chop-date: - type: string - description: "ClickHouse operator build date" - chop-ip: - type: string - description: "IP address of the operator's pod which managed this CHI" - clusters: - type: integer - minimum: 0 - description: "Clusters count" - shards: - type: integer - minimum: 0 - description: "Shards count" - replicas: - type: integer - minimum: 0 - description: "Replicas count" - hosts: - type: integer - minimum: 0 - description: "Hosts count" - status: - type: string - description: "Status" - taskID: - type: string - description: "Current task id" - taskIDsStarted: - type: array - description: "Started task ids" - items: - type: string - taskIDsCompleted: - type: array - description: "Completed task ids" - items: - type: string - action: - type: string - description: "Action" - actions: - type: array - description: "Actions" - items: - type: string - error: - type: string - description: "Last error" - errors: - type: array - description: "Errors" - items: - type: string - updated: - type: integer - minimum: 0 - description: "Updated Hosts count" - added: - type: integer - minimum: 0 - description: "Added Hosts count" - deleted: - type: integer - minimum: 0 - description: "Deleted Hosts count" - delete: - type: integer - minimum: 0 - description: "About to delete Hosts count" - pods: - type: array - description: "Pods" - items: - type: string - pod-ips: - type: array - description: "Pod IPs" - items: - type: string - fqdns: - type: array - description: "Pods FQDNs" - items: - type: string - endpoint: - type: string - description: "Endpoint" - generation: - type: integer - minimum: 0 - description: "Generation" - normalized: - type: object - description: "Normalized CHI requested" - x-kubernetes-preserve-unknown-fields: true - normalizedCompleted: - type: object - description: "Normalized CHI completed" - x-kubernetes-preserve-unknown-fields: true - spec: - type: object - # x-kubernetes-preserve-unknown-fields: true - description: | - Specification of the desired behavior of one or more ClickHouse clusters - More info: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md - properties: - taskID: - type: string - description: | - Allows to define custom taskID for named update operation and watch status of this update execution in .status.taskIDs field. - By default every update of chi manifest will generate random taskID - stop: &TypeStringBool - type: string - description: | - Allow stop all ClickHouse clusters described in current chi. - Stop mechanism works as follows: - - When `stop` is `1` then setup `Replicas: 0` in each related to current `chi` StatefulSet resource, all `Pods` and `Service` resources will desctroy, but PVCs still live - - When `stop` is `0` then `Pods` will created again and will attach retained PVCs and `Service` also will created again - enum: - # List StringBoolXXX constants from model - - "" - - "0" - - "1" - - "False" - - "false" - - "True" - - "true" - - "No" - - "no" - - "Yes" - - "yes" - - "Off" - - "off" - - "On" - - "on" - - "Disable" - - "disable" - - "Enable" - - "enable" - - "Disabled" - - "disabled" - - "Enabled" - - "enabled" - restart: - type: string - description: "This is a 'soft restart' button. When set to 'RollingUpdate' operator will restart ClickHouse pods in a graceful way. Remove it after the use in order to avoid unneeded restarts" - enum: - - "" - - "RollingUpdate" - troubleshoot: - <<: *TypeStringBool - description: "allows troubleshoot Pods during CrashLoopBack state, when you apply wrong configuration, `clickhouse-server` wouldn't startup" - namespaceDomainPattern: - type: string - description: "custom domain suffix which will add to end of `Service` or `Pod` name, use it when you use custom cluster domain in your Kubernetes cluster" - templating: - type: object - # nullable: true - description: "optional, define policy for auto applying ClickHouseInstallationTemplate inside ClickHouseInstallation" - properties: - policy: - type: string - description: "when defined as `auto` inside ClickhouseInstallationTemplate, it will auto add into all ClickHouseInstallation, manual value is default" - enum: - - "auto" - - "manual" - reconciling: - type: object - description: "optional, allows tuning reconciling cycle for ClickhouseInstallation from clickhouse-operator side" - # nullable: true - properties: - policy: - type: string - description: DEPRECATED - configMapPropagationTimeout: - type: integer - description: | - timeout in seconds when `clickhouse-operator` will wait when applied `ConfigMap` during reconcile `ClickhouseInstallation` pods will updated from cache - see details: https://kubernetes.io/docs/concepts/configuration/configmap/#mounted-configmaps-are-updated-automatically - minimum: 0 - maximum: 3600 - cleanup: - type: object - description: "optional, define behavior for cleanup Kubernetes resources during reconcile cycle" - # nullable: true - properties: - unknownObjects: - type: object - description: "what clickhouse-operator shall do when found Kubernetes resources which should be managed with clickhouse-operator, but not have `ownerReference` to any currently managed `ClickHouseInstallation` resource, default behavior is `Delete`" - # nullable: true - properties: - statefulSet: &TypeObjectsCleanup - type: string - description: "behavior policy for unknown StatefulSet, Delete by default" - enum: - # List ObjectsCleanupXXX constants from model - - "Retain" - - "Delete" - pvc: - type: string - <<: *TypeObjectsCleanup - description: "behavior policy for unknown PVC, Delete by default" - configMap: - <<: *TypeObjectsCleanup - description: "behavior policy for unknown ConfigMap, Delete by default" - service: - <<: *TypeObjectsCleanup - description: "behavior policy for unknown Service, Delete by default" - reconcileFailedObjects: - type: object - description: "what clickhouse-operator shall do when reconciling Kubernetes resources are failed, default behavior is `Retain`" - # nullable: true - properties: - statefulSet: - <<: *TypeObjectsCleanup - description: "behavior policy for failed StatefulSet reconciling, Retain by default" - pvc: - <<: *TypeObjectsCleanup - description: "behavior policy for failed PVC reconciling, Retain by default" - configMap: - <<: *TypeObjectsCleanup - description: "behavior policy for failed ConfigMap reconciling, Retain by default" - service: - <<: *TypeObjectsCleanup - description: "behavior policy for failed Service reconciling, Retain by default" - defaults: - type: object - description: | - define default behavior for whole ClickHouseInstallation, some behavior can be re-define on cluster, shard and replica level - More info: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specdefaults - # nullable: true - properties: - replicasUseFQDN: - <<: *TypeStringBool - description: | - define should replicas be specified by FQDN in ``. - In case of "no" will use short hostname and clickhouse-server will use kubernetes default suffixes for DNS lookup - "yes" by default - distributedDDL: - type: object - description: | - allows change `` settings - More info: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#server-settings-distributed_ddl - # nullable: true - properties: - profile: - type: string - description: "Settings from this profile will be used to execute DDL queries" - storageManagement: - type: object - description: default storage management options - properties: - provisioner: &TypePVCProvisioner - type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" - enum: - - "" - - "StatefulSet" - - "Operator" - reclaimPolicy: &TypePVCReclaimPolicy - type: string - description: | - defines behavior of `PVC` deletion. - `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet - enum: - - "" - - "Retain" - - "Delete" - templates: &TypeTemplateNames - type: object - description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" - # nullable: true - properties: - hostTemplate: - type: string - description: "optional, template name from chi.spec.templates.hostTemplates, which will apply to configure every `clickhouse-server` instance during render ConfigMap resources which will mount into `Pod`" - podTemplate: - type: string - description: "optional, template name from chi.spec.templates.podTemplates, allows customization each `Pod` resource during render and reconcile each StatefulSet.spec resource described in `chi.spec.configuration.clusters`" - dataVolumeClaimTemplate: - type: string - description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" - logVolumeClaimTemplate: - type: string - description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse log directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" - serviceTemplate: - type: string - description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for one `Service` resource which will created by `clickhouse-operator` which cover all clusters in whole `chi` resource" - clusterServiceTemplate: - type: string - description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each clickhouse cluster described in `chi.spec.configuration.clusters`" - shardServiceTemplate: - type: string - description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each shard inside clickhouse cluster described in `chi.spec.configuration.clusters`" - replicaServiceTemplate: - type: string - description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each replica inside each shard inside each clickhouse cluster described in `chi.spec.configuration.clusters`" - volumeClaimTemplate: - type: string - description: "DEPRECATED! VolumeClaimTemplate is deprecated in favor of DataVolumeClaimTemplate and LogVolumeClaimTemplate" - configuration: - type: object - description: "allows configure multiple aspects and behavior for `clickhouse-server` instance and also allows describe multiple `clickhouse-server` clusters inside one `chi` resource" - # nullable: true - properties: - zookeeper: &TypeZookeeperConfig - type: object - description: | - allows configure .. section in each `Pod` during generate `ConfigMap` which will mounted in `/etc/clickhouse-server/config.d/` - `clickhouse-operator` itself doesn't manage Zookeeper, please install Zookeeper separatelly look examples on https://github.com/Altinity/clickhouse-operator/tree/master/deploy/zookeeper/ - currently, zookeeper (or clickhouse-keeper replacement) used for *ReplicatedMergeTree table engines and for `distributed_ddl` - More details: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#server-settings_zookeeper - # nullable: true - properties: - nodes: - type: array - description: "describe every available zookeeper cluster node for interaction" - # nullable: true - items: - type: object - #required: - # - host - properties: - host: - type: string - description: "dns name or ip address for Zookeeper node" - port: - type: integer - description: "TCP port which used to connect to Zookeeper node" - minimum: 0 - maximum: 65535 - session_timeout_ms: - type: integer - description: "session timeout during connect to Zookeeper" - operation_timeout_ms: - type: integer - description: "one operation timeout during Zookeeper transactions" - root: - type: string - description: "optional root znode path inside zookeeper to store ClickHouse related data (replication queue or distributed DDL)" - identity: - type: string - description: "optional access credentials string with `user:password` format used when use digest authorization in Zookeeper" - users: - type: object - description: | - allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` - you can configure password hashed, authorization restrictions, database level security row filters etc. - More details: https://clickhouse.tech/docs/en/operations/settings/settings-users/ - Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationusers - # nullable: true - x-kubernetes-preserve-unknown-fields: true - profiles: - type: object - description: | - allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` - you can configure any aspect of settings profile - More details: https://clickhouse.tech/docs/en/operations/settings/settings-profiles/ - Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationprofiles - # nullable: true - x-kubernetes-preserve-unknown-fields: true - quotas: - type: object - description: | - allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` - you can configure any aspect of resource quotas - More details: https://clickhouse.tech/docs/en/operations/quotas/ - Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationquotas - # nullable: true - x-kubernetes-preserve-unknown-fields: true - settings: &TypeSettings - type: object - description: | - allows configure `clickhouse-server` settings inside ... tag in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` - More details: https://clickhouse.tech/docs/en/operations/settings/settings/ - Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationsettings - # nullable: true - x-kubernetes-preserve-unknown-fields: true - files: &TypeFiles - type: object - description: | - allows define content of any setting file inside each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` - every key in this object is the file name - every value in this object is the file content - you can use `!!binary |` and base64 for binary files, see details here https://yaml.org/type/binary.html - each key could contains prefix like USERS, COMMON, HOST or config.d, users.d, cond.d, wrong prefixes will ignored, subfolders also will ignored - More details: https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-05-files-nested.yaml - # nullable: true - x-kubernetes-preserve-unknown-fields: true - clusters: - type: array - description: | - describes ClickHouse clusters layout and allows change settings on cluster-level, shard-level and replica-level - every cluster is a set of StatefulSet, one StatefulSet contains only one Pod with `clickhouse-server` - all Pods will rendered in part of ClickHouse configs, mounted from ConfigMap as `/etc/clickhouse-server/config.d/chop-generated-remote_servers.xml` - Clusters will use for Distributed table engine, more details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ - If `cluster` contains zookeeper settings (could be inherited from top `chi` level), when you can create *ReplicatedMergeTree tables - # nullable: true - items: - type: object - #required: - # - name - properties: - name: - type: string - description: "cluster name, used to identify set of ClickHouse servers and wide used during generate names of related Kubernetes resources" - minLength: 1 - # See namePartClusterMaxLen const - maxLength: 15 - pattern: "^[a-zA-Z0-9-]{0,15}$" - zookeeper: - <<: *TypeZookeeperConfig - description: | - optional, allows configure .. section in each `Pod` only in current ClickHouse cluster, during generate `ConfigMap` which will mounted in `/etc/clickhouse-server/config.d/` - override top-level `chi.spec.configuration.zookeeper` settings - settings: - <<: *TypeSettings - description: | - optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` only in one cluster during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` - override top-level `chi.spec.configuration.settings` - More details: https://clickhouse.tech/docs/en/operations/settings/settings/ - files: - <<: *TypeFiles - description: | - optional, allows define content of any setting file inside each `Pod` on current cluster during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` - override top-level `chi.spec.configuration.files` - templates: - <<: *TypeTemplateNames - description: | - optional, configuration of the templates names which will use for generate Kubernetes resources according to selected cluster - override top-level `chi.spec.configuration.templates` - schemaPolicy: - type: object - description: | - describes how schema is propagated within replicas and shards - properties: - replica: - type: string - description: "how schema is propagated within a replica" - enum: - # List SchemaPolicyReplicaXXX constants from model - - "None" - - "All" - shard: - type: string - description: "how schema is propagated between shards" - enum: - # List SchemaPolicyShardXXX constants from model - - "None" - - "All" - - "DistributedTablesOnly" - secret: - type: object - description: "optional, shared secret value to secure cluster communications" - properties: - auto: - <<: *TypeStringBool - description: "Auto-generate shared secret value to secure cluster communications" - value: - description: "Cluster shared secret value in plain text" - type: string - valueFrom: - description: "Cluster shared secret source" - type: object - properties: - secretKeyRef: - description: | - Selects a key of a secret in the clickhouse installation namespace. - Should not be used if value is not empty. - type: object - properties: - name: - description: | - Name of the referent. More info: - https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - key: - description: The key of the secret to select from. Must be a valid secret key. - type: string - optional: - description: Specify whether the Secret or its key must be defined - type: boolean - required: - - name - - key - layout: - type: object - description: | - describe current cluster layout, how much shards in cluster, how much replica in shard - allows override settings on each shard and replica separatelly - # nullable: true - properties: - type: - type: string - description: "DEPRECATED - to be removed soon" - shardsCount: - type: integer - description: "how much shards for current ClickHouse cluster will run in Kubernetes, each shard contains shared-nothing part of data and contains set of replicas, cluster contains 1 shard by default" - replicasCount: - type: integer - description: "how much replicas in each shards for current ClickHouse cluster will run in Kubernetes, each replica is a separate `StatefulSet` which contains only one `Pod` with `clickhouse-server` instance, every shard contains 1 replica by default" - shards: - type: array - description: "optional, allows override top-level `chi.spec.configuration`, cluster-level `chi.spec.configuration.clusters` settings for each shard separately, use it only if you fully understand what you do" - # nullable: true - items: - type: object - properties: - name: - type: string - description: "optional, by default shard name is generated, but you can override it and setup custom name" - minLength: 1 - # See namePartShardMaxLen const - maxLength: 15 - pattern: "^[a-zA-Z0-9-]{0,15}$" - definitionType: - type: string - description: "DEPRECATED - to be removed soon" - weight: - type: integer - description: | - optional, 1 by default, allows setup shard setting which will use during insert into tables with `Distributed` engine, - will apply in inside ConfigMap which will mount in /etc/clickhouse-server/config.d/chop-generated-remote_servers.xml - More details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ - internalReplication: - <<: *TypeStringBool - description: | - optional, `true` by default when `chi.spec.configuration.clusters[].layout.ReplicaCount` > 1 and 0 otherwise - allows setup setting which will use during insert into tables with `Distributed` engine for insert only in one live replica and other replicas will download inserted data during replication, - will apply in inside ConfigMap which will mount in /etc/clickhouse-server/config.d/chop-generated-remote_servers.xml - More details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ - settings: - <<: *TypeSettings - description: | - optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` only in one shard during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` - override top-level `chi.spec.configuration.settings` and cluster-level `chi.spec.configuration.clusters.settings` - More details: https://clickhouse.tech/docs/en/operations/settings/settings/ - files: - <<: *TypeFiles - description: | - optional, allows define content of any setting file inside each `Pod` only in one shard during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` - override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files` - templates: - <<: *TypeTemplateNames - description: | - optional, configuration of the templates names which will use for generate Kubernetes resources according to selected shard - override top-level `chi.spec.configuration.templates` and cluster-level `chi.spec.configuration.clusters.templates` - replicasCount: - type: integer - description: | - optional, how much replicas in selected shard for selected ClickHouse cluster will run in Kubernetes, each replica is a separate `StatefulSet` which contains only one `Pod` with `clickhouse-server` instance, - shard contains 1 replica by default - override cluster-level `chi.spec.configuration.clusters.layout.replicasCount` - minimum: 1 - replicas: - type: array - description: | - optional, allows override behavior for selected replicas from cluster-level `chi.spec.configuration.clusters` and shard-level `chi.spec.configuration.clusters.layout.shards` - # nullable: true - items: - # Host - type: object - properties: - name: - type: string - description: "optional, by default replica name is generated, but you can override it and setup custom name" - minLength: 1 - # See namePartReplicaMaxLen const - maxLength: 15 - pattern: "^[a-zA-Z0-9-]{0,15}$" - secure: - type: boolean - description: | - optional, setup `secure` inside `clickhouse-server` settings for each Pod where current template will apply - if specified - tcpPort: - type: integer - description: | - optional, setup `Pod.spec.containers.ports` with name `tcp` for selected replica, override `chi.spec.templates.hostTemplates.spec.tcpPort` - allows connect to `clickhouse-server` via TCP Native protocol via kubernetes `Service` - minimum: 1 - maximum: 65535 - httpPort: - type: integer - description: | - optional, setup `Pod.spec.containers.ports` with name `http` for selected replica, override `chi.spec.templates.hostTemplates.spec.httpPort` - allows connect to `clickhouse-server` via HTTP protocol via kubernetes `Service` - minimum: 1 - maximum: 65535 - interserverHTTPPort: - type: integer - description: | - optional, setup `Pod.spec.containers.ports` with name `interserver` for selected replica, override `chi.spec.templates.hostTemplates.spec.interserverHTTPPort` - allows connect between replicas inside same shard during fetch replicated data parts HTTP protocol - minimum: 1 - maximum: 65535 - settings: - <<: *TypeSettings - description: | - optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` - override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and shard-level `chi.spec.configuration.clusters.layout.shards.settings` - More details: https://clickhouse.tech/docs/en/operations/settings/settings/ - files: - <<: *TypeFiles - description: | - optional, allows define content of any setting file inside `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` - override top-level `chi.spec.configuration.files`, cluster-level `chi.spec.configuration.clusters.files` and shard-level `chi.spec.configuration.clusters.layout.shards.files` - templates: - <<: *TypeTemplateNames - description: | - optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica - override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates` and shard-level `chi.spec.configuration.clusters.layout.shards.templates` - replicas: - type: array - description: "optional, allows override top-level `chi.spec.configuration` and cluster-level `chi.spec.configuration.clusters` configuration for each replica and each shard relates to selected replica, use it only if you fully understand what you do" - # nullable: true - items: - type: object - properties: - name: - type: string - description: "optional, by default replica name is generated, but you can override it and setup custom name" - minLength: 1 - # See namePartShardMaxLen const - maxLength: 15 - pattern: "^[a-zA-Z0-9-]{0,15}$" - settings: - <<: *TypeSettings - description: | - optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` - override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and will ignore if shard-level `chi.spec.configuration.clusters.layout.shards` present - More details: https://clickhouse.tech/docs/en/operations/settings/settings/ - files: - <<: *TypeFiles - description: | - optional, allows define content of any setting file inside each `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` - override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files`, will ignore if `chi.spec.configuration.clusters.layout.shards` presents - templates: - <<: *TypeTemplateNames - description: | - optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica - override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates` - shardsCount: - type: integer - description: "optional, count of shards related to current replica, you can override each shard behavior on low-level `chi.spec.configuration.clusters.layout.replicas.shards`" - minimum: 1 - shards: - type: array - description: "optional, list of shards related to current replica, will ignore if `chi.spec.configuration.clusters.layout.shards` presents" - # nullable: true - items: - # Host - type: object - properties: - name: - type: string - description: "optional, by default shard name is generated, but you can override it and setup custom name" - minLength: 1 - # See namePartReplicaMaxLen const - maxLength: 15 - pattern: "^[a-zA-Z0-9-]{0,15}$" - secure: - type: boolean - description: | - optional, setup `secure` inside `clickhouse-server` settings for each Pod where current template will apply - if specified - tcpPort: - type: integer - description: | - optional, setup `Pod.spec.containers.ports` with name `tcp` for selected shard, override `chi.spec.templates.hostTemplates.spec.tcpPort` - allows connect to `clickhouse-server` via TCP Native protocol via kubernetes `Service` - minimum: 1 - maximum: 65535 - httpPort: - type: integer - description: | - optional, setup `Pod.spec.containers.ports` with name `http` for selected shard, override `chi.spec.templates.hostTemplates.spec.httpPort` - allows connect to `clickhouse-server` via HTTP protocol via kubernetes `Service` - minimum: 1 - maximum: 65535 - interserverHTTPPort: - type: integer - description: | - optional, setup `Pod.spec.containers.ports` with name `interserver` for selected shard, override `chi.spec.templates.hostTemplates.spec.interserverHTTPPort` - allows connect between replicas inside same shard during fetch replicated data parts HTTP protocol - minimum: 1 - maximum: 65535 - settings: - <<: *TypeSettings - description: | - optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one shard related to current replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` - override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and replica-level `chi.spec.configuration.clusters.layout.replicas.settings` - More details: https://clickhouse.tech/docs/en/operations/settings/settings/ - files: - <<: *TypeFiles - description: | - optional, allows define content of any setting file inside each `Pod` only in one shard related to current replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` - override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files`, will ignore if `chi.spec.configuration.clusters.layout.shards` presents - templates: - <<: *TypeTemplateNames - description: | - optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica - override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates` - templates: - type: object - description: "allows define templates which will use for render Kubernetes resources like StatefulSet, ConfigMap, Service, PVC, by default, clickhouse-operator have own templates, but you can override it" - # nullable: true - properties: - hostTemplates: - type: array - description: "hostTemplate will use during apply to generate `clickhose-server` config files" - # nullable: true - items: - type: object - #required: - # - name - properties: - name: - description: "template name, could use to link inside top-level `chi.spec.defaults.templates.hostTemplate`, cluster-level `chi.spec.configuration.clusters.templates.hostTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.hostTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.hostTemplate`" - type: string - portDistribution: - type: array - description: "define how will distribute numeric values of named ports in `Pod.spec.containers.ports` and clickhouse-server configs" - # nullable: true - items: - type: object - #required: - # - type - properties: - type: - type: string - description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" - enum: - # List PortDistributionXXX constants - - "" - - "Unspecified" - - "ClusterScopeIndex" - spec: - # Host - type: object - properties: - name: - type: string - description: "by default, hostname will generate, but this allows define custom name for each `clickhuse-server`" - minLength: 1 - # See namePartReplicaMaxLen const - maxLength: 15 - pattern: "^[a-zA-Z0-9-]{0,15}$" - secure: - type: boolean - description: | - optional, setup `secure` inside `clickhouse-server` settings for each Pod where current template will apply - if specified - tcpPort: - type: integer - description: | - optional, setup `tcp_port` inside `clickhouse-server` settings for each Pod where current template will apply - if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=tcp]` - More info: https://clickhouse.tech/docs/en/interfaces/tcp/ - minimum: 1 - maximum: 65535 - httpPort: - type: integer - description: | - optional, setup `http_port` inside `clickhouse-server` settings for each Pod where current template will apply - if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=http]` - More info: https://clickhouse.tech/docs/en/interfaces/http/ - minimum: 1 - maximum: 65535 - interserverHTTPPort: - type: integer - description: | - optional, setup `interserver_http_port` inside `clickhouse-server` settings for each Pod where current template will apply - if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=interserver]` - More info: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#interserver-http-port - minimum: 1 - maximum: 65535 - settings: - <<: *TypeSettings - description: | - optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` where this template will apply during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` - More details: https://clickhouse.tech/docs/en/operations/settings/settings/ - files: - <<: *TypeFiles - description: | - optional, allows define content of any setting file inside each `Pod` where this template will apply during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` - templates: - <<: *TypeTemplateNames - description: "be careful, this part of CRD allows override template inside template, don't use it if you don't understand what you do" - - podTemplates: - type: array - description: | - podTemplate will use during render `Pod` inside `StatefulSet.spec` and allows define rendered `Pod.spec`, pod scheduling distribution and pod zone - More information: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatespodtemplates - # nullable: true - items: - type: object - #required: - # - name - properties: - name: - type: string - description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" - generateName: - type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" - zone: - type: object - description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" - #required: - # - values - properties: - key: - type: string - description: "optional, if defined, allows select kubernetes nodes by label with `name` equal `key`" - values: - type: array - description: "optional, if defined, allows select kubernetes nodes by label with `value` in `values`" - # nullable: true - items: - type: string - distribution: - type: string - description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" - enum: - - "" - - "Unspecified" - - "OnePerHost" - podDistribution: - type: array - description: "define ClickHouse Pod distibution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" - # nullable: true - items: - type: object - #required: - # - type - properties: - type: - type: string - description: "you can define multiple affinity policy types" - enum: - # List PodDistributionXXX constants - - "" - - "Unspecified" - - "ClickHouseAntiAffinity" - - "ShardAntiAffinity" - - "ReplicaAntiAffinity" - - "AnotherNamespaceAntiAffinity" - - "AnotherClickHouseInstallationAntiAffinity" - - "AnotherClusterAntiAffinity" - - "MaxNumberPerNode" - - "NamespaceAffinity" - - "ClickHouseInstallationAffinity" - - "ClusterAffinity" - - "ShardAffinity" - - "ReplicaAffinity" - - "PreviousTailAffinity" - - "CircularReplication" - scope: - type: string - description: "scope for apply each podDistribution" - enum: - # list PodDistributionScopeXXX constants - - "" - - "Unspecified" - - "Shard" - - "Replica" - - "Cluster" - - "ClickHouseInstallation" - - "Namespace" - number: - type: integer - description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" - minimum: 0 - maximum: 65535 - topologyKey: - type: string - description: "use for inter-pod affinity look to `pod.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution.podAffinityTerm.topologyKey`, More info: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity" - spec: - # TODO specify PodSpec - type: object - description: "allows define whole Pod.spec inside StaefulSet.spec, look to https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates for details" - # nullable: true - x-kubernetes-preserve-unknown-fields: true - metadata: - type: object - description: | - allows pass standard object's metadata from template to Pod - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata - # nullable: true - x-kubernetes-preserve-unknown-fields: true - - volumeClaimTemplates: - type: array - description: "allows define template for rendering `PVC` kubernetes resource, which would use inside `Pod` for mount clickhouse `data`, clickhouse `logs` or something else" - # nullable: true - items: - type: object - #required: - # - name - # - spec - properties: - name: - type: string - description: | - template name, could use to link inside - top-level `chi.spec.defaults.templates.dataVolumeClaimTemplate` or `chi.spec.defaults.templates.logVolumeClaimTemplate`, - cluster-level `chi.spec.configuration.clusters.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.templates.logVolumeClaimTemplate`, - shard-level `chi.spec.configuration.clusters.layout.shards.temlates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.shards.temlates.logVolumeClaimTemplate` - replica-level `chi.spec.configuration.clusters.layout.replicas.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.replicas.templates.logVolumeClaimTemplate` - provisioner: *TypePVCProvisioner - reclaimPolicy: *TypePVCReclaimPolicy - metadata: - type: object - description: | - allows to pass standard object's metadata from template to PVC - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata - # nullable: true - x-kubernetes-preserve-unknown-fields: true - spec: - type: object - description: | - allows define all aspects of `PVC` resource - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims - # nullable: true - x-kubernetes-preserve-unknown-fields: true - serviceTemplates: - type: array - description: | - allows define template for rendering `Service` which would get endpoint from Pods which scoped chi-wide, cluster-wide, shard-wide, replica-wide level - # nullable: true - items: - type: object - #required: - # - name - # - spec - properties: - name: - type: string - description: | - template name, could use to link inside - chi-level `chi.spec.defaults.templates.serviceTemplate` - cluster-level `chi.spec.configuration.clusters.templates.clusterServiceTemplate` - shard-level `chi.spec.configuration.clusters.layout.shards.temlates.shardServiceTemplate` - replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` - generateName: - type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" - metadata: - # TODO specify ObjectMeta - type: object - description: | - allows pass standard object's metadata from template to Service - Could be use for define specificly for Cloud Provider metadata which impact to behavior of service - More info: https://kubernetes.io/docs/concepts/services-networking/service/ - # nullable: true - x-kubernetes-preserve-unknown-fields: true - spec: - # TODO specify ServiceSpec - type: object - description: | - describe behavior of generated Service - More info: https://kubernetes.io/docs/concepts/services-networking/service/ - # nullable: true - x-kubernetes-preserve-unknown-fields: true - useTemplates: - type: array - description: "list of `ClickHouseInstallationTemplate` (chit) resource names which will merge with current `Chi` manifest during render Kubernetes resources to create related ClickHouse clusters" - # nullable: true - items: - type: object - #required: - # - name - properties: - name: - type: string - description: "name of `ClickHouseInstallationTemplate` (chit) resource" - namespace: - type: string - description: "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clichouse-operator`" - useType: - type: string - description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit`" - enum: - # List useTypeXXX constants from model - - "" - - "merge" ---- -# Template Parameters: -# -# KIND=ClickHouseInstallationTemplate -# SINGULAR=clickhouseinstallationtemplate -# PLURAL=clickhouseinstallationtemplates -# SHORT=chit -# -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: clickhouseinstallationtemplates.clickhouse.altinity.com - labels: - clickhouse.altinity.com/chop: 0.20.0 -spec: - group: clickhouse.altinity.com - scope: Namespaced - names: - kind: ClickHouseInstallationTemplate - singular: clickhouseinstallationtemplate - plural: clickhouseinstallationtemplates - shortNames: - - chit - versions: - - name: v1 - served: true - storage: true - additionalPrinterColumns: - - name: version - type: string - description: Operator version - priority: 1 # show in wide view - jsonPath: .status.chop-version - - name: clusters - type: integer - description: Clusters count - priority: 0 # show in standard view - jsonPath: .status.clusters - - name: shards - type: integer - description: Shards count - priority: 1 # show in wide view - jsonPath: .status.shards - - name: hosts - type: integer - description: Hosts count - priority: 0 # show in standard view - jsonPath: .status.hosts - - name: taskID - type: string - description: TaskID - priority: 1 # show in wide view - jsonPath: .status.taskID - - name: status - type: string - description: CHI status - priority: 0 # show in standard view - jsonPath: .status.status - - name: updated - type: integer - description: Updated hosts count - priority: 1 # show in wide view - jsonPath: .status.updated - - name: added - type: integer - description: Added hosts count - priority: 1 # show in wide view - jsonPath: .status.added - - name: deleted - type: integer - description: Hosts deleted count - priority: 1 # show in wide view - jsonPath: .status.deleted - - name: delete - type: integer - description: Hosts to be deleted count - priority: 1 # show in wide view - jsonPath: .status.delete - - name: endpoint - type: string - description: Client access endpoint - priority: 1 # show in wide view - jsonPath: .status.endpoint - - name: age - type: date - description: Age of the resource - # Displayed in all priorities - jsonPath: .metadata.creationTimestamp - subresources: - status: {} - schema: - openAPIV3Schema: - description: "define a set of Kubernetes resources (StatefulSet, PVC, Service, ConfigMap) which describe behavior one or more ClickHouse clusters" - type: object - required: - - spec - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - status: - type: object - description: "Current ClickHouseInstallation manifest status, contains many fields like a normalized configuration, clickhouse-operator version, current action and all applied action list, current taskID and all applied taskIDs and other" - properties: - chop-version: - type: string - description: "ClickHouse operator version" - chop-commit: - type: string - description: "ClickHouse operator git commit SHA" - chop-date: - type: string - description: "ClickHouse operator build date" - chop-ip: - type: string - description: "IP address of the operator's pod which managed this CHI" - clusters: - type: integer - minimum: 0 - description: "Clusters count" - shards: - type: integer - minimum: 0 - description: "Shards count" - replicas: - type: integer - minimum: 0 - description: "Replicas count" - hosts: - type: integer - minimum: 0 - description: "Hosts count" - status: - type: string - description: "Status" - taskID: - type: string - description: "Current task id" - taskIDsStarted: - type: array - description: "Started task ids" - items: - type: string - taskIDsCompleted: - type: array - description: "Completed task ids" - items: - type: string - action: - type: string - description: "Action" - actions: - type: array - description: "Actions" - items: - type: string - error: - type: string - description: "Last error" - errors: - type: array - description: "Errors" - items: - type: string - updated: - type: integer - minimum: 0 - description: "Updated Hosts count" - added: - type: integer - minimum: 0 - description: "Added Hosts count" - deleted: - type: integer - minimum: 0 - description: "Deleted Hosts count" - delete: - type: integer - minimum: 0 - description: "About to delete Hosts count" - pods: - type: array - description: "Pods" - items: - type: string - pod-ips: - type: array - description: "Pod IPs" - items: - type: string - fqdns: - type: array - description: "Pods FQDNs" - items: - type: string - endpoint: - type: string - description: "Endpoint" - generation: - type: integer - minimum: 0 - description: "Generation" - normalized: - type: object - description: "Normalized CHI requested" - x-kubernetes-preserve-unknown-fields: true - normalizedCompleted: - type: object - description: "Normalized CHI completed" - x-kubernetes-preserve-unknown-fields: true - spec: - type: object - # x-kubernetes-preserve-unknown-fields: true - description: | - Specification of the desired behavior of one or more ClickHouse clusters - More info: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md - properties: - taskID: - type: string - description: | - Allows to define custom taskID for named update operation and watch status of this update execution in .status.taskIDs field. - By default every update of chi manifest will generate random taskID - stop: &TypeStringBool - type: string - description: | - Allow stop all ClickHouse clusters described in current chi. - Stop mechanism works as follows: - - When `stop` is `1` then setup `Replicas: 0` in each related to current `chi` StatefulSet resource, all `Pods` and `Service` resources will desctroy, but PVCs still live - - When `stop` is `0` then `Pods` will created again and will attach retained PVCs and `Service` also will created again - enum: - # List StringBoolXXX constants from model - - "" - - "0" - - "1" - - "False" - - "false" - - "True" - - "true" - - "No" - - "no" - - "Yes" - - "yes" - - "Off" - - "off" - - "On" - - "on" - - "Disable" - - "disable" - - "Enable" - - "enable" - - "Disabled" - - "disabled" - - "Enabled" - - "enabled" - restart: - type: string - description: "This is a 'soft restart' button. When set to 'RollingUpdate' operator will restart ClickHouse pods in a graceful way. Remove it after the use in order to avoid unneeded restarts" - enum: - - "" - - "RollingUpdate" - troubleshoot: - <<: *TypeStringBool - description: "allows troubleshoot Pods during CrashLoopBack state, when you apply wrong configuration, `clickhouse-server` wouldn't startup" - namespaceDomainPattern: - type: string - description: "custom domain suffix which will add to end of `Service` or `Pod` name, use it when you use custom cluster domain in your Kubernetes cluster" - templating: - type: object - # nullable: true - description: "optional, define policy for auto applying ClickHouseInstallationTemplate inside ClickHouseInstallation" - properties: - policy: - type: string - description: "when defined as `auto` inside ClickhouseInstallationTemplate, it will auto add into all ClickHouseInstallation, manual value is default" - enum: - - "auto" - - "manual" - reconciling: - type: object - description: "optional, allows tuning reconciling cycle for ClickhouseInstallation from clickhouse-operator side" - # nullable: true - properties: - policy: - type: string - description: DEPRECATED - configMapPropagationTimeout: - type: integer - description: | - timeout in seconds when `clickhouse-operator` will wait when applied `ConfigMap` during reconcile `ClickhouseInstallation` pods will updated from cache - see details: https://kubernetes.io/docs/concepts/configuration/configmap/#mounted-configmaps-are-updated-automatically - minimum: 0 - maximum: 3600 - cleanup: - type: object - description: "optional, define behavior for cleanup Kubernetes resources during reconcile cycle" - # nullable: true - properties: - unknownObjects: - type: object - description: "what clickhouse-operator shall do when found Kubernetes resources which should be managed with clickhouse-operator, but not have `ownerReference` to any currently managed `ClickHouseInstallation` resource, default behavior is `Delete`" - # nullable: true - properties: - statefulSet: &TypeObjectsCleanup - type: string - description: "behavior policy for unknown StatefulSet, Delete by default" - enum: - # List ObjectsCleanupXXX constants from model - - "Retain" - - "Delete" - pvc: - type: string - <<: *TypeObjectsCleanup - description: "behavior policy for unknown PVC, Delete by default" - configMap: - <<: *TypeObjectsCleanup - description: "behavior policy for unknown ConfigMap, Delete by default" - service: - <<: *TypeObjectsCleanup - description: "behavior policy for unknown Service, Delete by default" - reconcileFailedObjects: - type: object - description: "what clickhouse-operator shall do when reconciling Kubernetes resources are failed, default behavior is `Retain`" - # nullable: true - properties: - statefulSet: - <<: *TypeObjectsCleanup - description: "behavior policy for failed StatefulSet reconciling, Retain by default" - pvc: - <<: *TypeObjectsCleanup - description: "behavior policy for failed PVC reconciling, Retain by default" - configMap: - <<: *TypeObjectsCleanup - description: "behavior policy for failed ConfigMap reconciling, Retain by default" - service: - <<: *TypeObjectsCleanup - description: "behavior policy for failed Service reconciling, Retain by default" - defaults: - type: object - description: | - define default behavior for whole ClickHouseInstallation, some behavior can be re-define on cluster, shard and replica level - More info: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specdefaults - # nullable: true - properties: - replicasUseFQDN: - <<: *TypeStringBool - description: | - define should replicas be specified by FQDN in ``. - In case of "no" will use short hostname and clickhouse-server will use kubernetes default suffixes for DNS lookup - "yes" by default - distributedDDL: - type: object - description: | - allows change `` settings - More info: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#server-settings-distributed_ddl - # nullable: true - properties: - profile: - type: string - description: "Settings from this profile will be used to execute DDL queries" - storageManagement: - type: object - description: default storage management options - properties: - provisioner: &TypePVCProvisioner - type: string - description: "defines `PVC` provisioner - be it StatefulSet or the Operator" - enum: - - "" - - "StatefulSet" - - "Operator" - reclaimPolicy: &TypePVCReclaimPolicy - type: string - description: | - defines behavior of `PVC` deletion. - `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet - enum: - - "" - - "Retain" - - "Delete" - templates: &TypeTemplateNames - type: object - description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" - # nullable: true - properties: - hostTemplate: - type: string - description: "optional, template name from chi.spec.templates.hostTemplates, which will apply to configure every `clickhouse-server` instance during render ConfigMap resources which will mount into `Pod`" - podTemplate: - type: string - description: "optional, template name from chi.spec.templates.podTemplates, allows customization each `Pod` resource during render and reconcile each StatefulSet.spec resource described in `chi.spec.configuration.clusters`" - dataVolumeClaimTemplate: - type: string - description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" - logVolumeClaimTemplate: - type: string - description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse log directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" - serviceTemplate: - type: string - description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for one `Service` resource which will created by `clickhouse-operator` which cover all clusters in whole `chi` resource" - clusterServiceTemplate: - type: string - description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each clickhouse cluster described in `chi.spec.configuration.clusters`" - shardServiceTemplate: - type: string - description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each shard inside clickhouse cluster described in `chi.spec.configuration.clusters`" - replicaServiceTemplate: - type: string - description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each replica inside each shard inside each clickhouse cluster described in `chi.spec.configuration.clusters`" - volumeClaimTemplate: - type: string - description: "DEPRECATED! VolumeClaimTemplate is deprecated in favor of DataVolumeClaimTemplate and LogVolumeClaimTemplate" - configuration: - type: object - description: "allows configure multiple aspects and behavior for `clickhouse-server` instance and also allows describe multiple `clickhouse-server` clusters inside one `chi` resource" - # nullable: true - properties: - zookeeper: &TypeZookeeperConfig - type: object - description: | - allows configure .. section in each `Pod` during generate `ConfigMap` which will mounted in `/etc/clickhouse-server/config.d/` - `clickhouse-operator` itself doesn't manage Zookeeper, please install Zookeeper separatelly look examples on https://github.com/Altinity/clickhouse-operator/tree/master/deploy/zookeeper/ - currently, zookeeper (or clickhouse-keeper replacement) used for *ReplicatedMergeTree table engines and for `distributed_ddl` - More details: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#server-settings_zookeeper - # nullable: true - properties: - nodes: - type: array - description: "describe every available zookeeper cluster node for interaction" - # nullable: true - items: - type: object - #required: - # - host - properties: - host: - type: string - description: "dns name or ip address for Zookeeper node" - port: - type: integer - description: "TCP port which used to connect to Zookeeper node" - minimum: 0 - maximum: 65535 - session_timeout_ms: - type: integer - description: "session timeout during connect to Zookeeper" - operation_timeout_ms: - type: integer - description: "one operation timeout during Zookeeper transactions" - root: - type: string - description: "optional root znode path inside zookeeper to store ClickHouse related data (replication queue or distributed DDL)" - identity: - type: string - description: "optional access credentials string with `user:password` format used when use digest authorization in Zookeeper" - users: - type: object - description: | - allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` - you can configure password hashed, authorization restrictions, database level security row filters etc. - More details: https://clickhouse.tech/docs/en/operations/settings/settings-users/ - Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationusers - # nullable: true - x-kubernetes-preserve-unknown-fields: true - profiles: - type: object - description: | - allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` - you can configure any aspect of settings profile - More details: https://clickhouse.tech/docs/en/operations/settings/settings-profiles/ - Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationprofiles - # nullable: true - x-kubernetes-preserve-unknown-fields: true - quotas: - type: object - description: | - allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` - you can configure any aspect of resource quotas - More details: https://clickhouse.tech/docs/en/operations/quotas/ - Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationquotas - # nullable: true - x-kubernetes-preserve-unknown-fields: true - settings: &TypeSettings - type: object - description: | - allows configure `clickhouse-server` settings inside ... tag in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` - More details: https://clickhouse.tech/docs/en/operations/settings/settings/ - Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationsettings - # nullable: true - x-kubernetes-preserve-unknown-fields: true - files: &TypeFiles - type: object - description: | - allows define content of any setting file inside each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` - every key in this object is the file name - every value in this object is the file content - you can use `!!binary |` and base64 for binary files, see details here https://yaml.org/type/binary.html - each key could contains prefix like USERS, COMMON, HOST or config.d, users.d, cond.d, wrong prefixes will ignored, subfolders also will ignored - More details: https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-05-files-nested.yaml - # nullable: true - x-kubernetes-preserve-unknown-fields: true - clusters: - type: array - description: | - describes ClickHouse clusters layout and allows change settings on cluster-level, shard-level and replica-level - every cluster is a set of StatefulSet, one StatefulSet contains only one Pod with `clickhouse-server` - all Pods will rendered in part of ClickHouse configs, mounted from ConfigMap as `/etc/clickhouse-server/config.d/chop-generated-remote_servers.xml` - Clusters will use for Distributed table engine, more details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ - If `cluster` contains zookeeper settings (could be inherited from top `chi` level), when you can create *ReplicatedMergeTree tables - # nullable: true - items: - type: object - #required: - # - name - properties: - name: - type: string - description: "cluster name, used to identify set of ClickHouse servers and wide used during generate names of related Kubernetes resources" - minLength: 1 - # See namePartClusterMaxLen const - maxLength: 15 - pattern: "^[a-zA-Z0-9-]{0,15}$" - zookeeper: - <<: *TypeZookeeperConfig - description: | - optional, allows configure .. section in each `Pod` only in current ClickHouse cluster, during generate `ConfigMap` which will mounted in `/etc/clickhouse-server/config.d/` - override top-level `chi.spec.configuration.zookeeper` settings - settings: - <<: *TypeSettings - description: | - optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` only in one cluster during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` - override top-level `chi.spec.configuration.settings` - More details: https://clickhouse.tech/docs/en/operations/settings/settings/ - files: - <<: *TypeFiles - description: | - optional, allows define content of any setting file inside each `Pod` on current cluster during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` - override top-level `chi.spec.configuration.files` - templates: - <<: *TypeTemplateNames - description: | - optional, configuration of the templates names which will use for generate Kubernetes resources according to selected cluster - override top-level `chi.spec.configuration.templates` - schemaPolicy: - type: object - description: | - describes how schema is propagated within replicas and shards - properties: - replica: - type: string - description: "how schema is propagated within a replica" - enum: - # List SchemaPolicyReplicaXXX constants from model - - "None" - - "All" - shard: - type: string - description: "how schema is propagated between shards" - enum: - # List SchemaPolicyShardXXX constants from model - - "None" - - "All" - - "DistributedTablesOnly" - secret: - type: object - description: "optional, shared secret value to secure cluster communications" - properties: - auto: - <<: *TypeStringBool - description: "Auto-generate shared secret value to secure cluster communications" - value: - description: "Cluster shared secret value in plain text" - type: string - valueFrom: - description: "Cluster shared secret source" - type: object - properties: - secretKeyRef: - description: | - Selects a key of a secret in the clickhouse installation namespace. - Should not be used if value is not empty. - type: object - properties: - name: - description: | - Name of the referent. More info: - https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - key: - description: The key of the secret to select from. Must be a valid secret key. - type: string - optional: - description: Specify whether the Secret or its key must be defined - type: boolean - required: - - name - - key - layout: - type: object - description: | - describe current cluster layout, how much shards in cluster, how much replica in shard - allows override settings on each shard and replica separatelly - # nullable: true - properties: - type: - type: string - description: "DEPRECATED - to be removed soon" - shardsCount: - type: integer - description: "how much shards for current ClickHouse cluster will run in Kubernetes, each shard contains shared-nothing part of data and contains set of replicas, cluster contains 1 shard by default" - replicasCount: - type: integer - description: "how much replicas in each shards for current ClickHouse cluster will run in Kubernetes, each replica is a separate `StatefulSet` which contains only one `Pod` with `clickhouse-server` instance, every shard contains 1 replica by default" - shards: - type: array - description: "optional, allows override top-level `chi.spec.configuration`, cluster-level `chi.spec.configuration.clusters` settings for each shard separately, use it only if you fully understand what you do" - # nullable: true - items: - type: object - properties: - name: - type: string - description: "optional, by default shard name is generated, but you can override it and setup custom name" - minLength: 1 - # See namePartShardMaxLen const - maxLength: 15 - pattern: "^[a-zA-Z0-9-]{0,15}$" - definitionType: - type: string - description: "DEPRECATED - to be removed soon" - weight: - type: integer - description: | - optional, 1 by default, allows setup shard setting which will use during insert into tables with `Distributed` engine, - will apply in inside ConfigMap which will mount in /etc/clickhouse-server/config.d/chop-generated-remote_servers.xml - More details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ - internalReplication: - <<: *TypeStringBool - description: | - optional, `true` by default when `chi.spec.configuration.clusters[].layout.ReplicaCount` > 1 and 0 otherwise - allows setup setting which will use during insert into tables with `Distributed` engine for insert only in one live replica and other replicas will download inserted data during replication, - will apply in inside ConfigMap which will mount in /etc/clickhouse-server/config.d/chop-generated-remote_servers.xml - More details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ - settings: - <<: *TypeSettings - description: | - optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` only in one shard during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` - override top-level `chi.spec.configuration.settings` and cluster-level `chi.spec.configuration.clusters.settings` - More details: https://clickhouse.tech/docs/en/operations/settings/settings/ - files: - <<: *TypeFiles - description: | - optional, allows define content of any setting file inside each `Pod` only in one shard during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` - override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files` - templates: - <<: *TypeTemplateNames - description: | - optional, configuration of the templates names which will use for generate Kubernetes resources according to selected shard - override top-level `chi.spec.configuration.templates` and cluster-level `chi.spec.configuration.clusters.templates` - replicasCount: - type: integer - description: | - optional, how much replicas in selected shard for selected ClickHouse cluster will run in Kubernetes, each replica is a separate `StatefulSet` which contains only one `Pod` with `clickhouse-server` instance, - shard contains 1 replica by default - override cluster-level `chi.spec.configuration.clusters.layout.replicasCount` - minimum: 1 - replicas: - type: array - description: | - optional, allows override behavior for selected replicas from cluster-level `chi.spec.configuration.clusters` and shard-level `chi.spec.configuration.clusters.layout.shards` - # nullable: true - items: - # Host - type: object - properties: - name: - type: string - description: "optional, by default replica name is generated, but you can override it and setup custom name" - minLength: 1 - # See namePartReplicaMaxLen const - maxLength: 15 - pattern: "^[a-zA-Z0-9-]{0,15}$" - secure: - type: boolean - description: | - optional, setup `secure` inside `clickhouse-server` settings for each Pod where current template will apply - if specified - tcpPort: - type: integer - description: | - optional, setup `Pod.spec.containers.ports` with name `tcp` for selected replica, override `chi.spec.templates.hostTemplates.spec.tcpPort` - allows connect to `clickhouse-server` via TCP Native protocol via kubernetes `Service` - minimum: 1 - maximum: 65535 - httpPort: - type: integer - description: | - optional, setup `Pod.spec.containers.ports` with name `http` for selected replica, override `chi.spec.templates.hostTemplates.spec.httpPort` - allows connect to `clickhouse-server` via HTTP protocol via kubernetes `Service` - minimum: 1 - maximum: 65535 - interserverHTTPPort: - type: integer - description: | - optional, setup `Pod.spec.containers.ports` with name `interserver` for selected replica, override `chi.spec.templates.hostTemplates.spec.interserverHTTPPort` - allows connect between replicas inside same shard during fetch replicated data parts HTTP protocol - minimum: 1 - maximum: 65535 - settings: - <<: *TypeSettings - description: | - optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` - override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and shard-level `chi.spec.configuration.clusters.layout.shards.settings` - More details: https://clickhouse.tech/docs/en/operations/settings/settings/ - files: - <<: *TypeFiles - description: | - optional, allows define content of any setting file inside `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` - override top-level `chi.spec.configuration.files`, cluster-level `chi.spec.configuration.clusters.files` and shard-level `chi.spec.configuration.clusters.layout.shards.files` - templates: - <<: *TypeTemplateNames - description: | - optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica - override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates` and shard-level `chi.spec.configuration.clusters.layout.shards.templates` - replicas: - type: array - description: "optional, allows override top-level `chi.spec.configuration` and cluster-level `chi.spec.configuration.clusters` configuration for each replica and each shard relates to selected replica, use it only if you fully understand what you do" - # nullable: true - items: - type: object - properties: - name: - type: string - description: "optional, by default replica name is generated, but you can override it and setup custom name" - minLength: 1 - # See namePartShardMaxLen const - maxLength: 15 - pattern: "^[a-zA-Z0-9-]{0,15}$" - settings: - <<: *TypeSettings - description: | - optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` - override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and will ignore if shard-level `chi.spec.configuration.clusters.layout.shards` present - More details: https://clickhouse.tech/docs/en/operations/settings/settings/ - files: - <<: *TypeFiles - description: | - optional, allows define content of any setting file inside each `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` - override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files`, will ignore if `chi.spec.configuration.clusters.layout.shards` presents - templates: - <<: *TypeTemplateNames - description: | - optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica - override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates` - shardsCount: - type: integer - description: "optional, count of shards related to current replica, you can override each shard behavior on low-level `chi.spec.configuration.clusters.layout.replicas.shards`" - minimum: 1 - shards: - type: array - description: "optional, list of shards related to current replica, will ignore if `chi.spec.configuration.clusters.layout.shards` presents" - # nullable: true - items: - # Host - type: object - properties: - name: - type: string - description: "optional, by default shard name is generated, but you can override it and setup custom name" - minLength: 1 - # See namePartReplicaMaxLen const - maxLength: 15 - pattern: "^[a-zA-Z0-9-]{0,15}$" - secure: - type: boolean - description: | - optional, setup `secure` inside `clickhouse-server` settings for each Pod where current template will apply - if specified - tcpPort: - type: integer - description: | - optional, setup `Pod.spec.containers.ports` with name `tcp` for selected shard, override `chi.spec.templates.hostTemplates.spec.tcpPort` - allows connect to `clickhouse-server` via TCP Native protocol via kubernetes `Service` - minimum: 1 - maximum: 65535 - httpPort: - type: integer - description: | - optional, setup `Pod.spec.containers.ports` with name `http` for selected shard, override `chi.spec.templates.hostTemplates.spec.httpPort` - allows connect to `clickhouse-server` via HTTP protocol via kubernetes `Service` - minimum: 1 - maximum: 65535 - interserverHTTPPort: - type: integer - description: | - optional, setup `Pod.spec.containers.ports` with name `interserver` for selected shard, override `chi.spec.templates.hostTemplates.spec.interserverHTTPPort` - allows connect between replicas inside same shard during fetch replicated data parts HTTP protocol - minimum: 1 - maximum: 65535 - settings: - <<: *TypeSettings - description: | - optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one shard related to current replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` - override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and replica-level `chi.spec.configuration.clusters.layout.replicas.settings` - More details: https://clickhouse.tech/docs/en/operations/settings/settings/ - files: - <<: *TypeFiles - description: | - optional, allows define content of any setting file inside each `Pod` only in one shard related to current replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` - override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files`, will ignore if `chi.spec.configuration.clusters.layout.shards` presents - templates: - <<: *TypeTemplateNames - description: | - optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica - override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates` - templates: - type: object - description: "allows define templates which will use for render Kubernetes resources like StatefulSet, ConfigMap, Service, PVC, by default, clickhouse-operator have own templates, but you can override it" - # nullable: true - properties: - hostTemplates: - type: array - description: "hostTemplate will use during apply to generate `clickhose-server` config files" - # nullable: true - items: - type: object - #required: - # - name - properties: - name: - description: "template name, could use to link inside top-level `chi.spec.defaults.templates.hostTemplate`, cluster-level `chi.spec.configuration.clusters.templates.hostTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.hostTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.hostTemplate`" - type: string - portDistribution: - type: array - description: "define how will distribute numeric values of named ports in `Pod.spec.containers.ports` and clickhouse-server configs" - # nullable: true - items: - type: object - #required: - # - type - properties: - type: - type: string - description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" - enum: - # List PortDistributionXXX constants - - "" - - "Unspecified" - - "ClusterScopeIndex" - spec: - # Host - type: object - properties: - name: - type: string - description: "by default, hostname will generate, but this allows define custom name for each `clickhuse-server`" - minLength: 1 - # See namePartReplicaMaxLen const - maxLength: 15 - pattern: "^[a-zA-Z0-9-]{0,15}$" - secure: - type: boolean - description: | - optional, setup `secure` inside `clickhouse-server` settings for each Pod where current template will apply - if specified - tcpPort: - type: integer - description: | - optional, setup `tcp_port` inside `clickhouse-server` settings for each Pod where current template will apply - if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=tcp]` - More info: https://clickhouse.tech/docs/en/interfaces/tcp/ - minimum: 1 - maximum: 65535 - httpPort: - type: integer - description: | - optional, setup `http_port` inside `clickhouse-server` settings for each Pod where current template will apply - if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=http]` - More info: https://clickhouse.tech/docs/en/interfaces/http/ - minimum: 1 - maximum: 65535 - interserverHTTPPort: - type: integer - description: | - optional, setup `interserver_http_port` inside `clickhouse-server` settings for each Pod where current template will apply - if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=interserver]` - More info: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#interserver-http-port - minimum: 1 - maximum: 65535 - settings: - <<: *TypeSettings - description: | - optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` where this template will apply during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` - More details: https://clickhouse.tech/docs/en/operations/settings/settings/ - files: - <<: *TypeFiles - description: | - optional, allows define content of any setting file inside each `Pod` where this template will apply during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` - templates: - <<: *TypeTemplateNames - description: "be careful, this part of CRD allows override template inside template, don't use it if you don't understand what you do" - - podTemplates: - type: array - description: | - podTemplate will use during render `Pod` inside `StatefulSet.spec` and allows define rendered `Pod.spec`, pod scheduling distribution and pod zone - More information: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatespodtemplates - # nullable: true - items: - type: object - #required: - # - name - properties: - name: - type: string - description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" - generateName: - type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" - zone: - type: object - description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" - #required: - # - values - properties: - key: - type: string - description: "optional, if defined, allows select kubernetes nodes by label with `name` equal `key`" - values: - type: array - description: "optional, if defined, allows select kubernetes nodes by label with `value` in `values`" - # nullable: true - items: - type: string - distribution: - type: string - description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" - enum: - - "" - - "Unspecified" - - "OnePerHost" - podDistribution: - type: array - description: "define ClickHouse Pod distibution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" - # nullable: true - items: - type: object - #required: - # - type - properties: - type: - type: string - description: "you can define multiple affinity policy types" - enum: - # List PodDistributionXXX constants - - "" - - "Unspecified" - - "ClickHouseAntiAffinity" - - "ShardAntiAffinity" - - "ReplicaAntiAffinity" - - "AnotherNamespaceAntiAffinity" - - "AnotherClickHouseInstallationAntiAffinity" - - "AnotherClusterAntiAffinity" - - "MaxNumberPerNode" - - "NamespaceAffinity" - - "ClickHouseInstallationAffinity" - - "ClusterAffinity" - - "ShardAffinity" - - "ReplicaAffinity" - - "PreviousTailAffinity" - - "CircularReplication" - scope: - type: string - description: "scope for apply each podDistribution" - enum: - # list PodDistributionScopeXXX constants - - "" - - "Unspecified" - - "Shard" - - "Replica" - - "Cluster" - - "ClickHouseInstallation" - - "Namespace" - number: - type: integer - description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" - minimum: 0 - maximum: 65535 - topologyKey: - type: string - description: "use for inter-pod affinity look to `pod.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution.podAffinityTerm.topologyKey`, More info: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity" - spec: - # TODO specify PodSpec - type: object - description: "allows define whole Pod.spec inside StaefulSet.spec, look to https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates for details" - # nullable: true - x-kubernetes-preserve-unknown-fields: true - metadata: - type: object - description: | - allows pass standard object's metadata from template to Pod - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata - # nullable: true - x-kubernetes-preserve-unknown-fields: true - - volumeClaimTemplates: - type: array - description: "allows define template for rendering `PVC` kubernetes resource, which would use inside `Pod` for mount clickhouse `data`, clickhouse `logs` or something else" - # nullable: true - items: - type: object - #required: - # - name - # - spec - properties: - name: - type: string - description: | - template name, could use to link inside - top-level `chi.spec.defaults.templates.dataVolumeClaimTemplate` or `chi.spec.defaults.templates.logVolumeClaimTemplate`, - cluster-level `chi.spec.configuration.clusters.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.templates.logVolumeClaimTemplate`, - shard-level `chi.spec.configuration.clusters.layout.shards.temlates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.shards.temlates.logVolumeClaimTemplate` - replica-level `chi.spec.configuration.clusters.layout.replicas.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.replicas.templates.logVolumeClaimTemplate` - provisioner: *TypePVCProvisioner - reclaimPolicy: *TypePVCReclaimPolicy - metadata: - type: object - description: | - allows to pass standard object's metadata from template to PVC - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata - # nullable: true - x-kubernetes-preserve-unknown-fields: true - spec: - type: object - description: | - allows define all aspects of `PVC` resource - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims - # nullable: true - x-kubernetes-preserve-unknown-fields: true - serviceTemplates: - type: array - description: | - allows define template for rendering `Service` which would get endpoint from Pods which scoped chi-wide, cluster-wide, shard-wide, replica-wide level - # nullable: true - items: - type: object - #required: - # - name - # - spec - properties: - name: - type: string - description: | - template name, could use to link inside - chi-level `chi.spec.defaults.templates.serviceTemplate` - cluster-level `chi.spec.configuration.clusters.templates.clusterServiceTemplate` - shard-level `chi.spec.configuration.clusters.layout.shards.temlates.shardServiceTemplate` - replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` - generateName: - type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" - metadata: - # TODO specify ObjectMeta - type: object - description: | - allows pass standard object's metadata from template to Service - Could be use for define specificly for Cloud Provider metadata which impact to behavior of service - More info: https://kubernetes.io/docs/concepts/services-networking/service/ - # nullable: true - x-kubernetes-preserve-unknown-fields: true - spec: - # TODO specify ServiceSpec - type: object - description: | - describe behavior of generated Service - More info: https://kubernetes.io/docs/concepts/services-networking/service/ - # nullable: true - x-kubernetes-preserve-unknown-fields: true - useTemplates: - type: array - description: "list of `ClickHouseInstallationTemplate` (chit) resource names which will merge with current `Chi` manifest during render Kubernetes resources to create related ClickHouse clusters" - # nullable: true - items: - type: object - #required: - # - name - properties: - name: - type: string - description: "name of `ClickHouseInstallationTemplate` (chit) resource" - namespace: - type: string - description: "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clichouse-operator`" - useType: - type: string - description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit`" - enum: - # List useTypeXXX constants from model - - "" - - "merge" ---- -# Template Parameters: -# -# NONE -# -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: clickhouseoperatorconfigurations.clickhouse.altinity.com - labels: - clickhouse.altinity.com/chop: 0.20.0 -spec: - group: clickhouse.altinity.com - scope: Namespaced - names: - kind: ClickHouseOperatorConfiguration - singular: clickhouseoperatorconfiguration - plural: clickhouseoperatorconfigurations - shortNames: - - chopconf - versions: - - name: v1 - served: true - storage: true - additionalPrinterColumns: - - name: namespaces - type: string - description: Watch namespaces - priority: 0 # show in standard view - jsonPath: .status - - name: age - type: date - description: Age of the resource - # Displayed in all priorities - jsonPath: .metadata.creationTimestamp - schema: - openAPIV3Schema: - type: object - description: "allows customize `clickhouse-operator` settings, need restart clickhouse-operator pod after adding, more details https://github.com/Altinity/clickhouse-operator/blob/master/docs/operator_configuration.md" - x-kubernetes-preserve-unknown-fields: true - properties: - status: - type: object - x-kubernetes-preserve-unknown-fields: true - spec: - type: object - description: | - Allows to define settings of the clickhouse-operator. - More info: https://github.com/Altinity/clickhouse-operator/blob/master/config/config.yaml - Check into etc-clickhouse-operator* ConfigMaps if you need more control - x-kubernetes-preserve-unknown-fields: true - properties: - watch: - type: object - description: "Parameters for watch kubernetes resources which used by clickhouse-operator deployment" - properties: - namespaces: - type: array - description: "List of namespaces where clickhouse-operator watches for events." - items: - type: string - clickhouse: - type: object - description: "Clickhouse related parameters used by clickhouse-operator" - properties: - configuration: - type: object - properties: - file: - type: object - properties: - path: - type: object - properties: - common: - type: string - description: "Path to the folder where ClickHouse configuration files common for all instances within a CHI are located. Default - config.d" - host: - type: string - description: "Path to the folder where ClickHouse configuration files unique for each instance (host) within a CHI are located. Default - conf.d" - user: - type: string - description: "Path to the folder where ClickHouse configuration files with users settings are located. Files are common for all instances within a CHI. Default - users.d" - user: - type: object - description: "Default parameters for any user which will create" - properties: - default: - type: object - properties: - profile: - type: string - description: "ClickHouse server configuration `...` for any " - quota: - type: string - description: "ClickHouse server configuration `...` for any " - networksIP: - type: array - description: "ClickHouse server configuration `...` for any " - items: - type: string - password: - type: string - description: "ClickHouse server configuration `...` for any " - network: - type: object - description: "Default network parameters for any user which will create" - properties: - hostRegexpTemplate: - type: string - description: "ClickHouse server configuration `...` for any " - access: - type: object - description: "parameters which use for connect to clickhouse from clickhouse-operator deployment" - properties: - scheme: - type: string - description: "The scheme to user for connecting to ClickHouse. One of http or https" - username: - type: string - description: "ClickHouse username to be used by operator to connect to ClickHouse instances, deprecated, use chCredentialsSecretName" - password: - type: string - description: "ClickHouse password to be used by operator to connect to ClickHouse instances, deprecated, use chCredentialsSecretName" - rootCA: - type: string - description: "Root certificate authority that clients use when verifying server certificates. Used for https connection to ClickHouse" - secret: - type: object - properties: - namespace: - type: string - description: "Location of k8s Secret with username and password to be used by operator to connect to ClickHouse instances" - name: - type: string - description: "Name of k8s Secret with username and password to be used by operator to connect to ClickHouse instances" - port: - type: integer - minimum: 1 - maximum: 65535 - description: "Port to be used by operator to connect to ClickHouse instances" - timeouts: - type: object - description: "Timeouts used to limit connection and queries from the operator to ClickHouse instances, In seconds" - properties: - connect: - type: integer - minimum: 1 - maximum: 10 - description: "Connect timeout. In seconds." - query: - type: integer - minimum: 1 - maximum: 600 - description: "Query timeout. In seconds." - template: - type: object - description: "Parameters which are used if you want to generate ClickHouseInstallationTemplate custom resources from files which are stored inside clickhouse-operator deployment" - properties: - chi: - type: object - properties: - path: - type: string - description: "Path to folder where ClickHouseInstallationTemplate .yaml manifests are located." - reconcile: - type: object - description: "allow tuning reconciling process" - properties: - runtime: - type: object - description: "runtime parameters for clickhouse-operator process which use during reconciling" - properties: - threadsNumber: - type: integer - minimum: 1 - maximum: 65535 - description: "How many goroutines will be used to reconcile in parallel, 10 by default" - statefulSet: - type: object - description: "Allow change default behavior for reconciling StatefulSet which generated by clickhouse-operator" - properties: - create: - type: object - description: "Behavior during create StatefulSet" - properties: - onFailure: - type: string - description: | - What to do in case created StatefulSet is not in Ready after `statefulSetUpdateTimeout` seconds - Possible options: - 1. abort - do nothing, just break the process and wait for admin. - 2. delete - delete newly created problematic StatefulSet. - 3. ignore (default) - ignore error, pretend nothing happened and move on to the next StatefulSet. - update: - type: object - description: "Behavior during update StatefulSet" - properties: - timeout: - type: integer - description: "How many seconds to wait for created/updated StatefulSet to be Ready" - pollInterval: - type: integer - description: "How many seconds to wait between checks for created/updated StatefulSet status" - onFailure: - type: string - description: | - What to do in case updated StatefulSet is not in Ready after `statefulSetUpdateTimeout` seconds - Possible options: - 1. abort - do nothing, just break the process and wait for admin. - 2. rollback (default) - delete Pod and rollback StatefulSet to previous Generation. Pod would be recreated by StatefulSet based on rollback-ed configuration. - 3. ignore - ignore error, pretend nothing happened and move on to the next StatefulSet. - host: - type: object - description: "allow define how to wait host include to system.cluster behavior during scale up and scale down cluster operations" - properties: - wait: - type: object - properties: - exclude: &TypeStringBool - type: string - description: "wait when a pod will be removed from the cluster" - enum: - # List StringBoolXXX constants from model - - "" - - "0" - - "1" - - "False" - - "false" - - "True" - - "true" - - "No" - - "no" - - "Yes" - - "yes" - - "Off" - - "off" - - "On" - - "on" - - "Disable" - - "disable" - - "Enable" - - "enable" - - "Disabled" - - "disabled" - - "Enabled" - - "enabled" - include: - <<: *TypeStringBool - description: "wait when a pod will be added to the cluster" - annotation: - type: object - description: "defines which metadata.annotations items will include or exclude during render StatefulSet, Pod, PVC resources" - properties: - include: - type: array - description: | - When propagating labels from the chi's `metadata.annotations` section to child objects' `metadata.annotations`, - include annotations with names from the following list - items: - type: string - exclude: - type: array - description: | - When propagating labels from the chi's `metadata.annotations` section to child objects' `metadata.annotations`, - exclude annotations with names from the following list - items: - type: string - label: - type: object - description: "defines which metadata.labels will include or exclude during render StatefulSet, Pod, PVC resources" - properties: - include: - type: array - description: | - When propagating labels from the chi's `metadata.labels` section to child objects' `metadata.labels`, - include labels from the following list - items: - type: string - exclude: - type: array - items: - type: string - description: | - When propagating labels from the chi's `metadata.labels` section to child objects' `metadata.labels`, - exclude labels from the following list - appendScope: - <<: *TypeStringBool - description: | - Whether to append *Scope* labels to StatefulSet and Pod - - "LabelShardScopeIndex" - - "LabelReplicaScopeIndex" - - "LabelCHIScopeIndex" - - "LabelCHIScopeCycleSize" - - "LabelCHIScopeCycleIndex" - - "LabelCHIScopeCycleOffset" - - "LabelClusterScopeIndex" - - "LabelClusterScopeCycleSize" - - "LabelClusterScopeCycleIndex" - - "LabelClusterScopeCycleOffset" - statefulSet: - type: object - description: "define StatefulSet-specific parameters" - properties: - revisionHistoryLimit: - type: integer - description: | - revisionHistoryLimit is the maximum number of revisions that will be - maintained in the StatefulSet's revision history. - Look details in `statefulset.spec.revisionHistoryLimit` - pod: - type: object - description: "define pod specific parameters" - properties: - terminationGracePeriod: - type: integer - description: | - Optional duration in seconds the pod needs to terminate gracefully. - Look details in `pod.spec.terminationGracePeriodSeconds` - logger: - type: object - description: "allow setup clickhouse-operator logger behavior" - properties: - logtostderr: - type: string - description: "boolean, allows logs to stderr" - alsologtostderr: - type: string - description: "boolean allows logs to stderr and files both" - v: - type: string - description: "verbosity level of clickhouse-operator log, default - 1 max - 9" - stderrthreshold: - type: string - vmodule: - type: string - description: | - Comma-separated list of filename=N, where filename (can be a pattern) must have no .go ext, and N is a V level. - Ex.: file*=2 sets the 'V' to 2 in all files with names like file*. - log_backtrace_at: - type: string - description: | - It can be set to a file and line number with a logging line. - Ex.: file.go:123 - Each time when this line is being executed, a stack trace will be written to the Info log. ---- -# Template Parameters: -# -# COMMENT= -# NAMESPACE=kube-system -# NAME=clickhouse-operator -# -# Setup ServiceAccount -apiVersion: v1 -kind: ServiceAccount -metadata: - name: clickhouse-operator - namespace: kube-system - labels: - clickhouse.altinity.com/chop: 0.20.0 ---- -# Template Parameters: -# -# NAMESPACE=kube-system -# COMMENT=# -# ROLE_KIND=ClusterRole -# ROLE_NAME=clickhouse-operator-kube-system -# ROLE_BINDING_KIND=ClusterRoleBinding -# ROLE_BINDING_NAME=clickhouse-operator-kube-system -# -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: clickhouse-operator-kube-system - #namespace: kube-system - labels: - clickhouse.altinity.com/chop: 0.20.0 -rules: -- apiGroups: - - "" - resources: - - configmaps - - services - - persistentvolumeclaims - verbs: - - get - - list - - patch - - update - - watch - - create - - delete -- apiGroups: - - "" - resources: - - endpoints - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - events - verbs: - - create -- apiGroups: - - "" - resources: - - persistentvolumes - - pods - verbs: - - get - - list - - patch - - update - - watch -- apiGroups: - - apps - resources: - - statefulsets - verbs: - - get - - list - - patch - - update - - watch - - create - - delete -- apiGroups: - - apps - resources: - - replicasets - verbs: - - get - - patch - - update - - delete -- apiGroups: - - apps - resourceNames: - - clickhouse-operator - resources: - - deployments - verbs: - - get - - patch - - update - - delete -- apiGroups: - - policy - resources: - - poddisruptionbudgets - verbs: - - get - - list - - patch - - update - - watch - - create - - delete -- apiGroups: - - clickhouse.altinity.com - resources: - - clickhouseinstallations - verbs: - - get - - patch - - update - - delete -- apiGroups: - - clickhouse.altinity.com - resources: - - clickhouseinstallations - - clickhouseinstallationtemplates - - clickhouseoperatorconfigurations - verbs: - - get - - list - - watch -- apiGroups: - - clickhouse.altinity.com - resources: - - clickhouseinstallations/finalizers - - clickhouseinstallationtemplates/finalizers - - clickhouseoperatorconfigurations/finalizers - verbs: - - update -- apiGroups: - - clickhouse.altinity.com - resources: - - clickhouseinstallations/status - - clickhouseinstallationtemplates/status - - clickhouseoperatorconfigurations/status - verbs: - - get - - update - - patch - - create - - delete -- apiGroups: - - "" - resources: - - secrets - verbs: - - get - - list -- apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - get - - list ---- -# Setup ClusterRoleBinding between ClusterRole and ServiceAccount. -# ClusterRoleBinding is namespace-less and must have unique name -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: clickhouse-operator-kube-system - #namespace: kube-system - labels: - clickhouse.altinity.com/chop: 0.20.0 -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: clickhouse-operator-kube-system -subjects: -- kind: ServiceAccount - name: clickhouse-operator - namespace: kube-system ---- -# Template Parameters: -# -# NAME=etc-clickhouse-operator-files -# NAMESPACE=kube-system -# COMMENT= -# -apiVersion: v1 -kind: ConfigMap -metadata: - name: etc-clickhouse-operator-files - namespace: kube-system - labels: - clickhouse.altinity.com/chop: 0.20.0 - app: clickhouse-operator -data: - config.yaml: | - # IMPORTANT - # This file is auto-generated - # Do not edit this file - all changes would be lost - # Edit appropriate template in the following folder: - # deploy/builder/templates-config - # IMPORTANT - # - # Template parameters available: - # WATCH_NAMESPACES= - # CH_USERNAME_PLAIN= - # CH_PASSWORD_PLAIN= - # CH_CREDENTIALS_SECRET_NAMESPACE= - # CH_CREDENTIALS_SECRET_NAME=clickhouse-operator - - ################################################ - ## - ## Watch Section - ## - ################################################ - watch: - # List of namespaces where clickhouse-operator watches for events. - # Concurrently running operators should watch on different namespaces. - #namespaces: ["dev", "test"] - namespaces: [] - - clickhouse: - configuration: - ################################################ - ## - ## Configuration Files Section - ## - ################################################ - file: - path: - # Path to the folder where ClickHouse configuration files common for all instances within a CHI are located. - common: config.d - # Path to the folder where ClickHouse configuration files unique for each instance (host) within a CHI are located. - host: conf.d - # Path to the folder where ClickHouse configuration files with users' settings are located. - # Files are common for all instances within a CHI. - user: users.d - ################################################ - ## - ## Configuration Users Section - ## - ################################################ - user: - default: - # Default values for ClickHouse user configuration - # 1. user/profile - string - # 2. user/quota - string - # 3. user/networks/ip - multiple strings - # 4. user/password - string - profile: default - quota: default - networksIP: - - "::1" - - "127.0.0.1" - password: "default" - ################################################ - ## - ## Configuration Network Section - ## - ################################################ - network: - # Default host_regexp to limit network connectivity from outside - hostRegexpTemplate: "(chi-{chi}-[^.]+\\d+-\\d+|clickhouse\\-{chi})\\.{namespace}\\.svc\\.cluster\\.local$" - ################################################ - ## - ## Access to ClickHouse instances - ## - ################################################ - access: - # Possible values for `scheme` are: - # 1. http - # 2. https - scheme: "" - # ClickHouse credentials (username, password and port) to be used by the operator to connect to ClickHouse instances. - # Used for: - # 1. Metrics requests - # 2. Schema maintenance - # 3. DROP DNS CACHE - # User with these credentials can be specified in additional ClickHouse .xml config files, - # located in `clickhouse.configuration.file.path.user` folder - username: "" - password: "" - rootCA: "" - - # Location of the k8s Secret with username and password to be used by the operator to connect to ClickHouse instances. - # Can be used instead of explicitly specified username and password which are: - # clickhouse.access.username - # clickhouse.access.password - # Secret should have two keys: - # 1. username - # 2. password - secret: - # Empty `namespace` means that k8s secret would be looked in the same namespace where operator's pod is running. - namespace: "" - # Empty `name` means no k8s Secret would be looked for - name: "clickhouse-operator" - # Port where to connect to ClickHouse instances to - port: 8123 - - # Timeouts used to limit connection and queries from the operator to ClickHouse instances - # Specified in seconds. - timeouts: - connect: 2 - query: 5 - - ################################################ - ## - ## Templates Section - ## - ################################################ - template: - chi: - # Path to the folder where ClickHouseInstallation .yaml manifests are located. - # Manifests are applied in sorted alpha-numeric order. - path: templates.d - - ################################################ - ## - ## Reconcile Section - ## - ################################################ - reconcile: - runtime: - # Max number of concurrent reconciles in progress - threadsNumber: 10 - - statefulSet: - create: - # What to do in case created StatefulSet is not in 'Ready' after `reconcile.statefulSet.update.timeout` seconds - # Possible options: - # 1. abort - do nothing, just break the process and wait for an admin to assist - # 2. delete - delete newly created problematic StatefulSet - # 3. ignore - ignore an error, pretend nothing happened and move on to the next StatefulSet - onFailure: ignore - - update: - # How many seconds to wait for created/updated StatefulSet to be 'Ready' - timeout: 300 - # How many seconds to wait between checks/polls for created/updated StatefulSet status - pollInterval: 5 - # What to do in case updated StatefulSet is not in 'Ready' after `reconcile.statefulSet.update.timeout` seconds - # Possible options: - # 1. abort - do nothing, just break the process and wait for an admin to assist - # 2. rollback - delete Pod and rollback StatefulSet to previous Generation. - # Pod would be recreated by StatefulSet based on rollback-ed configuration - # 3. ignore - ignore an error, pretend nothing happened and move on to the next StatefulSet - onFailure: rollback - - host: - # Whether reconciler should wait for a host: - # - to be excluded from a cluster - # OR - # - to be included into a cluster - # respectfully - wait: - exclude: true - include: false - - ################################################ - ## - ## Annotations management - ## - ################################################ - annotation: - # Applied when: - # 1. Propagating annotations from the CHI's `metadata.annotations` to child objects' `metadata.annotations`, - # 2. Propagating annotations from the CHI Template's `metadata.annotations` to CHI's `metadata.annotations`, - # Include annotations from the following list: - # Applied only when not empty. Empty list means "include all, no selection" - include: [] - # Exclude annotations from the following list: - exclude: [] - - ################################################ - ## - ## Labels management - ## - ################################################ - label: - # Applied when: - # 1. Propagating labels from the CHI's `metadata.labels` to child objects' `metadata.labels`, - # 2. Propagating labels from the CHI Template's `metadata.labels` to CHI's `metadata.labels`, - # Include labels from the following list: - # Applied only when not empty. Empty list means "include all, no selection" - include: [] - # Exclude labels from the following list: - # Applied only when not empty. Empty list means "nothing to exclude, no selection" - exclude: [] - # Whether to append *Scope* labels to StatefulSet and Pod. - # Full list of available *scope* labels check in 'labeler.go' - # LabelShardScopeIndex - # LabelReplicaScopeIndex - # LabelCHIScopeIndex - # LabelCHIScopeCycleSize - # LabelCHIScopeCycleIndex - # LabelCHIScopeCycleOffset - # LabelClusterScopeIndex - # LabelClusterScopeCycleSize - # LabelClusterScopeCycleIndex - # LabelClusterScopeCycleOffset - appendScope: "no" - - ################################################ - ## - ## StatefulSet management - ## - ################################################ - statefulSet: - revisionHistoryLimit: 0 - - ################################################ - ## - ## Pod management - ## - ################################################ - pod: - # Grace period for Pod termination. - # How many seconds to wait between sending - # SIGTERM and SIGKILL during Pod termination process. - # Increase this number is case of slow shutdown. - terminationGracePeriod: 30 - - ################################################ - ## - ## Log parameters - ## - ################################################ - logger: - logtostderr: "true" - alsologtostderr: "false" - v: "1" - stderrthreshold: "" - vmodule: "" - log_backtrace_at: "" - ---- -# Template Parameters: -# -# NAME=etc-clickhouse-operator-confd-files -# NAMESPACE=kube-system -# COMMENT= -# -apiVersion: v1 -kind: ConfigMap -metadata: - name: etc-clickhouse-operator-confd-files - namespace: kube-system - labels: - clickhouse.altinity.com/chop: 0.20.0 - app: clickhouse-operator -data: ---- -# Template Parameters: -# -# NAME=etc-clickhouse-operator-configd-files -# NAMESPACE=kube-system -# COMMENT= -# -apiVersion: v1 -kind: ConfigMap -metadata: - name: etc-clickhouse-operator-configd-files - namespace: kube-system - labels: - clickhouse.altinity.com/chop: 0.20.0 - app: clickhouse-operator -data: - 01-clickhouse-01-listen.xml: | - - - - - - - - - :: - 0.0.0.0 - 1 - - - 01-clickhouse-02-logger.xml: | - - - - - - - - - - debug - /var/log/clickhouse-server/clickhouse-server.log - /var/log/clickhouse-server/clickhouse-server.err.log - 1000M - 10 - - 1 - - - - 01-clickhouse-03-query_log.xml: | - - - - - - - - - system - query_log
- Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 30 day - 7500 - - - - - 01-clickhouse-04-part_log.xml: | - - - - - - - - - system - part_log
- Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 30 day - 7500 - - - ---- -# Template Parameters: -# -# NAME=etc-clickhouse-operator-templatesd-files -# NAMESPACE=kube-system -# COMMENT= -# -apiVersion: v1 -kind: ConfigMap -metadata: - name: etc-clickhouse-operator-templatesd-files - namespace: kube-system - labels: - clickhouse.altinity.com/chop: 0.20.0 - app: clickhouse-operator -data: - 001-templates.json.example: | - { - "apiVersion": "clickhouse.altinity.com/v1", - "kind": "ClickHouseInstallationTemplate", - "metadata": { - "name": "01-default-volumeclaimtemplate" - }, - "spec": { - "templates": { - "volumeClaimTemplates": [ - { - "name": "chi-default-volume-claim-template", - "spec": { - "accessModes": [ - "ReadWriteOnce" - ], - "resources": { - "requests": { - "storage": "2Gi" - } - } - } - } - ], - "podTemplates": [ - { - "name": "chi-default-oneperhost-pod-template", - "distribution": "OnePerHost", - "spec": { - "containers" : [ - { - "name": "clickhouse", - "image": "clickhouse/clickhouse-server:22.3", - "ports": [ - { - "name": "http", - "containerPort": 8123 - }, - { - "name": "client", - "containerPort": 9000 - }, - { - "name": "interserver", - "containerPort": 9009 - } - ] - } - ] - } - } - ] - } - } - } - - default-pod-template.yaml.example: | - apiVersion: "clickhouse.altinity.com/v1" - kind: "ClickHouseInstallationTemplate" - metadata: - name: "default-oneperhost-pod-template" - spec: - templates: - podTemplates: - - name: default-oneperhost-pod-template - distribution: "OnePerHost" - default-storage-template.yaml.example: | - apiVersion: "clickhouse.altinity.com/v1" - kind: "ClickHouseInstallationTemplate" - metadata: - name: "default-storage-template-2Gi" - spec: - templates: - volumeClaimTemplates: - - name: default-storage-template-2Gi - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 2Gi - - readme: | - Templates in this folder are packaged with an operator and available via 'useTemplate' ---- -# Template Parameters: -# -# NAME=etc-clickhouse-operator-usersd-files -# NAMESPACE=kube-system -# COMMENT= -# -apiVersion: v1 -kind: ConfigMap -metadata: - name: etc-clickhouse-operator-usersd-files - namespace: kube-system - labels: - clickhouse.altinity.com/chop: 0.20.0 - app: clickhouse-operator -data: - 01-clickhouse-user.xml: | - - - - - - - - - - - - 127.0.0.1 - - clickhouse_operator - default - - - - - 0 - 1 - 10 - - - - - 02-clickhouse-default-profile.xml: | - - - - - - - - - - 1 - 1000 - 1 - 1 - - - - 03-database-ordinary.xml: | - - - - - - - - - - - Ordinary - - - ---- -# -# Template parameters available: -# NAMESPACE=kube-system -# COMMENT= -# OPERATOR_VERSION=0.20.0 -# CH_USERNAME_SECRET_PLAIN=clickhouse_operator -# CH_PASSWORD_SECRET_PLAIN=clickhouse_operator_password -# -apiVersion: v1 -kind: Secret -metadata: - name: clickhouse-operator - namespace: kube-system - labels: - clickhouse.altinity.com/chop: 0.20.0 - app: clickhouse-operator -type: Opaque -stringData: - username: clickhouse_operator - password: clickhouse_operator_password ---- -# Template Parameters: -# -# NAMESPACE=kube-system -# COMMENT= -# OPERATOR_IMAGE=altinity/clickhouse-operator:0.20.0 -# OPERATOR_IMAGE_PULL_POLICY=Always -# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.20.0 -# METRICS_EXPORTER_IMAGE_PULL_POLICY=Always -# -# Setup Deployment for clickhouse-operator -# Deployment would be created in kubectl-specified namespace -kind: Deployment -apiVersion: apps/v1 -metadata: - name: clickhouse-operator - namespace: kube-system - labels: - clickhouse.altinity.com/chop: 0.20.0 - app: clickhouse-operator -spec: - replicas: 1 - selector: - matchLabels: - app: clickhouse-operator - template: - metadata: - labels: - app: clickhouse-operator - annotations: - prometheus.io/port: '8888' - prometheus.io/scrape: 'true' - spec: - serviceAccountName: clickhouse-operator - volumes: - - name: etc-clickhouse-operator-folder - configMap: - name: etc-clickhouse-operator-files - - name: etc-clickhouse-operator-confd-folder - configMap: - name: etc-clickhouse-operator-confd-files - - name: etc-clickhouse-operator-configd-folder - configMap: - name: etc-clickhouse-operator-configd-files - - name: etc-clickhouse-operator-templatesd-folder - configMap: - name: etc-clickhouse-operator-templatesd-files - - name: etc-clickhouse-operator-usersd-folder - configMap: - name: etc-clickhouse-operator-usersd-files - containers: - - name: clickhouse-operator - image: altinity/clickhouse-operator:0.20.0 - imagePullPolicy: Always - volumeMounts: - - name: etc-clickhouse-operator-folder - mountPath: /etc/clickhouse-operator - - name: etc-clickhouse-operator-confd-folder - mountPath: /etc/clickhouse-operator/conf.d - - name: etc-clickhouse-operator-configd-folder - mountPath: /etc/clickhouse-operator/config.d - - name: etc-clickhouse-operator-templatesd-folder - mountPath: /etc/clickhouse-operator/templates.d - - name: etc-clickhouse-operator-usersd-folder - mountPath: /etc/clickhouse-operator/users.d - env: - # Pod-specific - # spec.nodeName: ip-172-20-52-62.ec2.internal - - name: OPERATOR_POD_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - # metadata.name: clickhouse-operator-6f87589dbb-ftcsf - - name: OPERATOR_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - # metadata.namespace: kube-system - - name: OPERATOR_POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - # status.podIP: 100.96.3.2 - - name: OPERATOR_POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - # spec.serviceAccount: clickhouse-operator - # spec.serviceAccountName: clickhouse-operator - - name: OPERATOR_POD_SERVICE_ACCOUNT - valueFrom: - fieldRef: - fieldPath: spec.serviceAccountName - - # Container-specific - - name: OPERATOR_CONTAINER_CPU_REQUEST - valueFrom: - resourceFieldRef: - containerName: clickhouse-operator - resource: requests.cpu - - name: OPERATOR_CONTAINER_CPU_LIMIT - valueFrom: - resourceFieldRef: - containerName: clickhouse-operator - resource: limits.cpu - - name: OPERATOR_CONTAINER_MEM_REQUEST - valueFrom: - resourceFieldRef: - containerName: clickhouse-operator - resource: requests.memory - - name: OPERATOR_CONTAINER_MEM_LIMIT - valueFrom: - resourceFieldRef: - containerName: clickhouse-operator - resource: limits.memory - - - name: metrics-exporter - image: altinity/metrics-exporter:0.20.0 - imagePullPolicy: Always - volumeMounts: - - name: etc-clickhouse-operator-folder - mountPath: /etc/clickhouse-operator - - name: etc-clickhouse-operator-confd-folder - mountPath: /etc/clickhouse-operator/conf.d - - name: etc-clickhouse-operator-configd-folder - mountPath: /etc/clickhouse-operator/config.d - - name: etc-clickhouse-operator-templatesd-folder - mountPath: /etc/clickhouse-operator/templates.d - - name: etc-clickhouse-operator-usersd-folder - mountPath: /etc/clickhouse-operator/users.d - env: - # Pod-specific - # spec.nodeName: ip-172-20-52-62.ec2.internal - - name: OPERATOR_POD_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - # metadata.name: clickhouse-operator-6f87589dbb-ftcsf - - name: OPERATOR_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - # metadata.namespace: kube-system - - name: OPERATOR_POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - # status.podIP: 100.96.3.2 - - name: OPERATOR_POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - # spec.serviceAccount: clickhouse-operator - # spec.serviceAccountName: clickhouse-operator - - name: OPERATOR_POD_SERVICE_ACCOUNT - valueFrom: - fieldRef: - fieldPath: spec.serviceAccountName - - # Container-specific - - name: OPERATOR_CONTAINER_CPU_REQUEST - valueFrom: - resourceFieldRef: - containerName: clickhouse-operator - resource: requests.cpu - - name: OPERATOR_CONTAINER_CPU_LIMIT - valueFrom: - resourceFieldRef: - containerName: clickhouse-operator - resource: limits.cpu - - name: OPERATOR_CONTAINER_MEM_REQUEST - valueFrom: - resourceFieldRef: - containerName: clickhouse-operator - resource: requests.memory - - name: OPERATOR_CONTAINER_MEM_LIMIT - valueFrom: - resourceFieldRef: - containerName: clickhouse-operator - resource: limits.memory - ports: - - containerPort: 8888 - name: metrics ---- -# Template Parameters: -# -# NAMESPACE=kube-system -# COMMENT= -# -# Setup ClusterIP Service to provide monitoring metrics for Prometheus -# Service would be created in kubectl-specified namespace -# In order to get access outside of k8s it should be exposed as: -# kubectl --namespace prometheus port-forward service/prometheus 9090 -# and point browser to localhost:9090 -kind: Service -apiVersion: v1 -metadata: - name: clickhouse-operator-metrics - namespace: kube-system - labels: - clickhouse.altinity.com/chop: 0.20.0 - app: clickhouse-operator -spec: - ports: - - port: 8888 - name: clickhouse-operator-metrics - selector: - app: clickhouse-operator diff --git a/clickhouse.yaml b/clickhouse.yaml new file mode 100644 index 0000000..17eebfb --- /dev/null +++ b/clickhouse.yaml @@ -0,0 +1,173 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: databases +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: clickhouse-data-pvc + namespace: databases +spec: + accessModes: + - ReadWriteOnce + storageClassName: longhorn + resources: + requests: + storage: 100Gi + limits: + storage: 100Gi +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: clickhouse-logs-pvc + namespace: databases +spec: + accessModes: + - ReadWriteOnce + storageClassName: longhorn + resources: + requests: + storage: 1Gi + limits: + storage: 1Gi +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: clickhouse-configmap + namespace: databases +data: + custom-config.xml: | + + 0.0.0.0 + 8123 + 9000 + + /metrics + 9363 + true + true + true + + + s3_plain + + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: clickhouse + namespace: databases +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + ms: clickhouse + template: + metadata: + labels: + ms: clickhouse + spec: + terminationGracePeriodSeconds: 120 + containers: + - name: clickhouse + image: clickhouse/clickhouse-server:23.10 + ports: + - containerPort: 8123 + - containerPort: 9000 + - containerPort: 9363 + volumeMounts: + - name: data + mountPath: /var/lib/clickhouse/ + - name: logs + mountPath: /var/log/clickhouse-server/ + - name: config + mountPath: /etc/clickhouse-server/config.d/ + env: + - name: CLICKHOUSE_PASSWORD + valueFrom: + secretKeyRef: + name: secrets + key: CLICKHOUSE_PASSWORD + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: secrets + key: AWS_ACCESS_KEY_ID + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: secrets + key: AWS_SECRET_ACCESS_KEY + volumes: + - name: data + persistentVolumeClaim: + claimName: clickhouse-data-pvc + - name: logs + persistentVolumeClaim: + claimName: clickhouse-logs-pvc + - name: config + configMap: + name: clickhouse-configmap +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: clickhouse-backup + namespace: databases +spec: + schedule: "0 0 * * *" + jobTemplate: + spec: + template: + spec: + restartPolicy: Never + containers: + - name: clickhouse-backup + image: clickhouse/clickhouse-server:23.10 + env: + - name: CLICKHOUSE_PASSWORD + valueFrom: + secretKeyRef: + name: secrets + key: CLICKHOUSE_PASSWORD + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: secrets + key: AWS_ACCESS_KEY_ID + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: secrets + key: AWS_SECRET_ACCESS_KEY + command: + - /bin/sh + - -c + - > + clickhouse client --host clickhouse.databases --user default --password $CLICKHOUSE_PASSWORD --query="BACKUP TABLE nocodelytics_production.events TO S3('https://eu2.contabostorage.com/clickhouse/backup/events/$(date +\%Y-\%m-\%d).zip', '$AWS_ACCESS_KEY_ID', '$AWS_SECRET_ACCESS_KEY');" +--- +apiVersion: v1 +kind: Service +metadata: + name: clickhouse + namespace: databases +spec: + type: ClusterIP + selector: + ms: clickhouse + ports: + - name: http + port: 8123 + targetPort: 8123 + - name: native + port: 9000 + targetPort: 9000 + - name: prometheus + port: 9363 + targetPort: 9363 diff --git a/container-registry.yaml b/container-registry.yaml index e836656..555028c 100644 --- a/container-registry.yaml +++ b/container-registry.yaml @@ -1,72 +1,8 @@ apiVersion: v1 -kind: Namespace -metadata: - name: container-registry ---- -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - name: letsencrypt-prod - namespace: container-registry -spec: - acme: - server: https://acme-v02.api.letsencrypt.org/directory - email: florian@nocodelytics.com - privateKeySecretRef: - name: letsencrypt-prod - solvers: - - http01: - ingress: - class: traefik ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: container-registry-server - namespace: container-registry -spec: - secretName: container-registry-server-net-tls - issuerRef: - name: letsencrypt-prod - kind: Issuer - commonName: container-registry.nocodelytics.com - dnsNames: - - container-registry.nocodelytics.com ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: container-registry-server-pvc - namespace: container-registry -spec: - accessModes: - - ReadWriteOnce - storageClassName: local-path - resources: - requests: - storage: 1Gi ---- -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - name: letsencrypt-prod - namespace: container-registry -spec: - acme: - server: https://acme-v02.api.letsencrypt.org/directory - email: florian@nocodelytics.com - privateKeySecretRef: - name: letsencrypt-prod - solvers: - - http01: - ingress: - class: traefik ---- -apiVersion: v1 kind: ConfigMap metadata: name: container-registry-server-config - namespace: container-registry + namespace: sysadmin data: config.yml: | version: 0.1 @@ -77,8 +13,9 @@ data: cache: blobdescriptor: inmemory s3: - region: eu-west-1 + region: eu bucket: container-registry + regionendpoint: https://eu2.contabostorage.com http: addr: :5000 headers: @@ -88,7 +25,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: container-registry-server - namespace: container-registry + namespace: sysadmin spec: replicas: 1 selector: @@ -103,8 +40,6 @@ spec: - name: container-registry-server image: registry:2 volumeMounts: - - name: volv - mountPath: /var/lib/registry - name: config-volume mountPath: /etc/docker/registry/config.yml subPath: config.yml @@ -128,14 +63,7 @@ spec: secretKeyRef: name: secrets key: AWS_SECRET_ACCESS_KEY - resources: - limits: - memory: "512Mi" - cpu: "100m" volumes: - - name: volv - persistentVolumeClaim: - claimName: container-registry-server-pvc - name: config-volume configMap: name: container-registry-server-config @@ -148,7 +76,7 @@ apiVersion: v1 kind: Service metadata: name: container-registry-server - namespace: container-registry + namespace: sysadmin spec: type: NodePort selector: @@ -157,16 +85,43 @@ spec: - port: 5000 targetPort: 5000 --- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + namespace: sysadmin + name: letsencrypt-prod +spec: + acme: + server: https://acme-v02.api.letsencrypt.org/directory + email: florian@nocodelytics.com + privateKeySecretRef: + name: letsencrypt-prod + solvers: + - http01: + ingress: + class: traefik +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + namespace: sysadmin + name: container-registry +spec: + secretName: container-registry-net-tls + issuerRef: + name: letsencrypt-prod + kind: Issuer + commonName: container-registry.nocodelytics.com + dnsNames: + - container-registry.nocodelytics.com +--- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: + namespace: sysadmin name: container-registry-nginx-ingress - namespace: container-registry annotations: - kubernetes.io/ingress.class: "traefik" - cert-manager.io/issuer: letsencrypt-prod - traefik.ingress.kubernetes.io/redirect-entry-point: https - cert-manager.io/acme-challenge-type: http01 + traefik.ingress.kubernetes.io/router.middlewares: default-https-redirect@kubernetescrd spec: rules: - host: container-registry.nocodelytics.com @@ -189,4 +144,4 @@ spec: tls: - hosts: - container-registry.nocodelytics.com - secretName: container-registry-server-net-tls + secretName: container-registry-net-tls diff --git a/grafana.yaml b/grafana.yaml new file mode 100644 index 0000000..69501fe --- /dev/null +++ b/grafana.yaml @@ -0,0 +1,150 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: sysadmin +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: grafana-pvc + namespace: sysadmin +spec: + accessModes: + - ReadWriteOnce + storageClassName: longhorn + resources: + requests: + storage: 10Gi + limits: + storage: 10Gi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: grafana + namespace: sysadmin +spec: + replicas: 1 + selector: + matchLabels: + ms: grafana + template: + metadata: + labels: + ms: grafana + spec: + securityContext: + fsGroup: 472 + supplementalGroups: + - 0 + containers: + - name: grafana + image: grafana/grafana:latest + imagePullPolicy: IfNotPresent + ports: + - containerPort: 3000 + name: http-grafana + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /robots.txt + port: 3000 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 30 + successThreshold: 1 + timeoutSeconds: 2 + livenessProbe: + failureThreshold: 3 + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + tcpSocket: + port: 3000 + timeoutSeconds: 1 + resources: + requests: + cpu: 250m + memory: 750Mi + volumeMounts: + - mountPath: /var/lib/grafana + name: grafana-pv + volumes: + - name: grafana-pv + persistentVolumeClaim: + claimName: grafana-pvc +--- +apiVersion: v1 +kind: Service +metadata: + name: grafana + namespace: sysadmin +spec: + type: NodePort + selector: + ms: grafana + ports: + - port: 3000 + targetPort: 3000 +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + namespace: sysadmin + name: letsencrypt-prod +spec: + acme: + server: https://acme-v02.api.letsencrypt.org/directory + email: florian@nocodelytics.com + privateKeySecretRef: + name: letsencrypt-prod + solvers: + - http01: + ingress: + class: traefik +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + namespace: sysadmin + name: grafana +spec: + secretName: grafana-net-tls + issuerRef: + name: letsencrypt-prod + kind: Issuer + commonName: grafana.nocodelytics.com + dnsNames: + - grafana.nocodelytics.com +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + namespace: sysadmin + name: grafana-nginx-ingress + annotations: + traefik.ingress.kubernetes.io/router.middlewares: default-https-redirect@kubernetescrd,default-http-auth@kubernetescrd +spec: + rules: + - host: grafana.nocodelytics.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: ssl-redirect + port: + name: use-annotation + - path: / + pathType: Prefix + backend: + service: + name: grafana + port: + number: 3000 + tls: + - hosts: + - grafana.nocodelytics.com + secretName: grafana-net-tls diff --git a/grafana/grafana-server.service b/grafana/grafana-server.service new file mode 100644 index 0000000..e9b6ccb --- /dev/null +++ b/grafana/grafana-server.service @@ -0,0 +1,53 @@ +[Unit] +Description=Grafana instance +Documentation=http://docs.grafana.org +Wants=network-online.target +After=network-online.target +After=postgresql.service mariadb.service mysql.service influxdb.service + +[Service] +EnvironmentFile=/etc/default/grafana-server +User=grafana +Group=grafana +Type=simple +Restart=on-failure +WorkingDirectory=/usr/share/grafana +RuntimeDirectory=grafana +RuntimeDirectoryMode=0750 +ExecStart=/usr/share/grafana/bin/grafana server \ + --config=${CONF_FILE} \ + --pidfile=${PID_FILE_DIR}/grafana-server.pid \ + --packaging=deb \ + cfg:default.paths.logs=${LOG_DIR} \ + cfg:default.paths.data=${DATA_DIR} \ + cfg:default.paths.plugins=${PLUGINS_DIR} \ + cfg:default.paths.provisioning=${PROVISIONING_CFG_DIR} + +LimitNOFILE=10000 +TimeoutStopSec=20 +CapabilityBoundingSet= +DeviceAllow= +LockPersonality=true +MemoryDenyWriteExecute=false +NoNewPrivileges=true +PrivateDevices=true +PrivateTmp=true +ProtectClock=true +ProtectControlGroups=true +ProtectHome=true +ProtectHostname=true +ProtectKernelLogs=true +ProtectKernelModules=true +ProtectKernelTunables=true +ProtectProc=invisible +ProtectSystem=full +RemoveIPC=true +RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX +RestrictNamespaces=true +RestrictRealtime=true +RestrictSUIDSGID=true +SystemCallArchitectures=native +UMask=0027 + +[Install] +WantedBy=multi-user.target diff --git a/grafana/grafana.ini b/grafana/grafana.ini new file mode 100755 index 0000000..87880ec --- /dev/null +++ b/grafana/grafana.ini @@ -0,0 +1,1468 @@ +# +# Everything has defaults so you only need to uncomment things you want to +# change + +# possible values : production, development +;app_mode = production + +# instance name, defaults to HOSTNAME environment variable value or hostname if HOSTNAME var is empty +;instance_name = ${HOSTNAME} + +# force migration will run migrations that might cause dataloss +;force_migration = false + +#################################### Paths #################################### +[paths] +# Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used) +;data = /var/lib/grafana + +# Temporary files in `data` directory older than given duration will be removed +;temp_data_lifetime = 24h + +# Directory where grafana can store logs +;logs = /var/log/grafana + +# Directory where grafana will automatically scan and look for plugins +;plugins = /var/lib/grafana/plugins + +# folder that contains provisioning config files that grafana will apply on startup and while running. +;provisioning = conf/provisioning + +#################################### Server #################################### +[server] +# Protocol (http, https, h2, socket) +;protocol = http + +# The ip address to bind to, empty will bind to all interfaces +;http_addr = + +# The http port to use +http_port = 5478 + +# The public facing domain name used to access grafana from a browser +;domain = localhost + +# Redirect to correct domain if host header does not match domain +# Prevents DNS rebinding attacks +;enforce_domain = false + +# The full public facing url you use in browser, used for redirects and emails +# If you use reverse proxy and sub path specify full url (with sub path) +;root_url = %(protocol)s://%(domain)s:%(http_port)s/ + +# Serve Grafana from subpath specified in `root_url` setting. By default it is set to `false` for compatibility reasons. +;serve_from_sub_path = false + +# Log web requests +;router_logging = false + +# the path relative working path +;static_root_path = public + +# enable gzip +;enable_gzip = false + +# https certs & key file +;cert_file = +;cert_key = + +# Unix socket gid +# Changing the gid of a file without privileges requires that the target group is in the group of the process and that the process is the file owner +# It is recommended to set the gid as http server user gid +# Not set when the value is -1 +;socket_gid = + +# Unix socket mode +;socket_mode = + +# Unix socket path +;socket = + +# CDN Url +;cdn_url = + +# Sets the maximum time using a duration format (5s/5m/5ms) before timing out read of an incoming request and closing idle connections. +# `0` means there is no timeout for reading the request. +;read_timeout = 0 + +# This setting enables you to specify additional headers that the server adds to HTTP(S) responses. +[server.custom_response_headers] +#exampleHeader1 = exampleValue1 +#exampleHeader2 = exampleValue2 + +#################################### GRPC Server ######################### +;[grpc_server] +;network = "tcp" +;address = "127.0.0.1:10000" +;use_tls = false +;cert_file = +;key_file = + +#################################### Database #################################### +[database] +# You can configure the database connection by specifying type, host, name, user and password +# as separate properties or as on string using the url properties. + +# Either "mysql", "postgres" or "sqlite3", it's your choice +type = postgres +host = 127.0.0.1:6389 +name = grafana +user = grafana +# If the password contains # or ; you have to wrap it with triple quotes. Ex """#password;""" +password = EYxv7Gy9O981 + +# Use either URL or the previous fields to configure the database +# Example: mysql://user:secret@host:port/database +;url = + +# For "postgres", use either "disable", "require" or "verify-full" +# For "mysql", use either "true", "false", or "skip-verify". +;ssl_mode = disable + +# Database drivers may support different transaction isolation levels. +# Currently, only "mysql" driver supports isolation levels. +# If the value is empty - driver's default isolation level is applied. +# For "mysql" use "READ-UNCOMMITTED", "READ-COMMITTED", "REPEATABLE-READ" or "SERIALIZABLE". +;isolation_level = + +;ca_cert_path = +;client_key_path = +;client_cert_path = +;server_cert_name = + +# For "sqlite3" only, path relative to data_path setting +;path = grafana.db + +# Max idle conn setting default is 2 +;max_idle_conn = 2 + +# Max conn setting default is 0 (mean not set) +;max_open_conn = + +# Connection Max Lifetime default is 14400 (means 14400 seconds or 4 hours) +;conn_max_lifetime = 14400 + +# Set to true to log the sql calls and execution times. +;log_queries = + +# For "sqlite3" only. cache mode setting used for connecting to the database. (private, shared) +;cache_mode = private + +# For "sqlite3" only. Enable/disable Write-Ahead Logging, https://sqlite.org/wal.html. Default is false. +;wal = false + +# For "mysql" only if migrationLocking feature toggle is set. How many seconds to wait before failing to lock the database for the migrations, default is 0. +;locking_attempt_timeout_sec = 0 + +# For "sqlite" only. How many times to retry query in case of database is locked failures. Default is 0 (disabled). +;query_retries = 0 + +# For "sqlite" only. How many times to retry transaction in case of database is locked failures. Default is 5. +;transaction_retries = 5 + +# Set to true to add metrics and tracing for database queries. +;instrument_queries = false + +################################### Data sources ######################### +[datasources] +# Upper limit of data sources that Grafana will return. This limit is a temporary configuration and it will be deprecated when pagination will be introduced on the list data sources API. +;datasource_limit = 5000 + +#################################### Cache server ############################# +[remote_cache] +# Either "redis", "memcached" or "database" default is "database" +;type = database + +# cache connectionstring options +# database: will use Grafana primary database. +# redis: config like redis server e.g. `addr=127.0.0.1:6379,pool_size=100,db=0,ssl=false`. Only addr is required. ssl may be 'true', 'false', or 'insecure'. +# memcache: 127.0.0.1:11211 +;connstr = + +# prefix prepended to all the keys in the remote cache +; prefix = + +# This enables encryption of values stored in the remote cache +;encryption = + +#################################### Data proxy ########################### +[dataproxy] + +# This enables data proxy logging, default is false +;logging = false + +# How long the data proxy waits to read the headers of the response before timing out, default is 30 seconds. +# This setting also applies to core backend HTTP data sources where query requests use an HTTP client with timeout set. +;timeout = 30 + +# How long the data proxy waits to establish a TCP connection before timing out, default is 10 seconds. +;dialTimeout = 10 + +# How many seconds the data proxy waits before sending a keepalive probe request. +;keep_alive_seconds = 30 + +# How many seconds the data proxy waits for a successful TLS Handshake before timing out. +;tls_handshake_timeout_seconds = 10 + +# How many seconds the data proxy will wait for a server's first response headers after +# fully writing the request headers if the request has an "Expect: 100-continue" +# header. A value of 0 will result in the body being sent immediately, without +# waiting for the server to approve. +;expect_continue_timeout_seconds = 1 + +# Optionally limits the total number of connections per host, including connections in the dialing, +# active, and idle states. On limit violation, dials will block. +# A value of zero (0) means no limit. +;max_conns_per_host = 0 + +# The maximum number of idle connections that Grafana will keep alive. +;max_idle_connections = 100 + +# How many seconds the data proxy keeps an idle connection open before timing out. +;idle_conn_timeout_seconds = 90 + +# If enabled and user is not anonymous, data proxy will add X-Grafana-User header with username into the request, default is false. +;send_user_header = false + +# Limit the amount of bytes that will be read/accepted from responses of outgoing HTTP requests. +;response_limit = 0 + +# Limits the number of rows that Grafana will process from SQL data sources. +;row_limit = 1000000 + +# Sets a custom value for the `User-Agent` header for outgoing data proxy requests. If empty, the default value is `Grafana/` (for example `Grafana/9.0.0`). +;user_agent = + +#################################### Analytics #################################### +[analytics] +# Server reporting, sends usage counters to stats.grafana.org every 24 hours. +# No ip addresses are being tracked, only simple counters to track +# running instances, dashboard and error counts. It is very helpful to us. +# Change this option to false to disable reporting. +;reporting_enabled = true + +# The name of the distributor of the Grafana instance. Ex hosted-grafana, grafana-labs +;reporting_distributor = grafana-labs + +# Set to false to disable all checks to https://grafana.com +# for new versions of grafana. The check is used +# in some UI views to notify that a grafana update exists. +# This option does not cause any auto updates, nor send any information +# only a GET request to https://raw.githubusercontent.com/grafana/grafana/main/latest.json to get the latest version. +;check_for_updates = true + +# Set to false to disable all checks to https://grafana.com +# for new versions of plugins. The check is used +# in some UI views to notify that a plugin update exists. +# This option does not cause any auto updates, nor send any information +# only a GET request to https://grafana.com to get the latest versions. +;check_for_plugin_updates = true + +# Google Analytics universal tracking code, only enabled if you specify an id here +;google_analytics_ua_id = + +# Google Analytics 4 tracking code, only enabled if you specify an id here +;google_analytics_4_id = + +# When Google Analytics 4 Enhanced event measurement is enabled, we will try to avoid sending duplicate events and let Google Analytics 4 detect navigation changes, etc. +;google_analytics_4_send_manual_page_views = false + +# Google Tag Manager ID, only enabled if you specify an id here +;google_tag_manager_id = + +# Rudderstack write key, enabled only if rudderstack_data_plane_url is also set +;rudderstack_write_key = + +# Rudderstack data plane url, enabled only if rudderstack_write_key is also set +;rudderstack_data_plane_url = + +# Rudderstack SDK url, optional, only valid if rudderstack_write_key and rudderstack_data_plane_url is also set +;rudderstack_sdk_url = + +# Rudderstack Config url, optional, used by Rudderstack SDK to fetch source config +;rudderstack_config_url = + +# Intercom secret, optional, used to hash user_id before passing to Intercom via Rudderstack +;intercom_secret = + +# Controls if the UI contains any links to user feedback forms +;feedback_links_enabled = true + +#################################### Security #################################### +[security] +# disable creation of admin user on first start of grafana +;disable_initial_admin_creation = false + +# default admin user, created on startup +;admin_user = admin + +# default admin password, can be changed before first start of grafana, or in profile settings +;admin_password = admin + +# default admin email, created on startup +;admin_email = admin@localhost + +# used for signing +;secret_key = SW2YcwTIb9zpOOhoPsMm + +# current key provider used for envelope encryption, default to static value specified by secret_key +;encryption_provider = secretKey.v1 + +# list of configured key providers, space separated (Enterprise only): e.g., awskms.v1 azurekv.v1 +;available_encryption_providers = + +# disable gravatar profile images +;disable_gravatar = false + +# data source proxy whitelist (ip_or_domain:port separated by spaces) +;data_source_proxy_whitelist = + +# disable protection against brute force login attempts +;disable_brute_force_login_protection = false + +# set to true if you host Grafana behind HTTPS. default is false. +;cookie_secure = false + +# set cookie SameSite attribute. defaults to `lax`. can be set to "lax", "strict", "none" and "disabled" +;cookie_samesite = lax + +# set to true if you want to allow browsers to render Grafana in a ,