working system

databases/clickhouse.yaml

@@ -72,6 +72,9 @@ spec:
     metadata:
       labels:
         ms: clickhouse
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "9363"
     spec:
       terminationGracePeriodSeconds: 120
       containers:

databases/nats.yaml

@@ -43,6 +43,8 @@ metadata:
   namespace: databases
 spec:
   replicas: 1
+  strategy:
+    type: Recreate
   selector:
     matchLabels:
       ms: nats
@@ -51,6 +53,7 @@ spec:
       labels:
         ms: nats
     spec:
+      terminationGracePeriodSeconds: 120
       containers:
         - name: nats
           image: nats:2.10.5
@@ -105,6 +108,9 @@ spec:
     metadata:
       labels:
         ms: nats-exporter
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "7777"
     spec:
       containers:
         - name: nats-exporter
@@ -121,16 +127,3 @@ spec:
             - http://nats.databases:8222
           ports:
             - containerPort: 7777
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: nats-exporter
-  namespace: databases
-spec:
-  type: NodePort
-  selector:
-    ms: nats-exporter
-  ports:
-    - port: 7777
-      targetPort: 7777

databases/postgres.sql

@@ -0,0 +1,6 @@
+create database nocodelytics_staging;
+create database nocodelytics_production;
+create database grafana;
+
+CREATE EXTENSION IF NOT EXISTS pg_stat_statements;
+CREATE EXTENSION IF NOT EXISTS pg_buffercache;

databases/postgres.yaml

@@ -14,14 +14,13 @@ spec:
       storage: 10Gi
 ---
 apiVersion: apps/v1
-kind: Deployment
+kind: StatefulSet
 metadata:
   name: postgres
   namespace: databases
 spec:
+  serviceName: "postgres"
   replicas: 1
-  strategy:
-    type: Recreate
   selector:
     matchLabels:
       name: postgres
@@ -89,6 +88,9 @@ spec:
     metadata:
       labels:
         name: postgres-exporter
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "9187"
     spec:
       containers:
         - name: postgres-exporter
@@ -104,19 +106,6 @@ spec:
             - name: DATA_SOURCE_NAME
               value: postgresql://postgres:$(POSTGRES_PASSWORD)@postgres.databases:5432/postgres?sslmode=disable
 ---
-apiVersion: v1
-kind: Service
-metadata:
-  name: postgres-exporter
-  namespace: databases
-spec:
-  type: NodePort
-  selector:
-    name: postgres-exporter
-  ports:
-    - port: 9187
-      targetPort: 9187
----
 apiVersion: batch/v1
 kind: CronJob
 metadata:
@@ -135,55 +124,18 @@ spec:
         spec:
           restartPolicy: Never
           containers:
-            - name: test-container
+            - name: postgres-backup
-              image: container-registry.nocodelytics.com/postgres-s3
+              image: localhost:30007/postgres-s3
               command:
                 - /bin/sh
                 - -c
                 - >
                   pg_dump -U postgres -h postgres.databases nocodelytics_production | gzip > /backup/nocodelytics_production_$(date +'%Y-%m-%d').sql.gzip &&
-                  rclone copy /backup/nocodelytics_production_$(date '+%Y-%m-%d').sql.gzip contabo:postgres
+                  rclone copy /backup/nocodelytics_production_$(date '+%Y-%m-%d').sql.gzip contabo:postgres &&
-              env:
+                  rm /backup/nocodelytics_production_$(date '+%Y-%m-%d').sql.gzip &&
-                - name: PGPASSWORD
+                  pg_dump -U postgres -h postgres.databases grafana | gzip > /backup/grafana_$(date +'%Y-%m-%d').sql.gzip &&
-                  valueFrom:
+                  rclone copy /backup/grafana_$(date '+%Y-%m-%d').sql.gzip contabo:postgres &&
-                    secretKeyRef:
+                  rm /backup/grafana_$(date '+%Y-%m-%d').sql.gzip
-                      name: secrets
-                      key: POSTGRES_PASSWORD
-                - name: RCLONE_CONFIG_CONTABO_TYPE
-                  value: "s3"
-                - name: RCLONE_CONFIG_CONTABO_PROVIDER
-                  value: "Other"
-                - name: RCLONE_CONFIG_CONTABO_ENV_AUTH
-                  value: "false"
-                - name: RCLONE_CONFIG_CONTABO_ENDPOINT
-                  value: "https://eu2.contabostorage.com"
-                - name: RCLONE_CONFIG_CONTABO_ACCESS_KEY_ID
-                  valueFrom:
-                    secretKeyRef:
-                      name: secrets
-                      key: AWS_ACCESS_KEY_ID
-                - name: RCLONE_CONFIG_CONTABO_SECRET_ACCESS_KEY
-                  valueFrom:
-                    secretKeyRef:
-                      name: secrets
-                      key: AWS_SECRET_ACCESS_KEY
-              volumeMounts:
-                - mountPath: /backup
-                  name: backup-volume
-          volumes:
-            - name: backup-volume
-              emptyDir: {}
----
-apiVersion: v1
-kind: Pod
-metadata:
-  name: test-pod
-  namespace: databases
-spec:
-  containers:
-    - name: test-container
-      image: container-registry.nocodelytics.com/postgres-s3
-      command: ["sh", "-c", "echo Image pulled successfully && sleep 3600"]
               env:
                 - name: PGPASSWORD
                   valueFrom:

dependencies/00-longhorn.yaml

@@ -4388,3 +4388,7 @@ spec:
           namespace: default
         - name: https-redirect
           namespace: default
+  tls:
+    certResolver: letsencrypt
+    domains:
+      - main: longhorn.nocodelytics.com

dependencies/01-namespaces.yml

@@ -2,3 +2,8 @@ apiVersion: v1
 kind: Namespace
 metadata:
   name: sysadmin
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: databases

dependencies/kube-state-metrics.yaml

@@ -0,0 +1,215 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    app.kubernetes.io/component: exporter
+    app.kubernetes.io/name: kube-state-metrics
+    app.kubernetes.io/version: 2.10.1
+  name: kube-state-metrics
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - configmaps
+      - secrets
+      - nodes
+      - pods
+      - services
+      - serviceaccounts
+      - resourcequotas
+      - replicationcontrollers
+      - limitranges
+      - persistentvolumeclaims
+      - persistentvolumes
+      - namespaces
+      - endpoints
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+      - apps
+    resources:
+      - statefulsets
+      - daemonsets
+      - deployments
+      - replicasets
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+      - batch
+    resources:
+      - cronjobs
+      - jobs
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+      - autoscaling
+    resources:
+      - horizontalpodautoscalers
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+      - authentication.k8s.io
+    resources:
+      - tokenreviews
+    verbs:
+      - create
+  - apiGroups:
+      - authorization.k8s.io
+    resources:
+      - subjectaccessreviews
+    verbs:
+      - create
+  - apiGroups:
+      - policy
+    resources:
+      - poddisruptionbudgets
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+      - certificates.k8s.io
+    resources:
+      - certificatesigningrequests
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+      - discovery.k8s.io
+    resources:
+      - endpointslices
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+      - storage.k8s.io
+    resources:
+      - storageclasses
+      - volumeattachments
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+      - admissionregistration.k8s.io
+    resources:
+      - mutatingwebhookconfigurations
+      - validatingwebhookconfigurations
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+      - networking.k8s.io
+    resources:
+      - networkpolicies
+      - ingressclasses
+      - ingresses
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+      - coordination.k8s.io
+    resources:
+      - leases
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+      - rbac.authorization.k8s.io
+    resources:
+      - clusterrolebindings
+      - clusterroles
+      - rolebindings
+      - roles
+    verbs:
+      - list
+      - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    app.kubernetes.io/component: exporter
+    app.kubernetes.io/name: kube-state-metrics
+    app.kubernetes.io/version: 2.10.1
+  name: kube-state-metrics
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kube-state-metrics
+subjects:
+  - kind: ServiceAccount
+    name: kube-state-metrics
+    namespace: kube-system
+---
+apiVersion: v1
+automountServiceAccountToken: false
+kind: ServiceAccount
+metadata:
+  labels:
+    app.kubernetes.io/component: exporter
+    app.kubernetes.io/name: kube-state-metrics
+    app.kubernetes.io/version: 2.10.1
+  name: kube-state-metrics
+  namespace: kube-system
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app.kubernetes.io/component: exporter
+    app.kubernetes.io/name: kube-state-metrics
+    app.kubernetes.io/version: 2.10.1
+  name: kube-state-metrics
+  namespace: kube-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: kube-state-metrics
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/component: exporter
+        app.kubernetes.io/name: kube-state-metrics
+        app.kubernetes.io/version: 2.10.1
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "8080"
+    spec:
+      automountServiceAccountToken: true
+      containers:
+        - image: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.10.1
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: 8080
+            initialDelaySeconds: 5
+            timeoutSeconds: 5
+          name: kube-state-metrics
+          ports:
+            - containerPort: 8080
+              name: http-metrics
+            - containerPort: 8081
+              name: telemetry
+          readinessProbe:
+            httpGet:
+              path: /
+              port: 8081
+            initialDelaySeconds: 5
+            timeoutSeconds: 5
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - ALL
+            readOnlyRootFilesystem: true
+            runAsNonRoot: true
+            runAsUser: 65534
+            seccompProfile:
+              type: RuntimeDefault
+      nodeSelector:
+        kubernetes.io/os: linux
+      serviceAccountName: kube-state-metrics

dependencies/node-exporter.yaml

@@ -0,0 +1,51 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  labels:
+    app.kubernetes.io/component: exporter
+    app.kubernetes.io/name: node-exporter
+  name: node-exporter
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/component: exporter
+      app.kubernetes.io/name: node-exporter
+  template:
+    metadata:
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "9100"
+      labels:
+        app.kubernetes.io/component: exporter
+        app.kubernetes.io/name: node-exporter
+    spec:
+      containers:
+        - args:
+            - --path.sysfs=/host/sys
+            - --path.rootfs=/host/root
+            - --no-collector.wifi
+            - --no-collector.hwmon
+            - --collector.filesystem.ignored-mount-points=^/(dev|proc|sys|var/lib/docker/.+|var/lib/kubelet/pods/.+)($|/)
+            - --collector.netclass.ignored-devices=^(veth.*)$
+          name: node-exporter
+          image: prom/node-exporter
+          ports:
+            - containerPort: 9100
+              protocol: TCP
+          volumeMounts:
+            - mountPath: /host/sys
+              mountPropagation: HostToContainer
+              name: sys
+              readOnly: true
+            - mountPath: /host/root
+              mountPropagation: HostToContainer
+              name: root
+              readOnly: true
+      volumes:
+        - hostPath:
+            path: /sys
+          name: sys
+        - hostPath:
+            path: /
+          name: root

dependencies/traefik-config.yaml

@@ -7,13 +7,23 @@ metadata:
 spec:
   valuesContent: |-
     deployment:
-      - name: volume-permissions
-        image: busybox:latest
-        command: ["sh", "-c", "touch /data/acme.json; chmod -v 600 /data/acme.json"]
       securityContext:
-          runAsNonRoot: true
+        runAsNonRoot: false
-          runAsGroup: 65532
+        runAsGroup: 0
-          runAsUser: 65532
+        runAsUser: 0
+      initContainers:
+        - name: volume-permissions
+          image: traefik
+          command:
+            [
+              "sh",
+              "-c",
+              "touch /data/acme.json; chown 65532 /data/acme.json; chmod -v 600 /data/acme.json",
+            ]
+          securityContext:
+            runAsNonRoot: false
+            runAsGroup: 0
+            runAsUser: 0
           volumeMounts:
             - name: traefik
               mountPath: /data

sysadmin/00-container-registry.yaml

@@ -90,6 +90,7 @@ spec:
     name: container-registry-server
   ports:
     - port: 5000
+      nodePort: 30007
       targetPort: 5000
 ---
 apiVersion: traefik.containo.us/v1alpha1
@@ -101,7 +102,7 @@ spec:
   entryPoints:
     - websecure
   routes:
-    - match: Host(`container-registry-server.nocodelytics.com`)
+    - match: Host(`container-registry.nocodelytics.com`)
       kind: Rule
       services:
         - name: container-registry-server
@@ -111,3 +112,7 @@ spec:
           namespace: default
         - name: https-redirect
           namespace: default
+  tls:
+    certResolver: letsencrypt
+    domains:
+      - main: container-registry.nocodelytics.com

sysadmin/grafana.yaml

@@ -13,6 +13,25 @@ spec:
     limits:
       storage: 10Gi
 ---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: grafana-configmap
+  namespace: sysadmin
+data:
+  defaults.ini: |
+    instance_name = grafana
+
+    [security]
+    admin_user = admin
+    admin_password = ${HTTP_SECRET}
+
+    [database]
+    type: postgres
+    host: postgres.databases
+    user: postgres
+    password: ${POSTGRES_PASSWORD}
+---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -27,6 +46,9 @@ spec:
     metadata:
       labels:
         ms: grafana
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "3000"
     spec:
       securityContext:
         fsGroup: 472
@@ -59,12 +81,29 @@ spec:
               port: 3000
             timeoutSeconds: 1
           volumeMounts:
-            - mountPath: /var/lib/grafana
+            - name: grafana-pv
-              name: grafana-pv
+              mountPath: /var/lib/grafana
+            - name: grafana-config
+              mountPath: /etc/grafana/grafana.ini
+              subPath: defaults.ini
+          env:
+            - name: POSTGRES_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: secrets
+                  key: POSTGRES_PASSWORD
+            - name: HTTP_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: secrets
+                  key: HTTP_SECRET
       volumes:
         - name: grafana-pv
           persistentVolumeClaim:
             claimName: grafana-pvc
+        - name: grafana-config
+          configMap:
+            name: grafana-configmap
 ---
 apiVersion: v1
 kind: Service

sysadmin/k8s-dashboard.yml

@@ -315,3 +315,7 @@ spec:
       middlewares:
         - name: https-redirect
           namespace: default
+  tls:
+    certResolver: letsencrypt
+    domains:
+      - main: k3s-dashboard.nocodelytics.com

sysadmin/longhorn.yaml

@@ -1,14 +0,0 @@
-apiVersion: traefik.containo.us/v1alpha1
-kind: IngressRoute
-metadata:
-  name: longhorn-frontend
-  namespace: longhorn-system
-spec:
-  entryPoints:
-    - websecure
-  routes:
-    - match: Host(`longhorn.nocodelytics.com`)
-      kind: Rule
-      services:
-        - name: longhorn-frontend
-          port: 80

sysadmin/prometheus-deployment.yaml

@@ -53,21 +53,6 @@ data:
             regex: ([^:]+)(?::\d+)?;(\d+)
             replacement: $1:$2
             target_label: __address__
-      - job_name: "node_exporter"
-        static_configs:
-          - targets: ["144.76.186.182:9100"]
-      - job_name: "postgres_exporter"
-        static_configs:
-          - targets: ["postgres-exporter.databases:9187"]
-      - job_name: "clickhouse_exporter"
-        static_configs:
-          - targets: ["clickhouse.databases:9363"]
-      - job_name: "nats_exporter"
-        static_configs:
-          - targets: ["nats-exporter.databases:7777"]
-      - job_name: "kube_exporter"
-        static_configs:
-          - targets: ["kube-state-metrics.kube-system.svc.cluster.local:8080"]
       - job_name: "kubernetes-cadvisor"
         scheme: https
         kubernetes_sd_configs:
@@ -100,9 +85,6 @@ data:
             regex: (.+)
             target_label: __metrics_path__
             replacement: /api/v1/nodes/${1}/proxy/metrics
-      - job_name: "longhorn_exporter"
-        static_configs:
-          - targets: ["longhorn-backend.longhorn-system:9500"]
 
 ---
 apiVersion: v1
@@ -134,6 +116,9 @@ spec:
     metadata:
       labels:
         ms: prometheus
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "9090"
     spec:
       containers:
         - name: prometheus