checkpoint

.gitignore

@@ -2,3 +2,4 @@
 kustomization/bases/charts
 *.tar.gz
 .DS_Store
+secrets/secrets.yaml

.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+  "yaml.schemas": {
+    "kubernetes://schema/traefik.containo.us/v1alpha1%40middleware": "file:///Users/florian/projects/nocodelytics-infrastructure/dependencies/traefik.yaml"
+  }
+}

README.md

@@ -1 +1,33 @@
 # Infrastructure
+
+## Backups
+
+### Longhorn
+
+```
+apt-get -y install open-iscsi nfs-common jq
+curl -sSfL https://raw.githubusercontent.com/longhorn/longhorn/v1.5.3/scripts/environment_check.sh | bash
+```
+
+### Velero
+
+```
+velero install \
+    --use-node-agent \
+    --privileged-node-agent \
+    --uploader-type=restic \
+    --features=EnableCSI \
+    --provider aws \
+    --plugins velero/velero-plugin-for-aws:v1.2.1 \
+    --bucket velero \
+    --secret-file ./secrets/credentials-velero \
+    --use-volume-snapshots=true \
+    --backup-location-config region=eu,s3ForcePathStyle="true",s3Url=https://eu2.contabostorage.com \
+    --wait
+```
+
+If there's an issue with the credentials:
+
+```
+kubectl create secret generic cloud-credentials --namespace velero --from-file=cloud=./secrets/credentials-velero --dry-run=client -o yaml | kubectl apply -f -
+```

databases/clickhouse.yaml

@@ -1,9 +1,4 @@
 apiVersion: v1
-kind: Namespace
-metadata:
-  name: databases
----
-apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
   name: clickhouse-data-pvc

databases/nats.yaml

@@ -1,9 +1,4 @@
 apiVersion: v1
-kind: Namespace
-metadata:
-  name: databases
----
-apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
   name: nats-pvc

databases/postgres.yaml

@@ -1,9 +1,4 @@
 apiVersion: v1
-kind: Namespace
-metadata:
-  name: databases
----
-apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
   name: postgres-pvc
@@ -29,11 +24,11 @@ spec:
     type: Recreate
   selector:
     matchLabels:
-      ms: postgres
+      name: postgres
   template:
     metadata:
       labels:
-        ms: postgres
+        name: postgres
     spec:
       terminationGracePeriodSeconds: 120
       containers:
@@ -75,7 +70,7 @@ metadata:
 spec:
   type: NodePort
   selector:
-    ms: postgres
+    name: postgres
   ports:
     - port: 5432
       targetPort: 5432
@@ -89,11 +84,11 @@ spec:
   replicas: 1
   selector:
     matchLabels:
-      ms: postgres-exporter
+      name: postgres-exporter
   template:
     metadata:
       labels:
-        ms: postgres-exporter
+        name: postgres-exporter
     spec:
       containers:
         - name: postgres-exporter
@@ -117,7 +112,7 @@ metadata:
 spec:
   type: NodePort
   selector:
-    ms: postgres-exporter
+    name: postgres-exporter
   ports:
     - port: 9187
       targetPort: 9187

dependencies/01-namespaces.yml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: sysadmin

dependencies/system-upgrade-controller.yaml

@@ -0,0 +1,132 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  labels:
+    pod-security.kubernetes.io/enforce: privileged
+  name: system-upgrade
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: system-upgrade
+  namespace: system-upgrade
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: system-upgrade
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+- kind: ServiceAccount
+  name: system-upgrade
+  namespace: system-upgrade
+---
+apiVersion: v1
+data:
+  SYSTEM_UPGRADE_CONTROLLER_DEBUG: "false"
+  SYSTEM_UPGRADE_CONTROLLER_THREADS: "2"
+  SYSTEM_UPGRADE_JOB_ACTIVE_DEADLINE_SECONDS: "900"
+  SYSTEM_UPGRADE_JOB_BACKOFF_LIMIT: "99"
+  SYSTEM_UPGRADE_JOB_IMAGE_PULL_POLICY: Always
+  SYSTEM_UPGRADE_JOB_KUBECTL_IMAGE: rancher/kubectl:v1.25.4
+  SYSTEM_UPGRADE_JOB_PRIVILEGED: "true"
+  SYSTEM_UPGRADE_JOB_TTL_SECONDS_AFTER_FINISH: "900"
+  SYSTEM_UPGRADE_PLAN_POLLING_INTERVAL: 15m
+kind: ConfigMap
+metadata:
+  name: default-controller-env
+  namespace: system-upgrade
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: system-upgrade-controller
+  namespace: system-upgrade
+spec:
+  selector:
+    matchLabels:
+      upgrade.cattle.io/controller: system-upgrade-controller
+  template:
+    metadata:
+      labels:
+        upgrade.cattle.io/controller: system-upgrade-controller
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: node-role.kubernetes.io/control-plane
+                operator: Exists
+      containers:
+      - env:
+        - name: SYSTEM_UPGRADE_CONTROLLER_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.labels['upgrade.cattle.io/controller']
+        - name: SYSTEM_UPGRADE_CONTROLLER_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        envFrom:
+        - configMapRef:
+            name: default-controller-env
+        image: rancher/system-upgrade-controller:v0.13.2
+        imagePullPolicy: IfNotPresent
+        name: system-upgrade-controller
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - ALL
+          runAsGroup: 65534
+          runAsNonRoot: true
+          runAsUser: 65534
+          seccompProfile:
+            type: RuntimeDefault
+        volumeMounts:
+        - mountPath: /etc/ssl
+          name: etc-ssl
+          readOnly: true
+        - mountPath: /etc/pki
+          name: etc-pki
+          readOnly: true
+        - mountPath: /etc/ca-certificates
+          name: etc-ca-certificates
+          readOnly: true
+        - mountPath: /tmp
+          name: tmp
+      serviceAccountName: system-upgrade
+      tolerations:
+      - key: CriticalAddonsOnly
+        operator: Exists
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/master
+        operator: Exists
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/controlplane
+        operator: Exists
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/control-plane
+        operator: Exists
+      - effect: NoExecute
+        key: node-role.kubernetes.io/etcd
+        operator: Exists
+      volumes:
+      - hostPath:
+          path: /etc/ssl
+          type: DirectoryOrCreate
+        name: etc-ssl
+      - hostPath:
+          path: /etc/pki
+          type: DirectoryOrCreate
+        name: etc-pki
+      - hostPath:
+          path: /etc/ca-certificates
+          type: DirectoryOrCreate
+        name: etc-ca-certificates
+      - emptyDir: {}
+        name: tmp

dependencies/traefik-config.yaml

@@ -0,0 +1,37 @@
+# scp dependencies/traefik-config.yaml root@[ip]:/var/lib/rancher/k3s/server/manifests/traefik-config.yaml
+apiVersion: helm.cattle.io/v1
+kind: HelmChartConfig
+metadata:
+  name: traefik
+  namespace: kube-system
+spec:
+  valuesContent: |-
+    deployment:
+      - name: volume-permissions
+        image: busybox:latest
+        command: ["sh", "-c", "touch /data/acme.json; chmod -v 600 /data/acme.json"]
+        securityContext:
+          runAsNonRoot: true
+          runAsGroup: 65532
+          runAsUser: 65532
+        volumeMounts:
+          - name: traefik
+            mountPath: /data
+    providers:
+      kubernetesCRD:
+        allowCrossNamespace: true
+    persistence:
+      enabled: true
+      name: traefik
+      accessMode: ReadWriteOnce
+      size: 128Mi
+      storageClass: longhorn
+      path: /data
+      annotations:
+        app: traefik
+    certResolvers:
+      letsencrypt:
+        email: help@nocodelytics.com
+        httpChallenge:
+          entryPoint: "web"
+        storage: /data/acme.json

dependencies/traefik.yaml

@@ -1,5 +1,16 @@
 apiVersion: traefik.containo.us/v1alpha1
 kind: Middleware
+metadata:
+  name: websocket-middleware
+  namespace: default
+spec:
+  headers:
+    customRequestHeaders:
+      Connection: Upgrade
+      Upgrade: websocket
+---
+apiVersion: traefik.containo.us/v1alpha1
+kind: Middleware
 metadata:
   name: https-redirect
   namespace: default

dependencies/velero.yaml

@@ -0,0 +1,16 @@
+apiVersion: velero.io/v1
+kind: BackupStorageLocation
+metadata:
+  name: default
+  namespace: velero
+spec:
+  backupSyncPeriod: 2m0s
+  provider: aws
+  objectStorage:
+    bucket: velero
+  credential:
+    name: cloud-credentials
+    key: cloud
+  config:
+    region: eu
+    profile: "default"

kuma.yml

@@ -1,126 +0,0 @@
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: kuma-pvc-2
-  namespace: default
-spec:
-  accessModes:
-    - ReadWriteOnce
-  storageClassName: local-path
-  resources:
-    requests:
-      storage: 1Gi
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: kuma
-  namespace: default
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      ms: kuma
-  template:
-    metadata:
-      labels:
-        ms: kuma
-    spec:
-      containers:
-        - name: kuma
-          image: louislam/uptime-kuma
-          volumeMounts:
-            - name: volv
-              mountPath: /app/data
-          # resources:
-          #   limits:
-          #     memory: "512Mi"
-          #     cpu: "100m"
-      volumes:
-        - name: volv
-          persistentVolumeClaim:
-            claimName: kuma-pvc-2
-      nodeSelector:
-        kubernetes.io/arch: arm64
-      tolerations:
-        - key: "arch"
-          operator: "Equal"
-          value: "arm64"
-          effect: "NoSchedule"
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: kuma
-  namespace: default
-spec:
-  type: NodePort
-  selector:
-    ms: kuma
-  ports:
-    - port: 3001
-      targetPort: 3001
----
-apiVersion: cert-manager.io/v1
-kind: Issuer
-metadata:
-  name: kuma-letsencrypt-prod
-  namespace: default
-spec:
-  acme:
-    server: https://acme-v02.api.letsencrypt.org/directory
-    email: florian@nocodelytics.com
-    privateKeySecretRef:
-      name: kuma-letsencrypt-prod
-    solvers:
-      - http01:
-          ingress:
-            class: traefik
----
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  name: kuma
-  namespace: default
-spec:
-  secretName: kuma-net-tls
-  issuerRef:
-    name: kuma-letsencrypt-prod
-    kind: Issuer
-  commonName: status.nocodelytics.com
-  dnsNames:
-    - status.nocodelytics.com
----
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: kuma-nginx-ingress
-  namespace: default
-  annotations:
-    kubernetes.io/ingress.class: "traefik"
-    cert-manager.io/issuer: letsencrypt-prod
-    traefik.ingress.kubernetes.io/redirect-entry-point: https
-    cert-manager.io/acme-challenge-type: http01
-spec:
-  rules:
-    - host: status.nocodelytics.com
-      http:
-        paths:
-          - path: /
-            pathType: Prefix
-            backend:
-              service:
-                name: ssl-redirect
-                port:
-                  name: use-annotation
-          - path: /
-            pathType: Prefix
-            backend:
-              service:
-                name: kuma
-                port:
-                  number: 3001
-  tls:
-    - hosts:
-        - status.nocodelytics.com
-      secretName: kuma-net-tls

longhorn.yaml

@@ -1,62 +0,0 @@
----
-apiVersion: cert-manager.io/v1
-kind: Issuer
-metadata:
-  namespace: longhorn-system
-  name: letsencrypt-prod
-spec:
-  acme:
-    server: https://acme-v02.api.letsencrypt.org/directory
-    email: florian@nocodelytics.com
-    privateKeySecretRef:
-      name: letsencrypt-prod
-    solvers:
-      - http01:
-          ingress:
-            class: traefik
----
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  namespace: longhorn-system
-  name: longhorn-system
-spec:
-  secretName: longhorn-system-net-tls
-  issuerRef:
-    name: letsencrypt-prod
-    kind: Issuer
-  commonName: longhorn.nocodelytics.com
-  dnsNames:
-    - longhorn.nocodelytics.com
----
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  namespace: longhorn-system
-  name: longhorn-system-nginx-ingress
-  annotations:
-    traefik.ingress.kubernetes.io/router.middlewares: default-https-redirect@kubernetescrd,default-http-auth@kubernetescrd
-
-spec:
-  rules:
-    - host: longhorn.nocodelytics.com
-      http:
-        paths:
-          - path: /
-            pathType: Prefix
-            backend:
-              service:
-                name: ssl-redirect
-                port:
-                  name: use-annotation
-          - path: /
-            pathType: Prefix
-            backend:
-              service:
-                name: longhorn-frontend
-                port:
-                  number: 80
-  tls:
-    - hosts:
-        - longhorn.nocodelytics.com
-      secretName: longhorn-system-net-tls

secrets.yaml

@@ -1,9 +0,0 @@
-apiVersion: v1
-kind: Secret
-type: Opaque
-metadata:
-  name: secrets
-  namespace: container-registry
-data:
-  AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
-  AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}

sysadmin/00-container-registry.yaml

@@ -28,13 +28,15 @@ metadata:
   namespace: sysadmin
 spec:
   replicas: 1
+  strategy:
+    type: Recreate
   selector:
     matchLabels:
-      ms: container-registry-server
+      name: container-registry-server
   template:
     metadata:
       labels:
-        ms: container-registry-server
+        name: container-registry-server
     spec:
       containers:
         - name: container-registry-server
@@ -52,7 +54,12 @@ spec:
             - name: REGISTRY_AUTH_HTPASSWD_REALM
               value: Registry Realm
             - name: REGISTRY_AUTH_HTPASSWD_PATH
-              value: /auth/htpasswd
+              value: /auth/docker-container-registry
+            - name: REGISTRY_HTTP_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: secrets
+                  key: HTTP_SECRET
             - name: REGISTRY_STORAGE_S3_ACCESSKEY
               valueFrom:
                 secretKeyRef:
@@ -80,68 +87,27 @@ metadata:
 spec:
   type: NodePort
   selector:
-    ms: container-registry-server
+    name: container-registry-server
   ports:
     - port: 5000
       targetPort: 5000
 ---
-apiVersion: cert-manager.io/v1
+apiVersion: traefik.containo.us/v1alpha1
-kind: Issuer
+kind: IngressRoute
 metadata:
-  namespace: sysadmin
-  name: letsencrypt-prod
-spec:
-  acme:
-    server: https://acme-v02.api.letsencrypt.org/directory
-    email: florian@nocodelytics.com
-    privateKeySecretRef:
-      name: letsencrypt-prod
-    solvers:
-      - http01:
-          ingress:
-            class: traefik
----
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  namespace: sysadmin
-  name: container-registry
-spec:
-  secretName: container-registry-net-tls
-  issuerRef:
-    name: letsencrypt-prod
-    kind: Issuer
-  commonName: container-registry.nocodelytics.com
-  dnsNames:
-    - container-registry.nocodelytics.com
----
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  namespace: sysadmin
-  name: container-registry-nginx-ingress
-  annotations:
-    traefik.ingress.kubernetes.io/router.middlewares: default-https-redirect@kubernetescrd
-spec:
-  rules:
-    - host: container-registry.nocodelytics.com
-      http:
-        paths:
-          - path: /
-            pathType: Prefix
-            backend:
-              service:
-                name: ssl-redirect
-                port:
-                  name: use-annotation
-          - path: /
-            pathType: Prefix
-            backend:
-              service:
   name: container-registry-server
-                port:
+  namespace: sysadmin
-                  number: 5000
+spec:
-  tls:
+  entryPoints:
-    - hosts:
+    - websecure
-        - container-registry.nocodelytics.com
+  routes:
-      secretName: container-registry-net-tls
+    - match: Host(`container-registry-server.nocodelytics.com`)
+      kind: Rule
+      services:
+        - name: container-registry-server
+          port: 5000
+      middlewares:
+        - name: websocket-middleware
+          namespace: default
+        - name: https-redirect
+          namespace: default

sysadmin/grafana.yaml

@@ -1,9 +1,4 @@
 apiVersion: v1
-kind: Namespace
-metadata:
-  name: sysadmin
----
-apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
   name: grafana-pvc
@@ -77,70 +72,28 @@ metadata:
   name: grafana
   namespace: sysadmin
 spec:
-  type: NodePort
+  type: ClusterIP
   selector:
     ms: grafana
   ports:
     - port: 3000
       targetPort: 3000
 ---
-apiVersion: cert-manager.io/v1
+apiVersion: traefik.containo.us/v1alpha1
-kind: Issuer
+kind: IngressRoute
 metadata:
-  namespace: sysadmin
-  name: letsencrypt-prod
-spec:
-  acme:
-    server: https://acme-v02.api.letsencrypt.org/directory
-    email: florian@nocodelytics.com
-    privateKeySecretRef:
-      name: letsencrypt-prod
-    solvers:
-      - http01:
-          ingress:
-            class: traefik
----
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  namespace: sysadmin
   name: grafana
-spec:
-  secretName: grafana-net-tls
-  issuerRef:
-    name: letsencrypt-prod
-    kind: Issuer
-  commonName: grafana.nocodelytics.com
-  dnsNames:
-    - grafana.nocodelytics.com
----
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
   namespace: sysadmin
-  name: grafana-nginx-ingress
-  annotations:
-    traefik.ingress.kubernetes.io/router.middlewares: default-https-redirect@kubernetescrd
 spec:
-  rules:
+  entryPoints:
-    - host: grafana.nocodelytics.com
+    - websecure
-      http:
+  routes:
-        paths:
+    - match: Host(`grafana.nocodelytics.com`)
-          - path: /
+      kind: Rule
-            pathType: Prefix
+      services:
-            backend:
+        - name: grafana
-              service:
+          port: 3000
-                name: ssl-redirect
-                port:
-                  name: use-annotation
-          - path: /
-            pathType: Prefix
-            backend:
-              service:
-                name: grafana
-                port:
-                  number: 3000
   tls:
-    - hosts:
+    certResolver: letsencrypt
-        - grafana.nocodelytics.com
+    domains:
-      secretName: grafana-net-tls
+      - main: grafana.nocodelytics.com

sysadmin/k8s-dashboard.yml

@@ -279,6 +279,12 @@ spec:
         - name: tmp-volume
           emptyDir: {}
 ---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: admin-user
+  namespace: kubernetes-dashboard
+---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
@@ -292,70 +298,20 @@ subjects:
     name: admin-user
     namespace: kubernetes-dashboard
 ---
-apiVersion: v1
+apiVersion: traefik.containo.us/v1alpha1
-kind: ServiceAccount
+kind: IngressRoute
 metadata:
-  name: admin-user
-  namespace: kubernetes-dashboard
----
-apiVersion: cert-manager.io/v1
-kind: Issuer
-metadata:
-  namespace: kubernetes-dashboard
-  name: letsencrypt-prod
-spec:
-  acme:
-    server: https://acme-v02.api.letsencrypt.org/directory
-    email: florian@nocodelytics.com
-    privateKeySecretRef:
-      name: letsencrypt-prod
-    solvers:
-      - http01:
-          ingress:
-            class: traefik
----
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  namespace: kubernetes-dashboard
   name: kubernetes-dashboard
-spec:
-  secretName: kubernetes-dashboard-net-tls
-  issuerRef:
-    name: letsencrypt-prod
-    kind: Issuer
-  commonName: k3s-dashboard.nocodelytics.com
-  dnsNames:
-    - k3s-dashboard.nocodelytics.com
----
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
   namespace: kubernetes-dashboard
-  name: kubernetes-dashboard-nginx-ingress
-  annotations:
-    traefik.ingress.kubernetes.io/router.middlewares: default-https-redirect@kubernetescrd
-
 spec:
-  rules:
+  entryPoints:
-    - host: k3s-dashboard.nocodelytics.com
+    - websecure
-      http:
+  routes:
-        paths:
+    - match: Host(`k3s-dashboard.nocodelytics.com`)
-          - path: /
+      kind: Rule
-            pathType: Prefix
+      services:
-            backend:
+        - name: kubernetes-dashboard
-              service:
+          port: 9090
-                name: ssl-redirect
+      middlewares:
-                port:
+        - name: https-redirect
-                  name: use-annotation
+          namespace: default
-          - path: /
-            pathType: Prefix
-            backend:
-              service:
-                name: kubernetes-dashboard
-                port:
-                  number: 9090
-  tls:
-    - hosts:
-        - k3s-dashboard.nocodelytics.com
-      secretName: kubernetes-dashboard-net-tls

sysadmin/kuma.yml

@@ -0,0 +1,69 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: kuma-pvc
+  namespace: sysadmin
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: longhorn
+  resources:
+    requests:
+      storage: 1Gi
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kuma
+  namespace: sysadmin
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      ms: kuma
+  template:
+    metadata:
+      labels:
+        ms: kuma
+    spec:
+      containers:
+        - name: kuma
+          image: louislam/uptime-kuma
+          volumeMounts:
+            - name: volv
+              mountPath: /app/data
+      volumes:
+        - name: volv
+          persistentVolumeClaim:
+            claimName: kuma-pvc
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: kuma
+  namespace: sysadmin
+spec:
+  selector:
+    ms: kuma
+  ports:
+    - port: 3001
+      targetPort: 3001
+---
+apiVersion: traefik.containo.us/v1alpha1
+kind: IngressRoute
+metadata:
+  name: kuma
+  namespace: sysadmin
+spec:
+  entryPoints:
+    - websecure
+  routes:
+    - match: Host(`status.nocodelytics.com`)
+      kind: Rule
+      services:
+        - name: kuma
+          port: 3001
+  tls:
+    certResolver: letsencrypt
+    domains:
+      - main: status.nocodelytics.com

sysadmin/loki.yaml

@@ -1,9 +1,4 @@
 apiVersion: v1
-kind: Namespace
-metadata:
-  name: sysadmin
----
-apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
   name: loki-pvc
@@ -27,6 +22,10 @@ data:
   loki-config.yaml: |
     auth_enabled: false
 
+    table_manager:
+      retention_deletes_enabled: true
+      retention_period: 336h
+
     server:
       http_listen_port: 3100
 

sysadmin/longhorn.yaml

@@ -0,0 +1,14 @@
+apiVersion: traefik.containo.us/v1alpha1
+kind: IngressRoute
+metadata:
+  name: longhorn-frontend
+  namespace: longhorn-system
+spec:
+  entryPoints:
+    - websecure
+  routes:
+    - match: Host(`longhorn.nocodelytics.com`)
+      kind: Rule
+      services:
+        - name: longhorn-frontend
+          port: 80

sysadmin/prometheus-deployment.yaml

@@ -1,8 +1,3 @@
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: sysadmin
----
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
@@ -42,6 +37,22 @@ data:
       scrape_interval: 60s
 
     scrape_configs:
+      - job_name: "kubernetes_pods"
+        kubernetes_sd_configs:
+          - role: pod
+        relabel_configs:
+          - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
+            action: keep
+            regex: true
+          - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
+            action: replace
+            target_label: __metrics_path__
+            regex: (.+)
+          - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
+            action: replace
+            regex: ([^:]+)(?::\d+)?;(\d+)
+            replacement: $1:$2
+            target_label: __address__
       - job_name: "node_exporter"
         static_configs:
           - targets: ["144.76.186.182:9100"]
@@ -102,12 +113,12 @@ metadata:
 spec:
   accessModes:
     - ReadWriteOnce
-  storageClassName: longhorn
+  storageClassName: local-path
   resources:
     requests:
-      storage: 10Gi
+      storage: 20Gi
     limits:
-      storage: 10Gi
+      storage: 20Gi
 ---
 apiVersion: apps/v1
 kind: Deployment
@@ -129,7 +140,7 @@ spec:
           image: prom/prometheus
           args:
             - --config.file=/etc/prometheus/prometheus.yml
-            - --storage.tsdb.retention.size=8GB
+            - --storage.tsdb.retention.size=18GB
           volumeMounts:
             - name: data
               mountPath: /prometheus/
@@ -159,63 +170,21 @@ spec:
     - port: 9090
       targetPort: 9090
 ---
-apiVersion: cert-manager.io/v1
+apiVersion: traefik.containo.us/v1alpha1
-kind: Issuer
+kind: IngressRoute
 metadata:
-  namespace: sysadmin
-  name: letsencrypt-prod
-spec:
-  acme:
-    server: https://acme-v02.api.letsencrypt.org/directory
-    email: florian@nocodelytics.com
-    privateKeySecretRef:
-      name: letsencrypt-prod
-    solvers:
-      - http01:
-          ingress:
-            class: traefik
----
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  namespace: sysadmin
   name: prometheus
-spec:
-  secretName: prometheus-net-tls
-  issuerRef:
-    name: letsencrypt-prod
-    kind: Issuer
-  commonName: prometheus.nocodelytics.com
-  dnsNames:
-    - prometheus.nocodelytics.com
----
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
   namespace: sysadmin
-  name: prometheus-nginx-ingress
-  annotations:
-    traefik.ingress.kubernetes.io/router.middlewares: default-https-redirect@kubernetescrd,default-http-auth@kubernetescrd
 spec:
-  rules:
+  entryPoints:
-    - host: prometheus.nocodelytics.com
+    - websecure
-      http:
+  routes:
-        paths:
+    - match: Host(`prometheus.nocodelytics.com`)
-          - path: /
+      kind: Rule
-            pathType: Prefix
+      services:
-            backend:
+        - name: prometheus
-              service:
+          port: 9090
-                name: ssl-redirect
-                port:
-                  name: use-annotation
-          - path: /
-            pathType: Prefix
-            backend:
-              service:
-                name: prometheus
-                port:
-                  number: 9090
   tls:
-    - hosts:
+    certResolver: letsencrypt
-        - prometheus.nocodelytics.com
+    domains:
-      secretName: prometheus-net-tls
+      - main: prometheus.nocodelytics.com