diff --git a/databases/clickhouse.yaml b/databases/clickhouse.yaml index 7291d51..aad1e6f 100644 --- a/databases/clickhouse.yaml +++ b/databases/clickhouse.yaml @@ -72,6 +72,9 @@ spec: metadata: labels: ms: clickhouse + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9363" spec: terminationGracePeriodSeconds: 120 containers: diff --git a/databases/nats.yaml b/databases/nats.yaml index ccd8931..18c053d 100644 --- a/databases/nats.yaml +++ b/databases/nats.yaml @@ -43,6 +43,8 @@ metadata: namespace: databases spec: replicas: 1 + strategy: + type: Recreate selector: matchLabels: ms: nats @@ -51,6 +53,7 @@ spec: labels: ms: nats spec: + terminationGracePeriodSeconds: 120 containers: - name: nats image: nats:2.10.5 @@ -105,6 +108,9 @@ spec: metadata: labels: ms: nats-exporter + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "7777" spec: containers: - name: nats-exporter @@ -121,16 +127,3 @@ spec: - http://nats.databases:8222 ports: - containerPort: 7777 ---- -apiVersion: v1 -kind: Service -metadata: - name: nats-exporter - namespace: databases -spec: - type: NodePort - selector: - ms: nats-exporter - ports: - - port: 7777 - targetPort: 7777 diff --git a/databases/postgres.sql b/databases/postgres.sql new file mode 100644 index 0000000..86dc435 --- /dev/null +++ b/databases/postgres.sql @@ -0,0 +1,6 @@ +create database nocodelytics_staging; +create database nocodelytics_production; +create database grafana; + +CREATE EXTENSION IF NOT EXISTS pg_stat_statements; +CREATE EXTENSION IF NOT EXISTS pg_buffercache; \ No newline at end of file diff --git a/databases/postgres.yaml b/databases/postgres.yaml index bc2df34..632b1d7 100644 --- a/databases/postgres.yaml +++ b/databases/postgres.yaml @@ -14,14 +14,13 @@ spec: storage: 10Gi --- apiVersion: apps/v1 -kind: Deployment +kind: StatefulSet metadata: name: postgres namespace: databases spec: + serviceName: "postgres" replicas: 1 - strategy: - type: Recreate selector: matchLabels: name: postgres @@ -89,6 +88,9 @@ spec: metadata: labels: name: postgres-exporter + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9187" spec: containers: - name: postgres-exporter @@ -104,19 +106,6 @@ spec: - name: DATA_SOURCE_NAME value: postgresql://postgres:$(POSTGRES_PASSWORD)@postgres.databases:5432/postgres?sslmode=disable --- -apiVersion: v1 -kind: Service -metadata: - name: postgres-exporter - namespace: databases -spec: - type: NodePort - selector: - name: postgres-exporter - ports: - - port: 9187 - targetPort: 9187 ---- apiVersion: batch/v1 kind: CronJob metadata: @@ -135,14 +124,18 @@ spec: spec: restartPolicy: Never containers: - - name: test-container - image: container-registry.nocodelytics.com/postgres-s3 + - name: postgres-backup + image: localhost:30007/postgres-s3 command: - /bin/sh - -c - > pg_dump -U postgres -h postgres.databases nocodelytics_production | gzip > /backup/nocodelytics_production_$(date +'%Y-%m-%d').sql.gzip && - rclone copy /backup/nocodelytics_production_$(date '+%Y-%m-%d').sql.gzip contabo:postgres + rclone copy /backup/nocodelytics_production_$(date '+%Y-%m-%d').sql.gzip contabo:postgres && + rm /backup/nocodelytics_production_$(date '+%Y-%m-%d').sql.gzip && + pg_dump -U postgres -h postgres.databases grafana | gzip > /backup/grafana_$(date +'%Y-%m-%d').sql.gzip && + rclone copy /backup/grafana_$(date '+%Y-%m-%d').sql.gzip contabo:postgres && + rm /backup/grafana_$(date '+%Y-%m-%d').sql.gzip env: - name: PGPASSWORD valueFrom: @@ -173,44 +166,3 @@ spec: volumes: - name: backup-volume emptyDir: {} ---- -apiVersion: v1 -kind: Pod -metadata: - name: test-pod - namespace: databases -spec: - containers: - - name: test-container - image: container-registry.nocodelytics.com/postgres-s3 - command: ["sh", "-c", "echo Image pulled successfully && sleep 3600"] - env: - - name: PGPASSWORD - valueFrom: - secretKeyRef: - name: secrets - key: POSTGRES_PASSWORD - - name: RCLONE_CONFIG_CONTABO_TYPE - value: "s3" - - name: RCLONE_CONFIG_CONTABO_PROVIDER - value: "Other" - - name: RCLONE_CONFIG_CONTABO_ENV_AUTH - value: "false" - - name: RCLONE_CONFIG_CONTABO_ENDPOINT - value: "https://eu2.contabostorage.com" - - name: RCLONE_CONFIG_CONTABO_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - name: secrets - key: AWS_ACCESS_KEY_ID - - name: RCLONE_CONFIG_CONTABO_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - name: secrets - key: AWS_SECRET_ACCESS_KEY - volumeMounts: - - mountPath: /backup - name: backup-volume - volumes: - - name: backup-volume - emptyDir: {} diff --git a/dependencies/00-longhorn.yaml b/dependencies/00-longhorn.yaml index affdc75..602014f 100644 --- a/dependencies/00-longhorn.yaml +++ b/dependencies/00-longhorn.yaml @@ -4388,3 +4388,7 @@ spec: namespace: default - name: https-redirect namespace: default + tls: + certResolver: letsencrypt + domains: + - main: longhorn.nocodelytics.com diff --git a/dependencies/01-namespaces.yml b/dependencies/01-namespaces.yml index eefea41..33367ee 100644 --- a/dependencies/01-namespaces.yml +++ b/dependencies/01-namespaces.yml @@ -2,3 +2,8 @@ apiVersion: v1 kind: Namespace metadata: name: sysadmin +--- +apiVersion: v1 +kind: Namespace +metadata: + name: databases diff --git a/dependencies/kube-state-metrics.yaml b/dependencies/kube-state-metrics.yaml new file mode 100644 index 0000000..f3da6a0 --- /dev/null +++ b/dependencies/kube-state-metrics.yaml @@ -0,0 +1,215 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: exporter + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/version: 2.10.1 + name: kube-state-metrics +rules: + - apiGroups: + - "" + resources: + - configmaps + - secrets + - nodes + - pods + - services + - serviceaccounts + - resourcequotas + - replicationcontrollers + - limitranges + - persistentvolumeclaims + - persistentvolumes + - namespaces + - endpoints + verbs: + - list + - watch + - apiGroups: + - apps + resources: + - statefulsets + - daemonsets + - deployments + - replicasets + verbs: + - list + - watch + - apiGroups: + - batch + resources: + - cronjobs + - jobs + verbs: + - list + - watch + - apiGroups: + - autoscaling + resources: + - horizontalpodautoscalers + verbs: + - list + - watch + - apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create + - apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create + - apiGroups: + - policy + resources: + - poddisruptionbudgets + verbs: + - list + - watch + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests + verbs: + - list + - watch + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + - volumeattachments + verbs: + - list + - watch + - apiGroups: + - admissionregistration.k8s.io + resources: + - mutatingwebhookconfigurations + - validatingwebhookconfigurations + verbs: + - list + - watch + - apiGroups: + - networking.k8s.io + resources: + - networkpolicies + - ingressclasses + - ingresses + verbs: + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - list + - watch + - apiGroups: + - rbac.authorization.k8s.io + resources: + - clusterrolebindings + - clusterroles + - rolebindings + - roles + verbs: + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/component: exporter + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/version: 2.10.1 + name: kube-state-metrics +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kube-state-metrics +subjects: + - kind: ServiceAccount + name: kube-state-metrics + namespace: kube-system +--- +apiVersion: v1 +automountServiceAccountToken: false +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/component: exporter + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/version: 2.10.1 + name: kube-state-metrics + namespace: kube-system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/component: exporter + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/version: 2.10.1 + name: kube-state-metrics + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: kube-state-metrics + template: + metadata: + labels: + app.kubernetes.io/component: exporter + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/version: 2.10.1 + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "8080" + spec: + automountServiceAccountToken: true + containers: + - image: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.10.1 + livenessProbe: + httpGet: + path: /healthz + port: 8080 + initialDelaySeconds: 5 + timeoutSeconds: 5 + name: kube-state-metrics + ports: + - containerPort: 8080 + name: http-metrics + - containerPort: 8081 + name: telemetry + readinessProbe: + httpGet: + path: / + port: 8081 + initialDelaySeconds: 5 + timeoutSeconds: 5 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 65534 + seccompProfile: + type: RuntimeDefault + nodeSelector: + kubernetes.io/os: linux + serviceAccountName: kube-state-metrics diff --git a/dependencies/node-exporter.yaml b/dependencies/node-exporter.yaml new file mode 100644 index 0000000..4b730e3 --- /dev/null +++ b/dependencies/node-exporter.yaml @@ -0,0 +1,51 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + app.kubernetes.io/component: exporter + app.kubernetes.io/name: node-exporter + name: node-exporter + namespace: kube-system +spec: + selector: + matchLabels: + app.kubernetes.io/component: exporter + app.kubernetes.io/name: node-exporter + template: + metadata: + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9100" + labels: + app.kubernetes.io/component: exporter + app.kubernetes.io/name: node-exporter + spec: + containers: + - args: + - --path.sysfs=/host/sys + - --path.rootfs=/host/root + - --no-collector.wifi + - --no-collector.hwmon + - --collector.filesystem.ignored-mount-points=^/(dev|proc|sys|var/lib/docker/.+|var/lib/kubelet/pods/.+)($|/) + - --collector.netclass.ignored-devices=^(veth.*)$ + name: node-exporter + image: prom/node-exporter + ports: + - containerPort: 9100 + protocol: TCP + volumeMounts: + - mountPath: /host/sys + mountPropagation: HostToContainer + name: sys + readOnly: true + - mountPath: /host/root + mountPropagation: HostToContainer + name: root + readOnly: true + volumes: + - hostPath: + path: /sys + name: sys + - hostPath: + path: / + name: root diff --git a/dependencies/traefik-config.yaml b/dependencies/traefik-config.yaml index 62078f0..a2a5250 100644 --- a/dependencies/traefik-config.yaml +++ b/dependencies/traefik-config.yaml @@ -7,16 +7,26 @@ metadata: spec: valuesContent: |- deployment: - - name: volume-permissions - image: busybox:latest - command: ["sh", "-c", "touch /data/acme.json; chmod -v 600 /data/acme.json"] - securityContext: - runAsNonRoot: true - runAsGroup: 65532 - runAsUser: 65532 - volumeMounts: - - name: traefik - mountPath: /data + securityContext: + runAsNonRoot: false + runAsGroup: 0 + runAsUser: 0 + initContainers: + - name: volume-permissions + image: traefik + command: + [ + "sh", + "-c", + "touch /data/acme.json; chown 65532 /data/acme.json; chmod -v 600 /data/acme.json", + ] + securityContext: + runAsNonRoot: false + runAsGroup: 0 + runAsUser: 0 + volumeMounts: + - name: traefik + mountPath: /data providers: kubernetesCRD: allowCrossNamespace: true diff --git a/sysadmin/00-container-registry.yaml b/sysadmin/00-container-registry.yaml index 3acd001..44aeaf0 100644 --- a/sysadmin/00-container-registry.yaml +++ b/sysadmin/00-container-registry.yaml @@ -90,6 +90,7 @@ spec: name: container-registry-server ports: - port: 5000 + nodePort: 30007 targetPort: 5000 --- apiVersion: traefik.containo.us/v1alpha1 @@ -101,7 +102,7 @@ spec: entryPoints: - websecure routes: - - match: Host(`container-registry-server.nocodelytics.com`) + - match: Host(`container-registry.nocodelytics.com`) kind: Rule services: - name: container-registry-server @@ -111,3 +112,7 @@ spec: namespace: default - name: https-redirect namespace: default + tls: + certResolver: letsencrypt + domains: + - main: container-registry.nocodelytics.com diff --git a/sysadmin/grafana.yaml b/sysadmin/grafana.yaml index 773f9d2..d276058 100644 --- a/sysadmin/grafana.yaml +++ b/sysadmin/grafana.yaml @@ -13,6 +13,25 @@ spec: limits: storage: 10Gi --- +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-configmap + namespace: sysadmin +data: + defaults.ini: | + instance_name = grafana + + [security] + admin_user = admin + admin_password = ${HTTP_SECRET} + + [database] + type: postgres + host: postgres.databases + user: postgres + password: ${POSTGRES_PASSWORD} +--- apiVersion: apps/v1 kind: Deployment metadata: @@ -27,6 +46,9 @@ spec: metadata: labels: ms: grafana + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "3000" spec: securityContext: fsGroup: 472 @@ -59,12 +81,29 @@ spec: port: 3000 timeoutSeconds: 1 volumeMounts: - - mountPath: /var/lib/grafana - name: grafana-pv + - name: grafana-pv + mountPath: /var/lib/grafana + - name: grafana-config + mountPath: /etc/grafana/grafana.ini + subPath: defaults.ini + env: + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: secrets + key: POSTGRES_PASSWORD + - name: HTTP_SECRET + valueFrom: + secretKeyRef: + name: secrets + key: HTTP_SECRET volumes: - name: grafana-pv persistentVolumeClaim: claimName: grafana-pvc + - name: grafana-config + configMap: + name: grafana-configmap --- apiVersion: v1 kind: Service diff --git a/sysadmin/k8s-dashboard.yml b/sysadmin/k8s-dashboard.yml index b644aa7..4ec9f44 100644 --- a/sysadmin/k8s-dashboard.yml +++ b/sysadmin/k8s-dashboard.yml @@ -315,3 +315,7 @@ spec: middlewares: - name: https-redirect namespace: default + tls: + certResolver: letsencrypt + domains: + - main: k3s-dashboard.nocodelytics.com diff --git a/sysadmin/longhorn.yaml b/sysadmin/longhorn.yaml deleted file mode 100644 index 5db9d3f..0000000 --- a/sysadmin/longhorn.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: traefik.containo.us/v1alpha1 -kind: IngressRoute -metadata: - name: longhorn-frontend - namespace: longhorn-system -spec: - entryPoints: - - websecure - routes: - - match: Host(`longhorn.nocodelytics.com`) - kind: Rule - services: - - name: longhorn-frontend - port: 80 diff --git a/sysadmin/prometheus-deployment.yaml b/sysadmin/prometheus-deployment.yaml index d78cb42..0f930fc 100644 --- a/sysadmin/prometheus-deployment.yaml +++ b/sysadmin/prometheus-deployment.yaml @@ -53,21 +53,6 @@ data: regex: ([^:]+)(?::\d+)?;(\d+) replacement: $1:$2 target_label: __address__ - - job_name: "node_exporter" - static_configs: - - targets: ["144.76.186.182:9100"] - - job_name: "postgres_exporter" - static_configs: - - targets: ["postgres-exporter.databases:9187"] - - job_name: "clickhouse_exporter" - static_configs: - - targets: ["clickhouse.databases:9363"] - - job_name: "nats_exporter" - static_configs: - - targets: ["nats-exporter.databases:7777"] - - job_name: "kube_exporter" - static_configs: - - targets: ["kube-state-metrics.kube-system.svc.cluster.local:8080"] - job_name: "kubernetes-cadvisor" scheme: https kubernetes_sd_configs: @@ -100,9 +85,6 @@ data: regex: (.+) target_label: __metrics_path__ replacement: /api/v1/nodes/${1}/proxy/metrics - - job_name: "longhorn_exporter" - static_configs: - - targets: ["longhorn-backend.longhorn-system:9500"] --- apiVersion: v1 @@ -134,6 +116,9 @@ spec: metadata: labels: ms: prometheus + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" spec: containers: - name: prometheus