wip nats

README.md

@@ -0,0 +1,8 @@
+# Infrastructure
+
+## Secrets
+
+1. Create a `.env` file
+2. Add base64 encoded secrets
+3. `export $(cat .env | xargs)`
+4. `cat secrets.yaml | envsubst | kubectl apply -f -`

container-registry.yaml

@@ -0,0 +1,192 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: container-registry
+---
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: letsencrypt-prod
+  namespace: container-registry
+spec:
+  acme:
+    server: https://acme-v02.api.letsencrypt.org/directory
+    email: florian@nocodelytics.com
+    privateKeySecretRef:
+      name: letsencrypt-prod
+    solvers:
+      - http01:
+          ingress:
+            class: traefik
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: container-registry-server
+  namespace: container-registry
+spec:
+  secretName: container-registry-server-net-tls
+  issuerRef:
+    name: letsencrypt-prod
+    kind: Issuer
+  commonName: container-registry.nocodelytics.com
+  dnsNames:
+    - container-registry.nocodelytics.com
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: container-registry-server-pvc
+  namespace: container-registry
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: local-path
+  resources:
+    requests:
+      storage: 1Gi
+---
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: letsencrypt-prod
+  namespace: container-registry
+spec:
+  acme:
+    server: https://acme-v02.api.letsencrypt.org/directory
+    email: florian@nocodelytics.com
+    privateKeySecretRef:
+      name: letsencrypt-prod
+    solvers:
+      - http01:
+          ingress:
+            class: traefik
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: container-registry-server-config
+  namespace: container-registry
+data:
+  config.yml: |
+    version: 0.1
+    log:
+      fields:
+        service: registry
+    storage:
+      cache:
+        blobdescriptor: inmemory
+      s3:
+        region: eu-west-1
+        bucket: container-registry
+    http:
+      addr: :5000
+      headers:
+        X-Content-Type-Options: [nosniff]
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: container-registry-server
+  namespace: container-registry
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      ms: container-registry-server
+  template:
+    metadata:
+      labels:
+        ms: container-registry-server
+    spec:
+      containers:
+        - name: container-registry-server
+          image: registry:2
+          volumeMounts:
+            - name: volv
+              mountPath: /var/lib/registry
+            - name: config-volume
+              mountPath: /etc/docker/registry/config.yml
+              subPath: config.yml
+            - name: secrets-volume
+              mountPath: /auth
+              readOnly: true
+          env:
+            - name: REGISTRY_AUTH
+              value: htpasswd
+            - name: REGISTRY_AUTH_HTPASSWD_REALM
+              value: Registry Realm
+            - name: REGISTRY_AUTH_HTPASSWD_PATH
+              value: /auth/htpasswd
+            - name: REGISTRY_STORAGE_S3_ACCESSKEY
+              valueFrom:
+                secretKeyRef:
+                  name: secrets
+                  key: AWS_ACCESS_KEY_ID
+            - name: REGISTRY_STORAGE_S3_SECRETKEY
+              valueFrom:
+                secretKeyRef:
+                  name: secrets
+                  key: AWS_SECRET_ACCESS_KEY
+          resources:
+            limits:
+              memory: "512Mi"
+              cpu: "100m"
+      volumes:
+        - name: volv
+          persistentVolumeClaim:
+            claimName: container-registry-server-pvc
+        - name: config-volume
+          configMap:
+            name: container-registry-server-config
+        - name: secrets-volume
+          secret:
+            secretName: container-registry
+            optional: false
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: container-registry-server
+  namespace: container-registry
+spec:
+  type: NodePort
+  selector:
+    ms: container-registry-server
+  ports:
+    - port: 5000
+      targetPort: 5000
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: container-registry-nginx-ingress
+  namespace: container-registry
+  annotations:
+    kubernetes.io/ingress.class: "traefik"
+    cert-manager.io/issuer: letsencrypt-prod
+    traefik.ingress.kubernetes.io/redirect-entry-point: https
+    cert-manager.io/acme-challenge-type: http01
+spec:
+  rules:
+    - host: container-registry.nocodelytics.com
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: ssl-redirect
+                port:
+                  name: use-annotation
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: container-registry-server
+                port:
+                  number: 5000
+  tls:
+    - hosts:
+        - container-registry.nocodelytics.com
+      secretName: container-registry-server-net-tls

kustomization/bases/kustomization.yaml

@@ -3,8 +3,15 @@ resources:
   - ./nocodelytics-dashboard.yaml
   # - ./nocodelytics-events-worker.yaml
   # - ./nocodelytics-tracker-api.yaml
-  # - ./clickhouse.yaml
   - ./cert-manager.yaml
+
+helmCharts:
+  - name: nats
+    repo: https://nats-io.github.io/k8s/helm/charts/
+    version: 0.19.1
+    releaseName: nocodelytics-nats
+    valuesFile: ./nats.yaml
+
 secretGenerator:
   - name: regcred
     type: kubernetes.io/dockerconfigjson

kustomization/bases/nats.yaml

@@ -1,20 +1,73 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nocodelytics-nats
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      ms: nocodelytics-nats
+  template:
+    metadata:
+      labels:
+        ms: nocodelytics-nats
+    spec:
+      imagePullSecrets:
+        - name: regcred
+      containers:
+        - name: nocodelytics-nats
+          imagePullPolicy: Always
+          image: nats
+          command:
+            - "nats-server"
+            - "--cluster_name"
+            - "nocodelytics-nats-cluster"
+            - "-js"
+            - "--user"
+            - "nats"
+            - "--pass"
+            - "hello"
+          readinessProbe:
+            tcpSocket:
+              port: 4222
+            initialDelaySeconds: 5
+            periodSeconds: 10
+          livenessProbe:
+            tcpSocket:
+              port: 4222
+            initialDelaySeconds: 10
+            failureThreshold: 5
+            periodSeconds: 10
+            terminationGracePeriodSeconds: 60
+          resources:
+            requests:
+              memory: "256Mi"
+              cpu: "50m"
+            limits:
+              memory: "512Mi"
+              cpu: "200m"
+      volumes:
+        - name: config-volume
+          configMap:
+            name: nats-server-config
+      nodeSelector:
+        kubernetes.io/arch: arm64
+        kubernetes.io/hostname: primary-b
+      tolerations:
+        - key: "arch"
+          operator: "Equal"
+          value: "arm64"
+          effect: "NoSchedule"
+---
 apiVersion: v1
 kind: Service
 metadata:
-  name: nats-lb
-  namespace: stage
+  name: nocodelytics-nats
 spec:
   type: NodePort
   selector:
-    app.kubernetes.io/name: nats
+    ms: nocodelytics-nats
   ports:
-    - protocol: TCP
-      port: 4222
+    - port: 4222
       targetPort: 4222
-      name: nats
-      nodePort: 30022
-    - protocol: TCP
-      port: 8222
-      targetPort: 8222
-      name: nats-monitor
-      nodePort: 30023
+      nodePort: 30040

kustomization/components/nats-0/deployment.yaml

@@ -0,0 +1,3 @@
+- op: replace
+  path: /spec/template/spec/nodeSelector/kubernetes.io~1hostname
+  value: "primary-a"

kustomization/components/nats-0/kustomization.yaml

@@ -0,0 +1,10 @@
+apiVersion: kustomize.config.k8s.io/v1alpha1 # <-- Component notation
+kind: Component
+
+patchesJson6902:
+  - target:
+      group: apps
+      version: v1
+      kind: Deployment
+      name: nocodelytics-nats
+    path: deployment.yaml

kustomization/overlays/staging/kustomization.yaml

@@ -1,8 +1,11 @@
 namespace: staging
 resources:
   - ../../bases
-patchesStrategicMerge:
-  - ./nocodelytics-dashboard.yaml
+components:
+  - ../../components/nats-0
+  - ../../components/nats-1
+# patchesStrategicMerge:
+#   - ./nocodelytics-dashboard.yaml
 # - ./nocodelytics-tracker-api.yaml
 patches:
   - target:

secrets.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+  name: secrets
+  namespace: container-registry
+data:
+  AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
+  AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}