remove http auth on gitea- breaks Drone and health checks

README.md

@@ -2,6 +2,18 @@
 
 [![Build Status](https://drone.nocodelytics.com/api/badges/nocodelytics/infrastructure/status.svg)](https://drone.nocodelytics.com/nocodelytics/infrastructure)
 
+## Logging into drone/gitea
+These are protected by basic HTTP auth and logging in is a pain the arse. To log in, temporarily disable it
+by commenting out the middlewares in `gitea.yaml` and `drone.yaml` and `minio.yaml`, then
+
+```angular2html
+kubectl apply -f apps
+```
+
+Now log in (make sure you click Remember me), then undo the yaml changes and re-apply
+
+PS use Chrome for Drone!!! Also, drone will not trigger on `git push` if HTTP auth is enabled for gitea- disable auth and push again.
+
 ## Setting up server
 
 - install docker
@@ -43,3 +55,25 @@ kubectl create secret generic cloud-credentials --namespace velero --from-file=c
 ### Connect to services
 
 Postgres: `kubectl -n databases port-forward pod/postgres-0 5432:5432`
+
+
+# Runbook
+
+## Failing health checks
+`KUBE_CONFIG` is a secret on Drone https://drone.nocodelytics.com/nocodelytics/healthcheck/settings/org-secrets
+Value needs to come from `/etc/rancher/k3s/k3s.yaml` from the server
+
+This will expire once a year, needs to be renewed per https://docs.k3s.io/cli/certificate
+```
+# ssh into server
+systemctl stop k3s
+k3s certificate rotate
+systemctl start k3s
+```
+
+Then base64 encode it `cat /etc/rancher/k3s/k3s.yaml | base64 -i -`
+
+The same kube config, NOT encoded, goes to `~/.kube/config`, but the `server` section needs to be edited to point to the server IP
+
+## Disk space issues
+Find the persistent volume that's full, eg in `clickhouse.yaml`, edit ONLY `resources.requests.storage` section, then `kubectl apply -f ...`

apps/drone.yaml

@@ -99,6 +99,11 @@ spec:
       services:
         - name: drone
           port: 80
+#      middlewares:
+#        - name: https-redirect
+#          namespace: default
+#        - name: http-auth
+#          namespace: default
   tls:
     certResolver: letsencrypt
     domains:

apps/gitea.yaml

@@ -133,6 +133,11 @@ spec:
       services:
         - name: gitea
           port: 3000
+#      middlewares:
+#        - name: https-redirect
+#          namespace: default
+#        - name: http-auth
+#          namespace: default
   tls:
     certResolver: letsencrypt
     domains:

apps/minio.yaml

@@ -129,6 +129,11 @@ spec:
       services:
         - name: gitea
           port: 3000
+      middlewares:
+        - name: https-redirect
+          namespace: default
+        - name: http-auth
+          namespace: default
   tls:
     certResolver: letsencrypt
     domains:

apps/tooljet.yaml

@@ -69,6 +69,11 @@ spec:
       services:
         - name: tooljet
           port: 80
+#      middlewares:
+#        - name: https-redirect
+#          namespace: default
+#        - name: http-auth
+#          namespace: default
   tls:
     certResolver: letsencrypt
     domains:

databases/clickhouse.yaml

@@ -9,7 +9,7 @@ spec:
   storageClassName: longhorn
   resources:
     requests:
-      storage: 100Gi
+      storage: 120Gi
     limits:
       storage: 100Gi
 ---

sysadmin/grafana.yaml

@@ -132,6 +132,11 @@ spec:
       services:
         - name: grafana
           port: 3000
+      middlewares:
+        - name: https-redirect
+          namespace: default
+        - name: http-auth
+          namespace: default
   tls:
     certResolver: letsencrypt
     domains:

sysadmin/loki.yaml

@@ -9,7 +9,7 @@ spec:
   storageClassName: longhorn
   resources:
     requests:
-      storage: 10Gi
+      storage: 20Gi
     limits:
       storage: 10Gi
 ---
@@ -41,7 +41,6 @@ data:
       retention_enabled: true
       retention_delete_delay: 1h
       retention_delete_worker_count: 150
-      delete_request_cancel_after: 12h
 
     limits_config:
       retention_period: 168h