infrastructure/sysadmin/prometheus-deployment.yaml

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: prometheus
rules:
  - apiGroups: [""]
    resources:
      - nodes
      - nodes/metrics
      - nodes/proxy
      - services
      - endpoints
      - pods
    verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: prometheus
subjects:
  - kind: ServiceAccount
    name: default
    namespace: sysadmin
roleRef:
  kind: ClusterRole
  name: prometheus
  apiGroup: rbac.authorization.k8s.io
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: prometheus-configmap
  namespace: sysadmin
data:
  prometheus.yml: |
    global:
      scrape_interval: 60s

    scrape_configs:
      - job_name: "kubernetes_pods"
        kubernetes_sd_configs:
          - role: pod
        relabel_configs:
          - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
            action: keep
            regex: true
          - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
            action: replace
            target_label: __metrics_path__
            regex: (.+)
          - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
            action: replace
            regex: ([^:]+)(?::\d+)?;(\d+)
            replacement: $1:$2
            target_label: __address__
      - job_name: "node_exporter"
        static_configs:
          - targets: ["144.76.186.182:9100"]
      - job_name: "postgres_exporter"
        static_configs:
          - targets: ["postgres-exporter.databases:9187"]
      - job_name: "clickhouse_exporter"
        static_configs:
          - targets: ["clickhouse.databases:9363"]
      - job_name: "nats_exporter"
        static_configs:
          - targets: ["nats-exporter.databases:7777"]
      - job_name: "kube_exporter"
        static_configs:
          - targets: ["kube-state-metrics.kube-system.svc.cluster.local:8080"]
      - job_name: "kubernetes-cadvisor"
        scheme: https
        kubernetes_sd_configs:
          - role: node
        tls_config:
          insecure_skip_verify: true
        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
        relabel_configs:
          - action: labelmap
            regex: __meta_kubernetes_node_label_(.+)
          - target_label: __address__
            replacement: kubernetes.default.svc:443
          - source_labels: [__meta_kubernetes_node_name]
            regex: (.+)
            target_label: __metrics_path__
            replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
      - job_name: "kubelet"
        scheme: https
        kubernetes_sd_configs:
          - role: node
        tls_config:
          insecure_skip_verify: true
        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
        relabel_configs:
          - action: labelmap
            regex: __meta_kubernetes_node_label_(.+)
          - target_label: __address__
            replacement: kubernetes.default.svc:443
          - source_labels: [__meta_kubernetes_node_name]
            regex: (.+)
            target_label: __metrics_path__
            replacement: /api/v1/nodes/${1}/proxy/metrics
      - job_name: "longhorn_exporter"
        static_configs:
          - targets: ["longhorn-backend.longhorn-system:9500"]    

---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: prometheus-pvc
  namespace: sysadmin
spec:
  accessModes:
    - ReadWriteOnce
  storageClassName: local-path
  resources:
    requests:
      storage: 20Gi
    limits:
      storage: 20Gi
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: prometheus
  namespace: sysadmin
spec:
  replicas: 1
  selector:
    matchLabels:
      ms: prometheus
  template:
    metadata:
      labels:
        ms: prometheus
    spec:
      containers:
        - name: prometheus
          image: prom/prometheus
          args:
            - --config.file=/etc/prometheus/prometheus.yml
            - --storage.tsdb.retention.size=18GB
          volumeMounts:
            - name: data
              mountPath: /prometheus/
            - name: config
              mountPath: /etc/prometheus/
      securityContext:
        runAsUser: 1000
        fsGroup: 2000
      volumes:
        - name: data
          persistentVolumeClaim:
            claimName: prometheus-pvc
        - name: config
          configMap:
            name: prometheus-configmap
---
apiVersion: v1
kind: Service
metadata:
  name: prometheus
  namespace: sysadmin
spec:
  type: NodePort
  selector:
    ms: prometheus
  ports:
    - port: 9090
      targetPort: 9090
---
apiVersion: traefik.containo.us/v1alpha1
kind: IngressRoute
metadata:
  name: prometheus
  namespace: sysadmin
spec:
  entryPoints:
    - websecure
  routes:
    - match: Host(`prometheus.nocodelytics.com`)
      kind: Rule
      services:
        - name: prometheus
          port: 9090
  tls:
    certResolver: letsencrypt
    domains:
      - main: prometheus.nocodelytics.com