kubernetes/apps/observability/loki/app/externalsecret.yaml

@@ -0,0 +1,28 @@
+---
+# yaml-language-server: $schema=https://ks.hsn.dev/external-secrets.io/externalsecret_v1beta1.json
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: loki
+spec:
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: onepassword-connect
+  target:
+    name: loki-secret
+    creationPolicy: Owner
+    template:
+      engineVersion: v2
+      data:
+        S3_BUCKET_NAME: loki
+        S3_BUCKET_HOST: s3.hsn.dev
+        S3_BUCKET_REGION: us-east-1
+        S3_ACCESS_KEY: "loki"
+        S3_SECRET_KEY: "{{ .minio_loki_secret_key }}"
+  dataFrom:
+    - extract:
+        key: minio
+      rewrite:
+        - regexp:
+            source: "(.*)"
+            target: "minio_$1"

kubernetes/apps/observability/loki/app/helmrelease.yaml

@@ -10,7 +10,7 @@ spec:
   chart:
     spec:
       chart: loki
-      version: 6.2.0
+      version: 6.1.0
       sourceRef:
         kind: HelmRepository
         name: grafana
@@ -21,8 +21,9 @@ spec:
   upgrade:
     cleanupOnFail: true
     remediation:
-      strategy: rollback
       retries: 3
+  uninstall:
+    keepHistory: false
   dependsOn:
     - name: openebs
       namespace: openebs-system
@@ -32,40 +33,25 @@ spec:
       namespace: observability
     - name: vector-aggregator
       namespace: observability
-  valuesFrom:
-    - targetPath: loki.storage.bucketNames.chunks
-      kind: ConfigMap
-      name: &cephBucket loki-bucket
-      valuesKey: BUCKET_NAME
-    - targetPath: loki.storage.s3.endpoint
-      kind: ConfigMap
-      name: *cephBucket
-      valuesKey: BUCKET_HOST
-    - targetPath: loki.storage.s3.region
-      kind: ConfigMap
-      name: *cephBucket
-      valuesKey: BUCKET_REGION
-    - targetPath: loki.storage.s3.accessKeyId
-      kind: Secret
-      name: *cephBucket
-      valuesKey: AWS_ACCESS_KEY_ID
-    - targetPath: loki.storage.s3.secretAccessKey
-      kind: Secret
-      name: *cephBucket
-      valuesKey: AWS_SECRET_ACCESS_KEY
   values:
     deploymentMode: SimpleScalable
+
     loki:
       podAnnotations:
-        configmap.reloader.stakater.com/reload: *cephBucket
-        secret.reloader.stakater.com/reload: *cephBucket
+        secret.reloader.stakater.com/reload: loki-secret
+
       ingester:
         chunk_encoding: snappy
+
       storage:
         type: s3
+        bucketNames:
+          chunks: loki-chunks
+          ruler: loki-ruler
+          admin: loki-admin
         s3:
           s3ForcePathStyle: true
-          insecure: true
+
       schemaConfig:
         configs:
           - from: "2024-04-01"
@@ -75,14 +61,17 @@ spec:
             index:
               prefix: loki_index_
               period: 24h
+
       structuredConfig:
         auth_enabled: false
+
         server:
           log_level: info
           http_listen_port: 3100
           grpc_listen_port: 9095
           grpc_server_max_recv_msg_size: 8388608
           grpc_server_max_send_msg_size: 8388608
+
         limits_config:
           ingestion_burst_size_mb: 128
           ingestion_rate_mb: 64
@@ -95,10 +84,13 @@ spec:
           shard_streams:
             enabled: true
           split_queries_by_interval: 1h
+
         query_scheduler:
           max_outstanding_requests_per_tenant: 4096
+
         frontend:
           max_outstanding_per_tenant: 4096
+
         ruler:
           enable_api: true
           enable_alertmanager_v2: true
@@ -107,18 +99,31 @@ spec:
             type: local
             local:
               directory: /rules
-          rule_path: /rules/fake
+          rule_path: /rules
+
         analytics:
           reporting_enabled: false
-    backend:
-      replicas: 2
-      persistence:
-        size: 20Gi
-        storageClass: openebs-hostpath
+
     gateway:
-      replicas: 2
+      replicas: 3
+      enabled: true
       image:
         registry: ghcr.io
+        repository: nginxinc/nginx-unprivileged
+        tag: 1.25-alpine@sha256:5b49ce26ad8555b649a5a5ea8ccdfabc742a284ab58cb7b7d7a56f178c5dd351
+      deploymentStrategy:
+        type: RollingUpdate
+        rollingUpdate:
+          maxUnavailable: 50%
+          maxSurge: 50%
+      topologySpreadConstraints:
+        - maxSkew: 2
+          topologyKey: kubernetes.io/hostname
+          whenUnsatisfiable: DoNotSchedule
+          labelSelector:
+            matchLabels:
+              app.kubernetes.io/name: loki
+              app.kubernetes.io/component: gateway
       ingress:
         enabled: true
         ingressClassName: internal-nginx
@@ -128,21 +133,55 @@ spec:
               - path: /
                 pathType: Prefix
         tls:
-          - hosts: [*host]
-    read:
-      replicas: 2
+          - hosts:
+              - *host
+
     write:
-      replicas: 2
+      replicas: 3
       persistence:
-        size: 20Gi
         storageClass: openebs-hostpath
-    sidecar:
-      image:
-        repository: ghcr.io/kiwigrid/k8s-sidecar
+    read:
+      replicas: 3
+
+    backend:
+      replicas: 3
+      persistence:
+        storageClass: openebs-hostpath
+    monitoring:
+      dashboards:
+        annotations:
+          grafana_folder: Loki
       rules:
-        searchNamespace: ALL
-        folder: /rules/fake
+        enabled: false
+      serviceMonitor:
+        enabled: false
+        metricsInstance:
+          enabled: false
+      selfMonitoring:
+        enabled: false
+        grafanaAgent:
+          installOperator: false
+
     lokiCanary:
       enabled: false
+
     test:
       enabled: false
+
+  valuesFrom:
+    - kind: Secret
+      name: loki-secret
+      valuesKey: S3_BUCKET_HOST
+      targetPath: loki.storage.s3.endpoint
+    - kind: Secret
+      name: loki-secret
+      valuesKey: S3_BUCKET_REGION
+      targetPath: loki.storage.s3.region
+    - kind: Secret
+      name: loki-secret
+      valuesKey: S3_ACCESS_KEY
+      targetPath: loki.storage.s3.accessKeyId
+    - kind: Secret
+      name: loki-secret
+      valuesKey: S3_SECRET_KEY
+      targetPath: loki.storage.s3.secretAccessKey

kubernetes/apps/observability/loki/app/kustomization.yaml

@@ -3,5 +3,11 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - ./objectbucketclaim.yaml
+  - ./externalsecret.yaml
   - ./helmrelease.yaml
+configMapGenerator:
+  - name: loki-alerting-rules
+    files:
+      - loki-alerting-rules.yaml=./rules/loki-alerting-rules.yml
+generatorOptions:
+  disableNameSuffixHash: true

kubernetes/apps/observability/loki/app/objectbucketclaim.yaml

@@ -3,7 +3,7 @@
 apiVersion: objectbucket.io/v1alpha1
 kind: ObjectBucketClaim
 metadata:
-  name: loki-bucket
+  name: loki-bucket-v2
 spec:
-  bucketName: loki
+  bucketName: loki-v2
   storageClassName: ceph-bucket

kubernetes/apps/observability/smartctl-exporter/app/kustomization.yaml

@@ -5,12 +5,3 @@ kind: Kustomization
 resources:
   - ./helmrelease.yaml
   - ./prometheusrule.yaml
-configMapGenerator:
-  - name: smartctl-exporter-loki-rules
-    files:
-      - smartctl-exporter.yaml=./resources/lokirule.yaml
-    options:
-      labels:
-        loki_rule: "true"
-generatorOptions:
-  disableNameSuffixHash: true