Adding Vector.
This commit is contained in:
parent
0aac2d2da9
commit
68738a9a1a
11 changed files with 454 additions and 1 deletions
|
@ -14,4 +14,4 @@ resources:
|
|||
# - ./snmp-exporter/ks.yaml
|
||||
- ./thanos/ks.yaml
|
||||
# - ./unpoller/ks.yaml
|
||||
# - ./vector/ks.yaml
|
||||
- ./vector/ks.yaml
|
107
kubernetes/apps/observability/vector/app/agent/helmrelease.yaml
Normal file
107
kubernetes/apps/observability/vector/app/agent/helmrelease.yaml
Normal file
|
@ -0,0 +1,107 @@
|
|||
---
|
||||
# yaml-language-server: $schema=https://ks.hsn.dev/helm.toolkit.fluxcd.io/helmrelease_v2beta2.json
|
||||
apiVersion: helm.toolkit.fluxcd.io/v2beta2
|
||||
kind: HelmRelease
|
||||
metadata:
|
||||
name: vector-agent
|
||||
spec:
|
||||
interval: 30m
|
||||
timeout: 15m
|
||||
chart:
|
||||
spec:
|
||||
chart: app-template
|
||||
version: 2.6.0
|
||||
sourceRef:
|
||||
kind: HelmRepository
|
||||
name: bjw-s
|
||||
namespace: flux-system
|
||||
install:
|
||||
remediation:
|
||||
retries: 3
|
||||
upgrade:
|
||||
cleanupOnFail: true
|
||||
remediation:
|
||||
retries: 3
|
||||
uninstall:
|
||||
keepHistory: false
|
||||
dependsOn:
|
||||
- name: vector-aggregator
|
||||
namespace: observability
|
||||
values:
|
||||
controllers:
|
||||
main:
|
||||
type: daemonset
|
||||
strategy: RollingUpdate
|
||||
annotations:
|
||||
reloader.stakater.com/auto: "true"
|
||||
containers:
|
||||
main:
|
||||
image:
|
||||
repository: docker.io/timberio/vector
|
||||
tag: 0.36.0-alpine@sha256:d481b891ca5b36e2ae2a66917cc252ba8aaf30a7904f84b4c7d74206bd3de2e9
|
||||
env:
|
||||
PROCFS_ROOT: /host/proc
|
||||
SYSFS_ROOT: /host/sys
|
||||
VECTOR_SELF_NODE_NAME:
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
apiVersion: v1
|
||||
fieldPath: spec.nodeName
|
||||
VECTOR_SELF_POD_NAME:
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
apiVersion: v1
|
||||
fieldPath: metadata.name
|
||||
VECTOR_SELF_POD_NAMESPACE:
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
apiVersion: v1
|
||||
fieldPath: metadata.namespace
|
||||
args: ["--config", "/etc/vector/vector.yaml"]
|
||||
securityContext:
|
||||
privileged: true
|
||||
service:
|
||||
main:
|
||||
enabled: false
|
||||
serviceAccount:
|
||||
create: true
|
||||
name: vector-agent
|
||||
persistence:
|
||||
config:
|
||||
enabled: true
|
||||
type: configMap
|
||||
name: vector-agent-configmap
|
||||
globalMounts:
|
||||
- path: /etc/vector/vector.yaml
|
||||
subPath: vector.yaml
|
||||
readOnly: true
|
||||
data:
|
||||
type: emptyDir
|
||||
globalMounts:
|
||||
- path: /vector-data-dir
|
||||
procfs:
|
||||
type: hostPath
|
||||
hostPath: /proc
|
||||
hostPathType: Directory
|
||||
globalMounts:
|
||||
- path: /host/proc
|
||||
readOnly: true
|
||||
sysfs:
|
||||
type: hostPath
|
||||
hostPath: /sys
|
||||
hostPathType: Directory
|
||||
globalMounts:
|
||||
- path: /host/sys
|
||||
readOnly: true
|
||||
var-lib:
|
||||
type: hostPath
|
||||
hostPath: /var/lib
|
||||
hostPathType: Directory
|
||||
globalMounts:
|
||||
- readOnly: true
|
||||
var-log:
|
||||
type: hostPath
|
||||
hostPath: /var/log
|
||||
hostPathType: Directory
|
||||
globalMounts:
|
||||
- readOnly: true
|
|
@ -0,0 +1,13 @@
|
|||
---
|
||||
# yaml-language-server: $schema=https://json.schemastore.org/kustomization
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- ./helmrelease.yaml
|
||||
- ./rbac.yaml
|
||||
configMapGenerator:
|
||||
- name: vector-agent-configmap
|
||||
files:
|
||||
- vector.yaml=./resources/vector.yaml
|
||||
generatorOptions:
|
||||
disableNameSuffixHash: true
|
22
kubernetes/apps/observability/vector/app/agent/rbac.yaml
Normal file
22
kubernetes/apps/observability/vector/app/agent/rbac.yaml
Normal file
|
@ -0,0 +1,22 @@
|
|||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: vector-agent
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["namespaces", "nodes", "pods"]
|
||||
verbs: ["list", "watch"]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: vector-agent
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: vector-agent
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: vector-agent
|
||||
namespace: observability
|
|
@ -0,0 +1,25 @@
|
|||
---
|
||||
data_dir: /vector-data-dir
|
||||
|
||||
sources:
|
||||
kubernetes_source:
|
||||
type: kubernetes_logs
|
||||
use_apiserver_cache: true
|
||||
pod_annotation_fields:
|
||||
container_image: container_image
|
||||
container_name: container_name
|
||||
pod_labels: pod_labels
|
||||
pod_name: pod_name
|
||||
pod_annotations: ""
|
||||
namespace_annotation_fields:
|
||||
namespace_labels: ""
|
||||
node_annotation_fields:
|
||||
node_labels: ""
|
||||
|
||||
sinks:
|
||||
kubernetes:
|
||||
type: vector
|
||||
compression: true
|
||||
version: "2"
|
||||
address: vector-aggregator.observability.svc.cluster.local:6010
|
||||
inputs: ["kubernetes_source"]
|
|
@ -0,0 +1,20 @@
|
|||
---
|
||||
# yaml-language-server: $schema=https://ks.hsn.dev/external-secrets.io/externalsecret_v1beta1.json
|
||||
apiVersion: external-secrets.io/v1beta1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: vector-aggregator
|
||||
spec:
|
||||
secretStoreRef:
|
||||
kind: ClusterSecretStore
|
||||
name: onepassword-connect
|
||||
target:
|
||||
name: vector-aggregator-secret
|
||||
template:
|
||||
engineVersion: v2
|
||||
data:
|
||||
GEOIPUPDATE_ACCOUNT_ID: "{{ .account_id }}"
|
||||
GEOIPUPDATE_LICENSE_KEY: "{{ .vector_license_key }}"
|
||||
dataFrom:
|
||||
- extract:
|
||||
key: maxmind
|
|
@ -0,0 +1,91 @@
|
|||
---
|
||||
# yaml-language-server: $schema=https://ks.hsn.dev/helm.toolkit.fluxcd.io/helmrelease_v2beta2.json
|
||||
apiVersion: helm.toolkit.fluxcd.io/v2beta2
|
||||
kind: HelmRelease
|
||||
metadata:
|
||||
name: &app vector-aggregator
|
||||
spec:
|
||||
interval: 30m
|
||||
timeout: 15m
|
||||
chart:
|
||||
spec:
|
||||
chart: app-template
|
||||
version: 2.6.0
|
||||
sourceRef:
|
||||
kind: HelmRepository
|
||||
name: bjw-s
|
||||
namespace: flux-system
|
||||
install:
|
||||
remediation:
|
||||
retries: 3
|
||||
upgrade:
|
||||
cleanupOnFail: true
|
||||
remediation:
|
||||
retries: 3
|
||||
uninstall:
|
||||
keepHistory: false
|
||||
values:
|
||||
controllers:
|
||||
main:
|
||||
replicas: 2
|
||||
strategy: RollingUpdate
|
||||
annotations:
|
||||
reloader.stakater.com/auto: "true"
|
||||
initContainers:
|
||||
init-geoip:
|
||||
image:
|
||||
repository: ghcr.io/maxmind/geoipupdate
|
||||
tag: v6.1.0@sha256:cdd36d36c2e1d353a990a48e5a36d42ee75089d5d9064d80d36d90c147fd2606
|
||||
env:
|
||||
GEOIPUPDATE_EDITION_IDS: GeoLite2-City
|
||||
GEOIPUPDATE_FREQUENCY: "0"
|
||||
GEOIPUPDATE_VERBOSE: "true"
|
||||
envFrom:
|
||||
- secretRef:
|
||||
name: vector-aggregator-secret
|
||||
containers:
|
||||
main:
|
||||
image:
|
||||
repository: docker.io/timberio/vector
|
||||
tag: 0.36.0-alpine@sha256:d481b891ca5b36e2ae2a66917cc252ba8aaf30a7904f84b4c7d74206bd3de2e9
|
||||
args: ["--config", "/etc/vector/vector.yaml"]
|
||||
pod:
|
||||
topologySpreadConstraints:
|
||||
- maxSkew: 1
|
||||
topologyKey: kubernetes.io/hostname
|
||||
whenUnsatisfiable: DoNotSchedule
|
||||
labelSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: *app
|
||||
service:
|
||||
main:
|
||||
type: LoadBalancer
|
||||
annotations:
|
||||
external-dns.alpha.kubernetes.io/hostname: vector.jahanson.tech
|
||||
io.cilium/lb-ipam-ips: 10.45.0.4
|
||||
ports:
|
||||
http:
|
||||
port: 8686
|
||||
journald:
|
||||
port: 6000
|
||||
kubernetes:
|
||||
port: 6010
|
||||
vyos:
|
||||
port: 6020
|
||||
persistence:
|
||||
config:
|
||||
enabled: true
|
||||
type: configMap
|
||||
name: vector-aggregator-configmap
|
||||
globalMounts:
|
||||
- path: /etc/vector/vector.yaml
|
||||
subPath: vector.yaml
|
||||
readOnly: true
|
||||
data:
|
||||
type: emptyDir
|
||||
globalMounts:
|
||||
- path: /vector-data-dir
|
||||
geoip:
|
||||
type: emptyDir
|
||||
globalMounts:
|
||||
- path: /usr/share/GeoIP
|
|
@ -0,0 +1,13 @@
|
|||
---
|
||||
# yaml-language-server: $schema=https://json.schemastore.org/kustomization
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- ./externalsecret.yaml
|
||||
- ./helmrelease.yaml
|
||||
configMapGenerator:
|
||||
- name: vector-aggregator-configmap
|
||||
files:
|
||||
- vector.yaml=./resources/vector.yaml
|
||||
generatorOptions:
|
||||
disableNameSuffixHash: true
|
|
@ -0,0 +1,132 @@
|
|||
---
|
||||
data_dir: /vector-data-dir
|
||||
api:
|
||||
enabled: true
|
||||
address: 0.0.0.0:8686
|
||||
|
||||
enrichment_tables:
|
||||
geoip_table:
|
||||
type: geoip
|
||||
path: /usr/share/GeoIP/GeoLite2-City.mmdb
|
||||
|
||||
#
|
||||
# Sources
|
||||
#
|
||||
|
||||
sources:
|
||||
journald_source:
|
||||
type: vector
|
||||
version: "2"
|
||||
address: 0.0.0.0:6000
|
||||
|
||||
kubernetes_source:
|
||||
type: vector
|
||||
version: "2"
|
||||
address: 0.0.0.0:6010
|
||||
|
||||
vyos_source:
|
||||
type: syslog
|
||||
address: 0.0.0.0:6020
|
||||
mode: tcp
|
||||
|
||||
#
|
||||
# Transforms
|
||||
#
|
||||
|
||||
transforms:
|
||||
kubernetes_remap:
|
||||
type: remap
|
||||
inputs: ["kubernetes_source"]
|
||||
source: |
|
||||
# Standardize 'app' index
|
||||
.custom_app_name = .pod_labels."app.kubernetes.io/name" || .pod_labels.app || .pod_labels."k8s-app" || "unknown"
|
||||
# Drop pod_labels
|
||||
del(.pod_labels)
|
||||
|
||||
# [63950.153039] [wan-local-default-D]IN=eth4 OUT= MAC=xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx SRC=xxx.xxx.xxx.xxx DST=xxx.xxx.xxx.xxx LEN=40 TOS=0x00 PREC=0x00 TTL=240 ID=60610 PROTO=TCP SPT=53451 DPT=2002 WINDOW=1024 RES=0x00 SYN URGP=0
|
||||
vyos_firewall_route:
|
||||
type: route
|
||||
inputs: ["vyos_source"]
|
||||
route:
|
||||
firewall: |
|
||||
.facility == "kern" && match!(.message, r'^\[(.*?)\].(.*)')
|
||||
|
||||
vyos_firewall_remap:
|
||||
type: remap
|
||||
inputs: ["vyos_firewall_route.firewall"]
|
||||
source: |
|
||||
# Parse firewall rule message
|
||||
split_message, split_err = parse_regex(.message, r'^\[.*\].\[(?P<rule>.*?)\](?P<fields>.*)')
|
||||
if split_err != null {
|
||||
abort
|
||||
}
|
||||
# Extract separate fields from message
|
||||
split_message.fields, split_err = strip_whitespace(split_message.fields)
|
||||
if split_err != null {
|
||||
abort
|
||||
}
|
||||
.message, parse_err = parse_key_value(split_message.fields, whitespace: "strict")
|
||||
if parse_err != null {
|
||||
abort
|
||||
}
|
||||
# Add more information about the triggered rule
|
||||
.message.RULE, parse_err = parse_regex(split_message.rule, r'^ipv4-(?P<from_zone>\w+)-(?P<to_zone>\w+)-(?P<id>\w+)-(?P<action>\w+)$')
|
||||
if parse_err != null {
|
||||
abort
|
||||
}
|
||||
|
||||
vyos_firewall_wan_route:
|
||||
type: route
|
||||
inputs: ["vyos_firewall_remap"]
|
||||
route:
|
||||
from_wan: .message.RULE.from_zone == "wan"
|
||||
|
||||
vyos_firewall_geoip_remap:
|
||||
type: remap
|
||||
inputs: ["vyos_firewall_wan_route.from_wan"]
|
||||
source: |
|
||||
.geoip = get_enrichment_table_record!(
|
||||
"geoip_table", {
|
||||
"ip": .message.SRC
|
||||
}
|
||||
)
|
||||
|
||||
#
|
||||
# Sinks
|
||||
#
|
||||
|
||||
sinks:
|
||||
journald:
|
||||
inputs: ["journald_source"]
|
||||
type: loki
|
||||
endpoint: http://loki-gateway.observability.svc.cluster.local
|
||||
encoding: { codec: json }
|
||||
out_of_order_action: accept
|
||||
remove_label_fields: true
|
||||
remove_timestamp: true
|
||||
labels:
|
||||
hostname: '{{ host }}'
|
||||
|
||||
kubernetes:
|
||||
inputs: ["kubernetes_remap"]
|
||||
type: loki
|
||||
endpoint: http://loki-gateway.observability.svc.cluster.local
|
||||
encoding: { codec: json }
|
||||
out_of_order_action: accept
|
||||
remove_label_fields: true
|
||||
remove_timestamp: true
|
||||
labels:
|
||||
app: '{{ custom_app_name }}'
|
||||
namespace: '{{ kubernetes.pod_namespace }}'
|
||||
node: '{{ kubernetes.pod_node_name }}'
|
||||
|
||||
vyos:
|
||||
inputs: ["vyos_source", "vyos_firewall_geoip_remap"]
|
||||
type: loki
|
||||
endpoint: http://loki-gateway.observability.svc.cluster.local
|
||||
encoding: { codec: json }
|
||||
out_of_order_action: accept
|
||||
remove_label_fields: true
|
||||
remove_timestamp: true
|
||||
labels:
|
||||
hostname: '{{ host }}'
|
|
@ -0,0 +1,7 @@
|
|||
---
|
||||
# yaml-language-server: $schema=https://json.schemastore.org/kustomization
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- ./agent
|
||||
- ./aggregator
|
23
kubernetes/apps/observability/vector/ks.yaml
Normal file
23
kubernetes/apps/observability/vector/ks.yaml
Normal file
|
@ -0,0 +1,23 @@
|
|||
---
|
||||
# yaml-language-server: $schema=https://ks.hsn.dev/kustomize.toolkit.fluxcd.io/kustomization_v1.json
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: &app vector
|
||||
namespace: flux-system
|
||||
spec:
|
||||
targetNamespace: observability
|
||||
commonMetadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: *app
|
||||
dependsOn:
|
||||
- name: external-secrets-stores
|
||||
path: ./kubernetes/apps/observability/vector/app
|
||||
prune: true
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: homelab
|
||||
wait: false
|
||||
interval: 30m
|
||||
retryInterval: 1m
|
||||
timeout: 15m
|
Loading…
Reference in a new issue