Adding Vector.

This commit is contained in:
Joseph Hanson 2024-02-29 11:02:27 -06:00
parent 0aac2d2da9
commit 68738a9a1a
11 changed files with 454 additions and 1 deletions

View file

@ -14,4 +14,4 @@ resources:
# - ./snmp-exporter/ks.yaml
- ./thanos/ks.yaml
# - ./unpoller/ks.yaml
# - ./vector/ks.yaml
- ./vector/ks.yaml

View file

@ -0,0 +1,107 @@
---
# yaml-language-server: $schema=https://ks.hsn.dev/helm.toolkit.fluxcd.io/helmrelease_v2beta2.json
apiVersion: helm.toolkit.fluxcd.io/v2beta2
kind: HelmRelease
metadata:
name: vector-agent
spec:
interval: 30m
timeout: 15m
chart:
spec:
chart: app-template
version: 2.6.0
sourceRef:
kind: HelmRepository
name: bjw-s
namespace: flux-system
install:
remediation:
retries: 3
upgrade:
cleanupOnFail: true
remediation:
retries: 3
uninstall:
keepHistory: false
dependsOn:
- name: vector-aggregator
namespace: observability
values:
controllers:
main:
type: daemonset
strategy: RollingUpdate
annotations:
reloader.stakater.com/auto: "true"
containers:
main:
image:
repository: docker.io/timberio/vector
tag: 0.36.0-alpine@sha256:d481b891ca5b36e2ae2a66917cc252ba8aaf30a7904f84b4c7d74206bd3de2e9
env:
PROCFS_ROOT: /host/proc
SYSFS_ROOT: /host/sys
VECTOR_SELF_NODE_NAME:
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: spec.nodeName
VECTOR_SELF_POD_NAME:
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.name
VECTOR_SELF_POD_NAMESPACE:
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
args: ["--config", "/etc/vector/vector.yaml"]
securityContext:
privileged: true
service:
main:
enabled: false
serviceAccount:
create: true
name: vector-agent
persistence:
config:
enabled: true
type: configMap
name: vector-agent-configmap
globalMounts:
- path: /etc/vector/vector.yaml
subPath: vector.yaml
readOnly: true
data:
type: emptyDir
globalMounts:
- path: /vector-data-dir
procfs:
type: hostPath
hostPath: /proc
hostPathType: Directory
globalMounts:
- path: /host/proc
readOnly: true
sysfs:
type: hostPath
hostPath: /sys
hostPathType: Directory
globalMounts:
- path: /host/sys
readOnly: true
var-lib:
type: hostPath
hostPath: /var/lib
hostPathType: Directory
globalMounts:
- readOnly: true
var-log:
type: hostPath
hostPath: /var/log
hostPathType: Directory
globalMounts:
- readOnly: true

View file

@ -0,0 +1,13 @@
---
# yaml-language-server: $schema=https://json.schemastore.org/kustomization
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ./helmrelease.yaml
- ./rbac.yaml
configMapGenerator:
- name: vector-agent-configmap
files:
- vector.yaml=./resources/vector.yaml
generatorOptions:
disableNameSuffixHash: true

View file

@ -0,0 +1,22 @@
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: vector-agent
rules:
- apiGroups: [""]
resources: ["namespaces", "nodes", "pods"]
verbs: ["list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: vector-agent
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: vector-agent
subjects:
- kind: ServiceAccount
name: vector-agent
namespace: observability

View file

@ -0,0 +1,25 @@
---
data_dir: /vector-data-dir
sources:
kubernetes_source:
type: kubernetes_logs
use_apiserver_cache: true
pod_annotation_fields:
container_image: container_image
container_name: container_name
pod_labels: pod_labels
pod_name: pod_name
pod_annotations: ""
namespace_annotation_fields:
namespace_labels: ""
node_annotation_fields:
node_labels: ""
sinks:
kubernetes:
type: vector
compression: true
version: "2"
address: vector-aggregator.observability.svc.cluster.local:6010
inputs: ["kubernetes_source"]

View file

@ -0,0 +1,20 @@
---
# yaml-language-server: $schema=https://ks.hsn.dev/external-secrets.io/externalsecret_v1beta1.json
apiVersion: external-secrets.io/v1beta1
kind: ExternalSecret
metadata:
name: vector-aggregator
spec:
secretStoreRef:
kind: ClusterSecretStore
name: onepassword-connect
target:
name: vector-aggregator-secret
template:
engineVersion: v2
data:
GEOIPUPDATE_ACCOUNT_ID: "{{ .account_id }}"
GEOIPUPDATE_LICENSE_KEY: "{{ .vector_license_key }}"
dataFrom:
- extract:
key: maxmind

View file

@ -0,0 +1,91 @@
---
# yaml-language-server: $schema=https://ks.hsn.dev/helm.toolkit.fluxcd.io/helmrelease_v2beta2.json
apiVersion: helm.toolkit.fluxcd.io/v2beta2
kind: HelmRelease
metadata:
name: &app vector-aggregator
spec:
interval: 30m
timeout: 15m
chart:
spec:
chart: app-template
version: 2.6.0
sourceRef:
kind: HelmRepository
name: bjw-s
namespace: flux-system
install:
remediation:
retries: 3
upgrade:
cleanupOnFail: true
remediation:
retries: 3
uninstall:
keepHistory: false
values:
controllers:
main:
replicas: 2
strategy: RollingUpdate
annotations:
reloader.stakater.com/auto: "true"
initContainers:
init-geoip:
image:
repository: ghcr.io/maxmind/geoipupdate
tag: v6.1.0@sha256:cdd36d36c2e1d353a990a48e5a36d42ee75089d5d9064d80d36d90c147fd2606
env:
GEOIPUPDATE_EDITION_IDS: GeoLite2-City
GEOIPUPDATE_FREQUENCY: "0"
GEOIPUPDATE_VERBOSE: "true"
envFrom:
- secretRef:
name: vector-aggregator-secret
containers:
main:
image:
repository: docker.io/timberio/vector
tag: 0.36.0-alpine@sha256:d481b891ca5b36e2ae2a66917cc252ba8aaf30a7904f84b4c7d74206bd3de2e9
args: ["--config", "/etc/vector/vector.yaml"]
pod:
topologySpreadConstraints:
- maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: DoNotSchedule
labelSelector:
matchLabels:
app.kubernetes.io/name: *app
service:
main:
type: LoadBalancer
annotations:
external-dns.alpha.kubernetes.io/hostname: vector.jahanson.tech
io.cilium/lb-ipam-ips: 10.45.0.4
ports:
http:
port: 8686
journald:
port: 6000
kubernetes:
port: 6010
vyos:
port: 6020
persistence:
config:
enabled: true
type: configMap
name: vector-aggregator-configmap
globalMounts:
- path: /etc/vector/vector.yaml
subPath: vector.yaml
readOnly: true
data:
type: emptyDir
globalMounts:
- path: /vector-data-dir
geoip:
type: emptyDir
globalMounts:
- path: /usr/share/GeoIP

View file

@ -0,0 +1,13 @@
---
# yaml-language-server: $schema=https://json.schemastore.org/kustomization
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ./externalsecret.yaml
- ./helmrelease.yaml
configMapGenerator:
- name: vector-aggregator-configmap
files:
- vector.yaml=./resources/vector.yaml
generatorOptions:
disableNameSuffixHash: true

View file

@ -0,0 +1,132 @@
---
data_dir: /vector-data-dir
api:
enabled: true
address: 0.0.0.0:8686
enrichment_tables:
geoip_table:
type: geoip
path: /usr/share/GeoIP/GeoLite2-City.mmdb
#
# Sources
#
sources:
journald_source:
type: vector
version: "2"
address: 0.0.0.0:6000
kubernetes_source:
type: vector
version: "2"
address: 0.0.0.0:6010
vyos_source:
type: syslog
address: 0.0.0.0:6020
mode: tcp
#
# Transforms
#
transforms:
kubernetes_remap:
type: remap
inputs: ["kubernetes_source"]
source: |
# Standardize 'app' index
.custom_app_name = .pod_labels."app.kubernetes.io/name" || .pod_labels.app || .pod_labels."k8s-app" || "unknown"
# Drop pod_labels
del(.pod_labels)
# [63950.153039] [wan-local-default-D]IN=eth4 OUT= MAC=xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx SRC=xxx.xxx.xxx.xxx DST=xxx.xxx.xxx.xxx LEN=40 TOS=0x00 PREC=0x00 TTL=240 ID=60610 PROTO=TCP SPT=53451 DPT=2002 WINDOW=1024 RES=0x00 SYN URGP=0
vyos_firewall_route:
type: route
inputs: ["vyos_source"]
route:
firewall: |
.facility == "kern" && match!(.message, r'^\[(.*?)\].(.*)')
vyos_firewall_remap:
type: remap
inputs: ["vyos_firewall_route.firewall"]
source: |
# Parse firewall rule message
split_message, split_err = parse_regex(.message, r'^\[.*\].\[(?P<rule>.*?)\](?P<fields>.*)')
if split_err != null {
abort
}
# Extract separate fields from message
split_message.fields, split_err = strip_whitespace(split_message.fields)
if split_err != null {
abort
}
.message, parse_err = parse_key_value(split_message.fields, whitespace: "strict")
if parse_err != null {
abort
}
# Add more information about the triggered rule
.message.RULE, parse_err = parse_regex(split_message.rule, r'^ipv4-(?P<from_zone>\w+)-(?P<to_zone>\w+)-(?P<id>\w+)-(?P<action>\w+)$')
if parse_err != null {
abort
}
vyos_firewall_wan_route:
type: route
inputs: ["vyos_firewall_remap"]
route:
from_wan: .message.RULE.from_zone == "wan"
vyos_firewall_geoip_remap:
type: remap
inputs: ["vyos_firewall_wan_route.from_wan"]
source: |
.geoip = get_enrichment_table_record!(
"geoip_table", {
"ip": .message.SRC
}
)
#
# Sinks
#
sinks:
journald:
inputs: ["journald_source"]
type: loki
endpoint: http://loki-gateway.observability.svc.cluster.local
encoding: { codec: json }
out_of_order_action: accept
remove_label_fields: true
remove_timestamp: true
labels:
hostname: '{{ host }}'
kubernetes:
inputs: ["kubernetes_remap"]
type: loki
endpoint: http://loki-gateway.observability.svc.cluster.local
encoding: { codec: json }
out_of_order_action: accept
remove_label_fields: true
remove_timestamp: true
labels:
app: '{{ custom_app_name }}'
namespace: '{{ kubernetes.pod_namespace }}'
node: '{{ kubernetes.pod_node_name }}'
vyos:
inputs: ["vyos_source", "vyos_firewall_geoip_remap"]
type: loki
endpoint: http://loki-gateway.observability.svc.cluster.local
encoding: { codec: json }
out_of_order_action: accept
remove_label_fields: true
remove_timestamp: true
labels:
hostname: '{{ host }}'

View file

@ -0,0 +1,7 @@
---
# yaml-language-server: $schema=https://json.schemastore.org/kustomization
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ./agent
- ./aggregator

View file

@ -0,0 +1,23 @@
---
# yaml-language-server: $schema=https://ks.hsn.dev/kustomize.toolkit.fluxcd.io/kustomization_v1.json
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: &app vector
namespace: flux-system
spec:
targetNamespace: observability
commonMetadata:
labels:
app.kubernetes.io/name: *app
dependsOn:
- name: external-secrets-stores
path: ./kubernetes/apps/observability/vector/app
prune: true
sourceRef:
kind: GitRepository
name: homelab
wait: false
interval: 30m
retryInterval: 1m
timeout: 15m