Added monitoring to civo cluster

Signed-off-by: Marcus Noble <github@marcusnoble.co.uk>
This commit is contained in:
Marcus Noble 2022-05-31 18:40:49 +01:00
parent b1d821922f
commit f942f9e358
Signed by: AverageMarcus
GPG Key ID: B8F2DB8A7AEBAF78
7 changed files with 982 additions and 0 deletions

View File

@ -0,0 +1,24 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: monitoring-civo
namespace: argocd
finalizers:
- resources-finalizer.argocd.argoproj.io
spec:
project: cluster.fun
destination:
namespace: monitoring
name: civo
source:
path: manifests/monitoring-civo
repoURL: "https://git.cluster.fun/AverageMarcus/cluster.fun.git"
targetRevision: HEAD
syncPolicy:
automated: {}
syncOptions:
- CreateNamespace=true
ignoreDifferences:
- kind: Secret
jsonPointers:
- /data

View File

@ -0,0 +1,24 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: proxy-civo
namespace: argocd
finalizers:
- resources-finalizer.argocd.argoproj.io
spec:
project: cluster.fun
destination:
namespace: proxy-civo
name: civo
source:
path: manifests/proxy-civo
repoURL: "https://git.cluster.fun/AverageMarcus/cluster.fun.git"
targetRevision: HEAD
syncPolicy:
automated: {}
syncOptions:
- CreateNamespace=true
ignoreDifferences:
- kind: Secret
jsonPointers:
- /data

View File

@ -0,0 +1,255 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: kube-state-metrics
namespace: monitoring
labels:
app.kubernetes.io/name: kube-state-metrics
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
app.kubernetes.io/name: kube-state-metrics
name: kube-state-metrics
rules:
- apiGroups: ["certificates.k8s.io"]
resources:
- certificatesigningrequests
verbs: ["list", "watch"]
- apiGroups: [""]
resources:
- configmaps
verbs: ["list", "watch"]
- apiGroups: ["batch"]
resources:
- cronjobs
verbs: ["list", "watch"]
- apiGroups: ["extensions", "apps"]
resources:
- daemonsets
verbs: ["list", "watch"]
- apiGroups: ["extensions", "apps"]
resources:
- deployments
verbs: ["list", "watch"]
- apiGroups: [""]
resources:
- endpoints
verbs: ["list", "watch"]
- apiGroups: ["autoscaling"]
resources:
- horizontalpodautoscalers
verbs: ["list", "watch"]
- apiGroups: ["extensions", "networking.k8s.io"]
resources:
- ingresses
verbs: ["list", "watch"]
- apiGroups: ["batch"]
resources:
- jobs
verbs: ["list", "watch"]
- apiGroups: [""]
resources:
- limitranges
verbs: ["list", "watch"]
- apiGroups: ["admissionregistration.k8s.io"]
resources:
- mutatingwebhookconfigurations
verbs: ["list", "watch"]
- apiGroups: [""]
resources:
- namespaces
verbs: ["list", "watch"]
- apiGroups: ["networking.k8s.io"]
resources:
- networkpolicies
verbs: ["list", "watch"]
- apiGroups: [""]
resources:
- nodes
verbs: ["list", "watch"]
- apiGroups: [""]
resources:
- persistentvolumeclaims
verbs: ["list", "watch"]
- apiGroups: [""]
resources:
- persistentvolumes
verbs: ["list", "watch"]
- apiGroups: ["policy"]
resources:
- poddisruptionbudgets
verbs: ["list", "watch"]
- apiGroups: [""]
resources:
- pods
verbs: ["list", "watch"]
- apiGroups: ["extensions", "apps"]
resources:
- replicasets
verbs: ["list", "watch"]
- apiGroups: [""]
resources:
- replicationcontrollers
verbs: ["list", "watch"]
- apiGroups: [""]
resources:
- resourcequotas
verbs: ["list", "watch"]
- apiGroups: [""]
resources:
- secrets
verbs: ["list", "watch"]
- apiGroups: [""]
resources:
- services
verbs: ["list", "watch"]
- apiGroups: ["apps"]
resources:
- statefulsets
verbs: ["list", "watch"]
- apiGroups: ["storage.k8s.io"]
resources:
- storageclasses
verbs: ["list", "watch"]
- apiGroups: ["admissionregistration.k8s.io"]
resources:
- validatingwebhookconfigurations
verbs: ["list", "watch"]
- apiGroups: ["storage.k8s.io"]
resources:
- volumeattachments
verbs: ["list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
labels:
app.kubernetes.io/name: kube-state-metrics
name: kube-state-metrics
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: kube-state-metrics
subjects:
- kind: ServiceAccount
name: kube-state-metrics
namespace: monitoring
---
apiVersion: v1
kind: Service
metadata:
name: kube-state-metrics
namespace: monitoring
labels:
app.kubernetes.io/name: kube-state-metrics
annotations:
prometheus.io/scrape: 'true'
spec:
type: "ClusterIP"
ports:
- name: "http"
protocol: TCP
port: 8080
targetPort: 8080
selector:
app.kubernetes.io/name: kube-state-metrics
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: kube-state-metrics
namespace: monitoring
labels:
app.kubernetes.io/name: kube-state-metrics
spec:
selector:
matchLabels:
app.kubernetes.io/name: kube-state-metrics
replicas: 1
template:
metadata:
labels:
app.kubernetes.io/name: kube-state-metrics
spec:
serviceAccountName: kube-state-metrics
securityContext:
fsGroup: 65534
runAsGroup: 65534
runAsUser: 65534
containers:
- name: kube-state-metrics
args:
#- --resources=certificatesigningrequests
- --resources=configmaps
- --resources=cronjobs
- --resources=daemonsets
- --resources=deployments
#- --resources=endpoints
#- --resources=horizontalpodautoscalers
- --resources=ingresses
- --resources=jobs
#- --resources=limitranges
- --resources=mutatingwebhookconfigurations
- --resources=namespaces
#- --resources=networkpolicies
- --resources=nodes
- --resources=persistentvolumeclaims
- --resources=persistentvolumes
- --resources=poddisruptionbudgets
- --resources=pods
- --resources=replicasets
#- --resources=replicationcontrollers
#- --resources=resourcequotas
- --resources=secrets
- --resources=services
- --resources=statefulsets
- --resources=storageclasses
- --resources=validatingwebhookconfigurations
#- --resources=volumeattachments
imagePullPolicy: IfNotPresent
image: "k8s.gcr.io/kube-state-metrics/kube-state-metrics:v2.4.2"
ports:
- containerPort: 8080
livenessProbe:
httpGet:
path: /healthz
port: 8080
initialDelaySeconds: 5
timeoutSeconds: 5
readinessProbe:
httpGet:
path: /
port: 8080
initialDelaySeconds: 5
timeoutSeconds: 5
---

View File

@ -0,0 +1,64 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: prometheus-server
namespace: monitoring
labels:
app.kubernetes.io/name: prometheus
app.kubernetes.io/component: server
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
app.kubernetes.io/name: prometheus
app.kubernetes.io/component: server
name: prometheus-server
rules:
- apiGroups:
- ""
resources:
- nodes
- nodes/proxy
- nodes/metrics
- services
- endpoints
- pods
- ingresses
- configmaps
verbs:
- get
- list
- watch
- apiGroups:
- "extensions"
- "networking.k8s.io"
resources:
- ingresses/status
- ingresses
verbs:
- get
- list
- watch
- nonResourceURLs:
- "/metrics"
verbs:
- get
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
labels:
app.kubernetes.io/name: prometheus
app.kubernetes.io/component: server
name: prometheus-server
subjects:
- kind: ServiceAccount
name: prometheus-server
namespace: monitoring
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: prometheus-server
---

View File

@ -0,0 +1,331 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: promtail
namespace: monitoring
labels:
app.kubernetes.io/name: promtail
---
apiVersion: policy/v1beta1
kind: PodSecurityPolicy
metadata:
name: promtail
namespace: monitoring
labels:
app.kubernetes.io/name: promtail
spec:
allowPrivilegeEscalation: false
fsGroup:
rule: RunAsAny
hostIPC: false
hostNetwork: false
hostPID: false
privileged: false
readOnlyRootFilesystem: true
requiredDropCapabilities:
- ALL
runAsUser:
rule: RunAsAny
seLinux:
rule: RunAsAny
supplementalGroups:
rule: RunAsAny
volumes:
- secret
- configMap
- hostPath
- projected
- downwardAPI
- emptyDir
---
apiVersion: v1
kind: ConfigMap
metadata:
name: promtail
namespace: monitoring
labels:
app.kubernetes.io/name: promtail
data:
promtail.yaml: |
client:
backoff_config:
max_period: 5m
max_retries: 10
min_period: 500ms
batchsize: 1048576
batchwait: 1s
external_labels: {}
timeout: 10s
positions:
filename: /run/promtail/positions.yaml
server:
http_listen_port: 3101
clients:
- url: http://loki.proxy-civo.svc:80/loki/api/v1/push
external_labels:
kubernetes_cluster: civo
target_config:
sync_period: 10s
scrape_configs:
- job_name: kubernetes-pods
pipeline_stages:
- docker: {}
- cri: {}
- match:
selector: '{app="weave-net"}'
action: drop
- match:
selector: '{filename=~".*konnectivity.*"}'
action: drop
- match:
selector: '{name=~".*"} |~ ".*/healthz.*"'
action: drop
- match:
selector: '{name=~".*"} |~ ".*/api/health.*"'
action: drop
- match:
selector: '{name=~".*"} |~ ".*kube-probe/.*"'
action: drop
- match:
selector: '{app="internal-proxy"}'
action: drop
- match:
selector: '{app="non-auth-proxy"}'
action: drop
- match:
selector: '{app="vpa"}'
action: drop
- match:
selector: '{app="promtail"}'
action: drop
- match:
selector: '{app="csi-node"}'
action: drop
- match:
selector: '{app="victoria-metrics"}'
action: drop
- match:
selector: '{app="git-sync"}'
action: drop
- match:
selector: '{app="ingress-nginx"}'
stages:
- json:
expressions:
request_host: host
request_path: path
request_method: method
response_status: status
- drop:
source: "request_path"
value: "/healthz"
- drop:
source: "request_path"
value: "/health"
- labels:
request_host:
request_method:
response_status:
kubernetes_sd_configs:
- role: pod
relabel_configs:
- source_labels:
- __meta_kubernetes_pod_controller_name
regex: ([0-9a-z-.]+?)(-[0-9a-f]{8,10})?
action: replace
target_label: __tmp_controller_name
- source_labels:
- __meta_kubernetes_pod_label_app_kubernetes_io_name
- __meta_kubernetes_pod_label_app
- __tmp_controller_name
- __meta_kubernetes_pod_name
regex: ^;*([^;]+)(;.*)?$
action: replace
target_label: app
- source_labels:
- __meta_kubernetes_pod_label_app_kubernetes_io_component
- __meta_kubernetes_pod_label_component
regex: ^;*([^;]+)(;.*)?$
action: replace
target_label: component
- action: replace
source_labels:
- __meta_kubernetes_pod_node_name
target_label: node_name
- action: replace
source_labels:
- __meta_kubernetes_namespace
target_label: namespace
- action: replace
replacement: $1
separator: /
source_labels:
- namespace
- app
target_label: job
- action: replace
source_labels:
- __meta_kubernetes_pod_name
target_label: pod
- action: replace
source_labels:
- __meta_kubernetes_pod_container_name
target_label: container
- action: replace
replacement: /var/log/pods/*$1/*.log
separator: /
source_labels:
- __meta_kubernetes_pod_uid
- __meta_kubernetes_pod_container_name
target_label: __path__
- action: replace
replacement: /var/log/pods/*$1/*.log
regex: true/(.*)
separator: /
source_labels:
- __meta_kubernetes_pod_annotationpresent_kubernetes_io_config_hash
- __meta_kubernetes_pod_annotation_kubernetes_io_config_hash
- __meta_kubernetes_pod_container_name
target_label: __path__
- action: labelmap
regex: __meta_kubernetes_pod_label_(.+)
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: promtail-clusterrole
labels:
app.kubernetes.io/name: promtail
rules:
- apiGroups: [""] # "" indicates the core API group
resources:
- nodes
- nodes/proxy
- services
- endpoints
- pods
verbs: ["get", "watch", "list"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: promtail-clusterrolebinding
labels:
app.kubernetes.io/name: promtail
subjects:
- kind: ServiceAccount
name: promtail
namespace: monitoring
roleRef:
kind: ClusterRole
name: promtail-clusterrole
apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: promtail
namespace: monitoring
labels:
app.kubernetes.io/name: promtail
rules:
- apiGroups: ['extensions']
resources: ['podsecuritypolicies']
verbs: ['use']
resourceNames: [promtail]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: promtail
namespace: monitoring
labels:
app.kubernetes.io/name: promtail
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: promtail
subjects:
- kind: ServiceAccount
name: promtail
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: promtail
namespace: monitoring
labels:
app.kubernetes.io/name: promtail
annotations:
configmap.reloader.stakater.com/reload: "promtail"
spec:
selector:
matchLabels:
app.kubernetes.io/name: promtail
template:
metadata:
labels:
app.kubernetes.io/name: promtail
annotations:
prometheus.io/port: http-metrics
prometheus.io/scrape: "true"
spec:
serviceAccountName: promtail
containers:
- name: promtail
image: "grafana/promtail:2.5.0"
imagePullPolicy: IfNotPresent
args:
- "-config.file=/etc/promtail/promtail.yaml"
volumeMounts:
- name: config
mountPath: /etc/promtail
- name: run
mountPath: /run/promtail
- mountPath: /var/lib/docker/containers
name: docker
readOnly: true
- mountPath: /var/log/pods
name: pods
readOnly: true
env:
- name: HOSTNAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
ports:
- containerPort: 3101
name: http-metrics
securityContext:
readOnlyRootFilesystem: true
runAsGroup: 0
runAsUser: 0
readinessProbe:
failureThreshold: 5
httpGet:
path: /ready
port: http-metrics
initialDelaySeconds: 10
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
tolerations:
- effect: NoSchedule
key: node-role.kubernetes.io/master
operator: Exists
volumes:
- name: config
configMap:
name: promtail
- name: run
hostPath:
path: /run/promtail
- hostPath:
path: /var/lib/docker/containers
name: docker
- hostPath:
path: /var/log/pods
name: pods
---

View File

@ -0,0 +1,153 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: vmagent
namespace: monitoring
labels:
app.kubernetes.io/name: victoria-metrics
app.kubernetes.io/component: agent
data:
prometheus.yml: |
global:
scrape_interval: 1m
external_labels:
source: civo
agent: vmagent
scrape_configs:
- job_name: 'vmagent'
static_configs:
- targets: ['localhost:8429']
- bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
job_name: kubernetes-nodes
kubernetes_sd_configs:
- role: node
relabel_configs:
- action: labelmap
regex: __meta_kubernetes_node_label_(.+)
- replacement: kubernetes.default.svc:443
target_label: __address__
- regex: (.+)
replacement: /api/v1/nodes/$1/proxy/metrics
source_labels:
- __meta_kubernetes_node_name
target_label: __metrics_path__
scheme: https
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
insecure_skip_verify: true
- job_name: kubernetes-service-endpoints
kubernetes_sd_configs:
- role: endpoints
relabel_configs:
- action: keep
regex: true
source_labels:
- __meta_kubernetes_service_annotation_prometheus_io_scrape
- action: replace
regex: (https?)
source_labels:
- __meta_kubernetes_service_annotation_prometheus_io_scheme
target_label: __scheme__
- action: replace
regex: (.+)
source_labels:
- __meta_kubernetes_service_annotation_prometheus_io_path
target_label: __metrics_path__
- action: replace
regex: ([^:]+)(?::\d+)?;(\d+)
replacement: $1:$2
source_labels:
- __address__
- __meta_kubernetes_service_annotation_prometheus_io_port
target_label: __address__
- action: labelmap
regex: __meta_kubernetes_service_label_(.+)
- action: replace
source_labels:
- __meta_kubernetes_namespace
target_label: kubernetes_namespace
- action: replace
source_labels:
- __meta_kubernetes_service_name
target_label: kubernetes_name
- action: replace
source_labels:
- __meta_kubernetes_pod_node_name
target_label: kubernetes_node
- job_name: kubernetes-pods
kubernetes_sd_configs:
- role: pod
relabel_configs:
- action: keep
regex: true
source_labels:
- __meta_kubernetes_pod_annotation_prometheus_io_scrape
- action: replace
regex: (.+)
source_labels:
- __meta_kubernetes_pod_annotation_prometheus_io_path
target_label: __metrics_path__
- action: replace
regex: ([^:]+)(?::\d+)?;(\d+)
replacement: $1:$2
source_labels:
- __address__
- __meta_kubernetes_pod_annotation_prometheus_io_port
target_label: __address__
- action: labelmap
regex: __meta_kubernetes_pod_label_(.+)
- action: replace
source_labels:
- __meta_kubernetes_namespace
target_label: kubernetes_namespace
- action: replace
source_labels:
- __meta_kubernetes_pod_name
target_label: kubernetes_pod_name
- action: drop
regex: Pending|Succeeded|Failed
source_labels:
- __meta_kubernetes_pod_phase
---
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app.kubernetes.io/name: victoria-metrics
app.kubernetes.io/component: agent
name: vmagent
namespace: monitoring
spec:
strategy:
type: Recreate
selector:
matchLabels:
app.kubernetes.io/name: victoria-metrics
app.kubernetes.io/component: agent
replicas: 1
template:
metadata:
labels:
app.kubernetes.io/name: victoria-metrics
app.kubernetes.io/component: agent
spec:
serviceAccountName: prometheus-server
containers:
- name: vmagent
image: "victoriametrics/vmagent:v1.77.2"
imagePullPolicy: "IfNotPresent"
args:
- -remoteWrite.url=http://prometheus.proxy-civo.svc/api/v1/write
- -remoteWrite.showURL
- -promscrape.config=/config/prometheus.yml
volumeMounts:
- name: config-volume
mountPath: /config
volumes:
- name: config-volume
configMap:
name: vmagent
---

View File

@ -0,0 +1,131 @@
apiVersion: v1
kind: Secret
metadata:
name: tailscale-auth
namespace: proxy-civo
annotations:
kube-1password: 2cqycmsgv5r7vcyvjpblcl2l4y
kube-1password/vault: Kubernetes
type: Opaque
---
apiVersion: v1
kind: ConfigMap
metadata:
name: host-mappings
namespace: proxy-civo
labels:
app: proxy
data:
mapping.json: |
{
"prometheus.proxy-civo.svc": "prometheus.cluster.local",
"loki.proxy-civo.svc": "loki.cluster.local",
"loki.proxy-civo.svc:80": "loki.cluster.local"
}
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: internal-proxy
namespace: proxy-civo
labels:
app: internal-proxy
annotations:
configmap.reloader.stakater.com/reload: "host-mappings"
spec:
replicas: 1
selector:
matchLabels:
app: internal-proxy
template:
metadata:
labels:
app: internal-proxy
spec:
dnsPolicy: None
dnsConfig:
nameservers:
- 100.100.100.100
containers:
- name: proxy
image: rg.fr-par.scw.cloud/averagemarcus/proxy:latest
imagePullPolicy: Always
env:
- name: PROXY_DESTINATION
value: talos.averagemarcus.github.beta.tailscale.net
- name: PORT
value: "8080"
ports:
- containerPort: 8080
protocol: TCP
volumeMounts:
- name: host-mappings
mountPath: /config/
- name: tailscale
image: ghcr.io/tailscale/tailscale:v1.22
imagePullPolicy: Always
env:
- name: AUTH_KEY
valueFrom:
secretKeyRef:
name: tailscale-auth
key: password
securityContext:
capabilities:
add:
- NET_ADMIN
command:
- sh
- -c
- |
export PATH=$PATH:/tailscale/bin
if [[ ! -d /dev/net ]]; then mkdir -p /dev/net; fi
if [[ ! -c /dev/net/tun ]]; then mknod /dev/net/tun c 10 200; fi
echo "Starting tailscaled"
tailscaled --socket=/tmp/tailscaled.sock &
PID=$!
echo "Running tailscale up"
tailscale --socket=/tmp/tailscaled.sock up \
--accept-dns=true \
--authkey=${AUTH_KEY} \
--hostname=proxy-civo-internal-proxy
wait ${PID}
volumes:
- name: host-mappings
configMap:
name: host-mappings
---
apiVersion: v1
kind: Service
metadata:
name: loki
namespace: proxy-civo
labels:
app: internal-proxy
spec:
ports:
- name: http
port: 80
protocol: TCP
targetPort: 8080
selector:
app: internal-proxy
type: ClusterIP
---
apiVersion: v1
kind: Service
metadata:
name: prometheus
namespace: proxy-civo
labels:
app: internal-proxy
spec:
ports:
- name: http
port: 80
protocol: TCP
targetPort: 8080
selector:
app: internal-proxy
type: ClusterIP
---