From 769fdff851cd6778b6f64e043f14a83c69307863 Mon Sep 17 00:00:00 2001 From: Marcus Noble Date: Sat, 27 Nov 2021 21:02:05 +0000 Subject: [PATCH] Updated promtail config Signed-off-by: Marcus Noble --- manifests/monitoring/promtail.yaml | 854 ++++++++++++++++------------- 1 file changed, 467 insertions(+), 387 deletions(-) diff --git a/manifests/monitoring/promtail.yaml b/manifests/monitoring/promtail.yaml index 8fb7705..217eb28 100644 --- a/manifests/monitoring/promtail.yaml +++ b/manifests/monitoring/promtail.yaml @@ -67,7 +67,7 @@ data: target_config: sync_period: 10s scrape_configs: - - job_name: kubernetes-pods-name + - job_name: kubernetes-pods pipeline_stages: - docker: {} - match: @@ -85,398 +85,478 @@ data: - match: selector: '{app="internal-proxy"}' action: drop - # - match: - # selector: '{k8s_app="traefik-ingress-lb"}' - # stages: - # - json: - # expressions: - # request_host: RequestHost - # request_path: RequestPath - # error: error - # - drop: - # source: "request_path" - # value: "/healthz" - # - template: - # source: has_error - # template: '{{ if .error }}true{{ else }}false{{ end }}' - # - labels: - # request_host: - # has_error: kubernetes_sd_configs: - - role: pod + - role: pod relabel_configs: - - source_labels: - - __meta_kubernetes_pod_label_name - target_label: __service__ - - source_labels: - - __meta_kubernetes_pod_node_name - target_label: __host__ - - action: drop - regex: '' - source_labels: - - __service__ - - action: labelmap - regex: __meta_kubernetes_pod_label_(.+) - - action: replace - replacement: $1 - separator: / - source_labels: - - __meta_kubernetes_namespace - - __service__ - target_label: job - - action: replace - source_labels: - - __meta_kubernetes_namespace - target_label: namespace - - action: replace - source_labels: - - __meta_kubernetes_pod_name - target_label: pod - - action: replace - source_labels: - - __meta_kubernetes_pod_container_name - target_label: container - - replacement: /var/log/pods/*$1/*.log - separator: / - source_labels: - - __meta_kubernetes_pod_uid - - __meta_kubernetes_pod_container_name - target_label: __path__ + - source_labels: + - __meta_kubernetes_pod_controller_name + regex: ([0-9a-z-.]+?)(-[0-9a-f]{8,10})? + action: replace + target_label: __tmp_controller_name + - source_labels: + - __meta_kubernetes_pod_label_app_kubernetes_io_name + - __meta_kubernetes_pod_label_app + - __tmp_controller_name + - __meta_kubernetes_pod_name + regex: ^;*([^;]+)(;.*)?$ + action: replace + target_label: app + - source_labels: + - __meta_kubernetes_pod_label_app_kubernetes_io_component + - __meta_kubernetes_pod_label_component + regex: ^;*([^;]+)(;.*)?$ + action: replace + target_label: component + - action: replace + source_labels: + - __meta_kubernetes_pod_node_name + target_label: node_name + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + replacement: $1 + separator: / + source_labels: + - namespace + - app + target_label: job + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: pod + - action: replace + source_labels: + - __meta_kubernetes_pod_container_name + target_label: container + - action: replace + replacement: /var/log/pods/*$1/*.log + separator: / + source_labels: + - __meta_kubernetes_pod_uid + - __meta_kubernetes_pod_container_name + target_label: __path__ + - action: replace + replacement: /var/log/pods/*$1/*.log + regex: true/(.*) + separator: / + source_labels: + - __meta_kubernetes_pod_annotationpresent_kubernetes_io_config_hash + - __meta_kubernetes_pod_annotation_kubernetes_io_config_hash + - __meta_kubernetes_pod_container_name + target_label: __path__ - - job_name: kubernetes-pods-app - pipeline_stages: - - docker: {} - - match: - selector: '{name="weave-net"}' - action: drop - - match: - selector: '{filename=~".*konnectivity.*"}' - action: drop - - match: - selector: '{name=~".*"} |~ ".*/healthz.*"' - action: drop - - match: - selector: '{name=~".*"} |~ ".*kube-probe/.*"' - action: drop - - match: - selector: '{app="internal-proxy"}' - action: drop - # - match: - # selector: '{k8s_app="traefik-ingress-lb"}' - # stages: - # - json: - # expressions: - # request_host: RequestHost - # request_path: RequestPath - # error: error - # - drop: - # source: "request_path" - # value: "/healthz" - # - template: - # source: has_error - # template: '{{ if .error }}true{{ else }}false{{ end }}' - # - labels: - # request_host: - # has_error: - kubernetes_sd_configs: - - role: pod - relabel_configs: - - action: drop - regex: .+ - source_labels: - - __meta_kubernetes_pod_label_name - - source_labels: - - __meta_kubernetes_pod_label_app - target_label: __service__ - - source_labels: - - __meta_kubernetes_pod_node_name - target_label: __host__ - - action: drop - regex: '' - source_labels: - - __service__ - - action: labelmap - regex: __meta_kubernetes_pod_label_(.+) - - action: replace - replacement: $1 - separator: / - source_labels: - - __meta_kubernetes_namespace - - __service__ - target_label: job - - action: replace - source_labels: - - __meta_kubernetes_namespace - target_label: namespace - - action: replace - source_labels: - - __meta_kubernetes_pod_name - target_label: pod - - action: replace - source_labels: - - __meta_kubernetes_pod_container_name - target_label: container - - replacement: /var/log/pods/*$1/*.log - separator: / - source_labels: - - __meta_kubernetes_pod_uid - - __meta_kubernetes_pod_container_name - target_label: __path__ + # - job_name: kubernetes-pods-name + # pipeline_stages: + # - docker: {} + # - match: + # selector: '{name="weave-net"}' + # action: drop + # - match: + # selector: '{filename=~".*konnectivity.*"}' + # action: drop + # - match: + # selector: '{name=~".*"} |~ ".*/healthz.*"' + # action: drop + # - match: + # selector: '{name=~".*"} |~ ".*kube-probe/.*"' + # action: drop + # - match: + # selector: '{app="internal-proxy"}' + # action: drop + # # - match: + # # selector: '{k8s_app="traefik-ingress-lb"}' + # # stages: + # # - json: + # # expressions: + # # request_host: RequestHost + # # request_path: RequestPath + # # error: error + # # - drop: + # # source: "request_path" + # # value: "/healthz" + # # - template: + # # source: has_error + # # template: '{{ if .error }}true{{ else }}false{{ end }}' + # # - labels: + # # request_host: + # # has_error: + # kubernetes_sd_configs: + # - role: pod + # relabel_configs: + # - source_labels: + # - __meta_kubernetes_pod_label_name + # target_label: __service__ + # - source_labels: + # - __meta_kubernetes_pod_node_name + # target_label: __host__ + # - action: drop + # regex: '' + # source_labels: + # - __service__ + # - action: labelmap + # regex: __meta_kubernetes_pod_label_(.+) + # - action: replace + # replacement: $1 + # separator: / + # source_labels: + # - __meta_kubernetes_namespace + # - __service__ + # target_label: job + # - action: replace + # source_labels: + # - __meta_kubernetes_namespace + # target_label: namespace + # - action: replace + # source_labels: + # - __meta_kubernetes_pod_name + # target_label: pod + # - action: replace + # source_labels: + # - __meta_kubernetes_pod_container_name + # target_label: container + # - replacement: /var/log/pods/*$1/*.log + # separator: / + # source_labels: + # - __meta_kubernetes_pod_uid + # - __meta_kubernetes_pod_container_name + # target_label: __path__ - - job_name: kubernetes-pods-direct-controllers - pipeline_stages: - - docker: {} - - match: - selector: '{name="weave-net"}' - action: drop - - match: - selector: '{filename=~".*konnectivity.*"}' - action: drop - - match: - selector: '{name=~".*"} |~ ".*/healthz.*"' - action: drop - - match: - selector: '{name=~".*"} |~ ".*kube-probe/.*"' - action: drop - - match: - selector: '{app="internal-proxy"}' - action: drop - # - match: - # selector: '{k8s_app="traefik-ingress-lb"}' - # stages: - # - json: - # expressions: - # request_host: RequestHost - # request_path: RequestPath - # error: error - # - drop: - # source: "request_path" - # value: "/healthz" - # - template: - # source: has_error - # template: '{{ if .error }}true{{ else }}false{{ end }}' - # - labels: - # request_host: - # has_error: - kubernetes_sd_configs: - - role: pod - relabel_configs: - - action: drop - regex: .+ - separator: '' - source_labels: - - __meta_kubernetes_pod_label_name - - __meta_kubernetes_pod_label_app - - action: drop - regex: '[0-9a-z-.]+-[0-9a-f]{8,10}' - source_labels: - - __meta_kubernetes_pod_controller_name - - source_labels: - - __meta_kubernetes_pod_controller_name - target_label: __service__ - - source_labels: - - __meta_kubernetes_pod_node_name - target_label: __host__ - - action: drop - regex: '' - source_labels: - - __service__ - - action: labelmap - regex: __meta_kubernetes_pod_label_(.+) - - action: replace - replacement: $1 - separator: / - source_labels: - - __meta_kubernetes_namespace - - __service__ - target_label: job - - action: replace - source_labels: - - __meta_kubernetes_namespace - target_label: namespace - - action: replace - source_labels: - - __meta_kubernetes_pod_name - target_label: pod - - action: replace - source_labels: - - __meta_kubernetes_pod_container_name - target_label: container - - replacement: /var/log/pods/*$1/*.log - separator: / - source_labels: - - __meta_kubernetes_pod_uid - - __meta_kubernetes_pod_container_name - target_label: __path__ + # - job_name: kubernetes-pods-app + # pipeline_stages: + # - docker: {} + # - match: + # selector: '{name="weave-net"}' + # action: drop + # - match: + # selector: '{filename=~".*konnectivity.*"}' + # action: drop + # - match: + # selector: '{name=~".*"} |~ ".*/healthz.*"' + # action: drop + # - match: + # selector: '{name=~".*"} |~ ".*kube-probe/.*"' + # action: drop + # - match: + # selector: '{app="internal-proxy"}' + # action: drop + # # - match: + # # selector: '{k8s_app="traefik-ingress-lb"}' + # # stages: + # # - json: + # # expressions: + # # request_host: RequestHost + # # request_path: RequestPath + # # error: error + # # - drop: + # # source: "request_path" + # # value: "/healthz" + # # - template: + # # source: has_error + # # template: '{{ if .error }}true{{ else }}false{{ end }}' + # # - labels: + # # request_host: + # # has_error: + # kubernetes_sd_configs: + # - role: pod + # relabel_configs: + # - action: drop + # regex: .+ + # source_labels: + # - __meta_kubernetes_pod_label_name + # - source_labels: + # - __meta_kubernetes_pod_label_app + # target_label: __service__ + # - source_labels: + # - __meta_kubernetes_pod_node_name + # target_label: __host__ + # - action: drop + # regex: '' + # source_labels: + # - __service__ + # - action: labelmap + # regex: __meta_kubernetes_pod_label_(.+) + # - action: replace + # replacement: $1 + # separator: / + # source_labels: + # - __meta_kubernetes_namespace + # - __service__ + # target_label: job + # - action: replace + # source_labels: + # - __meta_kubernetes_namespace + # target_label: namespace + # - action: replace + # source_labels: + # - __meta_kubernetes_pod_name + # target_label: pod + # - action: replace + # source_labels: + # - __meta_kubernetes_pod_container_name + # target_label: container + # - replacement: /var/log/pods/*$1/*.log + # separator: / + # source_labels: + # - __meta_kubernetes_pod_uid + # - __meta_kubernetes_pod_container_name + # target_label: __path__ - - job_name: kubernetes-pods-indirect-controller - pipeline_stages: - - docker: {} - - match: - selector: '{name="weave-net"}' - action: drop - - match: - selector: '{filename=~".*konnectivity.*"}' - action: drop - - match: - selector: '{name=~".*"} |~ ".*/healthz.*"' - action: drop - - match: - selector: '{name=~".*"} |~ ".*kube-probe/.*"' - action: drop - - match: - selector: '{app="internal-proxy"}' - action: drop - # - match: - # selector: '{k8s_app="traefik-ingress-lb"}' - # stages: - # - json: - # expressions: - # request_host: RequestHost - # request_path: RequestPath - # error: error - # - drop: - # source: "request_path" - # value: "/healthz" - # - template: - # source: has_error - # template: '{{ if .error }}true{{ else }}false{{ end }}' - # - labels: - # request_host: - # has_error: - kubernetes_sd_configs: - - role: pod - relabel_configs: - - action: drop - regex: .+ - separator: '' - source_labels: - - __meta_kubernetes_pod_label_name - - __meta_kubernetes_pod_label_app - - action: keep - regex: '[0-9a-z-.]+-[0-9a-f]{8,10}' - source_labels: - - __meta_kubernetes_pod_controller_name - - action: replace - regex: '([0-9a-z-.]+)-[0-9a-f]{8,10}' - source_labels: - - __meta_kubernetes_pod_controller_name - target_label: __service__ - - source_labels: - - __meta_kubernetes_pod_node_name - target_label: __host__ - - action: drop - regex: '' - source_labels: - - __service__ - - action: labelmap - regex: __meta_kubernetes_pod_label_(.+) - - action: replace - replacement: $1 - separator: / - source_labels: - - __meta_kubernetes_namespace - - __service__ - target_label: job - - action: replace - source_labels: - - __meta_kubernetes_namespace - target_label: namespace - - action: replace - source_labels: - - __meta_kubernetes_pod_name - target_label: pod - - action: replace - source_labels: - - __meta_kubernetes_pod_container_name - target_label: container - - replacement: /var/log/pods/*$1/*.log - separator: / - source_labels: - - __meta_kubernetes_pod_uid - - __meta_kubernetes_pod_container_name - target_label: __path__ + # - job_name: kubernetes-pods-direct-controllers + # pipeline_stages: + # - docker: {} + # - match: + # selector: '{name="weave-net"}' + # action: drop + # - match: + # selector: '{filename=~".*konnectivity.*"}' + # action: drop + # - match: + # selector: '{name=~".*"} |~ ".*/healthz.*"' + # action: drop + # - match: + # selector: '{name=~".*"} |~ ".*kube-probe/.*"' + # action: drop + # - match: + # selector: '{app="internal-proxy"}' + # action: drop + # # - match: + # # selector: '{k8s_app="traefik-ingress-lb"}' + # # stages: + # # - json: + # # expressions: + # # request_host: RequestHost + # # request_path: RequestPath + # # error: error + # # - drop: + # # source: "request_path" + # # value: "/healthz" + # # - template: + # # source: has_error + # # template: '{{ if .error }}true{{ else }}false{{ end }}' + # # - labels: + # # request_host: + # # has_error: + # kubernetes_sd_configs: + # - role: pod + # relabel_configs: + # - action: drop + # regex: .+ + # separator: '' + # source_labels: + # - __meta_kubernetes_pod_label_name + # - __meta_kubernetes_pod_label_app + # - action: drop + # regex: '[0-9a-z-.]+-[0-9a-f]{8,10}' + # source_labels: + # - __meta_kubernetes_pod_controller_name + # - source_labels: + # - __meta_kubernetes_pod_controller_name + # target_label: __service__ + # - source_labels: + # - __meta_kubernetes_pod_node_name + # target_label: __host__ + # - action: drop + # regex: '' + # source_labels: + # - __service__ + # - action: labelmap + # regex: __meta_kubernetes_pod_label_(.+) + # - action: replace + # replacement: $1 + # separator: / + # source_labels: + # - __meta_kubernetes_namespace + # - __service__ + # target_label: job + # - action: replace + # source_labels: + # - __meta_kubernetes_namespace + # target_label: namespace + # - action: replace + # source_labels: + # - __meta_kubernetes_pod_name + # target_label: pod + # - action: replace + # source_labels: + # - __meta_kubernetes_pod_container_name + # target_label: container + # - replacement: /var/log/pods/*$1/*.log + # separator: / + # source_labels: + # - __meta_kubernetes_pod_uid + # - __meta_kubernetes_pod_container_name + # target_label: __path__ - - job_name: kubernetes-pods-static - pipeline_stages: - - docker: {} - - match: - selector: '{name="weave-net"}' - action: drop - - match: - selector: '{filename=~".*konnectivity.*"}' - action: drop - - match: - selector: '{name=~".*"} |~ ".*/healthz.*"' - action: drop - - match: - selector: '{name=~".*"} |~ ".*kube-probe/.*"' - action: drop - - match: - selector: '{app="internal-proxy"}' - action: drop - # - match: - # selector: '{k8s_app="traefik-ingress-lb"}' - # stages: - # - json: - # expressions: - # request_host: RequestHost - # request_path: RequestPath - # error: error - # - drop: - # source: "request_path" - # value: "/healthz" - # - template: - # source: has_error - # template: '{{ if .error }}true{{ else }}false{{ end }}' - # - labels: - # request_host: - # has_error: - kubernetes_sd_configs: - - role: pod - relabel_configs: - - action: drop - regex: '' - source_labels: - - __meta_kubernetes_pod_annotation_kubernetes_io_config_mirror - - action: replace - source_labels: - - __meta_kubernetes_pod_label_component - target_label: __service__ - - source_labels: - - __meta_kubernetes_pod_node_name - target_label: __host__ - - action: drop - regex: '' - source_labels: - - __service__ - - action: labelmap - regex: __meta_kubernetes_pod_label_(.+) - - action: replace - replacement: $1 - separator: / - source_labels: - - __meta_kubernetes_namespace - - __service__ - target_label: job - - action: replace - source_labels: - - __meta_kubernetes_namespace - target_label: namespace - - action: replace - source_labels: - - __meta_kubernetes_pod_name - target_label: pod - - action: replace - source_labels: - - __meta_kubernetes_pod_container_name - target_label: container - - replacement: /var/log/pods/*$1/*.log - separator: / - source_labels: - - __meta_kubernetes_pod_annotation_kubernetes_io_config_mirror - - __meta_kubernetes_pod_container_name - target_label: __path__ + # - job_name: kubernetes-pods-indirect-controller + # pipeline_stages: + # - docker: {} + # - match: + # selector: '{name="weave-net"}' + # action: drop + # - match: + # selector: '{filename=~".*konnectivity.*"}' + # action: drop + # - match: + # selector: '{name=~".*"} |~ ".*/healthz.*"' + # action: drop + # - match: + # selector: '{name=~".*"} |~ ".*kube-probe/.*"' + # action: drop + # - match: + # selector: '{app="internal-proxy"}' + # action: drop + # # - match: + # # selector: '{k8s_app="traefik-ingress-lb"}' + # # stages: + # # - json: + # # expressions: + # # request_host: RequestHost + # # request_path: RequestPath + # # error: error + # # - drop: + # # source: "request_path" + # # value: "/healthz" + # # - template: + # # source: has_error + # # template: '{{ if .error }}true{{ else }}false{{ end }}' + # # - labels: + # # request_host: + # # has_error: + # kubernetes_sd_configs: + # - role: pod + # relabel_configs: + # - action: drop + # regex: .+ + # separator: '' + # source_labels: + # - __meta_kubernetes_pod_label_name + # - __meta_kubernetes_pod_label_app + # - action: keep + # regex: '[0-9a-z-.]+-[0-9a-f]{8,10}' + # source_labels: + # - __meta_kubernetes_pod_controller_name + # - action: replace + # regex: '([0-9a-z-.]+)-[0-9a-f]{8,10}' + # source_labels: + # - __meta_kubernetes_pod_controller_name + # target_label: __service__ + # - source_labels: + # - __meta_kubernetes_pod_node_name + # target_label: __host__ + # - action: drop + # regex: '' + # source_labels: + # - __service__ + # - action: labelmap + # regex: __meta_kubernetes_pod_label_(.+) + # - action: replace + # replacement: $1 + # separator: / + # source_labels: + # - __meta_kubernetes_namespace + # - __service__ + # target_label: job + # - action: replace + # source_labels: + # - __meta_kubernetes_namespace + # target_label: namespace + # - action: replace + # source_labels: + # - __meta_kubernetes_pod_name + # target_label: pod + # - action: replace + # source_labels: + # - __meta_kubernetes_pod_container_name + # target_label: container + # - replacement: /var/log/pods/*$1/*.log + # separator: / + # source_labels: + # - __meta_kubernetes_pod_uid + # - __meta_kubernetes_pod_container_name + # target_label: __path__ + + # - job_name: kubernetes-pods-static + # pipeline_stages: + # - docker: {} + # - match: + # selector: '{name="weave-net"}' + # action: drop + # - match: + # selector: '{filename=~".*konnectivity.*"}' + # action: drop + # - match: + # selector: '{name=~".*"} |~ ".*/healthz.*"' + # action: drop + # - match: + # selector: '{name=~".*"} |~ ".*kube-probe/.*"' + # action: drop + # - match: + # selector: '{app="internal-proxy"}' + # action: drop + # # - match: + # # selector: '{k8s_app="traefik-ingress-lb"}' + # # stages: + # # - json: + # # expressions: + # # request_host: RequestHost + # # request_path: RequestPath + # # error: error + # # - drop: + # # source: "request_path" + # # value: "/healthz" + # # - template: + # # source: has_error + # # template: '{{ if .error }}true{{ else }}false{{ end }}' + # # - labels: + # # request_host: + # # has_error: + # kubernetes_sd_configs: + # - role: pod + # relabel_configs: + # - action: drop + # regex: '' + # source_labels: + # - __meta_kubernetes_pod_annotation_kubernetes_io_config_mirror + # - action: replace + # source_labels: + # - __meta_kubernetes_pod_label_component + # target_label: __service__ + # - source_labels: + # - __meta_kubernetes_pod_node_name + # target_label: __host__ + # - action: drop + # regex: '' + # source_labels: + # - __service__ + # - action: labelmap + # regex: __meta_kubernetes_pod_label_(.+) + # - action: replace + # replacement: $1 + # separator: / + # source_labels: + # - __meta_kubernetes_namespace + # - __service__ + # target_label: job + # - action: replace + # source_labels: + # - __meta_kubernetes_namespace + # target_label: namespace + # - action: replace + # source_labels: + # - __meta_kubernetes_pod_name + # target_label: pod + # - action: replace + # source_labels: + # - __meta_kubernetes_pod_container_name + # target_label: container + # - replacement: /var/log/pods/*$1/*.log + # separator: / + # source_labels: + # - __meta_kubernetes_pod_annotation_kubernetes_io_config_mirror + # - __meta_kubernetes_pod_container_name + # target_label: __path__ --- kind: ClusterRole