From 0e858ec1e0c3e1fedb5e1ef9a25628bda5bc5428 Mon Sep 17 00:00:00 2001 From: Marcus Noble Date: Thu, 14 Apr 2022 09:12:16 +0000 Subject: [PATCH] Tweaked prometheus config --- .../monitoring/{inges.yaml => ingess.yaml} | 0 manifests/monitoring/prometheus-server.yaml | 331 ++++++++---------- 2 files changed, 149 insertions(+), 182 deletions(-) rename manifests/monitoring/{inges.yaml => ingess.yaml} (100%) diff --git a/manifests/monitoring/inges.yaml b/manifests/monitoring/ingess.yaml similarity index 100% rename from manifests/monitoring/inges.yaml rename to manifests/monitoring/ingess.yaml diff --git a/manifests/monitoring/prometheus-server.yaml b/manifests/monitoring/prometheus-server.yaml index 99f5e0f..32fd82c 100644 --- a/manifests/monitoring/prometheus-server.yaml +++ b/manifests/monitoring/prometheus-server.yaml @@ -35,21 +35,21 @@ data: static_configs: - targets: - localhost:9090 - - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token - job_name: kubernetes-apiservers - kubernetes_sd_configs: - - role: endpoints - relabel_configs: - - action: keep - regex: default;kubernetes;https - source_labels: - - __meta_kubernetes_namespace - - __meta_kubernetes_service_name - - __meta_kubernetes_endpoint_port_name - scheme: https - tls_config: - ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - insecure_skip_verify: true + # - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + # job_name: kubernetes-apiservers + # kubernetes_sd_configs: + # - role: endpoints + # relabel_configs: + # - action: keep + # regex: default;kubernetes;https + # source_labels: + # - __meta_kubernetes_namespace + # - __meta_kubernetes_service_name + # - __meta_kubernetes_endpoint_port_name + # scheme: https + # tls_config: + # ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + # insecure_skip_verify: true - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token job_name: kubernetes-nodes kubernetes_sd_configs: @@ -68,24 +68,24 @@ data: tls_config: ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt insecure_skip_verify: true - - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token - job_name: kubernetes-nodes-cadvisor - kubernetes_sd_configs: - - role: node - relabel_configs: - - action: labelmap - regex: __meta_kubernetes_node_label_(.+) - - replacement: kubernetes.default.svc:443 - target_label: __address__ - - regex: (.+) - replacement: /api/v1/nodes/$1/proxy/metrics/cadvisor - source_labels: - - __meta_kubernetes_node_name - target_label: __metrics_path__ - scheme: https - tls_config: - ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - insecure_skip_verify: true + # - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + # job_name: kubernetes-nodes-cadvisor + # kubernetes_sd_configs: + # - role: node + # relabel_configs: + # - action: labelmap + # regex: __meta_kubernetes_node_label_(.+) + # - replacement: kubernetes.default.svc:443 + # target_label: __address__ + # - regex: (.+) + # replacement: /api/v1/nodes/$1/proxy/metrics/cadvisor + # source_labels: + # - __meta_kubernetes_node_name + # target_label: __metrics_path__ + # scheme: https + # tls_config: + # ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + # insecure_skip_verify: true - job_name: kubernetes-service-endpoints kubernetes_sd_configs: - role: endpoints @@ -125,84 +125,75 @@ data: source_labels: - __meta_kubernetes_pod_node_name target_label: kubernetes_node - - job_name: kubernetes-service-endpoints-slow - kubernetes_sd_configs: - - role: endpoints - relabel_configs: - - action: keep - regex: true - source_labels: - - __meta_kubernetes_service_annotation_prometheus_io_scrape_slow - - action: replace - regex: (https?) - source_labels: - - __meta_kubernetes_service_annotation_prometheus_io_scheme - target_label: __scheme__ - - action: replace - regex: (.+) - source_labels: - - __meta_kubernetes_service_annotation_prometheus_io_path - target_label: __metrics_path__ - - action: replace - regex: ([^:]+)(?::\d+)?;(\d+) - replacement: $1:$2 - source_labels: - - __address__ - - __meta_kubernetes_service_annotation_prometheus_io_port - target_label: __address__ - - action: labelmap - regex: __meta_kubernetes_service_label_(.+) - - action: replace - source_labels: - - __meta_kubernetes_namespace - target_label: kubernetes_namespace - - action: replace - source_labels: - - __meta_kubernetes_service_name - target_label: kubernetes_name - - action: replace - source_labels: - - __meta_kubernetes_pod_node_name - target_label: kubernetes_node - scrape_interval: 5m - scrape_timeout: 30s - - honor_labels: true - job_name: prometheus-pushgateway - kubernetes_sd_configs: - - role: service - relabel_configs: - - action: keep - regex: pushgateway - source_labels: - - __meta_kubernetes_service_annotation_prometheus_io_probe - - job_name: kubernetes-services - kubernetes_sd_configs: - - role: service - metrics_path: /probe - params: - module: - - http_2xx - relabel_configs: - - action: keep - regex: true - source_labels: - - __meta_kubernetes_service_annotation_prometheus_io_probe - - source_labels: - - __address__ - target_label: __param_target - - replacement: blackbox - target_label: __address__ - - source_labels: - - __param_target - target_label: instance - - action: labelmap - regex: __meta_kubernetes_service_label_(.+) - - source_labels: - - __meta_kubernetes_namespace - target_label: kubernetes_namespace - - source_labels: - - __meta_kubernetes_service_name - target_label: kubernetes_name + # - job_name: kubernetes-service-endpoints-slow + # kubernetes_sd_configs: + # - role: endpoints + # relabel_configs: + # - action: keep + # regex: true + # source_labels: + # - __meta_kubernetes_service_annotation_prometheus_io_scrape_slow + # - action: replace + # regex: (https?) + # source_labels: + # - __meta_kubernetes_service_annotation_prometheus_io_scheme + # target_label: __scheme__ + # - action: replace + # regex: (.+) + # source_labels: + # - __meta_kubernetes_service_annotation_prometheus_io_path + # target_label: __metrics_path__ + # - action: replace + # regex: ([^:]+)(?::\d+)?;(\d+) + # replacement: $1:$2 + # source_labels: + # - __address__ + # - __meta_kubernetes_service_annotation_prometheus_io_port + # target_label: __address__ + # - action: labelmap + # regex: __meta_kubernetes_service_label_(.+) + # - action: replace + # source_labels: + # - __meta_kubernetes_namespace + # target_label: kubernetes_namespace + # - action: replace + # source_labels: + # - __meta_kubernetes_service_name + # target_label: kubernetes_name + # - action: replace + # source_labels: + # - __meta_kubernetes_pod_node_name + # target_label: kubernetes_node + # scrape_interval: 5m + # scrape_timeout: 30s + # - job_name: kubernetes-services + # kubernetes_sd_configs: + # - role: service + # metrics_path: /probe + # params: + # module: + # - http_2xx + # relabel_configs: + # - action: keep + # regex: true + # source_labels: + # - __meta_kubernetes_service_annotation_prometheus_io_probe + # - source_labels: + # - __address__ + # target_label: __param_target + # - replacement: blackbox + # target_label: __address__ + # - source_labels: + # - __param_target + # target_label: instance + # - action: labelmap + # regex: __meta_kubernetes_service_label_(.+) + # - source_labels: + # - __meta_kubernetes_namespace + # target_label: kubernetes_namespace + # - source_labels: + # - __meta_kubernetes_service_name + # target_label: kubernetes_name - job_name: kubernetes-pods kubernetes_sd_configs: - role: pod @@ -237,75 +228,49 @@ data: regex: Pending|Succeeded|Failed source_labels: - __meta_kubernetes_pod_phase - - job_name: kubernetes-pods-slow - kubernetes_sd_configs: - - role: pod - relabel_configs: - - action: keep - regex: true - source_labels: - - __meta_kubernetes_pod_annotation_prometheus_io_scrape_slow - - action: replace - regex: (.+) - source_labels: - - __meta_kubernetes_pod_annotation_prometheus_io_path - target_label: __metrics_path__ - - action: replace - regex: ([^:]+)(?::\d+)?;(\d+) - replacement: $1:$2 - source_labels: - - __address__ - - __meta_kubernetes_pod_annotation_prometheus_io_port - target_label: __address__ - - action: labelmap - regex: __meta_kubernetes_pod_label_(.+) - - action: replace - source_labels: - - __meta_kubernetes_namespace - target_label: kubernetes_namespace - - action: replace - source_labels: - - __meta_kubernetes_pod_name - target_label: kubernetes_pod_name - - action: drop - regex: Pending|Succeeded|Failed - source_labels: - - __meta_kubernetes_pod_phase - scrape_interval: 5m - scrape_timeout: 30s - - job_name: 'prometheus-blackbox-exporter-ping' - metrics_path: /probe - params: - module: [icmp_ping] - static_configs: - - targets: [] - relabel_configs: - - source_labels: [__address__] - target_label: __param_target - - source_labels: [__param_target] - target_label: instance - - target_label: __address__ - replacement: blackbox-exporter:9115 - - job_name: 'prometheus-blackbox-exporter-http' - metrics_path: /probe - params: - module: [http_2xx] - static_configs: - - targets: [] - relabel_configs: - - source_labels: [__address__] - target_label: __param_target - - source_labels: [__param_target] - target_label: instance - - target_label: __address__ - replacement: blackbox-exporter:9115 - - job_name: 'node-exporter' - kubernetes_sd_configs: - - role: endpoints - relabel_configs: - - source_labels: [__meta_kubernetes_endpoints_name] - regex: 'node-exporter' - action: keep + # - job_name: kubernetes-pods-slow + # kubernetes_sd_configs: + # - role: pod + # relabel_configs: + # - action: keep + # regex: true + # source_labels: + # - __meta_kubernetes_pod_annotation_prometheus_io_scrape_slow + # - action: replace + # regex: (.+) + # source_labels: + # - __meta_kubernetes_pod_annotation_prometheus_io_path + # target_label: __metrics_path__ + # - action: replace + # regex: ([^:]+)(?::\d+)?;(\d+) + # replacement: $1:$2 + # source_labels: + # - __address__ + # - __meta_kubernetes_pod_annotation_prometheus_io_port + # target_label: __address__ + # - action: labelmap + # regex: __meta_kubernetes_pod_label_(.+) + # - action: replace + # source_labels: + # - __meta_kubernetes_namespace + # target_label: kubernetes_namespace + # - action: replace + # source_labels: + # - __meta_kubernetes_pod_name + # target_label: kubernetes_pod_name + # - action: drop + # regex: Pending|Succeeded|Failed + # source_labels: + # - __meta_kubernetes_pod_phase + # scrape_interval: 5m + # scrape_timeout: 30s + # - job_name: 'node-exporter' + # kubernetes_sd_configs: + # - role: endpoints + # relabel_configs: + # - source_labels: [__meta_kubernetes_endpoints_name] + # regex: 'node-exporter' + # action: keep - job_name: "synapse" scrape_interval: 15s @@ -451,7 +416,7 @@ spec: image: "prom/prometheus:v2.27.1" imagePullPolicy: "IfNotPresent" args: - - --storage.tsdb.retention.time=15d + - --storage.tsdb.retention.time=5d - --config.file=/etc/config/prometheus.yml - --storage.tsdb.path=/data - --web.console.libraries=/etc/prometheus/console_libraries @@ -479,7 +444,9 @@ spec: successThreshold: 1 resources: requests: - memory: 1250Mi + memory: 1500Mi + limits: + memory: 2000Mi volumeMounts: - name: config-volume mountPath: /etc/config