Tweaked prometheus config

2022-04-14 09:12:16 +00:00 · 2022-04-14 09:12:16 +00:00 · 0e858ec1e0
commit 0e858ec1e0
parent 22d07c1526
2 changed files with 149 additions and 182 deletions
--- a/manifests/monitoring/ingess.yaml
+++ b/manifests/monitoring/ingess.yaml
--- a/manifests/monitoring/prometheus-server.yaml
+++ b/manifests/monitoring/prometheus-server.yaml
@ -35,21 +35,21 @@ data:
      static_configs:
      - targets:
        - localhost:9090
-    - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
-      job_name: kubernetes-apiservers
-      kubernetes_sd_configs:
-      - role: endpoints
-      relabel_configs:
-      - action: keep
-        regex: default;kubernetes;https
-        source_labels:
-        - __meta_kubernetes_namespace
-        - __meta_kubernetes_service_name
-        - __meta_kubernetes_endpoint_port_name
-      scheme: https
-      tls_config:
-        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
-        insecure_skip_verify: true
+    # - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+    #   job_name: kubernetes-apiservers
+    #   kubernetes_sd_configs:
+    #   - role: endpoints
+    #   relabel_configs:
+    #   - action: keep
+    #     regex: default;kubernetes;https
+    #     source_labels:
+    #     - __meta_kubernetes_namespace
+    #     - __meta_kubernetes_service_name
+    #     - __meta_kubernetes_endpoint_port_name
+    #   scheme: https
+    #   tls_config:
+    #     ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+    #     insecure_skip_verify: true
    - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
      job_name: kubernetes-nodes
      kubernetes_sd_configs:
@ -68,24 +68,24 @@ data:
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        insecure_skip_verify: true
-    - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
-      job_name: kubernetes-nodes-cadvisor
-      kubernetes_sd_configs:
-      - role: node
-      relabel_configs:
-      - action: labelmap
-        regex: __meta_kubernetes_node_label_(.+)
-      - replacement: kubernetes.default.svc:443
-        target_label: __address__
-      - regex: (.+)
-        replacement: /api/v1/nodes/$1/proxy/metrics/cadvisor
-        source_labels:
-        - __meta_kubernetes_node_name
-        target_label: __metrics_path__
-      scheme: https
-      tls_config:
-        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
-        insecure_skip_verify: true
+    # - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+    #   job_name: kubernetes-nodes-cadvisor
+    #   kubernetes_sd_configs:
+    #   - role: node
+    #   relabel_configs:
+    #   - action: labelmap
+    #     regex: __meta_kubernetes_node_label_(.+)
+    #   - replacement: kubernetes.default.svc:443
+    #     target_label: __address__
+    #   - regex: (.+)
+    #     replacement: /api/v1/nodes/$1/proxy/metrics/cadvisor
+    #     source_labels:
+    #     - __meta_kubernetes_node_name
+    #     target_label: __metrics_path__
+    #   scheme: https
+    #   tls_config:
+    #     ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+    #     insecure_skip_verify: true
    - job_name: kubernetes-service-endpoints
      kubernetes_sd_configs:
      - role: endpoints
@ -125,84 +125,75 @@ data:
        source_labels:
        - __meta_kubernetes_pod_node_name
        target_label: kubernetes_node
-    - job_name: kubernetes-service-endpoints-slow
-      kubernetes_sd_configs:
-      - role: endpoints
-      relabel_configs:
-      - action: keep
-        regex: true
-        source_labels:
-        - __meta_kubernetes_service_annotation_prometheus_io_scrape_slow
-      - action: replace
-        regex: (https?)
-        source_labels:
-        - __meta_kubernetes_service_annotation_prometheus_io_scheme
-        target_label: __scheme__
-      - action: replace
-        regex: (.+)
-        source_labels:
-        - __meta_kubernetes_service_annotation_prometheus_io_path
-        target_label: __metrics_path__
-      - action: replace
-        regex: ([^:]+)(?::\d+)?;(\d+)
-        replacement: $1:$2
-        source_labels:
-        - __address__
-        - __meta_kubernetes_service_annotation_prometheus_io_port
-        target_label: __address__
-      - action: labelmap
-        regex: __meta_kubernetes_service_label_(.+)
-      - action: replace
-        source_labels:
-        - __meta_kubernetes_namespace
-        target_label: kubernetes_namespace
-      - action: replace
-        source_labels:
-        - __meta_kubernetes_service_name
-        target_label: kubernetes_name
-      - action: replace
-        source_labels:
-        - __meta_kubernetes_pod_node_name
-        target_label: kubernetes_node
-      scrape_interval: 5m
-      scrape_timeout: 30s
-    - honor_labels: true
-      job_name: prometheus-pushgateway
-      kubernetes_sd_configs:
-      - role: service
-      relabel_configs:
-      - action: keep
-        regex: pushgateway
-        source_labels:
-        - __meta_kubernetes_service_annotation_prometheus_io_probe
-    - job_name: kubernetes-services
-      kubernetes_sd_configs:
-      - role: service
-      metrics_path: /probe
-      params:
-        module:
-        - http_2xx
-      relabel_configs:
-      - action: keep
-        regex: true
-        source_labels:
-        - __meta_kubernetes_service_annotation_prometheus_io_probe
-      - source_labels:
-        - __address__
-        target_label: __param_target
-      - replacement: blackbox
-        target_label: __address__
-      - source_labels:
-        - __param_target
-        target_label: instance
-      - action: labelmap
-        regex: __meta_kubernetes_service_label_(.+)
-      - source_labels:
-        - __meta_kubernetes_namespace
-        target_label: kubernetes_namespace
-      - source_labels:
-        - __meta_kubernetes_service_name
-        target_label: kubernetes_name
+    # - job_name: kubernetes-service-endpoints-slow
+    #   kubernetes_sd_configs:
+    #   - role: endpoints
+    #   relabel_configs:
+    #   - action: keep
+    #     regex: true
+    #     source_labels:
+    #     - __meta_kubernetes_service_annotation_prometheus_io_scrape_slow
+    #   - action: replace
+    #     regex: (https?)
+    #     source_labels:
+    #     - __meta_kubernetes_service_annotation_prometheus_io_scheme
+    #     target_label: __scheme__
+    #   - action: replace
+    #     regex: (.+)
+    #     source_labels:
+    #     - __meta_kubernetes_service_annotation_prometheus_io_path
+    #     target_label: __metrics_path__
+    #   - action: replace
+    #     regex: ([^:]+)(?::\d+)?;(\d+)
+    #     replacement: $1:$2
+    #     source_labels:
+    #     - __address__
+    #     - __meta_kubernetes_service_annotation_prometheus_io_port
+    #     target_label: __address__
+    #   - action: labelmap
+    #     regex: __meta_kubernetes_service_label_(.+)
+    #   - action: replace
+    #     source_labels:
+    #     - __meta_kubernetes_namespace
+    #     target_label: kubernetes_namespace
+    #   - action: replace
+    #     source_labels:
+    #     - __meta_kubernetes_service_name
+    #     target_label: kubernetes_name
+    #   - action: replace
+    #     source_labels:
+    #     - __meta_kubernetes_pod_node_name
+    #     target_label: kubernetes_node
+    #   scrape_interval: 5m
+    #   scrape_timeout: 30s
+    # - job_name: kubernetes-services
+    #   kubernetes_sd_configs:
+    #   - role: service
+    #   metrics_path: /probe
+    #   params:
+    #     module:
+    #     - http_2xx
+    #   relabel_configs:
+    #   - action: keep
+    #     regex: true
+    #     source_labels:
+    #     - __meta_kubernetes_service_annotation_prometheus_io_probe
+    #   - source_labels:
+    #     - __address__
+    #     target_label: __param_target
+    #   - replacement: blackbox
+    #     target_label: __address__
+    #   - source_labels:
+    #     - __param_target
+    #     target_label: instance
+    #   - action: labelmap
+    #     regex: __meta_kubernetes_service_label_(.+)
+    #   - source_labels:
+    #     - __meta_kubernetes_namespace
+    #     target_label: kubernetes_namespace
+    #   - source_labels:
+    #     - __meta_kubernetes_service_name
+    #     target_label: kubernetes_name
    - job_name: kubernetes-pods
      kubernetes_sd_configs:
      - role: pod
@ -237,75 +228,49 @@ data:
        regex: Pending|Succeeded|Failed
        source_labels:
        - __meta_kubernetes_pod_phase
-    - job_name: kubernetes-pods-slow
-      kubernetes_sd_configs:
-      - role: pod
-      relabel_configs:
-      - action: keep
-        regex: true
-        source_labels:
-        - __meta_kubernetes_pod_annotation_prometheus_io_scrape_slow
-      - action: replace
-        regex: (.+)
-        source_labels:
-        - __meta_kubernetes_pod_annotation_prometheus_io_path
-        target_label: __metrics_path__
-      - action: replace
-        regex: ([^:]+)(?::\d+)?;(\d+)
-        replacement: $1:$2
-        source_labels:
-        - __address__
-        - __meta_kubernetes_pod_annotation_prometheus_io_port
-        target_label: __address__
-      - action: labelmap
-        regex: __meta_kubernetes_pod_label_(.+)
-      - action: replace
-        source_labels:
-        - __meta_kubernetes_namespace
-        target_label: kubernetes_namespace
-      - action: replace
-        source_labels:
-        - __meta_kubernetes_pod_name
-        target_label: kubernetes_pod_name
-      - action: drop
-        regex: Pending|Succeeded|Failed
-        source_labels:
-        - __meta_kubernetes_pod_phase
-      scrape_interval: 5m
-      scrape_timeout: 30s
-    - job_name: 'prometheus-blackbox-exporter-ping'
-      metrics_path: /probe
-      params:
-        module: [icmp_ping]
-      static_configs:
-        - targets: []
-      relabel_configs:
-        - source_labels: [__address__]
-          target_label: __param_target
-        - source_labels: [__param_target]
-          target_label: instance
-        - target_label: __address__
-          replacement: blackbox-exporter:9115
-    - job_name: 'prometheus-blackbox-exporter-http'
-      metrics_path: /probe
-      params:
-        module: [http_2xx]
-      static_configs:
-        - targets: []
-      relabel_configs:
-        - source_labels: [__address__]
-          target_label: __param_target
-        - source_labels: [__param_target]
-          target_label: instance
-        - target_label: __address__
-          replacement: blackbox-exporter:9115
-    - job_name: 'node-exporter'
-      kubernetes_sd_configs:
-        - role: endpoints
-      relabel_configs:
-      - source_labels: [__meta_kubernetes_endpoints_name]
-        regex: 'node-exporter'
-        action: keep
+    # - job_name: kubernetes-pods-slow
+    #   kubernetes_sd_configs:
+    #   - role: pod
+    #   relabel_configs:
+    #   - action: keep
+    #     regex: true
+    #     source_labels:
+    #     - __meta_kubernetes_pod_annotation_prometheus_io_scrape_slow
+    #   - action: replace
+    #     regex: (.+)
+    #     source_labels:
+    #     - __meta_kubernetes_pod_annotation_prometheus_io_path
+    #     target_label: __metrics_path__
+    #   - action: replace
+    #     regex: ([^:]+)(?::\d+)?;(\d+)
+    #     replacement: $1:$2
+    #     source_labels:
+    #     - __address__
+    #     - __meta_kubernetes_pod_annotation_prometheus_io_port
+    #     target_label: __address__
+    #   - action: labelmap
+    #     regex: __meta_kubernetes_pod_label_(.+)
+    #   - action: replace
+    #     source_labels:
+    #     - __meta_kubernetes_namespace
+    #     target_label: kubernetes_namespace
+    #   - action: replace
+    #     source_labels:
+    #     - __meta_kubernetes_pod_name
+    #     target_label: kubernetes_pod_name
+    #   - action: drop
+    #     regex: Pending|Succeeded|Failed
+    #     source_labels:
+    #     - __meta_kubernetes_pod_phase
+    #   scrape_interval: 5m
+    #   scrape_timeout: 30s
+    # - job_name: 'node-exporter'
+    #   kubernetes_sd_configs:
+    #     - role: endpoints
+    #   relabel_configs:
+    #   - source_labels: [__meta_kubernetes_endpoints_name]
+    #     regex: 'node-exporter'
+    #     action: keep

    - job_name: "synapse"
      scrape_interval: 15s
@ -451,7 +416,7 @@ spec:
          image: "prom/prometheus:v2.27.1"
          imagePullPolicy: "IfNotPresent"
          args:
-            - --storage.tsdb.retention.time=15d
+            - --storage.tsdb.retention.time=5d
            - --config.file=/etc/config/prometheus.yml
            - --storage.tsdb.path=/data
            - --web.console.libraries=/etc/prometheus/console_libraries
@ -479,7 +444,9 @@ spec:
            successThreshold: 1
          resources:
            requests:
-              memory: 1250Mi
+              memory: 1500Mi
+            limits:
+              memory: 2000Mi
          volumeMounts:
            - name: config-volume
              mountPath: /etc/config