AverageMarcus/cluster.fun
1
0
Fork 0
Code Issues 1 Pull Requests 2 Releases Activity

Tweaked prometheus config

Browse Source
This commit is contained in:
Marcus Noble 2022-04-14 09:12:16 +00:00
parent 22d07c1526
commit 0e858ec1e0
2 changed files with 149 additions and 182 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
manifests/monitoring/inges.yaml → manifests/monitoring/ingess.yaml
View File

331
manifests/monitoring/prometheus-server.yaml
View File

@ -35,21 +35,21 @@ data:
static_configs: static_configs:
- targets: - targets:
- localhost:9090 - localhost:9090
- bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token # - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
job_name: kubernetes-apiservers # job_name: kubernetes-apiservers
kubernetes_sd_configs: # kubernetes_sd_configs:
- role: endpoints # - role: endpoints
relabel_configs: # relabel_configs:
- action: keep # - action: keep
regex: default;kubernetes;https # regex: default;kubernetes;https
source_labels: # source_labels:
- __meta_kubernetes_namespace # - __meta_kubernetes_namespace
- __meta_kubernetes_service_name # - __meta_kubernetes_service_name
- __meta_kubernetes_endpoint_port_name # - __meta_kubernetes_endpoint_port_name
scheme: https # scheme: https
tls_config: # tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt # ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
insecure_skip_verify: true # insecure_skip_verify: true
- bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
job_name: kubernetes-nodes job_name: kubernetes-nodes
kubernetes_sd_configs: kubernetes_sd_configs:
@ -68,24 +68,24 @@ data:
tls_config: tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
insecure_skip_verify: true insecure_skip_verify: true
- bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token # - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
job_name: kubernetes-nodes-cadvisor # job_name: kubernetes-nodes-cadvisor
kubernetes_sd_configs: # kubernetes_sd_configs:
- role: node # - role: node
relabel_configs: # relabel_configs:
- action: labelmap # - action: labelmap
regex: __meta_kubernetes_node_label_(.+) # regex: __meta_kubernetes_node_label_(.+)
- replacement: kubernetes.default.svc:443 # - replacement: kubernetes.default.svc:443
target_label: __address__ # target_label: __address__
- regex: (.+) # - regex: (.+)
replacement: /api/v1/nodes/$1/proxy/metrics/cadvisor # replacement: /api/v1/nodes/$1/proxy/metrics/cadvisor
source_labels: # source_labels:
- __meta_kubernetes_node_name # - __meta_kubernetes_node_name
target_label: __metrics_path__ # target_label: __metrics_path__
scheme: https # scheme: https
tls_config: # tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt # ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
insecure_skip_verify: true # insecure_skip_verify: true
- job_name: kubernetes-service-endpoints - job_name: kubernetes-service-endpoints
kubernetes_sd_configs: kubernetes_sd_configs:
- role: endpoints - role: endpoints
@ -125,84 +125,75 @@ data:
source_labels: source_labels:
- __meta_kubernetes_pod_node_name - __meta_kubernetes_pod_node_name
target_label: kubernetes_node target_label: kubernetes_node
- job_name: kubernetes-service-endpoints-slow # - job_name: kubernetes-service-endpoints-slow
kubernetes_sd_configs: # kubernetes_sd_configs:
- role: endpoints # - role: endpoints
relabel_configs: # relabel_configs:
- action: keep # - action: keep
regex: true # regex: true
source_labels: # source_labels:
- __meta_kubernetes_service_annotation_prometheus_io_scrape_slow # - __meta_kubernetes_service_annotation_prometheus_io_scrape_slow
- action: replace # - action: replace
regex: (https?) # regex: (https?)
source_labels: # source_labels:
- __meta_kubernetes_service_annotation_prometheus_io_scheme # - __meta_kubernetes_service_annotation_prometheus_io_scheme
target_label: __scheme__ # target_label: __scheme__
- action: replace # - action: replace
regex: (.+) # regex: (.+)
source_labels: # source_labels:
- __meta_kubernetes_service_annotation_prometheus_io_path # - __meta_kubernetes_service_annotation_prometheus_io_path
target_label: __metrics_path__ # target_label: __metrics_path__
- action: replace # - action: replace
regex: ([^:]+)(?::\d+)?;(\d+) # regex: ([^:]+)(?::\d+)?;(\d+)
replacement: $1:$2 # replacement: $1:$2
source_labels: # source_labels:
- __address__ # - __address__
- __meta_kubernetes_service_annotation_prometheus_io_port # - __meta_kubernetes_service_annotation_prometheus_io_port
target_label: __address__ # target_label: __address__
- action: labelmap # - action: labelmap
regex: __meta_kubernetes_service_label_(.+) # regex: __meta_kubernetes_service_label_(.+)
- action: replace # - action: replace
source_labels: # source_labels:
- __meta_kubernetes_namespace # - __meta_kubernetes_namespace
target_label: kubernetes_namespace # target_label: kubernetes_namespace
- action: replace # - action: replace
source_labels: # source_labels:
- __meta_kubernetes_service_name # - __meta_kubernetes_service_name
target_label: kubernetes_name # target_label: kubernetes_name
- action: replace # - action: replace
source_labels: # source_labels:
- __meta_kubernetes_pod_node_name # - __meta_kubernetes_pod_node_name
target_label: kubernetes_node # target_label: kubernetes_node
scrape_interval: 5m # scrape_interval: 5m
scrape_timeout: 30s # scrape_timeout: 30s
- honor_labels: true # - job_name: kubernetes-services
job_name: prometheus-pushgateway # kubernetes_sd_configs:
kubernetes_sd_configs: # - role: service
- role: service # metrics_path: /probe
relabel_configs: # params:
- action: keep # module:
regex: pushgateway # - http_2xx
source_labels: # relabel_configs:
- __meta_kubernetes_service_annotation_prometheus_io_probe # - action: keep
- job_name: kubernetes-services # regex: true
kubernetes_sd_configs: # source_labels:
- role: service # - __meta_kubernetes_service_annotation_prometheus_io_probe
metrics_path: /probe # - source_labels:
params: # - __address__
module: # target_label: __param_target
- http_2xx # - replacement: blackbox
relabel_configs: # target_label: __address__
- action: keep # - source_labels:
regex: true # - __param_target
source_labels: # target_label: instance
- __meta_kubernetes_service_annotation_prometheus_io_probe # - action: labelmap
- source_labels: # regex: __meta_kubernetes_service_label_(.+)
- __address__ # - source_labels:
target_label: __param_target # - __meta_kubernetes_namespace
- replacement: blackbox # target_label: kubernetes_namespace
target_label: __address__ # - source_labels:
- source_labels: # - __meta_kubernetes_service_name
- __param_target # target_label: kubernetes_name
target_label: instance
- action: labelmap
regex: __meta_kubernetes_service_label_(.+)
- source_labels:
- __meta_kubernetes_namespace
target_label: kubernetes_namespace
- source_labels:
- __meta_kubernetes_service_name
target_label: kubernetes_name
- job_name: kubernetes-pods - job_name: kubernetes-pods
kubernetes_sd_configs: kubernetes_sd_configs:
- role: pod - role: pod
@ -237,75 +228,49 @@ data:
regex: Pending|Succeeded|Failed regex: Pending|Succeeded|Failed
source_labels: source_labels:
- __meta_kubernetes_pod_phase - __meta_kubernetes_pod_phase
- job_name: kubernetes-pods-slow # - job_name: kubernetes-pods-slow
kubernetes_sd_configs: # kubernetes_sd_configs:
- role: pod # - role: pod
relabel_configs: # relabel_configs:
- action: keep # - action: keep
regex: true # regex: true
source_labels: # source_labels:
- __meta_kubernetes_pod_annotation_prometheus_io_scrape_slow # - __meta_kubernetes_pod_annotation_prometheus_io_scrape_slow
- action: replace # - action: replace
regex: (.+) # regex: (.+)
source_labels: # source_labels:
- __meta_kubernetes_pod_annotation_prometheus_io_path # - __meta_kubernetes_pod_annotation_prometheus_io_path
target_label: __metrics_path__ # target_label: __metrics_path__
- action: replace # - action: replace
regex: ([^:]+)(?::\d+)?;(\d+) # regex: ([^:]+)(?::\d+)?;(\d+)
replacement: $1:$2 # replacement: $1:$2
source_labels: # source_labels:
- __address__ # - __address__
- __meta_kubernetes_pod_annotation_prometheus_io_port # - __meta_kubernetes_pod_annotation_prometheus_io_port
target_label: __address__ # target_label: __address__
- action: labelmap # - action: labelmap
regex: __meta_kubernetes_pod_label_(.+) # regex: __meta_kubernetes_pod_label_(.+)
- action: replace # - action: replace
source_labels: # source_labels:
- __meta_kubernetes_namespace # - __meta_kubernetes_namespace
target_label: kubernetes_namespace # target_label: kubernetes_namespace
- action: replace # - action: replace
source_labels: # source_labels:
- __meta_kubernetes_pod_name # - __meta_kubernetes_pod_name
target_label: kubernetes_pod_name # target_label: kubernetes_pod_name
- action: drop # - action: drop
regex: Pending|Succeeded|Failed # regex: Pending|Succeeded|Failed
source_labels: # source_labels:
- __meta_kubernetes_pod_phase # - __meta_kubernetes_pod_phase
scrape_interval: 5m # scrape_interval: 5m
scrape_timeout: 30s # scrape_timeout: 30s
- job_name: 'prometheus-blackbox-exporter-ping' # - job_name: 'node-exporter'
metrics_path: /probe # kubernetes_sd_configs:
params: # - role: endpoints
module: [icmp_ping] # relabel_configs:
static_configs: # - source_labels: [__meta_kubernetes_endpoints_name]
- targets: [] # regex: 'node-exporter'
relabel_configs: # action: keep
- source_labels: [__address__]
target_label: __param_target
- source_labels: [__param_target]
target_label: instance
- target_label: __address__
replacement: blackbox-exporter:9115
- job_name: 'prometheus-blackbox-exporter-http'
metrics_path: /probe
params:
module: [http_2xx]
static_configs:
- targets: []
relabel_configs:
- source_labels: [__address__]
target_label: __param_target
- source_labels: [__param_target]
target_label: instance
- target_label: __address__
replacement: blackbox-exporter:9115
- job_name: 'node-exporter'
kubernetes_sd_configs:
- role: endpoints
relabel_configs:
- source_labels: [__meta_kubernetes_endpoints_name]
regex: 'node-exporter'
action: keep
- job_name: "synapse" - job_name: "synapse"
scrape_interval: 15s scrape_interval: 15s
@ -451,7 +416,7 @@ spec:
image: "prom/prometheus:v2.27.1" image: "prom/prometheus:v2.27.1"
imagePullPolicy: "IfNotPresent" imagePullPolicy: "IfNotPresent"
args: args:
- --storage.tsdb.retention.time=15d - --storage.tsdb.retention.time=5d
- --config.file=/etc/config/prometheus.yml - --config.file=/etc/config/prometheus.yml
- --storage.tsdb.path=/data - --storage.tsdb.path=/data
- --web.console.libraries=/etc/prometheus/console_libraries - --web.console.libraries=/etc/prometheus/console_libraries
@ -479,7 +444,9 @@ spec:
successThreshold: 1 successThreshold: 1
resources: resources:
requests: requests:
memory: 1250Mi memory: 1500Mi
limits:
memory: 2000Mi
volumeMounts: volumeMounts:
- name: config-volume - name: config-volume
mountPath: /etc/config mountPath: /etc/config