From 94b62b4c75e142cf21dd0c871f4adf8a9159eecd Mon Sep 17 00:00:00 2001 From: Marcus Noble Date: Fri, 29 Jan 2021 21:54:47 +0000 Subject: [PATCH] Update loki and grafana --- manifests/loki_chart.yaml | 61 ++++++++++++++++++++++------------ terraform/bucket.tf | 9 ----- terraform/kubernetes-charts.tf | 5 --- 3 files changed, 39 insertions(+), 36 deletions(-) diff --git a/manifests/loki_chart.yaml b/manifests/loki_chart.yaml index ec49f03..99e8484 100644 --- a/manifests/loki_chart.yaml +++ b/manifests/loki_chart.yaml @@ -13,17 +13,6 @@ metadata: kube-1password/vault: Kubernetes type: Opaque --- -apiVersion: v1 -kind: Secret -metadata: - name: loki-values - namespace: logging - annotations: - kube-1password: 6u6ebtofcxbr5r4q7k5wkc5zea - kube-1password/vault: Kubernetes - kube-1password/secret-text-key: values.yaml -type: Opaque ---- apiVersion: helm.fluxcd.io/v1 kind: HelmRelease metadata: @@ -31,22 +20,24 @@ metadata: namespace: logging spec: chart: - repository: https://grafana.github.io/loki/charts + repository: https://grafana.github.io/helm-charts name: loki-stack - version: 2.0.2 + version: 2.3.1 maxHistory: 4 skipCRDs: false - valuesFrom: - - secretKeyRef: - name: loki-values - namespace: logging - key: values.yaml - optional: false values: - fluent-bit: - enabled: "true" promtail: enabled: "true" + loki: + config: + table_manager: + retention_deletes_enabled: true + retention_period: 720h + persistence: + enabled: true + accessModes: + - ReadWriteOnce + size: 10Gi --- @@ -59,7 +50,7 @@ spec: chart: repository: https://grafana.github.io/helm-charts name: grafana - version: 6.1.3 + version: 6.2.1 maxHistory: 4 skipCRDs: false values: @@ -77,6 +68,32 @@ spec: access: proxy jsonData: maxLines: 1000 + grafana.ini: + analytics: + reporting_enabled: false + check_for_updates: false + dataproxy: + timeout: 300 + auth.anonymous: + enabled: true + org_role: Admin + dashboardProviders: + dashboardproviders.yaml: + apiVersion: 1 + providers: + - name: 'default' + orgId: 1 + folder: '' + type: file + disableDeletion: false + editable: true + options: + path: /var/lib/grafana/dashboards/default + dashboards: + default: + analytics: + json: | + {"annotations":{"list":[{"builtIn":1,"datasource":"-- Grafana --","enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","type":"dashboard"}]},"editable":true,"gnetId":null,"graphTooltip":0,"iteration":1611959416070,"links":[],"panels":[{"datasource":"Loki","description":"","fieldConfig":{"defaults":{"custom":{"align":null,"filterable":false},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]}},"overrides":[{"matcher":{"id":"byRegexp","options":"(4|5).+"},"properties":[{"id":"color","value":{"fixedColor":"red","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"3.+"},"properties":[{"id":"color","value":{"fixedColor":"light-blue","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total Requests"},"properties":[{"id":"color","value":{"mode":"fixed"}}]}]},"gridPos":{"h":13,"w":9,"x":0,"y":0},"id":2,"options":{"colorMode":"value","graphMode":"none","justifyMode":"center","orientation":"auto","reduceOptions":{"calcs":["sum"],"fields":"","values":false},"textMode":"value_and_name"},"pluginVersion":"7.3.1","targets":[{"expr":"sum (count_over_time({k8s_app=\"traefik-ingress-lb\"} | json | RequestHost=~\"${host:text}\" error=\"\" [$__interval]))","legendFormat":"Total Requests","refId":"B"},{"expr":"sum by (DownstreamStatus) (count_over_time({k8s_app=\"traefik-ingress-lb\"} | json | RequestHost=~\"${host:text}\" error=\"\" [$__interval]))","legendFormat":"{{DownstreamStatus}}","queryType":"randomWalk","refId":"A"}],"timeFrom":null,"timeShift":null,"title":"Requests Per Status Code","transformations":[],"type":"stat"},{"datasource":"Loki","fieldConfig":{"defaults":{"custom":{"align":null,"filterable":false},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]}},"overrides":[]},"gridPos":{"h":13,"w":7,"x":9,"y":0},"id":6,"options":{"showHeader":true},"pluginVersion":"7.3.1","targets":[{"expr":"topk(25, sum by (RequestPath) (count_over_time({k8s_app=\"traefik-ingress-lb\"} | json | RequestHost=~\"${host:text}\" error=\"\" RequestPath=~\"^/(.*(/|html?))?$\" [$__interval])))","legendFormat":"{{RequestPath}}","queryType":"randomWalk","refId":"A"}],"timeFrom":null,"timeShift":null,"title":"Top Viewed Pages","transformations":[{"id":"reduce","options":{"reducers":["sum"]}},{"id":"organize","options":{"excludeByName":{},"indexByName":{},"renameByName":{"Field":"Page"}}}],"type":"table"},{"datasource":"Loki","fieldConfig":{"defaults":{"custom":{"align":null,"filterable":false},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]}},"overrides":[]},"gridPos":{"h":13,"w":8,"x":16,"y":0},"id":5,"options":{"showHeader":true,"sortBy":[{"desc":true,"displayName":"Total"}]},"pluginVersion":"7.3.1","targets":[{"expr":"topk(25, sum by (request_Referer) (count_over_time({k8s_app=\"traefik-ingress-lb\"} | json | RequestHost=~\"${host:text}\" error=\"\" request_Referer!=\"\" request_Referer!~\"https://.*marcusnoble.co.uk/.*\" [$__interval])))","legendFormat":"{{request_Referer}}","queryType":"randomWalk","refId":"A"}],"timeFrom":null,"timeShift":null,"title":"Top Referers","transformations":[{"id":"reduce","options":{"reducers":["sum"]}},{"id":"organize","options":{"excludeByName":{},"indexByName":{},"renameByName":{"Field":"Referer"}}}],"type":"table"},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"Loki","decimals":0,"description":"","fieldConfig":{"defaults":{"custom":{"align":null,"filterable":false},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]}},"overrides":[]},"fill":1,"fillGradient":0,"gridPos":{"h":18,"w":9,"x":0,"y":13},"hiddenSeries":false,"id":3,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":true,"values":true},"lines":true,"linewidth":1,"nullPointMode":"null","options":{"alertThreshold":false},"percentage":false,"pluginVersion":"7.3.1","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"sum by (DownstreamStatus) (count_over_time({k8s_app=\"traefik-ingress-lb\"} | json | RequestHost=~\"${host:text}\" error=\"\" [$__interval]))","legendFormat":"{{DownstreamStatus}}","queryType":"randomWalk","refId":"A"}],"thresholds":[],"timeFrom":null,"timeRegions":[],"timeShift":null,"title":"Requests Per Status Code","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"transformations":[],"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"decimals":0,"format":"short","label":"No. of Requests","logBase":1,"max":null,"min":null,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}],"yaxis":{"align":false,"alignLevel":null}},{"datasource":"Loki","fieldConfig":{"defaults":{"custom":{"align":null,"filterable":false},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]}},"overrides":[]},"gridPos":{"h":18,"w":7,"x":9,"y":13},"id":8,"options":{"showHeader":true,"sortBy":[{"desc":true,"displayName":"Total"}]},"pluginVersion":"7.3.1","targets":[{"expr":"topk(25, sum by (RequestPath) (count_over_time({k8s_app=\"traefik-ingress-lb\"} | json | RequestHost=~\"${host:text}\" error=\"\" DownstreamStatus=\"404\" [$__interval])))","legendFormat":"{{RequestPath}}","queryType":"randomWalk","refId":"A"}],"timeFrom":null,"timeShift":null,"title":"Top 404","transformations":[{"id":"reduce","options":{"reducers":["sum"]}},{"id":"organize","options":{"excludeByName":{},"indexByName":{},"renameByName":{"Field":"Requests"}}}],"type":"table"},{"datasource":"Loki","fieldConfig":{"defaults":{"custom":{"align":null,"filterable":false},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]}},"overrides":[]},"gridPos":{"h":18,"w":8,"x":16,"y":13},"id":7,"options":{"showHeader":true,"sortBy":[{"desc":true,"displayName":"Total"}]},"pluginVersion":"7.3.1","targets":[{"expr":"topk(25, sum by (request_User_Agent) (count_over_time({k8s_app=\"traefik-ingress-lb\"} | json | RequestHost=~\"${host:text}\" error=\"\" request_User_Agent!=\"\" [$__interval])))","legendFormat":"{{request_User_Agent}}","queryType":"randomWalk","refId":"A"}],"timeFrom":null,"timeShift":null,"title":"Top User Agents","transformations":[{"id":"reduce","options":{"reducers":["sum"]}},{"id":"organize","options":{"excludeByName":{},"indexByName":{},"renameByName":{"Field":"User Agent"}}}],"type":"table"}],"refresh":false,"schemaVersion":26,"style":"dark","tags":[],"templating":{"list":[{"allValue":null,"current":{"selected":false,"text":"marcusnoble.co.uk","value":"marcusnoble.co.uk"},"error":null,"hide":0,"includeAll":false,"label":"Host","multi":false,"name":"host","options":[{"selected":true,"text":"marcusnoble.co.uk","value":"marcusnoble.co.uk"},{"selected":false,"text":"til.marcusnoble.co.uk","value":"til.marcusnoble.co.uk"},{"selected":false,"text":"dash.cluster.fun","value":"dash.cluster.fun"}],"query":"marcusnoble.co.uk,til.marcusnoble.co.uk,dash.cluster.fun","queryValue":"","skipUrlSync":false,"type":"custom"}]},"time":{"from":"now-24h","to":"now"},"timepicker":{},"timezone":"","title":"Analytics","uid":"3hJJhOLMz","version":1} --- apiVersion: v1 diff --git a/terraform/bucket.tf b/terraform/bucket.tf index 92630ee..f9ec84f 100644 --- a/terraform/bucket.tf +++ b/terraform/bucket.tf @@ -24,12 +24,3 @@ resource "scaleway_object_bucket" "outline" { output "outline-bucket_id" { value = scaleway_object_bucket.outline.id } - -resource "scaleway_object_bucket" "logs" { - name = "cluster.fun-logs" - acl = "private" -} - -output "logs-bucket_id" { - value = scaleway_object_bucket.logs.id -} diff --git a/terraform/kubernetes-charts.tf b/terraform/kubernetes-charts.tf index d2a3e83..d4d7653 100644 --- a/terraform/kubernetes-charts.tf +++ b/terraform/kubernetes-charts.tf @@ -9,11 +9,6 @@ provider "helm" { } } -data "helm_repository" "stable" { - name = "stable" - url = "https://kubernetes-charts.storage.googleapis.com" -} - data "helm_repository" "fluxcd" { name = "fluxcd" url = "https://charts.fluxcd.io"