apiVersion: v1 kind: Secret metadata: name: grafana-credentials namespace: logging annotations: kube-1password: wpynfxkdipeeacyfxkvtdsuj54 kube-1password/vault: Kubernetes type: Opaque --- apiVersion: helm.fluxcd.io/v1 kind: HelmRelease metadata: name: loki namespace: logging spec: chart: repository: https://grafana.github.io/helm-charts name: loki-stack version: 2.3.1 maxHistory: 4 skipCRDs: false values: promtail: enabled: "true" loki: config: table_manager: retention_deletes_enabled: true retention_period: 720h persistence: enabled: true accessModes: - ReadWriteOnce size: 10Gi --- apiVersion: helm.fluxcd.io/v1 kind: HelmRelease metadata: name: grafana namespace: logging spec: chart: repository: https://grafana.github.io/helm-charts name: grafana version: 6.2.1 maxHistory: 4 skipCRDs: false values: admin: existingSecret: "grafana-credentials" userKey: username passwordKey: password datasources: datasources.yaml: apiVersion: 1 datasources: - name: Loki type: loki url: http://logging-loki.logging:3100 access: proxy jsonData: maxLines: 1000 grafana.ini: analytics: reporting_enabled: false check_for_updates: false dataproxy: timeout: 300 auth.anonymous: enabled: true org_role: Admin dashboardProviders: dashboardproviders.yaml: apiVersion: 1 providers: - name: 'default' orgId: 1 folder: '' type: file disableDeletion: false allowUiUpdates: true editable: true options: path: /var/lib/grafana/dashboards/default dashboards: default: analytics: json: | {"annotations":{"list":[{"builtIn":1,"datasource":"-- Grafana --","enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","type":"dashboard"}]},"editable":true,"gnetId":null,"graphTooltip":0,"id":1,"iteration":1617614944424,"links":[],"panels":[{"datasource":"Loki","description":"","fieldConfig":{"defaults":{"custom":{"align":null,"filterable":false},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]}},"overrides":[{"matcher":{"id":"byRegexp","options":"(4|5).+"},"properties":[{"id":"color","value":{"fixedColor":"red","mode":"fixed"}}]},{"matcher":{"id":"byRegexp","options":"3.+"},"properties":[{"id":"color","value":{"fixedColor":"light-blue","mode":"fixed"}}]},{"matcher":{"id":"byName","options":"Total Requests"},"properties":[{"id":"color","value":{"mode":"fixed"}},{"id":"links","value":[{"targetBlank":true,"title":"Show query","url":"https://grafana.cluster.fun/explore?orgId=1&left=%5B%22${__from}%22,%22${__to}%22,%22Loki%22,%7B%22expr%22:%22%7Bk8s_app%3D%5C%22traefik-ingress-lb%5C%22%7D%20%7C%20json%20%7C%20RequestHost%3D~%5C%22${host:text}%5C%22%20error%3D%5C%22%5C%22%22%7D%5D"}]}]},{"matcher":{"id":"byRegexp","options":"[0-9][0-9][0-9]"},"properties":[{"id":"links","value":[{"targetBlank":true,"title":"Show query","url":"https://grafana.cluster.fun/explore?orgId=1&left=%5B%22${__from}%22,%22${__to}%22,%22Loki%22,%7B%22expr%22:%22%7Bk8s_app%3D%5C%22traefik-ingress-lb%5C%22%7D%20%7C%20json%20%7C%20RequestHost%3D~%5C%22${host:text}%5C%22%20error%3D%5C%22%5C%22 DownstreamStatus%3D\\\"${__field.labels.DownstreamStatus}\\\"\"}]"}]}]}]},"gridPos":{"h":10,"w":9,"x":0,"y":0},"id":2,"options":{"colorMode":"value","graphMode":"none","justifyMode":"center","orientation":"auto","reduceOptions":{"calcs":["sum"],"fields":"","values":false},"textMode":"value_and_name"},"pluginVersion":"7.3.5","targets":[{"expr":"sum (count_over_time({k8s_app=\"traefik-ingress-lb\"} | json | RequestHost=~\"${host:text}\" error=\"\" [$__interval]))","legendFormat":"Total Requests","refId":"B"},{"expr":"sum by (DownstreamStatus) (count_over_time({k8s_app=\"traefik-ingress-lb\"} | json | RequestHost=~\"${host:text}\" error=\"\" [$__interval]))","legendFormat":"{{DownstreamStatus}}","queryType":"randomWalk","refId":"A"}],"timeFrom":null,"timeShift":null,"title":"Requests Per Status Code","transformations":[],"type":"stat"},{"datasource":"Loki","fieldConfig":{"defaults":{"custom":{"align":null,"filterable":false},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]}},"overrides":[{"matcher":{"id":"byName","options":"Total"},"properties":[{"id":"custom.width","value":80}]},{"matcher":{"id":"byName","options":"Page"},"properties":[{"id":"links","value":[{"targetBlank":true,"title":"","url":"https://${host:text}${__data.fields.Page}"}]}]}]},"gridPos":{"h":10,"w":7,"x":9,"y":0},"id":6,"options":{"showHeader":true,"sortBy":[{"desc":true,"displayName":"Total"}]},"pluginVersion":"7.3.5","targets":[{"expr":"topk(25, sum by (RequestPath) (count_over_time({k8s_app=\"traefik-ingress-lb\"} | json | RequestHost=~\"${host:text}\" error=\"\" RequestPath=~\"^/([a-zA-Z0-9\\\\-_]+(/|(ht|x)ml?)?)?$\" DownstreamStatus=~\"2.+\" [$__interval])))","legendFormat":"{{RequestPath}}","queryType":"randomWalk","refId":"A"}],"timeFrom":null,"timeShift":null,"title":"Top Viewed Pages","transformations":[{"id":"reduce","options":{"reducers":["sum"]}},{"id":"organize","options":{"excludeByName":{},"indexByName":{},"renameByName":{"Field":"Page"}}}],"type":"table"},{"datasource":"Loki","fieldConfig":{"defaults":{"custom":{"align":null,"filterable":false},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]}},"overrides":[{"matcher":{"id":"byName","options":"Total"},"properties":[{"id":"custom.width","value":80}]}]},"gridPos":{"h":10,"w":8,"x":16,"y":0},"id":5,"options":{"showHeader":true,"sortBy":[{"desc":true,"displayName":"Total"}]},"pluginVersion":"7.3.5","targets":[{"expr":"topk(25, sum by (request_Referer) (count_over_time({k8s_app=\"traefik-ingress-lb\"} | json | RequestHost=~\"${host:text}\" error=\"\" request_Referer!=\"\" request_Referer!~\"https?://.*${host:text}/?.*\" \n | label_format request_Referer=`{{ regexReplaceAll \"https?://(.*)google.co.uk\" .request_Referer \"Google UK\" }}`\n | label_format request_Referer=`{{ regexReplaceAll \"https?://(.*)google..+\" .request_Referer \"Google\" }}`\n | label_format request_Referer=`{{ regexReplaceAll \"https?://t.co/.*\" .request_Referer \"Twitter\" }}` \n | label_format request_Referer=`{{ regexReplaceAll \"https?://.*bing.com/.*\" .request_Referer \"Bing\" }}` \n | label_format request_Referer=`{{ regexReplaceAll \"https?://.*baidu.com/.*\" .request_Referer \"Baidu\" }}` \n | label_format request_Referer=`{{ regexReplaceAll \"https?://.*duckduckgo.com/?\" .request_Referer \"DuckDuckGo\" }}` \n[$__interval])))","legendFormat":"{{request_Referer}}","queryType":"randomWalk","refId":"A"}],"timeFrom":null,"timeShift":null,"title":"Top Referers","transformations":[{"id":"reduce","options":{"reducers":["sum"]}},{"id":"organize","options":{"excludeByName":{},"indexByName":{},"renameByName":{"Field":"Referer"}}}],"type":"table"},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"Loki","decimals":0,"description":"","fieldConfig":{"defaults":{"custom":{"align":null,"filterable":false},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]}},"overrides":[]},"fill":1,"fillGradient":10,"gridPos":{"h":10,"w":9,"x":0,"y":10},"hiddenSeries":false,"id":3,"interval":"30m","legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":true,"values":true},"lines":true,"linewidth":1,"nullPointMode":"null","options":{"alertThreshold":false},"percentage":false,"pluginVersion":"7.3.5","pointradius":1,"points":true,"renderer":"flot","seriesOverrides":[{}],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"sum by (DownstreamStatus) (count_over_time({k8s_app=\"traefik-ingress-lb\"} | json | RequestHost=~\"${host:text}\" error=\"\" [$__interval]))","legendFormat":"{{DownstreamStatus}}","queryType":"randomWalk","refId":"A"}],"thresholds":[],"timeFrom":null,"timeRegions":[],"timeShift":null,"title":"Requests Per Status Code","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"transformations":[],"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"decimals":0,"format":"short","label":"No. of Requests","logBase":1,"max":null,"min":null,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}],"yaxis":{"align":false,"alignLevel":null}},{"datasource":"Loki","fieldConfig":{"defaults":{"custom":{"align":null,"filterable":false},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]}},"overrides":[{"matcher":{"id":"byName","options":"Total"},"properties":[{"id":"custom.width","value":80}]}]},"gridPos":{"h":10,"w":7,"x":9,"y":10},"id":8,"options":{"showHeader":true,"sortBy":[{"desc":true,"displayName":"Total"}]},"pluginVersion":"7.3.5","targets":[{"expr":"topk(25, sum by (RequestPath) (count_over_time({k8s_app=\"traefik-ingress-lb\"} | json | RequestHost=~\"${host:text}\" error=\"\" DownstreamStatus=\"404\" [$__interval])))","legendFormat":"{{RequestPath}}","queryType":"randomWalk","refId":"A"}],"timeFrom":null,"timeShift":null,"title":"Top 404","transformations":[{"id":"reduce","options":{"reducers":["sum"]}},{"id":"organize","options":{"excludeByName":{},"indexByName":{},"renameByName":{"Field":"Requests"}}}],"type":"table"},{"datasource":"Loki","fieldConfig":{"defaults":{"custom":{"align":null,"filterable":false},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]}},"overrides":[{"matcher":{"id":"byName","options":"Total"},"properties":[{"id":"custom.width","value":80}]}]},"gridPos":{"h":10,"w":8,"x":16,"y":10},"id":7,"options":{"showHeader":true,"sortBy":[{"desc":true,"displayName":"Total"}]},"pluginVersion":"7.3.5","targets":[{"expr":"topk(25, sum by (request_User_Agent) (count_over_time({k8s_app=\"traefik-ingress-lb\"} | json | RequestHost=~\"${host:text}\" error=\"\" request_User_Agent!=\"\" [$__interval])))","legendFormat":"{{request_User_Agent}}","queryType":"randomWalk","refId":"A"}],"timeFrom":null,"timeShift":null,"title":"Top User Agents","transformations":[{"id":"reduce","options":{"reducers":["sum"]}},{"id":"organize","options":{"excludeByName":{},"indexByName":{},"renameByName":{"Field":"User Agent"}}}],"type":"table"},{"datasource":"Loki","description":"","fieldConfig":{"defaults":{"custom":{"align":null,"filterable":false},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]}},"overrides":[{"matcher":{"id":"byName","options":"Total"},"properties":[{"id":"custom.width","value":116},{"id":"links","value":[]}]},{"matcher":{"id":"byName","options":"Requests"},"properties":[{"id":"custom.width","value":1132}]}]},"gridPos":{"h":12,"w":9,"x":0,"y":20},"id":9,"options":{"showHeader":true,"sortBy":[{"desc":true,"displayName":"Total"}]},"pluginVersion":"7.3.5","targets":[{"expr":"sum by (RequestMethod, RequestPath, DownstreamStatus) (count_over_time({k8s_app=\"traefik-ingress-lb\"} | json | RequestHost=~\"${host:text}\" error=\"\" [$__interval]))","legendFormat":"{{RequestMethod}} - {{RequestPath}} - {{DownstreamStatus}}","queryType":"randomWalk","refId":"A"}],"timeFrom":null,"timeShift":null,"title":"All Requests","transformations":[{"id":"reduce","options":{"reducers":["sum"]}},{"id":"organize","options":{"excludeByName":{},"indexByName":{},"renameByName":{"Field":"Requests"}}}],"type":"table"},{"datasource":"Loki","fieldConfig":{"defaults":{"custom":{"align":null,"filterable":false},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]}},"overrides":[{"matcher":{"id":"byName","options":"Total"},"properties":[{"id":"custom.width","value":80}]}]},"gridPos":{"h":12,"w":7,"x":9,"y":20},"id":10,"options":{"showHeader":true,"sortBy":[{"desc":true,"displayName":"Total"}]},"pluginVersion":"7.3.5","targets":[{"expr":"topk(25, sum by (DownstreamStatus, RequestPath, downstream_Location) (count_over_time({k8s_app=\"traefik-ingress-lb\"} | json | RequestHost=~\"${host:text}\" error=\"\" DownstreamStatus=~\"3.+\" [$__interval])))","legendFormat":"{{DownstreamStatus}} - {{RequestPath}} => {{downstream_Location}}","queryType":"randomWalk","refId":"A"}],"timeFrom":null,"timeShift":null,"title":"Top Redirects","transformations":[{"id":"reduce","options":{"reducers":["sum"]}},{"id":"organize","options":{"excludeByName":{},"indexByName":{},"renameByName":{"Field":"Requests"}}}],"type":"table"},{"datasource":"Loki","fieldConfig":{"defaults":{"custom":{"align":null,"filterable":false},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]}},"overrides":[{"matcher":{"id":"byName","options":"Total"},"properties":[{"id":"custom.width","value":80}]}]},"gridPos":{"h":12,"w":8,"x":16,"y":20},"id":11,"options":{"showHeader":true,"sortBy":[{"desc":true,"displayName":"Total"}]},"pluginVersion":"7.3.5","targets":[{"expr":"topk(25, sum by (RequestMethod, DownstreamStatus, RequestPath) (count_over_time({k8s_app=\"traefik-ingress-lb\"} | json | RequestHost=~\"${host:text}\" error=\"\" DownstreamStatus=~\"[54].+\" DownstreamStatus!=\"404\" [$__interval])))","legendFormat":"{{DownstreamStatus}} - {{RequestMethod}} - {{RequestPath}}","queryType":"randomWalk","refId":"A"}],"timeFrom":null,"timeShift":null,"title":"Top Errors","transformations":[{"id":"reduce","options":{"reducers":["sum"]}},{"id":"organize","options":{"excludeByName":{},"indexByName":{},"renameByName":{"Field":"Requests"}}}],"type":"table"}],"refresh":false,"schemaVersion":26,"style":"dark","tags":[],"templating":{"list":[{"allValue":null,"current":{"selected":true,"text":"marcusnoble.co.uk","value":"marcusnoble.co.uk"},"error":null,"hide":0,"includeAll":false,"label":"Host","multi":false,"name":"host","options":[{"selected":false,"text":"marcusnoble.co.uk","value":"marcusnoble.co.uk"},{"selected":false,"text":"til.marcusnoble.co.uk","value":"til.marcusnoble.co.uk"},{"selected":false,"text":"dash.cluster.fun","value":"dash.cluster.fun"},{"selected":true,"text":"tweet.cluster.fun","value":"tweet.cluster.fun"},{"selected":false,"text":"base64.cluster.fun","value":"base64.cluster.fun"},{"selected":false,"text":"feed-fetcher.cluster.fun","value":"feed-fetcher.cluster.fun"},{"selected":false,"text":"cors-proxy.cluster.fun","value":"cors-proxy.cluster.fun"}],"query":"marcusnoble.co.uk,til.marcusnoble.co.uk,dash.cluster.fun,tweet.cluster.fun,base64.cluster.fun,feed-fetcher.cluster.fun,cors-proxy.cluster.fun","queryValue":"","skipUrlSync":false,"type":"custom"}]},"time":{"from":"now-24h","to":"now"},"timepicker":{},"timezone":"","title":"Analytics","uid":"3hJJhOLMz","version":9} --- apiVersion: v1 kind: Secret metadata: name: grafana-auth namespace: logging annotations: kube-1password: mr6spkkx7n3memkbute6ojaarm kube-1password/vault: Kubernetes type: Opaque --- apiVersion: apps/v1 kind: Deployment metadata: name: grafana-auth namespace: logging labels: app: grafana-auth spec: replicas: 1 selector: matchLabels: app: grafana-auth template: metadata: labels: app: grafana-auth spec: containers: - args: - --cookie-secure=false - --provider=oidc - --provider-display-name=Auth0 - --upstream=http://logging-grafana.logging.svc.cluster.local - --http-address=$(HOST_IP):8080 - --redirect-url=https://grafana.cluster.fun/oauth2/callback - --email-domain=marcusnoble.co.uk - --pass-basic-auth=false - --pass-access-token=false - --oidc-issuer-url=https://marcusnoble.eu.auth0.com/ - --cookie-secret=KDGD6rrK6cBmryyZ4wcJ9xAUNW9AQN env: - name: HOST_IP valueFrom: fieldRef: apiVersion: v1 fieldPath: status.podIP - name: OAUTH2_PROXY_CLIENT_ID valueFrom: secretKeyRef: key: username name: grafana-auth - name: OAUTH2_PROXY_CLIENT_SECRET valueFrom: secretKeyRef: key: password name: grafana-auth image: quay.io/oauth2-proxy/oauth2-proxy:v5.1.1 name: oauth-proxy ports: - containerPort: 8080 protocol: TCP --- apiVersion: v1 kind: Service metadata: name: grafana-auth namespace: logging labels: app: grafana-auth spec: ports: - name: http port: 80 protocol: TCP targetPort: 8080 selector: app: grafana-auth type: ClusterIP --- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: grafana-auth namespace: logging labels: app: grafana-auth annotations: cert-manager.io/cluster-issuer: letsencrypt traefik.ingress.kubernetes.io/frontend-entry-points: http,https traefik.ingress.kubernetes.io/redirect-entry-point: https traefik.ingress.kubernetes.io/redirect-permanent: "true" spec: tls: - hosts: - grafana.cluster.fun secretName: grafana-ingress rules: - host: grafana.cluster.fun http: paths: - backend: service: name: grafana-auth port: number: 80 path: / pathType: ImplementationSpecific --- kind: Service apiVersion: v1 metadata: name: prometheus-local namespace: logging spec: type: ClusterIP ports: - port: 80 protocol: TCP targetPort: 8000 selector: app: inlets