amkartashov/gf-k8s
1
0
Fork 0
mirror of https://github.com/amkartashov/gf-k8s.git synced 2026-01-10 17:39:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

logging: replace graylog/fluentbit with loki/promtail

Browse source
This commit is contained in:
Andrey Kartashov 2020-06-20 18:42:16 +07:00
parent 2e89abbd22
commit 4b39c6efd9
8 changed files with 70 additions and 139 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
apps/system/logging/Chart.yaml
View file

@ -1 +1,2 @@
name: logging
version: 0.1

18
apps/system/logging/requirements.yaml
View file

@ -1,13 +1,7 @@
dependencies:
- name: graylog
version: '1.6.2'
repository: '@stable'
- name: mongodb
version: '7.8.9'
repository: '@stable'
- name: elasticsearch
version: '7.6.1'
repository: https://helm.elastic.co
- name: fluent-bit
version: '2.8.11'
repository: '@stable'
- name: loki
version: '0.30.0'
repository: https://grafana.github.io/loki/charts
- name: promtail
version: '0.23.1'
repository: https://grafana.github.io/loki/charts

11
apps/system/logging/templates/elasticsearch-volume.yaml
View file

@ -1,11 +0,0 @@
apiVersion: v1
kind: PersistentVolume
metadata:
name: elasticsearch
spec:
capacity:
storage: 30Gi
accessModes:
- ReadWriteOnce
hostPath:
path: /data/elasticsearch

6
apps/system/logging/templates/mongodb-volume.yaml → apps/system/logging/templates/loki-volume.yaml
View file

@ -1,11 +1,11 @@
apiVersion: v1
kind: PersistentVolume
metadata:
name: mongodb
name: loki
spec:
capacity:
storage: 1Gi
storage: 10Gi
accessModes:
- ReadWriteOnce
hostPath:
path: /data/mongodb
path: /data/loki

6
apps/system/logging/templates/mongodb-volumeclaim.yaml → apps/system/logging/templates/loki-volumeclaim.yaml
View file

@ -1,12 +1,12 @@
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: mongodb
name: loki
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1Gi
storage: 10Gi
volumeMode: Filesystem
volumeName: mongodb
volumeName: loki

149
apps/system/logging/values.yaml
View file

@ -1,112 +1,41 @@
# don't install as graylog deps, because versions are outdated
tags:
install-elasticsearch: false
install-mongodb: false
mongodb:
nameOverride: mongodb
fullnameOverride: mongodb
usePassword: false
loki:
#extraArgs:
# log.level: debug
nameOverride: loki
fullnameOverride: loki
serviceAccount:
create: true
annotations:
iam.gke.io/gcp-service-account: loki-sa@projectid.iam.gserviceaccount.com # will be updated in parameters
serviceMonitor:
enabled: true
interval: 10s
scrapeTimeout: 5s
resources:
limits:
cpu: 150m
#memory: 256Mi
requests:
cpu: 50m
memory: 128Mi
persistence:
existingClaim: mongodb
elasticsearch:
nameOverride: elasticsearch
fullnameOverride: elasticsearch
imageTag: '6.8.7'
replicas: 1
minimumMasterNodes: 1
volumeClaimTemplate:
volumeName: elasticsearch
fluent-bit:
nameOverride: fluent-bit
fullnameOverride: fluent-bit
image:
fluent_bit:
tag: 1.4.1
input:
systemd:
enabled: false
filter:
mergeJSONLog: false
trackOffsets: true
backend:
# no support for gelf in chart, configure manually
type: none
extraEntries:
output: |
[OUTPUT]
Name gelf
Match kube.*
Host graylog-udp
Port 12201
Mode udp
Gelf_Short_Message_Key log
rawConfig: |
@INCLUDE fluent-bit-service.conf
@INCLUDE fluent-bit-input.conf
@INCLUDE fluent-bit-filter.conf
[FILTER]
Name nest
Match kube.*
Operation lift
Nested_under kubernetes
[FILTER]
Name modify
Match kube.*
Rename container_name container
Rename namespace_name namespace
Rename pod_name pod
[FILTER]
Name record_modifier
Match kube.*
Remove_key time
@INCLUDE fluent-bit-output.conf
graylog:
nameOverride: graylog
fullnameOverride: graylog
graylog:
replicas: 1
persistence:
enabled: false
ingress:
enabled: true
annotations:
cert-manager.io/cluster-issuer: letsencrypt
nginx.ingress.kubernetes.io/auth-url: https://oauth2.gorilych.ru/oauth2/auth
nginx.ingress.kubernetes.io/auth-signin: https://oauth2.gorilych.ru/oauth2/start?rd=$scheme://$host$request_uri
nginx.ingress.kubernetes.io/auth-response-headers: X-Forwarded-Email,X-Forwarded-User,X-Roles
nginx.ingress.kubernetes.io/configuration-snippet: |
auth_request_set $name_upstream_1 $upstream_cookie_name_1;
auth_request_set $user $upstream_http_x_auth_request_user;
auth_request_set $email $upstream_http_x_auth_request_email;
proxy_set_header X-Forwarded-User $user;
proxy_set_header X-Forwarded-Email $email;
access_by_lua_block {
if ngx.var.name_upstream_1 ~= "" then
ngx.header["Set-Cookie"] = "name_1=" .. ngx.var.name_upstream_1 .. ngx.var.auth_cookie:match("(; .*)")
end
}
hosts:
- graylog.gorilych.ru
tls:
- hosts:
- graylog.gorilych.ru
secretName: graylog-tls
input:
udp:
service:
type: ClusterIP
ports:
- name: gelf
port: 12201
mongodb:
uri: mongodb://mongodb:27017/graylog
elasticsearch:
hosts: http://elasticsearch:9200
plugins:
- name: graylog-plugin-auth-sso-3.1.0.jar
url: https://github.com/Graylog2/graylog-plugin-auth-sso/releases/download/3.1.0/graylog-plugin-auth-sso-3.1.0.jar
enabled: true
size: 10Gi
existingClaim: loki
promtail:
nameOverride: promtail
fullnameOverride: promtail
serviceMonitor:
enabled: true
interval: 10s
scrapeTimeout: 5s
loki:
serviceName: loki
resources:
limits:
cpu: 100m
#memory: 128Mi
requests:
cpu: 50m
memory: 128Mi

15
apps/system/monitoring/templates/datasources.yaml Normal file
View file

@ -0,0 +1,15 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: datasources
labels:
grafana_datasource: "1"
data:
loki-datasource.yaml: |-
apiVersion: 1
datasources:
- name: Loki
type: loki
access: proxy
url: http://loki.logging:3100
version: 1

3
apps/system/monitoring/values.yaml
View file

@ -88,6 +88,9 @@ prometheus-operator:
gnetId: 9614
revision: 1
datasource: Prometheus
sidecar:
datasources:
enabled: true
alertmanager:
enabled: false
defaultRules: