From 978e2cf56d2bcde9645d7bd26d767531b35147ea Mon Sep 17 00:00:00 2001 From: richard Date: Tue, 6 Mar 2018 17:11:57 -0500 Subject: [PATCH] setup fluentd prometheus exporter --- dockerfiles/services/build_apps.sh | 2 + dockerfiles/services/launch_apps.sh | 2 + dockerfiles/services/prometheus/Dockerfile | 2 + .../prometheus/docker-compose-prod.yml | 111 ++++++++++++++++++ .../services/prometheus/docker-compose.yml | 57 +++++---- .../services/prometheus/prometheus.yml | 32 +++++ .../services/services/fluentd/Dockerfile | 2 +- .../services/fluentd/docker-compose-prod.yml | 20 +++- .../services/fluentd/docker-compose.yml | 2 + .../services/services/fluentd/fluent.conf | 6 + 10 files changed, 203 insertions(+), 33 deletions(-) create mode 100644 dockerfiles/services/prometheus/Dockerfile create mode 100644 dockerfiles/services/prometheus/docker-compose-prod.yml create mode 100644 dockerfiles/services/prometheus/prometheus.yml diff --git a/dockerfiles/services/build_apps.sh b/dockerfiles/services/build_apps.sh index aa125ed..745486a 100644 --- a/dockerfiles/services/build_apps.sh +++ b/dockerfiles/services/build_apps.sh @@ -5,7 +5,9 @@ docker login -u richard -p $DKREGISTRY_PASS $DKREGISTRY (cd nextcloud && docker-compose -f docker-compose.yml build) (cd sslproxy && docker-compose -f docker-compose.yml build) (cd letsencrypt && docker-compose -f docker-compose.yml build) +(cd prometheus && docker-compose -f docker-compose.yml build) docker push dkregistry.xai-corp.net:5000/xaicorp/nextcloud:latest docker push dkregistry.xai-corp.net:5000/sslproxy:2.1 docker push dkregistry.xai-corp.net:5000/letsencrypt:latest +docker push dkregistry.xai-corp.net:5000/xaicorp/prometheus:latest diff --git a/dockerfiles/services/launch_apps.sh b/dockerfiles/services/launch_apps.sh index bdce756..6683d1d 100644 --- a/dockerfiles/services/launch_apps.sh +++ b/dockerfiles/services/launch_apps.sh @@ -13,3 +13,5 @@ docker stack deploy --with-registry-auth --prune -c ui/docker-compose.yml dkui docker stack deploy --with-registry-auth --prune -c sslproxy/docker-compose-prod.yml sslproxy docker stack deploy --with-registry-auth --prune -c letsencrypt/docker-compose-update.yml letsencrypt + +docker stack deploy --with-registry-auth --prune -c prometheus/docker-compose-prod.yml metrics diff --git a/dockerfiles/services/prometheus/Dockerfile b/dockerfiles/services/prometheus/Dockerfile new file mode 100644 index 0000000..0e07679 --- /dev/null +++ b/dockerfiles/services/prometheus/Dockerfile @@ -0,0 +1,2 @@ +FROM prom/prometheus +ADD prometheus.yml /etc/prometheus/ diff --git a/dockerfiles/services/prometheus/docker-compose-prod.yml b/dockerfiles/services/prometheus/docker-compose-prod.yml new file mode 100644 index 0000000..5ba3c49 --- /dev/null +++ b/dockerfiles/services/prometheus/docker-compose-prod.yml @@ -0,0 +1,111 @@ +--- +# docker-compose file for prometheus metrics server + +# docker login dkregistry.xai-corp.net:5000 +# docker-compose build && docker push dkregistry.xai-corp.net:5000/xaicorp/prometheus:latest + +# DOCKER_HOST=dkmanager:2376 docker stack deploy -c docker-compose.yml metrics +# DOCKER_HOST=dkmanager:2376 docker stack ps metrics + +version: '3.4' +services: + + prometheus: + image: "dkregistry.xai-corp.net:5000/xaicorp/prometheus:latest" + ports: + - 9090:9090 +# logging: +# driver: fluentd +# options: +# fluentd-address: "logs.xai-corp.net:24224" +# fluentd-async-connect: 'true' +# tag: postgres + deploy: + mode: replicated + replicas: 1 + restart_policy: + condition: any + delay: "1s" + max_attempts: 5 + update_config: + parallelism: 1 + delay: 2s + order: start-first + resources: + limits: +# cpus: '1' + memory: 512M + networks: + - ingress + - prod + + graphana: + image: grafana/grafana + ports: + - 3001:3000 + environment: + GF_SECURITY_ADMIN_PASSWORD: sakas +# volumes: +# - /opt/shared/nextcloud/data:/var/www/html/data + networks: + - ingress + + postgres_exported: + image: wrouesnel/postgres_exporter + environment: + DATA_SOURCE_NAME: "postgresql://xaicorp_admin:snqioxni1sw@tasks.postgres:5432/?sslmode=disable" + deploy: + mode: replicated + replicas: 0 + restart_policy: + condition: any + delay: "1s" + max_attempts: 5 + update_config: + parallelism: 1 + delay: 2s + order: start-first + resources: + limits: +# cpus: '1' + memory: 512M + networks: + - prod + + fluentd_exporter: + image: vect0r/fluentd_exporter + ports: + - 9309:9309 + command: + - go-wrapper + - run + - -scrape_uri + - http://tasks.fluentd:24220/api/plugins.json + deploy: + mode: replicated + replicas: 1 + restart_policy: + condition: any + delay: "1s" + max_attempts: 5 + update_config: + parallelism: 1 + delay: 2s + order: start-first + resources: + limits: +# cpus: '1' + memory: 512M + networks: + - prod + +networks: + default: + external: + name: ingress + ingress: + external: + name: ingress + prod: + external: + name: prod diff --git a/dockerfiles/services/prometheus/docker-compose.yml b/dockerfiles/services/prometheus/docker-compose.yml index 03640f8..3829702 100644 --- a/dockerfiles/services/prometheus/docker-compose.yml +++ b/dockerfiles/services/prometheus/docker-compose.yml @@ -1,27 +1,30 @@ --- -# docker-compose file for owncloud server +# docker-compose file for prometheus metrics server -# DOCKER_HOST=dkhost:2376 docker stack deploy -c docker-compose.yml metrics -# DOCKER_HOST=dkhost:2376 docker stack ps metrics +# docker login dkregistry.xai-corp.net:5000 +# docker-compose build && docker push dkregistry.xai-corp.net:5000/xaicorp/prometheus:latest + +# DOCKER_HOST=dkmanager:2376 docker stack deploy -c docker-compose.yml metrics +# DOCKER_HOST=dkmanager:2376 docker stack ps metrics version: '3.4' services: prometheus: - image: prom/prometheus + image: "dkregistry.xai-corp.net:5000/xaicorp/prometheus:latest" + build: + context: . + dockerfile: Dockerfile ports: - 9090:9090 -# volumes: -# - /opt/shared/nextcloud/data:/var/www/html/data - logging: - driver: fluentd - options: - fluentd-address: "logs.xai-corp.net:24224" - fluentd-async-connect: 'true' - tag: postgres - networks: - - ingress - - prod + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml +# logging: +# driver: fluentd +# options: +# fluentd-address: "logs.xai-corp.net:24224" +# fluentd-async-connect: 'true' +# tag: postgres graphana: image: grafana/grafana @@ -31,25 +34,19 @@ services: GF_SECURITY_ADMIN_PASSWORD: sakas # volumes: # - /opt/shared/nextcloud/data:/var/www/html/data - networks: - - ingress postgres_exported: image: wrouesnel/postgres_exporter environment: DATA_SOURCE_NAME: "postgresql://xaicorp_admin:snqioxni1sw@tasks.postgres:5432/?sslmode=disable" - networks: - - ingress - - prod + fluentd_exporter: + image: vect0r/fluentd_exporter +# ports: +# - 9309:9309 + command: + - go-wrapper + - run + - -scrape_uri + - http://tasks.fluentd:24220/api/plugins.json -networks: - default: - external: - name: ingress - ingress: - external: - name: ingress - prod: - external: - name: prod diff --git a/dockerfiles/services/prometheus/prometheus.yml b/dockerfiles/services/prometheus/prometheus.yml new file mode 100644 index 0000000..c4afc05 --- /dev/null +++ b/dockerfiles/services/prometheus/prometheus.yml @@ -0,0 +1,32 @@ +--- +# Prometheus config file + +# my global config +global: + scrape_interval: 15s + evaluation_interval: 30s + # scrape_timeout is set to the global default (10s). + + external_labels: + monitor: test + foo: bar + + +scrape_configs: + +- job_name: glusterfs + metrics_path: /metrics + scrape_interval: 30s + scrape_timeout: 14s + static_configs: + - targets: + - 'cubox-i.xai-corp.net:9189' + - 'home.xai-corp.net:9189' + +- job_name: fluentd + metrics_path: /metrics + scrape_interval: 10s + scrape_timeout: 9s + static_configs: + - targets: + - 'fluentd_exporter:9309' diff --git a/dockerfiles/services/services/fluentd/Dockerfile b/dockerfiles/services/services/fluentd/Dockerfile index 2785f61..2444aff 100644 --- a/dockerfiles/services/services/fluentd/Dockerfile +++ b/dockerfiles/services/services/fluentd/Dockerfile @@ -13,4 +13,4 @@ RUN apk add --update --virtual .build-deps \ && rm -rf /var/cache/apk/* \ /home/fluent/.gem/ruby/2.3.0/cache/*.gem -EXPOSE 24224 +EXPOSE 24224 24220 diff --git a/dockerfiles/services/services/fluentd/docker-compose-prod.yml b/dockerfiles/services/services/fluentd/docker-compose-prod.yml index 51275ae..8247241 100644 --- a/dockerfiles/services/services/fluentd/docker-compose-prod.yml +++ b/dockerfiles/services/services/fluentd/docker-compose-prod.yml @@ -18,9 +18,13 @@ services: ports: - "24224:24224" - "24224:24224/udp" + - 24220:24220 + expose: + - "24220" deploy: - mode: global + mode: replicated + replicas: 1 restart_policy: condition: any delay: "1s" @@ -31,8 +35,20 @@ services: order: start-first resources: limits: - cpus: '0.5' + cpus: '1' memory: 64M networks: - ingress + - prod + +networks: + default: + external: + name: ingress + ingress: + external: + name: ingress + prod: + external: + name: prod diff --git a/dockerfiles/services/services/fluentd/docker-compose.yml b/dockerfiles/services/services/fluentd/docker-compose.yml index f4590d4..643e90b 100644 --- a/dockerfiles/services/services/fluentd/docker-compose.yml +++ b/dockerfiles/services/services/fluentd/docker-compose.yml @@ -21,3 +21,5 @@ services: ports: - "24224:24224" - "24224:24224/udp" + expose: + - "24220" diff --git a/dockerfiles/services/services/fluentd/fluent.conf b/dockerfiles/services/services/fluentd/fluent.conf index 7a2e31b..330436d 100644 --- a/dockerfiles/services/services/fluentd/fluent.conf +++ b/dockerfiles/services/services/fluentd/fluent.conf @@ -4,6 +4,12 @@ port 24224 + + @type monitor_agent + bind 0.0.0.0 + port 24220 + + @type loggly loggly_url https://logs-01.loggly.com/inputs/22feaabc-c080-4407-86c3-ae9052999786/tag/fluentd