setup fluentd prometheus exporter

dockerfiles/services/build_apps.sh

@@ -5,7 +5,9 @@ docker login -u richard -p $DKREGISTRY_PASS $DKREGISTRY
 (cd nextcloud && docker-compose -f docker-compose.yml build)
 (cd sslproxy && docker-compose -f docker-compose.yml build)
 (cd letsencrypt && docker-compose -f docker-compose.yml build)
+(cd prometheus && docker-compose -f docker-compose.yml build)
 
 docker push dkregistry.xai-corp.net:5000/xaicorp/nextcloud:latest
 docker push dkregistry.xai-corp.net:5000/sslproxy:2.1
 docker push dkregistry.xai-corp.net:5000/letsencrypt:latest
+docker push dkregistry.xai-corp.net:5000/xaicorp/prometheus:latest

dockerfiles/services/launch_apps.sh

@@ -13,3 +13,5 @@ docker stack deploy --with-registry-auth --prune -c ui/docker-compose.yml dkui
 docker stack deploy --with-registry-auth --prune -c sslproxy/docker-compose-prod.yml sslproxy
 
 docker stack deploy --with-registry-auth --prune -c letsencrypt/docker-compose-update.yml letsencrypt
+
+docker stack deploy --with-registry-auth --prune -c prometheus/docker-compose-prod.yml metrics

dockerfiles/services/prometheus/Dockerfile

@@ -0,0 +1,2 @@
+FROM prom/prometheus
+ADD prometheus.yml /etc/prometheus/

dockerfiles/services/prometheus/docker-compose-prod.yml

@@ -0,0 +1,111 @@
+---
+# docker-compose file for prometheus metrics server
+
+# docker login dkregistry.xai-corp.net:5000
+# docker-compose build && docker push dkregistry.xai-corp.net:5000/xaicorp/prometheus:latest
+
+# DOCKER_HOST=dkmanager:2376 docker stack deploy -c docker-compose.yml metrics
+# DOCKER_HOST=dkmanager:2376 docker stack ps metrics
+
+version: '3.4'
+services:
+
+  prometheus:
+    image: "dkregistry.xai-corp.net:5000/xaicorp/prometheus:latest"
+    ports:
+      - 9090:9090
+#    logging:
+#      driver: fluentd
+#      options:
+#        fluentd-address: "logs.xai-corp.net:24224"
+#        fluentd-async-connect: 'true'
+#        tag: postgres
+    deploy:
+      mode: replicated
+      replicas: 1
+      restart_policy:
+        condition: any
+        delay: "1s"
+        max_attempts: 5
+      update_config:
+        parallelism: 1
+        delay: 2s
+        order: start-first
+      resources:
+        limits:
+#          cpus: '1'
+          memory: 512M
+    networks:
+    - ingress
+    - prod
+
+  graphana:
+    image: grafana/grafana
+    ports:
+      - 3001:3000
+    environment:
+      GF_SECURITY_ADMIN_PASSWORD: sakas
+#    volumes:
+#      - /opt/shared/nextcloud/data:/var/www/html/data
+    networks:
+    - ingress
+
+  postgres_exported:
+    image: wrouesnel/postgres_exporter
+    environment:
+      DATA_SOURCE_NAME: "postgresql://xaicorp_admin:snqioxni1sw@tasks.postgres:5432/?sslmode=disable"
+    deploy:
+      mode: replicated
+      replicas: 0
+      restart_policy:
+        condition: any
+        delay: "1s"
+        max_attempts: 5
+      update_config:
+        parallelism: 1
+        delay: 2s
+        order: start-first
+      resources:
+        limits:
+#          cpus: '1'
+          memory: 512M
+    networks:
+    - prod
+
+  fluentd_exporter:
+    image: vect0r/fluentd_exporter
+    ports:
+      - 9309:9309
+    command:
+      - go-wrapper
+      - run
+      - -scrape_uri
+      - http://tasks.fluentd:24220/api/plugins.json
+    deploy:
+      mode: replicated
+      replicas: 1
+      restart_policy:
+        condition: any
+        delay: "1s"
+        max_attempts: 5
+      update_config:
+        parallelism: 1
+        delay: 2s
+        order: start-first
+      resources:
+        limits:
+#          cpus: '1'
+          memory: 512M
+    networks:
+    - prod
+
+networks:
+  default:
+    external:
+      name: ingress
+  ingress:
+    external:
+      name: ingress
+  prod:
+    external:
+      name: prod

dockerfiles/services/prometheus/docker-compose.yml

@@ -1,27 +1,30 @@
 ---
-# docker-compose file for owncloud server
+# docker-compose file for prometheus metrics server
 
-# DOCKER_HOST=dkhost:2376 docker stack deploy -c docker-compose.yml metrics
+# docker login dkregistry.xai-corp.net:5000
-# DOCKER_HOST=dkhost:2376 docker stack ps metrics
+# docker-compose build && docker push dkregistry.xai-corp.net:5000/xaicorp/prometheus:latest
+
+# DOCKER_HOST=dkmanager:2376 docker stack deploy -c docker-compose.yml metrics
+# DOCKER_HOST=dkmanager:2376 docker stack ps metrics
 
 version: '3.4'
 services:
 
   prometheus:
-    image: prom/prometheus
+    image: "dkregistry.xai-corp.net:5000/xaicorp/prometheus:latest"
+    build:
+      context: .
+      dockerfile: Dockerfile
     ports:
       - 9090:9090
-#    volumes:
+    volumes:
-#      - /opt/shared/nextcloud/data:/var/www/html/data
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml
-    logging:
+#    logging:
-      driver: fluentd
+#      driver: fluentd
-      options:
+#      options:
-        fluentd-address: "logs.xai-corp.net:24224"
+#        fluentd-address: "logs.xai-corp.net:24224"
-        fluentd-async-connect: 'true'
+#        fluentd-async-connect: 'true'
-        tag: postgres
+#        tag: postgres
-    networks:
-    - ingress
-    - prod
 
   graphana:
     image: grafana/grafana
@@ -31,25 +34,19 @@ services:
       GF_SECURITY_ADMIN_PASSWORD: sakas
 #    volumes:
 #      - /opt/shared/nextcloud/data:/var/www/html/data
-    networks:
-    - ingress
 
   postgres_exported:
     image: wrouesnel/postgres_exporter
     environment:
       DATA_SOURCE_NAME: "postgresql://xaicorp_admin:snqioxni1sw@tasks.postgres:5432/?sslmode=disable"
-    networks:
-    - ingress
-    - prod
 
+  fluentd_exporter:
+    image: vect0r/fluentd_exporter
+#    ports:
+#      - 9309:9309
+    command:
+      - go-wrapper
+      - run
+      - -scrape_uri
+      - http://tasks.fluentd:24220/api/plugins.json
 
-networks:
-  default:
-    external:
-      name: ingress
-  ingress:
-    external:
-      name: ingress
-  prod:
-    external:
-      name: prod

dockerfiles/services/prometheus/prometheus.yml

@@ -0,0 +1,32 @@
+---
+# Prometheus config file
+
+# my global config
+global:
+  scrape_interval:     15s
+  evaluation_interval: 30s
+  # scrape_timeout is set to the global default (10s).
+
+  external_labels:
+    monitor: test
+    foo:     bar
+
+
+scrape_configs:
+
+- job_name: glusterfs
+  metrics_path: /metrics
+  scrape_interval: 30s
+  scrape_timeout:  14s
+  static_configs:
+    - targets:
+      - 'cubox-i.xai-corp.net:9189'
+      - 'home.xai-corp.net:9189'
+
+- job_name: fluentd
+  metrics_path: /metrics
+  scrape_interval: 10s
+  scrape_timeout:  9s
+  static_configs:
+    - targets:
+      - 'fluentd_exporter:9309'

dockerfiles/services/services/fluentd/Dockerfile

@@ -13,4 +13,4 @@ RUN apk add --update --virtual .build-deps \
  && rm -rf /var/cache/apk/* \
            /home/fluent/.gem/ruby/2.3.0/cache/*.gem
 
-EXPOSE 24224
+EXPOSE 24224 24220

dockerfiles/services/services/fluentd/docker-compose-prod.yml

@@ -18,9 +18,13 @@ services:
     ports:
       - "24224:24224"
       - "24224:24224/udp"
+      - 24220:24220
+    expose:
+      - "24220"
 
     deploy:
-      mode: global
+      mode: replicated
+      replicas: 1
       restart_policy:
         condition: any
         delay: "1s"
@@ -31,8 +35,20 @@ services:
         order: start-first
       resources:
         limits:
-          cpus: '0.5'
+          cpus: '1'
           memory: 64M
 
     networks:
       - ingress
+      - prod
+
+networks:
+  default:
+    external:
+      name: ingress
+  ingress:
+    external:
+      name: ingress
+  prod:
+    external:
+      name: prod

dockerfiles/services/services/fluentd/docker-compose.yml

@@ -21,3 +21,5 @@ services:
     ports:
       - "24224:24224"
       - "24224:24224/udp"
+    expose:
+      - "24220"

dockerfiles/services/services/fluentd/fluent.conf

@@ -4,6 +4,12 @@
   port  24224
 </source>
 
+<source>
+  @type monitor_agent
+  bind 0.0.0.0
+  port 24220
+</source>
+
 <match **>
     @type loggly
     loggly_url https://logs-01.loggly.com/inputs/22feaabc-c080-4407-86c3-ae9052999786/tag/fluentd