fluentd container logging to loggly.

dkhost.xai-corp.net.yml

@@ -85,8 +85,9 @@
 
   roles:
     - dockerhost
+    - rsyslog
 #    - geerlingguy.nginx
-    - certbot
+    #- certbot
 #    - { role: Datadog.datadog, when: ansible_architecture != 'armv7l' } #does not support armhf architecture. should switch to fluentd or logstash
 
 

dkswarm.xai-corp.net.yml

@@ -12,6 +12,7 @@
       managers:
         - dkhost01
         - dkhost03
+        - home
       workers: []
       removed:
         - dkhost04

dockerfiles/services/dkregistry/docker-compose.yml

@@ -40,3 +40,9 @@ services:
         limits:
           cpus: '0.1'
           memory: 16M
+
+    logging:
+      driver: fluentd
+      options:
+        fluentd-address: "logs.xai-corp.net:24224"
+        tag: dkregistry

dockerfiles/services/jenkins/docker-compose.yml

@@ -26,6 +26,12 @@ services:
           cpus: '2'
           memory: 1024M
 
+    logging:
+      driver: fluentd
+      options:
+        fluentd-address: "logs.xai-corp.net:24224"
+        tag: jenkins
+
 networks:
   default:
     external:

dockerfiles/services/letsencrypt/docker-compose-update.yml

@@ -42,3 +42,9 @@ services:
         limits:
           cpus: '0.5'
           memory: 16M
+
+    logging:
+      driver: fluentd
+      options:
+        fluentd-address: "logs.xai-corp.net:24224"
+        tag: letsencrypt-update

dockerfiles/services/nextcloud/docker-compose-prod.yml

@@ -2,6 +2,7 @@
 # docker-compose file for owncloud server
 
 # DOCKER_HOST=dkhost:2376 docker stack deploy -c docker-compose-prod.yml owncloud
+# DOCKER_HOST=dkhost:2376 docker stack ps owncloud
 
 version: '3'
 services:
@@ -28,6 +29,12 @@ services:
           cpus: '1'
           memory: 512M
 
+    logging:
+      driver: fluentd
+      options:
+        fluentd-address: "logs.xai-corp.net:24224"
+        tag: nextcloud
+
   collabora:
     image: collabora/code
     ports:
@@ -40,7 +47,7 @@ services:
 
     deploy:
       mode: replicated
-      replicas: 1
+      replicas: 0
       restart_policy:
         condition: any
         delay: "1s"

dockerfiles/services/services/fluentd/Dockerfile

@@ -0,0 +1,17 @@
+FROM fluent/fluentd:v0.14-onbuild
+MAINTAINER Richard Morgan <r_morgan@sympatico.ca>
+
+RUN apk add --update --virtual .build-deps \
+        sudo build-base ruby-dev \
+
+ # cutomize following instruction as you wish
+ && sudo gem install \
+        fluent-plugin-secure-forward \
+        fluent-plugin-loggly \
+
+ && sudo gem sources --clear-all \
+ && apk del .build-deps \
+ && rm -rf /var/cache/apk/* \
+           /home/fluent/.gem/ruby/2.3.0/cache/*.gem
+
+EXPOSE 24224

dockerfiles/services/services/fluentd/docker-compose.yml

@@ -0,0 +1,36 @@
+---
+# Based on https://hub.docker.com/r/fluent/fluentd/
+#
+# DOCKER_HOST=192.168.2.41:2376 docker-compose up -d
+#
+# docker login dkregistry.xai-corp.net:5000
+# docker-compose build && docker push dkregistry.xai-corp.net:5000/fluentd:latest
+# DOCKER_HOST=dkhost:2376 docker stack deploy --with-registry-auth -c docker-compose.yml services
+# DOCKER_HOST=dkhost:2376 docker stack ps services
+
+version: '3'
+services:
+
+  fluentd:
+    build:
+      context: ""
+      dockerfile: Dockerfile
+    image: "dkregistry.xai-corp.net:5000/fluentd:latest"
+    volumes:
+      - /opt/shared/logs:/fluentd/log
+    ports:
+      - "24224:24224"
+
+
+    deploy:
+      mode: replicated
+
+      replicas: 1
+      restart_policy:
+        condition: any
+        delay: "1s"
+        max_attempts: 3
+      resources:
+        limits:
+          cpus: '0.5'
+          memory: 64M

dockerfiles/services/services/fluentd/fluent.conf

@@ -0,0 +1,10 @@
+<source>
+  @type  forward
+  @id    input1
+  port  24224
+</source>
+
+<match **>
+    @type loggly
+    loggly_url https://logs-01.loggly.com/inputs/22feaabc-c080-4407-86c3-ae9052999786/tag/fluentd
+</match>

dockerfiles/services/services/loggly/docker-compose.yml

@@ -0,0 +1,32 @@
+---
+# docker-compose file for loggly container
+# DOCKER_HOST=dkhost:2376 docker stack deploy -c docker-compose.yml services
+
+version: '3'
+services:
+
+  loggly:
+    image: "sendgridlabs/loggly-docker"
+    ports:
+      - 31514:514/udp
+    environment:
+      - TOKEN=22feaabc-c080-4407-86c3-ae9052999786
+      - TAG=Docker
+
+    deploy:
+      mode: replicated
+
+      replicas: 1
+      restart_policy:
+        condition: any
+        delay: "1s"
+        max_attempts: 3
+      resources:
+        limits:
+          cpus: '0.1'
+          memory: 16M
+
+networks:
+  default:
+    external:
+      name: prod-private

dockerfiles/services/services/mariadb/docker-compose.yml

@@ -27,6 +27,13 @@ services:
           cpus: '0.5'
           memory: 512M
 
+
+    logging:
+      driver: fluentd
+      options:
+        fluentd-address: "logs.xai-corp.net:24224"
+        tag: mariadb
+
 networks:
   default:
     external:

dockerfiles/services/services/memcached/docker-compose.yml

@@ -26,6 +26,12 @@ services:
           cpus: '0.5'
           memory: 65M
 
+    logging:
+      driver: fluentd
+      options:
+        fluentd-address: "logs.xai-corp.net:24224"
+        tag: memcached
+
 networks:
   default:
     external:

dockerfiles/services/services/redis/docker-compose.yml

@@ -22,6 +22,12 @@ services:
           cpus: '0.5'
           memory: 256M
 
+    logging:
+      driver: fluentd
+      options:
+        fluentd-address: "logs.xai-corp.net:24224"
+        tag: redis
+
 networks:
   default:
     external:

dockerfiles/services/sslproxy/docker-compose-prod.yml

@@ -19,10 +19,22 @@ services:
 #      options:
 #        syslog-address: "tcp+tls://logs6.papertrailapp.com:38577"
 #        tag: "{{.Name}}/{{.ID}}"
+#
+#    logging:
+#      driver: fluentd
+#      options:
+#        fluentd-address: "logs.xai-corp.net:24224"
+#        tag: sslproxy
+
+#    logging:
+#      driver: syslog
+#      options:
+#        syslog-address: "udp://logs.xai-corp.net:31514"
+#        tag: sslproxy
 
     deploy:
       mode: replicated
-      replicas: 2
+      replicas: 1
       restart_policy:
         condition: any
         delay: 5s

inventory.conf

@@ -15,6 +15,7 @@ dkhost01        ansible_ssh_host=192.168.2.41
 #dkhost02       ansible_ssh_host=192.168.2.43
 dkhost03        ansible_ssh_host=192.168.2.53
 #dkhost04        ansible_ssh_host=192.168.2.54
+home            ansible_ssh_host=192.168.2.11
 
 [ns]
 home02          ansible_ssh_host=192.168.2.22

roles/rsyslog/templates/49-shiptograylog.j2

@@ -1 +1 @@
-# *.* @logs.xai-corp.net:514
+*.* @logs.xai-corp.net:31514