add service scripts

dkswarm.xai-corp.net.yml

@@ -11,12 +11,12 @@
    - swarm:
       managers:
         - dkhost01
-        - dkhost03
+        - dkhost02
         - home
       workers: []
       removed:
+        - dkhost03
         - dkhost04
-        - dkhost02
         - dkhost05
 
       history: 1

dockerfiles/services/build_apps.sh

@@ -0,0 +1,9 @@
+#!/bin/bash -ex
+
+docker login -u richard -p $DKREGISTRY_PASS $DKREGISTRY
+
+(cd nextcloud && docker-compose -f docker-compose.yml build)
+(cd sslproxy && docker-compose -f docker-compose.yml build)
+
+docker push dkregistry.xai-corp.net:5000/xaicorp/nextcloud:latest
+docker push dkregistry.xai-corp.net:5000/sslproxy:2.1

dockerfiles/services/gitea/docker-compose.yml

@@ -2,7 +2,7 @@
 # DOCKER_HOST=192.168.2.41:2376 docker-compose up -d
 # DOCKER_HOST=dkhost:2376 docker stack deploy -c docker-compose.yml gitea
 
-version: '3'
+version: '3.4'
 services:
 
   app:
@@ -20,6 +20,10 @@ services:
         condition: any
         delay: "5s"
         max_attempts: 3
+      update_config:
+        parallelism: 1
+        delay: 2s
+        order: start-first
       labels:
         net.xai-corp.sslproxy.description:  proxy ssl calls to non ssl containers
       resources:
@@ -33,3 +37,14 @@ services:
         fluentd-address: "logs.xai-corp.net:24224"
         fluentd-async-connect: 'true'
         tag: gitea
+
+networks:
+  default:
+    external:
+      name: ingress
+  ingress:
+    external:
+      name: ingress
+  prod:
+    external:
+      name: prod

dockerfiles/services/launch_apps.sh

@@ -0,0 +1,13 @@
+#!/bin/bash -ex
+
+export DOCKER_HOST=dkmanager:2376
+
+docker login -u richard -p $DKREGISTRY_PASS $DKREGISTRY
+
+docker stack deploy --with-registry-auth --prune -c gitea/docker-compose.yml gitea
+
+docker stack deploy --with-registry-auth --prune -c nextcloud/docker-compose-prod.yml nextcloud
+
+docker stack deploy --with-registry-auth --prune -c ui/docker-compose.yml dkui
+
+docker stack deploy --with-registry-auth --prune -c sslproxy/docker-compose-prod.yml sslproxy

dockerfiles/services/nextcloud/docker-compose-prod.yml

@@ -4,10 +4,10 @@
 # DOCKER_HOST=dkhost:2376 docker stack deploy -c docker-compose-prod.yml owncloud
 # DOCKER_HOST=dkhost:2376 docker stack ps owncloud
 
-version: '3'
+version: '3.4'
 services:
 
-  owncloud:
+  app:
     image: dkregistry.xai-corp.net:5000/xaicorp/nextcloud:latest
     ports:
       - 8083:80
@@ -24,6 +24,10 @@ services:
         condition: any
         delay: "1s"
         max_attempts: 5
+      update_config:
+        parallelism: 1
+        delay: 2s
+        order: start-first
       resources:
         limits:
           cpus: '1'
@@ -36,24 +40,39 @@ services:
         fluentd-async-connect: 'true'
         tag: nextcloud
 
-  collabora:
+#  collabora:
-    image: collabora/code
+#    image: collabora/code
-    ports:
+#    ports:
-      - 9980:9980
+#      - 9980:9980
-    environment:
+#    environment:
-      domain: office\\.xai-corp\\.net
+#      domain: office\\.xai-corp\\.net
-      username: admin
+#      username: admin
-      password: ah8031qhnbc
+#      password: ah8031qhnbc
-      server_name: office.xai-corp.net
+#      server_name: office.xai-corp.net
+#
+#    deploy:
+#      mode: replicated
+#      replicas: 0
+#      restart_policy:
+#        condition: any
+#        delay: "1s"
+#        max_attempts: 5
+#      update_config:
+#        parallelism: 1
+#        delay: 2s
+#        order: start-first
+#      resources:
+#        limits:
+#          cpus: '1'
+#          memory: 512M
 
-    deploy:
+networks:
-      mode: replicated
+  default:
-      replicas: 0
+    external:
-      restart_policy:
+      name: ingress
-        condition: any
+  ingress:
-        delay: "1s"
+    external:
-        max_attempts: 5
+      name: ingress
-      resources:
+  prod:
-        limits:
+    external:
-          cpus: '1'
+      name: prod
-          memory: 512M

dockerfiles/services/nextcloud/docker-compose.yml

@@ -18,16 +18,16 @@ services:
 #      - 9083:9000
     volumes:
       - ./data:/var/www/html
-
+#
-  collabora:
+#  collabora:
-    image: collabora/code
+#    image: collabora/code
-    ports:
+#    ports:
-      - 9980:9980
+#      - 9980:9980
-    environment:
+#    environment:
-      domain: office\\.xai-corp\\.net
+#      domain: office\\.xai-corp\\.net
-      username: admin
+#      username: admin
-      password: ah8031qhnbc
+#      password: ah8031qhnbc
-      server_name: office.xai-corp.net
+#      server_name: office.xai-corp.net
 
 #  letsencrypt:
 #    image: linuxserver/letsencrypt

dockerfiles/services/services/build_services.sh

@@ -0,0 +1,7 @@
+#!/bin/bash -ex
+
+docker login -u richard -p $DKREGISTRY_PASS $DKREGISTRY
+
+(cd fluentd && docker-compose -f docker-compose.yml build)
+
+docker push dkregistry.xai-corp.net:5000/fluentd:latest

dockerfiles/services/services/datadog/docker-compose.yml

@@ -0,0 +1,44 @@
+---
+# docker-compose file for datadog agent
+# DOCKER_HOST=dkhost:2376 docker-compose up -d
+#
+# DOCKER_HOST=dkhost:2376 docker stack deploy -c docker-compose.yml services
+#
+# DOCKER_HOST=dkhost:2376  docker stack ps services
+
+version: '3.4'
+services:
+
+  datadog:
+    image: datadog/docker-dd-agent:latest-dogstatsd
+    hostname: home
+    environment:
+      API_KEY: ca0faf176c4aedd4f547ed7cf85615eb
+      LOG_LEVEL: INFO
+    ports:
+      - "8125:8125/udp"
+    deploy:
+      mode: global
+      restart_policy:
+        condition: any
+        delay: "1m"
+        max_attempts: 1
+      update_config:
+        parallelism: 1
+        delay: 2s
+        order: start-first
+      resources:
+        limits:
+          cpus: '1.5'
+          memory: 512M
+
+networks:
+  default:
+    external:
+      name: prod
+  ingress:
+    external:
+      name: ingress
+  prod:
+    external:
+      name: prod

dockerfiles/services/services/dd-agent/docker-compose.yml

@@ -1,38 +0,0 @@
----
-# docker-compose file for datadog agent
-# DOCKER_HOST=dkhost:2376 docker-compose up -d
-#
-# DOCKER_HOST=dkhost:2376 docker stack deploy -c docker-compose.yml services
-#
-# DOCKER_HOST=dkhost:2376  docker stack ps services
-
-version: '3'
-services:
-
-  dd-agent:
-    image: "datadog/docker-dd-agent:latest"
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock:ro
-      - /proc/:/host/proc/:ro
-      - /sys/fs/cgroup/:/host/sys/fs/cgroup:ro
-    ports:
-      - "8125:8125/udp"
-    env_file:
-      .env
-
-    deploy:
-      mode: global
-
-      restart_policy:
-        condition: any
-        delay: "1s"
-        max_attempts: 3
-      resources:
-        limits:
-          cpus: '2'
-          memory: 512M
-
-#networks:
-#  default:
-#    external:
-#      name: prod

dockerfiles/services/services/dkregistry/docker-compose.yml

@@ -5,7 +5,7 @@
 # DOCKER_HOST=dkhost:2376 docker-compose up -d
 # DOCKER_HOST=dkhost:2376 docker stack deploy -c docker-compose.yml services
 
-version: '3'
+version: '3.4'
 services:
 
   registry:
@@ -36,6 +36,10 @@ services:
         condition: any
         delay: "1s"
         max_attempts: 1
+      update_config:
+        parallelism: 1
+        delay: 2s
+        order: start-first
       resources:
         limits:
           cpus: '0.2'
@@ -47,3 +51,17 @@ services:
         fluentd-address: "logs.xai-corp.net:24224"
         fluentd-async-connect: 'true'
         tag: dkregistry
+
+    networks:
+      - ingress
+
+networks:
+  default:
+    external:
+      name: prod
+  ingress:
+    external:
+      name: ingress
+  prod:
+    external:
+      name: prod

dockerfiles/services/services/fluentd/Dockerfile

@@ -8,7 +8,6 @@ RUN apk add --update --virtual .build-deps \
  && sudo gem install \
         fluent-plugin-secure-forward \
         fluent-plugin-loggly \
-
  && sudo gem sources --clear-all \
  && apk del .build-deps \
  && rm -rf /var/cache/apk/* \

dockerfiles/services/services/fluentd/docker-compose-prod.yml

@@ -0,0 +1,38 @@
+---
+# Based on https://hub.docker.com/r/fluent/fluentd/
+#
+# DOCKER_HOST=192.168.2.41:2376 docker-compose up -d
+#
+# docker login dkregistry.xai-corp.net:5000
+# docker-compose build && docker push dkregistry.xai-corp.net:5000/fluentd:latest
+# DOCKER_HOST=dkhost:2376 docker stack deploy --with-registry-auth -c docker-compose-prod.yml services
+# DOCKER_HOST=dkhost:2376 docker stack ps services
+
+version: '3.4'
+services:
+
+  fluentd:
+    image: "dkregistry.xai-corp.net:5000/fluentd:latest"
+    volumes:
+      - /opt/shared/logs:/fluentd/log
+    ports:
+      - "24224:24224"
+      - "24224:24224/udp"
+
+    deploy:
+      mode: global
+      restart_policy:
+        condition: any
+        delay: "1s"
+        max_attempts: 3
+      update_config:
+        parallelism: 1
+        delay: 2s
+        order: start-first
+      resources:
+        limits:
+          cpus: '0.5'
+          memory: 64M
+
+    networks:
+      - ingress

dockerfiles/services/services/fluentd/docker-compose.yml

@@ -8,12 +8,12 @@
 # DOCKER_HOST=dkhost:2376 docker stack deploy --with-registry-auth -c docker-compose.yml services
 # DOCKER_HOST=dkhost:2376 docker stack ps services
 
-version: '3'
+version: '3.4'
 services:
 
   fluentd:
     build:
-      context: ""
+      context: .
       dockerfile: Dockerfile
     image: "dkregistry.xai-corp.net:5000/fluentd:latest"
     volumes:
@@ -21,16 +21,3 @@ services:
     ports:
       - "24224:24224"
       - "24224:24224/udp"
-
-
-    deploy:
-      mode: global
-
-      restart_policy:
-        condition: any
-        delay: "1s"
-        max_attempts: 3
-      resources:
-        limits:
-          cpus: '0.5'
-          memory: 64M

dockerfiles/services/services/launch_services.sh

@@ -0,0 +1,28 @@
+#!/bin/bash -ex
+
+export DOCKER_HOST=dkmanager:2376
+
+CONFIG=stack.tmp.yml
+SERVICE=services
+#NETWORK=prod
+
+docker stack deploy --with-registry-auth -c dkregistry/docker-compose.yml $SERVICE
+
+docker login -u richard -p $DKREGISTRY_PASS $DKREGISTRY
+
+docker network inspect $NETWORK > /dev/null || docker network create -d overlay $NETWORK
+
+docker-compose \
+    -f network.yml \
+    -f fluentd/docker-compose-prod.yml \
+    -f mariadb/docker-compose.yml \
+    -f postgres/docker-compose.yml \
+    -f memcached/docker-compose.yml \
+    -f redis/docker-compose.yml \
+    -f datadog/docker-compose.yml \
+    config > $CONFIG
+
+docker stack deploy  --prune --with-registry-auth -c $CONFIG $SERVICE
+
+# Cleanup
+rm $CONFIG

dockerfiles/services/services/mariadb/docker-compose.yml

@@ -3,7 +3,7 @@
 # DOCKER_HOST=dkhost:2376 docker-compose up -d
 # DOCKER_HOST=dkhost:2376 docker stack deploy -c docker-compose.yml services
 
-version: '3'
+version: '3.4'
 services:
 
   mysql:
@@ -22,19 +22,22 @@ services:
         condition: any
         delay: "1s"
         max_attempts: 3
+      update_config:
+        parallelism: 1
+        delay: 2s
+        order: start-first
       resources:
         limits:
           cpus: '0.5'
           memory: 512M
 
-
     logging:
       driver: fluentd
       options:
         fluentd-address: "logs.xai-corp.net:24224"
+        fluentd-async-connect: 'true'
         tag: mariadb
 
-networks:
+    networks:
-  default:
+      - prod
-    external:
+      - ingress
-      name: prod

dockerfiles/services/services/memcached/docker-compose.yml

@@ -4,7 +4,7 @@
 # DOCKER_HOST=dkhost:2376 docker stack deploy -c docker-compose.yml services
 # DOCKER_HOST=dkhost:2376 docker service ps services_memcached
 
-version: '3'
+version: '3.4'
 services:
 
   memcached:
@@ -15,25 +15,27 @@ services:
       - memcached
       - -m64
 
-    deploy:
-      mode: replicated
-      replicas: 3
-      restart_policy:
-        condition: any
-        delay: "1s"
-        max_attempts: 3
-      resources:
-        limits:
-          cpus: '0.5'
-          memory: 65M
-
     logging:
       driver: fluentd
       options:
         fluentd-address: "logs.xai-corp.net:24224"
         tag: memcached
 
-networks:
+    deploy:
-  default:
+      mode: replicated
-    external:
+      replicas: 1
-      name: prod
+      restart_policy:
+        condition: any
+        delay: "1s"
+        max_attempts: 3
+      update_config:
+        parallelism: 1
+        delay: 2s
+        order: start-first
+      resources:
+        limits:
+          cpus: '0.5'
+          memory: 65M
+
+    networks:
+      - prod

dockerfiles/services/services/network.yml

@@ -0,0 +1,15 @@
+---
+# docker-compose file for defining networks used by services
+
+version: '3.4'
+
+networks:
+  default:
+    external:
+      name: prod
+  ingress:
+    external:
+      name: ingress
+  prod:
+    external:
+      name: prod

dockerfiles/services/services/postgres/docker-compose.yml

@@ -1,7 +1,7 @@
 # DOCKER_HOST=dkhost:2376 docker stack deploy -c docker-compose.yml services
 # DOCKER_HOST=dkhost:2376 docker service ps services_postgres
 
-version: '3'
+version: '3.4'
 services:
   postgres:
     volumes:
@@ -15,6 +15,13 @@ services:
     ports:
       - 5432:5432
 
+    logging:
+      driver: fluentd
+      options:
+        fluentd-address: "logs.xai-corp.net:24224"
+        fluentd-async-connect: 'true'
+        tag: postgres
+
     deploy:
       mode: replicated
       replicas: 1
@@ -22,12 +29,14 @@ services:
         condition: any
         delay: "1s"
         max_attempts: 1
+      update_config:
+        parallelism: 1
+        delay: 2s
+        order: start-first
       resources:
         limits:
           cpus: '0.5'
           memory: 128M
 
-networks:
+    networks:
-  default:
+      - prod
-    external:
-      name: prod

dockerfiles/services/services/redis/docker-compose.yml

@@ -2,7 +2,7 @@
 # docker-compose file for redis
 # DOCKER_HOST=dkhost:2376 docker stack deploy -c docker-compose.yml services
 
-version: '3'
+version: '3.4'
 services:
 
   redis:
@@ -17,6 +17,10 @@ services:
         condition: any
         delay: "1s"
         max_attempts: 3
+      update_config:
+        parallelism: 1
+        delay: 2s
+        order: start-first
       resources:
         limits:
           cpus: '0.5'
@@ -26,9 +30,8 @@ services:
       driver: fluentd
       options:
         fluentd-address: "logs.xai-corp.net:24224"
+        fluentd-async-connect: 'true'
         tag: redis
 
-networks:
+    networks:
-  default:
+      - prod
-    external:
-      name: prod

dockerfiles/services/sslproxy/docker-compose-prod.yml

@@ -5,7 +5,7 @@
 # DOCKER_HOST=dkhost:2376 docker stack deploy --with-registry-auth -c docker-compose-prod.yml sslproxy
 # DOCKER_HOST=dkhost:2376 docker stack ps sslproxy
 
-version: '3'
+version: '3.4'
 services:
   app:
     image: "dkregistry.xai-corp.net:5000/sslproxy:2.1"
@@ -21,13 +21,13 @@ services:
 #        syslog-address: "tcp+tls://logs6.papertrailapp.com:38577"
 #        tag: "{{.Name}}/{{.ID}}"
 
-#    logging:
+    logging:
-#      driver: fluentd#        tag: sslproxy
+      driver: fluentd
+        tag: sslproxy
+      options:
+        fluentd-address: "logs.xai-corp.net:24224"
+        fluentd-async-connect: 'true'
 
-
-#      options:
-#        fluentd-address: "logs.xai-corp.net:24224"
-#        fluentd-async-connect: 'true'
 #    logging:
 #      driver: syslog
 #      options:
@@ -39,11 +39,11 @@ services:
       restart_policy:
         condition: any
         delay: 6s
-        max_attempts: 10
+        max_attempts: 3
-#      update_config:
+      update_config:
-#        parallelism: 2
+        parallelism: 1
-#        delay: 2s
+        delay: 2s
-#        order: start-first
+        order: start-first
       resources:
         limits:
           cpus: '0.1'
@@ -52,14 +52,14 @@ services:
       labels:
         net.xai-corp.sslproxy.description:  proxy ssl calls to non ssl containers
 
-    networks:
+#    networks:
-    - ingress
+#    - ingress
 #    - prod
 
-networks:
+#networks:
-  ingress:
+#  ingress:
-    external:
+#    external:
-      name: ingress
+#      name: ingress
-  prod:
+#  prod:
-    external:
+#    external:
-      name: prod
+#      name: prod

dockerfiles/services/sslproxy/hosts/abcapi.xai-corp.net.conf

@@ -5,10 +5,15 @@ server {
     ssl_certificate_key /etc/letsencrypt/live/xai-corp.net/privkey.pem;
 
     #Strict-Transport-Security: max-age=15768000
-    #add_header Strict-Transport-Security "max-age=15768000; includeSubDomains" always;
+    add_header Strict-Transport-Security "max-age=15768000; includeSubDomains" always;
 
     location / {
         proxy_pass http://dkhost.xai-corp.net:8084;
+        proxy_set_header Host              $host;   # required for docker client's sake
+        proxy_set_header X-Real-IP         $remote_addr; # pass on real client's IP
+        proxy_set_header X-Forwarded-For   $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header Connection $http_connection;
     }
 
 }

dockerfiles/services/ui/docker-compose.yml

@@ -2,11 +2,11 @@
 # DOCKER_HOST=192.168.2.41:2376 docker-compose up -d
 # DOCKER_HOST=dkhost:2376 docker stack deploy -c docker-compose.yml dkui
 
-version: '3'
+version: '3.4'
 services:
   app:
-    restart: always
+#    restart: always
-    privileged: true
+#    privileged: true
     image: "portainer/portainer"
 
     volumes:
@@ -23,6 +23,10 @@ services:
         condition: any
         delay: 5s
         max_attempts: 10
+      update_config:
+        parallelism: 1
+        delay: 2s
+        order: start-first
       labels:
         net.xai-corp.dkui.description: portainer ui for docker host and swarm management
       resources:
@@ -37,3 +41,14 @@ services:
         fluentd-address: "logs.xai-corp.net:24224"
         fluentd-async-connect: 'true'
         tag: dkui
+
+networks:
+  default:
+    external:
+      name: ingress
+  ingress:
+    external:
+      name: ingress
+  prod:
+    external:
+      name: prod

inventory.conf

@@ -4,7 +4,7 @@ localhost       ansible_connection=local
 home            ansible_ssh_host=192.168.2.11
 home02          ansible_ssh_host=192.168.2.22
 dkhost01        ansible_ssh_host=192.168.2.41
-#dkhost02       ansible_ssh_host=192.168.2.43
+dkhost02       ansible_ssh_host=192.168.2.43
 dkhost03        ansible_ssh_host=192.168.2.53
 #dkhost04        ansible_ssh_host=192.168.2.54
 #dkhost05        ansible_ssh_host=192.168.2.55
@@ -12,13 +12,13 @@ dkhost03        ansible_ssh_host=192.168.2.53
 cubox-i         ansible_ssh_host=192.168.2.12
 
 [dkhost]
-#dkhost01        ansible_ssh_host=192.168.2.41
+dkhost01        ansible_ssh_host=192.168.2.41
-#dkhost02       ansible_ssh_host=192.168.2.43
+dkhost02        ansible_ssh_host=192.168.2.52
 #dkhost03        ansible_ssh_host=192.168.2.53
 #dkhost04        ansible_ssh_host=192.168.2.54
 #dkhost05        ansible_ssh_host=192.168.2.55
-#home            ansible_ssh_host=192.168.2.11
+home            ansible_ssh_host=192.168.2.11
-localhost       ansible_connection=local
+#localhost       ansible_connection=local
 
 [ns]
 home02          ansible_ssh_host=192.168.2.22

roles/ns.xai-corp.net/templates/xai-corp.net.internal.j2

@@ -27,7 +27,7 @@ cubox-i                 IN A    192.168.2.12
 ; virtual machine servers
 home02                  IN A    192.168.2.22
 dkhost01                IN A    192.168.2.41
-dkhost02                IN A    192.168.2.43
+dkhost02                IN A    192.168.2.52
 dkhost03                IN A    192.168.2.53
 dkhost04                IN A    192.168.2.54
 dkhost05                IN A    192.168.2.55
@@ -41,12 +41,18 @@ gluster                 IN A    192.168.2.11
 gluster                 IN A    192.168.2.12
 
 ; docker swarm nodes
-dkhost                  IN A    192.168.2.41
+dkhost                  IN A    192.168.2.11
-;dkhost                  IN A    192.168.2.43
+;dkhost                  IN A    192.168.2.41
-dkhost                  IN A    192.168.2.53
+dkhost                  IN A    192.168.2.52
-;dkhost                  IN A    192.168.2.54
+;dkhost                  IN A    192.168.2.53
+dkhost                  IN A    192.168.2.54
 ;dkhost                  IN A    192.168.2.55
 
+; docker swarm managers
+dkmanager               IN A    192.168.2.11
+dkmanager               IN A    192.168.2.52
+dkmanager               IN A    192.168.2.54
+
 ; service domains
 fs                      IN CNAME    dkhost
 git                     IN CNAME    dkhost

scripts/reboot.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+# run ansible playbook to update name servers
+
+ansible-playbook -vv ns.xai-corp.net.yml -i inventory.conf -u ansible