add service scripts

2018-02-03 09:02:45 -05:00
parent 9cfcc7de0b
commit d817d13607
26 changed files with 360 additions and 158 deletions
--- a/dockerfiles/services/services/build_services.sh
+++ b/dockerfiles/services/services/build_services.sh
@@ -0,0 +1,7 @@
+#!/bin/bash -ex
+
+docker login -u richard -p $DKREGISTRY_PASS $DKREGISTRY
+
+(cd fluentd && docker-compose -f docker-compose.yml build)
+
+docker push dkregistry.xai-corp.net:5000/fluentd:latest
--- a/dockerfiles/services/services/dd-agent/.env
+++ b/dockerfiles/services/services/dd-agent/.env
--- a/dockerfiles/services/services/datadog/docker-compose.yml
+++ b/dockerfiles/services/services/datadog/docker-compose.yml
@@ -0,0 +1,44 @@
+---
+# docker-compose file for datadog agent
+# DOCKER_HOST=dkhost:2376 docker-compose up -d
+#
+# DOCKER_HOST=dkhost:2376 docker stack deploy -c docker-compose.yml services
+#
+# DOCKER_HOST=dkhost:2376  docker stack ps services
+
+version: '3.4'
+services:
+
+  datadog:
+    image: datadog/docker-dd-agent:latest-dogstatsd
+    hostname: home
+    environment:
+      API_KEY: ca0faf176c4aedd4f547ed7cf85615eb
+      LOG_LEVEL: INFO
+    ports:
+      - "8125:8125/udp"
+    deploy:
+      mode: global
+      restart_policy:
+        condition: any
+        delay: "1m"
+        max_attempts: 1
+      update_config:
+        parallelism: 1
+        delay: 2s
+        order: start-first
+      resources:
+        limits:
+          cpus: '1.5'
+          memory: 512M
+
+networks:
+  default:
+    external:
+      name: prod
+  ingress:
+    external:
+      name: ingress
+  prod:
+    external:
+      name: prod
--- a/dockerfiles/services/services/dd-agent/docker-compose.yml
+++ b/dockerfiles/services/services/dd-agent/docker-compose.yml
@@ -1,38 +0,0 @@
---
-# docker-compose file for datadog agent
-# DOCKER_HOST=dkhost:2376 docker-compose up -d
-#
-# DOCKER_HOST=dkhost:2376 docker stack deploy -c docker-compose.yml services
-#
-# DOCKER_HOST=dkhost:2376  docker stack ps services
-
-version: '3'
-services:
-
-  dd-agent:
-    image: "datadog/docker-dd-agent:latest"
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock:ro
-      - /proc/:/host/proc/:ro
-      - /sys/fs/cgroup/:/host/sys/fs/cgroup:ro
-    ports:
-      - "8125:8125/udp"
-    env_file:
-      .env
-
-    deploy:
-      mode: global
-
-      restart_policy:
-        condition: any
-        delay: "1s"
-        max_attempts: 3
-      resources:
-        limits:
-          cpus: '2'
-          memory: 512M
-
-#networks:
-#  default:
-#    external:
-#      name: prod
--- a/dockerfiles/services/services/dkregistry/docker-compose.yml
+++ b/dockerfiles/services/services/dkregistry/docker-compose.yml
@@ -0,0 +1,67 @@
+---
+# docker-compose file for logstash
+# - see https://www.elastic.co/guide/en/logstash/current/_pulling_the_image.html
+#
+# DOCKER_HOST=dkhost:2376 docker-compose up -d
+# DOCKER_HOST=dkhost:2376 docker stack deploy -c docker-compose.yml services
+
+version: '3.4'
+services:
+
+  registry:
+    image: registry:2
+    ports:
+      - 5000:5000
+    environment:
+      REGISTRY_HTTP_SECRET: aabuioqlwlcpp2
+      REGISTRY_HTTP_TLS_CERTIFICATE: /certs/live/xai-corp.net/fullchain.pem
+      REGISTRY_HTTP_TLS_KEY: /certs/live/xai-corp.net/privkey.pem
+#      REGISTRY_HTTP_LETSENCRYPT_CACHEFILE: /var/run/letsencrypt.cache
+#      REGISTRY_HTTP_LETSENCRYPT_EMAIL: r_morgan@sympatico.ca
+      REGISTRY_HTTP_HOST: https://dkregistry.xai-corp.net:5000
+#      REGISTRY_HTTP_ADDR: dkregistry.xai-corp.net:5000
+      REGISTRY_AUTH: htpasswd
+      REGISTRY_AUTH_HTPASSWD_PATH: /auth/htpasswd
+      REGISTRY_AUTH_HTPASSWD_REALM: Registry Realm
+    volumes:
+      - /opt/shared/dkregistry/data:/var/lib/registry
+      - /opt/shared/letsencrypt-2:/certs
+      - /opt/shared/letsencrypt-2:/etc/letsencrypt
+      - /opt/shared/dkregistry/auth:/auth
+
+    deploy:
+      mode: replicated
+      replicas: 2
+      restart_policy:
+        condition: any
+        delay: "1s"
+        max_attempts: 1
+      update_config:
+        parallelism: 1
+        delay: 2s
+        order: start-first
+      resources:
+        limits:
+          cpus: '0.2'
+          memory: 64M
+
+    logging:
+      driver: fluentd
+      options:
+        fluentd-address: "logs.xai-corp.net:24224"
+        fluentd-async-connect: 'true'
+        tag: dkregistry
+
+    networks:
+      - ingress
+
+networks:
+  default:
+    external:
+      name: prod
+  ingress:
+    external:
+      name: ingress
+  prod:
+    external:
+      name: prod
--- a/dockerfiles/services/services/fluentd/Dockerfile
+++ b/dockerfiles/services/services/fluentd/Dockerfile
@@ -8,7 +8,6 @@ RUN apk add --update --virtual .build-deps \
 && sudo gem install \
        fluent-plugin-secure-forward \
        fluent-plugin-loggly \
-
 && sudo gem sources --clear-all \
 && apk del .build-deps \
 && rm -rf /var/cache/apk/* \
--- a/dockerfiles/services/services/fluentd/docker-compose-prod.yml
+++ b/dockerfiles/services/services/fluentd/docker-compose-prod.yml
@@ -0,0 +1,38 @@
+---
+# Based on https://hub.docker.com/r/fluent/fluentd/
+#
+# DOCKER_HOST=192.168.2.41:2376 docker-compose up -d
+#
+# docker login dkregistry.xai-corp.net:5000
+# docker-compose build && docker push dkregistry.xai-corp.net:5000/fluentd:latest
+# DOCKER_HOST=dkhost:2376 docker stack deploy --with-registry-auth -c docker-compose-prod.yml services
+# DOCKER_HOST=dkhost:2376 docker stack ps services
+
+version: '3.4'
+services:
+
+  fluentd:
+    image: "dkregistry.xai-corp.net:5000/fluentd:latest"
+    volumes:
+      - /opt/shared/logs:/fluentd/log
+    ports:
+      - "24224:24224"
+      - "24224:24224/udp"
+
+    deploy:
+      mode: global
+      restart_policy:
+        condition: any
+        delay: "1s"
+        max_attempts: 3
+      update_config:
+        parallelism: 1
+        delay: 2s
+        order: start-first
+      resources:
+        limits:
+          cpus: '0.5'
+          memory: 64M
+
+    networks:
+      - ingress
--- a/dockerfiles/services/services/fluentd/docker-compose.yml
+++ b/dockerfiles/services/services/fluentd/docker-compose.yml
@@ -8,12 +8,12 @@
 # DOCKER_HOST=dkhost:2376 docker stack deploy --with-registry-auth -c docker-compose.yml services
 # DOCKER_HOST=dkhost:2376 docker stack ps services

-version: '3'
+version: '3.4'
 services:

  fluentd:
    build:
-      context: ""
+      context: .
      dockerfile: Dockerfile
    image: "dkregistry.xai-corp.net:5000/fluentd:latest"
    volumes:
@@ -21,16 +21,3 @@ services:
    ports:
      - "24224:24224"
      - "24224:24224/udp"
-
-
-    deploy:
-      mode: global
-
-      restart_policy:
-        condition: any
-        delay: "1s"
-        max_attempts: 3
-      resources:
-        limits:
-          cpus: '0.5'
-          memory: 64M
--- a/dockerfiles/services/services/launch_services.sh
+++ b/dockerfiles/services/services/launch_services.sh
@@ -0,0 +1,28 @@
+#!/bin/bash -ex
+
+export DOCKER_HOST=dkmanager:2376
+
+CONFIG=stack.tmp.yml
+SERVICE=services
+#NETWORK=prod
+
+docker stack deploy --with-registry-auth -c dkregistry/docker-compose.yml $SERVICE
+
+docker login -u richard -p $DKREGISTRY_PASS $DKREGISTRY
+
+docker network inspect $NETWORK > /dev/null || docker network create -d overlay $NETWORK
+
+docker-compose \
+    -f network.yml \
+    -f fluentd/docker-compose-prod.yml \
+    -f mariadb/docker-compose.yml \
+    -f postgres/docker-compose.yml \
+    -f memcached/docker-compose.yml \
+    -f redis/docker-compose.yml \
+    -f datadog/docker-compose.yml \
+    config > $CONFIG
+
+docker stack deploy  --prune --with-registry-auth -c $CONFIG $SERVICE
+
+# Cleanup
+rm $CONFIG
--- a/dockerfiles/services/services/mariadb/docker-compose.yml
+++ b/dockerfiles/services/services/mariadb/docker-compose.yml
@@ -3,7 +3,7 @@
 # DOCKER_HOST=dkhost:2376 docker-compose up -d
 # DOCKER_HOST=dkhost:2376 docker stack deploy -c docker-compose.yml services

-version: '3'
+version: '3.4'
 services:

  mysql:
@@ -22,19 +22,22 @@ services:
        condition: any
        delay: "1s"
        max_attempts: 3
+      update_config:
+        parallelism: 1
+        delay: 2s
+        order: start-first
      resources:
        limits:
          cpus: '0.5'
          memory: 512M

-
    logging:
      driver: fluentd
      options:
        fluentd-address: "logs.xai-corp.net:24224"
+        fluentd-async-connect: 'true'
        tag: mariadb

-networks:
-  default:
-    external:
-      name: prod
+    networks:
+      - prod
+      - ingress
--- a/dockerfiles/services/services/memcached/docker-compose.yml
+++ b/dockerfiles/services/services/memcached/docker-compose.yml
@@ -4,7 +4,7 @@
 # DOCKER_HOST=dkhost:2376 docker stack deploy -c docker-compose.yml services
 # DOCKER_HOST=dkhost:2376 docker service ps services_memcached

-version: '3'
+version: '3.4'
 services:

  memcached:
@@ -15,25 +15,27 @@ services:
      - memcached
      - -m64

-    deploy:
-      mode: replicated
-      replicas: 3
-      restart_policy:
-        condition: any
-        delay: "1s"
-        max_attempts: 3
-      resources:
-        limits:
-          cpus: '0.5'
-          memory: 65M
-
    logging:
      driver: fluentd
      options:
        fluentd-address: "logs.xai-corp.net:24224"
        tag: memcached

-networks:
-  default:
-    external:
-      name: prod
+    deploy:
+      mode: replicated
+      replicas: 1
+      restart_policy:
+        condition: any
+        delay: "1s"
+        max_attempts: 3
+      update_config:
+        parallelism: 1
+        delay: 2s
+        order: start-first
+      resources:
+        limits:
+          cpus: '0.5'
+          memory: 65M
+
+    networks:
+      - prod
--- a/dockerfiles/services/services/network.yml
+++ b/dockerfiles/services/services/network.yml
@@ -0,0 +1,15 @@
+---
+# docker-compose file for defining networks used by services
+
+version: '3.4'
+
+networks:
+  default:
+    external:
+      name: prod
+  ingress:
+    external:
+      name: ingress
+  prod:
+    external:
+      name: prod
--- a/dockerfiles/services/services/postgres/docker-compose.yml
+++ b/dockerfiles/services/services/postgres/docker-compose.yml
@@ -1,7 +1,7 @@
 # DOCKER_HOST=dkhost:2376 docker stack deploy -c docker-compose.yml services
 # DOCKER_HOST=dkhost:2376 docker service ps services_postgres

-version: '3'
+version: '3.4'
 services:
  postgres:
    volumes:
@@ -15,6 +15,13 @@ services:
    ports:
      - 5432:5432

+    logging:
+      driver: fluentd
+      options:
+        fluentd-address: "logs.xai-corp.net:24224"
+        fluentd-async-connect: 'true'
+        tag: postgres
+
    deploy:
      mode: replicated
      replicas: 1
@@ -22,12 +29,14 @@ services:
        condition: any
        delay: "1s"
        max_attempts: 1
+      update_config:
+        parallelism: 1
+        delay: 2s
+        order: start-first
      resources:
        limits:
          cpus: '0.5'
          memory: 128M

-networks:
-  default:
-    external:
-      name: prod
+    networks:
+      - prod
--- a/dockerfiles/services/services/redis/docker-compose.yml
+++ b/dockerfiles/services/services/redis/docker-compose.yml
@@ -2,7 +2,7 @@
 # docker-compose file for redis
 # DOCKER_HOST=dkhost:2376 docker stack deploy -c docker-compose.yml services

-version: '3'
+version: '3.4'
 services:

  redis:
@@ -17,6 +17,10 @@ services:
        condition: any
        delay: "1s"
        max_attempts: 3
+      update_config:
+        parallelism: 1
+        delay: 2s
+        order: start-first
      resources:
        limits:
          cpus: '0.5'
@@ -26,9 +30,8 @@ services:
      driver: fluentd
      options:
        fluentd-address: "logs.xai-corp.net:24224"
+        fluentd-async-connect: 'true'
        tag: redis

-networks:
-  default:
-    external:
-      name: prod
+    networks:
+      - prod