setting up graylog docker swarm services

dockerfiles/gitea/docker-compose.yml

@@ -1,10 +1,24 @@
-version: '2'
+---
+# DOCKER_HOST=192.168.2.41:2376 docker-compose up -d
+# DOCKER_HOST=dkhost03:2376 docker stack deploy -c docker-compose.yml sslproxy
+
+version: '3'
 services:
+
   app:
-    restart: always
     image: "gitea/gitea:latest"
     volumes:
       - /var/lib/gitea:/data
     ports:
       - "10022:22"
       - "10080:3000"
+
+    deploy:
+      mode: replicated
+      replicas: 1
+      restart_policy:
+        condition: any
+        delay: "5s"
+        max_attempts: 10
+      labels:
+        net.xai-corp.sslproxy.description:  proxy ssl calls to non ssl containers

dockerfiles/graylog/docker-compose-elasticsearch.yml

@@ -0,0 +1,35 @@
+---
+# docker-compose file for graylog
+# DOCKER_HOST=dkhost03:2376 docker-compose up -d
+# DOCKER_HOST=dkhost03:2376 docker stack deploy -c docker-compose-elasticsearch.yml services
+
+version: '3'
+services:
+
+  elasticsearch:
+    image: "elasticsearch:2"
+#    image: docker.elastic.co/elasticsearch/elasticsearch:5.4.1
+    command: "elasticsearch -Des.cluster.name='es.xai-corp.net'"
+    volumes:
+      - /opt/shared/graylog/data/elasticsearch:/usr/share/elasticsearch/data
+    ports:
+#      - "9350:9350"
+#      - "9300:9300"
+      - "9200:9200"
+#      - "10091:80"
+    deploy:
+      mode: replicated
+      replicas: 1
+      restart_policy:
+        condition: any
+        delay: "1s"
+        max_attempts: 3
+      resources:
+        limits:
+          cpus: '0.5'
+          memory: 512M
+
+networks:
+  default:
+    external:
+      name: prod-private

dockerfiles/graylog/docker-compose-graylog.yml

@@ -0,0 +1,40 @@
+---
+# docker-compose file for graylog
+# DOCKER_HOST=dkhost03:2376 docker-compose up -d
+# DOCKER_HOST=dkhost03:2376 docker stack deploy -c docker-compose.yml graylog
+
+version: '3'
+services:
+
+  graylog:
+    image: graylog2/server:latest
+    volumes:
+      - /opt/shared/graylog/data/journal:/usr/share/graylog/data/journal
+      - /opt/shared/graylog/config:/usr/share/graylog/data/config
+    environment:
+      GRAYLOG_PASSWORD_SECRET: gsahu1dj901hdaiuafg3g1q
+      GRAYLOG_ROOT_PASSWORD_SHA2: d0fc133359968fbc38a267f29606c9dc805af7bcc231df9b9acd2e8e6e894ede
+      GRAYLOG_WEB_ENDPOINT_URI: http://logs.xai-corp.net:10090/api/
+      GRAYLOG_REST_LISTEN_URI: http://0.0.0.0:9000/api/
+      GRAYLOG_MONGODB_URI: mongodb://mongo/graylog
+      GRAYLOG_ELASTICSEARCH_CLUSTER_NAME: es.xai-corp.net
+    depends_on:
+      - mongo
+      - elasticsearch
+    ports:
+      - "10090:9000"
+      - "12201:12201/udp"
+      - "1514:1514/udp"
+      - "514:514/udp
+    deploy:
+      mode: replicated
+      replicas: 1
+      restart_policy:
+        condition: any
+        delay: "1s"
+        max_attempts: 3
+
+networks:
+  default:
+    external:
+      name: prod-private

dockerfiles/graylog/docker-compose-mongodb.yml

@@ -0,0 +1,34 @@
+---
+# docker-compose file for graylog
+# DOCKER_HOST=dkhost03:2376 docker-compose up -d
+# DOCKER_HOST=dkhost03:2376 docker stack deploy -c docker-compose-mongodb.yml services
+
+version: '3'
+services:
+
+  mongo:
+    image: "mongo:3"
+    volumes:
+      - /opt/shared/graylog/data/mongo:/data/db
+#    ports:
+#      - "27017:27017"
+    networks:
+      default:
+        aliases:
+          - nosql
+    deploy:
+      mode: replicated
+      replicas: 1
+      restart_policy:
+        condition: any
+        delay: "1s"
+        max_attempts: 10
+      resources:
+        limits:
+          cpus: '0.1'
+          memory: 512M
+
+networks:
+  default:
+    external:
+      name: prod-private

dockerfiles/graylog/docker-compose-test.yml

@@ -0,0 +1,26 @@
+---
+# docker-compose file for graylog
+# DOCKER_HOST=dkhost03:2376 docker-compose up -d
+# DOCKER_HOST=dkhost03:2376 docker stack deploy -c docker-compose.yml graylog
+
+version: '3'
+services:
+
+  test:
+    image: alpine
+    command: ping nosql
+    depends_on:
+      - mongo
+      - elasticsearch
+    deploy:
+      mode: replicated
+      replicas: 1
+      restart_policy:
+        condition: any
+        delay: "1s"
+        max_attempts: 3
+
+networks:
+  default:
+    external:
+      name: prod-private

dockerfiles/graylog/docker-compose.yml

@@ -0,0 +1,73 @@
+---
+# docker-compose file for graylog
+# DOCKER_HOST=dkhost03:2376 docker-compose up -d
+# DOCKER_HOST=dkhost03:2376 docker stack deploy -c docker-compose.yml graylog
+
+version: '3'
+services:
+
+  mongo:
+    image: "mongo:3"
+    volumes:
+      - /opt/shared/graylog/data/mongo:/data/db
+    ports:
+      - "27017:27017"
+    deploy:
+      mode: replicated
+      replicas: 1
+      restart_policy:
+        condition: any
+        delay: "1s"
+        max_attempts: 10
+      resources:
+        limits:
+          cpus: '0.1'
+          memory: 512M
+
+  elasticsearch:
+    image: "elasticsearch:2"
+    command: "elasticsearch -Des.cluster.name='graylog'"
+    volumes:
+      - /opt/shared/graylog/data/elasticsearch:/usr/share/elasticsearch/data
+    ports:
+      - "9350:9350"
+      - "9300:9300"
+      - "9200:9200"
+      - "10091:80"
+    deploy:
+      mode: replicated
+      replicas: 1
+      restart_policy:
+        condition: any
+        delay: "1s"
+        max_attempts: 10
+      resources:
+        limits:
+          cpus: '0.1'
+          memory: 512M
+
+  graylog:
+    image: graylog2/server:latest
+    volumes:
+      - /opt/shared/graylog/data/journal:/usr/share/graylog/data/journal
+      - /opt/shared/graylog/config:/usr/share/graylog/data/config
+    environment:
+      GRAYLOG_PASSWORD_SECRET: gsahu1dj901hdaiuafg3g1q
+      GRAYLOG_ROOT_PASSWORD_SHA2: d0fc133359968fbc38a267f29606c9dc805af7bcc231df9b9acd2e8e6e894ede
+      GRAYLOG_WEB_ENDPOINT_URI: http://logs.xai-corp.net:10090/api/
+      GRAYLOG_REST_LISTEN_URI: http://0.0.0.0:9000/api/
+      GRAYLOG_MONGODB_URI: mongodb://mongo/graylog
+    depends_on:
+      - mongo
+      - elasticsearch
+    ports:
+      - "10090:9000"
+      - "12201:12201/udp"
+      - "1514:1514/udp"
+    deploy:
+      mode: replicated
+      replicas: 1
+      restart_policy:
+        condition: any
+        delay: "1s"
+        max_attempts: 10

dockerfiles/shipyard/docker-compose.yml

@@ -0,0 +1,43 @@
+---
+# docker-compose file for docker shipyard
+# DOCKER_HOST=dkhost03:2376 docker-compose up -d
+
+version: '2'
+services:
+
+  shipyard-rethinkdb:
+    restart: always
+    image: rethinkdb
+
+  shipyard-discovery:
+    restart: always
+    image: microbox/etcd
+    ports:
+      - 4001:4001
+      - 7001:7001
+    command:
+      - -name
+      - discovery
+
+  shipyard-swarm-manager:
+    restart: always
+    image: swarm:latest
+    command:
+      - manage
+      - --host
+      - tcp://0.0.0.0:3375
+      - etcd://192.168.2.53:4001
+
+  shipyard-controller:
+    restart: always
+    privileged: true
+    image: "shipyard/shipyard:latest"
+    ports:
+      - "8080:8080"
+    links:
+      - shipyard-rethinkdb:rethinkdb
+      - shipyard-swarm-manager:swarm
+    command:
+      - server
+      - -d
+      - tcp://swarm:3375

dockerfiles/sslproxy/docker-compose.prod.yml

@@ -1,7 +0,0 @@
----
-version: '2'
-services:
-  app:
-    restart: always
-    volumes:
-      - /opt/shared/fileserver:/www/data:ro

dockerfiles/sslproxy/docker-compose.yml

@@ -1,5 +1,8 @@
 ---
-version: '2'
+# DOCKER_HOST=192.168.2.41:2376 docker-compose up -d
+# DOCKER_HOST=dkhost03:2376 docker stack deploy -c docker-compose.yml sslproxy
+
+version: '3'
 services:
   app:
     restart: always
@@ -11,5 +14,14 @@ services:
     volumes:
       - /etc/letsencrypt:/etc/letsencrypt:ro
     ports:
-#       - "80:80"
        - "443:443"
+
+    deploy:
+      mode: replicated
+      replicas: 1
+      restart_policy:
+        condition: any
+        delay: 5s
+        max_attempts: 10
+      labels:
+        net.xai-corp.sslproxy.description:  proxy ssl calls to non ssl containers

dockerfiles/sslproxy/host.conf

@@ -17,7 +17,7 @@ server {
 # git.xai-corp.net
 server {
     listen  443 ssl;
-    server_name git.xai-corp.net docker.dev;
+    server_name git.xai-corp.net;
     ssl_certificate     /etc/letsencrypt/live/git.xai-corp.net/cert.pem;
     ssl_certificate_key /etc/letsencrypt/live/git.xai-corp.net/privkey.pem;
 
@@ -33,7 +33,7 @@ server {
 # tripbuilder.xai-corp.net
 server {
     listen  443 ssl;
-    server_name tripbuilder.xai-corp.net docker.dev;
+    server_name tripbuilder.xai-corp.net;
     ssl_certificate     /etc/letsencrypt/live/tripbuilder.xai-corp.net/cert.pem;
     ssl_certificate_key /etc/letsencrypt/live/tripbuilder.xai-corp.net/privkey.pem;
 
@@ -48,7 +48,7 @@ server {
 # jenkins.xai-corp.net
 server {
     listen  443 ssl;
-    server_name jenkins.xai-corp.net docker.dev;
+    server_name jenkins.xai-corp.net;
     ssl_certificate     /etc/letsencrypt/live/jenkins.xai-corp.net/cert.pem;
     ssl_certificate_key /etc/letsencrypt/live/jenkins.xai-corp.net/privkey.pem;
 
@@ -64,9 +64,10 @@ server {
 # dkui.xai-corp.net
 server {
     listen  443 ssl;
-    server_name dkui.xai-corp.net docker.dev;
+    server_name dkui.xai-corp.net;
     ssl_certificate     /etc/letsencrypt/live/dkui.xai-corp.net/cert.pem;
     ssl_certificate_key /etc/letsencrypt/live/dkui.xai-corp.net/privkey.pem;
+
     #Strict-Transport-Security: max-age=15768000
     add_header Strict-Transport-Security "max-age=600; includeSubDomains" always;
 
@@ -77,10 +78,26 @@ server {
 
 }
 
+# logs.xai-corp.net
+server {
+    listen  443 ssl;
+    server_name logs.xai-corp.net;
+    ssl_certificate     /etc/letsencrypt/live/logs.xai-corp.net/cert.pem;
+    ssl_certificate_key /etc/letsencrypt/live/logs.xai-corp.net/privkey.pem;
+    #Strict-Transport-Security: max-age=15768000
+    #add_header Strict-Transport-Security "max-age=600; includeSubDomains" always;
+
+    location / {
+        proxy_set_header Connection $http_connection;
+        proxy_pass http://dkhost04.xai-corp.net:10090;
+    }
+
+}
+
 # sql.xai-corp.net
 server {
     listen  443 ssl;
-    server_name sql.xai-corp.net docker.dev;
+    server_name sql.xai-corp.net;
     ssl_certificate     /etc/letsencrypt/live/sql.xai-corp.net/cert.pem;
     ssl_certificate_key /etc/letsencrypt/live/sql.xai-corp.net/privkey.pem;
     #Strict-Transport-Security: max-age=15768000
@@ -95,7 +112,7 @@ server {
 # www.xai-corp.net
 server {
     listen  443 ssl;
-    server_name www.xai-corp.net xai-corp.net docker.dev;
+    server_name www.xai-corp.net xai-corp.net;
     ssl_certificate     /etc/letsencrypt/live/www.xai-corp.net/cert.pem;
     ssl_certificate_key /etc/letsencrypt/live/www.xai-corp.net/privkey.pem;
     #Strict-Transport-Security: max-age=15768000

dockerfiles/ui/docker-compose.yml

@@ -1,7 +1,8 @@
 ---
 # DOCKER_HOST=192.168.2.41:2376 docker-compose up -d
+# DOCKER_HOST=dkhost03:2376 docker stack deploy -c docker-compose.yml dkui
 
-version: '2'
+version: '3'
 services:
   app:
     restart: always
@@ -10,6 +11,18 @@ services:
 
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
+      - /opt/shared/portainer/data:/data
     ports:
 #       - "80:80"
       - "9000:9000"
+
+    deploy:
+      mode: replicated
+      replicas: 1
+      restart_policy:
+        condition: any
+        delay: 5s
+        max_attempts: 10
+      labels:
+        net.xai-corp.dkui.description: portainer ui for docker host and swarm management
+

logs.xai-corp.net.yml

@@ -0,0 +1,16 @@
+---
+# playbook for logs.xai-corp.net
+# configure hosts for running graylog
+
+
+- hosts: dkhost
+  remote_user: ansible
+  gather_facts: yes
+  become: true
+
+  vars:
+
+
+  roles:
+    - docker_graylog
+

roles/certbot/tasks/main.yml

@@ -12,6 +12,10 @@
     - "letsencrypt"
   when: ansible_os_family == "Debian"
 
+- name: create webroot /var/www/xai-corp.net
+  file:
+    state: directory
+    path: /var/www/xai-corp.net
 
 - name: create first certificates
   command: "letsencrypt certonly --webroot -w /var/www/xai-corp.net -d {{ item }}"

roles/docker_graylog/tasks/main.yml

@@ -1,8 +1,10 @@
 ---
 # main tasks for running graylog on a docker host
 
+# configure host for graylog
 # create folders for certs, data,
 - name: create data folders (/opt/dkregistry)
+  run_once: true
   file:
     path: "{{ item }}"
     state: directory
@@ -17,6 +19,7 @@
     - /opt/shared/graylog/data/mongo
 
 - name: install default config files
+  run_once: true
   copy:
     src: "{{ item }}"
     dest: "/opt/shared/graylog/config/{{ item }}"
@@ -24,12 +27,9 @@
     - graylog.conf
     - log4j2.xml
 
-- name: copy composer file
+# setup graylog docker service
-  copy:
+#- name: run docker up
-    src: docker-compose.yml
+#  shell: "DOCKER_HOST=dkhost03:2376 docker stack deploy -c docker-compose.yml graylog"
-    dest: /opt/shared/graylog/docker-compose.yml
+#  run_once: true
-
+#  args:
-- name: run docker up
+#    chdir: roles/docker_graylog/files
-  shell: "docker-compose down && docker-compose up -d"
-  args:
-    chdir: /opt/shared/graylog

roles/dockerhost/tasks/install-xenial.yml

@@ -25,8 +25,14 @@
     repo: deb https://apt.dockerproject.org/repo ubuntu-xenial main
     state: present
 
+- name: import repo key
+  apt_key:
+    id: 58118E89F3A912897C070ADBF76221572C52609D
+    keyserver: "hkp://ha.pool.sks-keyservers.net:80"
+    state: present
+
 - name: install prerequisits
-  shell: apt-get install linux-image-extra-$(uname -r) linux-image-extra-virtual
+  shell: apt-get -y install linux-image-extra-$(uname -r) linux-image-extra-virtual
 
 
 - name: create docker group
@@ -72,7 +78,7 @@
 ## install docker-compose
 - name: install docker-compose from git repo
   shell: |
-          curl -L https://github.com/docker/compose/releases/download/1.9.0/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose
+          curl -L https://github.com/docker/compose/releases/download/1.13.0/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose
           chmod +x /usr/local/bin/docker-compose
 
 ## expose the docker daemon on tcp

roles/ns.xai-corp.net/templates/xai-corp.net.internal.j2

@@ -29,16 +29,22 @@ gluster                 IN A    192.168.2.12
 
 home02                  IN A    192.168.2.22
 
+dkhost                  IN A    192.168.2.41
+dkhost                  IN A    192.168.2.43
+dkhost                  IN A    192.168.2.53
+dkhost                  IN A    192.168.2.54
+
 dkhost01                IN A    192.168.2.41
 dkregistry              IN A    192.168.2.41
 sql                     IN A    192.168.2.41
 tripbuilder             IN A    192.168.2.41
 
-logs                    IN A    192.168.2.42
-
 dkhost02                IN A    192.168.2.43
 fs                      IN A    192.168.2.43
 git                     IN A    192.168.2.43
 dkui                    IN A    192.168.2.43
 jenkins                 IN A    192.168.2.43
+logs                    IN A    192.168.2.43
 
+dkhost03                IN A    192.168.2.53
+dkhost04                IN A    192.168.2.54

roles/ns.xai-corp.net/templates/xai-corp.net.reverse.j2

@@ -21,3 +21,5 @@ $ORIGIN 2.168.192.IN-ADDR.ARPA.
 22              IN PTR  home02.xai-corp.net.
 41              IN PTR  dkhost01.xai-corp.net.
 43              IN PTR  dkhost02.xai-corp.net.
+53              IN PTR  dkhost03.xai-corp.net.
+54              IN PTR  dkhost04.xai-corp.net.

xai-corp1.test.yml

@@ -4,16 +4,40 @@
 - hosts: localhost
   become: true
 
-  vars:
+#  vars:
-    java_packages:
+#    java_packages:
-      - openjdk-7-jdk
+#      - openjdk-7-jdk
+#
+#  roles:
+#    - jenkins
+#    - devtools
+#    - {
+#        role: website,
+#        server_hostname: "htmlgames.xai-corp.net",
+#        server_root: "/var/www/{{ server_hostname }}",
+#        repo: "/home/richard/Documents/Aptana\ Studio\ 3/xai-corp\ workspace/htmlgames/"
+#    }
 
-  roles:
+  tasks:
-    - jenkins
+
-    - devtools
+    - name: install apt repo
-    - {
+      apt_repository:
-        role: website,
+        repo: deb https://apt.dockerproject.org/repo ubuntu-xenial main
-        server_hostname: "htmlgames.xai-corp.net",
+        state: present
-        server_root: "/var/www/{{ server_hostname }}",
+
-        repo: "/home/richard/Documents/Aptana\ Studio\ 3/xai-corp\ workspace/htmlgames/"
+    - name: import repo key
-    }
+      apt_key:
+        id: 58118E89F3A912897C070ADBF76221572C52609D
+        keyserver: "hkp://ha.pool.sks-keyservers.net:80"
+        state: present
+
+    - name: install prerequisits
+      shell: apt-get -y install linux-image-extra-$(uname -r) linux-image-extra-virtual
+
+    - name: install via apt
+      apt:
+        state: latest
+        update_cache: true
+        package: "{{ item }}"
+      with_items:
+        - docker-engine