fix elasticsearch by creating a distributed glusterfs filesystem for elasticsearch data

update graylog

dkhost.xai-corp.net.yml

@@ -56,10 +56,11 @@
       jenkins:
         host: gluster:/jenkins
         mount: /var/lib/jenkins
+      elasticsearch:
+        host: gluster:/elasticsearch
+        mount: /data/elasticsearch
 
   roles:
-#    - _install_updates
-#    - Datadog.datadog
     - dockerhost
     - geerlingguy.nginx
     - certbot

dockerfiles/graylog/docker-compose-graylog.yml

@@ -1,40 +0,0 @@
----
-# docker-compose file for graylog
-# DOCKER_HOST=dkhost03:2376 docker-compose up -d
-# DOCKER_HOST=dkhost03:2376 docker stack deploy -c docker-compose.yml graylog
-
-version: '3'
-services:
-
-  graylog:
-    image: graylog2/server:latest
-    volumes:
-      - /opt/shared/graylog/data/journal:/usr/share/graylog/data/journal
-      - /opt/shared/graylog/config:/usr/share/graylog/data/config
-    environment:
-      GRAYLOG_PASSWORD_SECRET: gsahu1dj901hdaiuafg3g1q
-      GRAYLOG_ROOT_PASSWORD_SHA2: d0fc133359968fbc38a267f29606c9dc805af7bcc231df9b9acd2e8e6e894ede
-      GRAYLOG_WEB_ENDPOINT_URI: http://logs.xai-corp.net:10090/api/
-      GRAYLOG_REST_LISTEN_URI: http://0.0.0.0:9000/api/
-      GRAYLOG_MONGODB_URI: mongodb://mongo/graylog
-      GRAYLOG_ELASTICSEARCH_CLUSTER_NAME: es.xai-corp.net
-    depends_on:
-      - mongo
-      - elasticsearch
-    ports:
-      - "10090:9000"
-      - "12201:12201/udp"
-      - "1514:1514/udp"
-      - "514:514/udp
-    deploy:
-      mode: replicated
-      replicas: 1
-      restart_policy:
-        condition: any
-        delay: "1s"
-        max_attempts: 3
-
-networks:
-  default:
-    external:
-      name: prod-private

dockerfiles/graylog/docker-compose-test.yml

@@ -1,26 +0,0 @@
----
-# docker-compose file for graylog
-# DOCKER_HOST=dkhost03:2376 docker-compose up -d
-# DOCKER_HOST=dkhost03:2376 docker stack deploy -c docker-compose.yml graylog
-
-version: '3'
-services:
-
-  test:
-    image: alpine
-    command: ping nosql
-    depends_on:
-      - mongo
-      - elasticsearch
-    deploy:
-      mode: replicated
-      replicas: 1
-      restart_policy:
-        condition: any
-        delay: "1s"
-        max_attempts: 3
-
-networks:
-  default:
-    external:
-      name: prod-private

dockerfiles/graylog/docker-compose.yml

@@ -6,46 +6,6 @@
 version: '3'
 services:
 
-  mongo:
-    image: "mongo:3"
-    volumes:
-      - /opt/shared/graylog/data/mongo:/data/db
-    ports:
-      - "27017:27017"
-    deploy:
-      mode: replicated
-      replicas: 1
-      restart_policy:
-        condition: any
-        delay: "1s"
-        max_attempts: 10
-      resources:
-        limits:
-          cpus: '0.1'
-          memory: 512M
-
-  elasticsearch:
-    image: "elasticsearch:2"
-    command: "elasticsearch -Des.cluster.name='graylog'"
-    volumes:
-      - /opt/shared/graylog/data/elasticsearch:/usr/share/elasticsearch/data
-    ports:
-      - "9350:9350"
-      - "9300:9300"
-      - "9200:9200"
-      - "10091:80"
-    deploy:
-      mode: replicated
-      replicas: 1
-      restart_policy:
-        condition: any
-        delay: "1s"
-        max_attempts: 10
-      resources:
-        limits:
-          cpus: '0.1'
-          memory: 512M
-
   graylog:
     image: graylog2/server:latest
     volumes:
@@ -54,9 +14,10 @@ services:
     environment:
       GRAYLOG_PASSWORD_SECRET: gsahu1dj901hdaiuafg3g1q
       GRAYLOG_ROOT_PASSWORD_SHA2: d0fc133359968fbc38a267f29606c9dc805af7bcc231df9b9acd2e8e6e894ede
-      GRAYLOG_WEB_ENDPOINT_URI: http://logs.xai-corp.net:10090/api/
+      GRAYLOG_WEB_ENDPOINT_URI: https://logs.xai-corp.net/api/
       GRAYLOG_REST_LISTEN_URI: http://0.0.0.0:9000/api/
       GRAYLOG_MONGODB_URI: mongodb://mongo/graylog
+      GRAYLOG_ELASTICSEARCH_CLUSTER_NAME: es.xai-corp.net
     depends_on:
       - mongo
       - elasticsearch
@@ -64,10 +25,16 @@ services:
       - "10090:9000"
       - "12201:12201/udp"
       - "1514:1514/udp"
+      - "514:514/udp"
     deploy:
       mode: replicated
       replicas: 1
       restart_policy:
         condition: any
         delay: "1s"
-        max_attempts: 10
+        max_attempts: 3
+
+networks:
+  default:
+    external:
+      name: prod-private

dockerfiles/services/elasticsearch/docker-compose.yml

@@ -1,7 +1,7 @@
 ---
 # docker-compose file for graylog
 # DOCKER_HOST=dkhost03:2376 docker-compose up -d
-# DOCKER_HOST=dkhost03:2376 docker stack deploy -c docker-compose-elasticsearch.yml services
+# DOCKER_HOST=dkhost03:2376 docker stack deploy -c docker-compose.yml services
 
 version: '3'
 services:
@@ -11,8 +11,8 @@ services:
 #    image: docker.elastic.co/elasticsearch/elasticsearch:5.4.1
     command: "elasticsearch -Des.cluster.name='es.xai-corp.net'"
     volumes:
-      - /opt/shared/elasticsearch/data:/usr/share/elasticsearch/data
-      - /opt/shared/elasticsearch/confg:/usr/share/elasticsearch/config
+      - /data/elasticsearch/data:/usr/share/elasticsearch/data
+      - /data/elasticsearch/config:/usr/share/elasticsearch/config
     ports:
 #      - "9350:9350"
 #      - "9300:9300"
@@ -24,23 +24,23 @@ services:
       restart_policy:
         condition: any
         delay: "1s"
-        max_attempts: 3
+        max_attempts: 1
       resources:
         limits:
           cpus: '1'
           memory: 512M
 
   ui:
-    image: appbaseio/dejavu
+    image: mobz/elasticsearch-head:2
     ports:
-      - "1358:1358"
+      - "1358:9100"
     deploy:
       mode: replicated
       replicas: 1
       restart_policy:
         condition: any
         delay: "1s"
-        max_attempts: 3
+        max_attempts: 1
 
 
 networks:

dockerfiles/services/mongo/docker-compose.yml

@@ -1,7 +1,7 @@
 ---
 # docker-compose file for graylog
 # DOCKER_HOST=dkhost03:2376 docker-compose up -d
-# DOCKER_HOST=dkhost03:2376 docker stack deploy -c docker-compose-mongodb.yml services
+# DOCKER_HOST=dkhost03:2376 docker stack deploy -c docker-compose.yml services
 
 version: '3'
 services:

glusterfs.yml

@@ -16,6 +16,7 @@
       - /data/glusterfs/gitea/brick1
       - /data/glusterfs/jenkins/brick1
       - /data/glusterfs/tmp/brick1
+      - /data/glusterfs/elasticsearch/brick1
     volumes:
       vmshares:
         - /data/glusterfs/vmshares/brick1/brick
@@ -23,8 +24,11 @@
         - /data/glusterfs/gitea/brick1/brick
       jenkins:
         - /data/glusterfs/jenkins/brick1/brick
-#      tmp:
-#        - /data/glusterfs/tmp/brick1/brick
+    distributed_volumes:
+      tmp:
+        - /data/glusterfs/tmp/brick1/brick
+      elasticsearch:
+        - /data/glusterfs/elasticsearch/brick1/brick
 
   roles:
     - glusterfs

roles/glusterfs/tasks/main.yml

@@ -34,13 +34,23 @@
   with_dict: "{{volumes}}"
   run_once: true
 
-# manage volumes
-- name: create tmp gluster volumes
+#- name: create tmp gluster volumes
+#  gluster_volume:
+#    state: present
+#    name: "tmp"
+#    brick: '/data/glusterfs/tmp/brick1/brick'
+#    rebalance: no
+#    replicas: 1
+#    cluster: "{{cluster}}"
+#  run_once: true
+
+- name: create distributed gluster volumes
   gluster_volume:
     state: present
-    name: "tmp"
-    brick: '/data/glusterfs/tmp/brick1/brick'
+    name: "{{ item.key }}"
+    brick: '{{ item.value | join(",")}}'
     rebalance: no
-    replicas: 2
+    replicas: false
     cluster: "{{cluster}}"
+  with_dict: "{{distributed_volumes}}"
   run_once: true