From 76c0d85fb990c4c5b32c2649aafbf0f59b34eac0 Mon Sep 17 00:00:00 2001 From: richard Date: Fri, 16 Jun 2017 22:47:03 -0400 Subject: [PATCH] fix elasticsearch by creating a distributed glusterfs filesystem for elasticsearch data update graylog --- dkhost.xai-corp.net.yml | 5 +- .../graylog/docker-compose-graylog.yml | 40 --------------- dockerfiles/graylog/docker-compose-test.yml | 26 ---------- dockerfiles/graylog/docker-compose.yml | 51 ++++--------------- .../elasticsearch/docker-compose.yml} | 14 ++--- .../mongo/docker-compose.yml} | 2 +- glusterfs.yml | 8 ++- roles/glusterfs/tasks/main.yml | 20 ++++++-- 8 files changed, 41 insertions(+), 125 deletions(-) delete mode 100644 dockerfiles/graylog/docker-compose-graylog.yml delete mode 100644 dockerfiles/graylog/docker-compose-test.yml rename dockerfiles/{graylog/docker-compose-elasticsearch.yml => services/elasticsearch/docker-compose.yml} (76%) rename dockerfiles/{graylog/docker-compose-mongodb.yml => services/mongo/docker-compose.yml} (96%) diff --git a/dkhost.xai-corp.net.yml b/dkhost.xai-corp.net.yml index 47bc197..3afee43 100644 --- a/dkhost.xai-corp.net.yml +++ b/dkhost.xai-corp.net.yml @@ -56,10 +56,11 @@ jenkins: host: gluster:/jenkins mount: /var/lib/jenkins + elasticsearch: + host: gluster:/elasticsearch + mount: /data/elasticsearch roles: -# - _install_updates -# - Datadog.datadog - dockerhost - geerlingguy.nginx - certbot diff --git a/dockerfiles/graylog/docker-compose-graylog.yml b/dockerfiles/graylog/docker-compose-graylog.yml deleted file mode 100644 index 876bc6f..0000000 --- a/dockerfiles/graylog/docker-compose-graylog.yml +++ /dev/null @@ -1,40 +0,0 @@ ---- -# docker-compose file for graylog -# DOCKER_HOST=dkhost03:2376 docker-compose up -d -# DOCKER_HOST=dkhost03:2376 docker stack deploy -c docker-compose.yml graylog - -version: '3' -services: - - graylog: - image: graylog2/server:latest - volumes: - - /opt/shared/graylog/data/journal:/usr/share/graylog/data/journal - - /opt/shared/graylog/config:/usr/share/graylog/data/config - environment: - GRAYLOG_PASSWORD_SECRET: gsahu1dj901hdaiuafg3g1q - GRAYLOG_ROOT_PASSWORD_SHA2: d0fc133359968fbc38a267f29606c9dc805af7bcc231df9b9acd2e8e6e894ede - GRAYLOG_WEB_ENDPOINT_URI: http://logs.xai-corp.net:10090/api/ - GRAYLOG_REST_LISTEN_URI: http://0.0.0.0:9000/api/ - GRAYLOG_MONGODB_URI: mongodb://mongo/graylog - GRAYLOG_ELASTICSEARCH_CLUSTER_NAME: es.xai-corp.net - depends_on: - - mongo - - elasticsearch - ports: - - "10090:9000" - - "12201:12201/udp" - - "1514:1514/udp" - - "514:514/udp - deploy: - mode: replicated - replicas: 1 - restart_policy: - condition: any - delay: "1s" - max_attempts: 3 - -networks: - default: - external: - name: prod-private diff --git a/dockerfiles/graylog/docker-compose-test.yml b/dockerfiles/graylog/docker-compose-test.yml deleted file mode 100644 index 5c9499f..0000000 --- a/dockerfiles/graylog/docker-compose-test.yml +++ /dev/null @@ -1,26 +0,0 @@ ---- -# docker-compose file for graylog -# DOCKER_HOST=dkhost03:2376 docker-compose up -d -# DOCKER_HOST=dkhost03:2376 docker stack deploy -c docker-compose.yml graylog - -version: '3' -services: - - test: - image: alpine - command: ping nosql - depends_on: - - mongo - - elasticsearch - deploy: - mode: replicated - replicas: 1 - restart_policy: - condition: any - delay: "1s" - max_attempts: 3 - -networks: - default: - external: - name: prod-private diff --git a/dockerfiles/graylog/docker-compose.yml b/dockerfiles/graylog/docker-compose.yml index fabc665..0279cdc 100644 --- a/dockerfiles/graylog/docker-compose.yml +++ b/dockerfiles/graylog/docker-compose.yml @@ -6,46 +6,6 @@ version: '3' services: - mongo: - image: "mongo:3" - volumes: - - /opt/shared/graylog/data/mongo:/data/db - ports: - - "27017:27017" - deploy: - mode: replicated - replicas: 1 - restart_policy: - condition: any - delay: "1s" - max_attempts: 10 - resources: - limits: - cpus: '0.1' - memory: 512M - - elasticsearch: - image: "elasticsearch:2" - command: "elasticsearch -Des.cluster.name='graylog'" - volumes: - - /opt/shared/graylog/data/elasticsearch:/usr/share/elasticsearch/data - ports: - - "9350:9350" - - "9300:9300" - - "9200:9200" - - "10091:80" - deploy: - mode: replicated - replicas: 1 - restart_policy: - condition: any - delay: "1s" - max_attempts: 10 - resources: - limits: - cpus: '0.1' - memory: 512M - graylog: image: graylog2/server:latest volumes: @@ -54,9 +14,10 @@ services: environment: GRAYLOG_PASSWORD_SECRET: gsahu1dj901hdaiuafg3g1q GRAYLOG_ROOT_PASSWORD_SHA2: d0fc133359968fbc38a267f29606c9dc805af7bcc231df9b9acd2e8e6e894ede - GRAYLOG_WEB_ENDPOINT_URI: http://logs.xai-corp.net:10090/api/ + GRAYLOG_WEB_ENDPOINT_URI: https://logs.xai-corp.net/api/ GRAYLOG_REST_LISTEN_URI: http://0.0.0.0:9000/api/ GRAYLOG_MONGODB_URI: mongodb://mongo/graylog + GRAYLOG_ELASTICSEARCH_CLUSTER_NAME: es.xai-corp.net depends_on: - mongo - elasticsearch @@ -64,10 +25,16 @@ services: - "10090:9000" - "12201:12201/udp" - "1514:1514/udp" + - "514:514/udp" deploy: mode: replicated replicas: 1 restart_policy: condition: any delay: "1s" - max_attempts: 10 + max_attempts: 3 + +networks: + default: + external: + name: prod-private diff --git a/dockerfiles/graylog/docker-compose-elasticsearch.yml b/dockerfiles/services/elasticsearch/docker-compose.yml similarity index 76% rename from dockerfiles/graylog/docker-compose-elasticsearch.yml rename to dockerfiles/services/elasticsearch/docker-compose.yml index ed350e0..c0b2734 100644 --- a/dockerfiles/graylog/docker-compose-elasticsearch.yml +++ b/dockerfiles/services/elasticsearch/docker-compose.yml @@ -1,7 +1,7 @@ --- # docker-compose file for graylog # DOCKER_HOST=dkhost03:2376 docker-compose up -d -# DOCKER_HOST=dkhost03:2376 docker stack deploy -c docker-compose-elasticsearch.yml services +# DOCKER_HOST=dkhost03:2376 docker stack deploy -c docker-compose.yml services version: '3' services: @@ -11,8 +11,8 @@ services: # image: docker.elastic.co/elasticsearch/elasticsearch:5.4.1 command: "elasticsearch -Des.cluster.name='es.xai-corp.net'" volumes: - - /opt/shared/elasticsearch/data:/usr/share/elasticsearch/data - - /opt/shared/elasticsearch/confg:/usr/share/elasticsearch/config + - /data/elasticsearch/data:/usr/share/elasticsearch/data + - /data/elasticsearch/config:/usr/share/elasticsearch/config ports: # - "9350:9350" # - "9300:9300" @@ -24,23 +24,23 @@ services: restart_policy: condition: any delay: "1s" - max_attempts: 3 + max_attempts: 1 resources: limits: cpus: '1' memory: 512M ui: - image: appbaseio/dejavu + image: mobz/elasticsearch-head:2 ports: - - "1358:1358" + - "1358:9100" deploy: mode: replicated replicas: 1 restart_policy: condition: any delay: "1s" - max_attempts: 3 + max_attempts: 1 networks: diff --git a/dockerfiles/graylog/docker-compose-mongodb.yml b/dockerfiles/services/mongo/docker-compose.yml similarity index 96% rename from dockerfiles/graylog/docker-compose-mongodb.yml rename to dockerfiles/services/mongo/docker-compose.yml index 0531324..44e3b5a 100644 --- a/dockerfiles/graylog/docker-compose-mongodb.yml +++ b/dockerfiles/services/mongo/docker-compose.yml @@ -1,7 +1,7 @@ --- # docker-compose file for graylog # DOCKER_HOST=dkhost03:2376 docker-compose up -d -# DOCKER_HOST=dkhost03:2376 docker stack deploy -c docker-compose-mongodb.yml services +# DOCKER_HOST=dkhost03:2376 docker stack deploy -c docker-compose.yml services version: '3' services: diff --git a/glusterfs.yml b/glusterfs.yml index cf9800d..3682550 100644 --- a/glusterfs.yml +++ b/glusterfs.yml @@ -16,6 +16,7 @@ - /data/glusterfs/gitea/brick1 - /data/glusterfs/jenkins/brick1 - /data/glusterfs/tmp/brick1 + - /data/glusterfs/elasticsearch/brick1 volumes: vmshares: - /data/glusterfs/vmshares/brick1/brick @@ -23,8 +24,11 @@ - /data/glusterfs/gitea/brick1/brick jenkins: - /data/glusterfs/jenkins/brick1/brick -# tmp: -# - /data/glusterfs/tmp/brick1/brick + distributed_volumes: + tmp: + - /data/glusterfs/tmp/brick1/brick + elasticsearch: + - /data/glusterfs/elasticsearch/brick1/brick roles: - glusterfs diff --git a/roles/glusterfs/tasks/main.yml b/roles/glusterfs/tasks/main.yml index d6b93a2..ec0dae4 100644 --- a/roles/glusterfs/tasks/main.yml +++ b/roles/glusterfs/tasks/main.yml @@ -34,13 +34,23 @@ with_dict: "{{volumes}}" run_once: true -# manage volumes -- name: create tmp gluster volumes +#- name: create tmp gluster volumes +# gluster_volume: +# state: present +# name: "tmp" +# brick: '/data/glusterfs/tmp/brick1/brick' +# rebalance: no +# replicas: 1 +# cluster: "{{cluster}}" +# run_once: true + +- name: create distributed gluster volumes gluster_volume: state: present - name: "tmp" - brick: '/data/glusterfs/tmp/brick1/brick' + name: "{{ item.key }}" + brick: '{{ item.value | join(",")}}' rebalance: no - replicas: 2 + replicas: false cluster: "{{cluster}}" + with_dict: "{{distributed_volumes}}" run_once: true