From c597fdf80d1f9d0564bb6cf19e1e4c4696b50362 Mon Sep 17 00:00:00 2001 From: Richard Morgan Date: Mon, 22 Oct 2018 00:18:56 -0400 Subject: [PATCH] - ansible docker images - ansible play for cleaning hosts (apt autoremove) - mondrian(incomplete) - lock postgres to version 10 - update jenkins dockerfile --- .../services/jenkins/docker-compose.yml | 2 + .../services/jenkins/startHostProxy.sh | 3 + dockerfiles/services/launch_apps.sh | 2 + .../services/dkregistry/docker-compose.yml | 27 +++++++ .../services/services/launch_services.sh | 3 +- .../services/postgres/docker-compose.yml | 2 +- .../abcapi.xai-corp.net.conf | 0 dockerfiles/tasks/ansible/Dockerfile-2.0 | 73 +++++++++++++++++++ dockerfiles/tasks/ansible/Dockerfile-2.4 | 70 ++++++++++++++++++ dockerfiles/tasks/ansible/Dockerfile-2.5 | 70 ++++++++++++++++++ dockerfiles/tasks/ansible/Dockerfile-2.7 | 70 ++++++++++++++++++ dockerfiles/tasks/ansible/build.sh | 30 ++++++++ dockerfiles/tasks/mondrian/Dockerfile | 10 +++ dockerfiles/tasks/mondrian/build.sh | 7 ++ managed_clean.yml | 18 +++++ 15 files changed, 385 insertions(+), 2 deletions(-) create mode 100755 dockerfiles/services/jenkins/startHostProxy.sh rename dockerfiles/services/sslproxy/{hosts => hosts-disabled}/abcapi.xai-corp.net.conf (100%) create mode 100644 dockerfiles/tasks/ansible/Dockerfile-2.0 create mode 100644 dockerfiles/tasks/ansible/Dockerfile-2.4 create mode 100644 dockerfiles/tasks/ansible/Dockerfile-2.5 create mode 100644 dockerfiles/tasks/ansible/Dockerfile-2.7 create mode 100755 dockerfiles/tasks/ansible/build.sh create mode 100644 dockerfiles/tasks/mondrian/Dockerfile create mode 100644 dockerfiles/tasks/mondrian/build.sh create mode 100644 managed_clean.yml diff --git a/dockerfiles/services/jenkins/docker-compose.yml b/dockerfiles/services/jenkins/docker-compose.yml index 1f2a9f5..b2f51b9 100644 --- a/dockerfiles/services/jenkins/docker-compose.yml +++ b/dockerfiles/services/jenkins/docker-compose.yml @@ -17,6 +17,8 @@ services: ports: - "8080:8080" - "50000:50000" + environment: + - "JAVA_OPTS=-Dhudson.model.DirectoryBrowserSupport.CSP=" deploy: mode: replicated replicas: 1 diff --git a/dockerfiles/services/jenkins/startHostProxy.sh b/dockerfiles/services/jenkins/startHostProxy.sh new file mode 100755 index 0000000..cb41b3c --- /dev/null +++ b/dockerfiles/services/jenkins/startHostProxy.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +DOCKER_HOST=dkhost:2376 docker run -d --rm --name 'dockerhost' --cap-add=NET_ADMIN --cap-add=NET_RAW qoomon/docker-host diff --git a/dockerfiles/services/launch_apps.sh b/dockerfiles/services/launch_apps.sh index bb7a6f3..e934ece 100644 --- a/dockerfiles/services/launch_apps.sh +++ b/dockerfiles/services/launch_apps.sh @@ -10,6 +10,8 @@ docker stack deploy --with-registry-auth --prune -c nextcloud/docker-compose-pro docker stack deploy --with-registry-auth --prune -c ui/docker-compose.yml dkui +docker stack deploy --with-registry-auth --prune -c jenkins/docker-compose.yml jenkins + #docker stack deploy --with-registry-auth --prune -c letsencrypt/docker-compose-update.yml letsencrypt docker stack deploy --with-registry-auth --prune -c prometheus/docker-compose-prod.yml metrics diff --git a/dockerfiles/services/services/dkregistry/docker-compose.yml b/dockerfiles/services/services/dkregistry/docker-compose.yml index ea9a34c..b60728c 100644 --- a/dockerfiles/services/services/dkregistry/docker-compose.yml +++ b/dockerfiles/services/services/dkregistry/docker-compose.yml @@ -51,3 +51,30 @@ services: fluentd-address: "logs.xai-corp.net:24224" fluentd-async-connect: 'true' tag: dkregistry + +# registry-ui: +# image: hyper/docker-registry-web +# ports: +# - 8087:8080 +# environment: +# REGISTRY_URL: https://dkregistry.xai-corp.net:5000/v2 +# REGISTRY_NAME: dkregistry +# REGISTRY_READONLY: 'false' +# REGISTRY_AUTH_ENABLED: 'true' +# REGISTRY_BASIC_AUTH: $apr1$2vrW.sPv$aIZ6xnQcvde6.kX7KvWm5/ +# +# deploy: +# mode: replicated +# replicas: 1 +# restart_policy: +# condition: any +# delay: "1s" +# max_attempts: 1 +# update_config: +# parallelism: 1 +# delay: 2s +# order: start-first +# resources: +# limits: +# cpus: '2' +# memory: 2048M diff --git a/dockerfiles/services/services/launch_services.sh b/dockerfiles/services/services/launch_services.sh index 4c254b4..43607ac 100755 --- a/dockerfiles/services/services/launch_services.sh +++ b/dockerfiles/services/services/launch_services.sh @@ -15,16 +15,17 @@ docker login -u richard -p $DKREGISTRY_PASS $DKREGISTRY docker-compose \ -f network.yml \ -f postgres/docker-compose.yml \ - -f dkregistry/docker-compose.yml \ -f mariadb/docker-compose.yml \ -f fluentd/docker-compose-prod.yml \ -f memcached/docker-compose.yml \ -f redis/docker-compose.yml \ -f datadog/docker-compose.yml \ + -f dkregistry/docker-compose.yml \ -f cron/docker-compose.yml \ config > $CONFIG + docker stack deploy --prune --with-registry-auth -c $CONFIG $SERVICE # Cleanup diff --git a/dockerfiles/services/services/postgres/docker-compose.yml b/dockerfiles/services/services/postgres/docker-compose.yml index 505291a..46b7a30 100644 --- a/dockerfiles/services/services/postgres/docker-compose.yml +++ b/dockerfiles/services/services/postgres/docker-compose.yml @@ -6,7 +6,7 @@ services: postgres: volumes: - /opt/shared/postgres/data:/data - image: postgres:alpine + image: postgres:10-alpine environment: POSTGRES_PASSWORD: snqioxni1sw POSTGRES_USER: xaicorp_admin diff --git a/dockerfiles/services/sslproxy/hosts/abcapi.xai-corp.net.conf b/dockerfiles/services/sslproxy/hosts-disabled/abcapi.xai-corp.net.conf similarity index 100% rename from dockerfiles/services/sslproxy/hosts/abcapi.xai-corp.net.conf rename to dockerfiles/services/sslproxy/hosts-disabled/abcapi.xai-corp.net.conf diff --git a/dockerfiles/tasks/ansible/Dockerfile-2.0 b/dockerfiles/tasks/ansible/Dockerfile-2.0 new file mode 100644 index 0000000..97be1ac --- /dev/null +++ b/dockerfiles/tasks/ansible/Dockerfile-2.0 @@ -0,0 +1,73 @@ +#http://ruleoftech.com/2017/dockerizing-all-the-things-running-ansible-inside-docker-container +FROM alpine:3.7 + +ENV ANSIBLE_VERSION 2.0.0 + +ENV BUILD_PACKAGES \ + bash \ + curl \ + tar \ + openssh-client \ + sshpass \ + git \ + python \ + py-boto \ + py-dateutil \ + py-httplib2 \ + py-jinja2 \ + py-paramiko \ + py-pip \ + py-yaml \ + ca-certificates + +# If installing ansible@testing +#RUN \ +# echo "@testing http://nl.alpinelinux.org/alpine/edge/testing" >> #/etc/apk/repositories + +RUN set -x && \ + \ + echo "==> Adding build-dependencies..." && \ + apk --update add --virtual build-dependencies \ + gcc \ + musl-dev \ + libffi-dev \ + openssl-dev \ + python-dev && \ + \ + echo "==> Upgrading apk and system..." && \ + apk update && apk upgrade && \ + \ + echo "==> Adding Python runtime..." && \ + apk add --no-cache ${BUILD_PACKAGES} && \ + pip install --upgrade pip && \ + pip install python-keyczar docker-py && \ + \ + echo "==> Installing Ansible..." && \ + pip install ansible==${ANSIBLE_VERSION} && \ + \ + echo "==> Cleaning up..." && \ + apk del build-dependencies && \ + rm -rf /var/cache/apk/* && \ + \ + echo "==> Adding hosts for convenience..." && \ + mkdir -p /etc/ansible /ansible && \ + echo "[local]" >> /etc/ansible/hosts && \ + echo "localhost" >> /etc/ansible/hosts + +ENV ANSIBLE_GATHERING smart +ENV ANSIBLE_HOST_KEY_CHECKING false +ENV ANSIBLE_RETRY_FILES_ENABLED false +ENV ANSIBLE_ROLES_PATH /ansible/playbooks/roles +ENV ANSIBLE_SSH_PIPELINING True +ENV PYTHONPATH /ansible/lib +ENV PATH /ansible/bin:$PATH +ENV ANSIBLE_LIBRARY /ansible/library + +WORKDIR /ansible/playbooks + +ENTRYPOINT ["ansible-playbook"] + +RUN addgroup ansible \ + && adduser -h /home/ansible -u 1000 -S -G ansible ansible + +USER ansible diff --git a/dockerfiles/tasks/ansible/Dockerfile-2.4 b/dockerfiles/tasks/ansible/Dockerfile-2.4 new file mode 100644 index 0000000..c63381b --- /dev/null +++ b/dockerfiles/tasks/ansible/Dockerfile-2.4 @@ -0,0 +1,70 @@ +#http://ruleoftech.com/2017/dockerizing-all-the-things-running-ansible-inside-docker-container +FROM alpine:3.7 + +ENV ANSIBLE_VERSION 2.4.0 + +ENV BUILD_PACKAGES \ + bash \ + curl \ + tar \ + openssh-client \ + sshpass \ + git \ + python \ + py-boto \ + py-dateutil \ + py-httplib2 \ + py-jinja2 \ + py-paramiko \ + py-pip \ + py-yaml \ + ca-certificates + +# If installing ansible@testing +#RUN \ +# echo "@testing http://nl.alpinelinux.org/alpine/edge/testing" >> #/etc/apk/repositories + +RUN set -x && \ + \ + echo "==> Adding build-dependencies..." && \ + apk --update add --virtual build-dependencies \ + gcc \ + musl-dev \ + libffi-dev \ + openssl-dev \ + python-dev && \ + \ + echo "==> Upgrading apk and system..." && \ + apk update && apk upgrade && \ + \ + echo "==> Adding Python runtime..." && \ + apk add --no-cache ${BUILD_PACKAGES} && \ + pip install --upgrade pip && \ + pip install python-keyczar docker-py && \ + \ + echo "==> Installing Ansible..." && \ + pip install ansible==${ANSIBLE_VERSION} && \ + \ + echo "==> Cleaning up..." && \ + apk del build-dependencies && \ + rm -rf /var/cache/apk/* && \ + \ + echo "==> Adding hosts for convenience..." && \ + mkdir -p /etc/ansible /ansible && \ + echo "[local]" >> /etc/ansible/hosts && \ + echo "localhost" >> /etc/ansible/hosts + +ENV ANSIBLE_GATHERING smart +ENV ANSIBLE_HOST_KEY_CHECKING false +ENV ANSIBLE_RETRY_FILES_ENABLED false +ENV ANSIBLE_ROLES_PATH /ansible/playbooks/roles +ENV ANSIBLE_SSH_PIPELINING True +ENV PYTHONPATH /ansible/lib +ENV PATH /ansible/bin:$PATH +ENV ANSIBLE_LIBRARY /ansible/library + +WORKDIR /ansible/playbooks + +ENTRYPOINT ["ansible-playbook"] + +RUN mkdir -p /.ansible && chmod 777 /.ansible diff --git a/dockerfiles/tasks/ansible/Dockerfile-2.5 b/dockerfiles/tasks/ansible/Dockerfile-2.5 new file mode 100644 index 0000000..7836814 --- /dev/null +++ b/dockerfiles/tasks/ansible/Dockerfile-2.5 @@ -0,0 +1,70 @@ +#http://ruleoftech.com/2017/dockerizing-all-the-things-running-ansible-inside-docker-container +FROM alpine:3.7 + +ENV ANSIBLE_VERSION 2.5.0 + +ENV BUILD_PACKAGES \ + bash \ + curl \ + tar \ + openssh-client \ + sshpass \ + git \ + python \ + py-boto \ + py-dateutil \ + py-httplib2 \ + py-jinja2 \ + py-paramiko \ + py-pip \ + py-yaml \ + ca-certificates + +# If installing ansible@testing +#RUN \ +# echo "@testing http://nl.alpinelinux.org/alpine/edge/testing" >> #/etc/apk/repositories + +RUN set -x && \ + \ + echo "==> Adding build-dependencies..." && \ + apk --update add --virtual build-dependencies \ + gcc \ + musl-dev \ + libffi-dev \ + openssl-dev \ + python-dev && \ + \ + echo "==> Upgrading apk and system..." && \ + apk update && apk upgrade && \ + \ + echo "==> Adding Python runtime..." && \ + apk add --no-cache ${BUILD_PACKAGES} && \ + pip install --upgrade pip && \ + pip install python-keyczar docker-py && \ + \ + echo "==> Installing Ansible..." && \ + pip install ansible==${ANSIBLE_VERSION} && \ + \ + echo "==> Cleaning up..." && \ + apk del build-dependencies && \ + rm -rf /var/cache/apk/* && \ + \ + echo "==> Adding hosts for convenience..." && \ + mkdir -p /etc/ansible /ansible && \ + echo "[local]" >> /etc/ansible/hosts && \ + echo "localhost" >> /etc/ansible/hosts + +ENV ANSIBLE_GATHERING smart +ENV ANSIBLE_HOST_KEY_CHECKING false +ENV ANSIBLE_RETRY_FILES_ENABLED false +ENV ANSIBLE_ROLES_PATH /ansible/playbooks/roles +ENV ANSIBLE_SSH_PIPELINING True +ENV PYTHONPATH /ansible/lib +ENV PATH /ansible/bin:$PATH +ENV ANSIBLE_LIBRARY /ansible/library + +WORKDIR /ansible/playbooks + +ENTRYPOINT ["ansible-playbook"] + +RUN mkdir -p /.ansible && chmod 777 /.ansible diff --git a/dockerfiles/tasks/ansible/Dockerfile-2.7 b/dockerfiles/tasks/ansible/Dockerfile-2.7 new file mode 100644 index 0000000..4bcb009 --- /dev/null +++ b/dockerfiles/tasks/ansible/Dockerfile-2.7 @@ -0,0 +1,70 @@ +#http://ruleoftech.com/2017/dockerizing-all-the-things-running-ansible-inside-docker-container +FROM alpine:3.7 + +ENV ANSIBLE_VERSION 2.7.0 + +ENV BUILD_PACKAGES \ + bash \ + curl \ + tar \ + openssh-client \ + sshpass \ + git \ + python \ + py-boto \ + py-dateutil \ + py-httplib2 \ + py-jinja2 \ + py-paramiko \ + py-pip \ + py-yaml \ + ca-certificates + +# If installing ansible@testing +#RUN \ +# echo "@testing http://nl.alpinelinux.org/alpine/edge/testing" >> #/etc/apk/repositories + +RUN set -x && \ + \ + echo "==> Adding build-dependencies..." && \ + apk --update add --virtual build-dependencies \ + gcc \ + musl-dev \ + libffi-dev \ + openssl-dev \ + python-dev && \ + \ + echo "==> Upgrading apk and system..." && \ + apk update && apk upgrade && \ + \ + echo "==> Adding Python runtime..." && \ + apk add --no-cache ${BUILD_PACKAGES} && \ + pip install --upgrade pip && \ + pip install python-keyczar docker-py && \ + \ + echo "==> Installing Ansible..." && \ + pip install ansible==${ANSIBLE_VERSION} && \ + \ + echo "==> Cleaning up..." && \ + apk del build-dependencies && \ + rm -rf /var/cache/apk/* && \ + \ + echo "==> Adding hosts for convenience..." && \ + mkdir -p /etc/ansible /ansible && \ + echo "[local]" >> /etc/ansible/hosts && \ + echo "localhost" >> /etc/ansible/hosts + +ENV ANSIBLE_GATHERING smart +ENV ANSIBLE_HOST_KEY_CHECKING false +ENV ANSIBLE_RETRY_FILES_ENABLED false +ENV ANSIBLE_ROLES_PATH /ansible/playbooks/roles +ENV ANSIBLE_SSH_PIPELINING True +ENV PYTHONPATH /ansible/lib +ENV PATH /ansible/bin:$PATH +ENV ANSIBLE_LIBRARY /ansible/library + +WORKDIR /ansible/playbooks + +ENTRYPOINT ["ansible-playbook"] + +RUN mkdir -p /.ansible && chmod 777 /.ansible diff --git a/dockerfiles/tasks/ansible/build.sh b/dockerfiles/tasks/ansible/build.sh new file mode 100755 index 0000000..2c7cf4b --- /dev/null +++ b/dockerfiles/tasks/ansible/build.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -e + +if [[ -z "${DKREGISTRY_PASS}" ]]; then + docker login -u richard -p $DKREGISTRY_PASS $DKREGISTRY +fi + +docker build --rm -f Dockerfile-2.0 -t xaicorp/ansible:2.0 . +docker run --rm --entrypoint ansible xaicorp/ansible:2.0 --version | grep 'ansible 2.0.' +docker tag xaicorp/ansible:2.0 dkregistry.xai-corp.net:5000/xaicorp/ansible:2.0 +docker push dkregistry.xai-corp.net:5000/xaicorp/ansible:2.0 + +docker build --rm -f Dockerfile-2.4 -t xaicorp/ansible:2.4 . +docker run --rm --entrypoint ansible xaicorp/ansible:2.4 --version | grep 'ansible 2.4.' +docker tag xaicorp/ansible:2.4 dkregistry.xai-corp.net:5000/xaicorp/ansible:2.4 +docker push dkregistry.xai-corp.net:5000/xaicorp/ansible:2.4 + +docker build --rm -f Dockerfile-2.5 -t xaicorp/ansible:2.5 . +docker run --rm --entrypoint ansible xaicorp/ansible:2.5 --version | grep 'ansible 2.5.' +docker tag xaicorp/ansible:2.5 dkregistry.xai-corp.net:5000/xaicorp/ansible:2.5 +docker push dkregistry.xai-corp.net:5000/xaicorp/ansible:2.5 + +docker build --rm -f Dockerfile-2.7 -t xaicorp/ansible:2.7 . +docker run --rm --entrypoint ansible xaicorp/ansible:2.7 --version | grep 'ansible 2.7.' +docker tag xaicorp/ansible:2.7 dkregistry.xai-corp.net:5000/xaicorp/ansible:2.7 +docker push dkregistry.xai-corp.net:5000/xaicorp/ansible:2.7 + +docker tag xaicorp/ansible:2.5 xaicorp/ansible:latest +docker tag xaicorp/ansible:2.5 dkregistry.xai-corp.net:5000/xaicorp/ansible:latest +docker push dkregistry.xai-corp.net:5000/xaicorp/ansible:latest diff --git a/dockerfiles/tasks/mondrian/Dockerfile b/dockerfiles/tasks/mondrian/Dockerfile new file mode 100644 index 0000000..1b3f479 --- /dev/null +++ b/dockerfiles/tasks/mondrian/Dockerfile @@ -0,0 +1,10 @@ +# docker build --rm -f Dockerfile -t mondrian:latest . +FROM dkregistry.xai-corp.net:5000/xaicorp/php:7.2-dev +MAINTAINER Richard Morgan + + +RUN apt-get update && apt-get install -y wget \ + && wget https://github.com/Trismegiste/Mondrian/raw/master/bin/box/mondrian.phar \ + && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +CMD ["mondrian.phar"] diff --git a/dockerfiles/tasks/mondrian/build.sh b/dockerfiles/tasks/mondrian/build.sh new file mode 100644 index 0000000..a5c4906 --- /dev/null +++ b/dockerfiles/tasks/mondrian/build.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +docker login -u richard -p $DKREGISTRY_PASS $DKREGISTRY + +docker build --rm -f Dockerfile -t mondrian:latest . +docker run --rm mondrian:latest --version | grep 'Mondrian' +docker tag mondrian:latest dkregistry.xai-corp.net:5000/xaicorp/qa/mondrian:latest +docker push dkregistry.xai-corp.net:5000/xaicorp/qa/mondrian:latest diff --git a/managed_clean.yml b/managed_clean.yml new file mode 100644 index 0000000..0c27ce3 --- /dev/null +++ b/managed_clean.yml @@ -0,0 +1,18 @@ +# playbook for all managed hosts + +# ansible-playbook managed_updates.yml -v --ask-become -u richard --ask-pass + + +- hosts: managed + remote_user: ansible + gather_facts: yes + become: True + + vars: + + roles: + + tasks: + - name: run apt autoremove + command: apt -y autoremove +