From 72cf28aa56dd95a021647caa4bfffddd66ad9514 Mon Sep 17 00:00:00 2001 From: richard Date: Tue, 5 Feb 2019 22:15:51 -0500 Subject: [PATCH] update launch app and launch services scripts to use timouts --- dockerfiles/services/launch_apps.sh | 21 +++-- .../services/services/fluentd/Dockerfile | 20 ++-- .../services/services/launch_services.sh | 27 ++++-- dockerfiles/services/services/timeout.sh | 91 +++++++++++++++++++ dockerfiles/services/timeout.sh | 2 +- 5 files changed, 137 insertions(+), 24 deletions(-) create mode 100755 dockerfiles/services/services/timeout.sh diff --git a/dockerfiles/services/launch_apps.sh b/dockerfiles/services/launch_apps.sh index f76bfca..b2112b7 100644 --- a/dockerfiles/services/launch_apps.sh +++ b/dockerfiles/services/launch_apps.sh @@ -1,20 +1,29 @@ #!/bin/bash -ex +trap "echo Booh!" SIGINT SIGTERM export DOCKER_HOST=dkmanager:2376 +scale_out() { + REPLICA_COUNT=`docker service inspect $1 | jq '.[0].Spec.Mode.Replicated.Replicas'` + ./timeout.sh -t $2 docker service scale $1=$(($REPLICA_COUNT*2)) + ./timeout.sh -t $2 docker service scale $1=$(($REPLICA_COUNT)) + docker service update -q $1 +} + + docker login -u richard -p $DKREGISTRY_PASS $DKREGISTRY docker stack deploy --with-registry-auth --prune -c gitea/docker-compose.yml gitea -docker service update -q gitea_app +scale_out gitea_app 180 docker stack deploy --with-registry-auth --prune -c nextcloud/docker-compose-prod.yml xaibox -docker service update -q xaibox_app +scale_out xaibox_app 180 docker stack deploy --with-registry-auth --prune -c ui/docker-compose.yml dkui -docker service update -q dkui_app +scale_out dkui_app 180 docker stack deploy --with-registry-auth --prune -c jenkins/docker-compose.yml jenkins -docker service update -q jenkins_app +scale_out jenkins_app 180 #docker stack deploy --with-registry-auth --prune -c letsencrypt/docker-compose-update.yml letsencrypt @@ -23,7 +32,5 @@ docker stack deploy --with-registry-auth --prune -c prometheus/docker-compose-pr sleep 5 docker stack deploy --with-registry-auth --prune -c sslproxy/docker-compose-prod.yml sslproxy -REPLICA_COUNT=`docker service inspect sslproxy_app | jq '.[0].Spec.Mode.Replicated.Replicas'` -./timeout.sh -t 120 docker service scale sslproxy_app=$(($REPLICA_COUNT*2)) -docker service update -q sslproxy_app +scale_out sslproxy_app 30 diff --git a/dockerfiles/services/services/fluentd/Dockerfile b/dockerfiles/services/services/fluentd/Dockerfile index a626d7f..8557b81 100644 --- a/dockerfiles/services/services/fluentd/Dockerfile +++ b/dockerfiles/services/services/fluentd/Dockerfile @@ -1,15 +1,21 @@ -FROM fluent/fluentd:v0.14-onbuild +FROM fluent/fluentd:v1.3-debian-onbuild-1 MAINTAINER Richard Morgan -RUN apk add --update --virtual .build-deps \ - sudo build-base ruby-dev \ - # cutomize following instruction as you wish +# Use root account to use apt +USER root + +RUN buildDeps="sudo make gcc g++ libc-dev ruby-dev" \ + && apt-get update \ + && apt-get install -y --no-install-recommends $buildDeps \ && sudo gem install \ fluent-plugin-secure-forward \ fluent-plugin-loggly \ && sudo gem sources --clear-all \ - && apk del .build-deps \ - && rm -rf /var/cache/apk/* \ + && SUDO_FORCE_REMOVE=yes \ + apt-get purge -y --auto-remove \ + -o APT::AutoRemove::RecommendsImportant=false \ + $buildDeps \ + && rm -rf /var/lib/apt/lists/* \ /home/fluent/.gem/ruby/2.3.0/cache/*.gem -EXPOSE 24224 24220 +USER fluent diff --git a/dockerfiles/services/services/launch_services.sh b/dockerfiles/services/services/launch_services.sh index 4e84d87..b362ecd 100755 --- a/dockerfiles/services/services/launch_services.sh +++ b/dockerfiles/services/services/launch_services.sh @@ -1,4 +1,5 @@ #!/bin/bash -ex +#trap "echo Booh!" SIGINT export DOCKER_HOST=dkmanager:2376 @@ -6,6 +7,13 @@ CONFIG=stack.tmp.yml SERVICE=services #NETWORK=prod +scale_out() { + REPLICA_COUNT=`docker service inspect $1 | jq '.[0].Spec.Mode.Replicated.Replicas'` + ./timeout.sh -t $2 docker service scale $1=$(($REPLICA_COUNT*2)) + ./timeout.sh -t $2 docker service scale $1=$(($REPLICA_COUNT)) + docker service update -q $1 +} + docker stack deploy -c dkregistry/docker-compose.yml $SERVICE docker service update -q services_registry docker login -u richard -p $DKREGISTRY_PASS $DKREGISTRY @@ -29,12 +37,13 @@ docker stack deploy --with-registry-auth -c $CONFIG $SERVICE # Cleanup rm $CONFIG -docker service update -q services_registry -docker service update -q services_postgres -docker service update -q services_mysql -docker service update -q services_memcached -docker service update -q services_redis -docker service update -q services_fluentd -docker service update -q services_datadog -docker service update -q services_cron -docker service update -q services_letsencrypt_updates +scale_out services_registry 60 +scale_out services_postgres 60 +scale_out services_mysql 60 +scale_out services_memcached 60 +scale_out services_redis 60 +scale_out services_fluentd 120 +#scale_out services_datadog 60 +scale_out services_cron 60 +scale_out services_letsencrypt_updates 60 + diff --git a/dockerfiles/services/services/timeout.sh b/dockerfiles/services/services/timeout.sh new file mode 100755 index 0000000..a291109 --- /dev/null +++ b/dockerfiles/services/services/timeout.sh @@ -0,0 +1,91 @@ +#!/bin/bash +# +# The Bash shell script executes a command with a time-out. +# Upon time-out expiration SIGTERM (15) is sent to the process. If the signal +# is blocked, then the subsequent SIGKILL (9) terminates it. +# +# Based on the Bash documentation example. + +# Hello Chet, +# please find attached a "little easier" :-) to comprehend +# time-out example. If you find it suitable, feel free to include +# anywhere: the very same logic as in the original examples/scripts, a +# little more transparent implementation to my taste. +# +# Dmitry V Golovashkin + +scriptName="${0##*/}" + +declare -i DEFAULT_TIMEOUT=9 +declare -i DEFAULT_INTERVAL=1 +declare -i DEFAULT_DELAY=1 + +# Timeout. +declare -i timeout=DEFAULT_TIMEOUT +# Interval between checks if the process is still alive. +declare -i interval=DEFAULT_INTERVAL +# Delay between posting the SIGTERM signal and destroying the process by SIGKILL. +declare -i delay=DEFAULT_DELAY + +function printUsage() { + cat < 0)); do + sleep $interval + kill -0 $$ || exit 0 + ((t -= interval)) + done + + # Be nice, post SIGTERM first. + # The 'exit 0' below will be executed if any preceeding command fails. + kill -s SIGINT $$ && kill -0 $$ || exit 0 + sleep $delay + kill -s SIGKILL $$ +) 2> /dev/null & + +exec "$@" diff --git a/dockerfiles/services/timeout.sh b/dockerfiles/services/timeout.sh index 5c19d2e..a291109 100755 --- a/dockerfiles/services/timeout.sh +++ b/dockerfiles/services/timeout.sh @@ -83,7 +83,7 @@ fi # Be nice, post SIGTERM first. # The 'exit 0' below will be executed if any preceeding command fails. - kill -s SIGTERM $$ && kill -0 $$ || exit 0 + kill -s SIGINT $$ && kill -0 $$ || exit 0 sleep $delay kill -s SIGKILL $$ ) 2> /dev/null &