update launch app and launch services scripts to use timouts

dockerfiles/services/launch_apps.sh

@@ -1,20 +1,29 @@
 #!/bin/bash -ex
+trap "echo Booh!" SIGINT SIGTERM
 
 export DOCKER_HOST=dkmanager:2376
 
+scale_out() {
+    REPLICA_COUNT=`docker service inspect $1 | jq '.[0].Spec.Mode.Replicated.Replicas'`
+    ./timeout.sh -t $2 docker service scale $1=$(($REPLICA_COUNT*2))
+    ./timeout.sh -t $2 docker service scale $1=$(($REPLICA_COUNT))
+    docker service update -q $1
+}
+
+
 docker login -u richard -p $DKREGISTRY_PASS $DKREGISTRY
 
 docker stack deploy --with-registry-auth --prune -c gitea/docker-compose.yml gitea
-docker service update -q gitea_app
+scale_out gitea_app 180
 
 docker stack deploy --with-registry-auth --prune -c nextcloud/docker-compose-prod.yml xaibox
-docker service update -q xaibox_app
+scale_out xaibox_app 180
 
 docker stack deploy --with-registry-auth --prune -c ui/docker-compose.yml dkui
-docker service update -q dkui_app
+scale_out dkui_app 180
 
 docker stack deploy --with-registry-auth --prune -c jenkins/docker-compose.yml jenkins
-docker service update -q jenkins_app
+scale_out jenkins_app 180
 
 #docker stack deploy --with-registry-auth --prune -c letsencrypt/docker-compose-update.yml letsencrypt
 
@@ -23,7 +32,5 @@ docker stack deploy --with-registry-auth --prune -c prometheus/docker-compose-pr
 sleep 5
 
 docker stack deploy --with-registry-auth --prune -c sslproxy/docker-compose-prod.yml sslproxy
-REPLICA_COUNT=`docker service inspect sslproxy_app | jq '.[0].Spec.Mode.Replicated.Replicas'`
+scale_out sslproxy_app 30
-./timeout.sh -t 120 docker service scale sslproxy_app=$(($REPLICA_COUNT*2))
-docker service update -q sslproxy_app
 

dockerfiles/services/services/fluentd/Dockerfile

@@ -1,15 +1,21 @@
-FROM fluent/fluentd:v0.14-onbuild
+FROM fluent/fluentd:v1.3-debian-onbuild-1
 MAINTAINER Richard Morgan <r_morgan@sympatico.ca>
 
-RUN apk add --update --virtual .build-deps \
+# Use root account to use apt
-        sudo build-base ruby-dev \
+USER root
- # cutomize following instruction as you wish
+
+RUN buildDeps="sudo make gcc g++ libc-dev ruby-dev" \
+ && apt-get update \
+ && apt-get install -y --no-install-recommends $buildDeps \
  && sudo gem install \
         fluent-plugin-secure-forward \
         fluent-plugin-loggly \
  && sudo gem sources --clear-all \
- && apk del .build-deps \
+ && SUDO_FORCE_REMOVE=yes \
- && rm -rf /var/cache/apk/* \
+    apt-get purge -y --auto-remove \
+                  -o APT::AutoRemove::RecommendsImportant=false \
+                  $buildDeps \
+ && rm -rf /var/lib/apt/lists/* \
            /home/fluent/.gem/ruby/2.3.0/cache/*.gem
 
-EXPOSE 24224 24220
+USER fluent

dockerfiles/services/services/launch_services.sh

@@ -1,4 +1,5 @@
 #!/bin/bash -ex
+#trap "echo Booh!" SIGINT
 
 export DOCKER_HOST=dkmanager:2376
 
@@ -6,6 +7,13 @@ CONFIG=stack.tmp.yml
 SERVICE=services
 #NETWORK=prod
 
+scale_out() {
+    REPLICA_COUNT=`docker service inspect $1 | jq '.[0].Spec.Mode.Replicated.Replicas'`
+    ./timeout.sh -t $2 docker service scale $1=$(($REPLICA_COUNT*2))
+    ./timeout.sh -t $2 docker service scale $1=$(($REPLICA_COUNT))
+    docker service update -q $1
+}
+
 docker stack deploy -c dkregistry/docker-compose.yml $SERVICE
 docker service update -q services_registry
 docker login -u richard -p $DKREGISTRY_PASS $DKREGISTRY
@@ -29,12 +37,13 @@ docker stack deploy --with-registry-auth -c $CONFIG $SERVICE
 # Cleanup
 rm $CONFIG
 
-docker service update -q services_registry
+scale_out services_registry 60
-docker service update -q services_postgres
+scale_out services_postgres 60
-docker service update -q services_mysql
+scale_out services_mysql 60
-docker service update -q services_memcached
+scale_out services_memcached 60
-docker service update -q services_redis
+scale_out services_redis 60
-docker service update -q services_fluentd
+scale_out services_fluentd 120
-docker service update -q services_datadog
+#scale_out services_datadog 60
-docker service update -q services_cron
+scale_out services_cron 60
-docker service update -q services_letsencrypt_updates
+scale_out services_letsencrypt_updates 60
+

dockerfiles/services/services/timeout.sh

@@ -0,0 +1,91 @@
+#!/bin/bash
+#
+# The Bash shell script executes a command with a time-out.
+# Upon time-out expiration SIGTERM (15) is sent to the process. If the signal
+# is blocked, then the subsequent SIGKILL (9) terminates it.
+#
+# Based on the Bash documentation example.
+
+# Hello Chet,
+# please find attached a "little easier"  :-)  to comprehend
+# time-out example.  If you find it suitable, feel free to include
+# anywhere: the very same logic as in the original examples/scripts, a
+# little more transparent implementation to my taste.
+#
+# Dmitry V Golovashkin <Dmitry.Golovashkin@sas.com>
+
+scriptName="${0##*/}"
+
+declare -i DEFAULT_TIMEOUT=9
+declare -i DEFAULT_INTERVAL=1
+declare -i DEFAULT_DELAY=1
+
+# Timeout.
+declare -i timeout=DEFAULT_TIMEOUT
+# Interval between checks if the process is still alive.
+declare -i interval=DEFAULT_INTERVAL
+# Delay between posting the SIGTERM signal and destroying the process by SIGKILL.
+declare -i delay=DEFAULT_DELAY
+
+function printUsage() {
+    cat <<EOF
+
+Synopsis
+    $scriptName [-t timeout] [-i interval] [-d delay] command
+    Execute a command with a time-out.
+    Upon time-out expiration SIGTERM (15) is sent to the process. If SIGTERM
+    signal is blocked, then the subsequent SIGKILL (9) terminates it.
+
+    -t timeout
+        Number of seconds to wait for command completion.
+        Default value: $DEFAULT_TIMEOUT seconds.
+
+    -i interval
+        Interval between checks if the process is still alive.
+        Positive integer, default value: $DEFAULT_INTERVAL seconds.
+
+    -d delay
+        Delay between posting the SIGTERM signal and destroying the
+        process by SIGKILL. Default value: $DEFAULT_DELAY seconds.
+
+As of today, Bash does not support floating point arithmetic (sleep does),
+therefore all delay/time values must be integers.
+EOF
+}
+
+# Options.
+while getopts ":t:i:d:" option; do
+    case "$option" in
+        t) timeout=$OPTARG ;;
+        i) interval=$OPTARG ;;
+        d) delay=$OPTARG ;;
+        *) printUsage; exit 1 ;;
+    esac
+done
+shift $((OPTIND - 1))
+
+# $# should be at least 1 (the command to execute), however it may be strictly
+# greater than 1 if the command itself has options.
+if (($# == 0 || interval <= 0)); then
+    printUsage
+    exit 1
+fi
+
+# kill -0 pid   Exit code indicates if a signal may be sent to $pid process.
+(
+    ((t = timeout))
+
+    while ((t > 0)); do
+        sleep $interval
+        kill -0 $$ || exit 0
+        ((t -= interval))
+    done
+
+    # Be nice, post SIGTERM first.
+    # The 'exit 0' below will be executed if any preceeding command fails.
+    kill -s SIGINT $$ && kill -0 $$ || exit 0
+    sleep $delay
+    kill -s SIGKILL $$
+) 2> /dev/null &
+
+exec "$@"

dockerfiles/services/timeout.sh

@@ -83,7 +83,7 @@ fi
 
     # Be nice, post SIGTERM first.
     # The 'exit 0' below will be executed if any preceeding command fails.
-    kill -s SIGTERM $$ && kill -0 $$ || exit 0
+    kill -s SIGINT $$ && kill -0 $$ || exit 0
     sleep $delay
     kill -s SIGKILL $$
 ) 2> /dev/null &