From 5b0746fcd634b8e84291ef00328781f7ef681df7 Mon Sep 17 00:00:00 2001 From: richard Date: Fri, 2 Apr 2021 07:31:03 -0400 Subject: [PATCH] prometheus xai cli tool setup WIP --- dockerfiles/services/prometheus/Dockerfile | 4 +- dockerfiles/services/prometheus/cli/.bash_cli | 0 .../services/prometheus/cli/_bootstrap.sh | 33 +++++ dockerfiles/services/prometheus/cli/build | 65 ++++++++ .../services/prometheus/cli/build.help | 5 + .../services/prometheus/cli/build.usage | 1 + dockerfiles/services/prometheus/cli/deploy | 140 ++++++++++++++++++ .../services/prometheus/cli/deploy.help | 3 + .../services/prometheus/cli/deploy.usage | 1 + dockerfiles/services/prometheus/cli/exec | 22 +++ dockerfiles/services/prometheus/cli/rollback | 89 +++++++++++ .../services/prometheus/cli/rollback.help | 3 + .../services/prometheus/cli/rollback.usage | 1 + .../prometheus/docker-compose.build.yml | 20 +++ .../services/prometheus/docker-compose.yml | 57 ++++--- 15 files changed, 413 insertions(+), 31 deletions(-) create mode 100644 dockerfiles/services/prometheus/cli/.bash_cli create mode 100644 dockerfiles/services/prometheus/cli/_bootstrap.sh create mode 100755 dockerfiles/services/prometheus/cli/build create mode 100644 dockerfiles/services/prometheus/cli/build.help create mode 100644 dockerfiles/services/prometheus/cli/build.usage create mode 100755 dockerfiles/services/prometheus/cli/deploy create mode 100644 dockerfiles/services/prometheus/cli/deploy.help create mode 100644 dockerfiles/services/prometheus/cli/deploy.usage create mode 100755 dockerfiles/services/prometheus/cli/exec create mode 100755 dockerfiles/services/prometheus/cli/rollback create mode 100644 dockerfiles/services/prometheus/cli/rollback.help create mode 100644 dockerfiles/services/prometheus/cli/rollback.usage create mode 100644 dockerfiles/services/prometheus/docker-compose.build.yml diff --git a/dockerfiles/services/prometheus/Dockerfile b/dockerfiles/services/prometheus/Dockerfile index c4f5e38..fad7a8e 100644 --- a/dockerfiles/services/prometheus/Dockerfile +++ b/dockerfiles/services/prometheus/Dockerfile @@ -1,2 +1,4 @@ -FROM prom/prometheus:v2.9.2 +ARG PROM_VERSION=v2.9.2 +FROM prom/prometheus:${PROM_VERSION} + ADD prometheus.yml /etc/prometheus/ diff --git a/dockerfiles/services/prometheus/cli/.bash_cli b/dockerfiles/services/prometheus/cli/.bash_cli new file mode 100644 index 0000000..e69de29 diff --git a/dockerfiles/services/prometheus/cli/_bootstrap.sh b/dockerfiles/services/prometheus/cli/_bootstrap.sh new file mode 100644 index 0000000..99236d2 --- /dev/null +++ b/dockerfiles/services/prometheus/cli/_bootstrap.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +# see jenkinsfile for version to build +PROM_VERSION=${PROM_VERSION:-v2.25.0} +LOCAL_IMAGE=xaicorp/prometheus +TAG=${PROM_VERSION}-${BUILD_NUMBER:-dev} +REMOTE_IMAGE=dkregistry.xai-corp.net:5000/${LOCAL_IMAGE}:${TAG} + +LOG=$(mktemp) + +export LOCAL_IMAGE +export REMOTE_IMAGE +export TAG + + + +function trap_exit() { + code=$? + if [ $code -gt 0 ]; then + echo + cat "$LOG" + rm "$LOG" + dc logs --tail=10 + dc down + echo -e "\033[31mFailed to build functional image\033[39m" + exit $code + fi + + dc down + rm "$LOG" + echo -e "\033[32mSuccess:\033[39m ${LOCAL_IMAGE}:${TAG} successfully built" +} +trap trap_exit EXIT diff --git a/dockerfiles/services/prometheus/cli/build b/dockerfiles/services/prometheus/cli/build new file mode 100755 index 0000000..4f66ef9 --- /dev/null +++ b/dockerfiles/services/prometheus/cli/build @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +set -e +set -x + +# include common parts +source ./cli/_bootstrap.sh + +dc() { + export PROM_VERSION=${PROM_VERSION} + # shellcheck disable=SC2068 + docker-compose \ + -f docker-compose.yml \ + -f docker-compose.build.yml \ + $@ +} + +#### +build() { + echo -e "\e[33m building the image\e[39m" + dc build +} + +build_test() { + echo -e "\e[33m testing the image\e[39m" + + dc run --rm --entrypoint prometheus prometheus --version | grep ${PROM_VERSION} + + + #test for smbclient installed +# dc run --rm --user www-data app which smbclient +} + + + +build_save() { + echo -e "\e[33m saving the image\e[39m" + + docker tag "$LOCAL_IMAGE:$TAG" "$REMOTE_IMAGE" + docker push "$REMOTE_IMAGE" +} + +print_usage() { + printf "Usage: %s: [-b] [-t] [-s] \n" "$0" + echo -b build + echo -t test + echo -s push to registry + echo -h help + exit 0 +} + +###### +if [ -z "$1" ]; then + build && build_test && build_save + exit +fi + +while getopts btdhs name +do + case $name in + b) build;; + t) build_test;; + s) build_save;; + *) print_usage;; + esac +done diff --git a/dockerfiles/services/prometheus/cli/build.help b/dockerfiles/services/prometheus/cli/build.help new file mode 100644 index 0000000..d274fdc --- /dev/null +++ b/dockerfiles/services/prometheus/cli/build.help @@ -0,0 +1,5 @@ +-b build image +-t test image +-s save image + +default: xai build -bts diff --git a/dockerfiles/services/prometheus/cli/build.usage b/dockerfiles/services/prometheus/cli/build.usage new file mode 100644 index 0000000..4f68e25 --- /dev/null +++ b/dockerfiles/services/prometheus/cli/build.usage @@ -0,0 +1 @@ +[-b][-t][-s] diff --git a/dockerfiles/services/prometheus/cli/deploy b/dockerfiles/services/prometheus/cli/deploy new file mode 100755 index 0000000..7082bef --- /dev/null +++ b/dockerfiles/services/prometheus/cli/deploy @@ -0,0 +1,140 @@ +#!/usr/bin/env bash +set -e +#set -x + +NC_VERSION=${NC_VERSION:-18} +LOCAL_IMAGE=xaicorp/nextcloud +TAG=${NC_VERSION}.0-${BUILD_NUMBER:-dev} +REMOTE_IMAGE=dkregistry.xai-corp.net:5000/${LOCAL_IMAGE}:${TAG} +STACK_NAME=xaibox +SERVICE_NAME=app +APP_NAME=${STACK_NAME}_${SERVICE_NAME} + +LOG=$(mktemp) + +export LOCAL_IMAGE +export REMOTE_IMAGE +export TAG + +export DOCKER_HOST=${DOCKER_HOST:-'dkhost:2376'} + +### +function deploy() { + docker pull "$REMOTE_IMAGE" + + docker stack deploy \ + --with-registry-auth \ + -c docker-compose.prod.yml \ + ${STACK_NAME} + + sleep 2 + docker stack ps ${STACK_NAME} + + wait_for_completed + + occ upgrade + occ app:update --all + occ db:add-missing-indices +# occ db:convert-filecache-bigint +} + +wait_for_completed() { + #states supported: "rollback_completed", "updating", "completed" + state=$(docker service inspect ${APP_NAME} | jq -r .[0].UpdateStatus.State) + while [ "completed" != "$state" ]; do + echo "$state" + sleep 3 + state=$(docker service inspect ${APP_NAME} | jq -r .[0].UpdateStatus.State) + done + sleep 5 +} + +function deploy_test() { + docker ps | grep ${APP_NAME} + + assertOK https xaibox.xai-corp.net + + assertNetwork prod_ui + assertNetwork prod_app + assertNetwork prod_db + assertNetwork prod_cache + + occ check + occ status +} + +function deploy_save() { + #tag as latest + echo -e "\033[94mTagging image as latest\033[39m" + docker tag "$REMOTE_IMAGE" "${REMOTE_IMAGE//${TAG}/latest}" + docker push "${REMOTE_IMAGE//${TAG}/latest}" +} + +dc() { + # shellcheck disable=SC2068 + docker-compose \ + -f docker-compose.yml \ + -f docker-compose.prod.yml \ + $@ +} + +occ() { + container=$(docker ps -qn1) + # shellcheck disable=SC2068 + docker exec -t -u www-data "$container" php occ $@ +} + +function assertOK() { + proto=$1 + domain=$2 + set -e + echo -e "\033[94m${proto}://${domain}\033[39m" + curl -IskH "Host: ${domain}" "${proto}://dkhost.xai-corp.net" | + tee "$LOG" | grep -P "200 OK|302 Found|403 Forbidden" +} + +function assertNetwork() { + network=$1 + echo -e "\033[94minspecting network\033[39m $network" + docker network inspect "$network" | jq -r .[].Containers[].Name | tee "$LOG" | grep ${APP_NAME} +} + +function trap_exit() { + code=$? + docker service ls | grep "${APP_NAME}" + if [ $code -gt 0 ]; then + echo + cat "$LOG" + rm "$LOG" + echo -e "\033[31mFailed to deploy ${REMOTE_IMAGE} \033[39m" + exit $code + fi + + rm "$LOG" + echo -e "\033[32mSuccess:\033[39m ${REMOTE_IMAGE} successfully deployed" +} +trap trap_exit EXIT + +print_usage() { + printf "Usage: %s: [-b] [-t] [-s] \n" "$0" + echo -d deploy + echo -t smoke tests + echo -s tag as latest + echo -h help + exit 0 +} + +###### +if [ -z "$1" ]; then + deploy && deploy_test && deploy_save + exit +fi + +while getopts tdhs name; do + case $name in + d) deploy ;; + t) deploy_test ;; + s) deploy_save ;; + *) print_usage ;; + esac +done diff --git a/dockerfiles/services/prometheus/cli/deploy.help b/dockerfiles/services/prometheus/cli/deploy.help new file mode 100644 index 0000000..7aa5493 --- /dev/null +++ b/dockerfiles/services/prometheus/cli/deploy.help @@ -0,0 +1,3 @@ +ARGS - The arguments you wish to provide to this command + +TODO: Fill out the help information for this command. diff --git a/dockerfiles/services/prometheus/cli/deploy.usage b/dockerfiles/services/prometheus/cli/deploy.usage new file mode 100644 index 0000000..5226895 --- /dev/null +++ b/dockerfiles/services/prometheus/cli/deploy.usage @@ -0,0 +1 @@ +ARGS... diff --git a/dockerfiles/services/prometheus/cli/exec b/dockerfiles/services/prometheus/cli/exec new file mode 100755 index 0000000..5f91af0 --- /dev/null +++ b/dockerfiles/services/prometheus/cli/exec @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -e + +export DOCKER_HOST=${DOCKER_HOST:-'dkhost:2376'} + +container=$(docker ps -qn1) + +while getopts c: name +do + case $name in + c) + container=$OPTARG + ;; + *) + ;; + esac +done +shift $((OPTIND -1)) + +echo using container "${container}" +# shellcheck disable=SC2068 +docker exec -it -u www-data "$container" $@ diff --git a/dockerfiles/services/prometheus/cli/rollback b/dockerfiles/services/prometheus/cli/rollback new file mode 100755 index 0000000..0a6b9d0 --- /dev/null +++ b/dockerfiles/services/prometheus/cli/rollback @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +set -e +#set -x + +LOCAL_IMAGE=xaicorp/nextcloud +TAG=${NC_VERSION}.0-${BUILD_NUMBER:-dev} +REMOTE_IMAGE=dkregistry.xai-corp.net:5000/${LOCAL_IMAGE}:${TAG} +STACK_NAME=xaibox +SERVICE_NAME=app +APP_NAME=${STACK_NAME}_${SERVICE_NAME} + +LOG=$(mktemp) + +#export LOCAL_IMAGE +export REMOTE_IMAGE +#export TAG + +export DOCKER_HOST=${DOCKER_HOST:-'dkhost:2376'} + +### + +function rollback() { +# docker service inspect ${APP_NAME} + docker service update --rollback "${APP_NAME}" + + wait_for_completed +# docker service scale "${APP_NAME}=2" +} + +wait_for_completed() { + #states supported: "rollback_completed", "updating", "completed" + state=$(docker service inspect ${APP_NAME} | jq -r .[0].UpdateStatus.State) + while [ "rollback_completed" != "$state" ]; do + echo "$state" + sleep 3 + state=$(docker service inspect ${APP_NAME} | jq -r .[0].UpdateStatus.State) + done +} + +function rollback_test() { + docker service ps --filter "desired-state=Healthy" ${APP_NAME} + + docker ps | grep "${APP_NAME}" +} + +function rollback_save() { + echo TODO +} + + +function trap_exit() { + code=$? + docker service ls | grep ${APP_NAME} + if [ $code -gt 0 ]; then + echo + rm "$LOG" + echo -e "\033[31mFailed rolling back ${APP_NAME} \033[39m" + exit $code + fi + + rm "$LOG" + echo -e "\033[32mSuccess:\033[39m ${APP_NAME} successfully rolled back" +} +trap trap_exit EXIT + +print_usage() { + printf "Usage: %s: [-b] [-t] [-s] \n" "$0" + echo -r rollback + echo -t smoke tests + echo -s tag as latest + echo -h help + exit 0 +} + +###### +if [ -z "$1" ]; then + rollback && rollback_test && rollback_save + exit +fi + +while getopts tdhs name +do + case $name in + d) rollback;; + t) rollback_test;; + s) rollback_save;; + *) print_usage;; + esac +done diff --git a/dockerfiles/services/prometheus/cli/rollback.help b/dockerfiles/services/prometheus/cli/rollback.help new file mode 100644 index 0000000..7aa5493 --- /dev/null +++ b/dockerfiles/services/prometheus/cli/rollback.help @@ -0,0 +1,3 @@ +ARGS - The arguments you wish to provide to this command + +TODO: Fill out the help information for this command. diff --git a/dockerfiles/services/prometheus/cli/rollback.usage b/dockerfiles/services/prometheus/cli/rollback.usage new file mode 100644 index 0000000..5226895 --- /dev/null +++ b/dockerfiles/services/prometheus/cli/rollback.usage @@ -0,0 +1 @@ +ARGS... diff --git a/dockerfiles/services/prometheus/docker-compose.build.yml b/dockerfiles/services/prometheus/docker-compose.build.yml new file mode 100644 index 0000000..3ed1c15 --- /dev/null +++ b/dockerfiles/services/prometheus/docker-compose.build.yml @@ -0,0 +1,20 @@ +--- +# docker-compose file for prometheus metrics server + +# docker login dkregistry.xai-corp.net:5000 +# docker-compose build && docker push dkregistry.xai-corp.net:5000/xaicorp/prometheus:latest + +# DOCKER_HOST=dkmanager:2376 docker stack deploy --prune --with-registry-auth -c docker-compose-prod.yml metrics +# DOCKER_HOST=dkmanager:2376 docker stack ps metrics + +version: '3.4' +services: + + prometheus: + build: + context: . + dockerfile: Dockerfile + args: + PROM_VERSION: ${PROM_VERSION} +# volumes: +# - ./prometheus.yml:/etc/prometheus/prometheus.yml diff --git a/dockerfiles/services/prometheus/docker-compose.yml b/dockerfiles/services/prometheus/docker-compose.yml index 48b5c09..b363a6c 100644 --- a/dockerfiles/services/prometheus/docker-compose.yml +++ b/dockerfiles/services/prometheus/docker-compose.yml @@ -12,9 +12,6 @@ services: prometheus: image: "dkregistry.xai-corp.net:5000/xaicorp/prometheus:latest" - build: - context: . - dockerfile: Dockerfile ports: - 9090:9090 # volumes: @@ -26,31 +23,31 @@ services: # fluentd-async-connect: 'true' # tag: postgres - graphana: - image: grafana/grafana - ports: - - 3001:3000 - environment: - GF_SECURITY_ADMIN_PASSWORD: sakas -# volumes: -# - /opt/shared/nextcloud/data:/var/www/html/data - - postgres_exported: - image: wrouesnel/postgres_exporter - environment: - DATA_SOURCE_NAME: "postgresql://xaicorp_admin:snqioxni1sw@tasks.postgres:5432/?sslmode=disable" - - fluentd_exporter: - image: vect0r/fluentd_exporter +# graphana: +# image: grafana/grafana # ports: -# - 9309:9309 - command: - - go-wrapper - - run - - -scrape_uri - - http://tasks.fluentd:24220/api/plugins.json - - node_exporter: - image: quay.io/prometheus/node-exporter - pid: "host" - +# - 3001:3000 +# environment: +# GF_SECURITY_ADMIN_PASSWORD: sakas +## volumes: +## - /opt/shared/nextcloud/data:/var/www/html/data +# +# postgres_exported: +# image: wrouesnel/postgres_exporter +# environment: +# DATA_SOURCE_NAME: "postgresql://xaicorp_admin:snqioxni1sw@tasks.postgres:5432/?sslmode=disable" +# +# fluentd_exporter: +# image: vect0r/fluentd_exporter +## ports: +## - 9309:9309 +# command: +# - go-wrapper +# - run +# - -scrape_uri +# - http://tasks.fluentd:24220/api/plugins.json +# +# node_exporter: +# image: quay.io/prometheus/node-exporter +# pid: "host" +#