From ce0563a739465ffde8fba3987a8cc08ceaf20e23 Mon Sep 17 00:00:00 2001 From: YueSong Jiang <1259012537@qq.com> Date: Mon, 9 Mar 2020 10:50:27 +0800 Subject: [PATCH 1/5] Merge with master --- docker-entrypoint.sh | 66 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 55 insertions(+), 11 deletions(-) diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index b07cf28..853c8b9 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -1,27 +1,71 @@ #!/bin/bash set -e +STOP_LOOP="false" + +# if DATABASE_NAME is not provided use default one: "docker" +export DATABASE_NAME="${DATABASE_NAME:-docker}" +# if DATABASE_PASSWORD is provided, use it as DB password, otherwise empty password +if [ -n "$DATABASE_PASSWORD" ]; then export DBPW="-p $DATABASE_PASSWORD" VSQLPW="-w $DATABASE_PASSWORD"; else export DBPW="" VSQLPW=""; fi + # Vertica should be shut down properly function shut_down() { echo "Shutting Down" - gosu dbadmin /opt/vertica/bin/admintools -t stop_db -d docker -i - exit + vertica_proper_shutdown + echo 'Saving configuration' + mkdir -p ${VERTICADATA}/config + /bin/cp /opt/vertica/config/admintools.conf ${VERTICADATA}/config/admintools.conf + echo 'Stopping loop' + STOP_LOOP="true" +} + +function vertica_proper_shutdown() { + echo 'Vertica: Closing active sessions' + /bin/su - dbadmin -c "/opt/vertica/bin/vsql -U dbadmin -d ${DATABASE_NAME} ${VSQLPW} -c 'SELECT CLOSE_ALL_SESSIONS();'" + echo 'Vertica: Flushing everything on disk' + /bin/su - dbadmin -c "/opt/vertica/bin/vsql -U dbadmin -d ${DATABASE_NAME} ${VSQLPW} -c 'SELECT MAKE_AHM_NOW();'" + echo 'Vertica: Stopping database' + /bin/su - dbadmin -c "/opt/vertica/bin/admintools -t stop_db ${DBPW} -d ${DATABASE_NAME} -i" } -trap "shut_down" SIGKILL SIGTERM SIGHUP SIGINT EXIT +function fix_filesystem_permissions() { + chown -R dbadmin:verticadba "${VERTICADATA}" + chown dbadmin:verticadba /opt/vertica/config/admintools.conf +} + +trap "shut_down" SIGKILL SIGTERM SIGHUP SIGINT -chown -R dbadmin:verticadba "$VERTICADATA" -if [ -z "$(ls -A "$VERTICADATA")" ]; then - echo "Creating database" - gosu dbadmin /opt/vertica/bin/admintools -t drop_db -d docker - gosu dbadmin /opt/vertica/bin/admintools -t create_db -s localhost -d docker -c /home/dbadmin/docker/catalog -D /home/dbadmin/docker/data +echo 'Starting up' +if [ ! -f ${VERTICADATA}/config/admintools.conf ]; then + echo 'Fixing filesystem permissions' + fix_filesystem_permissions + echo 'Creating database' + su - dbadmin -c "/opt/vertica/bin/admintools -t create_db --skip-fs-checks -s localhost -d ${DATABASE_NAME} ${DBPW} -c ${VERTICADATA}/catalog -D ${VERTICADATA}/data" else - gosu dbadmin /opt/vertica/bin/admintools -t start_db -d docker -i + echo 'Restoring configuration' + cp ${VERTICADATA}/config/admintools.conf /opt/vertica/config/admintools.conf + echo 'Fixing filesystem permissions' + fix_filesystem_permissions + echo 'Starting Database' + su - dbadmin -c "/opt/vertica/bin/admintools -t start_db -d ${DATABASE_NAME} ${DBPW} --noprompts --timeout=never" +fi + +echo +if [ -d /docker-entrypoint-initdb.d/ ]; then + echo "Running entrypoint scripts ..." + for f in $(ls /docker-entrypoint-initdb.d/* | sort); do + case "$f" in + *.sh) echo "$0: running $f"; . "$f" ;; + *.sql) echo "$0: running $f"; su - dbadmin -c "/opt/vertica/bin/vsql -d ${DATABASE_NAME} ${VSQLPW} -f $f"; echo ;; + *) echo "$0: ignoring $f" ;; + esac + echo + done fi echo "Vertica is now running" -while true; do +while [ "${STOP_LOOP}" == "false" ]; do sleep 1 -done +done \ No newline at end of file From c742153b70571fe3136b8cee4d59de5709f62daa Mon Sep 17 00:00:00 2001 From: YueSong Jiang <1259012537@qq.com> Date: Mon, 9 Mar 2020 10:55:19 +0800 Subject: [PATCH 2/5] add vertica running state check --- docker-entrypoint.sh | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 853c8b9..5c880e1 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -33,7 +33,14 @@ function fix_filesystem_permissions() { chown dbadmin:verticadba /opt/vertica/config/admintools.conf } -trap "shut_down" SIGKILL SIGTERM SIGHUP SIGINT +function containsElement() { + local e match="$1" + shift + for e; do [[ "$e" == "$match" ]] && return 0; done + return 1 +} + +trap "shut_down" SIGTERM SIGHUP SIGINT echo 'Starting up' @@ -67,5 +74,11 @@ fi echo "Vertica is now running" while [ "${STOP_LOOP}" == "false" ]; do - sleep 1 + sleep 5 + down_db=$(su - dbadmin -c "/opt/vertica/bin/admintools -t db_status -s DOWN") + IFS=', ' read -r -a db_array <<< "$down_db" + if containsElement "$DATABASE_NAME" "${db_array[@]}"; then + echo "database $DATABASE_NAME is already stop, exit" + shut_down + fi done \ No newline at end of file From 5d72908c07cb92372a4ffed2b24e4aecd1d4ee46 Mon Sep 17 00:00:00 2001 From: YueSong Jiang <1259012537@qq.com> Date: Mon, 9 Mar 2020 10:59:24 +0800 Subject: [PATCH 3/5] finxed start flow: 1. if you change the database name via env, it will create new database instead of throw error 2. for single vertica, use wos may case database crash when lastgoodepoch is slow than current epoch, add recorvery function if start failed --- docker-entrypoint.sh | 52 +++++++++++++++++++++++++++++++++----------- 1 file changed, 39 insertions(+), 13 deletions(-) diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 5c880e1..54254c0 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -42,20 +42,46 @@ function containsElement() { trap "shut_down" SIGTERM SIGHUP SIGINT - echo 'Starting up' -if [ ! -f ${VERTICADATA}/config/admintools.conf ]; then - echo 'Fixing filesystem permissions' - fix_filesystem_permissions - echo 'Creating database' - su - dbadmin -c "/opt/vertica/bin/admintools -t create_db --skip-fs-checks -s localhost -d ${DATABASE_NAME} ${DBPW} -c ${VERTICADATA}/catalog -D ${VERTICADATA}/data" -else - echo 'Restoring configuration' - cp ${VERTICADATA}/config/admintools.conf /opt/vertica/config/admintools.conf - echo 'Fixing filesystem permissions' - fix_filesystem_permissions - echo 'Starting Database' - su - dbadmin -c "/opt/vertica/bin/admintools -t start_db -d ${DATABASE_NAME} ${DBPW} --noprompts --timeout=never" + +echo 'check if database exit' + +# this info was get from admintools.conf +# so that file must be persistent +if [[ -f "${VERTICADATA}"/config/admintools.conf ]];then + echo 'Restoring configuration' + /bin/cp -vf "${VERTICADATA}"/config/admintools.conf /opt/vertica/config/admintools.conf +fi + +databases=$(su - dbadmin -c "/opt/vertica/bin/admintools -t db_status -s ALL") +IFS=', ' read -r -a database_array <<< "$databases" + +if ! containsElement "$DATABASE_NAME" "${database_array[@]}"; then + echo "database with name [$DATABASE_NAME] not exist" + echo 'Fixing filesystem permissions' + fix_filesystem_permissions + echo 'Creating database' + # if you delete admintools.conf but data still exist + # thi will give "Catalog parent directory already exists" error + su - dbadmin -c "/opt/vertica/bin/admintools -t create_db --skip-fs-checks -s localhost -d ${DATABASE_NAME} ${DBPW} -c ${VERTICADATA}/catalog -D ${VERTICADATA}/data" + echo "Backup configuration" + mkdir -p "${VERTICADATA}"/config + /bin/cp -vf /opt/vertica/config/admintools.conf "${VERTICADATA}"/config/admintools.conf + /opt/vertica/sbin/vertica_agent start +else + echo 'Fixing filesystem permissions' + fix_filesystem_permissions + echo 'Starting Database' + su - dbadmin -c "/opt/vertica/bin/admintools -t start_db -d ${DATABASE_NAME} ${DBPW} --noprompts --timeout=${DATABASE_TIMEOUT}" || true # use true to ignore error + # check if start success + databases=$(su - dbadmin -c "/opt/vertica/bin/admintools -t db_status -s UP") + IFS=', ' read -r -a database_array <<< "$databases" + if ! containsElement "$DATABASE_NAME" "${database_array[@]}"; then + # start failed , may because the wos problem + echo "recovery database" + su - dbadmin -c "/opt/vertica/bin/admintools -t restart_db -e 'last' -d ${DATABASE_NAME} ${DBPW} --noprompts --timeout=${DATABASE_TIMEOUT}" + fi + /opt/vertica/sbin/vertica_agent start fi echo From 9d392835b22c65ec16451ce26862b172a713cbf0 Mon Sep 17 00:00:00 2001 From: YueSong Jiang <1259012537@qq.com> Date: Mon, 9 Mar 2020 10:59:54 +0800 Subject: [PATCH 4/5] format code --- docker-entrypoint.sh | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 54254c0..bafff69 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -86,15 +86,22 @@ fi echo if [ -d /docker-entrypoint-initdb.d/ ]; then - echo "Running entrypoint scripts ..." - for f in $(ls /docker-entrypoint-initdb.d/* | sort); do - case "$f" in - *.sh) echo "$0: running $f"; . "$f" ;; - *.sql) echo "$0: running $f"; su - dbadmin -c "/opt/vertica/bin/vsql -d ${DATABASE_NAME} ${VSQLPW} -f $f"; echo ;; - *) echo "$0: ignoring $f" ;; - esac - echo - done + echo "Running entrypoint scripts ..." + for f in $(find /docker-entrypoint-initdb.d/ | sort); do + case "$f" in + *.sh) + echo "$0: running $f" + ."$f" + ;; + *.sql) + echo "$0: running $f" + su - dbadmin -c "/opt/vertica/bin/vsql -d ${DATABASE_NAME} ${DBPW} -f $f" + echo + ;; + *) echo "$0: ignoring $f" ;; + esac + echo + done fi echo "Vertica is now running" From c5bfb3e11de0d0537ee053433beef3914933e301 Mon Sep 17 00:00:00 2001 From: YueSong Jiang <1259012537@qq.com> Date: Mon, 9 Mar 2020 11:07:34 +0800 Subject: [PATCH 5/5] cp use -f command add database start timeout env --- docker-entrypoint.sh | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index bafff69..0cc3156 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -5,18 +5,19 @@ STOP_LOOP="false" # if DATABASE_NAME is not provided use default one: "docker" export DATABASE_NAME="${DATABASE_NAME:-docker}" +export DATABASE_TIMEOUT="${DATABASE_TIMEOUT:-300}" # if DATABASE_PASSWORD is provided, use it as DB password, otherwise empty password if [ -n "$DATABASE_PASSWORD" ]; then export DBPW="-p $DATABASE_PASSWORD" VSQLPW="-w $DATABASE_PASSWORD"; else export DBPW="" VSQLPW=""; fi # Vertica should be shut down properly function shut_down() { - echo "Shutting Down" - vertica_proper_shutdown - echo 'Saving configuration' - mkdir -p ${VERTICADATA}/config - /bin/cp /opt/vertica/config/admintools.conf ${VERTICADATA}/config/admintools.conf - echo 'Stopping loop' - STOP_LOOP="true" + echo "Shutting Down" + vertica_proper_shutdown + echo 'Saving configuration' + mkdir -p "${VERTICADATA}"/config + /bin/cp -vf /opt/vertica/config/admintools.conf "${VERTICADATA}"/config/admintools.conf + echo 'Stopping loop' + STOP_LOOP="true" } function vertica_proper_shutdown() {