From: Frank Brehm Date: Thu, 23 May 2019 09:47:15 +0000 (+0200) Subject: Adding bin/backup-pgsql.sh X-Git-Tag: 0.4.1~12^2 X-Git-Url: https://git.uhu-banane.net/?a=commitdiff_plain;h=7a81274a1a2b5936970ec7535b39051a4291512c;p=pixelpark%2Fpp-admin-tools.git Adding bin/backup-pgsql.sh --- diff --git a/bin/backup-pgsql.sh b/bin/backup-pgsql.sh new file mode 100755 index 0000000..195ae0f --- /dev/null +++ b/bin/backup-pgsql.sh @@ -0,0 +1,634 @@ +#!/usr/bin/env bash + +########################### +# REQUIREMENTS +########################### +# +# * Required commands: +# + pg_dump +# + du +# + tee +# + bzip2 # If bzip2 is not available, change 'CMD_COMPRESS' +# # to use 'gzip' or whatever compress command you want. +# + +########################### +# USAGE +########################### +# +# * It stores all backup copies in directory '/var/vmail/backup' by default, +# You can change it in variable $BACKUP_ROOTDIR below or via the -b parameter. +# +# * Set correct values for below variables: +# +# PGSQL_SYS_USER +# BACKUP_ROOTDIR +# +# * Add crontab job for root user (or whatever user you want): +# +# # crontab -e -u postgres +# 1 4 * * * bash /path/to/backup_pgsql.sh -q +# +# * Make sure 'crond' service is running. +# + +set -e +set -u + +export LC_ALL=C +export LANG=C + +VERBOSE="n" +DEBUG="n" +QUIET='n' + +VERSION="3.1" + +# console colors: +RED="" +YELLOW="" +GREEN="" +BLUE="" +NORMAL="" + +HAS_TTY='y' + +BASENAME="$(basename ${0})" +BASE_DIR="$(dirname ${0})" + +declare -a DATABASES=() + +######################################################### +# Modify below variables to fit your need ---- +######################################################### +# Keep backup for how many days. Default is 90 days. +KEEP_DAYS='30' + +# System user used to run PostgreSQL daemon. +# - On Linux, it's postgres. +# - On FreeBSD, it's pgsql. +# - On OpenBSD, it's _postgresql. +PGSQL_SYS_USER="postgres" + +# Where to store backup copies. +BACKUP_ROOTDIR="/var/backup/pgsql" + +# Date. +YEAR="$( date +%Y)" +MONTH="$( date +%m)" +DAY="$( date +%d)" +TIME="$( date +%H:%M:%S)" +TIMESTAMP="${YEAR}-${MONTH}-${DAY}-${TIME}" + +# Pre-defined backup status +BACKUP_SUCCESS='YES' + +# Define, check, create directories. +BACKUP_DIR="${BACKUP_ROOTDIR}/${YEAR}/${MONTH}/${DAY}" +TMP_DIR= +LOGFILE="/dev/null" + +BYTES_TOTAL="0" + +#------------------------------------------------------------------- +detect_color() { + + local safe_term="${TERM//[^[:alnum:]]/?}" + local match_lhs="" + local use_color="false" + [[ -f ~/.dir_colors ]] && match_lhs="${match_lhs}$(<~/.dir_colors)" + [[ -f /etc/DIR_COLORS ]] && match_lhs="${match_lhs}$(/dev/null \ + && match_lhs=$(dircolors --print-database) + [[ $'\n'${match_lhs} == *$'\n'"TERM "${safe_term}* ]] && use_color="true" + + # console colors: + if [ "${use_color}" = "true" ] ; then + RED="\033[38;5;196m" + YELLOW="\033[38;5;226m" + GREEN="\033[38;5;46m" + BLUE="\033[38;5;27m" + NORMAL="\033[39m" + else + RED="" + YELLOW="" + GREEN="" + BLUE="" + NORMAL="" + fi + + local my_tty=$(tty) + if [[ "${my_tty}" =~ 'not a tty' ]] ; then + my_tty='-' + fi + + if [[ "${my_tty}" = '-' || "${safe_term}" = "dump" ]] ; then + HAS_TTY='n' + fi + +} +detect_color + +#------------------------------------------------------------------------------ +description() { + echo -e $( cat <<-EOF + Creates a backup of all databases of the PostgreSQL installatio + on the current host. + + Only the user '${GREEN}${PGSQL_SYS_USER}${NORMAL}' may execute this script. + + EOF + ) +} + +#------------------------------------------------------------------------------ +usage() { + cat <<-EOF + Usage: ${BASENAME} [-K DAYS|--keep=DAYS] [-b DIR|--backupdir=DIR] [-d|--debug] [[-v|--verbose] | [-q|--quiet]]] [--nocolor] + ${BASENAME} [-h|--help] + ${BASENAME} [-V|--version] + + Options: + -K|--keep DAYS Keep the backup files of the last DAYS. Default: ${KEEP_DAYS} days. + -b|--backupdir DIR + Set root backup directory. Default: ${BACKUP_ROOTDIR} + -d|--debug Debug output (bash -x). + -v|--verbose Set verbosity on. Mutually exclusive to '--quiet'. + -q|--quiet Quiet execution, only errors and warnings are shown. + --nocolor Don't use colors on display. + -h|--help Show this output and exit. + -V|--version prints out version number of the script and exit + EOF +} + + +#------------------------------------------------------------------------------ +get_options() { + + local tmp= + local base_dir= + + set +e + tmp=$( getopt -o K:b:dvqhV \ + --long keep:,backupdir:,debug,verbose,quiet,nocolor,help,version \ + -n "${BASENAME}" -- "$@" ) + if [[ $? != 0 ]] ; then + echo "" >&2 + usage >&2 + exit 1 + fi + set -e + + # Note the quotes around `$TEMP': they are essential! + eval set -- "${tmp}" + + local p= + + while true ; do + case "$1" in + -K|--keep) + KEEP_DAYS="$2" + shift + shift + ;; + -b|--backupdir) + BACKUP_ROOTDIR="$2" + BACKUP_DIR="${BACKUP_ROOTDIR}/${YEAR}/${MONTH}/${DAY}" + shift + shift + ;; + -d|--debug) + DEBUG="y" + shift + ;; + -v|--verbose) + VERBOSE="y" + shift + ;; + -q|--quiet) + QUIET="y" + RED="" + YELLOW="" + GREEN="" + BLUE="" + NORMAL="" + shift + ;; + --nocolor) + RED="" + YELLOW="" + GREEN="" + BLUE="" + NORMAL="" + shift + ;; + -h|--help) + description + echo + usage + exit 0 + ;; + -V|--version) + echo "${BASENAME} version: ${VERSION}" + exit 0 + ;; + --) shift + break + ;; + *) echo "Internal error!" + exit 1 + ;; + esac + done + + if [[ "${DEBUG}" = "y" ]] ; then + set -x + fi + if [[ "${VERBOSE}" == "y" && "${QUIET}" == "y" ]] ; then + error "The parameters '${RED}${VERBOSE}${NORMAL}' and '${RED}${VERBOSE}${NORMAL}' are mutually exclusive." + usage >&2 + exit 1 + fi + + local keep_int=$(( $KEEP_DAYS + 0 )) + if [[ "${keep_int}" -le "0" ]] ; then + error "Invalid number of days '${RED}${KEEP_DAYS}${NORMAL}' to keep backup files." + echo >&2 + description >&2 + echo + usage >&2 + exit 1 + fi + debug "Keeping backupfiles, which are not older than ${keep_int} days." + KEEP_DAYS="${keep_int}" + + local cur_user=$( id -u -n ) + if [[ "${cur_user}" != "${PGSQL_SYS_USER}" ]] ; then + error "Wrong user '${RED}${cur_user}${NORMAL}'." + echo >&2 + description >&2 + echo + usage >&2 + exit 1 + fi + +} + +######################################### +# Some often used funktions + +#------------------------------------------------------------------------------ +my_date() { + date +'%F %T.%N %:::z' +} + +#------------------------------------------------------------------------------ +debug() { + if [[ "${VERBOSE}" != "y" ]] ; then + return 0 + fi + echo -e " * [$(my_date)] [${BASENAME}:DEBUG]: $@" | tee -a "${LOGFILE}" +} + +#------------------------------------------------------------------------------ +info() { + if [[ "${QUIET}" == "y" ]] ; then + echo -e " * [$(my_date)] [${BASENAME}:INFO] : $@" >> "${LOGFILE}" + return 0 + fi + echo -e " ${GREEN}*${NORMAL} [$(my_date)] [${BASENAME}:${GREEN}INFO${NORMAL}] : $@" | tee -a "${LOGFILE}" +} + +#------------------------------------------------------------------------------ +warn() { + echo -e " ${YELLOW}*${NORMAL} [$(my_date)] [${BASENAME}:${YELLOW}WARN${NORMAL}] : $@" | tee -a "${LOGFILE}" +} + +#------------------------------------------------------------------------------ +error() { + echo -e " ${RED}*${NORMAL} [$(my_date)] [${BASENAME}:${RED}ERROR${NORMAL}]: $@" | tee -a "${LOGFILE}" +} + +#------------------------------------------------------------------------------ +MKDIR() { + local cmd="mkdir" + if [[ "${VERBOSE}" == "y" ]] ; then + cmd+=" --verbose" + fi + eval ${cmd} "$@" 2>&1 | tee -a "${LOGFILE}" +} + +#------------------------------------------------------------------------------ +RM() { + local cmd="rm" + if [[ "${VERBOSE}" == "y" ]] ; then + cmd+=" --verbose" + fi + eval ${cmd} "$@" 2>&1 | tee -a "${LOGFILE}" +} + +#------------------------------------------------------------------------------ +MV() { + local cmd="mv" + if [[ "${VERBOSE}" == "y" ]] ; then + cmd+=" --verbose" + fi + eval ${cmd} "$@" 2>&1 | tee -a "${LOGFILE}" +} + +#------------------------------------------------------------------------------ +RMDIR() { + local cmd="rmdir" + if [[ "${VERBOSE}" == "y" ]] ; then + cmd+=" --verbose" + fi + eval ${cmd} "$@" 2>&1 | tee -a "${LOGFILE}" +} + +#------------------------------------------------------------------------------ +LN() { + local cmd="ln" + if [[ "${VERBOSE}" == "y" ]] ; then + cmd+=" --verbose" + fi + eval ${cmd} "$@" 2>&1 | tee -a "${LOGFILE}" +} + +#------------------------------------------------------------------------------ +empty_line() { + if [[ "${QUIET}" == "y" ]] ; then + echo >> "${LOGFILE}" + return 0 + fi + echo 2>&1 | tee -a "${LOGFILE}" +} + +################################################################################ + +get_databases() { + + debug "Detecting databases to backup ..." + local db= + for db in $( psql --list \ + --tuples-only \ + --no-align \ + --no-readline \ + --expanded \ + --field-separator=',' | \ + grep -i '^Name' | \ + awk -F ',' '{print $2}' ) ; do + DATABASES+=( "${db}" ) + done + + if [[ "${VERBOSE}" == "y" ]] ; then + echo | tee -a "${LOGFILE}" + echo "Databases to backup:" | tee -a "${LOGFILE}" + for db in "${DATABASES[@]}" ; do + echo " * '${db}'" | tee -a "${LOGFILE}" + done + echo | tee -a "${LOGFILE}" + fi + +} + +#------------------------------------------------------------------------------ +cleanup_tmp_dir() { + if [[ -n "${TMP_DIR}" ]] ; then + debug "Removing temporary directory is '${TMP_DIR}' ..." + RM --force --recursive "${TMP_DIR}" + fi +} + +#------------------------------------------------------------------------------ +prepare_dirs() { + + if [[ ! -d "${BACKUP_ROOTDIR}" ]] ; then + error "Directory '${RED}${BACKUP_ROOTDIR}${NORMAL}' does not exists or is not a directory." + exit 5 + fi + if [[ ! -w "${BACKUP_ROOTDIR}" ]] ; then + error "No write access to '${RED}${BACKUP_ROOTDIR}${NORMAL}'." + exit 6 + fi + + info "Creating all necessary directories ..." + MKDIR -p "${BACKUP_DIR}" + TMP_DIR=$( mktemp -d -p "${HOME}" backup.XXXXXXXX.d ) + debug "Temporary directory is '${TMP_DIR}'." + LOGFILE="${BACKUP_DIR}/${TIMESTAMP}.log" + + debug "Creating trap to cleanup temporary directory ..." + trap cleanup_tmp_dir INT TERM EXIT ABRT + +} + +#------------------------------------------------------------------------------ +cleanup_old_backups() { + + info "Cleaning up old backup files and directories ..." + + local verbose_option="" + if [[ "${VERBOSE}" == "y" ]] ; then + verbose_option="--verbose" + fi + + find "${BACKUP_ROOTDIR}" -type f -mtime +${KEEP_DAYS} -print0 | \ + xargs --null --no-run-if-empty rm ${verbose_option} 2>&1 | tee -a "${LOGFILE}" + + local year= + local month= + local day= + + for year in $( ls -1 "${BACKUP_ROOTDIR}" ); do + local y_dir="${BACKUP_ROOTDIR}/${year}" + if [[ -d "${y_dir}" ]] ; then + for month in $( ls -1 "${y_dir}" ); do + local m_dir="${y_dir}/${month}" + if [[ -d "${m_dir}" ]] ; then + for day in $( ls -1 "${m_dir}" ); do + local d_dir="${m_dir}/${day}" + if [[ -d "${d_dir}" && "${d_dir}" != "${BACKUP_DIR}" ]] ; then + rmdir --ignore-fail-on-non-empty "${d_dir}" + if [[ ! -d "${d_dir}" ]] ; then + debug "Removed directory '${d_dir}'." + fi + fi + done + rmdir --ignore-fail-on-non-empty "${m_dir}" + if [[ ! -d "${m_dir}" ]] ; then + debug "Removed directory '${m_dir}'." + fi + fi + done + rmdir --ignore-fail-on-non-empty "${y_dir}" + if [[ ! -d "${y_dir}" ]] ; then + debug "Removed directory '${y_dir}'." + fi + fi + done + +} + +#------------------------------------------------------------------------------ +backup_globals() { + + empty_line + info "Backing up ${GREEN}globals${NORMAL} ..." + + local output_sql="globals-${TIMESTAMP}.sql" + local output_sql_compressed="${output_sql}.bz2" + local out_sql_tmp="${TMP_DIR}/${output_sql}" + local out_sql_tmp_compressed="${TMP_DIR}/${output_sql_compressed}" + local out_sql_tgt="${BACKUP_DIR}/${output_sql}" + local out_sql_tgt_compressed="${BACKUP_DIR}/${output_sql_compressed}" + local out_sql_tgt_latest="${BACKUP_ROOTDIR}/globals-latest.sql.bz2" + + local verbose_option="" + if [[ "${VERBOSE}" == "y" ]] ; then + verbose_option="--verbose" + fi + + pg_dumpall --globals-only ${verbose_option} 2>&1 >"${out_sql_tmp}" | tee -a "${LOGFILE}" + + local blocks=$(stat -c "%b" "${out_sql_tmp}") + local bs=$(stat -c "%B" "${out_sql_tmp}") + local bytes=$(stat -c "%s" "${out_sql_tmp}") + local b_bytes=$(( ${blocks} * ${bs} )) + local k_bytes=$(( ${b_bytes} / 1024 )) + local m_bytes=$(( ${k_bytes} / 1024 )) + local msg=$( printf "Original size of %-50s %10d Bytes => %7d KiB => %4d MiB" \ + "'${output_sql}':" "${bytes}" "${k_bytes}" "${m_bytes}" ) + info "${msg}" + + debug "Compressing '${out_sql_tmp}' ..." + bzip2 ${verbose_option} --best "${out_sql_tmp}" 2>&1 | tee -a "${LOGFILE}" + + blocks=$(stat -c "%b" "${out_sql_tmp_compressed}") + bs=$(stat -c "%B" "${out_sql_tmp_compressed}") + bytes=$(stat -c "%s" "${out_sql_tmp_compressed}") + b_bytes=$(( ${blocks} * ${bs} )) + k_bytes=$(( ${b_bytes} / 1024 )) + m_bytes=$(( ${k_bytes} / 1024 )) + + BYTES_TOTAL=$(( ${BYTES_TOTAL} + ${b_bytes} )) + + local msg=$( printf "Compressed size of %-50s %10d Bytes => %7d KiB => %4d MiB" \ + "'${output_sql}':" "${bytes}" "${k_bytes}" "${m_bytes}" ) + info "${msg}" + + debug "Moving '${out_sql_tmp_compressed}' => '${BACKUP_DIR}' ..." + MV -i "${out_sql_tmp_compressed}" "${BACKUP_DIR}" + + info "Updating reference '${out_sql_tgt_latest}' -> '${out_sql_tgt_compressed}'" + LN -sf "${out_sql_tgt_compressed}" "${out_sql_tgt_latest}" + +} + +#------------------------------------------------------------------------------ +backup_databases() { + + local db= + for db in "${DATABASES[@]}" ; do + backup_database "${db}" + done + + empty_line + local k_bytes=$(( ${BYTES_TOTAL} / 1024 )) + local m_bytes=$(( ${k_bytes} / 1024 )) + local msg=$( printf "Total compressed size: %10d Bytes => %7d KiB => %4d MiB" \ + "${BYTES_TOTAL}" "${k_bytes}" "${m_bytes}" ) + info "${msg}" +} + +#------------------------------------------------------------------------------ +backup_database() { + + local db="$1" + + empty_line + info "Backing up database '${GREEN}${db}${NORMAL}' ..." + + local output_sql="${db}-${TIMESTAMP}.sql" + local output_sql_compressed="${output_sql}.bz2" + local out_sql_tmp="${TMP_DIR}/${output_sql}" + local out_sql_tmp_compressed="${TMP_DIR}/${output_sql_compressed}" + local out_sql_tgt="${BACKUP_DIR}/${output_sql}" + local out_sql_tgt_compressed="${BACKUP_DIR}/${output_sql_compressed}" + local out_sql_tgt_latest="${BACKUP_ROOTDIR}/${db}-latest.sql.bz2" + + local verbose_option="" + if [[ "${VERBOSE}" == "y" ]] ; then + verbose_option="--verbose" + fi + + pg_dump ${verbose_option} --blobs --clean \ + --create --if-exists --serializable-deferrable \ + "${db}" 2>&1 >"${out_sql_tmp}" | tee -a "${LOGFILE}" + + local blocks=$(stat -c "%b" "${out_sql_tmp}") + local bs=$(stat -c "%B" "${out_sql_tmp}") + local bytes=$(stat -c "%s" "${out_sql_tmp}") + local b_bytes=$(( ${blocks} * ${bs} )) + local k_bytes=$(( ${b_bytes} / 1024 )) + local m_bytes=$(( ${k_bytes} / 1024 )) + local msg=$( printf "Original size of %-50s %10d Bytes => %7d KiB => %4d MiB" \ + "'${output_sql}':" "${bytes}" "${k_bytes}" "${m_bytes}" ) + info "${msg}" + + debug "Compressing '${out_sql_tmp}' ..." + bzip2 ${verbose_option} --best "${out_sql_tmp}" 2>&1 | tee -a "${LOGFILE}" + + blocks=$(stat -c "%b" "${out_sql_tmp_compressed}") + bs=$(stat -c "%B" "${out_sql_tmp_compressed}") + bytes=$(stat -c "%s" "${out_sql_tmp_compressed}") + b_bytes=$(( ${blocks} * ${bs} )) + k_bytes=$(( ${b_bytes} / 1024 )) + m_bytes=$(( ${k_bytes} / 1024 )) + + BYTES_TOTAL=$(( ${BYTES_TOTAL} + ${b_bytes} )) + + local msg=$( printf "Compressed size of %-50s %10d Bytes => %7d KiB => %4d MiB" \ + "'${output_sql}':" "${bytes}" "${k_bytes}" "${m_bytes}" ) + info "${msg}" + + debug "Moving '${out_sql_tmp_compressed}' => '${BACKUP_DIR}' ..." + MV -i "${out_sql_tmp_compressed}" "${BACKUP_DIR}" + + info "Updating reference '${out_sql_tgt_latest}' -> '${out_sql_tgt_compressed}'" + LN -sf "${out_sql_tgt_compressed}" "${out_sql_tgt_latest}" +} + + + +################################################################################ +## +## Main +## +################################################################################ + +#------------------------------------------------------------------------------ +main() { + + get_options "$@" + + prepare_dirs + info "Starting backup ..." + get_databases + backup_globals + cleanup_old_backups + backup_databases + + empty_line + debug "Deactivating trap." + trap - INT TERM EXIT ABRT + cleanup_tmp_dir + info "Finished." + +} + +main "$@" + +exit 0 + +# vim: ts=4 et list