Skip to content

Commit

Permalink
backup-mysql.sh: old cruft removed, and we now compress via zstd.
Browse files Browse the repository at this point in the history
  • Loading branch information
@ckujau
ckujau committed Oct 21, 2022
1 parent 37f98cd commit d703a24
Showing 1 changed file with 90 additions and 78 deletions.
168 changes: 90 additions & 78 deletions backup-mysql.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,15 +7,29 @@
# $ mysqldump -AcfFl --flush-privileges | pbzip2 -c > backup.sql.bz2
#
# However, most of the databases do not change much or do not change at all
# over the day, yet the resulting .bz2 had to be generated every day. The
# compressed result _did_ change however and to be transferred in full over a
# 128 kbps line to our backup host. This one is slightly more complicated,
# but should suffice.
# over the day, yet the resulting compressed dump had to be generated every
# day, *did* change and had to be transferred in full over a 128 kbps line
# to our backup host. This one is slightly more complicated, but should do
# the trick.
#
# Privileges needed: SELECT, RELOAD, LOCK TABLES
# Notes:
# - If INFORMATION_SCHEMA.TABLES would work for InnoDB tables, we could query if the
# database has changed, and only backup if needed.
# * https://bugs.mysql.com/bug.php?id=2681
# Ability to determine when DB was last modified (generic method)
#
PATH=/bin:/usr/bin:/usr/local/bin:/opt/csw/bin:/usr/sfw/bin
HASH=sha1 # See 'openssl dgst -h' for possible values
# - We had to drop the sys.metrics view. The whole sys schema may not be needed at all.
# * https://mariadb.com/kb/en/sys-schema/
# * https://mariadb.com/kb/en/sys-schema/
# * https://forums.cpanel.net/threads/remove-sys-database-after-upgrade-to-from-mysql-5-7-to-mariadb-10-3.674045/
# > "MariaDB does not utilize the sys schema. If you upgrade from MySQL 5.7 to MariaDB, you must
# > manually remove the sys database, because it can cause unnecessary errors during certain check table calls."
#
#
PATH=/bin:/usr/bin:/usr/local/bin
RUNAS=mysql # Privileges needed: SELECT, RELOAD, LOCK TABLES
COMPRESS=zstd
LOGFILE=backup-mysql.log

# unset me!
# DEBUG=echo
Expand All @@ -25,103 +39,101 @@ if [ ! -d "$1" ]; then
exit 1
else
DIR="$1"
cd "$DIR" || exit 3
fi

# Use 'gdate' if available, 'date' otherwise
if [ "$(which gdate)" ]; then
DATE="gdate"
else
DATE="date"
# Don't run as root, but we need to own $DIR
if [ ! "$(whoami)" = "${RUNAS}" ]; then
echo "Please execute as user \"${RUNAS}\"!"
exit 2
fi

$DATE
# Be nice to others
renice 20 $$ > /dev/null

# The date should end up in a logfile
date >> "${LOGFILE}"

# We have checks too :-)
if [ "$2" = "-c" ]; then
for f in "${DIR}"/*.bz2; do
printf '..%s..' "Processing ${f} ($(stat -c 'scale=0; %s / 1024' "${f}" | bc -l)KB)..."
grep -q "$(bzip2 -dc "$f" | openssl ${HASH}) $(echo "${f}" | sed "s/.bz2$/.${HASH}/")"
if [ $? = 0 ]; then
for f in *.zst; do
printf "%s" "Processing \"$f\" ($(stat -c 'scale=0; %s / 1024' "$f" | bc -l) KB)..."
if grep -q "$(${COMPRESS} -qdc "$f" | sha1sum | awk '{print $1}')" "${f%%.zst}".sha1; then
echo "checksum OK"
else
echo "checksum FAILED"
fi
done
exit 0
exit $?
fi

BEGIN=$($DATE +%s)
for db in $(mysql --batch --skip-column-names -e 'show databases' | sort); do
printf '..%s..' "Backing up ${db}...."
# - Use multiple-row INSERT syntax that include several VALUES lists
# - Continue even if an SQL error occurs during a table dump
# - Flush the MySQL server log files before starting the dump
# - Send a FLUSH PRIVILEGES statement to the server after dumping the mysql database
# - Lock all tables to be dumped before dumping them
# - Dump binary columns using hexadecimal notation
# - Using --skip-dump-date (added in v5.0.52) so that the dump won't change unnecessarily.
# - Included stored routines
# - Include triggers for each dumped table
# DB credentials
OPTIONS="--user=backup --password=XXXX" # Can't we just use ~/.my.cnf?

# Main loop
BEGIN=$(date +%s)
for db in $(mysql ${OPTIONS} --batch --skip-column-names -e 'show databases' | sort); do
case "${db}" in
performance_schema|information_schema)
information_schema|performance_schema)
# Access denied for user 'root'@'localhost' to database 'information_schema' when using LOCK TABLES
# > mysqldump incorrectly tries to LOCK TABLES on the information_schema database.
# > https://bugs.mysql.com/bug.php?id=21527 (closed)
#
# > mysqldump can not dump INFORMATION_SCHEMA
# > https://bugs.mysql.com/bug.php?id=33762 (closed)
#
# > mysqldump: problems dumping INFORMATION_SCHEMA
# > https://bugs.mysql.com/bug.php?id=49633 (verified)
OPTIONS="--extended-insert --force --flush-logs --flush-privileges --skip-lock-tables --hex-blob --routines --triggers"
# http://bugs.mysql.com/bug.php?id=21527 (closed)
# http://bugs.mysql.com/bug.php?id=33762 (closed)
# http://bugs.mysql.com/bug.php?id=49633
# OPTIONS="${OPTIONS} --skip-lock-tables"
continue
;;

*)
OPTIONS="--extended-insert --force --flush-logs --flush-privileges --lock-tables --hex-blob --routines --triggers"
;;
# mysql)
# # - Skip mysql.event, http://bugs.mysql.com/bug.php?id=68376
# # - We used to add --skip-events b/c of http://bugs.debian.org/673572 - but this triggers #68376 again!
# OPTIONS="${OPTIONS} --ignore-table=mysql.event"
# ;;
esac

if [ -n "${DEBUG}" ]; then
${DEBUG} mysqldump "${OPTIONS}" "${db}" grep -Ev -- '^-- Dump completed on' "${DIR}/DB_${db}.sql.new"
else
${DEBUG} mysqldump "${OPTIONS}" "${db}" | grep -Ev -- '^-- Dump completed on' > "${DIR}/DB_${db}.sql.new"
fi
# Backup!
$DEBUG mysqldump ${OPTIONS} --lock-tables --skip-dump-date --result-file="DB_${db}.sql.new" --databases "${db}"

# We're comparing checksum rather than the whole dump, so that we
# can compress them afterwards and still be able to compare tomorrow's dump.
# We're comparing checksums rather than the whole dump, so that we can compress
# them afterwards and still be able to compare tomorrow's dump.
# - If a checksum file is present, create a new one and compare them
# - If no checksum file is present, create one
if [ -f "${DIR}/DB_${db}.sql.${HASH}" ]; then
if [ -n "${DEBUG}" ]; then
${DEBUG} openssl ${HASH} "${DIR}/DB_${db}.sql.new" sed 's/\.new$//' "${DIR}/DB_${db}.sql.new.${HASH}"
else
${DEBUG} openssl ${HASH} "${DIR}/DB_${db}.sql.new" | sed 's/\.new$//' > "${DIR}/DB_${db}.sql.new.${HASH}"
fi
H_OLD=$(awk '{print $NF}' "${DIR}/DB_${db}.sql.${HASH}" 2>/dev/null)
H_NEW=$(awk '{print $NF}' "${DIR}/DB_${db}.sql.new.${HASH}" 2>/dev/null)
if [ -f DB_"${db}".sql.sha1 ]; then
$DEBUG sha1sum DB_"${db}".sql.new > DB_"${db}".sql.new.sha1
sed 's/\.new$//' -i DB_"${db}".sql.new.sha1

H_OLD=$(awk '{print $1}' DB_"${db}".sql.sha1 2>/dev/null)
H_NEW=$(awk '{print $1}' DB_"${db}".sql.new.sha1 2>/dev/null)

# - If they are equal, delete our new one, otherwise update the old one
# If they are equal, delete our new one, otherwise update the old one
if [ "$H_OLD" = "$H_NEW" ]; then
echo "database ${db} has not changed, nothing to do."
${DEBUG} rm "${DIR}/DB_${db}.sql.new" "${DIR}/DB_${db}.sql.new.${HASH}"
echo "Database ${db} has not changed, nothing to do" >> "${LOGFILE}"
$DEBUG rm DB_"${db}".sql.new DB_"${db}".sql.new.sha1
else
echo "database ${db} changed!"
${DEBUG} mv "${DIR}/DB_${db}.sql.new.${HASH}" "${DIR}/DB_${db}.sql.${HASH}"
${DEBUG} mv "${DIR}/DB_${db}.sql.new" "${DIR}/DB_${db}.sql"
${DEBUG} pbzip2 -f "${DIR}/DB_${db}.sql"
echo "Database ${db} has changed, discarding the old dump." >> "${LOGFILE}"
$DEBUG mv -f DB_"${db}".sql.new.sha1 DB_"${db}".sql.sha1
$DEBUG mv -f DB_"${db}".sql.new DB_"${db}".sql
$DEBUG ${COMPRESS} --rm -9qf DB_"${db}".sql
fi
else
# - We have nothing to compare
echo "database ${db} must be new?"
${DEBUG} mv "${DIR}/DB_${db}.sql.new" "${DIR}/DB_${db}.sql"
if [ -n "${DEBUG}" ]; then
${DEBUG} openssl ${HASH} "${DIR}/DB_${db}.sql" "${DIR}/DB_${db}.sql.${HASH}"
else
${DEBUG} openssl ${HASH} "${DIR}/DB_${db}.sql" > "${DIR}/DB_${db}.sql.${HASH}"
fi
${DEBUG} pbzip2 -f "${DIR}/DB_${db}.sql"
# We have nothing to compare
echo "No checksum found for database ${db}." >> "${LOGFILE}"
$DEBUG mv -f DB_"${db}".sql.new DB_"${db}".sql
$DEBUG sha1sum DB_"${db}".sql > DB_"${db}".sql.sha1
$DEBUG ${COMPRESS} --rm -9qf DB_"${db}".sql
fi
${DEBUG}
done
END=$(${DATE} +%s)
echo "$0 finished after $(echo "${END}" - "${BEGIN}" | bc) seconds."
echo
END=$(date +%s)
echo "${0} finished after $(echo \( "${END}" - "${BEGIN}" \) / 60 | bc) minutes." >> "${LOGFILE}"
echo >> "${LOGFILE}"

### OLD ###
# printf "%s" "Backing up \"${db}\"...." >> "${LOGFILE}"
# - Use multiple-row INSERT syntax that include several VALUES lists
# - Continue even if an SQL error occurs during a table dump
# - Flush the MySQL server log files before starting the dump
# - Send a FLUSH PRIVILEGES statement to the server after dumping the mysql database
# - Dump binary columns using hexadecimal notation
# - Using --skip-dump-date (added in v5.0.52) so that the dump won't change unnecessarily.
# - Included stored routines
# - Include triggers for each dumped table
# OPTIONS="${OPTIONS} --extended-insert --force --flush-logs --flush-privileges --hex-blob --skip-dump-date --routines --triggers"

0 comments on commit d703a24

Please sign in to comment.