diff --git a/README.md b/README.md index ae16927..7bc5d2a 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,8 @@ -# Some GNU/Linux tools, for fun... +## Some GNU/Linux tools, for fun... -## bash/: +### bash -- trans.sh: a [linguee.com](https://linguee.com) based command-line translator. +- **trans.sh**: a [linguee.com](https://linguee.com) based command-line translator. + +- **sync.sh**: a rsync/ssh backup tool. + - **sync-conf-example.sh**: configuration example. diff --git a/bash/sync-conf-example.sh b/bash/sync-conf-example.sh new file mode 100644 index 0000000..f8c64a5 --- /dev/null +++ b/bash/sync-conf-example.sh @@ -0,0 +1,84 @@ +#!/bin/bash +# +# sync-conf-example.sh - a "sync.sh" configuration file example. +# +# (C) Bruno Raoult ("br"), 2007-2021 +# Licensed under the Mozilla Public License (MPL) version 2.0. +# Some rights reserved. See COPYING. +# +# You should have received a copy of the Mozilla Public License along with this +# program. If not, see +# +# SPDX-License-Identifier: MPL-2.0 +# +# USAGE: +# sync.sh -rfu /path/to/sync-conf-example.sh + +# full source path +SOURCEDIR=/example-srcdir +# server name. Could also be user@hostname +SERVER=backuphost +# full destination path on target machine (or relative to home directory) +DESTDIR=/mnt/array3+4/example-destdir + +# backups to keep +NYEARS=2 +NMONTHS=12 +NWEEKS=4 +NDAYS=7 + +# FILTER can be used to filter directories to include/exclude. See rsync(1) for +# details. +FILTER=--filter="dir-merge .rsync-filter-br" + +# other rsync options +RSYNCOPTS="" + +# functions run just before and after the rsync. Could be useful to create +# database dumps, etc... +# Warning: avoid using "cd", or be sure to come back to current dir +# before returning from functions + +# example below will create a dump +function beforesync() { + # next line may be removed if you do something. bash does not like empty + # functions + : + + # log is a sync.sh function. + log -s -t "calling user beforesync: mysql databases dumps..." + + datadir=$(mysql -sN -u root -e 'select @@datadir') + # log "mysql datadir=${datadir}" + rm -f "$datadir/$FILTERNAME" + databases=($(mysql -sN -u root -e "SHOW DATABASES;")) + + for db in "${databases[@]}" + do + # exclude database directory itself + echo "- /${db}/*" >> "$datadir/$FILTERNAME" + + log -n "${db}... " + case "$db" in + information_schema|performance_schema) + log "skipped." + ;; + *) + log -n "dumping to ${datadir}${db}.sql... " + mysqldump --user=root --routines "$db" > "$datadir/$db.sql" + # log -n "compressing... " + gzip "$datadir/$db.sql" + log "done." + esac + done + # log "filtername contains:" + # cat ${datadir}/${FILTERNAME} +} + +function aftersync() { + # next line may be removed if you do something. bash does not like empty + # functions + : + # we may remove the dump here... + log -s -t "calling user aftersync" +} diff --git a/bash/sync.sh b/bash/sync.sh new file mode 100755 index 0000000..f555295 --- /dev/null +++ b/bash/sync.sh @@ -0,0 +1,458 @@ +#!/bin/bash +# +# sync.sh - a backup utility using ssh/rsync facilities. +# +# (C) Bruno Raoult ("br"), 2007-2021 +# Licensed under the Mozilla Public License (MPL) version 2.0. +# Some rights reserved. See COPYING. +# +# You should have received a copy of the Mozilla Public License along with this +# program. If not, see +# +# SPDX-License-Identifier: MPL-2.0 +# +# NAME +# sync.sh - a backup utility using ssh/rsync facilities. +# +# SYNOPSIS +# sync.sh [-ymwdznt] CONFIG +# +# DESCRIPTION +# Performs a backup to a local or remote destination, keeping different +# versions (daily, weekly, monthly, yearly). All options can be set in +# CONFIG file, which is mandatory. +# +# OPTIONS +# -y, -m, -w, -d +# yearly/monthly/weekly/daily backup. If none of these options is set, +# and none of the corresponding variables set to "y" in configuration +# file, the script will determine itself what should be done, +# depending on the current day or date: daily backup every day, +# weekly every sunday, monthly every first day of month, and yearly +# every Jan 1st. +# -n +# do not send mail report (which is the default if MAILTO environment +# is set). Practically, this option only unsets MAILTO. +# -f +# filter some rsync output, such as hard and soft links, dirs, etc. +# -r +# resume an interrupted transfer (rsync --partial option). It should +# be safe to use this option, as it has no effect in usual case. +# -z +# enable rsync compression. Should be used when the transport is more +# expensive than CPU (typically slow connections). +# -u +# will use numeric IDs (uid and gid) instead of usernames/groupnames. +# This could be preferable in case of backup, to avoid any issue when +# getting back the file (for instance via a mount). +# -v +# adds sub-tasks information, with timestamps. +# -D +# by default, this script will re-route all outputs (stdout and stderr) +# to a temporary file after basic initialization (mainly options +# checks and configuration file evaluation), so that we can format +# the output before displaying or mailing it. +# This option disables this redirection. It is useful (together with +# bash's -x option) when some errors are difficult to track. +# +# GENERAL +# Ensure your ssh setup is correct: You must be able to ssh the target +# machine without password. +# You should avoid modifying variables in this script. +# Instead, use the configuration file, whenever possible. +# +# All options can be set in configuration file (which will always have +# the "last word"). Please check the "getopts" section in the code below. +# +# By default, the output is displayed (or mailed) when the script exits. +# The -D option allows to get real-time output. +# +# CONFIGURATION FILE +# TODO: Write documentation. See example (sync-conf-example.sh). +# +# BUGS +# Many. +# This was written for a "terastation" NAS server, which is a kind of +# light Linux running busybox and an old version of rsync. Therefore, +# some useful rsync options were not used (such as server logs, etc). +# This script has been tested on Linux/MacOS clients, with bash versions +# 3.X and 4.X. Associative arrays cannot be used in bash < V4. Therefore +# the code is not as efficient as it could be. +# +# TODO +# - allow a reverse backup (from remote to local). Would be useful for +# non-rooted Android devices (rooted ones could probably run the script +# by themselves - to be confirmed. +# - add more logs options (maybe with a mask?) +# - replace y/n values with empty/not-empty. A step to avoid config file +# - set default compress value on local/non-local. Step to avoid config +# file +# - replace y/m/w/d with numerical values ? Step to avoid config file +# - manage more errors (instead of relying on traps) +# - the deletion of oldest backup directories takes ages. This could be +# avoided (for example we could move them to a temp dir and remove it +# in background). +# - configuration file could be avoided by adding a few options, such +# as source and destination directories, and maybe also an rsync +# exclude pattern (or more generally some more options to rsync). +# - rewrite exit_handler which is really ugly. +# - instead of filtering output, it could be better to create the +# destination from the previous backup, and perform a usual backup +# on this new directory. This should be easy, but my guess is that +# it could be slower (1 first pass on server is added before the +# normal backup. +# - replace getopts(1) to have a better options parsing. GNU's getopt(1) +# could be an option, but not available everywhere (for example on +# MacOS). Likely impossible to keep this script portable. +# +# AUTHOR +# Bruno Raoult. +# +# + +######################### options default values. +# These ones can be set by command-line. +# They can also be overwritten in configuration file (prefered option). +YEARLY=n # (-y) yearly backup (y/n) +MONTHLY=n # (-m) monthly backup (y/n) +WEEKLY=n # (-w) weekly backup (y/n) +DAILY=n # (-d) daily backup (y/n) +FILTERLNK=n # (-f) rsync logs filter: links, dirs... (y/n) +RESUME=n # (-r) resume backup (y/n) +COMPRESS="" # (-z) rsync compression +NUMID="" # (-u) use numeric IDs +#VERBOSE=0 # TODO: (-v) logs level (0/1) +DEBUG=n # (-D) debug: no I/O redirect (y/n) +MAILTO=${MAILTO:-""} # (-n) mail recipient. -n sets it to "" + +# options not available on command line, but settable in config file. +NYEARS=3 # keep # years (int) +NMONTHS=12 # keep # months (int) +NWEEKS=4 # keep # weeks (int) +NDAYS=7 # keep # days (int) +RSYNCOPTS="" # other rsync options +SOURCEDIR="." # source dir +DESTDIR="." # destination dir +MODIFYWINDOW=1 # accuracy for mod time comparison + +# these 2 functions can be overwritten in data file, to run specific actions +# just before and after the actual sync +function beforesync () { + log calling default beforesync... +} +function aftersync () { + log calling default aftersync... +} + +# internal variables, cannot (and *should not*) be changed unless you +# understand exactly what you do. +# Some variables were moved into the code (example: in the log() function), +# for practical reasons, the absence of associative arrays being one of them. +LOCKED=n # indicates if we created lock file. +SUBJECT="${0##*/} ${*##*/}" # mail subject (removes paths) +ERROR=0 # set by error_handler when called +STARTTIME=$(date +%s) # time since epoch in seconds + +usage () { + echo "usage: ${0##*/} [-ymwdnfrzuD]" config-file + exit 1 +} + +# command-line options parsing. +OPTIND=1 +while getopts ymwdfrnzuD todo +do + case "${todo}" in + y) YEARLY=y;; + m) MONTHLY=y;; + w) WEEKLY=y;; + d) DAILY=y;; + f) FILTERLNK=y;; + r) RESUME=y;; + n) MAILTO="";; + z) COMPRESS=-y;; # rsync compression. Depends on net/CPU perfs + u) NUMID="--numeric-ids";; + D) DEBUG=y;; + *) usage;; + esac +done +# Now check remaining argument (configuration file), which should be unique, +# and read the file. +shift $((OPTIND - 1)) +(( $# != 1 )) && usage +CONFIG="$1" +if [[ ! -f "$CONFIG" ]]; then + echo "No $CONFIG file." + usage +fi +source "$CONFIG" + +# we set backups to be done if none has been set yet (i.e. none is "y"). +# Note: we use the form +%-d to avoid zero padding. +# for bash, starting with 0 => octal => 08 is invalid +if ! [[ "${DAILY}${WEEKLY}${MONTHLY}${YEARLY}" =~ .*y.* ]]; then + (( $(date +%u) == 7 )) && WEEKLY=y + (( $(date +%-d) == 1 )) && MONTHLY=y + (( $(date +%-d) == 1 && $(date +%-m) == 1 )) && YEARLY=y + DAILY=y +fi + +# set final variables values +LOCKFILE=".sync-${SERVER}-${CONFIG##*/}.lock" + +# log function +# parameters: +# -l, -s: long, or short prefix (default: none). Last one is used. +# -t: timestamp +# -n: no newline +log() { + timestr="" + prefix="" + opt=y + newline=y + while [[ $opt = y ]]; do + case $1 in + -l) prefix=$(printf "*%.s" {1..30});; + -s) prefix=$(printf "*%.s" {1..5});; + -n) newline=n;; + -t) timestr=$(date "+%F %T%z - ");; + *) opt=n;; + esac + [[ $opt = y ]] && shift + done + [[ $prefix != "" ]] && printf "%s " "$prefix" + printf "%s%s" "$timestr" "$*" + [[ $newline = y ]] && echo + return 0 +} + +# After these basic initializations, errors will be managed by the +# following handler. It is better to do this before the redirections below. +error_handler() { + ERROR=$2 + echo "FATAL: Error line $1, exit code $2. Aborting." + exit "$ERROR" +} + +trap 'error_handler $LINENO $?' ERR SIGHUP SIGINT SIGTERM + +exit_handler() { + # we dont need lock file anymore (another backup could start from now). + log "exit_handler LOCKED=$LOCKED" + #if [[ "$LOCKED" = y ]]; then + rm --dir --verbose "${LOCKFILE}" + #fi + + if (( ERROR == 0 )); then + SUBJECT="Successful $SUBJECT" + else + SUBJECT="Failure in $SUBJECT" + fi + + log -l -t "Ending backup." + + if [[ $DEBUG = n ]]; then + # restore stdout (not necessary), set temp file as stdin, close fd 3. + # remove temp file (as still opened by stdin, will still be readable). + exec 1<&3 3>&- 0<"$TMPFILE" + rm -f "$TMPFILE" + else + echo 222 + exec 0<<<"" # force empty input for the following + fi + + # Warning: no logs allowed here (before next braces), as stdout is no + # more handled the final way. + { + SECS=$(($(date +%s)-STARTTIME)) + + # we write these logs here so that they are on top if no DEBUG. + log "Exit code: $ERROR" + log "$(printf "Elapsed time: $SECS seconds (%d:%02d:%02d)\n\n" \ + $((SECS/3600)) $((SECS%3600/60)) $((SECS%60)))" + + if [[ -n $FILTERLNK ]]; then + grep -vE "^(hf|cd|cL)[ \+]" ; + else + cat ; + fi + } | + { + if [[ -n $MAILTO ]]; then + mail -s "${SUBJECT}" "${MAILTO}"; + else + cat; + fi + } +} +trap 'exit_handler' EXIT + +# standard descriptors redirection. +# if not DEBUG, save stdout as fd 3, and redirect stdout to temp file. +# in case of DEBUG, we could close stdin, but there could be side effects, +# such as ^C handling, etc... So we keep the keyboard available. +if [[ $DEBUG = n ]]; then + TMPFILE=$(mktemp /tmp/sync-log.XXXXXX) + exec 3<&1 >"${TMPFILE}" # no more output on screen from now. +fi +exec 2>&1 +if [[ ! -d $SOURCEDIR ]]; then + log -s "Source directory (\"${SOURCEDIR}\") is not a valid directory." + error_handler $LINENO 1 +fi +cd ${SOURCEDIR} + +# prepare list of backups, such as "daily 7 weekly 4", etc... +TODO=() +[[ $DAILY = y && $NDAYS -gt 0 ]] && TODO+=(daily "$NDAYS") +[[ $WEEKLY = y && $NWEEKS -gt 0 ]] && TODO+=(weekly "$NWEEKS") +[[ $MONTHLY = y && $NMONTHS -gt 0 ]] && TODO+=(monthly "$NMONTHS") +[[ $YEARLY = y && $NYEARS -gt 0 ]] && TODO+=(yearly "$NYEARS") + +log -l -t "Starting backup." +log "Config : ${CONFIG}" +log "Src dir: ${SOURCEDIR}" +log "Dst dir: ${SERVER}:${DESTDIR}" +log "Actions: ${TODO[@]}" + +# select handling depending on local or networked target (ssh or not). +if [[ $SERVER = local ]]; then # local backup + DOIT="" + DEST=${DESTDIR} +else # remote backup + DOIT="ssh ${SERVER}" + DEST="${SERVER}:${DESTDIR}" +fi + +# commands and specific variables. +EXIST="${DOIT} test -e" +MOVE="${DOIT} mv" +REMOVE="${DOIT} rm -rf" +COPYHARD="${DOIT} rsync -ar" + +# prints out and run a command. Used mainly for rsync debug. +echorun () { + log "$@" + "$@" + return $? +} + +# rotate files. arguments are a string and a number. For instance $1=weekly, +# $2=3. +# we first build a list from $2 to zero, with 2 padded digits: 03 02 01 00 +# then we remove $1-03, and move $1-02 to $1-03, $1-01 to $1-02, etc... +rotate-files () { + files=( $(seq -f "${DESTDIR}/${1}-%02g" "${2}" -1 0) ) + log -s -t -n "${files[0]##*/} deletion... " + status=0 + ${REMOVE} "${files[0]}" || status=$? + if (( status != 0 )); then + # this should never happen. + # But I saw this event in case of a file system corruption. Better + # is to stop immediately instead of accepting strange side effects. + if ${EXIST} "${files[0]}" ; then + log -s "Could not remove ${files[0]}. This SHOULD NOT happen." + error_handler $LINENO ${status} + fi + fi + log "done." + + log -s -t -n "${1} rotation... " + while (( ${#files[@]} > 1 )) + do + if ${EXIST} "${files[1]}" ; then + [[ $DEBUG = y ]] && log -n "${files[1]:(-2)} " + ${MOVE} "${files[1]}" "${files[0]}" + fi + unset files[0] # shift and pack array + files=( "${files[@]}" ) + done + log "done." + return 0 +} + +# create lock file +if ! mkdir "${LOCKFILE}"; then + log -s "Cannot create lock file. Exiting." + error_handler $LINENO 1 +fi +LOCKED=y + +# main loop. +while [[ ${TODO[0]} != "" ]] +do + # these variables to make the script easier to read. + todo="${TODO[0]}" # daily, weekly, etc... + keep="${TODO[1]}" # # of versions to keep for $todo set + todop="${DESTDIR}/${todo}" # prefix for backup (e.g. "destdir/daily") + tdest="${todop}-00" # target full path (e.g. "destdir/daily-00") + ldest="${DESTDIR}/daily-01" # link-dest dir (always daily-01) + + log -l -t "${todo} backup..." + + # check if target (XX-00) directory exists. If yes, we must have the + # resume option to go on. + if ${EXIST} "${tdest}"; then + if [[ $RESUME = n ]]; then + log -s "${tdest} already exists, and no \"resume\" option." + error_handler $LINENO 1 + fi + log -s "Warning: Resuming ${todo} partial backup.". + fi + + # daily backup. + # as we already checked the existence of the -00 file, we can keep + # the --partial option safely. + if [[ $todo = daily ]]; then + beforesync # script to run before the sync + log -s -t "rsync copy..." + # Do the sync. Run in a subshell to avoid the error handling, as + # we want to ignore "acceptable" errors, such as: + # - "vanished file" (exit code 24). + # - others? + status=0 + echorun rsync \ + -aHixv \ + "${FILTER}" \ + ${RSYNCOPTS} \ + ${COMPRESS} \ + ${NUMID} \ + --delete \ + --delete-during \ + --delete-excluded \ + --modify-window=${MODIFYWINDOW} \ + --partial \ + --link-dest="${ldest}" \ + . \ + "${DEST}/daily-00" || status=$? + # error 24 is "vanished source file", and should be ignored. + if (( status != 24 && status != 0)); then + error_handler $LINENO $status + fi + aftersync # script to run after the sync + else # non-daily case. + status=0 + ${EXIST} "${ldest}" || status=$? + if ((status == 0 )); then + log -s -t "${tdest} update..." + ${COPYHARD} --link-dest="${ldest}" "${ldest}/" "${tdest}" + else + log "No ${ldest} directory. Skipping ${todo} backup." + fi + fi + rotate-files "${todo}" "${keep}" + + # shift and pack TODO array + unset 'TODO[0]' 'TODO[1]' + TODO=( "${TODO[@]}" ) +done + +exit 0 + +# Indent style for emacs +# Local Variables: +# sh-basic-offset: 4 +# sh-indentation: 4 +# indent-tabs-mode: nil +# comment-column: 32 +# End: diff --git a/bash/trans.sh b/bash/trans.sh index c574fe4..7df7218 100755 --- a/bash/trans.sh +++ b/bash/trans.sh @@ -2,7 +2,7 @@ # # trans.sh - Translate words using linguee.com. # -# (C) Bruno Raoult, 2021 +# (C) Bruno Raoult ("br"), 2021 # Licensed under the Mozilla Public License (MPL) version 2.0. # Some rights reserved. See COPYING. #