btrfs-sync

#!/bin/bash

#
# Simple script that synchronizes BTRFS snapshots locally.
# Features compression, retention policy and automatic incremental sync
#

set -e
set -o pipefail
set -o errtrace

print_usage() {
  echo "Usage:
  $BIN [options] <src> [<src>...] <dir>

  -k|--keep NUM     keep only last <NUM> sync'ed snapshots
  -d|--delete       delete snapshots in <dst> that don't exist in <src>
  -q|--quiet        don't display progress
  -v|--verbose      display more information
  -h|--help         show usage

<src> can either be a single snapshot, or a folder containing snapshots
"
}

echov() { if [[ "$VERBOSE" == 1 ]]; then echo "$@"; fi }

#----------------------------------------------------------------------------------------------------------

# preliminary checks
BIN="${0##*/}"
[[ $# -lt 2      ]] && { print_usage                                ; exit 1; }
[[ ${EUID} -ne 0 ]] && { echo "Must be run as root. Try 'sudo $BIN'"; exit 1; }

# parse arguments
KEEP=0

OPTS=$( getopt -o hqzZk:p:dv -l quiet -l help -l keep: -l delete -l verbose -- "$@" 2>/dev/null )
[[ $? -ne 0 ]] && { echo "error parsing arguments"; exit 1; }
eval set -- "$OPTS"

while true; do
  case "$1" in
    -h|--help   ) print_usage; exit  0 ;;
    -q|--quiet  ) QUIET=1    ; shift 1 ;;
    -d|--delete ) DELETE=1   ; shift 1 ;;
    -k|--keep   ) KEEP=$2    ; shift 2 ;;
    -v|--verbose) VERBOSE=1  ; shift 1 ;;
    --)                shift;  break   ;;
  esac
done

# detect src and dst arguments
SRC=( "${@:1:$#-1}" )
DST="${@: -1}"

test -x "$SRC" &>/dev/null || {
  echo "Access error. Do you have adequate permissions for $SRC?"
  exit 1
}

test -x "$DST" &>/dev/null || {
  echo "Access error. Do you have adequate permissions for $DST?"
  exit 1
}

#----------------------------------------------------------------------------------------------------------

# more checks

## don't overlap
if pgrep -F /run/btrfs-sync.pid &>/dev/null; then
  echo "$BIN is already running"
  exit 1
fi
echo $$ > /run/btrfs-sync.pid

## src checks
echov "* Check source"
SRCS=()
SRCS_BASE=()
for s in "${SRC[@]}"; do
  src="$(realpath "$s")"
  if ! test -e "$src"; then
    echo "$s not found"
    exit 1
  fi
  # check if the src is a read-only subvolume
  if btrfs subvolume show "$src" &>/dev/null && [[ "$(btrfs property get -ts "$src")" == "ro=true" ]]; then
    SRCS+=("$src")
    SRCS_BASE+=("$src")
  else
    for dir in $( find "$src" -maxdepth 2 -type d ); do
      # check if the src is a read-only subvolume
      if btrfs subvolume show "$dir" &>/dev/null && [[ "$(btrfs property get -ts "$dir")" == "ro=true" ]]; then
        SRCS+=("$dir")
        SRCS_BASE+=("$src")
      fi
    done
  fi
done
if [[ ${#SRCS[@]} -eq 0 ]]; then
  echo "no BTRFS subvolumes found"
  exit 1
fi

## use 'pv' command if available
PV=( pv -F"time elapsed [%t] | rate %r | total size [%b]" )
if [[ "$QUIET" == "1" ]]; then
  PV=( cat )
else
  if ! type pv &>/dev/null; then
    echo "INFO: install the 'pv' package in order to get a progress indicator"
    PV=( cat )
  fi
fi

#----------------------------------------------------------------------------------------------------------

# sync snapshots

get_dst_snapshots() {      # sets DSTS DST_UUIDS
  local DST="$1"
  DSTS=()
  DST_UUIDS=()
  for dir in $( find "$DST" -maxdepth 2 -type d ); do
    if btrfs subvolume show "$dir" &>/dev/null; then
      local UUID=$( btrfs subvolume show "$dir" 2>/dev/null | grep 'Received UUID' | awk '{ print $3 }' )
      if [[ "$UUID" != "-" ]] && [[ "$UUID" != "" ]]; then
        DSTS+=("$dir")
        DST_UUIDS+=("$UUID")
      fi
    fi
  done
}

choose_seed() {      # sets SEED
  local SRC="$1"
  local SRC_BASE="$2"

  SEED="$SEED_NEXT"
  if [[ "$SEED" == "" ]]; then
    # try to get most recent src snapshot that exists in dst to use as a seed
    local RXID_CALCULATED=0
    declare -A PATH_RXID DATE_RXID SHOWP RXIDP DATEP
    local LIST="$( btrfs subvolume list -su "$SRC" )"
    local SEED_CANDIDATES=()
    for id in "${DST_UUIDS[@]}"; do
      # try to match by UUID
      local PATH_=$( awk "{ if ( \$14 == \"$id\" ) print \$16       }" <<<"$LIST" )
      local DATE=$(  awk "{ if ( \$14 == \"$id\" ) print \$11, \$12 }" <<<"$LIST" )

      # try to match by received UUID, only if necessary
      if [[ "$PATH_" == "" ]]; then
        if [[ "$RXID_CALCULATED" == "0" ]]; then # create table during the first iteration if needed
          local PATHS=( $( btrfs subvolume list -u "$SRC" | awk '{ print $11 }' ) )
          for p in "${PATHS[@]}"; do
            SHOWP="$( btrfs subvolume show "$( dirname "$SRC" )/$( basename "$p" )" 2>/dev/null )"
            RXIDP="$( grep 'Received UUID' <<<"$SHOWP" | awk '{ print $3     }' )"
            DATEP="$( grep 'Creation time' <<<"$SHOWP" | awk '{ print $3, $4 }' )"
            [[ "$RXIDP" == "" ]] && continue
            PATH_RXID["$RXIDP"]="$p"
            DATE_RXID["$RXIDP"]="$DATEP"
          done
          RXID_CALCULATED=1
        fi
        PATH_="${PATH_RXID["$id"]}"
        DATE="${DATE_RXID["$id"]}"
      fi

      if [[ "$PATH_" == "" ]] || [[ "$PATH_" == "$( basename "$SRC" )" ]]; then
        continue
      fi

      # if the path does not exist, it is likely relative to the root subvolume
      # rather than the mounted subvolume
      if ! test -d "$PATH_" && mountpoint -q "$SRC_BASE"; then
        local SRC_BASE_SUBVOL=$(findmnt -n -o OPTIONS "$SRC_BASE" | tr "," "\n" | grep "subvol=" | awk -F '=' '{ print $2 }')
        # drop the leading slash
        SRC_BASE_SUBVOL="${SRC_BASE_SUBVOL#/}"
        # replace the prefix in $PATH_
        if [[ "$PATH_" =~ "$SRC_BASE_SUBVOL"* ]]; then
          PATH_="${PATH_#${SRC_BASE_SUBVOL}}"
          PATH_="$SRC_BASE/$PATH_"
        fi
      fi

      local SECS=$( date -d "$DATE" +"%s" )
      SEED_CANDIDATES+=("$SECS|$PATH_")
    done
    SEED=$(IFS=$'\n' echo "${SEED_CANDIDATES[@]}" | sort -V | tail -1 | cut -f2 -d'|')
  fi
}

exists_at_dst() {
  local SHOW="$( btrfs subvolume show "$SRC" )"

  local SRC_UUID="$( grep 'UUID:' <<< "$SHOW" | head -1 | awk '{ print $2 }' )"
  grep -q "$SRC_UUID" <<<"${DST_UUIDS[@]}" && return 0;

  local SRC_RXID="$( grep 'Received UUID' <<< "$SHOW"   | awk '{ print $3 }' )"
  grep -q "^-$"       <<<"$SRC_RXID"       && return 1;
  grep -q "$SRC_RXID" <<<"${DST_UUIDS[@]}" && return 0;

  return 1
}

## sync incrementally
sync_snapshot() {
  local SRC="$1"
  local SRC_BASE="$2"
  if ! test -d "$SRC" || ! test -d "$SRC_BASE"; then
    return
  fi

  if exists_at_dst "$SRC"; then
    echov "* Skip existing '$SRC'"
    return 0
  fi

  choose_seed "$SRC" "$SRC_BASE"  # sets SEED
  echo "SEED=$SEED"

  # incremental sync argument
  if [[ "$SEED" != "" ]]; then
    if test -d "$SEED"; then
      # Sends the difference between the new snapshot and old snapshot to the
      # backup location. Using the -c flag instead of -p tells it that there
      # is an identical subvolume to the old snapshot at the receiving
      # location where it can get its data. This helps speed up the transfer.
      local SEED_ARG=( -c "$SEED" )
    else
      echo "INFO: couldn't find $SEED. Non-incremental mode"
    fi
  fi

  # destination path where the subvolume will be sent
  local DST_SUBVOL="$DST/$( realpath --relative-to "$SRC_BASE" "$SRC" )"
  if test -d "$DST_SUBVOL"; then
    echo "ERROR: destination directory $DST_SUBVOL already exists, but was not detected as a Btrfs subvolume." >&2
    return 1
  fi

  # create the parent directory at destination
  mkdir -p "$(dirname "$DST_SUBVOL")"

  # print info
  echo -n "* Synchronizing '$SRC' to '$DST_SUBVOL'"
  if [[ "$SEED" != "" ]]; then
    echov -n " using seed '$SEED'"
  fi
  echo "..."

  # do it
  btrfs send -q "${SEED_ARG[@]}" "$SRC" \
    | "${PV[@]}" \
    | btrfs receive "$(dirname "$DST_SUBVOL")" 2>&1 \
    | (grep -v -e'^At subvol ' -e'^At snapshot ' || true) \
    || {
      btrfs subvolume delete "$DST_SUBVOL" 2>/dev/null
      return 1;
    }

  # update DST list
  DSTS+=("$DST_SUBVOL")
  DST_UUIDS+=("$SRC_UUID")
  SEED_NEXT="$SRC"
}

#----------------------------------------------------------------------------------------------------------

# sync all snapshots found in src
echov "* Check destination"
get_dst_snapshots "$DST" # sets DSTS DST_UUIDS
for (( i=0; i<"${#SRCS[@]}"; i++ )); do
  src="${SRCS[$i]}"
  src_base="${SRCS_BASE[$i]}"
  sync_snapshot "$src" "$src_base" && RET=0 || RET=1
#  for i in 1 2; do
#    [[ "$RET" != "1" ]] && break
#    echo "* Retrying '$src'..."
#    sync_snapshot "$src" && RET=0 || RET=1
#  done
  if [[ "$RET" == "1" ]]; then
    echo "Abort"
    exit 1
  fi
done

#----------------------------------------------------------------------------------------------------------

# retention policy
if [[ "$KEEP" != 0 ]] && [[ ${#DSTS[@]} -gt $KEEP ]]; then
  echo "* Pruning old snapshots..."
  for (( i=0; i < $(( ${#DSTS[@]} - KEEP )); i++ )); do
    PRUNE_LIST+=( "${DSTS[$i]}" )
  done
  btrfs subvolume delete "${PRUNE_LIST[@]}"
fi

# delete flag
if [[ "$DELETE" == 1 ]]; then
  for dst in "${DSTS[@]}"; do
    FOUND=0
#    for src in "${SRCS[@]}"; do
for (( i=0; i<"${#SRCS[@]}"; i++ )); do
  src="${SRCS[$i]}"
  echo "checking $src"
      if [[ "$( basename $src )" == "$( basename $dst )" ]]; then
        FOUND=1
        break
      fi
    done
    if [[ "$FOUND" == 0 ]]; then
      DEL_LIST+=( "$dst" )
    fi
  done
  if [[ "$DEL_LIST" != "" ]]; then
    echo "* Deleting non existent snapshots..."
    btrfs subvolume delete "${DEL_LIST[@]}"
  fi
fi