zfs_sync_vol

#!/bin/sh -f
#
# Helper to migrate (and maintain) a copy of a ZFS volume
# through ssh if needed (mbuffer for data stream if wanted)
#
# TODO:
#  * suppress new excludes on dest
#  * transmit new non-excludes before
#
# TOTEST:
#  * restore
#  * pre-flight checks on mountpoints (check source and already corrected dest)

PATH=/sbin:/bin:/usr/sbin:/usr/bin

# VARIABLES that change things: (no options for those ones)
# - CHANGEDSTHOST <oldname> : will use oldname to search snapshots
#   (and clean them)
##################
# default config #
##################
INCOPT=${INCOPT:-"-i"}
SENDOPT=${SENDOPT:-""}
RECVOPT=${RECVOPT:-""}
JUSTFORFUN=${JUSTFORFUN:-""}
MYNAME=${MYNAME:-$(hostname -s)}
VERBOSE=${VERBOSE:-0}
SSH_CIPHERS=${SSH_CIPHERS:-""}
PERSISTSSH=${PERSISTSSH:-""}

SRCZFSLIST=${TMPDIR:-/var/tmp}/srczfslist.$$
DSTZFSLIST=${TMPDIR:-/var/tmp}/dstzfslist.$$

usage() {
  echo "# $0 [-k ssh_key] [-R [-x <file>]] [-B] [-u] [-U] [-I [-m <strftime format>]]"
  echo "         [-t <command>] [-o \"transport send opts\"] [-O \"transport recv opts\"]"
  echo "         [-b <snapshot>] [-s] [-S] <SRCVOL>[@host] <DSTVOL>[@host]"
  echo "      -k ssh_key: use this ssk private key for remote commands - MANDATORY if opearating remotly"
  echo "      -R: use 'zfs send -R' and snapshot -r ('full replication stream', **see zfs(1M) !**)"
  echo "      -r: recurse"
  echo "      -x <file>: exclude zfs volumes matching <file> (with -R or -r)"
  echo "      -B: Rollback to last common snapshot (zfs receive -F)"
  echo "      -u: do not mount received volume(s) on destination (usefull for 'full replication stream' backups !)"
  echo "      -U: safe mountpoints (backed up to orig:mountpoint if outside of destination"
  echo "      -z: replace safe mountpoints (using orig:mountpoint if exists)"
  echo "      -C: create missing destination volume (or let 'zfs receive -F' create it)"
  echo "      -c /mountpoint: specify mountpoint for created volume"
  echo "      -I: use 'zfs send -I' instead of 'send -i' (sends all intermediary snapshots - zfs(1M))"
  echo "          WARNING: if source has 'daily.0' names (REnamed snapshots)"
  echo "          you HAVE to use -R (AND understand what it does)"
  echo "      -m cmd: strftime format of a snapshot to be created before sync (to be used with -I)"
  echo "          snapshot will be destroyed on source after sync"
  echo "      -t <command>: use 'command' to send data between hosts," 
  echo "          otherwise ssh is used (CPU/speed vs security)"
  echo "      TODO: make this work without terminal (eg: cron) \!\!\!"
  echo "      -o \"transport opts\": specify options for transport server - (source)"
  echo "      -O \"transport opts\": specify options for transport client - (destination)"
  echo "      -p \"use ssh's ControlMaster feature to reduce ssh connections\""
  echo "      -s: avoid deleting old sync snapshots on source after incremental transfer"
  echo "      -S: avoid deleting old sync snapshots on destination after incremental transfer"
  echo "      -T: use a temporary 'known_hosts' and skip ssh key check (restauration case)"
  echo "      -b SNAP: specify the snapshot to be send instead of creating a new one (add -TBzs to restore)"
  echo "      -l (facility|none): syslog with defined facility instead on user (local0, ...)"
  echo "      -H SHORTNAME override MYNAME (defaults to $(uname -s))"
  echo "      -j: relax source snapshot regex (in case dst/src have changed, or 'After-Murphy' emergency)"
  echo "      -v: verbose (-vv to debug)"
  echo "      -n: do not launch write command for real (display them)"
  echo
  echo " <SRCVOL> can be given as its mountpoint, as long as it's mounted on source"
  echo " If there is no volume in <DSTVOL>, the volume name <SRCVOL> will be used as destination volume too"
  echo " If you use a distant destination, destination host must be able to resolve and connect $MYNAME"
  echo "   (or use -H hostname to specify this host's name for him)"
  exit 0
}

shortusage() {
  echo "usage: $0 [OPTIONS] [-k ssh_key] <SRCVOL>[@host] <DSTVOL>[@host]"
  echo "  $0 -k /root/.ssh/id_rsa_zfs zpool/myvol zpool/hisvol@destination.example.com"
  echo "      will do a single volume sync (incremental if not the first)"
  echo "  $0 -k /root/.ssh/id_rsa_zfs -R -uU -I -m 'GMT-%Y%m%d-%H%M%S' zpool@myserver zdata/sav/myserver"
  echo "      will make a full recursive (-R) backup of myserver with history snapshots (-I -m)"
  echo "      and will protect the backup machine mountpoints (disable mountpoints outside /sav/myserver)"
  echo "  $0 -k /root/.ssh/id_rsa_restore -T -RBsz -b GMT-20140115-08h15m61s zdata/sav/myserver@backuphost zroot"
  echo "      will restore backup from backuphost:zdata/sav/myserver@GMT-20140115-08h15m61s to zroot pool"
  echo "      (for use in mfsbsd live-system)"
  echo "  $0 -h for full doc"
  exit 0
}

ORIGARGS=$@
while getopts IRrk:t:sSo:pO:H:hCc:nuUzTb:ax:Bm:l:jv option
do
  case $option in
    I)
      INCOPT="-I"
    ;;
    R)
      SENDOPT="-R"
      GRECURSE="FULL"
#      DOROLLBACK="LOOSELY"
    ;;
    r)
      GRECURSE="MANUAL"
    ;;
    B)
      RECVOPT='-F'
      #DOROLLBACK='YES_ROLLBACK'
      #ROLLBACKOPT='-Rf'
    ;;
    u)
      NODSTMOUNT="YES_NODSTMOUNTS"
    ;;
    U)
      CORRECTMOUNTS="YES_CORRECTMOUNTS"
    ;;
    C)
      CREATEDEST="YES_CREATEDEST"
    ;;
    c)
      CREATEMOUNTPOINT=$OPTARG
    ;;
    k)
      SSHKEY=$OPTARG
    ;;
    t)
      TRANSPORT_CMD=$OPTARG
    ;;
    T)
      TEMPSSH="YES_TEMPSSH"
    ;;
    s)
      KEEPSRCSNAPS="YES_KEEPSRCSNAPS"
    ;;
    S)
      KEEPDSTSNAPS="YES_KEEPDSTSNAPS"
    ;;
    m)
      SRCPRESNAPFORMAT=$OPTARG
    ;;
    o)
      TRANSPORT_SRC_OPTS=$OPTARG
    ;;
    p)
      PERSISTSSH="YES"
    ;;
    O)
      TRANSPORT_DST_OPTS=$OPTARG
    ;;
    H)
      MYNAME=$OPTARG
    ;;
    n)
      JUSTFORFUN="YES_JUSTFORFUN"
    ;;
    x)
      if [ -f $OPTARG ]; then
        EXCLUDEFILE=$OPTARG
      else
        echo "-x $OPTARG but $OPTARG doen't exist ... ignoring" >&2
      fi
    ;;
    b)
      NEWSNAP=$OPTARG
    ;;
    z)
      DSTREMOUNTORIG="YES_DSTREMOUNTORIG"
    ;;
    l)
      SYSLOG_ID=$OPTARG
    ;;
    j)
      RELAXSNAPREGEX="YES_RELAXSNAPREGEX"
      KEEPSRCSNAPS="YES_KEEPSRCSNAPS"
    ;;
    v)
      VERBOSE=$(( $VERBOSE + 1 ))
      if [ $VERBOSE -gt 2 ]; then
        GROSDEBUG="YES_GROSDEBUG"
      fi
    ;;
    h)
      usage
    ;;
    *)
      shortusage
    ;;
  esac
done

# shift getopt args from ARGV
shift $(expr $OPTIND - 1)

erreur() {
  case $1 in
    [0-9])
      ret=$1
      shift
    ;;
  esac
  echo $* 1>&2
  if [ ! -z "$SYSLOG_ID" ] && [ "$SYSLOG_ID" != "none" ]; then
    logger -p ${SYSLOG_ID}.error -t "${0##*/}["$$"]" -- $@
  fi
  exit ${ret:-1}
}

inform() {
  if [ ! -z "$TERM" -a -z "$QUIET" ] || [ $VERBOSE -gt 0 ]; then
    echo "$(date +%Y%m%d-%H:%M:%S) ${0##*/}: " $* >&2
  fi
  if [ ! -z "$SYSLOG_ID" ] && [ "$SYSLOG_ID" != "none" ]; then
    logger -p ${SYSLOG_ID}.info -t "${0##*/}["$$"]" -- $@
  fi
}

warne() {
  inform $@ >&2
}

debug() {
  if [ $VERBOSE -gt 1 ]; then
    inform $@
  fi
}

missingvol() {
  erreur "Volume $1 doesn't exist on "${2:-"localhost"}"." "you have to create the volume before (or use -C)"
}

dst_run() {
  debug "dst_run $dstcmd $*" >&2
  if [ -z "$JUSTFORFUN" ]; then
    if [ -t 0 ]; then
      $dstcmd $*
    else
      $dstcmd $*
    fi
  fi
}

dst_ro_run() {
  debug "dst_ro_run $dstcmd $*" >&2
  if [ -t 0 ]; then
    $dstcmd $@
  else
    $dstcmd $@
  fi
}
src_run() {
  debug "src_run $srccmd $*" >&2
  if [ -z "$JUSTFORFUN" ]; then
    if [ -t 0 ]; then
      $srccmd $@
    else
      $srccmd $@
    fi
  fi
}

src_ro_run() {
  debug "src_ro_run $srccmd $*" >&2
  if [ -t 0 ]; then
    $srccmd $*
  else
    $srccmd $*
  fi
}
src_zfs_list() {
  if [ ! -s "$SRCZFSLIST" ]; then
    # WARNING: there are TABs in next line !
    #src_ro_run "zfs list -H -t all -oname,mountpoint,lastbackup:${DSTSNAPHOST:-$MYNAME},origin,jailed -S creation ${LIMIT_RECURSION:+-d1} -r $SRCVOL" | grep -v 'on$' | sed 's/	/\!/g' > $SRCZFSLIST
    src_ro_run "zfs list -H -t all -oname,mountpoint,lastbackup:${DSTSNAPHOST:-$MYNAME},origin,jailed -S creation ${LIMIT_RECURSION:+-d1} -r $SRCVOL" | awk '/on$/{j=$1;gsub("^.*jails/","",j);gsub("/.*$","",j); gsub(".*","/iocage/jails/"j"/root"$2,$2);}{printf("%s\t%s\t%s\t%s\t%s\n",$1,$2,$3,$4,$5);}' | sed 's/	/\!/g' > $SRCZFSLIST
    src_ro_run "zfs get ${GRECURSE:+-r} -Hp -t filesystem -oname,value,source mountpoint $SRCVOL" | sed 's/	/\!/g' > $SRCZFSLIST.mounts
  fi
  cat $SRCZFSLIST
}

dst_zfs_list() {
  if [ ! -s "$DSTZFSLIST" ]; then
    # WARNING: there are TABs in next line !
    #$dstrocmd "zfs list -H -t filesystem,volume -oname,mountpoint,orig:mountpoint -S creation ${GRECURSE:+-r} $DSTVOL 2>/dev/null | sed 's/	/\!/g'; zfs list -H -t snapshot -oname,mountpoint,orig:mountpoint -S creation ${LIMIT_RECURSION:+-d1} -r $DSTVOL 2>/dev/null | sed 's/	/\!/g'" > $DSTZFSLIST
    dst_ro_run "zfs list -H -t filesystem,volume -oname,mountpoint,orig:mountpoint,jailed -S creation ${GRECURSE:+-r} $DSTVOL 2>/dev/null; zfs list -H -t snapshot -oname,mountpoint,orig:mountpoint -S creation ${LIMIT_RECURSION:+-d1} -r $DSTVOL 2>/dev/null" | sed 's/	/\!/g' > $DSTZFSLIST
  fi
  cat $DSTZFSLIST
}

# tree tools
has_excludes() {
  test -s "$EXCLUDEFILE" || return 1
  # excludes as zfs path
  grep '^'$1'/.*' $EXCLUDEFILE && return 0
  # excludes as unix path
  for efs in $(src_zfs_list | grep '^'$1'[^@]*!' | cut -d'!' -f2 | sort); do
    grep "^$efs$" $EXCLUDEFILE && return 0
  done
  return 1
}

is_excluded() {
  test -s "$EXCLUDEFILE" || return 1
  for excluded in $(grep -v '^#' $EXCLUDEFILE | grep -v '^[[:space:]]*$' | sed 's@/@\\/@g; s@\.@\\.@g;'); do
    # excludes as unix path => translate to zfs path
    expr "$excluded" : "/*" > /dev/null && excluded=$(src_zfs_list | grep '^[^!]*!'$1'!' | cut -d'!' -f1)
    [ -z "$excluded" ] && return 1
    expr "$1" : "${excluded}" > /dev/null && return 0
  done
  return 1
}


# args: dstvol
make_source_snapshot() {
  DELSNAPS=""
  # build list of snapshots to be deleted before
  if has_excludes ${1%@*} > /dev/null; then
    for fs in $(src_zfs_list | cut -d'!' -f1 | grep '^'${1%@*}'/[^@]*$'); do
      if is_excluded $fs; then
        if [ -z "$GRECURSE" ] || ! echo $DELSNAPS | grep -q ' '$fs' '; then
          DELSNAPS=$DELSNAPS"${fs}@${1#*@} "
        fi
      else
        for sfs in $(has_excludes $fs | sort -r); do
          DELSNAPS=$DELSNAPS"$sfs@${1#*@} "
        done
      fi
    done
  fi
  src_run "zfs snapshot ${GRECURSE:+-r} $SRCVOL@${1#*@}; ${DELSNAPS:+for i in $DELSNAPS; do zfs destroy -r \$i; done}"
  rm $SRCZFSLIST
}

# args: srcvol dstvol
# set LASTCOMMONSNAP if not set
get_last_common_snapshot() {
  debug "get_last_common_snapshot($*)"
#  if [ ! -z "$LASTCOMMONSNAP" ]; then
#    return 0
#  fi
  # if stored in source property, try this one first
  local SRCVOL=${1:-$SRCVOL}
  local DSTVOL=${2:-$DSTVOL}
  local LASTCOMMONSNAP=""
  test1=$(src_zfs_list | grep -E '^'${SRCVOL}'!.*![a-zA-Z0-9]+[-a-zA-Z0-9]+' | cut -d'!' -f3)
  if [ ! -z "$test1" ]; then
    dst_zfs_list | grep -Eq '^'${DSTVOL}'@'${test1}'!' && lastknownsnap=${test1}
  fi
  debug "get_last_common_snapshot($*) lastknownsnap=$lastknownsnap"
  if [ "$RELAXSNAPREGEX" = "YES_RELAXSNAPREGEX" ]; then
    SNAPCANDIDATES=$(dst_zfs_list | cut -d'!' -f1 | grep -E '^'${DSTVOL}'@.*$')
  else
    SNAPCANDIDATES=$(dst_zfs_list | cut -d'!' -f1 | grep -E '^'${DSTVOL}'@'${SRCSNAPHOST:-$MYNAME}'-'${DSTSEARCHHOST:-$MYNAME}'-[[:digit:]]{8}-[[:digit:]]{2}h[[:digit:]]{2}m[[:digit:]]{2}s$')
  fi
  snapcandidate=""
  for snap in ${DSTVOL}@$lastknownsnap $SNAPCANDIDATES; do
    if [ ! -z "${snap#*@}" ] && src_zfs_list | cut -d'!' -f1 | grep -Eq '^'${SRCVOL}'@'${snap#*@}'$'; then
      snapcandidate=${snap#*@}
      # search for snapcandidate on all existing subvolumes on dst (but excluded or nor part of SRCVOL)
      for dstfs in $(dst_zfs_list | cut -d'!' -f1 | grep '^'${DSTVOL}'/[^@]*$'); do
        if ! dst_zfs_list | cut -d'!' -f1 | grep -q '^'$dstfs'@'${snapcandidate}'$'; then
          if is_excluded ${SRCVOL}${dstfs#$DSTVOL}; then
            warne "excluded fs existing on dest: $dstfs : should delete it ?"
            #dst_run "zfs destroy -r $dstfs"
            rm $DSTZFSLIST
          elif src_zfs_list | cut -d'!' -f1 | sed 's|'${SRCVOL}'|'${DSTVOL}'|' | grep -q '^'$dstfs; then
            # this volume is part of $SRCVOL :(
            debug "snapshot ${snapcandidate} missing on $dstfs"
#            snapcandidate=""
            break 1
          fi
        fi
      done
      [ -z "$snapcandidate" ] && continue
      # search for snapcandidate all subvolumes *but new or excluded* on src
      for fs in $(src_zfs_list | cut -d'!' -f1 | grep -E '^'${SRCVOL}'[^@]*$'); do
        # if the snapshot doesn't exists on a sub-volume on source...
        if ! src_zfs_list | cut -d'!' -f1 | grep -q '^'$fs'@'${snapcandidate}'$' && ! is_excluded $fs; then
          # ..check that the volume doesn't exist on ${DSTVOL} (new vol on $SRC), 
          # otherwise this snap wouldn't work
          if dst_zfs_list | cut -d'!' -f1 | grep -q '^'${DSTVOL}${fs#${SRCVOL}}'!'; then
            debug "$snapcandidate non valide (manque sur ${DSTVOL}${fs#${SRCVOL}})"
            snapcandidate=""
            break 1
          fi
        fi
      done
      # get it !
      if [ ! -z "$snapcandidate" ]; then
        LASTCOMMONSNAP=$snapcandidate
        break
      else
        LASTCOMMONSNAP=""
      fi
    fi
  done
  debug "get_last_common_snapshot($*) -> $LASTCOMMONSNAP"
  if [ -z "$LASTCOMMONSNAP" ]; then
    return 1
  fi
  echo $LASTCOMMONSNAP
}

rollback_to_last_common_snapshot() {
  inform "Rollback to ${DSTVOL}@${LASTCOMMONSNAP}${DSTHOST:+ on $DSTHOST}"
  dst_run "zfs rollback $ROLLBACKOPT ${DSTVOL}@${LASTCOMMONSNAP}"
}

# usage: correct_mountpoints $DSTVOL $DSTMOUNTBASE
correct_mountpoints() {
  local DSTVOL=$1
  local DSTMOUNTBASE=$2

  # correction des points de montage
  if [ -z "$DSTREMOUNTORIG" -a ! -z "$CORRECTMOUNTS" ]; then
    inform "Enregistrement+modification des points de montage originaux sur ${DSTVOL}"
    dst_run 'zfs get -Hp -r -t filesystem -oname,value,source mountpoint '${DSTVOL}' | sort -r | grep 'received$' | while read vol val src; do
      if [ "$val" != "legacy" ] && [ "$val" != "none" ]; then
        zfs set mountpoint=${DSTMOUNTBASE}${val#$SRCMOUNTBASE} $vol;
        zfs set orig:mountpoint=$val $vol;
      fi;
    done'
  fi

  # si restauration, on remet les points de montage originaux
  if [ ! -z "$DSTREMOUNTORIG" ]; then
    inform "Restauration des points de montage originaux sur ${DSTVOL}"
    dst_run 'zfs get -Hp -t filesystem -r -oname,value,source orig:mountpoint '${DSTVOL}' | sort -r | while read vol val src; do
      if [ "$src" = "received" -a "$val" != "-" ]; then
        zfs set mountpoint=$val $vol
        zfs inherit orig:mountpoint $vol
      fi
    done'
  fi
}

# usage: sync_zfs srcvol dstvol [NR]
#
# do the work
sync_zfs() {
  debug "sync_zfs $1 $2 $3 (SRCVOL=$SRCVOL DSTVOL=$DSTVOL RECURSE=$3)"
  local SRCV=${1}
  shift
  local DSTV=${1%@*}
  shift
  local RET=0
  if [ $# -eq 1 -a "$1" = "NR" ]; then
    shift
    LRECURSE=""
    if ! dst_zfs_list | grep -q '^'${DSTV}'!'; then
      debug "sync_zfs: create_zfsdest $DSTV $SRCV"
      create_zfsdest ${DSTV} ${SRCV}
    fi
  else
    LRECURSE="$GRECURSE"
  fi

  local SENDOPT=$SENDOPT
  local RECVOPT=$RECVOPT
  if [ "$LRECURSE" != "FULL" ]; then
    SENDOPT="-p"
  else
    SENDOPT="-R"
  fi
  L=$LASTCOMMONSNAP
  LASTCOMMONSNAP=$(get_last_common_snapshot ${SRCV} ${DSTV})
  if [ -z "$LASTCOMMONSNAP" ]; then
    R=$RELAXSNAPREGEX
    RELAXSNAPREGEX="YES_RELAXSNAPREGEX"
    KEEPSRCSNAPS="YES"
    LASTCOMMONSNAP=$(get_last_common_snapshot ${SRCV} ${DSTV})
    if [ -z "$LASTCOMMONSNAP" ]; then
      debug "get_last_common_snapshot ${SRCV} ${DSTV} fails (new zfs ?)"
    fi
  fi
  debug sync_zfs lastcommon for ${SRCV} ${DSTV}: $LASTCOMMONSNAP
  local SRC_SEND_CMD="zfs send ${SENDOPT:+${SENDOPT}} ${LASTCOMMONSNAP:+$INCOPT @$LASTCOMMONSNAP} ${SRCV}@${NEWSNAP}"
  local DST_RECV_CMD="zfs receive -v ${RECVOPT:+${RECVOPT}} ${NODSTMOUNT:+ -u} ${DSTV}"
  LASTCOMMONSNAP=$L

  inform "Sending ${SRCV}@${NEWSNAP}${MBUFFER_SRC_CMD:+ with mbuffer}${TRANSPORT_CMD:+ via TRANSPORT_CMD}"
  if [ ! -z "$TRANSPORT_CMD" ]; then
    src_run "$SRC_SEND_CMD | $TRANSPORT_SRC_CMD $TRANSPORT_SRC_OPTS" &
    sleep 1
    dst_run "$TRANSPORT_DST_CMD $TRANSPORT_DST_OPTS | $DST_RECV_CMD"
    wait
    RET=$?
  else
    src_run "$SRC_SEND_CMD ${MBUFFER_SRC_CMD:+ | $MBUFFER_SRC_CMD}" | dst_run "${MBUFFER_DST_CMD:+$MBUFFER_DST_CMD | }$DST_RECV_CMD"
    RET=$?
  fi
  if [ $RET -ne 0 ]; then
    warne "Error transfering from ${SRCHOST:+"$SRCHOST:"}${SRCV}@${NEWSNAP} to ${DSTHOST:+"$DSTHOST:"}${DSTV}${LASTCOMMONSNAP:+" (inc. based on @$LASTCOMMONSNAP)"}. zfs returns $RET"
    warne "  send command: $SRC_SEND_CMD"
    warne "  receive command: $DST_RECV_CMD"
    warne "you may retry with -B (rollback forced) and/or -j (loosy snapshot search)"
#    [ -z "$KEEPSRCSNAPS" ] && SRCSNAPSTODEL=$SRCSNAPSTODEL${NEWSNAP:+" ${SRCV}@${NEWSNAP}"}
#    [ -z "$KEEPDSTSNAPS" ] && DSTSNAPSTODEL=$DSTSNAPSTODEL${NEWSNAP:+" ${DSTV}@${NEWSNAP}"}
  else
    inform "Transfer ok :)"
    [ -z "$KEEPSRCSNAPS" ] && SRCSNAPSTODEL=$SRCSNAPSTODEL${LASTCOMMONSNAP:+" ${SRCV}@${LASTCOMMONSNAP}"}
    [ -z "$KEEPDSTSNAPS" ] && DSTSNAPSTODEL=$DSTSNAPSTODEL${LASTCOMMONSNAP:+" ${DSTV}@${LASTCOMMONSNAP}"}
  fi
  return $RET
}

create_zfsdest() {
  debug "create_zfsdest($*)"
  local DSTV=${1:-$DSTVOL}
  local SRCV=${2:-$SRCVOL}
  local MNTPOINT=${3:-$(get_mountpoint $DSTV)}
  local VOLSIZE=""
  src_zfs_list | grep -q '^'${SRCV}'!.*!on$' && local j=1
  rm $DSTZFSLIST
  dst_zfs_list | grep -q '^'${DSTV}'!' && return 0
  cmd="zfs create"$(src_ro_run "zfs get -H -s local,received -oproperty,value all ${SRCV}" | while read prop val; do
     if [ "$prop" = "mountpoint" ]; then
       if [ "$j" = "1" ]; then
         debug "create_zfsdest($*): jailed fs"
       elif [ "$val" != "none" -a "$val" != "legacy" ]; then
         echo -n " -o orig:mountpoint=$val";
       else
         echo -n " -o $prop=\"$val\"";
       fi
     elif [ "$prop" = "volsize" ]; then
         echo -n " -V $val";
     else
       echo -n " -o $prop=\"$val\"";
     fi
    done)
  if [ "$j" != "1" ] && ! echo $cmd | grep -q -- "-V"; then
    [ "$(get_mountpoint ${DSTV%/*})" = "${MNTPOINT%/*}" ] || \
      echo "$cmd" | fgrep -q -- "-o mountpoint=" || cmd=${cmd}${MNTPOINT:+" -o mountpoint=$MNTPOINT"}
  fi
  #echo $cmd | grep -q -- "-V" || dst_run "$cmd ${DSTV}" || erreur "Failed to create $DSTV localhost (-C) ? ($cmd)"
  dst_run "$cmd ${DSTV}" || erreur "Failed to create $DSTV localhost (-C) ? ($cmd)"
  rm $DSTZFSLIST
  RECVOPT="-F"
}

get_mountpoint() {
  local DSTVOL=$1
  local DSTMNT=""
  [ -n "$CREATEMOUNTPOINT" ] && echo $CREATEMOUNTPOINT && return 0

  DSTMNT=$(dst_ro_run "zfs list -H -o mountpoint $DSTVOL 2>/dev/null")
  if [ -z "$DSTMNT" -o "$DSTMNT" = "none" -o "$DSTMNT" = "legacy" -o "$DSTMNT" = "-" ]; then
    local DSTUP=${DSTVOL%/*}
    local DSTUPMNT=$(dst_ro_run "zfs list -H -o mountpoint ${DSTUP}")
    [ -n "$DSTUPMNT" ] && DSTMNT=${DSTUPMNT%/}${DSTVOL#$DSTUP}
  fi
  [ -z "$DSTMNT" -o "$DSTMNT" = "none" -o "$DSTMNT" = "legacy" ] && \
    erreur 5 "Impossible de determiner le repertoire de montage"

  echo $DSTMNT
}

# usage: get_zfses src/vol dst/vol
#
# recursively get all non-excluded sub-volumes (-R when possible)
#
get_zfses() {
  debug get_zfses $*
  local ERRS=0
  local LOOPSRCVOL=${1%@*}
  local SUBVOL=$(echo ${1} | sed 's!'${SRCVOL}'!!; s/@.*$//;')
  local LOOPDSTVOL=${2:-${DSTVOL}${SUBVOL}}
  local DSTMNTPOINT=${3:-$(get_mountpoint ${LOOPDSTVOL})}
  if has_excludes ${LOOPSRCVOL} > /dev/null || [ "$GRECURSE" = "MANUAL" ]; then
    debug "get_zfses($1): $LOOPSRCVOL has excludes or manual recursion required"
    if ! is_excluded ${LOOPSRCVOL}; then
      DSTMNTPOINT=$(get_mountpoint ${LOOPDSTVOL})
      sync_zfs ${LOOPSRCVOL} ${LOOPDSTVOL} NR || return 1
    else
      debug "${LOOPSRCVOL} excluded"
    fi
    debug get_zfs subfses: $(src_zfs_list | cut -d'!' -f1 | grep '^'${LOOPSRCVOL}'/[^@/]*$' | sed 's*'$LOOPSRCVOL'**')
    local fs
    for fs in $(src_zfs_list | cut -d'!' -f1 | grep '^'${LOOPSRCVOL}'/[^@/]*$' | sed 's*'$LOOPSRCVOL'**'); do
      debug get_zfses $LOOPSRCVOL fs=$fs
      if ! is_excluded ${LOOPSRCVOL}${fs}; then
        get_zfses ${LOOPSRCVOL}${fs} ${LOOPDSTVOL}${fs} || ERRS=$((ERRS + $?))
      else
        debug "${LOOPSRCVOL}${fs} excluded"
      fi
    done
  else
    debug "get_zfses($1): $LOOPSRCVOL has NO excludes"
    DSTMNTPOINT=$(get_mountpoint ${LOOPDSTVOL})
    sync_zfs ${LOOPSRCVOL} ${LOOPDSTVOL} || ERRS=$((ERRS + 1))
  fi
  return $ERRS
}

# sanity check args
if [ ! -z "$SRCPRESNAPFORMAT" ] && [ "$INCOPT" != "-I" ]; then
  warne "$0: -m needs -I to get the snapshot"
  exec $0 -0
fi
if [ ! -z "$TRANSPORT_CMD" ] && [ -z "$TRANSPORT_SRC_OPTS" -o -z "$TRANSPORT_DST_OPTS" ]; then
  warne "-t $TRANSPORT_CMD may need args with -o and -O"
  exec $0 -0
fi
if [ ! -z "$EXCLUDEFILE" ]; then
  if [ ! -f "$EXCLUDEFILE" ]; then
    erreur "$EXCLUDEFILE doesn't exist"
  fi
  if [ -z "$GRECURSE" ]; then
    erreur "-x <file> only makes sense with -R or -r"
  fi
fi
if [ "$GRECURSE" = "FULL" ]; then
  LIMIT_RECURSION="YES_LIMIT"
fi

#TRANSPORTCMD=${TRANSPORTCMD:-""}

### check args
if [ $# -eq 2 ]; then
  SRC=$1
  DST=$2
else
  shortusage
fi

### lock myself if not already
if [ ! -n "$ZFSSYNCLOCKED" ]; then
  LOCKFILE=${TMPDIR:-/var/tmp}/LCK.$(echo "$SRC $DST" | sed 's/[^-a-zA-Z0-9_]/_/g')
  export ZFSSYNCLOCKED=$LOCKFILE
  exec lockf -t 0 $LOCKFILE /bin/sh -f ${GROSDEBUG:+-x} $0 $ORIGARGS
fi

TBEGIN=$(date +%s)

SYSLOG_ID=${SYSLOG_ID:-user}

####################
### process args ###
####################
if [ "x$PERSISTSSH" = "xYES" ]; then
  if [ -z "$TMPDIR" ]; then
    export TMPDIR=/var/tmp/sshs
    mkdir -m 0700 $TMPDIR
  fi
fi
SSHOPTS="-T"${SSHKEY:+" -i $SSHKEY"}${TEMPSSH:+" -oUserKnownHostsFile=${TMPDIR:-/tmp}/kkk -oCheckHostIP=no"}${PERSISTSSH:+" -oControlMaster=auto -oControlPath=$TMPDIR/%C -oControlPersist=yes"}

# determine source and dest hosts, volumes,...
SRCHOST=""
DSTHOST=""
SRCVOL=${SRC%%@*}
DSTVOL=${DST%%@*}

srccmd="eval"
dstcmd="eval"

if echo "$SRC" | grep '@' >/dev/null; then
  SRCHOST=${SRC##*@}
  SRCSNAPHOST=${SRCHOST%%.*}
  srccmd="ssh ${SSHOPTS} ${SRCHOST}"
  #ZSRCVOL=$(src_zfs_list | grep '^'${SRCVOL}'!' | cut -d'!' -f1)
  test ! -z "$SRCVOL" || erreur "Impossible de se connecter a ${SRCHOST} ou de determiner la source zfs pour $SRCVOL"
  #SRCVOL=$ZSRCVOL
fi

if echo "$DST" | grep '@' >/dev/null; then
  DSTHOST=${DST##*@}
  DSTSNAPHOST=${DSTHOST%%.*}
  DSTSEARCHHOST=${CHANGEDSTHOST:-$DSTSNAPHOST}
  dstcmd="ssh ${SSHOPTS} ${DSTHOST}"
  dst_run "echo ok" | grep ok >/dev/null || erreur "Impossible de se connecter a ${DSTHOST}"
fi

if [ -n "$SRCHOST" -o -n "$DSTHOST" ] && [ -z "$SSHKEY" ] && [ -z "$SSH_AUTH_SOCK" ]; then
  erreur "Missing ssh key (-k or agent)"
fi

export srcrocmd=$srccmd
export dstrocmd=$dstcmd

#if [ ! -z "$JUSTFORFUN" ]; then
#  inform "Running just for fun, don't worry :)"
#  srccmd='echo DID NOT LAUNCH: # '$srccmd
#  dstcmd='echo DID NOT LAUNCH: # '$dstcmd
#fi

# nc|mbuffer|...
if [ ! -z "$TRANSPORT_CMD" ]; then
  TRANSPORT_SRC_CMD=$(src_ro_run "which $TRANSPORT_CMD" | fgrep -v 'not found')
  [ -z "$TRANSPORT_SRC_CMD" ] && erreur "$TRANSPORT_CMD introuvable ${SRCHOST:+sur $SRCHOST}"
  TRANSPORT_DST_CMD=$(dst_ro_run "which $TRANSPORT_CMD" | fgrep -v 'not found')
  [ -z "$TRANSPORT_DST_CMD" ] && erreur "$TRANSPORT_CMD introuvable ${DSTHOST:+sur $DSTHOST}"
else
  # try mbuffer
  MBUFFER_SRC_CMD=$(src_ro_run "which mbuffer" | fgrep -v 'not found')
  [ ! -z "$MBUFFER_SRC_CMD" ] && MBUFFER_SRC_CMD=$MBUFFER_SRC_CMD" -m 128M -q"
  MBUFFER_DST_CMD=$(dst_ro_run "which mbuffer" | fgrep -v 'not found')
  [ ! -z "$MBUFFER_DST_CMD" ] && MBUFFER_DST_CMD=$MBUFFER_DST_CMD" -m 128M -q"
fi

# check and resolve source
if ! src_zfs_list | cut -d'!' -f1 | grep -q '^'${SRCVOL}'$'; then
  if src_zfs_list | cut -d'!' -f2 | grep -q '^'${SRCVOL}'$'; then
    SRCVOL=$(src_zfs_list | fgrep -q '!'${SRCVOL}'!' | cut -d'!' -f1)
  else
    erreur "Le volume source ${SRCVOL} n'existe pas"
    src_zfs_list | fgrep '!'${SRCVOL}'!'
  fi
fi

DSTBASE=$(dst_ro_run "zfs list -H -o mountpoint ${DSTVOL%/*}")
if ! echo $DSTBASE | grep -q ^/; then
  volbase=${DSTVOL%/*}
  DSTBASE=$(dst_ro_run "zfs list -H -o mountpoint ${volbase%/*}")/${volbase##*/}/${DSTVOL##*/}
fi

if [ "$CORRECTMOUNTS" = "YES_CORRECTMOUNTS" ] && ( [ -z "$DSTBASE" ] || ! echo $DSTBASE | grep -q ^/ ); then
  erreur "Impossible de definir le point de montage parent de $DSTVOL :("
fi

DSTMOUNTBASE=$(dst_zfs_list | grep '^'${DSTVOL}'!' | cut -d'!' -f2 2>/dev/null)

echo "$DSTMOUNTBASE" | grep -q ^/ || DSTMOUNTBASE=${DSTBASE%/}/${DSTVOL##*/}

echo $DSTMOUNTBASE | grep -q ^$DSTBASE || erreur "$DSTMOUNTBASE n'est pas dans $DSTBASE ???"

SRCMOUNTBASE=$(src_zfs_list | grep '^'${SRCVOL}'!' | cut -d'!' -f2 2>/dev/null)

# create destination if needed && -C
if [ -z "$DSTMOUNTBASE" -o "$DSTMOUNTBASE" = "none" -o "$DSTMOUNTBASE" = "legacy" ]; then
  DSTMNTPOINT=$(get_mountpoint ${DSTVOL})
  if [ "$CREATEDEST" = "YES_CREATEDEST" ]; then
    # ... and if not clone
#    if [ "$(src_zfs_list | grep '^'${SRCVOL} | cut -d'|' -f4)" = "-" ]; then
    if src_zfs_list | grep -q '^'${SRCVOL}'!'; then
      RECVOPT='-F'
      cmd=""
#      if [ -n "$CREATEMOUNTPOINT" ]; then
#      if [ -n "$CREATEMOUNTPOINT" ]; then
#      create_zfsdest ${DSTVOL} ${SRCVOL} ${DSTMNTPOINT}
#      fi
#      fi

    else
      inform "${SRCVOL} disapeard"
      src_zfs_list|grep '^'${SRCVOL}'!'
      inform "Lord, is ${SRCVOL} a clone ?"
    fi
  else
    erreur 6 "La destination (${DSTVOL}) n'existe pas et pas -C :("
  fi
fi

# check DSTMOUNTBASE must exist
echo $DSTMOUNTBASE | grep -q '^/[a-zA-Z0-9]' || echo $DSTMOUNTBASE | grep -q '\(none\|legacy\)' || erreur "DSTMOUNTBASE mal defini ($DSTMOUNTBASE)"

# check for dangerous new fixed mountpoints on source for new volumes
for fs in $(grep '!\(local\|received\)$' $SRCZFSLIST.mounts | cut -d'!' -f1 | sed 's|'${SRCVOL}'|'${DSTVOL}'|'); do
  dst_zfs_list | grep '^'$fs'!' | grep -v 'on$' | cut -d'!' -f2 | while read dstmountpoint; do
    if ! expr "$dstmountpoint" : "$DSTMOUNTBASE" > /dev/null && [ -z "$CORRECTMOUNTS" -a -z "$DSTREMOUNTORIG" ]; then
      erreur "DANGEROUS: some mountpoints out of $DSTMOUNTBASE (and no -U !)"
    fi
  done
done

# check for dangerous mountpoints on dest
if [ -n "$(dst_zfs_list | grep -Ev '^'${DSTVOL}'.*!(none|legacy|-)!' | grep -qv '!'${DSTMOUNTBASE}'!' | grep -v '!on$')" ]; then
  if [ -z "$CORRECTMOUNTS" -a -z "$DSTREMOUNTORIG" ]; then
    erreur "WARNING: certains points de montage sont hors de $DSTMOUNTBASE (et -U absent) : $(dst_zfs_list | grep -Ev '^'${DSTVOL}'.*!(none|legacy|-)!' | grep -v '!'${DSTMOUNTBASE} | sed 's/^\(.*\)!\(.*\)!.*$/\1 -> \2/')"
  fi
fi

# locate last snapshot (if any)
LASTCOMMONSNAP=$(get_last_common_snapshot)
if [ -z "$LASTCOMMONSNAP" ] && dst_zfs_list | grep -q '^'${DSTVOL}'@'; then
  # If no common snapshot AND DSTVOL has snapshots, zfs receive would fail.
  erreur 7 "ERREUR: des snapshots existent dans ${DSTVOL} et aucun snapshot commun existant n'a ete trouve... Re-essayez avec -j ou supprimez tout ${DSTVOL}"
fi

TCHECKSOK=$(date +%s)
TCHECKS=$(($TCHECKSOK - $TBEGIN))

if [ ! -z "$LASTCOMMONSNAP" -a "$DOROLLBACK" = "YES_ROLLBACK" ]; then
  rollback_to_last_common_snapshot || erreur "rollback impossible"
fi

# pre-snapshot on source if -m
if [ ! -z "$SRCPRESNAPFORMAT" ]; then
  PRESNAP=${SRCVOL}@$(TZ=UTC date +$SRCPRESNAPFORMAT)
  make_source_snapshot ${PRESNAP} && SRCSNAPSTODEL=$SRCSNAPSTODEL" "$PRESNAP
fi

# create source snapshot if ! -b
if [ -z "$NEWSNAP" ]; then
  # create snapshot on source (this one is quick)
  NEWSNAP=${SRCSNAPHOST:-$MYNAME}'-'${DSTHOST:-$MYNAME}'-'$(date +%Y%m%d-%Hh%Mm%Ss)
  # workaround: si on demande un incremental trop vite, on triche d'une minute
  if [ "$NEWSNAP" = "$LASTCOMMONSNAP" ]; then
    NEWSNAP=${SRCSNAPHOST:-$MYNAME}'-'${DSTSNAPHOST:-$MYNAME}'-'$(date -v +1S +%Y%m%d-%Hh%Mm%Ss)
  fi
  inform "Create source snapshot ${SRCHOST:+"$SRCHOST:"}${SRCVOL}@${NEWSNAP}"
  make_source_snapshot ${SRCVOL}@${NEWSNAP}
fi

TMKSNAPOK=$(date +%s)
TMKSNAP=$(($TMKSNAPOK - $TCHECKSOK))

DONE=0
DSTSNAPSTODEL=""

# get all
get_zfses ${SRCVOL}@${NEWSNAP} ${DSTVOL} ${DSTMOUNTBASE}
RET=$?

TTRANSOK=$(date +%s)
TTRANSFER=$(($TTRANSOK - $TMKSNAPOK))

#TTRANSFER=$(awk 'BEGIN { T=0; } { T=$1; } END { print t; }' /tmp/$$)

correct_mountpoints ${DSTVOL} ${DSTMOUNTBASE}

TPOSTCHECKSOK=$(date +%s)
TPOSTCHECKS=$(($TPOSTCHECKSOK - $TTRANSOK))

if [ -z "$KEEPSRCSNAPS" -a $RET -eq 0 ]; then
  SRCSNAPSTODEL=$SRCSNAPSTODEL$(src_zfs_list | cut -d'!' -f1 | \
    grep -E '@'${SRCSNAPHOST:-$MYNAME}'-'${DSTSEARCHHOST:-$MYNAME}'-[[:digit:]]{8}-[[:digit:]]{2}h[[:digit:]]{2}m[[:digit:]]{2}s$' | \
    grep -Fv ${LASTCOMMONSNAP:-"THISCANNOTBEASNAPSHOTNAME"} | fgrep -v ${NEWSNAP} | \
    while read s; do if ! is_excluded ${s%@*} && [ "${s#*@}" != "$LASTCOMMONSNAP" ] && [ "${s#*@}" != "$NEWSNAP" ]; then echo " $s"; fi; done)
fi
if [ -z "$KEEPDSTSNAPS" -a $RET -eq 0 ]; then
  DSTSNAPSTODEL=$DSTSNAPSTODEL$(dst_zfs_list | cut -d'!' -f1 | \
    grep -E '@'${SRCSNAPHOST:-$MYNAME}'-'${DSTSEARCHHOST:-$MYNAME}'-[[:digit:]]{8}-[[:digit:]]{2}h[[:digit:]]{2}m[[:digit:]]{2}s$' | \
    fgrep -v ${LASTCOMMONSNAP:-"THISCANNOTBEASNAPSHOTNAME"} | fgrep -v ${NEWSNAP} | \
    while read s; do if ! is_excluded ${s%@*} && [ "${s#*@}" != "$LASTCOMMONSNAP" ] && [ "${s#*@}" != "$NEWSNAP" ] && src_zfs_list | ( v=${s%@*}; grep -q '^'${SRCVOL}${v#$DSTVOL}'!'); then echo " $s"; fi; done)
fi
if [ ! -z "$SRCSNAPSTODEL" ]; then
  inform "Cleaning snapshots${SRCHOST:+ on $SRCHOST}: $SRCSNAPSTODEL" 
  src_run 'for snap in '$SRCSNAPSTODEL'; do zfs destroy '${GRECURSE:+-r}' -d $snap 2>&1; done | fgrep -v "not exist"'
fi
if [ ! -z "$DSTSNAPSTODEL" ]; then
  inform "Cleaning snapshots${DSTHOST:+ on $DSTHOST}: $DSTSNAPSTODEL" 
  dst_run 'for snap in '$DSTSNAPSTODEL'; do zfs destroy '${GRECURSE:+-r}' -d $snap 2>&1; done | fgrep -v "not exist"'
fi

# write newsnap name on lastbackup:$MYNAME property if ok
if [ $RET -eq 0 ]; then
  src_run "zfs set lastbackup:${DSTSNAPHOST:-$MYNAME}=${NEWSNAP} ${SRCVOL}"
fi
rm -f $SRCZFSLIST $SRCZFSLIST.mounts $DSTZFSLIST

TCLEANUPOK=$(date +%s)
TCLEANUP=$(($TCLEANUPOK - $TPOSTCHECKSOK))

if [ "x$PERSISTSSH" = "xYES" ]; then
  SSHOPTS=$SSHOPTS" -O exit"
  src_run "echo end ${SRCHOST} ${SRCVOL}" 2>/dev/null
  dst_run "echo end ${SRCHOST} ${SRCVOL}" 2>/dev/null
fi

TEND=$(($TCLEANUPOK - $TBEGIN))
inform "${0##*/} end: ${TCHECKS}s checking, ${TMKSNAP}s snapshot, ${TTRANSFER}s transfer, ${TPOSTCHECKS}s checking, ${TCLEANUP}s cleanup, ${TEND}s total."
return $RET