#!/bin/sh
set -u

#
# vstackos.config-backup.sh -- collect node configuration and state before a
# vStack OS update, so that a node can be rebuilt or rolled back by hand.
#
# Read-only for the system: the only writes go into the output directory.
# POSIX sh, no network access, works on FreeBSD 13.2 / 14.2 / 15.x and on
# vStack OS 2.x / 3.x / 4.x, on SDS-only, HCP and standalone nodes.
#

VERSION="1.6.0"

usage() {
  echo "Usage: $0 [OPTIONS]"
  echo ""
  echo "Options:"
  echo "  -o, --out DIR       output directory"
  echo "                      (default: /root/vstackos-config-backup/<cluster>/<UTC-date>/<hostname>)"
  echo "      --tag TEXT      free-form tag recorded in SUMMARY.txt (target version, change id)"
  echo "      --no-secrets    drop master.passwd, spwd.db and ssh host private keys"
  echo "                      from the copy (restore of accounts/host identity then needs"
  echo "                      another source)"
  echo "      --with-logs     also copy tails of /var/log (messages, ccrd, vstack)"
  echo "      --with-smart    also collect smartctl per disk (slow on large JBOD)"
  echo "      --full-sysctl   also dump the whole sysctl -a"
  echo "      --copy-to DIR   after collecting, duplicate the result into DIR and verify it there
                      by checksum (repeatable: mounted flash, another pool, an NFS mount).
                      A single copy on the system disk does not survive a failed update
      --tar           additionally pack the result into vstackos-config-backup-<host>-<date>.tar.gz
      --verify DIR    verify an earlier collection in DIR against its own MANIFEST
                      and exit (no collecting)"
  echo "      --timeout SEC   per-command timeout, default 60"
  echo "      --dry-run       print what would be collected, write nothing"
  echo "  -h, --help          this message"
  echo ""
  echo "Exit codes: 0 ok, 1 fatal, 2 finished with warnings."
  echo ""
  echo "Remote use:  ssh root@<node> sh -s -- --tag 3.2.0 < vstackos.config-backup.sh"
  echo "Off-node:    scp -r root@<node>:<out> ."
  echo ""
  echo "Keep the result in three places: the node, outside the node (jump host or"
  echo "workstation) and on the boot flash of the node. The default output lives on"
  echo "the system disk -- the very disk a failed update rewrites."
}

OUT=""
TAG=""
OPT_NO_SECRETS=0
OPT_WITH_LOGS=0
OPT_WITH_SMART=0
OPT_FULL_SYSCTL=0
OPT_TAR=0
OPT_DRY=0
OPT_VERIFY=""
COPY_TARGETS=""
CMD_TIMEOUT=60

while [ "$#" -gt 0 ]; do
  case "$1" in
    -o|--out)       OUT="$2"; shift 2 ;;
    --tag)          TAG="$2"; shift 2 ;;
    --no-secrets)   OPT_NO_SECRETS=1; shift ;;
    --with-logs)    OPT_WITH_LOGS=1; shift ;;
    --with-smart)   OPT_WITH_SMART=1; shift ;;
    --full-sysctl)  OPT_FULL_SYSCTL=1; shift ;;
    --tar)          OPT_TAR=1; shift ;;
    --copy-to)      COPY_TARGETS="${COPY_TARGETS} $2"; shift 2 ;;
    --verify)       OPT_VERIFY="$2"; shift 2 ;;
    --timeout)      CMD_TIMEOUT="$2"; shift 2 ;;
    --dry-run)      OPT_DRY=1; shift ;;
    -h|--help)      usage; exit 0 ;;
    *) printf 'ERROR: unknown argument: %s\n' "$1" >&2; usage >&2; exit 1 ;;
  esac
done

# the manifest is written on a FreeBSD node but verified wherever the copy lands
# (jump host, workstation): pick the digest by the manifest and use whatever tool
# for it this machine has
digest_of() {
  case "$1" in
    sha256)
      command -v sha256    >/dev/null 2>&1 && { sha256 -q "$2"; return $?; }
      command -v sha256sum >/dev/null 2>&1 && { sha256sum "$2" | awk '{print $1}'; return $?; }
      command -v shasum    >/dev/null 2>&1 && { shasum -a 256 "$2" | awk '{print $1}'; return $?; }
      ;;
    md5)
      command -v md5       >/dev/null 2>&1 && { md5 -q "$2"; return $?; }
      command -v md5sum    >/dev/null 2>&1 && { md5sum "$2" | awk '{print $1}'; return $?; }
      ;;
  esac
  return 1
}

# a node keeps vStack.conf and vstack.conf apart; FAT flashes and macOS volumes
# do not, and a copy onto one of them silently loses a file
fs_case_sensitive() {
  _probe="$1/.preupd_case_probe"
  rm -f "${_probe}" 2>/dev/null
  : > "${_probe}" 2>/dev/null || return 2
  if [ -e "$1/.PREUPD_CASE_PROBE" ]; then
    rm -f "${_probe}"
    return 1
  fi
  rm -f "${_probe}"
  return 0
}

verify_collection() {
  _dir="$1"
  _man="${_dir}/00-summary/MANIFEST.sha256"
  if [ ! -s "${_man}" ]; then
    printf 'ERROR: no manifest in %s\n' "${_dir}" >&2
    return 1
  fi
  _len="$( awk 'NR==1 {print length($1); exit}' "${_man}" )"
  case "${_len}" in
    64) _algo=sha256 ;;
    32) _algo=md5 ;;
    *)  printf 'ERROR: unrecognised digest length %s in %s\n' "${_len}" "${_man}" >&2; return 1 ;;
  esac
  if ! digest_of "${_algo}" "${_man}" >/dev/null 2>&1; then
    printf 'ERROR: no %s tool here (tried %s, %ssum, shasum)\n' "${_algo}" "${_algo}" "${_algo}" >&2
    return 1
  fi
  _bad=0
  _seen=0
  while read -r _sum _file; do
    [ -n "${_sum}" ] || continue
    _seen=$(( _seen + 1 ))
    _cur="$( cd "${_dir}" && digest_of "${_algo}" "${_file}" 2>/dev/null )"
    if [ "${_cur}" != "${_sum}" ]; then
      printf 'MISMATCH %s\n' "${_file}"
      _bad=$(( _bad + 1 ))
    fi
  done < "${_man}"
  printf 'verified %s files in %s with %s, mismatched %s\n' "${_seen}" "${_dir}" "${_algo}" "${_bad}"
  [ "${_bad}" -eq 0 ]
}

if [ -n "${OPT_VERIFY}" ]; then
  if verify_collection "${OPT_VERIFY}"; then
    exit 0
  fi
  exit 2
fi

if [ "$( id -u )" -ne 0 ]; then
  printf 'ERROR: collecting must run as root (--verify does not)\n' >&2
  exit 1
fi

TS="$( date -u '+%Y%m%dT%H%M%SZ' )"
START_EPOCH="$( date -u '+%s' )"
DAY="$( date -u '+%Y-%m-%d' )"
HOST="$( hostname -s 2>/dev/null || hostname )"

# Collections are looked up by stand first and by date second, so the path is
# <cluster>/<date>/<node>. The cluster name is what people call the stand: the
# hostname prefix for qNNNuNNN nodes, otherwise the installed cluster id.
CLUSTER="$( printf '%s' "${HOST}" | sed -n 's/^\(..*\)u[0-9][0-9]*$/\1/p' )"
CLUSTER_SRC="hostname"
if [ -z "${CLUSTER}" ]; then
  _cid="$( sed -n 's/^cluster_id="*\([^"]*\)"*$/\1/p' /usr/local/etc/vstack-env.conf 2>/dev/null )"
  if [ -n "${_cid}" ]; then
    CLUSTER="q${_cid}"
    CLUSTER_SRC="vstack-env.conf"
  else
    CLUSTER="${HOST}"
    CLUSTER_SRC="fallback: node has no cluster name"
  fi
fi

if [ -z "${OUT}" ]; then
  OUT="/root/vstackos-config-backup/${CLUSTER}/${DAY}/${HOST}"
fi

TIMEOUT_BIN="$( command -v timeout 2>/dev/null || true )"
WARN_COUNT=0
LOG=""
WARNFILE=""

msg() {
  printf '%s\n' "$*"
  [ -n "${LOG}" ] && [ "${OPT_DRY}" -eq 0 ] && printf '%s %s\n' "$( date -u '+%Y-%m-%dT%H:%M:%SZ' )" "$*" >> "${LOG}"
  return 0
}

warn() {
  WARN_COUNT=$(( WARN_COUNT + 1 ))
  printf 'WARN: %s\n' "$*" >&2
  if [ -n "${WARNFILE}" ] && [ "${OPT_DRY}" -eq 0 ]; then
    printf '%s WARN %s\n' "$( date -u '+%Y-%m-%dT%H:%M:%SZ' )" "$*" >> "${WARNFILE}"
    printf '%s WARN %s\n' "$( date -u '+%Y-%m-%dT%H:%M:%SZ' )" "$*" >> "${LOG}"
  fi
  return 0
}

have() {
  command -v "$1" >/dev/null 2>&1
}

# run <destfile> <command...> -- stdout+stderr of the command into destfile
run() {
  _dst="$1"; shift
  if [ "${OPT_DRY}" -eq 1 ]; then
    printf 'would run : %-52s < %s\n' "${_dst#${OUT}/}" "$*"
    return 0
  fi
  if ! have "$1"; then
    warn "tool not found, skipped: $1 (for ${_dst#${OUT}/})"
    return 1
  fi
  if [ -n "${TIMEOUT_BIN}" ]; then
    "${TIMEOUT_BIN}" "${CMD_TIMEOUT}" "$@" > "${_dst}" 2>&1
    _rc=$?
  else
    "$@" > "${_dst}" 2>&1
    _rc=$?
  fi
  if [ "${_rc}" -eq 124 ]; then
    warn "timed out after ${CMD_TIMEOUT}s: $*"
  elif [ "${_rc}" -ne 0 ]; then
    printf '%s rc=%s %s\n' "$( date -u '+%Y-%m-%dT%H:%M:%SZ' )" "${_rc}" "$*" >> "${LOG}"
  fi
  return 0
}

# rpc <destfile> <method> -- SPDK JSON-RPC over the local TCP socket
rpc() {
  _dst="$1"; _method="$2"
  if [ "${OPT_DRY}" -eq 1 ]; then
    printf 'would rpc : %-52s < %s\n' "${_dst#${OUT}/}" "${_method}"
    return 0
  fi
  have nc || { warn "nc not found, SPDK RPC skipped: ${_method}"; return 1; }
  if [ -n "${TIMEOUT_BIN}" ]; then
    printf '%s' "{\"jsonrpc\":\"2.0\",\"id\":\"1\",\"method\":\"${_method}\"}" \
      | "${TIMEOUT_BIN}" 15 nc -N 127.0.0.1 "${SPDK_PORT}" > "${_dst}" 2>&1
  else
    printf '%s' "{\"jsonrpc\":\"2.0\",\"id\":\"1\",\"method\":\"${_method}\"}" \
      | nc -N 127.0.0.1 "${SPDK_PORT}" > "${_dst}" 2>&1
  fi
  [ -s "${_dst}" ] || warn "empty SPDK RPC reply: ${_method}"
  return 0
}

# cpf <src> <dstdir> [newname]
cpf() {
  _src="$1"; _dstdir="$2"; _name="${3:-}"
  if [ -L "${_src}" ] && [ ! -e "${_src}" ]; then
    warn "dangling symlink, copied as a link: ${_src} -> $( readlink "${_src}" 2>/dev/null )"
    [ "${OPT_DRY}" -eq 1 ] && return 0
    cp -Pp "${_src}" "${_dstdir}/${_name:-}" 2>>"${LOG}"
    return 0
  fi
  [ -e "${_src}" ] || return 1
  if [ -d "${_src}" ]; then
    # some builds ship an empty directory where another ships a file
    return 1
  fi
  if [ "${OPT_DRY}" -eq 1 ]; then
    printf 'would copy: %s\n' "${_src}"
    return 0
  fi
  if [ -n "${_name}" ]; then
    cp -p "${_src}" "${_dstdir}/${_name}" 2>>"${LOG}" || warn "copy failed: ${_src}"
  else
    cp -p "${_src}" "${_dstdir}/" 2>>"${LOG}" || warn "copy failed: ${_src}"
  fi
  return 0
}

# cpt <srcdir> <dstdir>
cpt() {
  _src="$1"; _dst="$2"
  [ -d "${_src}" ] || return 1
  if [ "${OPT_DRY}" -eq 1 ]; then
    printf 'would copy tree: %s (%s KB)\n' "${_src}" "$( du -sk "${_src}" 2>/dev/null | awk '{print $1}' )"
    return 0
  fi
  cp -a "${_src}" "${_dst}" 2>>"${LOG}" || warn "tree copy failed: ${_src}"
  return 0
}

mkd() {
  [ "${OPT_DRY}" -eq 1 ] && return 0
  mkdir -p "$1" || { printf 'ERROR: cannot create %s\n' "$1" >&2; exit 1; }
  return 0
}

#
# node profile
#

FBSD_RUNNING="$( freebsd-version -k 2>/dev/null || uname -r )"
FBSD_MAJOR="$( uname -r | sed 's/\..*//' )"
OS_NAME="$( sed -n 's/^NAME=//p' /etc/os-release 2>/dev/null | tr -d '"' )"
OS_VER="$( sed -n 's/^VERSION=//p' /etc/os-release 2>/dev/null | tr -d '"' )"
[ -n "${OS_NAME}" ] || OS_NAME="unknown"
[ -n "${OS_VER}" ] || OS_VER="unknown"

PKGBASE="no"
if have pkg && pkg query '%n' 2>/dev/null | grep -q '^FreeBSD-runtime$'; then
  PKGBASE="yes"
fi

CCRD_GEN="absent"
if have ccrd_cli; then
  if pkg query '%n' 2>/dev/null | grep -q '^vStack-ccrd-ng$'; then
    CCRD_GEN="ng"
  else
    CCRD_GEN="legacy"
  fi
fi

SAN_CLI="none"
have vstack-san && SAN_CLI="vstack-san"
[ "${SAN_CLI}" = "none" ] && have vstack-san-cli && SAN_CLI="vstack-san-cli"

SPDK_PORT=7000
SPDK_PROC="none"
if have sockstat; then
  if sockstat -4l 2>/dev/null | grep -q "spdk_tgt"; then SPDK_PROC="spdk_tgt"; fi
  if [ "${SPDK_PROC}" = "none" ] && sockstat -4l 2>/dev/null | grep -q "nvmf_tgt"; then SPDK_PROC="nvmf_tgt"; fi
fi

POOLS="$( zpool list -H -o name 2>/dev/null | tr '\n' ' ' )"
VMJSON_COUNT="$( ls -1 /pools/*/.vm_V4/*/vm.json 2>/dev/null | wc -l | tr -d ' ' )"
VOLJSON_COUNT="$( ls -1 /pools/*/.vol/spdk_info.json 2>/dev/null | wc -l | tr -d ' ' )"

ROLE="standalone"
[ "${CCRD_GEN}" != "absent" ] && ROLE="cluster-node"
[ "${VMJSON_COUNT}" -gt 0 ] && ROLE="${ROLE}/hcp"
[ "${VOLJSON_COUNT}" -gt 0 ] && ROLE="${ROLE}/sds"

#
# preflight
#

if [ "${OPT_DRY}" -eq 0 ]; then
  if [ -e "${OUT}" ] && [ -n "$( ls -A "${OUT}" 2>/dev/null )" ]; then
    printf 'ERROR: output directory is not empty: %s\n' "${OUT}" >&2
    printf 'Pick another --out or move the old collection aside.\n' >&2
    exit 1
  fi
  _parent="$( dirname "${OUT}" )"
  mkdir -p "${_parent}" || { printf 'ERROR: cannot create %s\n' "${_parent}" >&2; exit 1; }
  _free="$( df -k "${_parent}" 2>/dev/null | awk 'NR==2 {print $4}' )"
  case "${_free}" in
    ''|*[!0-9]*) _free=0 ;;
  esac
  if [ "${_free}" -lt 262144 ]; then
    printf 'ERROR: less than 256 MB free on %s (%s KB) -- pick --out on another filesystem\n' \
      "${_parent}" "${_free}" >&2
    exit 1
  fi
fi

if [ "${OPT_DRY}" -eq 0 ]; then
  for t in ${COPY_TARGETS}; do
    if [ ! -d "${t}" ]; then
      printf 'ERROR: --copy-to target is not a directory: %s\n' "${t}" >&2
      printf 'Mount the flash or the share first, then run again.\n' >&2
      exit 1
    fi
    fs_case_sensitive "${t}"
    case $? in
      1) printf 'ERROR: %s is on a case-insensitive filesystem (FAT, exFAT, HFS+)\n' "${t}" >&2
         printf 'The node has files whose names differ only in case, so the copy would\n' >&2
         printf 'silently lose one of each pair. Use a UFS/ZFS target, or take the copy\n' >&2
         printf 'as an archive with --tar.\n' >&2
         exit 1 ;;
      2) printf 'ERROR: cannot write into --copy-to target: %s\n' "${t}" >&2
         exit 1 ;;
    esac
  done
fi

for d in 00-summary 10-identity 20-packages 30-config 40-storage 50-disks \
         60-cluster 70-san 80-vm 90-network 95-runtime; do
  mkd "${OUT}/${d}"
done

if [ "${OPT_DRY}" -eq 0 ]; then
  chmod 700 "${OUT}" 2>/dev/null || true
  LOG="${OUT}/00-summary/collect.log"
  WARNFILE="${OUT}/00-summary/warnings.txt"
  : > "${LOG}"
  : > "${WARNFILE}"
fi

msg "vstackos.config-backup ${VERSION} on ${HOST} at ${TS}"
msg "profile: ${OS_NAME} ${OS_VER}, base ${FBSD_RUNNING}, pkgbase=${PKGBASE}, ccrd=${CCRD_GEN}, san=${SAN_CLI}, role=${ROLE}"
msg "cluster: ${CLUSTER} (${CLUSTER_SRC})"
msg "output: ${OUT}"

if [ "${CCRD_GEN}" != "absent" ] && [ -z "${POOLS}" ]; then
  warn "no pool is imported on this node -- the share layout and pool properties cannot be captured. Collect BEFORE stopping the cluster layer, or run this on the node that currently owns the pools"
fi
if [ "${VOLJSON_COUNT}" -gt 0 ] && [ "${SPDK_PROC}" = "none" ]; then
  warn "share configuration is on the pool but no spdk_tgt/nvmf_tgt is listening -- runtime layout will be missing from this collection"
fi

#
# 10 identity
#

D="${OUT}/10-identity"
run "${D}/date-utc.txt"          date -u
run "${D}/hostname.txt"          hostname
run "${D}/uname.txt"             uname -a
run "${D}/freebsd-version.txt"   freebsd-version -kru
run "${D}/uptime.txt"            uptime
run "${D}/kenv.txt"              kenv
cpf /etc/hostid                  "${D}"
cpf /etc/os-release              "${D}"
cpf /usr/local/etc/vstack-env.conf "${D}"
if [ "${OPT_DRY}" -eq 0 ]; then
  {
    echo "smbios.system.uuid    = $( kenv smbios.system.uuid 2>/dev/null )"
    echo "smbios.system.serial  = $( kenv smbios.system.serial 2>/dev/null )"
    echo "smbios.system.maker   = $( kenv smbios.system.maker 2>/dev/null )"
    echo "smbios.system.product = $( kenv smbios.system.product 2>/dev/null )"
    echo "kern.hostuuid         = $( sysctl -n kern.hostuuid 2>/dev/null )"
  } > "${D}/smbios.txt" 2>/dev/null
fi

#
# 20 packages
#

D="${OUT}/20-packages"
if have pkg; then
  run "${D}/pkg-versions.txt"      pkg query '%n %v'
  run "${D}/pkg-versions-origin.txt" pkg query '%n %v %o %R'
  run "${D}/pkg-repos.txt"         pkg -vv
  run "${D}/pkg-abi.txt"           pkg config abi
else
  warn "pkg not available -- package inventory not collected"
fi
cpt /usr/local/etc/pkg/repos "${D}/repos"
# the local package cache is the only offline source for a downgrade
if [ "${OPT_DRY}" -eq 0 ] && [ -d /var/cache/pkg ]; then
  ls -1 /var/cache/pkg > "${D}/pkg-cache-listing.txt" 2>&1
  {
    printf 'files: %s\n' "$( ls -1 /var/cache/pkg 2>/dev/null | wc -l | tr -d ' ' )"
    printf 'size_kb: %s\n' "$( du -sk /var/cache/pkg 2>/dev/null | awk '{print $1}' )"
  } > "${D}/pkg-cache-summary.txt" 2>&1
fi
cpf /usr/share/keys/vstack/poudriere.pub "${D}" 2>/dev/null
cpf /usr/share/keys/vstack/vstackos.pub  "${D}" 2>/dev/null

#
# 30 config: /etc, /usr/local/etc, /boot conf files, /var/backups
#

D="${OUT}/30-config"
cpt /etc            "${D}/etc"           || warn "/etc not copied"
cpt /usr/local/etc  "${D}/usr-local-etc" || warn "/usr/local/etc not copied"

mkd "${D}/boot"
for f in /boot/loader.conf /boot/loader.conf.local /boot/loader.rc /boot/device.hints \
         /boot/menu.rc /boot/beastie.4th /boot/loader.conf.d; do
  [ -e "${f}" ] || continue
  if [ -d "${f}" ]; then cpt "${f}" "${D}/boot/$( basename "${f}" )"; else cpf "${f}" "${D}/boot"; fi
done
run "${D}/boot/boot-listing.txt" ls -la /boot

mkd "${D}/var-backups"
if [ "${OPT_DRY}" -eq 0 ]; then
  for f in /var/backups/*.bak /var/backups/*.bak2; do
    [ -f "${f}" ] || continue
    cpf "${f}" "${D}/var-backups"
  done
fi
run "${D}/var-backups/listing.txt" ls -la /var/backups

if [ "${OPT_NO_SECRETS}" -eq 1 ] && [ "${OPT_DRY}" -eq 0 ]; then
  rm -f "${D}/etc/master.passwd" "${D}/etc/spwd.db" 2>/dev/null
  for f in "${D}"/etc/ssh/ssh_host_*_key; do
    [ -f "${f}" ] && rm -f "${f}"
  done
  rm -f "${D}/var-backups/master.passwd.bak" 2>/dev/null
  msg "--no-secrets: password database and ssh host private keys removed from the copy"
fi

# runtime value for every tunable written in the config files
if [ "${OPT_DRY}" -eq 0 ]; then
  {
    echo "# key config_value runtime_value"
    for f in /etc/sysctl.conf /etc/sysctl.conf.local /boot/loader.conf /boot/loader.conf.local; do
      [ -f "${f}" ] || continue
      echo "## ${f}"
      grep -E '^[a-zA-Z_][a-zA-Z0-9_.]*=' "${f}" 2>/dev/null | while read -r line; do
        _k="${line%%=*}"
        _v="${line#*=}"
        _r="$( sysctl -n "${_k}" 2>/dev/null )"
        [ -n "${_r}" ] || _r="(not a live sysctl)"
        printf '%s config=%s runtime=%s\n' "${_k}" "${_v}" "${_r}"
      done
    done
  } > "${D}/tunables-config-vs-runtime.txt" 2>/dev/null
fi
run "${D}/md5-key-configs.txt" md5 /etc/rc.conf /etc/rc.conf.local /etc/sysctl.conf \
    /etc/sysctl.conf.local /boot/loader.conf /boot/loader.conf.local /usr/local/etc/vstack-env.conf

#
# 40 storage
#

D="${OUT}/40-storage"
run "${D}/zpool-list.txt"     zpool list -v
run "${D}/zpool-status.txt"   zpool status -v
run "${D}/zfs-list.txt"       zfs list -t all -o name,type,used,avail,refer,mountpoint
cpf /etc/zfs/zpool.cache      "${D}"
cpf /boot/zfs/zpool.cache     "${D}" zpool.cache.boot
for p in ${POOLS}; do
  run "${D}/zpool-get-${p}.txt"      zpool get all "${p}"
  run "${D}/zfs-local-props-${p}.txt" zfs get -r -s local -o name,property,value all "${p}"
  if [ "${OPT_DRY}" -eq 0 ] && have zpool; then
    zpool history "${p}" 2>/dev/null | tail -n 300 > "${D}/zpool-history-tail-${p}.txt"
  fi
done
if [ "${OPT_DRY}" -eq 0 ]; then
  for f in /pools/*/.vol/spdk_info.json; do
    [ -f "${f}" ] || continue
    _pool="$( echo "${f}" | awk -F/ '{print $3}' )"
    cpf "${f}" "${D}" "spdk_info.${_pool}.json"
  done
  ls -la /pools/*/.vol/ > "${D}/vol-dir-listing.txt" 2>&1
fi

#
# 50 disks and boot layout
#

D="${OUT}/50-disks"
run "${D}/kern-disks.txt"     sysctl -n kern.disks
run "${D}/gpart-show.txt"     gpart show -p
run "${D}/geom-disk-list.txt" geom disk list
run "${D}/camcontrol-devlist.txt" camcontrol devlist
run "${D}/nvmecontrol-devlist.txt" nvmecontrol devlist
run "${D}/efibootmgr.txt"     efibootmgr -v
# geom classes are only queried when already loaded: the g* utilities pull in
# their kernel module otherwise, and this script must not change the node
if kldstat -q -m g_label 2>/dev/null; then
  run "${D}/glabel-status.txt"  glabel status
else
  msg "g_label not loaded -- glabel skipped"
fi
if kldstat -q -m g_mirror 2>/dev/null; then
  run "${D}/gmirror-status.txt" gmirror status
  run "${D}/gmirror-list.txt"   gmirror list
else
  msg "g_mirror not loaded -- gmirror skipped"
fi
run "${D}/df.txt"             df -h
run "${D}/swapinfo.txt"       swapinfo -h
if [ "${OPT_DRY}" -eq 0 ] && have gpart; then
  mkd "${D}/gpart-backup"
  for d in $( sysctl -n kern.disks 2>/dev/null ); do
    gpart backup "${d}" > "${D}/gpart-backup/${d}.txt" 2>/dev/null || rm -f "${D}/gpart-backup/${d}.txt"
  done
  _cnt="$( ls -1 "${D}/gpart-backup" 2>/dev/null | wc -l | tr -d ' ' )"
  msg "gpart backup taken for ${_cnt} disks"
fi
if [ "${OPT_WITH_SMART}" -eq 1 ] && [ "${OPT_DRY}" -eq 0 ] && have smartctl; then
  mkd "${D}/smart"
  for d in $( sysctl -n kern.disks 2>/dev/null ); do
    smartctl -i "/dev/${d}" > "${D}/smart/${d}.txt" 2>&1
  done
fi

#
# 60 cluster (ccrd)
#

D="${OUT}/60-cluster"
if [ "${CCRD_GEN}" = "absent" ]; then
  msg "no ccrd on this node -- cluster section skipped"
else
  cpt /usr/local/etc/ccrd "${D}/ccrd-etc"
  cpf /var/db/ccrd.db "${D}" || warn "/var/db/ccrd.db not found"
  if [ "${CCRD_GEN}" = "ng" ]; then
    run "${D}/ccrd-service-list.txt" ccrd_cli service list
    run "${D}/ccrd-node-list.txt"    ccrd_cli node list
    run "${D}/ccrd-cluster-status.txt" ccrd_cli cluster status
  else
    run "${D}/ccrd-registrations.txt" ccrd_cli --command listRegistrations
    run "${D}/ccrd-state.txt"         ccrd_cli
  fi
fi

#
# 70 san / spdk
#

D="${OUT}/70-san"
if [ "${SPDK_PROC}" = "none" ]; then
  msg "no spdk_tgt/nvmf_tgt listening -- SPDK runtime section limited"
fi
for m in nvmf_get_subsystems nvmf_get_targets nvmf_get_transports \
         iscsi_get_target_nodes iscsi_get_portal_groups iscsi_get_initiator_groups \
         bdev_get_bdevs framework_get_config; do
  rpc "${D}/rpc-${m}.json" "${m}"
done
case "${SAN_CLI}" in
  vstack-san)
    run "${D}/cli-nvmf-target-list.txt"  vstack-san nvmf target list
    run "${D}/cli-iscsi-target-list.txt" vstack-san iscsi target list
    run "${D}/cli-export-list.txt"       vstack-san export list
    for p in ${POOLS}; do
      run "${D}/cli-pool-list-backups-${p}.txt" vstack-san pool list-backups "--pool=${p}"
    done
    ;;
  vstack-san-cli)
    run "${D}/cli-nvmf-get-targets.txt"  vstack-san-cli nvmf-get-targets
    run "${D}/cli-iscsi-get-targets.txt" vstack-san-cli iscsi-get-targets
    ;;
esac
if [ -e /dev/cam/ctl ]; then
  run "${D}/ctladm-portlist.txt" ctladm portlist -v
  run "${D}/ctladm-devlist.txt"  ctladm devlist -v
fi
# who is connected right now: both a record of consumers and the pre-window gate
if [ "${OPT_DRY}" -eq 0 ]; then
  netstat -an -f inet 2>/dev/null | grep -E '\.(4420|3260)[^0-9]' > "${D}/initiator-sessions.txt" 2>&1
  {
    printf 'nvme_tcp_4420_established: %s\n' \
      "$( netstat -an -f inet 2>/dev/null | grep '\.4420' | grep -c ESTABLISHED )"
    printf 'iscsi_3260_established: %s\n' \
      "$( netstat -an -f inet 2>/dev/null | grep '\.3260' | grep -c ESTABLISHED )"
  } > "${D}/initiator-sessions-count.txt" 2>&1
fi
cpf /etc/ctl.conf "${D}"

#
# 80 vm
#

D="${OUT}/80-vm"
if [ "${VMJSON_COUNT}" -eq 0 ]; then
  msg "no VM definitions on local pools -- vm section limited"
fi
run "${D}/vm-list.txt" vstack-vm vm list
run "${D}/dev-vmm.txt" ls -la /dev/vmm
if [ "${OPT_DRY}" -eq 0 ]; then
  ps -auxww 2>/dev/null | grep -E 'bhyve' | grep -v grep > "${D}/bhyve-processes.txt" 2>&1
  mkd "${D}/vm-json"
  for f in /pools/*/.vm_V4/*/vm.json; do
    [ -f "${f}" ] || continue
    _pool="$( echo "${f}" | awk -F/ '{print $3}' )"
    _vmid="$( echo "${f}" | awk -F/ '{print $5}' )"
    cpf "${f}" "${D}/vm-json" "${_pool}-${_vmid}-vm.json"
  done
  ls -la /pools/*/.vm_V4/ /pools/*/.vm_V4_state/ > "${D}/vm-dir-listing.txt" 2>&1
  ls -la /pools/*/.vm_images/ > "${D}/vm-images-listing.txt" 2>&1
fi

#
# 90 network
#

D="${OUT}/90-network"
run "${D}/ifconfig-av.txt"    ifconfig -av
run "${D}/ifconfig-l.txt"     ifconfig -l
run "${D}/netstat-rn.txt"     netstat -rnW
run "${D}/netstat-i.txt"      netstat -inW
run "${D}/arp.txt"            arp -an
run "${D}/sockstat.txt"       sockstat -4l -n
run "${D}/ntpq.txt"           ntpq -pn
run "${D}/host-interfaces.txt" vstack-net host list
run "${D}/virtual-networks.txt" vstack-net network list
# ipfw(8) loads the firewall module when it is absent, and an empty ruleset
# defaults to deny: only ask when the module is already there
if kldstat -q -m ipfw 2>/dev/null; then
  run "${D}/ipfw-list.txt" ipfw list
else
  msg "ipfw module not loaded -- firewall dump skipped"
fi

#
# 95 runtime
#

D="${OUT}/95-runtime"
run "${D}/service-enabled.txt" service -e
run "${D}/kldstat.txt"         kldstat -v
run "${D}/ps.txt"              ps -axwwj
run "${D}/pciconf.txt"         pciconf -lv
run "${D}/vmstat.txt"          vmstat -s
run "${D}/zfs-arcstats.txt"    sysctl kstat.zfs.misc.arcstats
cpf /var/run/dmesg.boot "${D}"
if [ "${OPT_FULL_SYSCTL}" -eq 1 ]; then
  run "${D}/sysctl-all.txt" sysctl -a
fi
if [ "${OPT_WITH_LOGS}" -eq 1 ] && [ "${OPT_DRY}" -eq 0 ]; then
  mkd "${D}/logs"
  for f in /var/log/messages /var/log/ccrd.log /var/log/cluster.log /var/log/vstack-san.log; do
    [ -f "${f}" ] || continue
    tail -n 2000 "${f}" > "${D}/logs/$( basename "${f}" ).tail" 2>/dev/null
  done
  for f in /var/log/ccrd_*.log; do
    [ -f "${f}" ] || continue
    tail -n 2000 "${f}" > "${D}/logs/$( basename "${f}" ).tail" 2>/dev/null
  done
fi

#
# checks and summary
#

if [ "${OPT_DRY}" -eq 1 ]; then
  printf '\ndry run finished, nothing written\n'
  exit 0
fi

# third argument: the file on the node this artefact comes from. An empty source
# is a finding about the node and must not read as a failed collection.
check() {
  _what="$1"; _path="$2"; _src="${3:-}"
  if [ -s "${OUT}/${_path}" ]; then
    printf 'PASS  %s\n' "${_what}"
  elif [ -n "${_src}" ] && [ -f "${_src}" ] && [ ! -s "${_src}" ]; then
    printf 'FAIL  %s is EMPTY ON THE NODE (%s) -- collection is fine, the node is not\n' \
      "${_what}" "${_src}"
    warn "${_src} is empty on the node"
  else
    printf 'FAIL  %s (%s missing or empty)\n' "${_what}" "${_path}"
    warn "check failed: ${_what}"
  fi
}

CHECKS="${OUT}/00-summary/checks.txt"
{
  check "rc.conf copied"            "30-config/etc/rc.conf"
  check "hostid copied"             "10-identity/hostid"
  check "loader.conf copied"        "30-config/boot/loader.conf"
  check "package inventory"         "20-packages/pkg-versions.txt"
  check "pkg repo configuration"    "20-packages/pkg-repos.txt"
  check "disk partitioning"         "50-disks/gpart-show.txt"
  check "zpool status"              "40-storage/zpool-status.txt"
  if [ -f /usr/local/etc/vstack-env.conf ]; then
    check "vstack-env.conf" "10-identity/vstack-env.conf" /usr/local/etc/vstack-env.conf
  fi
  if [ "${CCRD_GEN}" != "absent" ]; then
    check "ccrd config.json copied" "60-cluster/ccrd-etc/config.json"
    check "ccrd.db copied"          "60-cluster/ccrd.db"
  fi
  if [ "${VMJSON_COUNT}" -gt 0 ]; then
    _n="$( ls -1 "${OUT}/80-vm/vm-json" 2>/dev/null | wc -l | tr -d ' ' )"
    if [ "${_n}" -eq "${VMJSON_COUNT}" ]; then
      printf 'PASS  vm.json copied (%s)\n' "${_n}"
    else
      printf 'FAIL  vm.json copied %s of %s\n' "${_n}" "${VMJSON_COUNT}"
      warn "vm.json copied ${_n} of ${VMJSON_COUNT}"
    fi
  fi
  if [ "${VOLJSON_COUNT}" -gt 0 ]; then
    _n="$( ls -1 "${OUT}/40-storage"/spdk_info.*.json 2>/dev/null | wc -l | tr -d ' ' )"
    printf 'INFO  spdk_info.json copied: %s of %s\n' "${_n}" "${VOLJSON_COUNT}"
    # Count subsystem objects, not every "nqn" key. The RPC dump also carries
    # the NQN of every allowed initiator inside "hosts", and those were counted
    # too: a node with 3 targets and one initiator each reported 6 subsystems.
    # "subtype" appears once per subsystem and tells NVMe from Discovery, so it
    # needs no filtering. grep -c would count LINES and the dump is one line.
    _live_nqn="$( grep -o '"subtype": *"NVMe"' "${OUT}/70-san/rpc-nvmf_get_subsystems.json" 2>/dev/null \
                  | wc -l | tr -d ' ' )"
    [ -n "${_live_nqn}" ] || _live_nqn=0
    printf 'INFO  nvmf subsystems in runtime (without discovery): %s\n' "${_live_nqn}"
    # a stub file is the failure seen in the field: every list emptied at once,
    # a couple of hundred bytes. An empty nvmf list next to live iscsi data is
    # normal -- internal-fabric subsystems are rebuilt at start, not persisted.
    for _f in "${OUT}/40-storage"/spdk_info.*.json; do
      [ -f "${_f}" ] || continue
      _sz="$( wc -c < "${_f}" | tr -d ' ' )"
      if [ "${_sz}" -lt 256 ] && [ "${_live_nqn}" -gt 0 ]; then
        printf 'WARN  %s is a stub (%s bytes) while runtime serves %s subsystems\n' \
          "$( basename "${_f}" )" "${_sz}" "${_live_nqn}"
        warn "$( basename "${_f}" ) collapsed to a stub while the runtime is serving -- the RPC dump in 70-san is the only copy of the share layout"
      fi
    done
  fi
} > "${CHECKS}" 2>&1

cat > "${OUT}/00-summary/RESTORE-HINTS.txt" <<HINTS
What this collection is for
---------------------------
Rebuilding this node by hand after a failed update: same identity, same disk
layout, same services, same shares. It is not a system image and not a data
backup -- pool data is not here.

Where things go back
--------------------
30-config/etc/               -> /etc            (whole tree; the node identity lives here)
30-config/etc/hostid         -> /etc/hostid     UNIQUE PER NODE, never restore onto another node
30-config/etc/rc.conf*       -> /etc            what starts on boot
30-config/etc/ssh/           -> /etc/ssh        host keys; without them clients see a changed host
30-config/usr-local-etc/     -> /usr/local/etc  controller configuration, pkg repo layout
30-config/boot/loader.conf*  -> /boot           tunables; the updater does NOT back these up
10-identity/vstack-env.conf  -> /usr/local/etc  install-time node identity (cluster id, node id, laggs)
60-cluster/ccrd-etc/         -> /usr/local/etc/ccrd
60-cluster/ccrd.db           -> /var/db/ccrd.db
50-disks/gpart-backup/<d>.txt-> gpart restore -F /dev/<d> < file   (empty disk only)

Order that works
----------------
1  install the node from the memstick of the target version
2  restore /etc/hostid, /etc/rc.conf*, /etc/ssh, vstack-env.conf, loader.conf*
3  restore /usr/local/etc/pkg/repos, install the same package set (20-packages)
4  restore ccrd config.json and ccrd.db, start the cluster layer
5  import pools, compare the share layout with 40-storage/spdk_info.*.json
   and 70-san/rpc-nvmf_get_subsystems.json, then hand the pools back to ccrd

Read before restoring
---------------------
- Never mix collections of two nodes: hostid, node_id in vstack-env.conf and
  ccrd.db are per node.
- Keep this tree on a case-sensitive filesystem. Names here differ only in case
  (vStack.conf and vstack.conf side by side); a FAT flash, a Windows disk or a
  stock macOS volume merges them and drops a file. On such a machine keep the
  archive instead and unpack it where case matters.
- 70-san/rpc-*.json is the runtime truth about the share layout. If the
  persisted spdk_info.json on the pool was zeroed by the old daemon during
  shutdown, the RPC dump is the only copy of what was exported.
- Config files here are from BEFORE the update. After it, do not blindly
  overwrite new package configuration -- compare first.
- MANIFEST.sha256 covers every collected file. The run's own report files
  (SUMMARY.txt, collect.log, warnings.txt) are finished after it and are
  deliberately left out.
HINTS

# the run's own report files keep growing after this point, so they stay out of
# the manifest: it covers the collected material, which is what a copy must keep
MSKIP="! -name MANIFEST.sha256 ! -name collect.log ! -name warnings.txt ! -name SUMMARY.txt"
if have sha256; then
  ( cd "${OUT}" && find . -type f ${MSKIP} -exec sha256 -r {} + \
      > "${OUT}/00-summary/MANIFEST.sha256" 2>/dev/null )
elif have md5; then
  ( cd "${OUT}" && find . -type f ${MSKIP} -exec md5 -r {} + \
      > "${OUT}/00-summary/MANIFEST.sha256" 2>/dev/null )
else
  warn "no sha256/md5 -- manifest not written"
fi

COPIES_DONE=""
for t in ${COPY_TARGETS}; do
  _dst="${t}/$( basename "${OUT}" )"
  if [ -e "${_dst}" ]; then
    warn "copy target already exists, skipped: ${_dst}"
    continue
  fi
  if cp -a "${OUT}" "${_dst}" 2>>"${LOG}"; then
    if verify_collection "${_dst}" >> "${LOG}" 2>&1; then
      msg "copy verified: ${_dst}"
      COPIES_DONE="${COPIES_DONE} ${_dst}"
    else
      warn "copy made but checksums do not match: ${_dst}"
    fi
  else
    warn "copy failed: ${_dst}"
  fi
done

SIZE_KB="$( du -sk "${OUT}" 2>/dev/null | awk '{print $1}' )"
FILES="$( find "${OUT}" -type f 2>/dev/null | wc -l | tr -d ' ' )"

{
  echo "vstackos.config-backup ${VERSION}"
  echo ""
  echo "collected_at_utc : ${TS}"
  echo "hostname         : $( hostname )"
  echo "cluster          : ${CLUSTER} (${CLUSTER_SRC})"
  echo "tag              : ${TAG}"
  echo ""
  echo "os_name          : ${OS_NAME}"
  echo "os_version       : ${OS_VER}"
  echo "base_running     : ${FBSD_RUNNING}"
  echo "base_major       : ${FBSD_MAJOR}"
  echo "pkgbase          : ${PKGBASE}"
  echo "ccrd_generation  : ${CCRD_GEN}"
  echo "san_cli          : ${SAN_CLI}"
  echo "spdk_process     : ${SPDK_PROC}"
  echo "node_role        : ${ROLE}"
  echo "pools_imported   : ${POOLS}"
  echo "vm_definitions   : ${VMJSON_COUNT}"
  echo "vol_share_files  : ${VOLJSON_COUNT}"
  echo ""
  echo "elapsed_sec      : $(( $( date -u '+%s' ) - START_EPOCH ))"
  echo "files            : ${FILES}"
  echo "size_kb          : ${SIZE_KB}"
  echo "warnings         : ${WARN_COUNT}"
  echo "secrets_included : $( [ "${OPT_NO_SECRETS}" -eq 1 ] && echo no || echo yes )"
  echo ""
  echo "copies:"
  echo "  on node        : ${OUT}"
  if [ -n "${COPIES_DONE}" ]; then
    for c in ${COPIES_DONE}; do echo "  verified copy  : ${c}"; done
  else
    echo "  verified copy  : none (--copy-to not used)"
  fi
  echo "  off node       : operator step, see below"
  echo ""
  echo "checks:"
  sed 's/^/  /' "${CHECKS}"
} > "${OUT}/00-summary/SUMMARY.txt" 2>&1


msg ""
msg "done: ${FILES} files, ${SIZE_KB} KB, ${WARN_COUNT} warnings"
msg "summary : ${OUT}/00-summary/SUMMARY.txt"
msg "checks  : ${OUT}/00-summary/checks.txt"

if [ "${OPT_TAR}" -eq 1 ]; then
  # the archive travels on its own (ticket, mail), so its name repeats the tool
  _tar="$( dirname "${OUT}" )/vstackos-config-backup-${HOST}-${DAY}.tar.gz"
  if tar czf "${_tar}" -C "$( dirname "${OUT}" )" "$( basename "${OUT}" )" 2>>"${LOG}"; then
    msg "archive : ${_tar}"
  else
    warn "archive failed"
  fi
fi

msg ""
msg "keep this in three places:"
msg "  1 on the node   : ${OUT}   [done]"
if [ -n "${COPIES_DONE}" ]; then
  for c in ${COPIES_DONE}; do msg "  2 second media  : ${c}   [done, checksums match]"; done
else
  msg "  2 second media  : mount the boot flash and rerun with --copy-to <mountpoint>   [NOT done]"
fi
msg "  3 off the node  : scp -r root@$( hostname ):${OUT} .   [NOT done, operator step]"
msg "      then verify the copy: sh vstackos.config-backup.sh --verify <local-dir>"

if [ "${WARN_COUNT}" -gt 0 ]; then
  msg "finished with ${WARN_COUNT} warnings -- see 00-summary/warnings.txt"
  exit 2
fi
exit 0