#!/usr/bin/env bash set -euo pipefail # shellcheck disable=1091 . /usr/local/share/dao/config/dao.conf readonly LOG_PREFIX="[dao_am.service]" readonly MAX_RETRIES=90 readonly RETRY_DELAY=2 # Configuration declare -A RCLONE_MOUNTS=( ["koofr"]="${DAO_STORAGE_DIR}/koofr:false" ["koofr_vault"]="${DAO_STORAGE_DIR}/vault:true" ) declare -A SSHFS_MOUNTS=( ["hephaestus"]="/home/oc/dao:/home/jamie/dao/servers/hephaestus" ["pan_lms"]="/home/tc:/home/jamie/dao/servers/pan" ) # Options readonly BASE_RCLONE_OPTS=( --vfs-cache-mode writes --cache-dir /tmp/rclone-cache --dir-cache-time 5m --poll-interval 1m --timeout 1h --low-level-retries 10 --retries 3 --vfs-cache-max-size 10G --vfs-cache-max-age 24h --buffer-size 256M --transfers 8 --checkers 8 --allow-non-empty --allow-other --umask 000 ) readonly CRYPT_RCLONE_OPTS=( --buffer-size 256M --transfers 8 --vfs-read-ahead 256M --vfs-read-chunk-size 128M --vfs-read-chunk-size-limit 2G ) readonly SSHFS_OPTS=( -o allow_other -o reconnect -o ServerAliveInterval=30 -o ServerAliveCountMax=3 ) # Track mount attempts and failures declare -A RCLONE_ATTEMPTS declare -A SSHFS_ATTEMPTS log() { echo "${LOG_PREFIX} $*" >&2 } is_mounted() { local mount_point="$1" grep -q " ${mount_point} " /proc/mounts } mount_rclone() { local remote="$1" mount_point="$2" is_crypt="$3" if is_mounted "$mount_point"; then log "rclone $remote already mounted at $mount_point" return 0 fi local opts=("${BASE_RCLONE_OPTS[@]}") [[ "$is_crypt" == "true" ]] && opts+=("${CRYPT_RCLONE_OPTS[@]}") log "Mounting rclone: $remote -> $mount_point (attempt $((RCLONE_ATTEMPTS[$remote] + 1)))" /usr/bin/rclone mount "$remote:" "$mount_point" "${opts[@]}" & # Give it a moment to attempt the mount sleep 2 if is_mounted "$mount_point"; then log "Successfully mounted rclone: $remote" return 0 else log "Failed to mount rclone: $remote" return 1 fi } mount_sshfs() { local remote="$1" mount_point="$2" if is_mounted "$mount_point"; then log "sshfs $remote already mounted at $mount_point" return 0 fi log "Mounting sshfs: $remote -> $mount_point (attempt $((SSHFS_ATTEMPTS[$remote] + 1)))" if /usr/bin/sshfs "$remote" "$mount_point" "${SSHFS_OPTS[@]}" 2>/dev/null; then log "Successfully mounted sshfs: $remote" return 0 else log "Failed to mount sshfs: $remote" return 1 fi } ensure_mounts() { local failed_rclone=() local failed_sshfs=() # First pass: attempt all mounts log "First pass: attempting all mounts" # Handle rclone mounts for remote in "${!RCLONE_MOUNTS[@]}"; do IFS=':' read -r mount_point is_crypt <<<"${RCLONE_MOUNTS[$remote]}" RCLONE_ATTEMPTS[$remote]=0 if ! mount_rclone "$remote" "$mount_point" "$is_crypt"; then failed_rclone+=("$remote") fi done # Handle sshfs mounts for remote in "${!SSHFS_MOUNTS[@]}"; do IFS=':' read -r remote_path mount_point <<<"${SSHFS_MOUNTS[$remote]}" SSHFS_ATTEMPTS[$remote]=0 if ! mount_sshfs "${remote}:${remote_path}" "$mount_point"; then failed_sshfs+=("$remote") fi done # Retry failed mounts while [[ ${#failed_rclone[@]} -gt 0 || ${#failed_sshfs[@]} -gt 0 ]]; do log "Retrying failed mounts in ${RETRY_DELAY} seconds..." sleep "$RETRY_DELAY" # Clear failed arrays for this round local current_failed_rclone=() local current_failed_sshfs=() # Retry rclone mounts for remote in "${failed_rclone[@]}"; do IFS=':' read -r mount_point is_crypt <<<"${RCLONE_MOUNTS[$remote]}" RCLONE_ATTEMPTS[$remote]=$((RCLONE_ATTEMPTS[$remote] + 1)) if [[ ${RCLONE_ATTEMPTS[$remote]} -ge $MAX_RETRIES ]]; then log "rclone $remote: reached max retries ($MAX_RETRIES), giving up" continue fi if mount_rclone "$remote" "$mount_point" "$is_crypt"; then log "rclone $remote: mount successful on retry" else current_failed_rclone+=("$remote") fi done # Retry sshfs mounts for remote in "${failed_sshfs[@]}"; do IFS=':' read -r remote_path mount_point <<<"${SSHFS_MOUNTS[$remote]}" SSHFS_ATTEMPTS[$remote]=$((SSHFS_ATTEMPTS[$remote] + 1)) if [[ ${SSHFS_ATTEMPTS[$remote]} -ge $MAX_RETRIES ]]; then log "sshfs $remote: reached max retries ($MAX_RETRIES), giving up" continue fi if mount_sshfs "${remote}:${remote_path}" "$mount_point"; then log "sshfs $remote: mount successful on retry" else current_failed_sshfs+=("$remote") fi done # Update failed arrays for next iteration failed_rclone=("${current_failed_rclone[@]}") failed_sshfs=("${current_failed_sshfs[@]}") # If both arrays are empty, we're done if [[ ${#failed_rclone[@]} -eq 0 && ${#failed_sshfs[@]} -eq 0 ]]; then log "All mounts successful" break fi done } main() { log "Starting mount daemon" ensure_mounts log "Mount operations completed, sleeping" while true; do sleep 3600 # Sleep for an hour, then check again if needed done } main "$@"