dao_hades/scripts/reboot/mount.sh

#!/usr/bin/env bash
set -euo pipefail

# shellcheck disable=1091
. /usr/local/share/dao/config/dao.conf

readonly LOG_PREFIX="[dao_am.service]"
readonly MAX_RETRIES=30
readonly RETRY_DELAY=10

# Configuration
declare -A RCLONE_MOUNTS=(
	["koofr"]="${DAO_STORAGE_DIR}/koofr:false"
	["koofr_vault"]="${DAO_STORAGE_DIR}/vault:true"
)

declare -A SSHFS_MOUNTS=(
	["hephaestus"]="/home/oc/dao:/home/jamie/dao/storage/hephaestus"
)

# Options
readonly BASE_RCLONE_OPTS=(
	--vfs-cache-mode writes
	--cache-dir /tmp/rclone-cache
	--dir-cache-time 5m
	--poll-interval 1m
	--timeout 1h
	--low-level-retries 10
	--retries 3
	--vfs-cache-max-size 10G
	--vfs-cache-max-age 24h
	--buffer-size 256M
	--transfers 8
	--checkers 8
	--allow-non-empty
	--allow-other
	--umask 000
)

readonly CRYPT_RCLONE_OPTS=(
	--buffer-size 256M
	--transfers 8
	--vfs-read-ahead 256M
	--vfs-read-chunk-size 128M
	--vfs-read-chunk-size-limit 2G
)

readonly SSHFS_OPTS=(
	-o allow_other
	-o reconnect
	-o ServerAliveInterval=30
	-o ServerAliveCountMax=3
)

# Track mount attempts and failures
declare -A RCLONE_ATTEMPTS
declare -A SSHFS_ATTEMPTS

log() {
	echo "${LOG_PREFIX} $*" >&2
}

is_mounted() {
	local mount_point="$1"
	grep -q " ${mount_point} " /proc/mounts
}

mount_rclone() {
	local remote="$1" mount_point="$2" is_crypt="$3"

	if is_mounted "$mount_point"; then
		log "rclone $remote already mounted at $mount_point"
		return 0
	fi

	local opts=("${BASE_RCLONE_OPTS[@]}")
	[[ "$is_crypt" == "true" ]] && opts+=("${CRYPT_RCLONE_OPTS[@]}")

	log "Mounting rclone: $remote -> $mount_point (attempt $((RCLONE_ATTEMPTS[$remote] + 1)))"
	/usr/bin/rclone mount "$remote:" "$mount_point" "${opts[@]}" &

	# Give it a moment to attempt the mount
	sleep 2

	if is_mounted "$mount_point"; then
		log "Successfully mounted rclone: $remote"
		return 0
	else
		log "Failed to mount rclone: $remote"
		return 1
	fi
}

mount_sshfs() {
	local remote="$1" mount_point="$2"

	if is_mounted "$mount_point"; then
		log "sshfs $remote already mounted at $mount_point"
		return 0
	fi

	log "Mounting sshfs: $remote -> $mount_point (attempt $((SSHFS_ATTEMPTS[$remote] + 1)))"
	if /usr/bin/sshfs "$remote" "$mount_point" "${SSHFS_OPTS[@]}" 2>/dev/null; then
		log "Successfully mounted sshfs: $remote"
		return 0
	else
		log "Failed to mount sshfs: $remote"
		return 1
	fi
}

ensure_mounts() {
	local failed_rclone=()
	local failed_sshfs=()

	# First pass: attempt all mounts
	log "First pass: attempting all mounts"

	# Handle rclone mounts
	for remote in "${!RCLONE_MOUNTS[@]}"; do
		IFS=':' read -r mount_point is_crypt <<<"${RCLONE_MOUNTS[$remote]}"
		RCLONE_ATTEMPTS[$remote]=0
		if ! mount_rclone "$remote" "$mount_point" "$is_crypt"; then
			failed_rclone+=("$remote")
		fi
	done

	# Handle sshfs mounts
	for remote in "${!SSHFS_MOUNTS[@]}"; do
		IFS=':' read -r remote_path mount_point <<<"${SSHFS_MOUNTS[$remote]}"
		SSHFS_ATTEMPTS[$remote]=0
		if ! mount_sshfs "${remote}:${remote_path}" "$mount_point"; then
			failed_sshfs+=("$remote")
		fi
	done

	# Retry failed mounts
	while [[ ${#failed_rclone[@]} -gt 0 || ${#failed_sshfs[@]} -gt 0 ]]; do
		log "Retrying failed mounts in ${RETRY_DELAY} seconds..."
		sleep "$RETRY_DELAY"

		# Clear failed arrays for this round
		local current_failed_rclone=()
		local current_failed_sshfs=()

		# Retry rclone mounts
		for remote in "${failed_rclone[@]}"; do
			IFS=':' read -r mount_point is_crypt <<<"${RCLONE_MOUNTS[$remote]}"
			RCLONE_ATTEMPTS[$remote]=$((RCLONE_ATTEMPTS[$remote] + 1))

			if [[ ${RCLONE_ATTEMPTS[$remote]} -ge $MAX_RETRIES ]]; then
				log "rclone $remote: reached max retries ($MAX_RETRIES), giving up"
				continue
			fi

			if mount_rclone "$remote" "$mount_point" "$is_crypt"; then
				log "rclone $remote: mount successful on retry"
			else
				current_failed_rclone+=("$remote")
			fi
		done

		# Retry sshfs mounts
		for remote in "${failed_sshfs[@]}"; do
			IFS=':' read -r remote_path mount_point <<<"${SSHFS_MOUNTS[$remote]}"
			SSHFS_ATTEMPTS[$remote]=$((SSHFS_ATTEMPTS[$remote] + 1))

			if [[ ${SSHFS_ATTEMPTS[$remote]} -ge $MAX_RETRIES ]]; then
				log "sshfs $remote: reached max retries ($MAX_RETRIES), giving up"
				continue
			fi

			if mount_sshfs "${remote}:${remote_path}" "$mount_point"; then
				log "sshfs $remote: mount successful on retry"
			else
				current_failed_sshfs+=("$remote")
			fi
		done

		# Update failed arrays for next iteration
		failed_rclone=("${current_failed_rclone[@]}")
		failed_sshfs=("${current_failed_sshfs[@]}")

		# If both arrays are empty, we're done
		if [[ ${#failed_rclone[@]} -eq 0 && ${#failed_sshfs[@]} -eq 0 ]]; then
			log "All mounts successful"
			break
		fi
	done
}

main() {
	log "Starting mount daemon"
	ensure_mounts
		log "Mount operations completed, sleeping"

	while true; do
        sleep 3600  # Sleep for an hour, then check again if needed
    done
}

main "$@"