While we're working on an easier system for ERDA mounting on Slurm-HPC, we've tested and provided this script for use until then. Remember to add your own ERDA password and username into the script.

To test that your sftp username and password works, try the following command

rclone lsd :sftp: --sftp-host io.erda.au.dk --sftp-port 2222 --sftp-user username@ece.au.dk --sftp-pass "$(rclone obscure 'YOUR_SFTP_PASSWORD_HERE')"

Then you can try running this script locally first, ensure it works, and then run it with sbatch.

#!/bin/bash

#SBATCH --job-name=yourjobname
#SBATCH --partition=min


# --- SAFETY FLAGS ---
# -e: Exit immediately if a command exits with a non-zero status.
# -u: Treat unset variables as an error and exit immediately.
# -o pipefail: Return the exit status of the last command in the pipeline that failed.
set -euo pipefail

# ==== Pre-install rclone for password obscure ====
echo "Installing rclone..."
if ! command -v rclone >/dev/null 2>&1; then
  curl -fsSL https://rclone.org/install.sh | sudo bash
fi


# ===== CONFIG =====
REMOTE_NAME="ERDA"
SFTP_HOST="io.erda.au.dk"
SFTP_USERNAME="YOUR_ERDA_EMAIL_HERE"
SFTP_PORT="2222"
# for password - go into erda -> bottom left user icon -> setup -> SFTP -> put something in the password
SFTP_PASSWORD="$(rclone obscure 'YOUR_ERDA_PASSWORD_HERE')"
# ===== CONFIG END =====

# ==== System Config - Dont touch unless you know what youre doing ====

RCLONE_REMOTE="sftp-remote:"
MOUNT_POINT="/mnt/sftp"

TMP_DIR="$(mktemp -d)"
RCLONE_LOG="$TMP_DIR/rclone.log"
RCLONE_CONFIG="$TMP_DIR/rclone.conf"

# Clean up old runs

# If rclone exists, we want to try and gracefully shut it down, so we don't lose files that are currently being uploaded/downloaded
# We get the PID first so we can track it specifically
RCLONE_PID=$(pgrep -f "rclone")

if [ -n "$RCLONE_PID" ]; then
    echo "Found active rclone mount (PID: $RCLONE_PID). Signaling shutdown..."
    sudo kill "$RCLONE_PID"

    # Wait for the process to disappear (up to some timeout in seconds)
    TIMEOUT=60
    while kill -0 "$RCLONE_PID" 2>/dev/null && [ $TIMEOUT -gt 0 ]; do
        sleep 1
        ((TIMEOUT--))
        echo "Waiting for rclone to exit... ($TIMEOUTs remaining)"
    done

    # If it's still alive after the timeout, force it to die
    if kill -0 "$RCLONE_PID" 2>/dev/null; then
        echo "Rclone failed to exit gracefully. Forcing..."
        sudo kill -9 "$RCLONE_PID"
    fi
fi

# Final Cleanup of mount point
sudo umount -l "$MOUNT_POINT" 2>/dev/null
sudo rm -rf "$MOUNT_POINT"

cleanup() {
  fusermount -u "$MOUNT_POINT" 2>/dev/null || true
  rm -rf "$TMP_DIR"
  kill $RCLONE_PID 2>/dev/null || true
}
trap cleanup EXIT



# Install rclone and fusermount if not present

if ! command -v fusermount >/dev/null 2>&1 && ! command -v fusermount3 >/dev/null 2>&1; then
  echo "Installing fuse utilities..."
  sudo dnf install -y fuse fuse3 fuse-sshfs || true
fi

# Ensure fuse conf is readable by all.
sudo chmod +r /etc/fuse.conf


# There is some errors sometimes where rclone will try to use the old name for FUSE. We just symlink them here and it should be fine
if ! command -v fusermount >/dev/null 2>&1 && command -v fusermount3 >/dev/null 2>&1; then
  echo "Creating symlink for fusermount -> fusermount3"
  sudo ln -sf "$(which fusermount3)" /usr/local/bin/fusermount
fi

sudo mkdir -p "$MOUNT_POINT"
sudo chown "$USER":"$USER" "$MOUNT_POINT"

cat > "$RCLONE_CONFIG" <<EOF
[sftp-remote]
type = sftp
host = ${SFTP_HOST}
port = ${SFTP_PORT}
user = ${SFTP_USERNAME}
pass = ${SFTP_PASSWORD}
EOF

# Test remote reachability (optional quick TCP check)
echo "Testing SFTP host reachability..."
if ! timeout 10 bash -c "</dev/tcp/${SFTP_HOST}/${SFTP_PORT}" 2>/dev/null; then
  echo "WARNING: SFTP host not reachable. Proceeding; mount may fail."
fi

echo "Testing rclone remote access with 'lsf'..."
if ! rclone --config "$RCLONE_CONFIG" lsf "$RCLONE_REMOTE" -d --max-depth 1 --timeout=30s; then
  echo "ERROR: rclone remote '$RCLONE_REMOTE' is not accessible (credentials/network)."
  exit 1
fi

# Ensure FUSE allows access for non-owner (Docker daemon runs as root)
if sudo test -f /etc/fuse.conf && ! sudo grep -q '^user_allow_other' /etc/fuse.conf; then
  echo "Enabling user_allow_other in /etc/fuse.conf..."
  echo 'user_allow_other' | sudo tee -a /etc/fuse.conf >/dev/null
fi

# Mount remote using rclone (daemonized) with external log
echo "Mounting remote $RCLONE_REMOTE at $MOUNT_POINT using rclone..."
set +e
rclone --config "$RCLONE_CONFIG" mount "$RCLONE_REMOTE" "$MOUNT_POINT" \
  --vfs-cache-mode full \
  --timeout=60s \
  --contimeout=20s \
  --log-level DEBUG \
  --log-file "$RCLONE_LOG" \
  --allow-other &
RCLONE_PID=$!

RC=$?
set -e

if [ $RC -ne 0 ]; then
  echo "ERROR: rclone mount failed with exit code $RC"
  echo "----- rclone log (last 200 lines) -----"
  tail -n 200 "$RCLONE_LOG" || true
  exit $RC
fi

# Wait for mount readiness
echo "Waiting for rclone mount to become ready..."
READY=0
for i in {1..30}; do
  if timeout 5 ls "$MOUNT_POINT" >/dev/null 2>&1; then
    READY=1
    echo "rclone mount is ready at $MOUNT_POINT"
    break
  fi
  sleep 1
done
if [ $READY -ne 1 ]; then
  echo "ERROR: rclone mount did not become ready in time"
  echo "----- rclone log (last 200 lines) -----"
  tail -n 200 "$RCLONE_LOG" || true
  exit 1
fi

# Sleep a little for good luck, since rclone might sometimes not actually be fully synced even if its mounted.
sleep 10
# YOUR SCRIPT GOES HERE

echo sleeping 30
sleep 30