Disaster Recovery for Docker Stacks

1# Backup directory structure
2backups/
3├── daily/
4│   ├── 2024-01-15/
5│   │   ├── volumes/
6│   │   ├── configs/
7│   │   └── manifest.txt
8│   └── 2024-01-14/
9├── weekly/
10└── offsite/
11    └── synced-to-cloud/
12
13# What goes where:
14# - daily/: Last 7 days of backups
15# - weekly/: Keep 4 weekly backups
16# - offsite/: Sync to cloud storage daily

1# Method 1: Direct tar backup (container stopped)
2docker compose stop myservice
3docker run --rm -v myvolume:/data -v $(pwd)/backups:/backup alpine \
4  tar czf /backup/myvolume-$(date +%Y%m%d).tar.gz -C /data .
5docker compose start myservice
6
7# Method 2: Using docker-volume-backup container
8services:
9  backup:
10    image: offen/docker-volume-backup:latest
11    environment:
12      - BACKUP_CRON_EXPRESSION=0 2 * * *
13      - BACKUP_RETENTION_DAYS=7
14    volumes:
15      - myvolume:/backup/myvolume:ro
16      - ./backups:/archive
17      - /var/run/docker.sock:/var/run/docker.sock:ro
18
19# Method 3: Rsync from bind mounts
20rsync -av --delete /docker/appdata/ /backups/appdata/
21
22# Method 4: For databases, use native dump tools
23docker exec postgres pg_dump -U user dbname > backup.sql
24docker exec mariadb mysqldump -u root -p dbname > backup.sql

1#!/bin/bash
2# docker-backup.sh
3
4set -e
5
6BACKUP_ROOT="/backups"
7DATE=$(date +%Y%m%d-%H%M)
8BACKUP_DIR="$BACKUP_ROOT/daily/$DATE"
9COMPOSE_DIR="/docker"
10
11mkdir -p "$BACKUP_DIR"/{volumes,databases,configs}
12
13echo "Starting backup at $(date)"
14
15# Backup docker-compose files and configs
16cp -r "$COMPOSE_DIR"/*.yml "$BACKUP_DIR/configs/"
17cp -r "$COMPOSE_DIR"/*.env "$BACKUP_DIR/configs/" 2>/dev/null || true
18cp -r "$COMPOSE_DIR"/config "$BACKUP_DIR/configs/" 2>/dev/null || true
19
20# Database backups (while running)
21echo "Backing up databases..."
22docker exec postgres pg_dump -U postgres mydb | gzip > "$BACKUP_DIR/databases/postgres.sql.gz"
23docker exec mariadb mysqldump -u root -p"$MYSQL_ROOT_PASSWORD" --all-databases | gzip > "$BACKUP_DIR/databases/mariadb.sql.gz"
24
25# Volume backups
26echo "Backing up volumes..."
27for volume in $(docker volume ls -q); do
28  echo "  Backing up $volume..."
29  docker run --rm -v "$volume":/data -v "$BACKUP_DIR/volumes":/backup alpine \
30    tar czf "/backup/$volume.tar.gz" -C /data .
31done
32
33# Create manifest
34echo "Creating manifest..."
35ls -la "$BACKUP_DIR"/* > "$BACKUP_DIR/manifest.txt"
36docker ps -a >> "$BACKUP_DIR/manifest.txt"
37
38# Cleanup old backups (keep 7 days)
39find "$BACKUP_ROOT/daily" -maxdepth 1 -type d -mtime +7 -exec rm -rf {} \;
40
41echo "Backup complete at $(date)"
42echo "Backup size: $(du -sh "$BACKUP_DIR" | cut -f1)"

1# Option 1: Rclone to cloud storage
2# Configure rclone first: rclone config
3
4# Sync backups to cloud
5rclone sync /backups remote:docker-backups --progress
6
7# In cron (after local backup completes)
80 4 * * * rclone sync /backups/daily backblaze:mybucket/docker/daily
9
10# Option 2: Restic to various backends
11# Initialize repository
12restic -r b2:mybucket:docker init
13
14# Backup with deduplication
15restic -r b2:mybucket:docker backup /backups/daily/latest
16
17# Option 3: Duplicati (GUI-based)
18services:
19  duplicati:
20    image: linuxserver/duplicati:latest
21    volumes:
22      - /backups:/source:ro
23      - ./duplicati-config:/config
24    ports:
25      - "8200:8200"
26
27# Option 4: Simple rsync to another server
28rsync -avz --delete /backups/ user@backup-server:/docker-backups/

1#!/bin/bash
2# docker-restore.sh
3
4BACKUP_DIR=$1
5COMPOSE_DIR="/docker"
6
7if [ -z "$BACKUP_DIR" ]; then
8  echo "Usage: $0 /path/to/backup/2024-01-15"
9  exit 1
10fi
11
12echo "Restoring from $BACKUP_DIR"
13echo "This will overwrite current data. Continue? (yes/no)"
14read confirm
15[ "$confirm" != "yes" ] && exit 1
16
17# Stop all containers
18cd "$COMPOSE_DIR"
19docker compose down
20
21# Restore configs
22cp "$BACKUP_DIR/configs/"*.yml "$COMPOSE_DIR/"
23cp "$BACKUP_DIR/configs/"*.env "$COMPOSE_DIR/" 2>/dev/null || true
24
25# Restore volumes
26for archive in "$BACKUP_DIR/volumes/"*.tar.gz; do
27  volume=$(basename "$archive" .tar.gz)
28  echo "Restoring volume $volume..."
29  docker volume create "$volume" 2>/dev/null || true
30  docker run --rm -v "$volume":/data -v "$BACKUP_DIR/volumes":/backup alpine \
31    tar xzf "/backup/$volume.tar.gz" -C /data
32done
33
34# Start containers
35docker compose up -d
36
37echo "Restore complete. Verify services are working!"

1# Quarterly restore test procedure
2
3# 1. Spin up test environment (VM, spare machine, or cloud instance)
4# 2. Install Docker
5# 3. Copy latest backup to test environment
6# 4. Run restore script
7# 5. Verify all services start
8# 6. Verify data integrity:
9#    - Can you log into services?
10#    - Is recent data present?
11#    - Do databases respond correctly?
12# 7. Document results and any issues
13
14# Automated integrity check (run monthly)
15#!/bin/bash
16for archive in /backups/daily/latest/volumes/*.tar.gz; do
17  echo "Checking $archive..."
18  tar tzf "$archive" > /dev/null || echo "CORRUPT: $archive"
19done
20
21# Database backup verification
22gunzip -t /backups/daily/latest/databases/*.sql.gz || echo "CORRUPT backup found"