This comprehensive guide covers routine maintenance tasks, update procedures, backup operations, and troubleshooting for the Lucille4 homelab server.
docker ps --format "table {{.Names}}\t{{.Status}}"
df -h
docker system df
python3 check-updates.pysudo -E ./backup.shSafe to update anytime:
docker compose pull cyberchef it-tools regexr stirling-pdf
docker compose up -d cyberchef it-tools regexr stirling-pdf
Update during low usage:
docker compose pull grafana influxdb dozzle seq beszel
docker compose up -d grafana influxdb dozzle seq beszel
Update individually with testing:
## Update one at a time
docker compose pull n8n
docker compose up -d n8n
## Test core workflows before proceeding
docker compose pull jupyter code-server
docker compose up -d jupyter code-server
Requires backup and careful planning:
## Mandatory backup first!
sudo -E ./backup.sh
## Update authentication stack
docker compose pull auth-server auth-worker
docker compose up -d auth-server auth-worker
## Update document management
docker compose pull paperless
docker compose up -d paperless
Major versions require complete reindex:
## Stop service
docker compose stop meilisearch
## Clear data directory
rm -rf ./meili/data/*
## Update version in docker-compose.yml
vi docker-compose.yml
## Deploy new version
docker compose up -d meilisearch
## Reindex via Karakeep UI
Always check release notes:
## Pull new images
docker compose pull auth-server auth-worker
## Update with minimal downtime
docker compose up -d auth-server auth-worker
## Monitor logs for migration completion
docker logs -f auth-server
Requires dump and restore:
## Dump all databases
docker exec paperless-db pg_dumpall -U paperless > paperless_backup.sql
docker exec auth-db pg_dumpall -U authentik > authentik_backup.sql
docker exec n8n-db pg_dumpall -U n8n > n8n_backup.sql
## Stop databases
docker compose down paperless-db auth-db n8n-db
## Update versions in docker-compose.yml
## Deploy new versions
docker compose up -d paperless-db auth-db n8n-db
## Restore databases
docker exec -i paperless-db psql -U paperless < paperless_backup.sql
docker exec -i auth-db psql -U authentik < authentik_backup.sql
docker exec -i n8n-db psql -U n8n < n8n_backup.sql
## Root crontab entry
0 2 * * * /home/mspeicher/homelab-lucille4/backup.sh
## Full backup with environment variables
sudo -E ./backup.sh
## Backblaze snapshots
restic -r b2:homelab-backup snapshots
## Hetzner snapshots
restic -r sftp:u123456@u123456.your-storagebox.de:/backup snapshots
## Restore single file
restic -r b2:homelab-backup restore latest \
--target /tmp/restore \
--include /home/mspeicher/homelab-lucille4/paperless/data/documents/example.pdf
## Restore entire service
restic -r b2:homelab-backup restore latest \
--target /tmp/restore \
--include /home/mspeicher/homelab-lucille4/paperless
## Restore complete homelab directory
restic -r b2:homelab-backup restore latest \
--target /home/mspeicher/homelab-lucille4-restore
Check container logs:
Common failure causes:
Interactive debugging:
docker logs [container-name] --tail 50
docker compose logs [service-name] --tail 100
sudo netstat -tulpn | grep [port]docker volume ls | grep [volume-name]docker network inspect caddy internaldocker stats [container-name]
docker compose run --rm [service] sh
docker compose run --rm --entrypoint sh [service]
## Caddy Reverse Proxy Issues
### Certificate Problems
```bash
## View Caddy logs
docker logs caddy | grep -E "error|certificate|acme"
## Force certificate renewal
docker exec caddy caddy reload --force
## Check certificate status
docker exec caddy caddy list-certificates
docker-compose.ymlcaddy networknslookup service.speicher.familydocker exec caddy caddy validate
## Test database connectivity
docker exec -it paperless-db psql -U paperless -d paperless -c "SELECT 1"
## View PostgreSQL logs
docker logs paperless-db --tail 100 | grep -E "FATAL|ERROR"
## Check connections
docker exec paperless-db psql -U paperless -c "SELECT * FROM pg_stat_activity"
## Test Redis connectivity
docker exec -it paperless-broker redis-cli ping
## Check memory usage
docker exec -it paperless-broker redis-cli info memory
## Monitor real-time activity
docker exec -it paperless-broker redis-cli monitor
## Remove unused images (safe)
docker image prune -a
## Remove stopped containers
docker container prune
## Remove unused volumes (CAREFUL - data loss possible!)
docker volume prune
## Complete system cleanup (VERY CAREFUL!)
docker system prune -a --volumes
## Find large directories
du -h --max-depth=1 /home/mspeicher/homelab-lucille4 | sort -hr | head -20
## Docker-specific usage
docker system df -v
## Identify resource-hungry containers
docker stats --no-stream --format "table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}"
## Restart problematic service
docker compose restart [service-name]
## Adjust CPU limits in docker-compose.yml
deploy:
resources:
limits:
cpus: '2.0'
## Check current memory limits
docker inspect [container] | jq '.[0].HostConfig.Memory'
## Monitor memory usage
docker stats [container] --format "table {{.MemUsage}}\t{{.MemPerc}}"
## Update limits in docker-compose.yml
mem_limit: 4g
mem_reservation: 2g
## 1. Install prerequisites
sudo apt update && sudo apt install docker.io docker-compose git
## 2. Clone repository
git clone https://github.com/yourusername/homelab-lucille4
## 3. Restore .env from 1Password
op item get "Homelab Environment - Homelab" --fields notes > .env
## 4. Create networks and volumes
docker network create caddy internal
docker volume create influxdb_data grafana_data auth-db auth-redis \
caddy_data mealie_data paperless-db paperless-broker n8n-data
## 5. Restore from backup
restic -r b2:homelab-backup restore latest --target /
## 6. Deploy all services
docker compose up -d
## 7. Verify functionality
docker ps
curl -I https://auth.speicher.family
docker ps --format "table {{.Names}}\t{{.Status}}"df -h and docker system dfpython3 scripts/check-updates.pypython3 scripts/check-updates.py
Utilities (Low Risk):
docker compose pull cyberchef it-tools regexr
docker compose up -d cyberchef it-tools regexr
Monitoring (Medium Risk):
docker compose pull grafana influxdb dozzle seq
docker compose up -d grafana influxdb dozzle seq
Workflow Tools (Medium-High Risk):
## Update one at a time
docker compose pull n8n
docker compose up -d n8n
## Test functionality before proceeding
Critical Services (High Risk):
## Backup first!
sudo -E ./scripts/backup.sh
## Then update
docker compose pull paperless auth-server
docker compose up -d paperless auth-server
Requires data reset on major version updates:
docker compose stop meilisearch
rm -rf ./meili/data/*
## Update version in docker-compose.yml
docker compose up -d meilisearch
## Re-index in Karakeep UI
Check release notes for migration requirements:
## Review: https://goauthentik.io/docs/releases
docker compose pull auth-server auth-worker
docker compose up -d auth-server auth-worker
Major version upgrades require dump/restore:
## Dump all databases
docker exec paperless-db pg_dumpall -U paperless > backup.sql
docker exec auth-db pg_dumpall -U authentik >> backup.sql
## Update and restore
docker compose down paperless-db auth-db
## Update versions in docker-compose.yml
docker compose up -d paperless-db auth-db
## Restore databases
## Cron entry (root crontab)
0 2 * * * /home/mspeicher/homelab-lucille4/scripts/backup.sh
sudo -E ./scripts/backup.sh
See backup improvements for upcoming monitoring enhancements.
## Backblaze
restic -r b2:bucket-name snapshots
## Hetzner
restic -r sftp:user@host:/path snapshots
## Restore specific file
restic -r b2:bucket-name restore latest \
--target /tmp/restore \
--include /path/to/file
## Restore entire service data
restic -r b2:bucket-name restore latest \
--target /tmp/restore \
--include /home/mspeicher/homelab-lucille4/service-name
docker logs [container-name] --tail 50
docker compose logs [service-name]
netstat -tulpn | grep [port]docker volume lsdocker network lsdocker stats [container]
## Run interactively
docker compose run --rm [service] sh
## Override entrypoint
docker compose run --rm --entrypoint sh [service]
## Check Caddy logs
docker logs caddy | grep -i error
## Force certificate renewal
docker exec caddy caddy reload --force
caddy network
## Test connection
docker exec -it paperless-db psql -U paperless -d paperless
## Check postgres logs
docker logs paperless-db --tail 100
## Test Redis
docker exec -it paperless-broker redis-cli ping
## Check memory usage
docker exec -it paperless-broker redis-cli info memory
## Remove unused images
docker image prune -a
## Remove stopped containers
docker container prune
## Remove unused volumes (CAREFUL!)
docker volume prune
## Full cleanup
docker system prune -a --volumes
du -h --max-depth=1 /home/mspeicher/homelab-lucille4 | sort -hr
## Find culprit
docker stats --no-stream --format "table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}"
## Restart problematic service
docker compose restart [service]
## Check memory limits
docker inspect [container] | grep -i memory
## Adjust in docker-compose.yml
mem_limit: 4g
mem_reservation: 2g
.env from 1Passworddocker compose up -d
## Export service data
docker exec [service] [export-command]
## Copy volumes
docker run --rm -v [source]:/source -v [dest]:/dest \
alpine sh -c "cp -av /source/* /dest/"
## Import on new system
docker exec [service] [import-command]