feat: add infrastructure backup verification

This commit is contained in:
wlx committed 2026-08-06 18:42:07 +08:00
1 parent 885db5319f
commit 3cd0bebf4d
4 files changed
+230 -1

No files matched your search

+113
View File
@@ -0,0 +1,113 @@
#!/usr/bin/env bash
set -Eeuo pipefail
env_file="${1:-compose/.env}"
backup_root="${2:-${BACKUP_ROOT:-results/backups/infra}}"
compose_file="${COMPOSE_FILE:-compose/docker-compose.infra.yml}"
for required_command in docker id mktemp realpath sha256sum; do
if ! command -v "$required_command" >/dev/null 2>&1; then
printf 'Required command is unavailable: %s\n' "$required_command" >&2
exit 1
fi
done
if [[ ! -r "$env_file" ]]; then
printf 'Environment file not found: %s\n' "$env_file" >&2
exit 1
fi
env_value() {
local key="$1"
local value
value="$(awk -F= -v key="$key" '$1 == key {sub(/^[^=]*=/, ""); print; exit}' "$env_file")"
if [[ -z "$value" ]]; then
printf 'Required environment value is missing: %s\n' "$key" >&2
exit 1
fi
printf '%s' "$value"
}
compose_command=(docker compose --env-file "$env_file" -f "$compose_file")
"${compose_command[@]}" config --quiet
for service in postgres redis minio; do
if [[ -z "$("${compose_command[@]}" ps -q "$service")" ]]; then
printf 'Required service is not running: %s\n' "$service" >&2
exit 1
fi
done
umask 077
mkdir -p -- "$backup_root"
chmod 700 "$backup_root"
backup_root="$(realpath "$backup_root")"
env_file_absolute="$(realpath "$env_file")"
backup_timestamp="$(date +%Y%m%d-%H%M%S)"
final_directory="$backup_root/$backup_timestamp"
if [[ -e "$final_directory" ]]; then
printf 'Backup destination already exists: %s\n' "$final_directory" >&2
exit 1
fi
staging_directory="$(mktemp -d "$backup_root/.partial-${backup_timestamp}-XXXXXX")"
mkdir -p "$staging_directory/minio"
"${compose_command[@]}" exec -T postgres sh -ec \
'exec pg_dump --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" --format=custom --compress=6 --no-owner --no-acl' \
> "$staging_directory/postgres.dump"
"${compose_command[@]}" exec -T redis sh -ec \
'redis-cli --no-auth-warning -a "$REDIS_PASSWORD" SAVE >/dev/null'
redis_container_id="$("${compose_command[@]}" ps -q redis)"
docker cp "$redis_container_id:/data/dump.rdb" "$staging_directory/redis.rdb" >/dev/null
docker_network_name="$(env_value DOCKER_NETWORK_NAME)"
minio_bucket="$(env_value MINIO_BUCKET)"
minio_mc_image="$(env_value MINIO_MC_IMAGE)"
docker run --rm \
--network "$docker_network_name" \
--user "$(id -u):$(id -g)" \
--env-file "$env_file_absolute" \
--env HOME=/tmp \
--volume "$staging_directory/minio:/backup" \
--entrypoint /bin/sh \
"$minio_mc_image" -ec '
mc alias set local http://minio:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" >/dev/null
mc stat "local/$MINIO_BUCKET" >/dev/null
mc mirror --overwrite --preserve "local/$MINIO_BUCKET" /backup
'
deployment_repository_commit="unknown"
if command -v git >/dev/null 2>&1 && git rev-parse --is-inside-work-tree >/dev/null 2>&1; then
deployment_repository_commit="$(git rev-parse HEAD)"
fi
minio_object_count="$(find "$staging_directory/minio" -type f | wc -l | tr -d '[:space:]')"
cat > "$staging_directory/manifest.yml" <<EOF
schema_version: 1
created_at: "$(date --iso-8601=seconds)"
deployment_repository_commit: "$deployment_repository_commit"
compose_file: "$compose_file"
artifacts:
postgres: "postgres.dump"
redis: "redis.rdb"
minio: "minio/"
minio:
bucket: "$minio_bucket"
object_file_count: $minio_object_count
security:
credentials_included: false
customer_identifiers_included: false
EOF
(
cd "$staging_directory"
find manifest.yml postgres.dump redis.rdb minio -type f -print0 \
| LC_ALL=C sort -z \
| xargs -0 sha256sum
) > "$staging_directory/checksums.sha256"
chmod -R go-rwx "$staging_directory"
mv "$staging_directory" "$final_directory"
printf 'Infrastructure backup created: %s\n' "$final_directory"
printf 'Copy this backup to separately managed storage before relying on it.\n'
+105
View File
@@ -0,0 +1,105 @@
#!/usr/bin/env bash
set -Eeuo pipefail
env_file="${1:-compose/.env}"
backup_directory="${2:-}"
compose_file="${COMPOSE_FILE:-compose/docker-compose.infra.yml}"
if [[ -z "$backup_directory" ]]; then
printf 'Usage: %s [env-file] <backup-directory>\n' "$0" >&2
exit 1
fi
for required_command in docker id realpath sha256sum; do
if ! command -v "$required_command" >/dev/null 2>&1; then
printf 'Required command is unavailable: %s\n' "$required_command" >&2
exit 1
fi
done
if [[ ! -r "$env_file" ]]; then
printf 'Environment file not found: %s\n' "$env_file" >&2
exit 1
fi
if [[ ! -r "$backup_directory/checksums.sha256" ]]; then
printf 'Backup checksum file not found: %s\n' "$backup_directory/checksums.sha256" >&2
exit 1
fi
env_value() {
local key="$1"
local value
value="$(awk -F= -v key="$key" '$1 == key {sub(/^[^=]*=/, ""); print; exit}' "$env_file")"
if [[ -z "$value" ]]; then
printf 'Required environment value is missing: %s\n' "$key" >&2
exit 1
fi
printf '%s' "$value"
}
backup_directory="$(realpath "$backup_directory")"
env_file_absolute="$(realpath "$env_file")"
compose_command=(docker compose --env-file "$env_file" -f "$compose_file")
"${compose_command[@]}" config --quiet
postgres_user="$(env_value POSTGRES_USER)"
(
cd "$backup_directory"
sha256sum --check checksums.sha256
)
restore_database="restore_check_$(date +%s)_$$"
cleanup_postgres() {
"${compose_command[@]}" exec -T postgres dropdb \
--if-exists --username "$postgres_user" "$restore_database" >/dev/null 2>&1 || true
}
trap cleanup_postgres EXIT
"${compose_command[@]}" exec -T postgres createdb \
--username "$postgres_user" "$restore_database"
"${compose_command[@]}" exec -T postgres pg_restore \
--username "$postgres_user" \
--dbname "$restore_database" \
--no-owner \
--no-acl \
--exit-on-error \
< "$backup_directory/postgres.dump"
"${compose_command[@]}" exec -T postgres psql \
--username "$postgres_user" \
--dbname "$restore_database" \
--tuples-only \
--no-align \
--command 'SELECT 1' \
| grep -qx '1'
cleanup_postgres
trap - EXIT
redis_image="$(env_value REDIS_IMAGE)"
docker run --rm \
--volume "$backup_directory:/backup:ro" \
"$redis_image" redis-check-rdb /backup/redis.rdb
docker_network_name="$(env_value DOCKER_NETWORK_NAME)"
minio_mc_image="$(env_value MINIO_MC_IMAGE)"
restore_bucket="restore-check-$(date +%s)-$$"
docker run --rm \
--network "$docker_network_name" \
--user "$(id -u):$(id -g)" \
--env-file "$env_file_absolute" \
--env HOME=/tmp \
--env "RESTORE_BUCKET=$restore_bucket" \
--volume "$backup_directory/minio:/backup:ro" \
--entrypoint /bin/sh \
"$minio_mc_image" -ec '
mc alias set local http://minio:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" >/dev/null
cleanup_bucket() {
mc rm --recursive --force "local/$RESTORE_BUCKET" >/dev/null 2>&1 || true
mc rb --force "local/$RESTORE_BUCKET" >/dev/null 2>&1 || true
}
trap cleanup_bucket EXIT
mc mb "local/$RESTORE_BUCKET" >/dev/null
mc mirror --overwrite --preserve /backup "local/$RESTORE_BUCKET"
mc stat "local/$RESTORE_BUCKET" >/dev/null
'
printf 'Infrastructure backup verification passed: %s\n' "$backup_directory"