feat: add infrastructure backup verification
This commit is contained in:
1 parent
885db5319f
commit
3cd0bebf4d
4 files changed
+230
-1
No files matched your search
Executable
+113
@@ -0,0 +1,113 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
env_file="${1:-compose/.env}"
|
||||
backup_root="${2:-${BACKUP_ROOT:-results/backups/infra}}"
|
||||
compose_file="${COMPOSE_FILE:-compose/docker-compose.infra.yml}"
|
||||
|
||||
for required_command in docker id mktemp realpath sha256sum; do
|
||||
if ! command -v "$required_command" >/dev/null 2>&1; then
|
||||
printf 'Required command is unavailable: %s\n' "$required_command" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ ! -r "$env_file" ]]; then
|
||||
printf 'Environment file not found: %s\n' "$env_file" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
env_value() {
|
||||
local key="$1"
|
||||
local value
|
||||
value="$(awk -F= -v key="$key" '$1 == key {sub(/^[^=]*=/, ""); print; exit}' "$env_file")"
|
||||
if [[ -z "$value" ]]; then
|
||||
printf 'Required environment value is missing: %s\n' "$key" >&2
|
||||
exit 1
|
||||
fi
|
||||
printf '%s' "$value"
|
||||
}
|
||||
|
||||
compose_command=(docker compose --env-file "$env_file" -f "$compose_file")
|
||||
"${compose_command[@]}" config --quiet
|
||||
|
||||
for service in postgres redis minio; do
|
||||
if [[ -z "$("${compose_command[@]}" ps -q "$service")" ]]; then
|
||||
printf 'Required service is not running: %s\n' "$service" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
umask 077
|
||||
mkdir -p -- "$backup_root"
|
||||
chmod 700 "$backup_root"
|
||||
backup_root="$(realpath "$backup_root")"
|
||||
env_file_absolute="$(realpath "$env_file")"
|
||||
backup_timestamp="$(date +%Y%m%d-%H%M%S)"
|
||||
final_directory="$backup_root/$backup_timestamp"
|
||||
if [[ -e "$final_directory" ]]; then
|
||||
printf 'Backup destination already exists: %s\n' "$final_directory" >&2
|
||||
exit 1
|
||||
fi
|
||||
staging_directory="$(mktemp -d "$backup_root/.partial-${backup_timestamp}-XXXXXX")"
|
||||
mkdir -p "$staging_directory/minio"
|
||||
|
||||
"${compose_command[@]}" exec -T postgres sh -ec \
|
||||
'exec pg_dump --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" --format=custom --compress=6 --no-owner --no-acl' \
|
||||
> "$staging_directory/postgres.dump"
|
||||
|
||||
"${compose_command[@]}" exec -T redis sh -ec \
|
||||
'redis-cli --no-auth-warning -a "$REDIS_PASSWORD" SAVE >/dev/null'
|
||||
redis_container_id="$("${compose_command[@]}" ps -q redis)"
|
||||
docker cp "$redis_container_id:/data/dump.rdb" "$staging_directory/redis.rdb" >/dev/null
|
||||
|
||||
docker_network_name="$(env_value DOCKER_NETWORK_NAME)"
|
||||
minio_bucket="$(env_value MINIO_BUCKET)"
|
||||
minio_mc_image="$(env_value MINIO_MC_IMAGE)"
|
||||
docker run --rm \
|
||||
--network "$docker_network_name" \
|
||||
--user "$(id -u):$(id -g)" \
|
||||
--env-file "$env_file_absolute" \
|
||||
--env HOME=/tmp \
|
||||
--volume "$staging_directory/minio:/backup" \
|
||||
--entrypoint /bin/sh \
|
||||
"$minio_mc_image" -ec '
|
||||
mc alias set local http://minio:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" >/dev/null
|
||||
mc stat "local/$MINIO_BUCKET" >/dev/null
|
||||
mc mirror --overwrite --preserve "local/$MINIO_BUCKET" /backup
|
||||
'
|
||||
|
||||
deployment_repository_commit="unknown"
|
||||
if command -v git >/dev/null 2>&1 && git rev-parse --is-inside-work-tree >/dev/null 2>&1; then
|
||||
deployment_repository_commit="$(git rev-parse HEAD)"
|
||||
fi
|
||||
minio_object_count="$(find "$staging_directory/minio" -type f | wc -l | tr -d '[:space:]')"
|
||||
|
||||
cat > "$staging_directory/manifest.yml" <<EOF
|
||||
schema_version: 1
|
||||
created_at: "$(date --iso-8601=seconds)"
|
||||
deployment_repository_commit: "$deployment_repository_commit"
|
||||
compose_file: "$compose_file"
|
||||
artifacts:
|
||||
postgres: "postgres.dump"
|
||||
redis: "redis.rdb"
|
||||
minio: "minio/"
|
||||
minio:
|
||||
bucket: "$minio_bucket"
|
||||
object_file_count: $minio_object_count
|
||||
security:
|
||||
credentials_included: false
|
||||
customer_identifiers_included: false
|
||||
EOF
|
||||
|
||||
(
|
||||
cd "$staging_directory"
|
||||
find manifest.yml postgres.dump redis.rdb minio -type f -print0 \
|
||||
| LC_ALL=C sort -z \
|
||||
| xargs -0 sha256sum
|
||||
) > "$staging_directory/checksums.sha256"
|
||||
|
||||
chmod -R go-rwx "$staging_directory"
|
||||
mv "$staging_directory" "$final_directory"
|
||||
printf 'Infrastructure backup created: %s\n' "$final_directory"
|
||||
printf 'Copy this backup to separately managed storage before relying on it.\n'
|
||||
Executable
+105
@@ -0,0 +1,105 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
env_file="${1:-compose/.env}"
|
||||
backup_directory="${2:-}"
|
||||
compose_file="${COMPOSE_FILE:-compose/docker-compose.infra.yml}"
|
||||
|
||||
if [[ -z "$backup_directory" ]]; then
|
||||
printf 'Usage: %s [env-file] <backup-directory>\n' "$0" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for required_command in docker id realpath sha256sum; do
|
||||
if ! command -v "$required_command" >/dev/null 2>&1; then
|
||||
printf 'Required command is unavailable: %s\n' "$required_command" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ ! -r "$env_file" ]]; then
|
||||
printf 'Environment file not found: %s\n' "$env_file" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ ! -r "$backup_directory/checksums.sha256" ]]; then
|
||||
printf 'Backup checksum file not found: %s\n' "$backup_directory/checksums.sha256" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
env_value() {
|
||||
local key="$1"
|
||||
local value
|
||||
value="$(awk -F= -v key="$key" '$1 == key {sub(/^[^=]*=/, ""); print; exit}' "$env_file")"
|
||||
if [[ -z "$value" ]]; then
|
||||
printf 'Required environment value is missing: %s\n' "$key" >&2
|
||||
exit 1
|
||||
fi
|
||||
printf '%s' "$value"
|
||||
}
|
||||
|
||||
backup_directory="$(realpath "$backup_directory")"
|
||||
env_file_absolute="$(realpath "$env_file")"
|
||||
compose_command=(docker compose --env-file "$env_file" -f "$compose_file")
|
||||
"${compose_command[@]}" config --quiet
|
||||
postgres_user="$(env_value POSTGRES_USER)"
|
||||
|
||||
(
|
||||
cd "$backup_directory"
|
||||
sha256sum --check checksums.sha256
|
||||
)
|
||||
|
||||
restore_database="restore_check_$(date +%s)_$$"
|
||||
cleanup_postgres() {
|
||||
"${compose_command[@]}" exec -T postgres dropdb \
|
||||
--if-exists --username "$postgres_user" "$restore_database" >/dev/null 2>&1 || true
|
||||
}
|
||||
trap cleanup_postgres EXIT
|
||||
|
||||
"${compose_command[@]}" exec -T postgres createdb \
|
||||
--username "$postgres_user" "$restore_database"
|
||||
"${compose_command[@]}" exec -T postgres pg_restore \
|
||||
--username "$postgres_user" \
|
||||
--dbname "$restore_database" \
|
||||
--no-owner \
|
||||
--no-acl \
|
||||
--exit-on-error \
|
||||
< "$backup_directory/postgres.dump"
|
||||
"${compose_command[@]}" exec -T postgres psql \
|
||||
--username "$postgres_user" \
|
||||
--dbname "$restore_database" \
|
||||
--tuples-only \
|
||||
--no-align \
|
||||
--command 'SELECT 1' \
|
||||
| grep -qx '1'
|
||||
cleanup_postgres
|
||||
trap - EXIT
|
||||
|
||||
redis_image="$(env_value REDIS_IMAGE)"
|
||||
docker run --rm \
|
||||
--volume "$backup_directory:/backup:ro" \
|
||||
"$redis_image" redis-check-rdb /backup/redis.rdb
|
||||
|
||||
docker_network_name="$(env_value DOCKER_NETWORK_NAME)"
|
||||
minio_mc_image="$(env_value MINIO_MC_IMAGE)"
|
||||
restore_bucket="restore-check-$(date +%s)-$$"
|
||||
docker run --rm \
|
||||
--network "$docker_network_name" \
|
||||
--user "$(id -u):$(id -g)" \
|
||||
--env-file "$env_file_absolute" \
|
||||
--env HOME=/tmp \
|
||||
--env "RESTORE_BUCKET=$restore_bucket" \
|
||||
--volume "$backup_directory/minio:/backup:ro" \
|
||||
--entrypoint /bin/sh \
|
||||
"$minio_mc_image" -ec '
|
||||
mc alias set local http://minio:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" >/dev/null
|
||||
cleanup_bucket() {
|
||||
mc rm --recursive --force "local/$RESTORE_BUCKET" >/dev/null 2>&1 || true
|
||||
mc rb --force "local/$RESTORE_BUCKET" >/dev/null 2>&1 || true
|
||||
}
|
||||
trap cleanup_bucket EXIT
|
||||
mc mb "local/$RESTORE_BUCKET" >/dev/null
|
||||
mc mirror --overwrite --preserve /backup "local/$RESTORE_BUCKET"
|
||||
mc stat "local/$RESTORE_BUCKET" >/dev/null
|
||||
'
|
||||
|
||||
printf 'Infrastructure backup verification passed: %s\n' "$backup_directory"
|
||||
Reference in new issue
Block a user