fix: quarantine unavailable upstream routes and harden upgrades
This commit is contained in:
1 parent
9acc69d4c7
commit
59907cebdf
8 files changed
+424
-3
No files matched your search
@@ -0,0 +1,213 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Safely switch a running MOYI Compose project to a prepared release.
|
||||
|
||||
The old release owns the running containers, so the database backup must be
|
||||
performed through the old release's ``backup_database.py``. The new release
|
||||
is validated and started with the same Compose project name; no ``down`` or
|
||||
volume operation is performed. Every child process is detached from stdin so
|
||||
the command is safe to run from an SSH heredoc.
|
||||
"""
|
||||
import argparse
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from types import ModuleType
|
||||
|
||||
|
||||
class UpgradeError(RuntimeError):
|
||||
"""A deliberate safety check prevented the release switch."""
|
||||
|
||||
|
||||
def require(condition, message):
|
||||
if not condition:
|
||||
raise UpgradeError(message)
|
||||
|
||||
|
||||
def child(command, *, cwd=None, capture=False):
|
||||
"""Run a child process without inheriting an SSH/heredoc stdin stream."""
|
||||
result = subprocess.run(
|
||||
command,
|
||||
cwd=cwd,
|
||||
stdin=subprocess.DEVNULL,
|
||||
capture_output=capture,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
)
|
||||
if result.returncode:
|
||||
raise UpgradeError(
|
||||
"Command failed: " + " ".join(command[:3]) +
|
||||
". Check local configuration/permissions; no secret output is shown."
|
||||
)
|
||||
return result.stdout if capture else ""
|
||||
|
||||
|
||||
def load_module(path, name):
|
||||
spec = importlib.util.spec_from_file_location(name, path)
|
||||
if spec is None or spec.loader is None:
|
||||
raise UpgradeError("Cannot load release helper: " + path.name)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def discover_old_dir(container="moyi-test-moyi-api-1"):
|
||||
output = child(
|
||||
["docker", "inspect", "--format", "{{ index .Config.Labels \"com.docker.compose.project.working_dir\" }}", container],
|
||||
capture=True,
|
||||
).strip()
|
||||
require(output, "Cannot discover the running release directory; pass --old-dir explicitly.")
|
||||
return Path(output).resolve()
|
||||
|
||||
|
||||
def copy_runtime_file(old, new, name, mode):
|
||||
source = old / name
|
||||
destination = new / name
|
||||
require(source.is_file(), "Current release is missing " + name + ".")
|
||||
if destination.exists():
|
||||
require(destination.is_file() and destination.read_bytes() == source.read_bytes(),
|
||||
"Prepared release has a different " + name + "; refusing to overwrite it.")
|
||||
else:
|
||||
destination.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
shutil.copyfile(source, destination)
|
||||
os.chmod(destination, mode)
|
||||
|
||||
|
||||
def update_release_environment(old, new, release):
|
||||
"""Copy credentials and pin a prepared release to its immutable image tag."""
|
||||
old_env = old / ".env.cloud"
|
||||
new_env = new / ".env.cloud"
|
||||
require(old_env.is_file(), "Current release is missing .env.cloud.")
|
||||
if new_env.exists():
|
||||
require(new_env.is_file() and new_env.read_bytes() == old_env.read_bytes(),
|
||||
"Prepared .env.cloud differs from the current configuration; refusing to replace credentials.")
|
||||
else:
|
||||
shutil.copyfile(old_env, new_env)
|
||||
os.chmod(new_env, 0o600)
|
||||
text = new_env.read_text(encoding="utf-8-sig")
|
||||
|
||||
def replace_or_append(name, value):
|
||||
nonlocal text
|
||||
pattern = re.compile(r"^(\s*(?:export\s+)?" + re.escape(name) + r"\s*=).*$", re.MULTILINE)
|
||||
text, count = pattern.subn(lambda match: match.group(1) + value, text)
|
||||
if count == 0:
|
||||
text = text.rstrip("\r\n") + "\n" + name + "=" + value + "\n"
|
||||
|
||||
replace_or_append("MOYI_IMAGE_TAG", release)
|
||||
replace_or_append("MOYI_API_DOCKERFILE", "Dockerfile.runtime")
|
||||
replace_or_append("MOYI_FRONTEND_DOCKERFILE", "Dockerfile.runtime")
|
||||
new_env.write_text(text, encoding="utf-8", newline="\n")
|
||||
return new_env
|
||||
|
||||
|
||||
def manifest_for(new):
|
||||
root = new.parent.parent
|
||||
path = root / "release-manifest.json"
|
||||
require(path.is_file(), "Prepared release is missing release-manifest.json.")
|
||||
try:
|
||||
manifest = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError) as error:
|
||||
raise UpgradeError("Prepared release manifest is invalid.") from error
|
||||
release = str(manifest.get("release", ""))
|
||||
require(release and release == root.name, "Release manifest does not match its directory.")
|
||||
return manifest
|
||||
|
||||
|
||||
def image_tag(config, service):
|
||||
image = str(config.get("services", {}).get(service, {}).get("image", ""))
|
||||
return image.rsplit(":", 1)[-1] if ":" in image else ""
|
||||
|
||||
|
||||
def validate_new_config(old_config, new_config, release):
|
||||
old_name = old_config.get("name")
|
||||
new_name = new_config.get("name")
|
||||
if old_name and new_name:
|
||||
require(old_name == new_name, "Compose project name changed; refusing to switch releases.")
|
||||
for service in ("moyi-api", "frontend"):
|
||||
require(image_tag(new_config, service) == release,
|
||||
"Prepared " + service + " image is not tagged " + release + ".")
|
||||
dockerfile = str(new_config.get("services", {}).get(service, {}).get("build", {}).get("dockerfile", ""))
|
||||
require(dockerfile == "Dockerfile.runtime", "Prepared " + service + " is not using Dockerfile.runtime.")
|
||||
|
||||
|
||||
def backup_current(old, mode, output_root):
|
||||
"""Import and execute backup_database.py from the running release directory."""
|
||||
sys.path.insert(0, str(old))
|
||||
try:
|
||||
deploy = load_module(old / "deploy.py", "moyi_old_deploy")
|
||||
backup = load_module(old / "backup_database.py", "moyi_old_backup")
|
||||
return backup.backup(deploy, mode, output_root)
|
||||
finally:
|
||||
sys.path.remove(str(old))
|
||||
|
||||
|
||||
def start_new(deploy, config, env_file, mode):
|
||||
command = deploy.compose_command(env_file, mode)
|
||||
deploy.execute(["docker", "info", "--format", "{{.ServerVersion}}"], capture=True)
|
||||
# Build first, so a failed image build leaves the running release serving traffic.
|
||||
deploy.execute(command + ["build"])
|
||||
deploy.execute(command + ["up", "-d", "--wait", "--wait-timeout", "180", "--no-build", "moyi-api"])
|
||||
deploy.execute(command + ["run", "--rm", "--no-deps", "frontend", "nginx", "-t"])
|
||||
deploy.execute(command + ["up", "-d", "--wait", "--wait-timeout", "180", "--no-build"])
|
||||
deploy.execute(command + ["ps"])
|
||||
|
||||
|
||||
def upgrade(old_dir, new_dir, mode="https", output_root=Path("/opt/moyi/backups"),
|
||||
accept_migrations=False, network=False, run_smoke=False, login=False):
|
||||
old = Path(old_dir).resolve()
|
||||
new = Path(new_dir).resolve()
|
||||
require(old != new, "Old and new release directories must differ.")
|
||||
require(old.is_dir() and (old / "deploy.py").is_file(), "Current release directory is invalid.")
|
||||
require(new.is_dir() and (new / "deploy.py").is_file(), "Prepared release directory is invalid.")
|
||||
require(accept_migrations, "Back up the dedicated RDS database, then pass --accept-migrations.")
|
||||
|
||||
manifest = manifest_for(new)
|
||||
release = str(manifest["release"])
|
||||
old_deploy = load_module(old / "deploy.py", "moyi_running_deploy")
|
||||
old_config = old_deploy.preflight(old / ".env.cloud", mode, False)
|
||||
backup_dir = backup_current(old, mode, Path(output_root).resolve())
|
||||
require((Path(backup_dir) / "backup-complete.json").is_file(), "Database backup did not complete.")
|
||||
|
||||
update_release_environment(old, new, release)
|
||||
copy_runtime_file(old, new, "certs/fullchain.pem", 0o644)
|
||||
copy_runtime_file(old, new, "certs/privkey.pem", 0o600)
|
||||
new_deploy = load_module(new / "deploy.py", "moyi_new_deploy")
|
||||
new_config = new_deploy.preflight(new / ".env.cloud", mode, network)
|
||||
validate_new_config(old_config, new_config, release)
|
||||
start_new(new_deploy, new_config, new / ".env.cloud", mode)
|
||||
(new / ".deploy-mode").write_text(mode + "\n", encoding="ascii")
|
||||
if run_smoke:
|
||||
new_deploy.smoke(new_config, mode, login)
|
||||
print("UPGRADE_OK: backup complete; new release started without docker compose down.")
|
||||
return backup_dir
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--new-dir", type=Path, required=True, help="Prepared release infra/moyi directory")
|
||||
parser.add_argument("--old-dir", type=Path, help="Running release infra/moyi directory; otherwise inspect the API container")
|
||||
parser.add_argument("--container", default="moyi-test-moyi-api-1")
|
||||
parser.add_argument("--mode", choices=("tunnel", "https"), default="https")
|
||||
parser.add_argument("--output-root", type=Path, default=Path("/opt/moyi/backups"))
|
||||
parser.add_argument("--network", action="store_true")
|
||||
parser.add_argument("--accept-migrations", action="store_true")
|
||||
parser.add_argument("--smoke", action="store_true")
|
||||
parser.add_argument("--login", action="store_true")
|
||||
args = parser.parse_args()
|
||||
old = args.old_dir.resolve() if args.old_dir else discover_old_dir(args.container)
|
||||
upgrade(old, args.new_dir, args.mode, args.output_root, args.accept_migrations,
|
||||
args.network, args.smoke, args.login)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except (UpgradeError, OSError, ValueError, subprocess.SubprocessError) as error:
|
||||
message = str(error) if isinstance(error, (UpgradeError, ValueError)) else type(error).__name__ + ": upgrade failed; no secret output shown."
|
||||
print("ERROR: " + message, file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
Reference in new issue
Block a user