fix: quarantine unavailable upstream routes and harden upgrades

This commit is contained in:
wlx committed 2026-09-28 06:34:15 +08:00
1 parent 9acc69d4c7
commit 59907cebdf
8 files changed
+424 -3

No files matched your search

+213
View File
@@ -0,0 +1,213 @@
#!/usr/bin/env python3
"""Safely switch a running MOYI Compose project to a prepared release.
The old release owns the running containers, so the database backup must be
performed through the old release's ``backup_database.py``. The new release
is validated and started with the same Compose project name; no ``down`` or
volume operation is performed. Every child process is detached from stdin so
the command is safe to run from an SSH heredoc.
"""
import argparse
import importlib.util
import json
import os
from pathlib import Path
import re
import shutil
import subprocess
import sys
from types import ModuleType
class UpgradeError(RuntimeError):
"""A deliberate safety check prevented the release switch."""
def require(condition, message):
if not condition:
raise UpgradeError(message)
def child(command, *, cwd=None, capture=False):
"""Run a child process without inheriting an SSH/heredoc stdin stream."""
result = subprocess.run(
command,
cwd=cwd,
stdin=subprocess.DEVNULL,
capture_output=capture,
text=True,
encoding="utf-8",
errors="replace",
)
if result.returncode:
raise UpgradeError(
"Command failed: " + " ".join(command[:3]) +
". Check local configuration/permissions; no secret output is shown."
)
return result.stdout if capture else ""
def load_module(path, name):
spec = importlib.util.spec_from_file_location(name, path)
if spec is None or spec.loader is None:
raise UpgradeError("Cannot load release helper: " + path.name)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def discover_old_dir(container="moyi-test-moyi-api-1"):
output = child(
["docker", "inspect", "--format", "{{ index .Config.Labels \"com.docker.compose.project.working_dir\" }}", container],
capture=True,
).strip()
require(output, "Cannot discover the running release directory; pass --old-dir explicitly.")
return Path(output).resolve()
def copy_runtime_file(old, new, name, mode):
source = old / name
destination = new / name
require(source.is_file(), "Current release is missing " + name + ".")
if destination.exists():
require(destination.is_file() and destination.read_bytes() == source.read_bytes(),
"Prepared release has a different " + name + "; refusing to overwrite it.")
else:
destination.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
shutil.copyfile(source, destination)
os.chmod(destination, mode)
def update_release_environment(old, new, release):
"""Copy credentials and pin a prepared release to its immutable image tag."""
old_env = old / ".env.cloud"
new_env = new / ".env.cloud"
require(old_env.is_file(), "Current release is missing .env.cloud.")
if new_env.exists():
require(new_env.is_file() and new_env.read_bytes() == old_env.read_bytes(),
"Prepared .env.cloud differs from the current configuration; refusing to replace credentials.")
else:
shutil.copyfile(old_env, new_env)
os.chmod(new_env, 0o600)
text = new_env.read_text(encoding="utf-8-sig")
def replace_or_append(name, value):
nonlocal text
pattern = re.compile(r"^(\s*(?:export\s+)?" + re.escape(name) + r"\s*=).*$", re.MULTILINE)
text, count = pattern.subn(lambda match: match.group(1) + value, text)
if count == 0:
text = text.rstrip("\r\n") + "\n" + name + "=" + value + "\n"
replace_or_append("MOYI_IMAGE_TAG", release)
replace_or_append("MOYI_API_DOCKERFILE", "Dockerfile.runtime")
replace_or_append("MOYI_FRONTEND_DOCKERFILE", "Dockerfile.runtime")
new_env.write_text(text, encoding="utf-8", newline="\n")
return new_env
def manifest_for(new):
root = new.parent.parent
path = root / "release-manifest.json"
require(path.is_file(), "Prepared release is missing release-manifest.json.")
try:
manifest = json.loads(path.read_text(encoding="utf-8"))
except (OSError, ValueError) as error:
raise UpgradeError("Prepared release manifest is invalid.") from error
release = str(manifest.get("release", ""))
require(release and release == root.name, "Release manifest does not match its directory.")
return manifest
def image_tag(config, service):
image = str(config.get("services", {}).get(service, {}).get("image", ""))
return image.rsplit(":", 1)[-1] if ":" in image else ""
def validate_new_config(old_config, new_config, release):
old_name = old_config.get("name")
new_name = new_config.get("name")
if old_name and new_name:
require(old_name == new_name, "Compose project name changed; refusing to switch releases.")
for service in ("moyi-api", "frontend"):
require(image_tag(new_config, service) == release,
"Prepared " + service + " image is not tagged " + release + ".")
dockerfile = str(new_config.get("services", {}).get(service, {}).get("build", {}).get("dockerfile", ""))
require(dockerfile == "Dockerfile.runtime", "Prepared " + service + " is not using Dockerfile.runtime.")
def backup_current(old, mode, output_root):
"""Import and execute backup_database.py from the running release directory."""
sys.path.insert(0, str(old))
try:
deploy = load_module(old / "deploy.py", "moyi_old_deploy")
backup = load_module(old / "backup_database.py", "moyi_old_backup")
return backup.backup(deploy, mode, output_root)
finally:
sys.path.remove(str(old))
def start_new(deploy, config, env_file, mode):
command = deploy.compose_command(env_file, mode)
deploy.execute(["docker", "info", "--format", "{{.ServerVersion}}"], capture=True)
# Build first, so a failed image build leaves the running release serving traffic.
deploy.execute(command + ["build"])
deploy.execute(command + ["up", "-d", "--wait", "--wait-timeout", "180", "--no-build", "moyi-api"])
deploy.execute(command + ["run", "--rm", "--no-deps", "frontend", "nginx", "-t"])
deploy.execute(command + ["up", "-d", "--wait", "--wait-timeout", "180", "--no-build"])
deploy.execute(command + ["ps"])
def upgrade(old_dir, new_dir, mode="https", output_root=Path("/opt/moyi/backups"),
accept_migrations=False, network=False, run_smoke=False, login=False):
old = Path(old_dir).resolve()
new = Path(new_dir).resolve()
require(old != new, "Old and new release directories must differ.")
require(old.is_dir() and (old / "deploy.py").is_file(), "Current release directory is invalid.")
require(new.is_dir() and (new / "deploy.py").is_file(), "Prepared release directory is invalid.")
require(accept_migrations, "Back up the dedicated RDS database, then pass --accept-migrations.")
manifest = manifest_for(new)
release = str(manifest["release"])
old_deploy = load_module(old / "deploy.py", "moyi_running_deploy")
old_config = old_deploy.preflight(old / ".env.cloud", mode, False)
backup_dir = backup_current(old, mode, Path(output_root).resolve())
require((Path(backup_dir) / "backup-complete.json").is_file(), "Database backup did not complete.")
update_release_environment(old, new, release)
copy_runtime_file(old, new, "certs/fullchain.pem", 0o644)
copy_runtime_file(old, new, "certs/privkey.pem", 0o600)
new_deploy = load_module(new / "deploy.py", "moyi_new_deploy")
new_config = new_deploy.preflight(new / ".env.cloud", mode, network)
validate_new_config(old_config, new_config, release)
start_new(new_deploy, new_config, new / ".env.cloud", mode)
(new / ".deploy-mode").write_text(mode + "\n", encoding="ascii")
if run_smoke:
new_deploy.smoke(new_config, mode, login)
print("UPGRADE_OK: backup complete; new release started without docker compose down.")
return backup_dir
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--new-dir", type=Path, required=True, help="Prepared release infra/moyi directory")
parser.add_argument("--old-dir", type=Path, help="Running release infra/moyi directory; otherwise inspect the API container")
parser.add_argument("--container", default="moyi-test-moyi-api-1")
parser.add_argument("--mode", choices=("tunnel", "https"), default="https")
parser.add_argument("--output-root", type=Path, default=Path("/opt/moyi/backups"))
parser.add_argument("--network", action="store_true")
parser.add_argument("--accept-migrations", action="store_true")
parser.add_argument("--smoke", action="store_true")
parser.add_argument("--login", action="store_true")
args = parser.parse_args()
old = args.old_dir.resolve() if args.old_dir else discover_old_dir(args.container)
upgrade(old, args.new_dir, args.mode, args.output_root, args.accept_migrations,
args.network, args.smoke, args.login)
if __name__ == "__main__":
try:
main()
except (UpgradeError, OSError, ValueError, subprocess.SubprocessError) as error:
message = str(error) if isinstance(error, (UpgradeError, ValueError)) else type(error).__name__ + ": upgrade failed; no secret output shown."
print("ERROR: " + message, file=sys.stderr)
raise SystemExit(1)