fix: quarantine unavailable upstream routes and harden upgrades
This commit is contained in:
1 parent
9acc69d4c7
commit
59907cebdf
8 files changed
+424
-3
No files matched your search
@@ -40,6 +40,20 @@ docker compose --env-file .env.cloud -f docker-compose.cloud.yml build
|
||||
docker compose --env-file .env.cloud -f docker-compose.cloud.yml up -d
|
||||
```
|
||||
|
||||
### 切换预编译发布包
|
||||
|
||||
把新发布包解压到独立的 `releases/<版本>/` 目录后,从新目录执行升级脚本。脚本会先在当前运行目录执行数据库备份,再校验发布清单、复制证书和环境配置、固定镜像标签,最后使用同一个 Compose 项目名逐步启动新服务。它不会执行 `docker compose down`,也不会删除卷;`--accept-migrations` 是确认已完成数据库备份的显式开关。
|
||||
|
||||
```bash
|
||||
cd /opt/moyi/releases/<版本>/infra/moyi
|
||||
python3 upgrade_release.py \
|
||||
--new-dir /opt/moyi/releases/<版本>/infra/moyi \
|
||||
--old-dir /opt/moyi/releases/<当前版本>/infra/moyi \
|
||||
--mode https --network --accept-migrations --smoke --login
|
||||
```
|
||||
|
||||
不确定当前目录时可以省略 `--old-dir`,脚本会从 `moyi-test-moyi-api-1` 容器标签读取;如果备份、发布清单、镜像标签或配置校验失败,脚本会在启动新版前退出。通过 SSH heredoc 调用时,脚本及 `deploy.py` 会让子进程从 `/dev/null` 读取标准输入,避免 `docker compose run` 吃掉后续命令。
|
||||
|
||||
轻量应用服务器与RDS/Redis通常需要在阿里云控制台配置同地域内网互通和白名单;如果直接使用ECS并放在同一VPC,则不需要这层轻量服务器互通配置。OSS优先填写同地域内网Endpoint。
|
||||
|
||||
## 停止与数据
|
||||
|
||||
@@ -26,8 +26,12 @@ def fail(message):
|
||||
|
||||
|
||||
def execute(command, capture=False, data=None):
|
||||
result = subprocess.run(command, cwd=HERE, input=data, capture_output=capture,
|
||||
text=True, encoding="utf-8", errors="replace")
|
||||
# Never let docker compose or a diagnostic command consume the caller's
|
||||
# shell stdin. This is especially important when deploy.py is invoked from
|
||||
# an SSH heredoc; only an explicit data payload may be sent to a process.
|
||||
options = {"input": data} if data is not None else {"stdin": subprocess.DEVNULL}
|
||||
result = subprocess.run(command, cwd=HERE, capture_output=capture,
|
||||
text=True, encoding="utf-8", errors="replace", **options)
|
||||
if result.returncode:
|
||||
# Compose error output can contain expanded credentials. Do not echo it.
|
||||
fail("Command failed: " + " ".join(command[:3]) + ". Check local configuration/permissions; do not share secrets.")
|
||||
|
||||
@@ -37,6 +37,20 @@ def configuration():
|
||||
|
||||
|
||||
class DeploymentTests(unittest.TestCase):
|
||||
def test_execute_detaches_stdin_when_no_payload_is_explicitly_supplied(self):
|
||||
completed = SimpleNamespace(returncode=0, stdout="")
|
||||
with patch.object(deploy.subprocess, "run", return_value=completed) as runner:
|
||||
deploy.execute(["docker", "compose", "config"], capture=True)
|
||||
self.assertEqual(runner.call_args.kwargs["stdin"], subprocess.DEVNULL)
|
||||
self.assertNotIn("input", runner.call_args.kwargs)
|
||||
|
||||
def test_execute_passes_only_explicit_payload_to_stdin(self):
|
||||
completed = SimpleNamespace(returncode=0, stdout="")
|
||||
with patch.object(deploy.subprocess, "run", return_value=completed) as runner:
|
||||
deploy.execute(["psql"], data="SELECT 1", capture=True)
|
||||
self.assertEqual(runner.call_args.kwargs["input"], "SELECT 1")
|
||||
self.assertNotIn("stdin", runner.call_args.kwargs)
|
||||
|
||||
def test_duplicate_empty_secret_is_rejected_before_compose(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
env_file = Path(directory) / ".env.cloud"
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
|
||||
DIRECTORY = Path(__file__).resolve().parents[1]
|
||||
SPEC = importlib.util.spec_from_file_location("upgrade_release", DIRECTORY / "upgrade_release.py")
|
||||
upgrade_release = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(upgrade_release)
|
||||
|
||||
|
||||
def config(release, name="moyi-test"):
|
||||
return {
|
||||
"name": name,
|
||||
"services": {
|
||||
"moyi-api": {"image": "moyi-api:" + release, "build": {"dockerfile": "Dockerfile.runtime"}},
|
||||
"frontend": {"image": "moyi-frontend:" + release, "build": {"dockerfile": "Dockerfile.runtime"}},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class UpgradeReleaseTests(unittest.TestCase):
|
||||
def test_child_detaches_stdin(self):
|
||||
completed = SimpleNamespace(returncode=0, stdout="output")
|
||||
with patch.object(upgrade_release.subprocess, "run", return_value=completed) as runner:
|
||||
self.assertEqual(upgrade_release.child(["docker", "info"], capture=True), "output")
|
||||
self.assertEqual(runner.call_args.kwargs["stdin"], subprocess.DEVNULL)
|
||||
|
||||
def test_environment_is_pinned_to_manifest_release_and_runtime_dockerfiles(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
old = root / "old" / "infra" / "moyi"
|
||||
new = root / "20260928-000000-test" / "infra" / "moyi"
|
||||
old.mkdir(parents=True)
|
||||
new.mkdir(parents=True)
|
||||
(old / ".env.cloud").write_text("MOYI_IMAGE_TAG=old\n", encoding="utf-8")
|
||||
(new / "deploy.py").write_text("# test\n", encoding="utf-8")
|
||||
(new / ".env.cloud").write_text("MOYI_IMAGE_TAG=old\n", encoding="utf-8")
|
||||
env = upgrade_release.update_release_environment(old, new, "20260928-000000-test")
|
||||
text = env.read_text(encoding="utf-8")
|
||||
self.assertIn("MOYI_IMAGE_TAG=20260928-000000-test", text)
|
||||
self.assertIn("MOYI_API_DOCKERFILE=Dockerfile.runtime", text)
|
||||
self.assertIn("MOYI_FRONTEND_DOCKERFILE=Dockerfile.runtime", text)
|
||||
|
||||
def test_upgrade_backs_up_before_starting_new_project_without_down(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
release = "20260928-000000-test"
|
||||
old = root / "old" / "infra" / "moyi"
|
||||
new = root / release / "infra" / "moyi"
|
||||
old.mkdir(parents=True)
|
||||
new.mkdir(parents=True)
|
||||
(old / ".env.cloud").write_text("MOYI_IMAGE_TAG=old\n", encoding="utf-8")
|
||||
(old / "deploy.py").write_text("# test\n", encoding="utf-8")
|
||||
(new / "deploy.py").write_text("# test\n", encoding="utf-8")
|
||||
(old / "certs").mkdir()
|
||||
(old / "certs" / "fullchain.pem").write_text("cert\n", encoding="utf-8")
|
||||
(old / "certs" / "privkey.pem").write_text("key\n", encoding="utf-8")
|
||||
(new.parent.parent / "release-manifest.json").write_text(json.dumps({"release": release}), encoding="utf-8")
|
||||
backup_dir = root / "backups" / "pre-upgrade"
|
||||
backup_dir.mkdir(parents=True)
|
||||
(backup_dir / "backup-complete.json").write_text("{}", encoding="utf-8")
|
||||
events = []
|
||||
old_deploy = SimpleNamespace(preflight=Mock(side_effect=lambda *args: config("old")))
|
||||
|
||||
class NewDeploy:
|
||||
def preflight(self, *args):
|
||||
return config(release)
|
||||
|
||||
def compose_command(self, env_file, mode):
|
||||
return ["docker", "compose"]
|
||||
|
||||
def execute(self, command, capture=False):
|
||||
events.append(command)
|
||||
|
||||
with patch.object(upgrade_release, "load_module", side_effect=[old_deploy, NewDeploy()]), \
|
||||
patch.object(upgrade_release, "backup_current", side_effect=lambda *args: (events.append(["backup"]), backup_dir)[1]):
|
||||
upgrade_release.upgrade(old, new, output_root=root / "backups", accept_migrations=True)
|
||||
self.assertEqual(events[0], ["backup"])
|
||||
self.assertTrue(events)
|
||||
self.assertFalse(any("down" in event for event in events if isinstance(event, list)))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,213 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Safely switch a running MOYI Compose project to a prepared release.
|
||||
|
||||
The old release owns the running containers, so the database backup must be
|
||||
performed through the old release's ``backup_database.py``. The new release
|
||||
is validated and started with the same Compose project name; no ``down`` or
|
||||
volume operation is performed. Every child process is detached from stdin so
|
||||
the command is safe to run from an SSH heredoc.
|
||||
"""
|
||||
import argparse
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from types import ModuleType
|
||||
|
||||
|
||||
class UpgradeError(RuntimeError):
|
||||
"""A deliberate safety check prevented the release switch."""
|
||||
|
||||
|
||||
def require(condition, message):
|
||||
if not condition:
|
||||
raise UpgradeError(message)
|
||||
|
||||
|
||||
def child(command, *, cwd=None, capture=False):
|
||||
"""Run a child process without inheriting an SSH/heredoc stdin stream."""
|
||||
result = subprocess.run(
|
||||
command,
|
||||
cwd=cwd,
|
||||
stdin=subprocess.DEVNULL,
|
||||
capture_output=capture,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
)
|
||||
if result.returncode:
|
||||
raise UpgradeError(
|
||||
"Command failed: " + " ".join(command[:3]) +
|
||||
". Check local configuration/permissions; no secret output is shown."
|
||||
)
|
||||
return result.stdout if capture else ""
|
||||
|
||||
|
||||
def load_module(path, name):
|
||||
spec = importlib.util.spec_from_file_location(name, path)
|
||||
if spec is None or spec.loader is None:
|
||||
raise UpgradeError("Cannot load release helper: " + path.name)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def discover_old_dir(container="moyi-test-moyi-api-1"):
|
||||
output = child(
|
||||
["docker", "inspect", "--format", "{{ index .Config.Labels \"com.docker.compose.project.working_dir\" }}", container],
|
||||
capture=True,
|
||||
).strip()
|
||||
require(output, "Cannot discover the running release directory; pass --old-dir explicitly.")
|
||||
return Path(output).resolve()
|
||||
|
||||
|
||||
def copy_runtime_file(old, new, name, mode):
|
||||
source = old / name
|
||||
destination = new / name
|
||||
require(source.is_file(), "Current release is missing " + name + ".")
|
||||
if destination.exists():
|
||||
require(destination.is_file() and destination.read_bytes() == source.read_bytes(),
|
||||
"Prepared release has a different " + name + "; refusing to overwrite it.")
|
||||
else:
|
||||
destination.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
shutil.copyfile(source, destination)
|
||||
os.chmod(destination, mode)
|
||||
|
||||
|
||||
def update_release_environment(old, new, release):
|
||||
"""Copy credentials and pin a prepared release to its immutable image tag."""
|
||||
old_env = old / ".env.cloud"
|
||||
new_env = new / ".env.cloud"
|
||||
require(old_env.is_file(), "Current release is missing .env.cloud.")
|
||||
if new_env.exists():
|
||||
require(new_env.is_file() and new_env.read_bytes() == old_env.read_bytes(),
|
||||
"Prepared .env.cloud differs from the current configuration; refusing to replace credentials.")
|
||||
else:
|
||||
shutil.copyfile(old_env, new_env)
|
||||
os.chmod(new_env, 0o600)
|
||||
text = new_env.read_text(encoding="utf-8-sig")
|
||||
|
||||
def replace_or_append(name, value):
|
||||
nonlocal text
|
||||
pattern = re.compile(r"^(\s*(?:export\s+)?" + re.escape(name) + r"\s*=).*$", re.MULTILINE)
|
||||
text, count = pattern.subn(lambda match: match.group(1) + value, text)
|
||||
if count == 0:
|
||||
text = text.rstrip("\r\n") + "\n" + name + "=" + value + "\n"
|
||||
|
||||
replace_or_append("MOYI_IMAGE_TAG", release)
|
||||
replace_or_append("MOYI_API_DOCKERFILE", "Dockerfile.runtime")
|
||||
replace_or_append("MOYI_FRONTEND_DOCKERFILE", "Dockerfile.runtime")
|
||||
new_env.write_text(text, encoding="utf-8", newline="\n")
|
||||
return new_env
|
||||
|
||||
|
||||
def manifest_for(new):
|
||||
root = new.parent.parent
|
||||
path = root / "release-manifest.json"
|
||||
require(path.is_file(), "Prepared release is missing release-manifest.json.")
|
||||
try:
|
||||
manifest = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError) as error:
|
||||
raise UpgradeError("Prepared release manifest is invalid.") from error
|
||||
release = str(manifest.get("release", ""))
|
||||
require(release and release == root.name, "Release manifest does not match its directory.")
|
||||
return manifest
|
||||
|
||||
|
||||
def image_tag(config, service):
|
||||
image = str(config.get("services", {}).get(service, {}).get("image", ""))
|
||||
return image.rsplit(":", 1)[-1] if ":" in image else ""
|
||||
|
||||
|
||||
def validate_new_config(old_config, new_config, release):
|
||||
old_name = old_config.get("name")
|
||||
new_name = new_config.get("name")
|
||||
if old_name and new_name:
|
||||
require(old_name == new_name, "Compose project name changed; refusing to switch releases.")
|
||||
for service in ("moyi-api", "frontend"):
|
||||
require(image_tag(new_config, service) == release,
|
||||
"Prepared " + service + " image is not tagged " + release + ".")
|
||||
dockerfile = str(new_config.get("services", {}).get(service, {}).get("build", {}).get("dockerfile", ""))
|
||||
require(dockerfile == "Dockerfile.runtime", "Prepared " + service + " is not using Dockerfile.runtime.")
|
||||
|
||||
|
||||
def backup_current(old, mode, output_root):
|
||||
"""Import and execute backup_database.py from the running release directory."""
|
||||
sys.path.insert(0, str(old))
|
||||
try:
|
||||
deploy = load_module(old / "deploy.py", "moyi_old_deploy")
|
||||
backup = load_module(old / "backup_database.py", "moyi_old_backup")
|
||||
return backup.backup(deploy, mode, output_root)
|
||||
finally:
|
||||
sys.path.remove(str(old))
|
||||
|
||||
|
||||
def start_new(deploy, config, env_file, mode):
|
||||
command = deploy.compose_command(env_file, mode)
|
||||
deploy.execute(["docker", "info", "--format", "{{.ServerVersion}}"], capture=True)
|
||||
# Build first, so a failed image build leaves the running release serving traffic.
|
||||
deploy.execute(command + ["build"])
|
||||
deploy.execute(command + ["up", "-d", "--wait", "--wait-timeout", "180", "--no-build", "moyi-api"])
|
||||
deploy.execute(command + ["run", "--rm", "--no-deps", "frontend", "nginx", "-t"])
|
||||
deploy.execute(command + ["up", "-d", "--wait", "--wait-timeout", "180", "--no-build"])
|
||||
deploy.execute(command + ["ps"])
|
||||
|
||||
|
||||
def upgrade(old_dir, new_dir, mode="https", output_root=Path("/opt/moyi/backups"),
|
||||
accept_migrations=False, network=False, run_smoke=False, login=False):
|
||||
old = Path(old_dir).resolve()
|
||||
new = Path(new_dir).resolve()
|
||||
require(old != new, "Old and new release directories must differ.")
|
||||
require(old.is_dir() and (old / "deploy.py").is_file(), "Current release directory is invalid.")
|
||||
require(new.is_dir() and (new / "deploy.py").is_file(), "Prepared release directory is invalid.")
|
||||
require(accept_migrations, "Back up the dedicated RDS database, then pass --accept-migrations.")
|
||||
|
||||
manifest = manifest_for(new)
|
||||
release = str(manifest["release"])
|
||||
old_deploy = load_module(old / "deploy.py", "moyi_running_deploy")
|
||||
old_config = old_deploy.preflight(old / ".env.cloud", mode, False)
|
||||
backup_dir = backup_current(old, mode, Path(output_root).resolve())
|
||||
require((Path(backup_dir) / "backup-complete.json").is_file(), "Database backup did not complete.")
|
||||
|
||||
update_release_environment(old, new, release)
|
||||
copy_runtime_file(old, new, "certs/fullchain.pem", 0o644)
|
||||
copy_runtime_file(old, new, "certs/privkey.pem", 0o600)
|
||||
new_deploy = load_module(new / "deploy.py", "moyi_new_deploy")
|
||||
new_config = new_deploy.preflight(new / ".env.cloud", mode, network)
|
||||
validate_new_config(old_config, new_config, release)
|
||||
start_new(new_deploy, new_config, new / ".env.cloud", mode)
|
||||
(new / ".deploy-mode").write_text(mode + "\n", encoding="ascii")
|
||||
if run_smoke:
|
||||
new_deploy.smoke(new_config, mode, login)
|
||||
print("UPGRADE_OK: backup complete; new release started without docker compose down.")
|
||||
return backup_dir
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--new-dir", type=Path, required=True, help="Prepared release infra/moyi directory")
|
||||
parser.add_argument("--old-dir", type=Path, help="Running release infra/moyi directory; otherwise inspect the API container")
|
||||
parser.add_argument("--container", default="moyi-test-moyi-api-1")
|
||||
parser.add_argument("--mode", choices=("tunnel", "https"), default="https")
|
||||
parser.add_argument("--output-root", type=Path, default=Path("/opt/moyi/backups"))
|
||||
parser.add_argument("--network", action="store_true")
|
||||
parser.add_argument("--accept-migrations", action="store_true")
|
||||
parser.add_argument("--smoke", action="store_true")
|
||||
parser.add_argument("--login", action="store_true")
|
||||
args = parser.parse_args()
|
||||
old = args.old_dir.resolve() if args.old_dir else discover_old_dir(args.container)
|
||||
upgrade(old, args.new_dir, args.mode, args.output_root, args.accept_migrations,
|
||||
args.network, args.smoke, args.login)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except (UpgradeError, OSError, ValueError, subprocess.SubprocessError) as error:
|
||||
message = str(error) if isinstance(error, (UpgradeError, ValueError)) else type(error).__name__ + ": upgrade failed; no secret output shown."
|
||||
print("ERROR: " + message, file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
Reference in new issue
Block a user