fix: quarantine unavailable upstream routes and harden upgrades
This commit is contained in:
1 parent
9acc69d4c7
commit
59907cebdf
8 files changed
+424
-3
No files matched your search
+1
-1
@@ -195,7 +195,7 @@ public class OpenAiCompatibleProvider {
|
|||||||
if (status == 429 || details.matches(".*(rate.?limit|too many requests|concurr|并发|频率过高|请求过于频繁|限流).*")) {
|
if (status == 429 || details.matches(".*(rate.?limit|too many requests|concurr|并发|频率过高|请求过于频繁|限流).*")) {
|
||||||
return new ApiException("UPSTREAM_RATE_LIMITED", "上游渠道限流或并发已满,请稍后重试", 429);
|
return new ApiException("UPSTREAM_RATE_LIMITED", "上游渠道限流或并发已满,请稍后重试", 429);
|
||||||
}
|
}
|
||||||
if (status == 404 || details.matches(".*(no available channel|model.{0,16}(not found|does not exist|unavailable)|not_found|unsupported model|没有可用渠道|模型不存在|模型不可用).*")) {
|
if (status == 404 || details.matches(".*(no available channel|available channel.{0,12}(does not exist|not found)|channel.{0,12}(not found|does not exist|unavailable)|model.{0,16}(not found|does not exist|unavailable)|not_found|unsupported model|没有可用渠道|可用渠道不存在|渠道不存在|模型不存在|模型不可用).*")) {
|
||||||
return new ApiException("UPSTREAM_MODEL_UNAVAILABLE", "上游渠道没有可用的模型路由", 502);
|
return new ApiException("UPSTREAM_MODEL_UNAVAILABLE", "上游渠道没有可用的模型路由", 502);
|
||||||
}
|
}
|
||||||
return new ApiException("UPSTREAM_PROVIDER_ERROR", "上游模型供应商请求失败", 502);
|
return new ApiException("UPSTREAM_PROVIDER_ERROR", "上游模型供应商请求失败", 502);
|
||||||
|
|||||||
+79
@@ -0,0 +1,79 @@
|
|||||||
|
-- Keep the public catalog honest after the 2026-09-28 live acceptance run.
|
||||||
|
-- These routes returned a deterministic "no available channel"/unsupported
|
||||||
|
-- response from the named upstream. They remain soft-disabled and editable in
|
||||||
|
-- the admin catalog so an operator can re-enable them after the upstream adds
|
||||||
|
-- a channel. Routes for the same logical model on another provider are kept.
|
||||||
|
UPDATE model_catalog AS m
|
||||||
|
SET status = 'disabled', updated_at = now()
|
||||||
|
FROM model_providers AS p
|
||||||
|
WHERE m.provider_id = p.id
|
||||||
|
AND p.code = 'apipro'
|
||||||
|
AND m.upstream_model IN (
|
||||||
|
'Moonshot-Kimi-K2-Instruct',
|
||||||
|
'claude-fable-5',
|
||||||
|
'claude-haiku-4-5-20251001-thinking',
|
||||||
|
'claude-opus-4-1-20250805',
|
||||||
|
'claude-opus-4-1-20250805-thinking',
|
||||||
|
'claude-opus-4-20250514',
|
||||||
|
'claude-opus-4-20250514-thinking',
|
||||||
|
'claude-opus-4-5-20251101-thinking',
|
||||||
|
'claude-sonnet-4-20250514-thinking',
|
||||||
|
'doubao-seed-1-6-lite-251015',
|
||||||
|
'gemini-3-flash',
|
||||||
|
'glm-4.7',
|
||||||
|
'gpt-4.1-nano-2025-04-14',
|
||||||
|
'gpt-5-chat',
|
||||||
|
'gpt-5-chat-latest',
|
||||||
|
'gpt-5-nano-2025-08-07',
|
||||||
|
'gpt-5.1-codex',
|
||||||
|
'gpt-5.2-pro',
|
||||||
|
'gpt-5.4-pro',
|
||||||
|
'gpt-5.5-pro',
|
||||||
|
'kimi-k2-thinking',
|
||||||
|
'kimi-k2.5',
|
||||||
|
'kimi-k2.7-code',
|
||||||
|
'minimax-m2.7'
|
||||||
|
)
|
||||||
|
AND m.status = 'active'
|
||||||
|
AND m.deleted_at IS NULL;
|
||||||
|
|
||||||
|
-- TokenFoundryX's own playground currently reports these default-group routes
|
||||||
|
-- as "可用渠道不存在". Keep the successful TokenFoundryX routes visible and
|
||||||
|
-- quarantine only the models that failed the live probe.
|
||||||
|
UPDATE model_catalog AS m
|
||||||
|
SET status = 'disabled', updated_at = now()
|
||||||
|
FROM model_providers AS p
|
||||||
|
WHERE m.provider_id = p.id
|
||||||
|
AND p.code = 'tokenfoundryx'
|
||||||
|
AND m.source_label = 'TokenFoundryX 海外'
|
||||||
|
AND m.upstream_model IN (
|
||||||
|
'claude-fable-5'
|
||||||
|
)
|
||||||
|
AND m.status = 'active'
|
||||||
|
AND m.deleted_at IS NULL;
|
||||||
|
|
||||||
|
-- These aliases/wildcards are catalog placeholders rather than stable
|
||||||
|
-- OpenAI-compatible model IDs. A timeout or generic provider error is not
|
||||||
|
-- enough evidence to disable a real route, so only wildcard aliases and
|
||||||
|
-- deterministic "no available channel" results are quarantined here.
|
||||||
|
UPDATE model_catalog AS m
|
||||||
|
SET status = 'disabled', updated_at = now()
|
||||||
|
FROM model_providers AS p
|
||||||
|
WHERE m.provider_id = p.id
|
||||||
|
AND p.code = 'apipro'
|
||||||
|
AND (
|
||||||
|
m.upstream_model LIKE '%*'
|
||||||
|
)
|
||||||
|
AND m.status = 'active'
|
||||||
|
AND m.deleted_at IS NULL;
|
||||||
|
|
||||||
|
-- OCR requires an image payload, but the current chat gateway intentionally
|
||||||
|
-- accepts text-only messages and the async executor has no OCR adapter. Keep
|
||||||
|
-- it editable in the admin catalog, but do not publish a route that cannot be
|
||||||
|
-- called successfully through the public API.
|
||||||
|
UPDATE model_catalog
|
||||||
|
SET status = 'disabled',
|
||||||
|
updated_at = now()
|
||||||
|
WHERE upstream_model = 'deepseek-ocr'
|
||||||
|
AND provider_id IN (SELECT id FROM model_providers WHERE code = 'apipro')
|
||||||
|
AND deleted_at IS NULL;
|
||||||
+7
@@ -35,6 +35,13 @@ class ProviderErrorClassificationTest {
|
|||||||
assertEquals("上游渠道没有可用的模型路由", error.getMessage());
|
assertEquals("上游渠道没有可用的模型路由", error.getMessage());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void classifiesChineseMissingChannelErrors() {
|
||||||
|
var error = OpenAiCompatibleProvider.classifyUpstreamError(502, "分组 default 下模型 claude-fable-5 的可用渠道不存在(retry)");
|
||||||
|
assertEquals("UPSTREAM_MODEL_UNAVAILABLE", error.getCode());
|
||||||
|
assertEquals(502, error.getStatus());
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void classifiesErrorEnvelopeReturnedWithHttp200() {
|
void classifiesErrorEnvelopeReturnedWithHttp200() {
|
||||||
var error = OpenAiCompatibleProvider.classifyUpstreamError(200, "{\"error\":{\"code\":\"invalid_api_key\",\"message\":\"unauthorized\"}}");
|
var error = OpenAiCompatibleProvider.classifyUpstreamError(200, "{\"error\":{\"code\":\"invalid_api_key\",\"message\":\"unauthorized\"}}");
|
||||||
|
|||||||
@@ -40,6 +40,20 @@ docker compose --env-file .env.cloud -f docker-compose.cloud.yml build
|
|||||||
docker compose --env-file .env.cloud -f docker-compose.cloud.yml up -d
|
docker compose --env-file .env.cloud -f docker-compose.cloud.yml up -d
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### 切换预编译发布包
|
||||||
|
|
||||||
|
把新发布包解压到独立的 `releases/<版本>/` 目录后,从新目录执行升级脚本。脚本会先在当前运行目录执行数据库备份,再校验发布清单、复制证书和环境配置、固定镜像标签,最后使用同一个 Compose 项目名逐步启动新服务。它不会执行 `docker compose down`,也不会删除卷;`--accept-migrations` 是确认已完成数据库备份的显式开关。
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /opt/moyi/releases/<版本>/infra/moyi
|
||||||
|
python3 upgrade_release.py \
|
||||||
|
--new-dir /opt/moyi/releases/<版本>/infra/moyi \
|
||||||
|
--old-dir /opt/moyi/releases/<当前版本>/infra/moyi \
|
||||||
|
--mode https --network --accept-migrations --smoke --login
|
||||||
|
```
|
||||||
|
|
||||||
|
不确定当前目录时可以省略 `--old-dir`,脚本会从 `moyi-test-moyi-api-1` 容器标签读取;如果备份、发布清单、镜像标签或配置校验失败,脚本会在启动新版前退出。通过 SSH heredoc 调用时,脚本及 `deploy.py` 会让子进程从 `/dev/null` 读取标准输入,避免 `docker compose run` 吃掉后续命令。
|
||||||
|
|
||||||
轻量应用服务器与RDS/Redis通常需要在阿里云控制台配置同地域内网互通和白名单;如果直接使用ECS并放在同一VPC,则不需要这层轻量服务器互通配置。OSS优先填写同地域内网Endpoint。
|
轻量应用服务器与RDS/Redis通常需要在阿里云控制台配置同地域内网互通和白名单;如果直接使用ECS并放在同一VPC,则不需要这层轻量服务器互通配置。OSS优先填写同地域内网Endpoint。
|
||||||
|
|
||||||
## 停止与数据
|
## 停止与数据
|
||||||
|
|||||||
@@ -26,8 +26,12 @@ def fail(message):
|
|||||||
|
|
||||||
|
|
||||||
def execute(command, capture=False, data=None):
|
def execute(command, capture=False, data=None):
|
||||||
result = subprocess.run(command, cwd=HERE, input=data, capture_output=capture,
|
# Never let docker compose or a diagnostic command consume the caller's
|
||||||
text=True, encoding="utf-8", errors="replace")
|
# shell stdin. This is especially important when deploy.py is invoked from
|
||||||
|
# an SSH heredoc; only an explicit data payload may be sent to a process.
|
||||||
|
options = {"input": data} if data is not None else {"stdin": subprocess.DEVNULL}
|
||||||
|
result = subprocess.run(command, cwd=HERE, capture_output=capture,
|
||||||
|
text=True, encoding="utf-8", errors="replace", **options)
|
||||||
if result.returncode:
|
if result.returncode:
|
||||||
# Compose error output can contain expanded credentials. Do not echo it.
|
# Compose error output can contain expanded credentials. Do not echo it.
|
||||||
fail("Command failed: " + " ".join(command[:3]) + ". Check local configuration/permissions; do not share secrets.")
|
fail("Command failed: " + " ".join(command[:3]) + ". Check local configuration/permissions; do not share secrets.")
|
||||||
|
|||||||
@@ -37,6 +37,20 @@ def configuration():
|
|||||||
|
|
||||||
|
|
||||||
class DeploymentTests(unittest.TestCase):
|
class DeploymentTests(unittest.TestCase):
|
||||||
|
def test_execute_detaches_stdin_when_no_payload_is_explicitly_supplied(self):
|
||||||
|
completed = SimpleNamespace(returncode=0, stdout="")
|
||||||
|
with patch.object(deploy.subprocess, "run", return_value=completed) as runner:
|
||||||
|
deploy.execute(["docker", "compose", "config"], capture=True)
|
||||||
|
self.assertEqual(runner.call_args.kwargs["stdin"], subprocess.DEVNULL)
|
||||||
|
self.assertNotIn("input", runner.call_args.kwargs)
|
||||||
|
|
||||||
|
def test_execute_passes_only_explicit_payload_to_stdin(self):
|
||||||
|
completed = SimpleNamespace(returncode=0, stdout="")
|
||||||
|
with patch.object(deploy.subprocess, "run", return_value=completed) as runner:
|
||||||
|
deploy.execute(["psql"], data="SELECT 1", capture=True)
|
||||||
|
self.assertEqual(runner.call_args.kwargs["input"], "SELECT 1")
|
||||||
|
self.assertNotIn("stdin", runner.call_args.kwargs)
|
||||||
|
|
||||||
def test_duplicate_empty_secret_is_rejected_before_compose(self):
|
def test_duplicate_empty_secret_is_rejected_before_compose(self):
|
||||||
with tempfile.TemporaryDirectory() as directory:
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
env_file = Path(directory) / ".env.cloud"
|
env_file = Path(directory) / ".env.cloud"
|
||||||
|
|||||||
@@ -0,0 +1,90 @@
|
|||||||
|
import importlib.util
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
from types import SimpleNamespace
|
||||||
|
import subprocess
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import Mock, patch
|
||||||
|
|
||||||
|
|
||||||
|
DIRECTORY = Path(__file__).resolve().parents[1]
|
||||||
|
SPEC = importlib.util.spec_from_file_location("upgrade_release", DIRECTORY / "upgrade_release.py")
|
||||||
|
upgrade_release = importlib.util.module_from_spec(SPEC)
|
||||||
|
SPEC.loader.exec_module(upgrade_release)
|
||||||
|
|
||||||
|
|
||||||
|
def config(release, name="moyi-test"):
|
||||||
|
return {
|
||||||
|
"name": name,
|
||||||
|
"services": {
|
||||||
|
"moyi-api": {"image": "moyi-api:" + release, "build": {"dockerfile": "Dockerfile.runtime"}},
|
||||||
|
"frontend": {"image": "moyi-frontend:" + release, "build": {"dockerfile": "Dockerfile.runtime"}},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class UpgradeReleaseTests(unittest.TestCase):
|
||||||
|
def test_child_detaches_stdin(self):
|
||||||
|
completed = SimpleNamespace(returncode=0, stdout="output")
|
||||||
|
with patch.object(upgrade_release.subprocess, "run", return_value=completed) as runner:
|
||||||
|
self.assertEqual(upgrade_release.child(["docker", "info"], capture=True), "output")
|
||||||
|
self.assertEqual(runner.call_args.kwargs["stdin"], subprocess.DEVNULL)
|
||||||
|
|
||||||
|
def test_environment_is_pinned_to_manifest_release_and_runtime_dockerfiles(self):
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
root = Path(directory)
|
||||||
|
old = root / "old" / "infra" / "moyi"
|
||||||
|
new = root / "20260928-000000-test" / "infra" / "moyi"
|
||||||
|
old.mkdir(parents=True)
|
||||||
|
new.mkdir(parents=True)
|
||||||
|
(old / ".env.cloud").write_text("MOYI_IMAGE_TAG=old\n", encoding="utf-8")
|
||||||
|
(new / "deploy.py").write_text("# test\n", encoding="utf-8")
|
||||||
|
(new / ".env.cloud").write_text("MOYI_IMAGE_TAG=old\n", encoding="utf-8")
|
||||||
|
env = upgrade_release.update_release_environment(old, new, "20260928-000000-test")
|
||||||
|
text = env.read_text(encoding="utf-8")
|
||||||
|
self.assertIn("MOYI_IMAGE_TAG=20260928-000000-test", text)
|
||||||
|
self.assertIn("MOYI_API_DOCKERFILE=Dockerfile.runtime", text)
|
||||||
|
self.assertIn("MOYI_FRONTEND_DOCKERFILE=Dockerfile.runtime", text)
|
||||||
|
|
||||||
|
def test_upgrade_backs_up_before_starting_new_project_without_down(self):
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
root = Path(directory)
|
||||||
|
release = "20260928-000000-test"
|
||||||
|
old = root / "old" / "infra" / "moyi"
|
||||||
|
new = root / release / "infra" / "moyi"
|
||||||
|
old.mkdir(parents=True)
|
||||||
|
new.mkdir(parents=True)
|
||||||
|
(old / ".env.cloud").write_text("MOYI_IMAGE_TAG=old\n", encoding="utf-8")
|
||||||
|
(old / "deploy.py").write_text("# test\n", encoding="utf-8")
|
||||||
|
(new / "deploy.py").write_text("# test\n", encoding="utf-8")
|
||||||
|
(old / "certs").mkdir()
|
||||||
|
(old / "certs" / "fullchain.pem").write_text("cert\n", encoding="utf-8")
|
||||||
|
(old / "certs" / "privkey.pem").write_text("key\n", encoding="utf-8")
|
||||||
|
(new.parent.parent / "release-manifest.json").write_text(json.dumps({"release": release}), encoding="utf-8")
|
||||||
|
backup_dir = root / "backups" / "pre-upgrade"
|
||||||
|
backup_dir.mkdir(parents=True)
|
||||||
|
(backup_dir / "backup-complete.json").write_text("{}", encoding="utf-8")
|
||||||
|
events = []
|
||||||
|
old_deploy = SimpleNamespace(preflight=Mock(side_effect=lambda *args: config("old")))
|
||||||
|
|
||||||
|
class NewDeploy:
|
||||||
|
def preflight(self, *args):
|
||||||
|
return config(release)
|
||||||
|
|
||||||
|
def compose_command(self, env_file, mode):
|
||||||
|
return ["docker", "compose"]
|
||||||
|
|
||||||
|
def execute(self, command, capture=False):
|
||||||
|
events.append(command)
|
||||||
|
|
||||||
|
with patch.object(upgrade_release, "load_module", side_effect=[old_deploy, NewDeploy()]), \
|
||||||
|
patch.object(upgrade_release, "backup_current", side_effect=lambda *args: (events.append(["backup"]), backup_dir)[1]):
|
||||||
|
upgrade_release.upgrade(old, new, output_root=root / "backups", accept_migrations=True)
|
||||||
|
self.assertEqual(events[0], ["backup"])
|
||||||
|
self.assertTrue(events)
|
||||||
|
self.assertFalse(any("down" in event for event in events if isinstance(event, list)))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,213 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Safely switch a running MOYI Compose project to a prepared release.
|
||||||
|
|
||||||
|
The old release owns the running containers, so the database backup must be
|
||||||
|
performed through the old release's ``backup_database.py``. The new release
|
||||||
|
is validated and started with the same Compose project name; no ``down`` or
|
||||||
|
volume operation is performed. Every child process is detached from stdin so
|
||||||
|
the command is safe to run from an SSH heredoc.
|
||||||
|
"""
|
||||||
|
import argparse
|
||||||
|
import importlib.util
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
from types import ModuleType
|
||||||
|
|
||||||
|
|
||||||
|
class UpgradeError(RuntimeError):
|
||||||
|
"""A deliberate safety check prevented the release switch."""
|
||||||
|
|
||||||
|
|
||||||
|
def require(condition, message):
|
||||||
|
if not condition:
|
||||||
|
raise UpgradeError(message)
|
||||||
|
|
||||||
|
|
||||||
|
def child(command, *, cwd=None, capture=False):
|
||||||
|
"""Run a child process without inheriting an SSH/heredoc stdin stream."""
|
||||||
|
result = subprocess.run(
|
||||||
|
command,
|
||||||
|
cwd=cwd,
|
||||||
|
stdin=subprocess.DEVNULL,
|
||||||
|
capture_output=capture,
|
||||||
|
text=True,
|
||||||
|
encoding="utf-8",
|
||||||
|
errors="replace",
|
||||||
|
)
|
||||||
|
if result.returncode:
|
||||||
|
raise UpgradeError(
|
||||||
|
"Command failed: " + " ".join(command[:3]) +
|
||||||
|
". Check local configuration/permissions; no secret output is shown."
|
||||||
|
)
|
||||||
|
return result.stdout if capture else ""
|
||||||
|
|
||||||
|
|
||||||
|
def load_module(path, name):
|
||||||
|
spec = importlib.util.spec_from_file_location(name, path)
|
||||||
|
if spec is None or spec.loader is None:
|
||||||
|
raise UpgradeError("Cannot load release helper: " + path.name)
|
||||||
|
module = importlib.util.module_from_spec(spec)
|
||||||
|
spec.loader.exec_module(module)
|
||||||
|
return module
|
||||||
|
|
||||||
|
|
||||||
|
def discover_old_dir(container="moyi-test-moyi-api-1"):
|
||||||
|
output = child(
|
||||||
|
["docker", "inspect", "--format", "{{ index .Config.Labels \"com.docker.compose.project.working_dir\" }}", container],
|
||||||
|
capture=True,
|
||||||
|
).strip()
|
||||||
|
require(output, "Cannot discover the running release directory; pass --old-dir explicitly.")
|
||||||
|
return Path(output).resolve()
|
||||||
|
|
||||||
|
|
||||||
|
def copy_runtime_file(old, new, name, mode):
|
||||||
|
source = old / name
|
||||||
|
destination = new / name
|
||||||
|
require(source.is_file(), "Current release is missing " + name + ".")
|
||||||
|
if destination.exists():
|
||||||
|
require(destination.is_file() and destination.read_bytes() == source.read_bytes(),
|
||||||
|
"Prepared release has a different " + name + "; refusing to overwrite it.")
|
||||||
|
else:
|
||||||
|
destination.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||||
|
shutil.copyfile(source, destination)
|
||||||
|
os.chmod(destination, mode)
|
||||||
|
|
||||||
|
|
||||||
|
def update_release_environment(old, new, release):
|
||||||
|
"""Copy credentials and pin a prepared release to its immutable image tag."""
|
||||||
|
old_env = old / ".env.cloud"
|
||||||
|
new_env = new / ".env.cloud"
|
||||||
|
require(old_env.is_file(), "Current release is missing .env.cloud.")
|
||||||
|
if new_env.exists():
|
||||||
|
require(new_env.is_file() and new_env.read_bytes() == old_env.read_bytes(),
|
||||||
|
"Prepared .env.cloud differs from the current configuration; refusing to replace credentials.")
|
||||||
|
else:
|
||||||
|
shutil.copyfile(old_env, new_env)
|
||||||
|
os.chmod(new_env, 0o600)
|
||||||
|
text = new_env.read_text(encoding="utf-8-sig")
|
||||||
|
|
||||||
|
def replace_or_append(name, value):
|
||||||
|
nonlocal text
|
||||||
|
pattern = re.compile(r"^(\s*(?:export\s+)?" + re.escape(name) + r"\s*=).*$", re.MULTILINE)
|
||||||
|
text, count = pattern.subn(lambda match: match.group(1) + value, text)
|
||||||
|
if count == 0:
|
||||||
|
text = text.rstrip("\r\n") + "\n" + name + "=" + value + "\n"
|
||||||
|
|
||||||
|
replace_or_append("MOYI_IMAGE_TAG", release)
|
||||||
|
replace_or_append("MOYI_API_DOCKERFILE", "Dockerfile.runtime")
|
||||||
|
replace_or_append("MOYI_FRONTEND_DOCKERFILE", "Dockerfile.runtime")
|
||||||
|
new_env.write_text(text, encoding="utf-8", newline="\n")
|
||||||
|
return new_env
|
||||||
|
|
||||||
|
|
||||||
|
def manifest_for(new):
|
||||||
|
root = new.parent.parent
|
||||||
|
path = root / "release-manifest.json"
|
||||||
|
require(path.is_file(), "Prepared release is missing release-manifest.json.")
|
||||||
|
try:
|
||||||
|
manifest = json.loads(path.read_text(encoding="utf-8"))
|
||||||
|
except (OSError, ValueError) as error:
|
||||||
|
raise UpgradeError("Prepared release manifest is invalid.") from error
|
||||||
|
release = str(manifest.get("release", ""))
|
||||||
|
require(release and release == root.name, "Release manifest does not match its directory.")
|
||||||
|
return manifest
|
||||||
|
|
||||||
|
|
||||||
|
def image_tag(config, service):
|
||||||
|
image = str(config.get("services", {}).get(service, {}).get("image", ""))
|
||||||
|
return image.rsplit(":", 1)[-1] if ":" in image else ""
|
||||||
|
|
||||||
|
|
||||||
|
def validate_new_config(old_config, new_config, release):
|
||||||
|
old_name = old_config.get("name")
|
||||||
|
new_name = new_config.get("name")
|
||||||
|
if old_name and new_name:
|
||||||
|
require(old_name == new_name, "Compose project name changed; refusing to switch releases.")
|
||||||
|
for service in ("moyi-api", "frontend"):
|
||||||
|
require(image_tag(new_config, service) == release,
|
||||||
|
"Prepared " + service + " image is not tagged " + release + ".")
|
||||||
|
dockerfile = str(new_config.get("services", {}).get(service, {}).get("build", {}).get("dockerfile", ""))
|
||||||
|
require(dockerfile == "Dockerfile.runtime", "Prepared " + service + " is not using Dockerfile.runtime.")
|
||||||
|
|
||||||
|
|
||||||
|
def backup_current(old, mode, output_root):
|
||||||
|
"""Import and execute backup_database.py from the running release directory."""
|
||||||
|
sys.path.insert(0, str(old))
|
||||||
|
try:
|
||||||
|
deploy = load_module(old / "deploy.py", "moyi_old_deploy")
|
||||||
|
backup = load_module(old / "backup_database.py", "moyi_old_backup")
|
||||||
|
return backup.backup(deploy, mode, output_root)
|
||||||
|
finally:
|
||||||
|
sys.path.remove(str(old))
|
||||||
|
|
||||||
|
|
||||||
|
def start_new(deploy, config, env_file, mode):
|
||||||
|
command = deploy.compose_command(env_file, mode)
|
||||||
|
deploy.execute(["docker", "info", "--format", "{{.ServerVersion}}"], capture=True)
|
||||||
|
# Build first, so a failed image build leaves the running release serving traffic.
|
||||||
|
deploy.execute(command + ["build"])
|
||||||
|
deploy.execute(command + ["up", "-d", "--wait", "--wait-timeout", "180", "--no-build", "moyi-api"])
|
||||||
|
deploy.execute(command + ["run", "--rm", "--no-deps", "frontend", "nginx", "-t"])
|
||||||
|
deploy.execute(command + ["up", "-d", "--wait", "--wait-timeout", "180", "--no-build"])
|
||||||
|
deploy.execute(command + ["ps"])
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade(old_dir, new_dir, mode="https", output_root=Path("/opt/moyi/backups"),
|
||||||
|
accept_migrations=False, network=False, run_smoke=False, login=False):
|
||||||
|
old = Path(old_dir).resolve()
|
||||||
|
new = Path(new_dir).resolve()
|
||||||
|
require(old != new, "Old and new release directories must differ.")
|
||||||
|
require(old.is_dir() and (old / "deploy.py").is_file(), "Current release directory is invalid.")
|
||||||
|
require(new.is_dir() and (new / "deploy.py").is_file(), "Prepared release directory is invalid.")
|
||||||
|
require(accept_migrations, "Back up the dedicated RDS database, then pass --accept-migrations.")
|
||||||
|
|
||||||
|
manifest = manifest_for(new)
|
||||||
|
release = str(manifest["release"])
|
||||||
|
old_deploy = load_module(old / "deploy.py", "moyi_running_deploy")
|
||||||
|
old_config = old_deploy.preflight(old / ".env.cloud", mode, False)
|
||||||
|
backup_dir = backup_current(old, mode, Path(output_root).resolve())
|
||||||
|
require((Path(backup_dir) / "backup-complete.json").is_file(), "Database backup did not complete.")
|
||||||
|
|
||||||
|
update_release_environment(old, new, release)
|
||||||
|
copy_runtime_file(old, new, "certs/fullchain.pem", 0o644)
|
||||||
|
copy_runtime_file(old, new, "certs/privkey.pem", 0o600)
|
||||||
|
new_deploy = load_module(new / "deploy.py", "moyi_new_deploy")
|
||||||
|
new_config = new_deploy.preflight(new / ".env.cloud", mode, network)
|
||||||
|
validate_new_config(old_config, new_config, release)
|
||||||
|
start_new(new_deploy, new_config, new / ".env.cloud", mode)
|
||||||
|
(new / ".deploy-mode").write_text(mode + "\n", encoding="ascii")
|
||||||
|
if run_smoke:
|
||||||
|
new_deploy.smoke(new_config, mode, login)
|
||||||
|
print("UPGRADE_OK: backup complete; new release started without docker compose down.")
|
||||||
|
return backup_dir
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument("--new-dir", type=Path, required=True, help="Prepared release infra/moyi directory")
|
||||||
|
parser.add_argument("--old-dir", type=Path, help="Running release infra/moyi directory; otherwise inspect the API container")
|
||||||
|
parser.add_argument("--container", default="moyi-test-moyi-api-1")
|
||||||
|
parser.add_argument("--mode", choices=("tunnel", "https"), default="https")
|
||||||
|
parser.add_argument("--output-root", type=Path, default=Path("/opt/moyi/backups"))
|
||||||
|
parser.add_argument("--network", action="store_true")
|
||||||
|
parser.add_argument("--accept-migrations", action="store_true")
|
||||||
|
parser.add_argument("--smoke", action="store_true")
|
||||||
|
parser.add_argument("--login", action="store_true")
|
||||||
|
args = parser.parse_args()
|
||||||
|
old = args.old_dir.resolve() if args.old_dir else discover_old_dir(args.container)
|
||||||
|
upgrade(old, args.new_dir, args.mode, args.output_root, args.accept_migrations,
|
||||||
|
args.network, args.smoke, args.login)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
try:
|
||||||
|
main()
|
||||||
|
except (UpgradeError, OSError, ValueError, subprocess.SubprocessError) as error:
|
||||||
|
message = str(error) if isinstance(error, (UpgradeError, ValueError)) else type(error).__name__ + ": upgrade failed; no secret output shown."
|
||||||
|
print("ERROR: " + message, file=sys.stderr)
|
||||||
|
raise SystemExit(1)
|
||||||
Reference in new issue
Block a user