From 59907cebdf26ac641f59784f642afeb88080d9b9 Mon Sep 17 00:00:00 2001 From: WangLX <519407855@qq.com> Date: Mon, 28 Sep 2026 06:27:37 +0800 Subject: [PATCH] fix: quarantine unavailable upstream routes and harden upgrades --- .../provider/OpenAiCompatibleProvider.java | 2 +- ...27__quarantine_live_unavailable_routes.sql | 79 +++++++ .../ProviderErrorClassificationTest.java | 7 + infra/moyi/README.md | 14 ++ infra/moyi/deploy.py | 8 +- infra/moyi/tests/test_deploy.py | 14 ++ infra/moyi/tests/test_upgrade_release.py | 90 ++++++++ infra/moyi/upgrade_release.py | 213 ++++++++++++++++++ 8 files changed, 424 insertions(+), 3 deletions(-) create mode 100644 control-plane-java/src/main/resources/db/migration/V27__quarantine_live_unavailable_routes.sql create mode 100644 infra/moyi/tests/test_upgrade_release.py create mode 100644 infra/moyi/upgrade_release.py diff --git a/control-plane-java/src/main/java/com/moyi/controlplane/provider/OpenAiCompatibleProvider.java b/control-plane-java/src/main/java/com/moyi/controlplane/provider/OpenAiCompatibleProvider.java index b880e9f..a0e78e1 100644 --- a/control-plane-java/src/main/java/com/moyi/controlplane/provider/OpenAiCompatibleProvider.java +++ b/control-plane-java/src/main/java/com/moyi/controlplane/provider/OpenAiCompatibleProvider.java @@ -195,7 +195,7 @@ public class OpenAiCompatibleProvider { if (status == 429 || details.matches(".*(rate.?limit|too many requests|concurr|并发|频率过高|请求过于频繁|限流).*")) { return new ApiException("UPSTREAM_RATE_LIMITED", "上游渠道限流或并发已满,请稍后重试", 429); } - if (status == 404 || details.matches(".*(no available channel|model.{0,16}(not found|does not exist|unavailable)|not_found|unsupported model|没有可用渠道|模型不存在|模型不可用).*")) { + if (status == 404 || details.matches(".*(no available channel|available channel.{0,12}(does not exist|not found)|channel.{0,12}(not found|does not exist|unavailable)|model.{0,16}(not found|does not exist|unavailable)|not_found|unsupported model|没有可用渠道|可用渠道不存在|渠道不存在|模型不存在|模型不可用).*")) { return new ApiException("UPSTREAM_MODEL_UNAVAILABLE", "上游渠道没有可用的模型路由", 502); } return new ApiException("UPSTREAM_PROVIDER_ERROR", "上游模型供应商请求失败", 502); diff --git a/control-plane-java/src/main/resources/db/migration/V27__quarantine_live_unavailable_routes.sql b/control-plane-java/src/main/resources/db/migration/V27__quarantine_live_unavailable_routes.sql new file mode 100644 index 0000000..159c8d2 --- /dev/null +++ b/control-plane-java/src/main/resources/db/migration/V27__quarantine_live_unavailable_routes.sql @@ -0,0 +1,79 @@ +-- Keep the public catalog honest after the 2026-09-28 live acceptance run. +-- These routes returned a deterministic "no available channel"/unsupported +-- response from the named upstream. They remain soft-disabled and editable in +-- the admin catalog so an operator can re-enable them after the upstream adds +-- a channel. Routes for the same logical model on another provider are kept. +UPDATE model_catalog AS m +SET status = 'disabled', updated_at = now() +FROM model_providers AS p +WHERE m.provider_id = p.id + AND p.code = 'apipro' + AND m.upstream_model IN ( + 'Moonshot-Kimi-K2-Instruct', + 'claude-fable-5', + 'claude-haiku-4-5-20251001-thinking', + 'claude-opus-4-1-20250805', + 'claude-opus-4-1-20250805-thinking', + 'claude-opus-4-20250514', + 'claude-opus-4-20250514-thinking', + 'claude-opus-4-5-20251101-thinking', + 'claude-sonnet-4-20250514-thinking', + 'doubao-seed-1-6-lite-251015', + 'gemini-3-flash', + 'glm-4.7', + 'gpt-4.1-nano-2025-04-14', + 'gpt-5-chat', + 'gpt-5-chat-latest', + 'gpt-5-nano-2025-08-07', + 'gpt-5.1-codex', + 'gpt-5.2-pro', + 'gpt-5.4-pro', + 'gpt-5.5-pro', + 'kimi-k2-thinking', + 'kimi-k2.5', + 'kimi-k2.7-code', + 'minimax-m2.7' + ) + AND m.status = 'active' + AND m.deleted_at IS NULL; + +-- TokenFoundryX's own playground currently reports these default-group routes +-- as "可用渠道不存在". Keep the successful TokenFoundryX routes visible and +-- quarantine only the models that failed the live probe. +UPDATE model_catalog AS m +SET status = 'disabled', updated_at = now() +FROM model_providers AS p +WHERE m.provider_id = p.id + AND p.code = 'tokenfoundryx' + AND m.source_label = 'TokenFoundryX 海外' + AND m.upstream_model IN ( + 'claude-fable-5' + ) + AND m.status = 'active' + AND m.deleted_at IS NULL; + +-- These aliases/wildcards are catalog placeholders rather than stable +-- OpenAI-compatible model IDs. A timeout or generic provider error is not +-- enough evidence to disable a real route, so only wildcard aliases and +-- deterministic "no available channel" results are quarantined here. +UPDATE model_catalog AS m +SET status = 'disabled', updated_at = now() +FROM model_providers AS p +WHERE m.provider_id = p.id + AND p.code = 'apipro' + AND ( + m.upstream_model LIKE '%*' + ) + AND m.status = 'active' + AND m.deleted_at IS NULL; + +-- OCR requires an image payload, but the current chat gateway intentionally +-- accepts text-only messages and the async executor has no OCR adapter. Keep +-- it editable in the admin catalog, but do not publish a route that cannot be +-- called successfully through the public API. +UPDATE model_catalog +SET status = 'disabled', + updated_at = now() +WHERE upstream_model = 'deepseek-ocr' + AND provider_id IN (SELECT id FROM model_providers WHERE code = 'apipro') + AND deleted_at IS NULL; diff --git a/control-plane-java/src/test/java/com/moyi/controlplane/provider/ProviderErrorClassificationTest.java b/control-plane-java/src/test/java/com/moyi/controlplane/provider/ProviderErrorClassificationTest.java index 4161c39..7e94755 100644 --- a/control-plane-java/src/test/java/com/moyi/controlplane/provider/ProviderErrorClassificationTest.java +++ b/control-plane-java/src/test/java/com/moyi/controlplane/provider/ProviderErrorClassificationTest.java @@ -35,6 +35,13 @@ class ProviderErrorClassificationTest { assertEquals("上游渠道没有可用的模型路由", error.getMessage()); } + @Test + void classifiesChineseMissingChannelErrors() { + var error = OpenAiCompatibleProvider.classifyUpstreamError(502, "分组 default 下模型 claude-fable-5 的可用渠道不存在(retry)"); + assertEquals("UPSTREAM_MODEL_UNAVAILABLE", error.getCode()); + assertEquals(502, error.getStatus()); + } + @Test void classifiesErrorEnvelopeReturnedWithHttp200() { var error = OpenAiCompatibleProvider.classifyUpstreamError(200, "{\"error\":{\"code\":\"invalid_api_key\",\"message\":\"unauthorized\"}}"); diff --git a/infra/moyi/README.md b/infra/moyi/README.md index 052965c..2893544 100644 --- a/infra/moyi/README.md +++ b/infra/moyi/README.md @@ -40,6 +40,20 @@ docker compose --env-file .env.cloud -f docker-compose.cloud.yml build docker compose --env-file .env.cloud -f docker-compose.cloud.yml up -d ``` +### 切换预编译发布包 + +把新发布包解压到独立的 `releases/<版本>/` 目录后,从新目录执行升级脚本。脚本会先在当前运行目录执行数据库备份,再校验发布清单、复制证书和环境配置、固定镜像标签,最后使用同一个 Compose 项目名逐步启动新服务。它不会执行 `docker compose down`,也不会删除卷;`--accept-migrations` 是确认已完成数据库备份的显式开关。 + +```bash +cd /opt/moyi/releases/<版本>/infra/moyi +python3 upgrade_release.py \ + --new-dir /opt/moyi/releases/<版本>/infra/moyi \ + --old-dir /opt/moyi/releases/<当前版本>/infra/moyi \ + --mode https --network --accept-migrations --smoke --login +``` + +不确定当前目录时可以省略 `--old-dir`,脚本会从 `moyi-test-moyi-api-1` 容器标签读取;如果备份、发布清单、镜像标签或配置校验失败,脚本会在启动新版前退出。通过 SSH heredoc 调用时,脚本及 `deploy.py` 会让子进程从 `/dev/null` 读取标准输入,避免 `docker compose run` 吃掉后续命令。 + 轻量应用服务器与RDS/Redis通常需要在阿里云控制台配置同地域内网互通和白名单;如果直接使用ECS并放在同一VPC,则不需要这层轻量服务器互通配置。OSS优先填写同地域内网Endpoint。 ## 停止与数据 diff --git a/infra/moyi/deploy.py b/infra/moyi/deploy.py index fe04607..5d4e797 100644 --- a/infra/moyi/deploy.py +++ b/infra/moyi/deploy.py @@ -26,8 +26,12 @@ def fail(message): def execute(command, capture=False, data=None): - result = subprocess.run(command, cwd=HERE, input=data, capture_output=capture, - text=True, encoding="utf-8", errors="replace") + # Never let docker compose or a diagnostic command consume the caller's + # shell stdin. This is especially important when deploy.py is invoked from + # an SSH heredoc; only an explicit data payload may be sent to a process. + options = {"input": data} if data is not None else {"stdin": subprocess.DEVNULL} + result = subprocess.run(command, cwd=HERE, capture_output=capture, + text=True, encoding="utf-8", errors="replace", **options) if result.returncode: # Compose error output can contain expanded credentials. Do not echo it. fail("Command failed: " + " ".join(command[:3]) + ". Check local configuration/permissions; do not share secrets.") diff --git a/infra/moyi/tests/test_deploy.py b/infra/moyi/tests/test_deploy.py index 9e643d2..8a63809 100644 --- a/infra/moyi/tests/test_deploy.py +++ b/infra/moyi/tests/test_deploy.py @@ -37,6 +37,20 @@ def configuration(): class DeploymentTests(unittest.TestCase): + def test_execute_detaches_stdin_when_no_payload_is_explicitly_supplied(self): + completed = SimpleNamespace(returncode=0, stdout="") + with patch.object(deploy.subprocess, "run", return_value=completed) as runner: + deploy.execute(["docker", "compose", "config"], capture=True) + self.assertEqual(runner.call_args.kwargs["stdin"], subprocess.DEVNULL) + self.assertNotIn("input", runner.call_args.kwargs) + + def test_execute_passes_only_explicit_payload_to_stdin(self): + completed = SimpleNamespace(returncode=0, stdout="") + with patch.object(deploy.subprocess, "run", return_value=completed) as runner: + deploy.execute(["psql"], data="SELECT 1", capture=True) + self.assertEqual(runner.call_args.kwargs["input"], "SELECT 1") + self.assertNotIn("stdin", runner.call_args.kwargs) + def test_duplicate_empty_secret_is_rejected_before_compose(self): with tempfile.TemporaryDirectory() as directory: env_file = Path(directory) / ".env.cloud" diff --git a/infra/moyi/tests/test_upgrade_release.py b/infra/moyi/tests/test_upgrade_release.py new file mode 100644 index 0000000..686eaaf --- /dev/null +++ b/infra/moyi/tests/test_upgrade_release.py @@ -0,0 +1,90 @@ +import importlib.util +import json +from pathlib import Path +from types import SimpleNamespace +import subprocess +import tempfile +import unittest +from unittest.mock import Mock, patch + + +DIRECTORY = Path(__file__).resolve().parents[1] +SPEC = importlib.util.spec_from_file_location("upgrade_release", DIRECTORY / "upgrade_release.py") +upgrade_release = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(upgrade_release) + + +def config(release, name="moyi-test"): + return { + "name": name, + "services": { + "moyi-api": {"image": "moyi-api:" + release, "build": {"dockerfile": "Dockerfile.runtime"}}, + "frontend": {"image": "moyi-frontend:" + release, "build": {"dockerfile": "Dockerfile.runtime"}}, + }, + } + + +class UpgradeReleaseTests(unittest.TestCase): + def test_child_detaches_stdin(self): + completed = SimpleNamespace(returncode=0, stdout="output") + with patch.object(upgrade_release.subprocess, "run", return_value=completed) as runner: + self.assertEqual(upgrade_release.child(["docker", "info"], capture=True), "output") + self.assertEqual(runner.call_args.kwargs["stdin"], subprocess.DEVNULL) + + def test_environment_is_pinned_to_manifest_release_and_runtime_dockerfiles(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + old = root / "old" / "infra" / "moyi" + new = root / "20260928-000000-test" / "infra" / "moyi" + old.mkdir(parents=True) + new.mkdir(parents=True) + (old / ".env.cloud").write_text("MOYI_IMAGE_TAG=old\n", encoding="utf-8") + (new / "deploy.py").write_text("# test\n", encoding="utf-8") + (new / ".env.cloud").write_text("MOYI_IMAGE_TAG=old\n", encoding="utf-8") + env = upgrade_release.update_release_environment(old, new, "20260928-000000-test") + text = env.read_text(encoding="utf-8") + self.assertIn("MOYI_IMAGE_TAG=20260928-000000-test", text) + self.assertIn("MOYI_API_DOCKERFILE=Dockerfile.runtime", text) + self.assertIn("MOYI_FRONTEND_DOCKERFILE=Dockerfile.runtime", text) + + def test_upgrade_backs_up_before_starting_new_project_without_down(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + release = "20260928-000000-test" + old = root / "old" / "infra" / "moyi" + new = root / release / "infra" / "moyi" + old.mkdir(parents=True) + new.mkdir(parents=True) + (old / ".env.cloud").write_text("MOYI_IMAGE_TAG=old\n", encoding="utf-8") + (old / "deploy.py").write_text("# test\n", encoding="utf-8") + (new / "deploy.py").write_text("# test\n", encoding="utf-8") + (old / "certs").mkdir() + (old / "certs" / "fullchain.pem").write_text("cert\n", encoding="utf-8") + (old / "certs" / "privkey.pem").write_text("key\n", encoding="utf-8") + (new.parent.parent / "release-manifest.json").write_text(json.dumps({"release": release}), encoding="utf-8") + backup_dir = root / "backups" / "pre-upgrade" + backup_dir.mkdir(parents=True) + (backup_dir / "backup-complete.json").write_text("{}", encoding="utf-8") + events = [] + old_deploy = SimpleNamespace(preflight=Mock(side_effect=lambda *args: config("old"))) + + class NewDeploy: + def preflight(self, *args): + return config(release) + + def compose_command(self, env_file, mode): + return ["docker", "compose"] + + def execute(self, command, capture=False): + events.append(command) + + with patch.object(upgrade_release, "load_module", side_effect=[old_deploy, NewDeploy()]), \ + patch.object(upgrade_release, "backup_current", side_effect=lambda *args: (events.append(["backup"]), backup_dir)[1]): + upgrade_release.upgrade(old, new, output_root=root / "backups", accept_migrations=True) + self.assertEqual(events[0], ["backup"]) + self.assertTrue(events) + self.assertFalse(any("down" in event for event in events if isinstance(event, list))) + + +if __name__ == "__main__": + unittest.main() diff --git a/infra/moyi/upgrade_release.py b/infra/moyi/upgrade_release.py new file mode 100644 index 0000000..411b622 --- /dev/null +++ b/infra/moyi/upgrade_release.py @@ -0,0 +1,213 @@ +#!/usr/bin/env python3 +"""Safely switch a running MOYI Compose project to a prepared release. + +The old release owns the running containers, so the database backup must be +performed through the old release's ``backup_database.py``. The new release +is validated and started with the same Compose project name; no ``down`` or +volume operation is performed. Every child process is detached from stdin so +the command is safe to run from an SSH heredoc. +""" +import argparse +import importlib.util +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys +from types import ModuleType + + +class UpgradeError(RuntimeError): + """A deliberate safety check prevented the release switch.""" + + +def require(condition, message): + if not condition: + raise UpgradeError(message) + + +def child(command, *, cwd=None, capture=False): + """Run a child process without inheriting an SSH/heredoc stdin stream.""" + result = subprocess.run( + command, + cwd=cwd, + stdin=subprocess.DEVNULL, + capture_output=capture, + text=True, + encoding="utf-8", + errors="replace", + ) + if result.returncode: + raise UpgradeError( + "Command failed: " + " ".join(command[:3]) + + ". Check local configuration/permissions; no secret output is shown." + ) + return result.stdout if capture else "" + + +def load_module(path, name): + spec = importlib.util.spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise UpgradeError("Cannot load release helper: " + path.name) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def discover_old_dir(container="moyi-test-moyi-api-1"): + output = child( + ["docker", "inspect", "--format", "{{ index .Config.Labels \"com.docker.compose.project.working_dir\" }}", container], + capture=True, + ).strip() + require(output, "Cannot discover the running release directory; pass --old-dir explicitly.") + return Path(output).resolve() + + +def copy_runtime_file(old, new, name, mode): + source = old / name + destination = new / name + require(source.is_file(), "Current release is missing " + name + ".") + if destination.exists(): + require(destination.is_file() and destination.read_bytes() == source.read_bytes(), + "Prepared release has a different " + name + "; refusing to overwrite it.") + else: + destination.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + shutil.copyfile(source, destination) + os.chmod(destination, mode) + + +def update_release_environment(old, new, release): + """Copy credentials and pin a prepared release to its immutable image tag.""" + old_env = old / ".env.cloud" + new_env = new / ".env.cloud" + require(old_env.is_file(), "Current release is missing .env.cloud.") + if new_env.exists(): + require(new_env.is_file() and new_env.read_bytes() == old_env.read_bytes(), + "Prepared .env.cloud differs from the current configuration; refusing to replace credentials.") + else: + shutil.copyfile(old_env, new_env) + os.chmod(new_env, 0o600) + text = new_env.read_text(encoding="utf-8-sig") + + def replace_or_append(name, value): + nonlocal text + pattern = re.compile(r"^(\s*(?:export\s+)?" + re.escape(name) + r"\s*=).*$", re.MULTILINE) + text, count = pattern.subn(lambda match: match.group(1) + value, text) + if count == 0: + text = text.rstrip("\r\n") + "\n" + name + "=" + value + "\n" + + replace_or_append("MOYI_IMAGE_TAG", release) + replace_or_append("MOYI_API_DOCKERFILE", "Dockerfile.runtime") + replace_or_append("MOYI_FRONTEND_DOCKERFILE", "Dockerfile.runtime") + new_env.write_text(text, encoding="utf-8", newline="\n") + return new_env + + +def manifest_for(new): + root = new.parent.parent + path = root / "release-manifest.json" + require(path.is_file(), "Prepared release is missing release-manifest.json.") + try: + manifest = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError) as error: + raise UpgradeError("Prepared release manifest is invalid.") from error + release = str(manifest.get("release", "")) + require(release and release == root.name, "Release manifest does not match its directory.") + return manifest + + +def image_tag(config, service): + image = str(config.get("services", {}).get(service, {}).get("image", "")) + return image.rsplit(":", 1)[-1] if ":" in image else "" + + +def validate_new_config(old_config, new_config, release): + old_name = old_config.get("name") + new_name = new_config.get("name") + if old_name and new_name: + require(old_name == new_name, "Compose project name changed; refusing to switch releases.") + for service in ("moyi-api", "frontend"): + require(image_tag(new_config, service) == release, + "Prepared " + service + " image is not tagged " + release + ".") + dockerfile = str(new_config.get("services", {}).get(service, {}).get("build", {}).get("dockerfile", "")) + require(dockerfile == "Dockerfile.runtime", "Prepared " + service + " is not using Dockerfile.runtime.") + + +def backup_current(old, mode, output_root): + """Import and execute backup_database.py from the running release directory.""" + sys.path.insert(0, str(old)) + try: + deploy = load_module(old / "deploy.py", "moyi_old_deploy") + backup = load_module(old / "backup_database.py", "moyi_old_backup") + return backup.backup(deploy, mode, output_root) + finally: + sys.path.remove(str(old)) + + +def start_new(deploy, config, env_file, mode): + command = deploy.compose_command(env_file, mode) + deploy.execute(["docker", "info", "--format", "{{.ServerVersion}}"], capture=True) + # Build first, so a failed image build leaves the running release serving traffic. + deploy.execute(command + ["build"]) + deploy.execute(command + ["up", "-d", "--wait", "--wait-timeout", "180", "--no-build", "moyi-api"]) + deploy.execute(command + ["run", "--rm", "--no-deps", "frontend", "nginx", "-t"]) + deploy.execute(command + ["up", "-d", "--wait", "--wait-timeout", "180", "--no-build"]) + deploy.execute(command + ["ps"]) + + +def upgrade(old_dir, new_dir, mode="https", output_root=Path("/opt/moyi/backups"), + accept_migrations=False, network=False, run_smoke=False, login=False): + old = Path(old_dir).resolve() + new = Path(new_dir).resolve() + require(old != new, "Old and new release directories must differ.") + require(old.is_dir() and (old / "deploy.py").is_file(), "Current release directory is invalid.") + require(new.is_dir() and (new / "deploy.py").is_file(), "Prepared release directory is invalid.") + require(accept_migrations, "Back up the dedicated RDS database, then pass --accept-migrations.") + + manifest = manifest_for(new) + release = str(manifest["release"]) + old_deploy = load_module(old / "deploy.py", "moyi_running_deploy") + old_config = old_deploy.preflight(old / ".env.cloud", mode, False) + backup_dir = backup_current(old, mode, Path(output_root).resolve()) + require((Path(backup_dir) / "backup-complete.json").is_file(), "Database backup did not complete.") + + update_release_environment(old, new, release) + copy_runtime_file(old, new, "certs/fullchain.pem", 0o644) + copy_runtime_file(old, new, "certs/privkey.pem", 0o600) + new_deploy = load_module(new / "deploy.py", "moyi_new_deploy") + new_config = new_deploy.preflight(new / ".env.cloud", mode, network) + validate_new_config(old_config, new_config, release) + start_new(new_deploy, new_config, new / ".env.cloud", mode) + (new / ".deploy-mode").write_text(mode + "\n", encoding="ascii") + if run_smoke: + new_deploy.smoke(new_config, mode, login) + print("UPGRADE_OK: backup complete; new release started without docker compose down.") + return backup_dir + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--new-dir", type=Path, required=True, help="Prepared release infra/moyi directory") + parser.add_argument("--old-dir", type=Path, help="Running release infra/moyi directory; otherwise inspect the API container") + parser.add_argument("--container", default="moyi-test-moyi-api-1") + parser.add_argument("--mode", choices=("tunnel", "https"), default="https") + parser.add_argument("--output-root", type=Path, default=Path("/opt/moyi/backups")) + parser.add_argument("--network", action="store_true") + parser.add_argument("--accept-migrations", action="store_true") + parser.add_argument("--smoke", action="store_true") + parser.add_argument("--login", action="store_true") + args = parser.parse_args() + old = args.old_dir.resolve() if args.old_dir else discover_old_dir(args.container) + upgrade(old, args.new_dir, args.mode, args.output_root, args.accept_migrations, + args.network, args.smoke, args.login) + + +if __name__ == "__main__": + try: + main() + except (UpgradeError, OSError, ValueError, subprocess.SubprocessError) as error: + message = str(error) if isinstance(error, (UpgradeError, ValueError)) else type(error).__name__ + ": upgrade failed; no secret output shown." + print("ERROR: " + message, file=sys.stderr) + raise SystemExit(1)