fix: quarantine unavailable upstream routes and harden upgrades

This commit is contained in:
wlx committed 2026-09-28 06:34:15 +08:00
1 parent 9acc69d4c7
commit 59907cebdf
8 files changed
+424 -3

No files matched your search

@@ -195,7 +195,7 @@ public class OpenAiCompatibleProvider {
if (status == 429 || details.matches(".*(rate.?limit|too many requests|concurr|并发|频率过高|请求过于频繁|限流).*")) {
return new ApiException("UPSTREAM_RATE_LIMITED", "上游渠道限流或并发已满,请稍后重试", 429);
}
if (status == 404 || details.matches(".*(no available channel|model.{0,16}(not found|does not exist|unavailable)|not_found|unsupported model|没有可用渠道|模型不存在|模型不可用).*")) {
if (status == 404 || details.matches(".*(no available channel|available channel.{0,12}(does not exist|not found)|channel.{0,12}(not found|does not exist|unavailable)|model.{0,16}(not found|does not exist|unavailable)|not_found|unsupported model|没有可用渠道|可用渠道不存在|渠道不存在|模型不存在|模型不可用).*")) {
return new ApiException("UPSTREAM_MODEL_UNAVAILABLE", "上游渠道没有可用的模型路由", 502);
}
return new ApiException("UPSTREAM_PROVIDER_ERROR", "上游模型供应商请求失败", 502);
@@ -0,0 +1,79 @@
-- Keep the public catalog honest after the 2026-09-28 live acceptance run.
-- These routes returned a deterministic "no available channel"/unsupported
-- response from the named upstream. They remain soft-disabled and editable in
-- the admin catalog so an operator can re-enable them after the upstream adds
-- a channel. Routes for the same logical model on another provider are kept.
UPDATE model_catalog AS m
SET status = 'disabled', updated_at = now()
FROM model_providers AS p
WHERE m.provider_id = p.id
AND p.code = 'apipro'
AND m.upstream_model IN (
'Moonshot-Kimi-K2-Instruct',
'claude-fable-5',
'claude-haiku-4-5-20251001-thinking',
'claude-opus-4-1-20250805',
'claude-opus-4-1-20250805-thinking',
'claude-opus-4-20250514',
'claude-opus-4-20250514-thinking',
'claude-opus-4-5-20251101-thinking',
'claude-sonnet-4-20250514-thinking',
'doubao-seed-1-6-lite-251015',
'gemini-3-flash',
'glm-4.7',
'gpt-4.1-nano-2025-04-14',
'gpt-5-chat',
'gpt-5-chat-latest',
'gpt-5-nano-2025-08-07',
'gpt-5.1-codex',
'gpt-5.2-pro',
'gpt-5.4-pro',
'gpt-5.5-pro',
'kimi-k2-thinking',
'kimi-k2.5',
'kimi-k2.7-code',
'minimax-m2.7'
)
AND m.status = 'active'
AND m.deleted_at IS NULL;
-- TokenFoundryX's own playground currently reports these default-group routes
-- as "可用渠道不存在". Keep the successful TokenFoundryX routes visible and
-- quarantine only the models that failed the live probe.
UPDATE model_catalog AS m
SET status = 'disabled', updated_at = now()
FROM model_providers AS p
WHERE m.provider_id = p.id
AND p.code = 'tokenfoundryx'
AND m.source_label = 'TokenFoundryX 海外'
AND m.upstream_model IN (
'claude-fable-5'
)
AND m.status = 'active'
AND m.deleted_at IS NULL;
-- These aliases/wildcards are catalog placeholders rather than stable
-- OpenAI-compatible model IDs. A timeout or generic provider error is not
-- enough evidence to disable a real route, so only wildcard aliases and
-- deterministic "no available channel" results are quarantined here.
UPDATE model_catalog AS m
SET status = 'disabled', updated_at = now()
FROM model_providers AS p
WHERE m.provider_id = p.id
AND p.code = 'apipro'
AND (
m.upstream_model LIKE '%*'
)
AND m.status = 'active'
AND m.deleted_at IS NULL;
-- OCR requires an image payload, but the current chat gateway intentionally
-- accepts text-only messages and the async executor has no OCR adapter. Keep
-- it editable in the admin catalog, but do not publish a route that cannot be
-- called successfully through the public API.
UPDATE model_catalog
SET status = 'disabled',
updated_at = now()
WHERE upstream_model = 'deepseek-ocr'
AND provider_id IN (SELECT id FROM model_providers WHERE code = 'apipro')
AND deleted_at IS NULL;
@@ -35,6 +35,13 @@ class ProviderErrorClassificationTest {
assertEquals("上游渠道没有可用的模型路由", error.getMessage());
}
@Test
void classifiesChineseMissingChannelErrors() {
var error = OpenAiCompatibleProvider.classifyUpstreamError(502, "分组 default 下模型 claude-fable-5 的可用渠道不存在(retry)");
assertEquals("UPSTREAM_MODEL_UNAVAILABLE", error.getCode());
assertEquals(502, error.getStatus());
}
@Test
void classifiesErrorEnvelopeReturnedWithHttp200() {
var error = OpenAiCompatibleProvider.classifyUpstreamError(200, "{\"error\":{\"code\":\"invalid_api_key\",\"message\":\"unauthorized\"}}");
+14
View File
@@ -40,6 +40,20 @@ docker compose --env-file .env.cloud -f docker-compose.cloud.yml build
docker compose --env-file .env.cloud -f docker-compose.cloud.yml up -d
```
### 切换预编译发布包
把新发布包解压到独立的 `releases/<版本>/` 目录后,从新目录执行升级脚本。脚本会先在当前运行目录执行数据库备份,再校验发布清单、复制证书和环境配置、固定镜像标签,最后使用同一个 Compose 项目名逐步启动新服务。它不会执行 `docker compose down`,也不会删除卷;`--accept-migrations` 是确认已完成数据库备份的显式开关。
```bash
cd /opt/moyi/releases/<版本>/infra/moyi
python3 upgrade_release.py \
--new-dir /opt/moyi/releases/<版本>/infra/moyi \
--old-dir /opt/moyi/releases/<当前版本>/infra/moyi \
--mode https --network --accept-migrations --smoke --login
```
不确定当前目录时可以省略 `--old-dir`,脚本会从 `moyi-test-moyi-api-1` 容器标签读取;如果备份、发布清单、镜像标签或配置校验失败,脚本会在启动新版前退出。通过 SSH heredoc 调用时,脚本及 `deploy.py` 会让子进程从 `/dev/null` 读取标准输入,避免 `docker compose run` 吃掉后续命令。
轻量应用服务器与RDS/Redis通常需要在阿里云控制台配置同地域内网互通和白名单;如果直接使用ECS并放在同一VPC,则不需要这层轻量服务器互通配置。OSS优先填写同地域内网Endpoint。
## 停止与数据
+6 -2
View File
@@ -26,8 +26,12 @@ def fail(message):
def execute(command, capture=False, data=None):
result = subprocess.run(command, cwd=HERE, input=data, capture_output=capture,
text=True, encoding="utf-8", errors="replace")
# Never let docker compose or a diagnostic command consume the caller's
# shell stdin. This is especially important when deploy.py is invoked from
# an SSH heredoc; only an explicit data payload may be sent to a process.
options = {"input": data} if data is not None else {"stdin": subprocess.DEVNULL}
result = subprocess.run(command, cwd=HERE, capture_output=capture,
text=True, encoding="utf-8", errors="replace", **options)
if result.returncode:
# Compose error output can contain expanded credentials. Do not echo it.
fail("Command failed: " + " ".join(command[:3]) + ". Check local configuration/permissions; do not share secrets.")
+14
View File
@@ -37,6 +37,20 @@ def configuration():
class DeploymentTests(unittest.TestCase):
def test_execute_detaches_stdin_when_no_payload_is_explicitly_supplied(self):
completed = SimpleNamespace(returncode=0, stdout="")
with patch.object(deploy.subprocess, "run", return_value=completed) as runner:
deploy.execute(["docker", "compose", "config"], capture=True)
self.assertEqual(runner.call_args.kwargs["stdin"], subprocess.DEVNULL)
self.assertNotIn("input", runner.call_args.kwargs)
def test_execute_passes_only_explicit_payload_to_stdin(self):
completed = SimpleNamespace(returncode=0, stdout="")
with patch.object(deploy.subprocess, "run", return_value=completed) as runner:
deploy.execute(["psql"], data="SELECT 1", capture=True)
self.assertEqual(runner.call_args.kwargs["input"], "SELECT 1")
self.assertNotIn("stdin", runner.call_args.kwargs)
def test_duplicate_empty_secret_is_rejected_before_compose(self):
with tempfile.TemporaryDirectory() as directory:
env_file = Path(directory) / ".env.cloud"
+90
View File
@@ -0,0 +1,90 @@
import importlib.util
import json
from pathlib import Path
from types import SimpleNamespace
import subprocess
import tempfile
import unittest
from unittest.mock import Mock, patch
DIRECTORY = Path(__file__).resolve().parents[1]
SPEC = importlib.util.spec_from_file_location("upgrade_release", DIRECTORY / "upgrade_release.py")
upgrade_release = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(upgrade_release)
def config(release, name="moyi-test"):
return {
"name": name,
"services": {
"moyi-api": {"image": "moyi-api:" + release, "build": {"dockerfile": "Dockerfile.runtime"}},
"frontend": {"image": "moyi-frontend:" + release, "build": {"dockerfile": "Dockerfile.runtime"}},
},
}
class UpgradeReleaseTests(unittest.TestCase):
def test_child_detaches_stdin(self):
completed = SimpleNamespace(returncode=0, stdout="output")
with patch.object(upgrade_release.subprocess, "run", return_value=completed) as runner:
self.assertEqual(upgrade_release.child(["docker", "info"], capture=True), "output")
self.assertEqual(runner.call_args.kwargs["stdin"], subprocess.DEVNULL)
def test_environment_is_pinned_to_manifest_release_and_runtime_dockerfiles(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
old = root / "old" / "infra" / "moyi"
new = root / "20260928-000000-test" / "infra" / "moyi"
old.mkdir(parents=True)
new.mkdir(parents=True)
(old / ".env.cloud").write_text("MOYI_IMAGE_TAG=old\n", encoding="utf-8")
(new / "deploy.py").write_text("# test\n", encoding="utf-8")
(new / ".env.cloud").write_text("MOYI_IMAGE_TAG=old\n", encoding="utf-8")
env = upgrade_release.update_release_environment(old, new, "20260928-000000-test")
text = env.read_text(encoding="utf-8")
self.assertIn("MOYI_IMAGE_TAG=20260928-000000-test", text)
self.assertIn("MOYI_API_DOCKERFILE=Dockerfile.runtime", text)
self.assertIn("MOYI_FRONTEND_DOCKERFILE=Dockerfile.runtime", text)
def test_upgrade_backs_up_before_starting_new_project_without_down(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
release = "20260928-000000-test"
old = root / "old" / "infra" / "moyi"
new = root / release / "infra" / "moyi"
old.mkdir(parents=True)
new.mkdir(parents=True)
(old / ".env.cloud").write_text("MOYI_IMAGE_TAG=old\n", encoding="utf-8")
(old / "deploy.py").write_text("# test\n", encoding="utf-8")
(new / "deploy.py").write_text("# test\n", encoding="utf-8")
(old / "certs").mkdir()
(old / "certs" / "fullchain.pem").write_text("cert\n", encoding="utf-8")
(old / "certs" / "privkey.pem").write_text("key\n", encoding="utf-8")
(new.parent.parent / "release-manifest.json").write_text(json.dumps({"release": release}), encoding="utf-8")
backup_dir = root / "backups" / "pre-upgrade"
backup_dir.mkdir(parents=True)
(backup_dir / "backup-complete.json").write_text("{}", encoding="utf-8")
events = []
old_deploy = SimpleNamespace(preflight=Mock(side_effect=lambda *args: config("old")))
class NewDeploy:
def preflight(self, *args):
return config(release)
def compose_command(self, env_file, mode):
return ["docker", "compose"]
def execute(self, command, capture=False):
events.append(command)
with patch.object(upgrade_release, "load_module", side_effect=[old_deploy, NewDeploy()]), \
patch.object(upgrade_release, "backup_current", side_effect=lambda *args: (events.append(["backup"]), backup_dir)[1]):
upgrade_release.upgrade(old, new, output_root=root / "backups", accept_migrations=True)
self.assertEqual(events[0], ["backup"])
self.assertTrue(events)
self.assertFalse(any("down" in event for event in events if isinstance(event, list)))
if __name__ == "__main__":
unittest.main()
+213
View File
@@ -0,0 +1,213 @@
#!/usr/bin/env python3
"""Safely switch a running MOYI Compose project to a prepared release.
The old release owns the running containers, so the database backup must be
performed through the old release's ``backup_database.py``. The new release
is validated and started with the same Compose project name; no ``down`` or
volume operation is performed. Every child process is detached from stdin so
the command is safe to run from an SSH heredoc.
"""
import argparse
import importlib.util
import json
import os
from pathlib import Path
import re
import shutil
import subprocess
import sys
from types import ModuleType
class UpgradeError(RuntimeError):
"""A deliberate safety check prevented the release switch."""
def require(condition, message):
if not condition:
raise UpgradeError(message)
def child(command, *, cwd=None, capture=False):
"""Run a child process without inheriting an SSH/heredoc stdin stream."""
result = subprocess.run(
command,
cwd=cwd,
stdin=subprocess.DEVNULL,
capture_output=capture,
text=True,
encoding="utf-8",
errors="replace",
)
if result.returncode:
raise UpgradeError(
"Command failed: " + " ".join(command[:3]) +
". Check local configuration/permissions; no secret output is shown."
)
return result.stdout if capture else ""
def load_module(path, name):
spec = importlib.util.spec_from_file_location(name, path)
if spec is None or spec.loader is None:
raise UpgradeError("Cannot load release helper: " + path.name)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def discover_old_dir(container="moyi-test-moyi-api-1"):
output = child(
["docker", "inspect", "--format", "{{ index .Config.Labels \"com.docker.compose.project.working_dir\" }}", container],
capture=True,
).strip()
require(output, "Cannot discover the running release directory; pass --old-dir explicitly.")
return Path(output).resolve()
def copy_runtime_file(old, new, name, mode):
source = old / name
destination = new / name
require(source.is_file(), "Current release is missing " + name + ".")
if destination.exists():
require(destination.is_file() and destination.read_bytes() == source.read_bytes(),
"Prepared release has a different " + name + "; refusing to overwrite it.")
else:
destination.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
shutil.copyfile(source, destination)
os.chmod(destination, mode)
def update_release_environment(old, new, release):
"""Copy credentials and pin a prepared release to its immutable image tag."""
old_env = old / ".env.cloud"
new_env = new / ".env.cloud"
require(old_env.is_file(), "Current release is missing .env.cloud.")
if new_env.exists():
require(new_env.is_file() and new_env.read_bytes() == old_env.read_bytes(),
"Prepared .env.cloud differs from the current configuration; refusing to replace credentials.")
else:
shutil.copyfile(old_env, new_env)
os.chmod(new_env, 0o600)
text = new_env.read_text(encoding="utf-8-sig")
def replace_or_append(name, value):
nonlocal text
pattern = re.compile(r"^(\s*(?:export\s+)?" + re.escape(name) + r"\s*=).*$", re.MULTILINE)
text, count = pattern.subn(lambda match: match.group(1) + value, text)
if count == 0:
text = text.rstrip("\r\n") + "\n" + name + "=" + value + "\n"
replace_or_append("MOYI_IMAGE_TAG", release)
replace_or_append("MOYI_API_DOCKERFILE", "Dockerfile.runtime")
replace_or_append("MOYI_FRONTEND_DOCKERFILE", "Dockerfile.runtime")
new_env.write_text(text, encoding="utf-8", newline="\n")
return new_env
def manifest_for(new):
root = new.parent.parent
path = root / "release-manifest.json"
require(path.is_file(), "Prepared release is missing release-manifest.json.")
try:
manifest = json.loads(path.read_text(encoding="utf-8"))
except (OSError, ValueError) as error:
raise UpgradeError("Prepared release manifest is invalid.") from error
release = str(manifest.get("release", ""))
require(release and release == root.name, "Release manifest does not match its directory.")
return manifest
def image_tag(config, service):
image = str(config.get("services", {}).get(service, {}).get("image", ""))
return image.rsplit(":", 1)[-1] if ":" in image else ""
def validate_new_config(old_config, new_config, release):
old_name = old_config.get("name")
new_name = new_config.get("name")
if old_name and new_name:
require(old_name == new_name, "Compose project name changed; refusing to switch releases.")
for service in ("moyi-api", "frontend"):
require(image_tag(new_config, service) == release,
"Prepared " + service + " image is not tagged " + release + ".")
dockerfile = str(new_config.get("services", {}).get(service, {}).get("build", {}).get("dockerfile", ""))
require(dockerfile == "Dockerfile.runtime", "Prepared " + service + " is not using Dockerfile.runtime.")
def backup_current(old, mode, output_root):
"""Import and execute backup_database.py from the running release directory."""
sys.path.insert(0, str(old))
try:
deploy = load_module(old / "deploy.py", "moyi_old_deploy")
backup = load_module(old / "backup_database.py", "moyi_old_backup")
return backup.backup(deploy, mode, output_root)
finally:
sys.path.remove(str(old))
def start_new(deploy, config, env_file, mode):
command = deploy.compose_command(env_file, mode)
deploy.execute(["docker", "info", "--format", "{{.ServerVersion}}"], capture=True)
# Build first, so a failed image build leaves the running release serving traffic.
deploy.execute(command + ["build"])
deploy.execute(command + ["up", "-d", "--wait", "--wait-timeout", "180", "--no-build", "moyi-api"])
deploy.execute(command + ["run", "--rm", "--no-deps", "frontend", "nginx", "-t"])
deploy.execute(command + ["up", "-d", "--wait", "--wait-timeout", "180", "--no-build"])
deploy.execute(command + ["ps"])
def upgrade(old_dir, new_dir, mode="https", output_root=Path("/opt/moyi/backups"),
accept_migrations=False, network=False, run_smoke=False, login=False):
old = Path(old_dir).resolve()
new = Path(new_dir).resolve()
require(old != new, "Old and new release directories must differ.")
require(old.is_dir() and (old / "deploy.py").is_file(), "Current release directory is invalid.")
require(new.is_dir() and (new / "deploy.py").is_file(), "Prepared release directory is invalid.")
require(accept_migrations, "Back up the dedicated RDS database, then pass --accept-migrations.")
manifest = manifest_for(new)
release = str(manifest["release"])
old_deploy = load_module(old / "deploy.py", "moyi_running_deploy")
old_config = old_deploy.preflight(old / ".env.cloud", mode, False)
backup_dir = backup_current(old, mode, Path(output_root).resolve())
require((Path(backup_dir) / "backup-complete.json").is_file(), "Database backup did not complete.")
update_release_environment(old, new, release)
copy_runtime_file(old, new, "certs/fullchain.pem", 0o644)
copy_runtime_file(old, new, "certs/privkey.pem", 0o600)
new_deploy = load_module(new / "deploy.py", "moyi_new_deploy")
new_config = new_deploy.preflight(new / ".env.cloud", mode, network)
validate_new_config(old_config, new_config, release)
start_new(new_deploy, new_config, new / ".env.cloud", mode)
(new / ".deploy-mode").write_text(mode + "\n", encoding="ascii")
if run_smoke:
new_deploy.smoke(new_config, mode, login)
print("UPGRADE_OK: backup complete; new release started without docker compose down.")
return backup_dir
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--new-dir", type=Path, required=True, help="Prepared release infra/moyi directory")
parser.add_argument("--old-dir", type=Path, help="Running release infra/moyi directory; otherwise inspect the API container")
parser.add_argument("--container", default="moyi-test-moyi-api-1")
parser.add_argument("--mode", choices=("tunnel", "https"), default="https")
parser.add_argument("--output-root", type=Path, default=Path("/opt/moyi/backups"))
parser.add_argument("--network", action="store_true")
parser.add_argument("--accept-migrations", action="store_true")
parser.add_argument("--smoke", action="store_true")
parser.add_argument("--login", action="store_true")
args = parser.parse_args()
old = args.old_dir.resolve() if args.old_dir else discover_old_dir(args.container)
upgrade(old, args.new_dir, args.mode, args.output_root, args.accept_migrations,
args.network, args.smoke, args.login)
if __name__ == "__main__":
try:
main()
except (UpgradeError, OSError, ValueError, subprocess.SubprocessError) as error:
message = str(error) if isinstance(error, (UpgradeError, ValueError)) else type(error).__name__ + ": upgrade failed; no secret output shown."
print("ERROR: " + message, file=sys.stderr)
raise SystemExit(1)