feat: add model groups, verified registration and CNY credit grants

This commit is contained in:
wlx committed 2026-09-18 17:27:58 +08:00
1 parent f5ba6cb255
commit 7a18db2123
40 files changed
+1980 -263

No files matched your search

+17
View File
@@ -46,6 +46,23 @@ GATEWAY_DEFAULT_MODEL_CONCURRENCY=4
TASK_MAX_QUEUED_PER_MODEL=100
MOYI_TASK_WORKER_INTERVAL_MS=1000
# Registration requires a valid invitation AND an email/SMS verification code.
# Unconfigured channels fail closed; admins can still create user accounts.
# SMTP authorization password, not your mailbox login password where applicable.
MOYI_SMTP_HOST=
MOYI_SMTP_PORT=465
MOYI_SMTP_FROM=
MOYI_SMTP_USERNAME=
MOYI_SMTP_PASSWORD=
MOYI_SMTP_SSL=true
MOYI_SMTP_STARTTLS=false
# Aliyun domestic SMS: approved signature + template with a ${code} variable.
# Use a dedicated least-privilege RAM credential for SendSms only.
MOYI_SMS_ACCESS_KEY_ID=
MOYI_SMS_ACCESS_KEY_SECRET=
MOYI_SMS_SIGN_NAME=
MOYI_SMS_TEMPLATE_CODE=
# OPTIONAL: use admin UI OR environment for one synchronous provider.
# Environment credentials take precedence over admin UI credentials.
DEEPSEEK_API_KEY=
+278
View File
@@ -0,0 +1,278 @@
#!/usr/bin/env python3
"""Reset the existing owner of a single-instance MOYI cloud deployment."""
import argparse
import base64
import copy
import getpass
import hashlib
import importlib.util
import json
import os
from pathlib import Path
import re
import secrets
import shutil
import subprocess
import sys
import tempfile
import urllib.parse
class MaintenanceError(Exception):
pass
def require(condition, message):
if not condition:
raise MaintenanceError(message)
def run(command, *, cwd=None, env=None, sql=None, timeout=240):
options = {"input": sql} if sql is not None else {"stdin": subprocess.DEVNULL}
result = subprocess.run(command, cwd=cwd, env=env, capture_output=True,
text=True, encoding="utf-8", timeout=timeout, **options)
# Database/Compose error messages can contain secrets or SQL; never echo them.
require(result.returncode == 0, Path(command[0]).name + " failed; output withheld to protect credentials.")
return result.stdout.strip()
def password_hash(password, pepper):
salt = secrets.token_bytes(16)
key = hashlib.pbkdf2_hmac("sha256", (password + pepper).encode("utf-8"), salt, 210000, 32)
encode = lambda value: base64.urlsafe_b64encode(value).decode("ascii").rstrip("=")
return "pbkdf2$210000$" + encode(salt) + "$" + encode(key)
def sql_literal(value):
# Hex also prevents dollar-quote delimiters from appearing inside a DO body.
return "convert_from(decode('" + str(value).encode("utf-8").hex() + "', 'hex'), 'UTF8')"
def updated_environment(original, account, password):
require(password.strip() and not any(ord(char) < 32 or ord(char) == 127 for char in password),
"Password must be nonblank and cannot contain control characters.")
require(password != "admin123", "The development default password is not allowed.")
# Append effective overrides, then ask Compose to validate them. Do not parse
# or rewrite arbitrary existing dotenv values, which may be multiline/quoted.
values = {"ADMIN_ACCOUNT": account, "ADMIN_PASSWORD": password}
return original.rstrip("\r\n") + "\n\n# Owner credentials set by admin_credentials.py\n" + "".join(
key + "=" + json.dumps(value, ensure_ascii=False).replace("$", "$$") + "\n"
for key, value in values.items())
def validate_staged_config(before, after, account, password):
expected = copy.deepcopy(before)
expected["services"]["moyi-api"]["environment"].update(
ADMIN_ACCOUNT=account, ADMIN_PASSWORD=password)
require(after == expected, "Staged configuration differs unexpectedly. Check shell environment overrides; nothing was changed.")
def postgres_environment(api, expected_database):
url = str(api["SPRING_DATASOURCE_URL"])
require(url.startswith("jdbc:postgresql://"), "PostgreSQL JDBC URL required.")
target = urllib.parse.urlsplit(url[5:])
require(urllib.parse.unquote(target.path) == "/" + expected_database and target.hostname
and not target.username and not target.password, "Database target does not match --database.")
query = urllib.parse.parse_qs(target.query, keep_blank_values=True)
supported = {"sslmode": "PGSSLMODE", "sslrootcert": "PGSSLROOTCERT", "sslcert": "PGSSLCERT", "sslkey": "PGSSLKEY"}
require(all(key in supported and len(value) == 1 for key, value in query.items()),
"Unsupported JDBC parameters; review TLS settings before maintenance.")
env = {key: value for key, value in os.environ.items() if not key.startswith("PG")}
env.update(PGHOST=target.hostname, PGPORT=str(target.port or 5432), PGDATABASE=expected_database,
PGUSER=str(api["SPRING_DATASOURCE_USERNAME"]), PGPASSWORD=str(api["SPRING_DATASOURCE_PASSWORD"]),
PGCONNECT_TIMEOUT="10", PGAPPNAME="moyi-admin-maintenance", PGCLIENTENCODING="UTF8")
for key, value in query.items():
env[supported[key]] = value[0]
return env
def query_database(psql, env, sql):
return run([psql, "-X", "--no-password", "-qAt", "-v", "ON_ERROR_STOP=1", "-f", "-"], env=env, sql=sql)
def read_owner(psql, env, configured_account, new_account):
rows = json.loads(query_database(psql, env, """
SELECT COALESCE(json_agg(json_build_object('id', id, 'account', account, 'status', status)), '[]'::json)
FROM public.users WHERE role = 'super_admin';
"""))
require(len(rows) == 1 and rows[0]["status"] == "active", "Expected exactly one active super administrator.")
normalized = re.sub(r"[\s-]", "", configured_account.strip().lower())
# The second case permits retry after DB commit but before env installation.
require(rows[0]["account"] in (normalized, new_account.lower()),
"Existing owner differs from both configured and requested accounts; refusing to change it.")
occupied = query_database(psql, env, "SELECT count(*) FROM public.users WHERE account = "
+ sql_literal(new_account.lower()) + " AND id <> " + sql_literal(rows[0]["id"]) + "::uuid;")
require(occupied == "0", "Requested account is already used by another user; nothing changed.")
return rows[0]
def reset_sql(database, owner, account, encoded):
database, user_id, previous, account, encoded = map(sql_literal, (
database, owner["id"], owner["account"], account.lower(), encoded))
return f"""
BEGIN;
SET LOCAL standard_conforming_strings = on;
SET LOCAL lock_timeout = '10s';
SET LOCAL statement_timeout = '30s';
LOCK TABLE public.users IN SHARE ROW EXCLUSIVE MODE;
DO $moyi_reset$
BEGIN
IF current_database() <> {database} THEN
RAISE EXCEPTION 'Wrong database';
END IF;
IF (SELECT count(*) FROM public.users WHERE role = 'super_admin') <> 1
OR NOT EXISTS (SELECT 1 FROM public.users WHERE id = {user_id}::uuid
AND account = {previous} AND role = 'super_admin' AND status = 'active') THEN
RAISE EXCEPTION 'Owner changed; refusing reset';
END IF;
IF EXISTS (SELECT 1 FROM public.users WHERE account = {account} AND id <> {user_id}::uuid) THEN
RAISE EXCEPTION 'Account already exists';
END IF;
UPDATE public.users SET account = {account}, account_kind = 'username',
password_hash = {encoded}, updated_at = now() WHERE id = {user_id}::uuid;
UPDATE public.auth_sessions SET revoked_at = now()
WHERE user_id = {user_id}::uuid AND revoked_at IS NULL;
INSERT INTO public.auth_audit_log (actor_user_id, event_type, target_type, target_id, details)
VALUES ({user_id}::uuid, 'user.credentials_reset', 'user', {user_id},
json_build_object('fromAccount', {previous}, 'account', {account}, 'method', 'server_cli')::jsonb);
END;
$moyi_reset$;
COMMIT;
SELECT 'OWNER_RESET_COMMITTED';
"""
def write_private(path, data):
descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
with os.fdopen(descriptor, "wb") as stream:
stream.write(data)
stream.flush()
os.fsync(stream.fileno())
def install_environment(env_file, content):
temporary = env_file.with_name(".env.owner-" + secrets.token_hex(8))
write_private(temporary, content)
os.replace(temporary, env_file)
if os.name != "nt":
descriptor = os.open(env_file.parent, os.O_RDONLY | os.O_DIRECTORY)
try:
os.fsync(descriptor)
finally:
os.close(descriptor)
def check_container(deploy, config, env_file, mode):
compose = deploy.compose_command(env_file, mode)
ids = run(compose + ["ps", "-a", "-q", "moyi-api"], cwd=deploy.HERE).splitlines()
require(len(ids) == 1, "Expected one existing API container in this Compose project.")
item = json.loads(run(["docker", "inspect", ids[0]]))[0]
labels = item["Config"]["Labels"]
require(Path(labels["com.docker.compose.project.working_dir"]).resolve() == deploy.HERE.resolve(),
"Container belongs to another release directory.")
require(item["Config"]["Image"] == config["services"]["moyi-api"]["image"],
"Running image differs from deployment configuration.")
actual = dict(value.split("=", 1) for value in item["Config"]["Env"] if "=" in value)
expected = config["services"]["moyi-api"]["environment"]
for key in ("SPRING_DATASOURCE_URL", "SPRING_DATASOURCE_USERNAME", "SPRING_DATASOURCE_PASSWORD",
"AUTH_PASSWORD_PEPPER", "CREDENTIAL_ENCRYPTION_KEY"):
require(actual.get(key) == expected.get(key), "Container/database or encryption settings differ; refusing reset.")
return ids[0], compose
def apply_change(deploy, config, env_file, mode, account, password, database, psql, pg_dump, backup_root):
api = config["services"]["moyi-api"]["environment"]
pg_env = postgres_environment(api, database)
owner = read_owner(psql, pg_env, api["ADMIN_ACCOUNT"], account)
container, compose = check_container(deploy, config, env_file, mode)
original = env_file.read_bytes()
content = updated_environment(original.decode("utf-8-sig"), account, password).encode("utf-8")
backup_root.mkdir(mode=0o700, parents=True, exist_ok=True)
backup = Path(tempfile.mkdtemp(prefix="owner-reset-", dir=backup_root))
write_private(backup / "env.cloud.before", original)
write_private(backup / "env.cloud.after", content)
staged = deploy.load_config(backup / "env.cloud.after", mode)
deploy.validate_config(staged, mode)
validate_staged_config(config, staged, account, password)
print("Private recovery directory: " + str(backup), flush=True)
print("Stopping API for backup and session revocation...", flush=True)
mutation_started = False
try:
run(["docker", "stop", "--time", "40", container])
require(env_file.read_bytes() == original, "Environment file changed concurrently; refusing reset.")
dump = backup / "moyi_control.dump"
run([pg_dump, "--no-password", "--format=custom", "--quote-all-identifiers", "--lock-wait-timeout=10000",
"--file=" + str(dump)], env=pg_env)
require(dump.stat().st_size > 0, "Database backup is empty.")
with dump.open("rb") as stream:
digest = hashlib.file_digest(stream, "sha256").hexdigest()
write_private(backup / "moyi_control.dump.sha256", (digest + " moyi_control.dump\n").encode("ascii"))
encoded = password_hash(password, api["AUTH_PASSWORD_PEPPER"])
mutation_started = True
result = query_database(psql, pg_env, reset_sql(database, owner, account, encoded))
require(result == "OWNER_RESET_COMMITTED", "Database outcome uncertain; keep API stopped and rerun this tool.")
install_environment(env_file, content)
run(compose + ["up", "-d", "--no-deps", "--no-build", "--pull", "never", "--force-recreate",
"--wait", "--wait-timeout", "180", "moyi-api"], cwd=deploy.HERE)
run(compose + ["exec", "-T", "frontend", "nginx", "-t"], cwd=deploy.HERE)
run(compose + ["exec", "-T", "frontend", "nginx", "-s", "reload"], cwd=deploy.HERE)
deploy.smoke(staged, mode, login=True)
except BaseException:
if not mutation_started:
run(["docker", "start", container])
print("Reset was not attempted; original API container restarted.", flush=True)
else:
print("Maintenance did not finish. Do not restore the old env or database blindly. "
"Rerun this tool with the same account/password to reconcile and verify. Recovery: " + str(backup), flush=True)
raise
print("SUPER_ADMIN_RESET_OK: " + account + "; existing user ID preserved; previous sessions revoked.")
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--deployment-dir", type=Path, required=True)
parser.add_argument("--account", required=True)
parser.add_argument("--database", default="moyi_control")
parser.add_argument("--mode", choices=("https", "tunnel"), default="https")
parser.add_argument("--backup-root", type=Path, default=Path("/opt/moyi/backups"))
parser.add_argument("--apply", action="store_true", help="Stop API, back up, reset owner and restart; password is prompted twice")
args = parser.parse_args()
require(re.fullmatch(r"[A-Za-z][A-Za-z0-9_]{0,79}", args.account), "Use an ASCII username starting with a letter.")
require(re.fullmatch(r"[a-z][a-z0-9_]+", args.database) and args.database not in ("postgres", "template0", "template1"),
"Use a dedicated application database.")
require(not args.apply or (os.name == "posix" and os.geteuid() == 0), "Apply must run as root on the Linux server.")
directory = args.deployment_dir.resolve()
spec = importlib.util.spec_from_file_location("moyi_deploy", directory / "deploy.py")
deploy = importlib.util.module_from_spec(spec)
spec.loader.exec_module(deploy)
env_file = directory / ".env.cloud"
psql, pg_dump = shutil.which("psql"), shutil.which("pg_dump")
require(psql and pg_dump, "Install postgresql-client first.")
if args.apply:
import fcntl
os.umask(0o077)
lock = open(directory / ".owner-maintenance.lock", "a")
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
config = deploy.preflight(env_file, args.mode)
api = config["services"]["moyi-api"]["environment"]
env = postgres_environment(api, args.database)
owner = read_owner(psql, env, api["ADMIN_ACCOUNT"], args.account)
check_container(deploy, config, env_file, args.mode)
print("Database: " + args.database + "; existing super administrator: " + owner["account"])
if not args.apply:
print("Read-only check passed. Add --apply to change credentials.")
return
require(sys.stdin.isatty(), "Run interactively in SSH, not through a heredoc or pipe.")
password = getpass.getpass("New super administrator password (hidden): ")
require(password == getpass.getpass("Repeat new password (hidden): "), "Passwords do not match; nothing changed.")
apply_change(deploy, config, env_file, args.mode, args.account, password, args.database,
psql, pg_dump, args.backup_root.resolve())
if __name__ == "__main__":
try:
main()
except (MaintenanceError, ValueError, OSError, subprocess.SubprocessError, KeyboardInterrupt) as error:
message = str(error) if isinstance(error, MaintenanceError) else type(error).__name__ + ": maintenance stopped; no secret output shown."
print("ERROR: " + message, file=sys.stderr)
sys.exit(1)
+79
View File
@@ -0,0 +1,79 @@
#!/usr/bin/env python3
"""Back up the running MOYI database and its configuration before an upgrade."""
import argparse
import hashlib
import importlib.util
import json
import os
from pathlib import Path
import shutil
import subprocess
import sys
import tempfile
from datetime import datetime, timezone
from admin_credentials import MaintenanceError, check_container, postgres_environment, require, run, write_private
def backup(deploy, mode, output_root, database="moyi_control"):
require(database == "moyi_control", "This upgrade backup is restricted to moyi_control.")
programs = {name: shutil.which(name) for name in ("psql", "pg_dump", "pg_restore")}
require(all(programs.values()), "Install postgresql-client before backing up.")
env_file = deploy.HERE / ".env.cloud"
config = deploy.preflight(env_file, mode)
check_container(deploy, config, env_file, mode)
pg_env = postgres_environment(config["services"]["moyi-api"]["environment"], database)
actual = run([programs["psql"], "-X", "--no-password", "-qAt", "-v", "ON_ERROR_STOP=1",
"-c", "SELECT current_database()"], env=pg_env)
require(actual == database, "Connected database does not match moyi_control.")
original = env_file.read_bytes()
output_root.mkdir(mode=0o700, parents=True, exist_ok=True)
destination = Path(tempfile.mkdtemp(prefix="pre-upgrade-", dir=output_root))
print("Private backup directory: " + str(destination), flush=True)
write_private(destination / "env.cloud", original)
if mode == "https":
for name in ("fullchain.pem", "privkey.pem"):
write_private(destination / name, (deploy.HERE / "certs" / name).read_bytes())
dump = destination / "moyi_control.dump"
run([programs["pg_dump"], "--no-password", "--format=custom", "--quote-all-identifiers",
"--lock-wait-timeout=10000", "--file=" + str(dump)], env=pg_env)
require(dump.is_file() and dump.stat().st_size > 0, "Database backup is empty.")
# Read/decompress the whole archive without connecting to any database.
run([programs["pg_restore"], "--file=" + os.devnull, str(dump)])
require(env_file.read_bytes() == original, "Configuration changed during backup; repeat the backup.")
digests = {}
for item in destination.iterdir():
with item.open("rb") as stream:
digests[item.name] = hashlib.file_digest(stream, "sha256").hexdigest()
write_private(destination / "SHA256SUMS", "".join(
value + " " + name + "\n" for name, value in sorted(digests.items())).encode("ascii"))
# This marker is written only after export and full archive validation succeed.
write_private(destination / "backup-complete.json", json.dumps({
"database": database, "deployment": str(deploy.HERE),
"completedAt": datetime.now(timezone.utc).isoformat(), "sha256": digests,
}, indent=2).encode("utf-8"))
print("BACKUP_OK: moyi_control exported; archive read successfully; credentials not printed.")
return destination
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--deployment-dir", type=Path, required=True)
parser.add_argument("--mode", choices=("https", "tunnel"), default="https")
parser.add_argument("--output-root", type=Path, default=Path("/opt/moyi/backups"))
args = parser.parse_args()
os.umask(0o077)
directory = args.deployment_dir.resolve()
spec = importlib.util.spec_from_file_location("running_moyi_deploy", directory / "deploy.py")
deploy = importlib.util.module_from_spec(spec)
spec.loader.exec_module(deploy)
backup(deploy, args.mode, args.output_root.resolve())
if __name__ == "__main__":
try:
main()
except (MaintenanceError, ValueError, OSError, subprocess.SubprocessError) as error:
message = str(error) if isinstance(error, MaintenanceError) else type(error).__name__ + ": backup failed; no secret output shown."
print("ERROR: " + message, file=sys.stderr)
sys.exit(1)
+11
View File
@@ -32,6 +32,17 @@ services:
ADMIN_PASSWORD: ${ADMIN_PASSWORD:?Set a strong initial admin password}
ADMIN_DISPLAY_NAME: ${ADMIN_DISPLAY_NAME:-MOYI Admin}
SESSION_TTL_HOURS: ${SESSION_TTL_HOURS:-168}
MOYI_SMTP_HOST: ${MOYI_SMTP_HOST:-}
MOYI_SMTP_PORT: ${MOYI_SMTP_PORT:-465}
MOYI_SMTP_FROM: ${MOYI_SMTP_FROM:-}
MOYI_SMTP_USERNAME: ${MOYI_SMTP_USERNAME:-}
MOYI_SMTP_PASSWORD: ${MOYI_SMTP_PASSWORD:-}
MOYI_SMTP_SSL: ${MOYI_SMTP_SSL:-true}
MOYI_SMTP_STARTTLS: ${MOYI_SMTP_STARTTLS:-false}
MOYI_SMS_ACCESS_KEY_ID: ${MOYI_SMS_ACCESS_KEY_ID:-}
MOYI_SMS_ACCESS_KEY_SECRET: ${MOYI_SMS_ACCESS_KEY_SECRET:-}
MOYI_SMS_SIGN_NAME: ${MOYI_SMS_SIGN_NAME:-}
MOYI_SMS_TEMPLATE_CODE: ${MOYI_SMS_TEMPLATE_CODE:-}
GATEWAY_REQUESTS_PER_MINUTE: ${GATEWAY_REQUESTS_PER_MINUTE:-60}
GATEWAY_READS_PER_MINUTE: ${GATEWAY_READS_PER_MINUTE:-300}
GATEWAY_DEFAULT_MODEL_CONCURRENCY: ${GATEWAY_DEFAULT_MODEL_CONCURRENCY:-4}
+2
View File
@@ -17,6 +17,7 @@ location /api/ {
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $remote_addr;
proxy_set_header Forwarded "";
proxy_set_header X-Forwarded-Proto $scheme;
proxy_connect_timeout 5s;
proxy_read_timeout 120s;
@@ -28,6 +29,7 @@ location /v1/ {
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $remote_addr;
proxy_set_header Forwarded "";
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header Connection "";
proxy_buffering off;
+246
View File
@@ -0,0 +1,246 @@
import base64
import contextlib
import hashlib
import importlib.util
import io
import json
import os
from pathlib import Path
import shutil
import subprocess
import tempfile
from types import SimpleNamespace
import unittest
from unittest.mock import patch
import uuid
from test_deploy import configuration, deploy, DIRECTORY
SPEC = importlib.util.spec_from_file_location("admin_credentials", DIRECTORY / "admin_credentials.py")
maintenance = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(maintenance)
class CredentialMaintenanceTests(unittest.TestCase):
def test_password_format_matches_java_parameters(self):
for password in ("x", "Test42!", "a'b\\c$#\"", "\u6d4b\u8bd5A1!"):
with self.subTest(password=password):
encoded = maintenance.password_hash(password, "local-test-pepper")
name, iterations, salt, key = encoded.split("$")
self.assertEqual((name, iterations), ("pbkdf2", "210000"))
salt = base64.urlsafe_b64decode(salt + "==")
self.assertEqual(len(salt), 16)
actual = base64.urlsafe_b64decode(key + "=")
self.assertEqual(len(actual), 32)
expected = hashlib.pbkdf2_hmac("sha256", (password + "local-test-pepper").encode(), salt, 210000, 32)
self.assertEqual(expected, actual)
self.assertNotEqual(encoded, maintenance.password_hash(password, "local-test-pepper"))
def test_no_password_minimum_but_rejects_blank_and_control_characters(self):
self.assertIn('ADMIN_PASSWORD="x"', maintenance.updated_environment("OTHER=value\n", "Owner", "x"))
for password in ("", " ", "bad\nline", "bad\x00null", "admin123"):
with self.subTest(password=password), self.assertRaises(maintenance.MaintenanceError):
maintenance.updated_environment("OTHER=value\n", "Owner", password)
def test_database_target_guard_and_no_password_in_argv(self):
api = configuration()["services"]["moyi-api"]["environment"]
env = maintenance.postgres_environment(api, "moyi_control")
self.assertEqual(env["PGDATABASE"], "moyi_control")
with patch.object(maintenance, "run", return_value="ok") as runner:
maintenance.query_database("psql", env, "SELECT 1;")
args, kwargs = runner.call_args
self.assertNotIn(api["SPRING_DATASOURCE_PASSWORD"], " ".join(args[0]))
self.assertEqual(kwargs["env"]["PGPASSWORD"], api["SPRING_DATASOURCE_PASSWORD"])
with self.assertRaisesRegex(maintenance.MaintenanceError, "target"):
maintenance.postgres_environment(api, "other_app")
api["SPRING_DATASOURCE_URL"] += "?sslmode=verify-full&sslrootcert=%2Fopt%2Fmoyi%2Fca.pem"
env = maintenance.postgres_environment(api, "moyi_control")
self.assertEqual(env["PGSSLMODE"], "verify-full")
self.assertEqual(env["PGSSLROOTCERT"], "/opt/moyi/ca.pem")
api["SPRING_DATASOURCE_URL"] += "&unknown=unsafe"
with self.assertRaisesRegex(maintenance.MaintenanceError, "Unsupported"):
maintenance.postgres_environment(api, "moyi_control")
def test_secret_tool_errors_are_not_echoed_and_stdin_is_not_inherited(self):
with patch.object(maintenance.subprocess, "run", return_value=SimpleNamespace(returncode=1, stdout="secret", stderr="secret")) as runner:
with self.assertRaises(maintenance.MaintenanceError) as error:
maintenance.run(["docker", "compose"])
self.assertNotIn("secret", str(error.exception))
self.assertEqual(runner.call_args.kwargs["stdin"], subprocess.DEVNULL)
def test_owner_check_and_recovery_after_commit(self):
owner = {"id": str(uuid.uuid4()), "account": "adimin", "status": "active"}
for rows in ([], [owner, owner], [{**owner, "status": "disabled"}], [{**owner, "account": "other"}]):
with patch.object(maintenance, "query_database", return_value=json.dumps(rows)):
with self.assertRaises(maintenance.MaintenanceError):
maintenance.read_owner("psql", {}, "adimin", "NewOwner")
for account in ("adimin", "newowner"):
with patch.object(maintenance, "query_database", side_effect=[json.dumps([{**owner, "account": account}]), "0"]):
self.assertEqual(maintenance.read_owner("psql", {}, "adimin", "NewOwner")["id"], owner["id"])
with patch.object(maintenance, "query_database", side_effect=[json.dumps([owner]), "1"]):
with self.assertRaisesRegex(maintenance.MaintenanceError, "already used"):
maintenance.read_owner("psql", {}, "adimin", "NewOwner")
def test_config_change_guard_preserves_other_settings(self):
before = configuration()
after = json.loads(json.dumps(before))
after["services"]["moyi-api"]["environment"].update(ADMIN_ACCOUNT="Owner", ADMIN_PASSWORD="x")
maintenance.validate_staged_config(before, after, "Owner", "x")
after["services"]["moyi-api"]["environment"]["AUTH_PASSWORD_PEPPER"] = "changed"
with self.assertRaises(maintenance.MaintenanceError):
maintenance.validate_staged_config(before, after, "Owner", "x")
def test_environment_install_is_atomic(self):
with tempfile.TemporaryDirectory() as directory:
path = Path(directory) / ".env.cloud"
path.write_bytes(b"before")
with patch.object(maintenance.os, "replace", side_effect=OSError("test")):
with self.assertRaises(OSError):
maintenance.install_environment(path, b"after")
self.assertEqual(path.read_bytes(), b"before")
maintenance.install_environment(path, b"after")
self.assertEqual(path.read_bytes(), b"after")
@unittest.skipUnless(shutil.which("docker"), "Docker CLI required")
def test_real_compose_password_quoting_preserves_every_other_setting(self):
text = (DIRECTORY / ".env.cloud.example").read_text(encoding="utf-8")
for old, new in {"CHANGE_ME_RDS_HOST": "rds.internal", "CHANGE_ME_RDS_PASSWORD": "test-db",
"CHANGE_ME_TAIR_HOST": "redis.internal", "CHANGE_ME_TAIR_PASSWORD": "test-cache",
"GENERATE_AUTH_PEPPER": "p" * 64, "GENERATE_CREDENTIAL_KEY": "k" * 64,
"GENERATE_ADMIN_PASSWORD": "test-admin"}.items():
text = text.replace(old, new)
with tempfile.TemporaryDirectory() as directory:
path = Path(directory) / ".env.cloud"
maintenance.write_private(path, text.encode())
before = deploy.load_config(path, "https")
for password in ("x", "Test42!", "a'b\\c$#\"", "end\\", "$$literal${HOME}", "\u6d4b\u8bd5A1!", " spaced "):
with self.subTest(password=password):
path.write_text(maintenance.updated_environment(text, "TestOwner", password), encoding="utf-8")
after = deploy.load_config(path, "https")
maintenance.validate_staged_config(before, after, "TestOwner", password)
@unittest.skipUnless(os.environ.get("MOYI_CREDENTIAL_TEST_PG_BIN"), "Isolated local PostgreSQL required")
class CredentialDatabaseTests(unittest.TestCase):
def setUp(self):
self.psql = str(Path(os.environ["MOYI_CREDENTIAL_TEST_PG_BIN"]) / "psql.exe")
self.name = "moyi_credentials_test_" + uuid.uuid4().hex
self.env = {key: value for key, value in os.environ.items() if not key.startswith("PG")}
self.env.update(PGHOST="127.0.0.1", PGPORT="55439", PGUSER="moyi_role_test", PGDATABASE="moyi_role_test")
maintenance.query_database(self.psql, self.env, 'CREATE DATABASE "' + self.name + '";')
self.env["PGDATABASE"] = self.name
repo = DIRECTORY.parents[1]
migrations = repo / "control-plane-java/src/main/resources/db/migration"
maintenance.query_database(self.psql, self.env, (migrations / "V1__auth.sql").read_text() + (migrations / "V9__admin_roles.sql").read_text())
self.owner = {"id": str(uuid.uuid4()), "account": "oldowner", "status": "active"}
self.other = str(uuid.uuid4())
self.session = str(uuid.uuid4())
self.other_session = str(uuid.uuid4())
maintenance.query_database(self.psql, self.env, f"""
INSERT INTO public.users (id, account, account_kind, password_hash, display_name, role) VALUES
('{self.owner['id']}', 'oldowner', 'username', 'old-hash', 'Original name', 'super_admin'),
('{self.other}', 'ordinary', 'username', 'other-hash', 'Other name', 'user');
INSERT INTO public.auth_sessions (id, token_hash, user_id, expires_at) VALUES
('{self.session}', 'owner-token', '{self.owner['id']}', now() + interval '1 day'),
('{self.other_session}', 'other-token', '{self.other}', now() + interval '1 day');
INSERT INTO public.invitation_codes (id, code_hash, code_prefix, label, created_by) VALUES
('{uuid.uuid4()}', 'invite-hash', 'invite', 'Owner relation', '{self.owner['id']}');
""")
def tearDown(self):
self.env["PGDATABASE"] = "moyi_role_test"
maintenance.query_database(self.psql, self.env, 'DROP DATABASE "' + self.name + '";')
def query(self, sql):
return maintenance.query_database(self.psql, self.env, sql)
def test_reset_preserves_identity_relations_other_users_and_revokes_owner_sessions(self):
encoded = maintenance.password_hash("Test42!", "local-test-pepper")
self.assertEqual(self.query(maintenance.reset_sql(self.name, self.owner, "NewOwner", encoded)), "OWNER_RESET_COMMITTED")
data = json.loads(self.query("SELECT row_to_json(u) FROM public.users u WHERE role = 'super_admin'"))
self.assertEqual(data["id"], self.owner["id"])
self.assertEqual(data["account"], "newowner")
self.assertEqual(data["password_hash"], encoded)
self.assertEqual(data["display_name"], "Original name")
self.assertEqual(self.query("SELECT count(*) FROM public.users"), "2")
self.assertEqual(self.query("SELECT created_by FROM public.invitation_codes"), self.owner["id"])
self.assertEqual(self.query(f"SELECT revoked_at IS NOT NULL FROM public.auth_sessions WHERE id = '{self.session}'"), "t")
self.assertEqual(self.query(f"SELECT revoked_at IS NULL FROM public.auth_sessions WHERE id = '{self.other_session}'"), "t")
self.assertEqual(self.query(f"SELECT password_hash FROM public.users WHERE id = '{self.other}'"), "other-hash")
details = self.query("SELECT details FROM public.auth_audit_log WHERE event_type = 'user.credentials_reset'")
self.assertNotIn("password", details)
self.assertNotIn(encoded, details)
self.assertIn("newowner", details)
current = maintenance.read_owner(self.psql, self.env, "oldowner", "NewOwner")
self.assertEqual(self.query(maintenance.reset_sql(self.name, current, "NewOwner", encoded)), "OWNER_RESET_COMMITTED")
def test_conflict_wrong_database_and_owner_drift_leave_everything_unchanged(self):
for database, owner, account in ((self.name, self.owner, "ordinary"), ("wrong_database", self.owner, "NewOwner"),
(self.name, {**self.owner, "account": "changed"}, "NewOwner")):
with self.subTest(database=database, account=account):
with self.assertRaises(maintenance.MaintenanceError):
self.query(maintenance.reset_sql(database, owner, account, "new-hash"))
self.assertEqual(self.query("SELECT account FROM public.users WHERE role = 'super_admin'"), "oldowner")
self.assertEqual(self.query("SELECT count(*) FROM public.auth_audit_log"), "0")
self.assertEqual(self.query("SELECT count(*) FROM public.auth_sessions WHERE revoked_at IS NOT NULL"), "0")
def test_audit_failure_rolls_back_password_and_session_changes(self):
self.query("ALTER TABLE public.auth_audit_log ADD CONSTRAINT test_reject_audit CHECK (event_type <> 'user.credentials_reset')")
with self.assertRaises(maintenance.MaintenanceError):
self.query(maintenance.reset_sql(self.name, self.owner, "NewOwner", "new-hash"))
self.assertEqual(self.query("SELECT password_hash FROM public.users WHERE role = 'super_admin'"), "old-hash")
self.assertEqual(self.query("SELECT count(*) FROM public.auth_sessions WHERE revoked_at IS NOT NULL"), "0")
@unittest.skipUnless(shutil.which("docker"), "Docker CLI required for config parsing only")
def test_real_backup_and_recovery_when_env_install_fails_after_commit(self):
text = (DIRECTORY / ".env.cloud.example").read_text(encoding="utf-8")
for old, new in {"jdbc:postgresql://CHANGE_ME_RDS_HOST:5432/moyi_control": "jdbc:postgresql://127.0.0.1:55439/" + self.name,
"MOYI_DB_USERNAME=moyi_app": "MOYI_DB_USERNAME=moyi_role_test", "CHANGE_ME_RDS_PASSWORD": "test-only",
"CHANGE_ME_TAIR_HOST": "redis.internal", "CHANGE_ME_TAIR_PASSWORD": "test-cache",
"GENERATE_AUTH_PEPPER": "p" * 64, "GENERATE_CREDENTIAL_KEY": "k" * 64,
"GENERATE_ADMIN_PASSWORD": "test-admin", "ADMIN_ACCOUNT=adimin": "ADMIN_ACCOUNT=oldowner"}.items():
text = text.replace(old, new)
pg_dump = str(Path(self.psql).with_name("pg_dump.exe"))
real_run = maintenance.run
docker_calls = []
def run_command(command, **kwargs):
if command[0] == "docker":
docker_calls.append(command)
return ""
return real_run(command, **kwargs)
with tempfile.TemporaryDirectory() as temporary:
path = Path(temporary) / ".env.cloud"
backups = Path(temporary) / "backups"
maintenance.write_private(path, text.encode())
cfg = deploy.load_config(path, "https")
with patch.object(maintenance, "check_container", return_value=("test-container", ["docker", "compose"])), \
patch.object(maintenance, "run", side_effect=run_command), patch.object(deploy, "smoke") as smoke, \
contextlib.redirect_stdout(io.StringIO()):
with patch.object(maintenance, "install_environment", side_effect=OSError("injected failure")):
with self.assertRaises(OSError):
maintenance.apply_change(deploy, cfg, path, "https", "NewOwner", "x", self.name,
self.psql, pg_dump, backups)
self.assertEqual(path.read_bytes(), text.encode())
self.assertEqual(self.query("SELECT account FROM public.users WHERE role = 'super_admin'"), "newowner")
self.assertFalse(any(call[1] == "start" or "up" in call for call in docker_calls))
smoke.assert_not_called()
backup = next(backups.glob("owner-reset-*"))
pg_restore = str(Path(self.psql).with_name("pg_restore.exe"))
listing = real_run([pg_restore, "--list", str(backup / "moyi_control.dump")])
self.assertIn("TABLE DATA public users", listing)
real_run([pg_restore, "--file=" + os.devnull, str(backup / "moyi_control.dump")])
maintenance.apply_change(deploy, cfg, path, "https", "NewOwner", "x", self.name,
self.psql, pg_dump, backups)
smoke.assert_called_once()
self.assertTrue(smoke.call_args.kwargs["login"])
final = deploy.load_config(path, "https")
maintenance.validate_staged_config(cfg, final, "NewOwner", "x")
self.assertEqual(self.query("SELECT count(*) FROM public.users WHERE role = 'super_admin'"), "1")
self.assertEqual(self.query("SELECT id FROM public.users WHERE role = 'super_admin'"), self.owner["id"])
self.assertTrue(any("--force-recreate" in call and "--no-build" in call for call in docker_calls))
if __name__ == "__main__":
unittest.main()
+66
View File
@@ -0,0 +1,66 @@
import importlib.util
import json
from pathlib import Path
import sys
import tempfile
from types import SimpleNamespace
import unittest
from unittest.mock import patch
from test_deploy import configuration, DIRECTORY
sys.path.insert(0, str(DIRECTORY))
SPEC = importlib.util.spec_from_file_location("backup_database", DIRECTORY / "backup_database.py")
backup_database = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(backup_database)
class BackupTests(unittest.TestCase):
def test_wrong_database_is_rejected_before_export(self):
with tempfile.TemporaryDirectory() as root:
config = configuration()
config["services"]["moyi-api"]["environment"]["SPRING_DATASOURCE_URL"] = "jdbc:postgresql://rds.internal:5432/another_app"
deploy = SimpleNamespace(HERE=Path(root), preflight=lambda *args: config)
with patch.object(backup_database.shutil, "which", side_effect=lambda name: name), \
patch.object(backup_database, "check_container"), \
patch.object(backup_database, "run") as command:
with self.assertRaisesRegex(backup_database.MaintenanceError, "target"):
backup_database.backup(deploy, "tunnel", Path(root) / "backups")
command.assert_not_called()
def test_complete_marker_requires_successful_archive_validation(self):
for corrupt in (False, True):
with self.subTest(corrupt=corrupt), tempfile.TemporaryDirectory() as root:
directory = Path(root)
(directory / ".env.cloud").write_text("MOYI_SMTP_PASSWORD='test-only-secret'\n")
deploy = SimpleNamespace(HERE=directory, preflight=lambda *args: configuration())
seen = []
def command(argv, **kwargs):
seen.append(argv)
self.assertNotIn("test-only-rds-password", " ".join(argv))
if argv[0] == "psql":
return "moyi_control"
if argv[0] == "pg_dump":
Path(next(item[7:] for item in argv if item.startswith("--file="))).write_bytes(b"test archive")
if argv[0] == "pg_restore" and corrupt:
raise backup_database.MaintenanceError("Invalid archive")
return ""
with patch.object(backup_database.shutil, "which", side_effect=lambda name: name), \
patch.object(backup_database, "check_container"), patch.object(backup_database, "run", side_effect=command):
if corrupt:
with self.assertRaisesRegex(backup_database.MaintenanceError, "Invalid archive"):
backup_database.backup(deploy, "tunnel", directory / "backups")
self.assertEqual([], list((directory / "backups").rglob("backup-complete.json")))
else:
destination = backup_database.backup(deploy, "tunnel", directory / "backups")
report = json.loads((destination / "backup-complete.json").read_text())
self.assertEqual(report["database"], "moyi_control")
self.assertEqual((destination / "env.cloud").read_bytes(), (directory / ".env.cloud").read_bytes())
self.assertEqual(set(report["sha256"]), {"env.cloud", "moyi_control.dump"})
self.assertEqual([argv[0] for argv in seen], ["psql", "pg_dump", "pg_restore"])
if __name__ == "__main__":
unittest.main()
+6
View File
@@ -37,6 +37,12 @@ def configuration():
class DeploymentTests(unittest.TestCase):
def test_proxy_replaces_forwarding_headers_for_verification_limits(self):
routes = (DIRECTORY / "nginx" / "routes.conf").read_text(encoding="utf-8")
self.assertEqual(routes.count('proxy_set_header Forwarded "";'), 2)
self.assertEqual(routes.count('proxy_set_header X-Forwarded-For $remote_addr;'), 2)
self.assertNotIn('$proxy_add_x_forwarded_for', routes)
def test_login_smoke_requires_super_admin_and_checks_management(self):
cfg = configuration()
cfg["services"]["moyi-api"]["environment"]["ADMIN_ACCOUNT"] = "adimin"