feat: add zero-cost performance benchmark workflow

This commit is contained in:
wlx committed 2026-08-13 12:02:36 +08:00
1 parent 54851fc191
commit 5293d93a30
17 files changed
+1366 -8

No files matched your search

+79
View File
@@ -0,0 +1,79 @@
#!/usr/bin/env bash
set -Eeuo pipefail
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
env_file="${1:-${repo_root}/compose/.env}"
infra_compose="${repo_root}/compose/docker-compose.infra.yml"
backend_compose="${repo_root}/compose/docker-compose.backend.yml"
admin_compose="${repo_root}/compose/docker-compose.admin.yml"
if [[ ! -f "$env_file" ]]; then
echo "Environment file not found: $env_file" >&2
exit 1
fi
section() {
printf '\n## %s\n' "$1"
}
read_value() {
local key="$1"
sed -n "s/^${key}=//p" "$env_file" | head -n 1
}
infra_project="$(read_value COMPOSE_PROJECT_NAME)"
backend_project="$(read_value BACKEND_COMPOSE_PROJECT_NAME)"
admin_project="$(read_value ADMIN_COMPOSE_PROJECT_NAME)"
if [[ -z "$infra_project" || -z "$backend_project" || -z "$admin_project" ]]; then
echo "Compose project names are missing from the environment file." >&2
exit 1
fi
section "Runtime snapshot"
bash "${repo_root}/scripts/capture-runtime-snapshot.sh"
section "Application containers"
docker compose --project-name "$infra_project" --env-file "$env_file" -f "$infra_compose" ps --all
docker compose --project-name "$backend_project" --env-file "$env_file" -f "$backend_compose" ps --all
docker compose --project-name "$admin_project" --env-file "$env_file" -f "$admin_compose" ps --all
section "Container resource totals"
docker stats --no-stream --format 'table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.NetIO}}\t{{.BlockIO}}\t{{.PIDs}}'
section "PostgreSQL database statistics"
docker compose --project-name "$infra_project" --env-file "$env_file" -f "$infra_compose" exec -T postgres sh -ec '
psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" -P pager=off -c "
SELECT datname,
numbackends,
xact_commit,
xact_rollback,
blks_read,
blks_hit,
temp_files,
temp_bytes,
deadlocks,
pg_database_size(datname) AS size_bytes
FROM pg_stat_database
WHERE datname IN (current_database(), '\''mlkj_admin'\'')
ORDER BY datname;"
'
section "PostgreSQL connection states"
docker compose --project-name "$infra_project" --env-file "$env_file" -f "$infra_compose" exec -T postgres sh -ec '
psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" -P pager=off -c "
SELECT datname, state, wait_event_type, count(*)
FROM pg_stat_activity
WHERE datname IS NOT NULL
GROUP BY datname, state, wait_event_type
ORDER BY datname, state, wait_event_type;"
'
section "Redis statistics"
docker compose --project-name "$infra_project" --env-file "$env_file" -f "$infra_compose" exec -T redis sh -ec '
redis-cli --no-auth-warning -a "$REDIS_PASSWORD" INFO clients memory stats persistence |
grep -E "^(connected_clients|blocked_clients|used_memory:|used_memory_peak:|mem_fragmentation_ratio:|total_connections_received:|total_commands_processed:|instantaneous_ops_per_sec:|keyspace_hits:|keyspace_misses:|evicted_keys:|expired_keys:|rdb_last_bgsave_status:|aof_enabled:)"
'
section "Docker disk usage"
docker system df
+40 -2
View File
@@ -107,11 +107,49 @@ set_value OSS_PUBLIC_ENDPOINT "http://${public_host}:9000"
set_value CORS_ALLOWED_ORIGINS "http://${public_host},http://${public_host}:8080,http://${public_host}:8081,http://localhost:8080,http://127.0.0.1:8080"
set_value AI_SOULMATE_PUBLIC_URL "http://${public_host}:8080"
set_value FRONTEND_URL "http://${public_host}:8080"
if ! grep -q '^TRIPO3D_API_KEY=' "$env_file"; then
if ! grep -q '^TRIPO3D_MODE=' "$env_file"; then
set_value TRIPO3D_MODE disabled
fi
tripo_mode="$(env_value TRIPO3D_MODE)"
if [[ "$tripo_mode" == "mock" ]]; then
set_value TRIPO3D_API_KEY ""
set_value TRIPO3D_BASE_URL http://127.0.0.1:9/tripo-mock-blocked
else
if ! grep -q '^TRIPO3D_API_KEY=' "$env_file"; then
set_value TRIPO3D_API_KEY ""
fi
if ! grep -q '^TRIPO3D_BASE_URL=' "$env_file"; then
set_value TRIPO3D_BASE_URL https://api.tripo3d.com/v2/openapi
fi
fi
set_value TRIPO3D_BASE_URL https://api.tripo3d.com/v2/openapi
set_value TRIPO3D_DEFAULT_MODEL_VERSION v3.1-20260211
if ! grep -q '^TRIPO3D_MOCK_QUEUE_DELAY_SECONDS=' "$env_file"; then
set_value TRIPO3D_MOCK_QUEUE_DELAY_SECONDS 2
fi
if ! grep -q '^TRIPO3D_MOCK_RUN_DELAY_SECONDS=' "$env_file"; then
set_value TRIPO3D_MOCK_RUN_DELAY_SECONDS 8
fi
if ! grep -q '^TRIPO3D_MOCK_H_CONCURRENCY=' "$env_file"; then
set_value TRIPO3D_MOCK_H_CONCURRENCY 10
fi
if ! grep -q '^TRIPO3D_MOCK_P_CONCURRENCY=' "$env_file"; then
set_value TRIPO3D_MOCK_P_CONCURRENCY 5
fi
if ! grep -q '^TRIPO3D_MOCK_IMAGE_CONCURRENCY=' "$env_file"; then
set_value TRIPO3D_MOCK_IMAGE_CONCURRENCY 1
fi
if ! grep -q '^TRIPO3D_MOCK_ANIMATION_CONCURRENCY=' "$env_file"; then
set_value TRIPO3D_MOCK_ANIMATION_CONCURRENCY 10
fi
if ! grep -q '^TRIPO3D_MOCK_POSTPROCESS_CONCURRENCY=' "$env_file"; then
set_value TRIPO3D_MOCK_POSTPROCESS_CONCURRENCY 5
fi
if ! grep -q '^TRIPO3D_MOCK_MESH_CONCURRENCY=' "$env_file"; then
set_value TRIPO3D_MOCK_MESH_CONCURRENCY 10
fi
if ! grep -q '^TRIPO3D_MOCK_TASK_RETENTION_SECONDS=' "$env_file"; then
set_value TRIPO3D_MOCK_TASK_RETENTION_SECONDS 3600
fi
chmod 600 "$env_file"
printf 'Backend environment configured without displaying secrets.\n'
+11
View File
@@ -103,6 +103,17 @@ FRONTEND_URL=http://127.0.0.1:8080
TRIPO3D_API_KEY=
TRIPO3D_BASE_URL=https://api.tripo3d.com/v2/openapi
TRIPO3D_DEFAULT_MODEL_VERSION=v3.1-20260211
TRIPO3D_MODE=disabled
TRIPO3D_MOCK_PROFILE=not-selected
TRIPO3D_MOCK_QUEUE_DELAY_SECONDS=2
TRIPO3D_MOCK_RUN_DELAY_SECONDS=8
TRIPO3D_MOCK_H_CONCURRENCY=10
TRIPO3D_MOCK_P_CONCURRENCY=5
TRIPO3D_MOCK_IMAGE_CONCURRENCY=1
TRIPO3D_MOCK_ANIMATION_CONCURRENCY=10
TRIPO3D_MOCK_POSTPROCESS_CONCURRENCY=5
TRIPO3D_MOCK_MESH_CONCURRENCY=10
TRIPO3D_MOCK_TASK_RETENTION_SECONDS=3600
EOF
chmod 600 "$target_path"
+81
View File
@@ -0,0 +1,81 @@
#!/usr/bin/env bash
set -Eeuo pipefail
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
env_file="${1:-${repo_root}/compose/.env}"
profile="${2:-}"
if [[ ! -f "$env_file" ]]; then
echo "Environment file not found: $env_file" >&2
exit 1
fi
if [[ -z "$profile" ]]; then
echo "Choose a mock profile explicitly: rate-limits-page or model-pages-conservative" >&2
exit 1
fi
case "$profile" in
rate-limits-page)
h_concurrency=10
p_concurrency=5
image_concurrency=1
animation_concurrency=10
postprocess_concurrency=5
mesh_concurrency=10
;;
model-pages-conservative)
h_concurrency=3
p_concurrency=3
image_concurrency=1
animation_concurrency=3
postprocess_concurrency=5
mesh_concurrency=10
;;
*)
echo "Unknown mock profile: $profile" >&2
echo "Supported profiles: rate-limits-page, model-pages-conservative" >&2
exit 1
;;
esac
set_value() {
local key="$1"
local value="$2"
local temporary_file
temporary_file="$(mktemp "${env_file}.tmp.XXXXXX")"
awk -v key="$key" -v value="$value" '
BEGIN { prefix = key "="; found = 0 }
index($0, prefix) == 1 {
if (!found) print prefix value
found = 1
next
}
{ print }
END { if (!found) print prefix value }
' "$env_file" > "$temporary_file"
chmod 600 "$temporary_file"
mv -f "$temporary_file" "$env_file"
}
set_value TRIPO3D_MODE mock
set_value TRIPO3D_MOCK_PROFILE "$profile"
set_value TRIPO3D_API_KEY ""
set_value TRIPO3D_BASE_URL http://127.0.0.1:9/tripo-mock-blocked
set_value TRIPO3D_MOCK_H_CONCURRENCY "$h_concurrency"
set_value TRIPO3D_MOCK_P_CONCURRENCY "$p_concurrency"
set_value TRIPO3D_MOCK_IMAGE_CONCURRENCY "$image_concurrency"
set_value TRIPO3D_MOCK_ANIMATION_CONCURRENCY "$animation_concurrency"
set_value TRIPO3D_MOCK_POSTPROCESS_CONCURRENCY "$postprocess_concurrency"
set_value TRIPO3D_MOCK_MESH_CONCURRENCY "$mesh_concurrency"
set_value TRIPO3D_MOCK_QUEUE_DELAY_SECONDS 2
set_value TRIPO3D_MOCK_RUN_DELAY_SECONDS 8
set_value TRIPO3D_MOCK_TASK_RETENTION_SECONDS 3600
chmod 600 "$env_file"
echo "Tripo zero-cost mock profile enabled."
echo "Profile: $profile"
echo "H/P/Image/Animation/Postprocess/Mesh concurrency: ${h_concurrency}/${p_concurrency}/${image_concurrency}/${animation_concurrency}/${postprocess_concurrency}/${mesh_concurrency}"
echo "Real Tripo API key cleared and outbound base URL blocked."
+77
View File
@@ -0,0 +1,77 @@
#!/usr/bin/env bash
set -Eeuo pipefail
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
env_file="${1:-${repo_root}/compose/.env}"
run_id="${2:-$(date +%Y%m%d-%H%M%S)}"
request_count="${BENCHMARK_REQUESTS_PER_LEVEL:-200}"
output_root="${repo_root}/results/raw/${run_id}"
if [[ ! -f "$env_file" ]]; then
echo "Environment file not found: $env_file" >&2
exit 1
fi
read_value() {
local key="$1"
sed -n "s/^${key}=//p" "$env_file" | head -n 1
}
backend_bind_address="$(read_value BACKEND_BIND_ADDRESS)"
backend_host_port="$(read_value BACKEND_HOST_PORT)"
admin_backend_bind_address="$(read_value ADMIN_BACKEND_BIND_ADDRESS)"
admin_backend_host_port="$(read_value ADMIN_BACKEND_HOST_PORT)"
if [[ -z "$backend_bind_address" || -z "$backend_host_port" || -z "$admin_backend_bind_address" || -z "$admin_backend_host_port" ]]; then
echo "Backend benchmark endpoints are missing from the environment file." >&2
exit 1
fi
mkdir -p "$output_root"
sampler_stop="${output_root}/.resource-sampler-stop"
rm -f "$sampler_stop"
bash "${repo_root}/scripts/sample-benchmark-resources.sh" "$sampler_stop" \
> "${output_root}/resources-during-load.csv" &
sampler_pid=$!
stop_sampler() {
touch "$sampler_stop"
wait "$sampler_pid" 2>/dev/null || true
rm -f "$sampler_stop"
}
trap stop_sampler EXIT
declare -a names=(user-health admin-health)
declare -a urls=(
"http://${backend_bind_address}:${backend_host_port}/api/health"
"http://${admin_backend_bind_address}:${admin_backend_host_port}/actuator/health"
)
declare -a concurrencies=(1 5 10 20 40)
bash "${repo_root}/scripts/capture-benchmark-snapshot.sh" "$env_file" \
> "${output_root}/snapshot-before.txt"
for index in "${!names[@]}"; do
name="${names[$index]}"
url="${urls[$index]}"
for concurrency in "${concurrencies[@]}"; do
python3 "${repo_root}/benchmark/run-http-benchmark.py" \
--url "$url" \
--requests "$request_count" \
--concurrency "$concurrency" \
--warmup 5 \
--timeout 10 \
--expected-status 200 \
--output "${output_root}/${name}-c${concurrency}.json" \
> /dev/null
echo "Completed ${name} at concurrency ${concurrency}."
done
done
bash "${repo_root}/scripts/capture-benchmark-snapshot.sh" "$env_file" \
> "${output_root}/snapshot-after.txt"
stop_sampler
trap - EXIT
echo "HTTP benchmark suite completed: $output_root"
+71
View File
@@ -0,0 +1,71 @@
#!/usr/bin/env bash
set -Eeuo pipefail
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
env_file="${1:-${repo_root}/compose/.env}"
run_id="${2:-$(date +%Y%m%d-%H%M%S)}"
output_root="${repo_root}/results/raw/${run_id}"
compose_file="${repo_root}/compose/docker-compose.backend.yml"
if [[ ! -f "$env_file" ]]; then
echo "Environment file not found: $env_file" >&2
exit 1
fi
read_value() {
local key="$1"
sed -n "s/^${key}=//p" "$env_file" | head -n 1
}
project_name="$(read_value BACKEND_COMPOSE_PROJECT_NAME)"
profile="$(read_value TRIPO3D_MOCK_PROFILE)"
if [[ -z "$project_name" ]]; then
echo "BACKEND_COMPOSE_PROJECT_NAME is missing." >&2
exit 1
fi
compose_command=(
docker compose
--project-name "$project_name"
--env-file "$env_file"
-f "$compose_file"
)
bash "${repo_root}/scripts/verify-tripo-mock.sh" "$env_file"
mkdir -p "$output_root"
sampler_stop="${output_root}/.resource-sampler-stop"
rm -f "$sampler_stop"
bash "${repo_root}/scripts/sample-benchmark-resources.sh" "$sampler_stop" \
> "${output_root}/resources-during-tripo-mock.csv" &
sampler_pid=$!
stop_sampler() {
touch "$sampler_stop"
wait "$sampler_pid" 2>/dev/null || true
rm -f "$sampler_stop"
}
trap stop_sampler EXIT
bash "${repo_root}/scripts/capture-benchmark-snapshot.sh" "$env_file" \
> "${output_root}/tripo-mock-snapshot-before.txt"
container_id="$("${compose_command[@]}" ps -q backend)"
if [[ -z "$container_id" ]]; then
echo "Backend container is not running." >&2
exit 1
fi
container_result="/tmp/tripo-mock-${run_id}.json"
"${compose_command[@]}" exec -T backend python \
/app/tools/tripo_mock_benchmark.py \
--output "$container_result"
docker cp "${container_id}:${container_result}" \
"${output_root}/tripo-mock-${profile}.json"
bash "${repo_root}/scripts/capture-benchmark-snapshot.sh" "$env_file" \
> "${output_root}/tripo-mock-snapshot-after.txt"
stop_sampler
trap - EXIT
echo "Tripo mock benchmark completed: ${output_root}/tripo-mock-${profile}.json"
+50
View File
@@ -0,0 +1,50 @@
#!/usr/bin/env bash
set -Eeuo pipefail
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
env_file="${1:-${repo_root}/compose/.env}"
run_id="${2:-$(date +%Y%m%d-%H%M%S)}"
output_root="${repo_root}/results/raw/${run_id}"
if [[ ! -f "$env_file" ]]; then
echo "Environment file not found: $env_file" >&2
exit 1
fi
mkdir -p "$output_root"
bash "${repo_root}/scripts/verify-tripo-mock.sh" "$env_file"
sampler_stop="${output_root}/.resource-sampler-stop"
rm -f "$sampler_stop"
bash "${repo_root}/scripts/sample-benchmark-resources.sh" "$sampler_stop" \
> "${output_root}/resources-during-user-e2e.csv" &
sampler_pid=$!
stop_sampler() {
touch "$sampler_stop"
wait "$sampler_pid" 2>/dev/null || true
rm -f "$sampler_stop"
}
trap stop_sampler EXIT
bash "${repo_root}/scripts/capture-benchmark-snapshot.sh" "$env_file" \
> "${output_root}/user-e2e-snapshot-before.txt"
set +e
python3 "${repo_root}/benchmark/run-user-e2e-smoke.py" \
--base-url http://127.0.0.1:3001 \
--output "${output_root}/user-e2e-smoke.json"
smoke_status=$?
set -e
bash "${repo_root}/scripts/capture-benchmark-snapshot.sh" "$env_file" \
> "${output_root}/user-e2e-snapshot-after.txt"
stop_sampler
trap - EXIT
if [[ "$smoke_status" -ne 0 ]]; then
echo "User E2E smoke failed; inspect: ${output_root}/user-e2e-smoke.json" >&2
exit "$smoke_status"
fi
echo "User E2E smoke completed: ${output_root}/user-e2e-smoke.json"
+21
View File
@@ -0,0 +1,21 @@
#!/usr/bin/env bash
set -Eeuo pipefail
stop_file="${1:?stop file is required}"
interval_seconds="${2:-1}"
if ! [[ "$interval_seconds" =~ ^[1-9][0-9]*$ ]]; then
echo "Sampling interval must be a positive integer." >&2
exit 1
fi
printf '%s\n' 'captured_at,load_1m,mem_available_kib,container,cpu_percent,memory_usage,network_io,block_io,pids'
while [[ ! -e "$stop_file" ]]; do
captured_at="$(date --iso-8601=seconds)"
load_1m="$(awk '{print $1}' /proc/loadavg)"
mem_available_kib="$(awk '/^MemAvailable:/ {print $2}' /proc/meminfo)"
docker stats --no-stream \
--format "${captured_at},${load_1m},${mem_available_kib},{{.Name}},{{.CPUPerc}},{{.MemUsage}},{{.NetIO}},{{.BlockIO}},{{.PIDs}}" \
|| true
sleep "$interval_seconds"
done
+127
View File
@@ -0,0 +1,127 @@
#!/usr/bin/env bash
set -Eeuo pipefail
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
env_file="${1:-${repo_root}/compose/.env}"
compose_file="${repo_root}/compose/docker-compose.backend.yml"
if [[ ! -f "$env_file" ]]; then
echo "Environment file not found: $env_file" >&2
exit 1
fi
read_value() {
local key="$1"
sed -n "s/^${key}=//p" "$env_file" | tail -n 1
}
mode="$(read_value TRIPO3D_MODE)"
api_key="$(read_value TRIPO3D_API_KEY)"
base_url="$(read_value TRIPO3D_BASE_URL)"
project_name="$(read_value BACKEND_COMPOSE_PROJECT_NAME)"
profile="$(read_value TRIPO3D_MOCK_PROFILE)"
if [[ "$mode" != "mock" ]]; then
echo "TRIPO3D_MODE must be mock; found: ${mode:-unset}" >&2
exit 1
fi
if [[ -n "$api_key" ]]; then
echo "TRIPO3D_API_KEY must be empty in a zero-cost benchmark." >&2
exit 1
fi
if [[ "$base_url" != "http://127.0.0.1:9/tripo-mock-blocked" ]]; then
echo "TRIPO3D_BASE_URL is not the blocked loopback sentinel." >&2
exit 1
fi
case "$profile" in
rate-limits-page)
expected_limits=(10 5 1 10 5 10)
;;
model-pages-conservative)
expected_limits=(3 3 1 3 5 10)
;;
*)
echo "TRIPO3D_MOCK_PROFILE was not explicitly selected." >&2
exit 1
;;
esac
limit_keys=(
TRIPO3D_MOCK_H_CONCURRENCY
TRIPO3D_MOCK_P_CONCURRENCY
TRIPO3D_MOCK_IMAGE_CONCURRENCY
TRIPO3D_MOCK_ANIMATION_CONCURRENCY
TRIPO3D_MOCK_POSTPROCESS_CONCURRENCY
TRIPO3D_MOCK_MESH_CONCURRENCY
)
for index in "${!limit_keys[@]}"; do
actual_limit="$(read_value "${limit_keys[$index]}")"
if [[ "$actual_limit" != "${expected_limits[$index]}" ]]; then
echo "${limit_keys[$index]}=${actual_limit:-unset} does not match profile ${profile}." >&2
exit 1
fi
done
if [[ "$(read_value TRIPO3D_MOCK_QUEUE_DELAY_SECONDS)" != "2" ]] \
|| [[ "$(read_value TRIPO3D_MOCK_RUN_DELAY_SECONDS)" != "8" ]]; then
echo "Mock queue/run delays do not match the benchmark baseline." >&2
exit 1
fi
if [[ -z "$project_name" ]]; then
echo "BACKEND_COMPOSE_PROJECT_NAME is missing." >&2
exit 1
fi
compose_command=(
docker compose
--project-name "$project_name"
--env-file "$env_file"
-f "$compose_file"
)
"${compose_command[@]}" exec -T backend python -c '
import os
assert os.environ.get("TRIPO3D_MODE") == "mock"
assert not os.environ.get("TRIPO3D_API_KEY")
assert os.environ.get("TRIPO3D_BASE_URL") == "http://127.0.0.1:9/tripo-mock-blocked"
assert os.environ.get("TRIPO3D_MOCK_PROFILE") in {"rate-limits-page", "model-pages-conservative"}
expected_profiles = {
"rate-limits-page": (10, 5, 1, 10, 5, 10),
"model-pages-conservative": (3, 3, 1, 3, 5, 10),
}
actual_limits = tuple(int(os.environ[name]) for name in (
"TRIPO3D_MOCK_H_CONCURRENCY",
"TRIPO3D_MOCK_P_CONCURRENCY",
"TRIPO3D_MOCK_IMAGE_CONCURRENCY",
"TRIPO3D_MOCK_ANIMATION_CONCURRENCY",
"TRIPO3D_MOCK_POSTPROCESS_CONCURRENCY",
"TRIPO3D_MOCK_MESH_CONCURRENCY",
))
assert actual_limits == expected_profiles[os.environ["TRIPO3D_MOCK_PROFILE"]]
assert float(os.environ["TRIPO3D_MOCK_QUEUE_DELAY_SECONDS"]) == 2
assert float(os.environ["TRIPO3D_MOCK_RUN_DELAY_SECONDS"]) == 8
from services.tripo3d import Tripo3DService
service = Tripo3DService()
assert service._api_url("task") == "mock://tripo-local/task"
assert service.billable_credits(40) == 0
print("Container mock invariants passed.")
'
backend_bind_address="$(read_value BACKEND_BIND_ADDRESS)"
backend_host_port="$(read_value BACKEND_HOST_PORT)"
health_json="$(curl --fail --silent --show-error --max-time 5 "http://${backend_bind_address}:${backend_host_port}/api/health")"
if ! printf '%s' "$health_json" | jq -e --arg profile "$profile" \
'.status == "healthy" and .database == "connected" and .tripo_mode == "mock" and .tripo_mock_profile == $profile' >/dev/null; then
echo "Backend health response does not report tripo_mode=mock." >&2
exit 1
fi
echo "Tripo zero-cost runtime verification passed."