Files
ai-soulmate-local-deployment/scripts/run-business-benchmark-suite.sh
T

169 lines
6.2 KiB
Bash
Executable File

#!/usr/bin/env bash
set -Eeuo pipefail
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
env_file="${1:-${repo_root}/compose/.env}"
run_id="${2:-business-baseline-$(date +%Y%m%d-%H%M%S)}"
request_count="${BUSINESS_BENCHMARK_REQUESTS_PER_LEVEL:-500}"
repeat_count="${BUSINESS_BENCHMARK_REPEATS:-3}"
output_root="${repo_root}/results/raw/${run_id}"
temporary_root=""
if [[ ! -f "$env_file" ]]; then
echo "Environment file not found: $env_file" >&2
exit 1
fi
if [[ ! "$run_id" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]{0,119}$ || "$run_id" == *..* ]]; then
echo "Run ID must be 1-120 safe filename characters and must not contain '..'." >&2
exit 1
fi
if ! [[ "$request_count" =~ ^[1-9][0-9]*$ ]] || (( request_count > 100000 )); then
echo "BUSINESS_BENCHMARK_REQUESTS_PER_LEVEL must be between 1 and 100000." >&2
exit 1
fi
if ! [[ "$repeat_count" =~ ^[1-9][0-9]*$ ]] || (( repeat_count > 10 )); then
echo "BUSINESS_BENCHMARK_REPEATS must be between 1 and 10." >&2
exit 1
fi
mkdir -p "$output_root"
runtime_root="/dev/shm"
if [[ ! -d "$runtime_root" || ! -w "$runtime_root" || "$(stat -f -c %T "$runtime_root")" != "tmpfs" ]]; then
echo "Writable tmpfs runtime directory is unavailable: $runtime_root" >&2
exit 1
fi
temporary_root="$(mktemp -d "${runtime_root}/ai-soulmate-business-benchmark.XXXXXX")"
headers_file="${temporary_root}/headers.json"
phase_file="${temporary_root}/phase.txt"
sampler_stop="${temporary_root}/sampler-stop"
cleanup() {
if [[ -n "${sampler_pid:-}" ]]; then
touch "$sampler_stop"
wait "$sampler_pid" 2>/dev/null || true
fi
if [[ -r "$headers_file" ]]; then
python3 "${repo_root}/benchmark/close-local-benchmark-session.py" \
--base-url http://127.0.0.1:3001 \
--headers-file "$headers_file" >/dev/null 2>&1 || true
fi
rm -f "$headers_file" "$phase_file" "$sampler_stop"
rmdir "$temporary_root" 2>/dev/null || true
}
trap cleanup EXIT
declare -a concurrencies=(1 5 10 20 40 80)
deployment_commit="$(git -C "$repo_root" rev-parse HEAD)"
if [[ -n "$(git -C "$repo_root" status --short --untracked-files=no)" ]]; then
deployment_dirty=true
else
deployment_dirty=false
fi
project_name="$(sed -n 's/^BACKEND_COMPOSE_PROJECT_NAME=//p' "$env_file" | head -n 1)"
project_name="${project_name:-ai-soulmate-backend}"
backend_container_id="$(docker compose --project-name "$project_name" --env-file "$env_file" -f "${repo_root}/compose/docker-compose.backend.yml" ps -q backend)"
if [[ -z "$backend_container_id" ]]; then
echo "Running backend container was not found." >&2
exit 1
fi
source_commit="$(docker inspect --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}' "$backend_container_id")"
backend_image="$(sed -n 's/^BACKEND_IMAGE=//p' "$env_file" | head -n 1)"
backend_image_id="$(docker inspect --format '{{.Image}}' "$backend_container_id")"
backend_container_started_at="$(docker inspect --format '{{.State.StartedAt}}' "$backend_container_id")"
bash "${repo_root}/scripts/verify-tripo-mock.sh" "$env_file"
bash "${repo_root}/scripts/verify-infra.sh" "$env_file" >/dev/null
python3 "${repo_root}/benchmark/prepare-local-benchmark-session.py" \
--base-url http://127.0.0.1:3001 \
--headers-output "$headers_file" \
--metadata-output "${output_root}/session-metadata.json" \
--deployment-commit "$deployment_commit" \
--deployment-dirty "$deployment_dirty" \
--source-commit "$source_commit" \
--backend-image "$backend_image" \
--backend-image-id "$backend_image_id" \
--backend-container-started-at "$backend_container_started_at" \
--requests-per-level "$request_count" \
--repeat-count "$repeat_count" \
--concurrencies "$(IFS=,; echo "${concurrencies[*]}")"
chmod 600 "$headers_file"
printf '%s\n' setup > "$phase_file"
bash "${repo_root}/scripts/sample-benchmark-resources.sh" \
"$sampler_stop" 1 "$phase_file" \
> "${output_root}/resources-during-load.csv" &
sampler_pid=$!
bash "${repo_root}/scripts/capture-benchmark-snapshot.sh" "$env_file" \
> "${output_root}/snapshot-before.txt"
declare -a names=(workflow-validate user-profile projects-list address-list device-list)
declare -a methods=(POST GET GET GET GET)
declare -a paths=(
/api/workflow/validate
/api/user/profile
/api/projects
/api/address
/api/device
)
for repeat in $(seq 1 "$repeat_count"); do
for index in "${!names[@]}"; do
name="${names[$index]}"
method="${methods[$index]}"
url="http://127.0.0.1:3001${paths[$index]}"
for concurrency in "${concurrencies[@]}"; do
phase="${name}-c${concurrency}-r${repeat}"
printf '%s\n' "$phase" > "$phase_file"
command=(
python3 "${repo_root}/benchmark/run-http-benchmark.py"
--url "$url"
--method "$method"
--requests "$request_count"
--concurrency "$concurrency"
--warmup 5
--timeout 10
--expected-status 200
--headers-file "$headers_file"
--allow-failures
--output "${output_root}/${phase}.json"
)
if [[ "$method" == POST ]]; then
command+=(--body-file "${repo_root}/benchmark/workflow-validate-body.json")
fi
"${command[@]}" > /dev/null
echo "Completed ${phase}."
sleep 1
done
done
done
printf '%s\n' snapshot-after > "$phase_file"
bash "${repo_root}/scripts/capture-benchmark-snapshot.sh" "$env_file" \
> "${output_root}/snapshot-after.txt"
touch "$sampler_stop"
if ! wait "$sampler_pid"; then
echo "Resource sampler failed during the benchmark." >&2
exit 1
fi
unset sampler_pid
expected_results="$(( ${#names[@]} * ${#concurrencies[@]} * repeat_count ))"
actual_results="$(find "$output_root" -maxdepth 1 -type f -name '*-c*-r*.json' | wc -l)"
if [[ "$actual_results" -ne "$expected_results" ]]; then
echo "Expected ${expected_results} benchmark result files, found ${actual_results}." >&2
exit 1
fi
if ! awk -F, 'NR > 1 && $2 != "" { found = 1 } END { exit(found ? 0 : 1) }' \
"${output_root}/resources-during-load.csv"; then
echo "Resource sampler did not capture any phase-tagged rows." >&2
exit 1
fi
python3 "${repo_root}/benchmark/summarize-business-results.py" "$output_root" \
--markdown-output "${output_root}/summary.md" \
--json-output "${output_root}/summary.json" \
--require-concurrency-one-success
echo "Authenticated business benchmark completed: $output_root"