169 lines
6.2 KiB
Bash
Executable File
169 lines
6.2 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -Eeuo pipefail
|
|
|
|
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
env_file="${1:-${repo_root}/compose/.env}"
|
|
run_id="${2:-business-baseline-$(date +%Y%m%d-%H%M%S)}"
|
|
request_count="${BUSINESS_BENCHMARK_REQUESTS_PER_LEVEL:-500}"
|
|
repeat_count="${BUSINESS_BENCHMARK_REPEATS:-3}"
|
|
output_root="${repo_root}/results/raw/${run_id}"
|
|
temporary_root=""
|
|
|
|
if [[ ! -f "$env_file" ]]; then
|
|
echo "Environment file not found: $env_file" >&2
|
|
exit 1
|
|
fi
|
|
if [[ ! "$run_id" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]{0,119}$ || "$run_id" == *..* ]]; then
|
|
echo "Run ID must be 1-120 safe filename characters and must not contain '..'." >&2
|
|
exit 1
|
|
fi
|
|
if ! [[ "$request_count" =~ ^[1-9][0-9]*$ ]] || (( request_count > 100000 )); then
|
|
echo "BUSINESS_BENCHMARK_REQUESTS_PER_LEVEL must be between 1 and 100000." >&2
|
|
exit 1
|
|
fi
|
|
if ! [[ "$repeat_count" =~ ^[1-9][0-9]*$ ]] || (( repeat_count > 10 )); then
|
|
echo "BUSINESS_BENCHMARK_REPEATS must be between 1 and 10." >&2
|
|
exit 1
|
|
fi
|
|
|
|
mkdir -p "$output_root"
|
|
runtime_root="/dev/shm"
|
|
if [[ ! -d "$runtime_root" || ! -w "$runtime_root" || "$(stat -f -c %T "$runtime_root")" != "tmpfs" ]]; then
|
|
echo "Writable tmpfs runtime directory is unavailable: $runtime_root" >&2
|
|
exit 1
|
|
fi
|
|
temporary_root="$(mktemp -d "${runtime_root}/ai-soulmate-business-benchmark.XXXXXX")"
|
|
headers_file="${temporary_root}/headers.json"
|
|
phase_file="${temporary_root}/phase.txt"
|
|
sampler_stop="${temporary_root}/sampler-stop"
|
|
|
|
cleanup() {
|
|
if [[ -n "${sampler_pid:-}" ]]; then
|
|
touch "$sampler_stop"
|
|
wait "$sampler_pid" 2>/dev/null || true
|
|
fi
|
|
if [[ -r "$headers_file" ]]; then
|
|
python3 "${repo_root}/benchmark/close-local-benchmark-session.py" \
|
|
--base-url http://127.0.0.1:3001 \
|
|
--headers-file "$headers_file" >/dev/null 2>&1 || true
|
|
fi
|
|
rm -f "$headers_file" "$phase_file" "$sampler_stop"
|
|
rmdir "$temporary_root" 2>/dev/null || true
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
declare -a concurrencies=(1 5 10 20 40 80)
|
|
deployment_commit="$(git -C "$repo_root" rev-parse HEAD)"
|
|
if [[ -n "$(git -C "$repo_root" status --short --untracked-files=no)" ]]; then
|
|
deployment_dirty=true
|
|
else
|
|
deployment_dirty=false
|
|
fi
|
|
project_name="$(sed -n 's/^BACKEND_COMPOSE_PROJECT_NAME=//p' "$env_file" | head -n 1)"
|
|
project_name="${project_name:-ai-soulmate-backend}"
|
|
backend_container_id="$(docker compose --project-name "$project_name" --env-file "$env_file" -f "${repo_root}/compose/docker-compose.backend.yml" ps -q backend)"
|
|
if [[ -z "$backend_container_id" ]]; then
|
|
echo "Running backend container was not found." >&2
|
|
exit 1
|
|
fi
|
|
source_commit="$(docker inspect --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}' "$backend_container_id")"
|
|
backend_image="$(sed -n 's/^BACKEND_IMAGE=//p' "$env_file" | head -n 1)"
|
|
backend_image_id="$(docker inspect --format '{{.Image}}' "$backend_container_id")"
|
|
backend_container_started_at="$(docker inspect --format '{{.State.StartedAt}}' "$backend_container_id")"
|
|
|
|
bash "${repo_root}/scripts/verify-tripo-mock.sh" "$env_file"
|
|
bash "${repo_root}/scripts/verify-infra.sh" "$env_file" >/dev/null
|
|
python3 "${repo_root}/benchmark/prepare-local-benchmark-session.py" \
|
|
--base-url http://127.0.0.1:3001 \
|
|
--headers-output "$headers_file" \
|
|
--metadata-output "${output_root}/session-metadata.json" \
|
|
--deployment-commit "$deployment_commit" \
|
|
--deployment-dirty "$deployment_dirty" \
|
|
--source-commit "$source_commit" \
|
|
--backend-image "$backend_image" \
|
|
--backend-image-id "$backend_image_id" \
|
|
--backend-container-started-at "$backend_container_started_at" \
|
|
--requests-per-level "$request_count" \
|
|
--repeat-count "$repeat_count" \
|
|
--concurrencies "$(IFS=,; echo "${concurrencies[*]}")"
|
|
chmod 600 "$headers_file"
|
|
|
|
printf '%s\n' setup > "$phase_file"
|
|
bash "${repo_root}/scripts/sample-benchmark-resources.sh" \
|
|
"$sampler_stop" 1 "$phase_file" \
|
|
> "${output_root}/resources-during-load.csv" &
|
|
sampler_pid=$!
|
|
|
|
bash "${repo_root}/scripts/capture-benchmark-snapshot.sh" "$env_file" \
|
|
> "${output_root}/snapshot-before.txt"
|
|
|
|
declare -a names=(workflow-validate user-profile projects-list address-list device-list)
|
|
declare -a methods=(POST GET GET GET GET)
|
|
declare -a paths=(
|
|
/api/workflow/validate
|
|
/api/user/profile
|
|
/api/projects
|
|
/api/address
|
|
/api/device
|
|
)
|
|
for repeat in $(seq 1 "$repeat_count"); do
|
|
for index in "${!names[@]}"; do
|
|
name="${names[$index]}"
|
|
method="${methods[$index]}"
|
|
url="http://127.0.0.1:3001${paths[$index]}"
|
|
for concurrency in "${concurrencies[@]}"; do
|
|
phase="${name}-c${concurrency}-r${repeat}"
|
|
printf '%s\n' "$phase" > "$phase_file"
|
|
command=(
|
|
python3 "${repo_root}/benchmark/run-http-benchmark.py"
|
|
--url "$url"
|
|
--method "$method"
|
|
--requests "$request_count"
|
|
--concurrency "$concurrency"
|
|
--warmup 5
|
|
--timeout 10
|
|
--expected-status 200
|
|
--headers-file "$headers_file"
|
|
--allow-failures
|
|
--output "${output_root}/${phase}.json"
|
|
)
|
|
if [[ "$method" == POST ]]; then
|
|
command+=(--body-file "${repo_root}/benchmark/workflow-validate-body.json")
|
|
fi
|
|
"${command[@]}" > /dev/null
|
|
echo "Completed ${phase}."
|
|
sleep 1
|
|
done
|
|
done
|
|
done
|
|
|
|
printf '%s\n' snapshot-after > "$phase_file"
|
|
bash "${repo_root}/scripts/capture-benchmark-snapshot.sh" "$env_file" \
|
|
> "${output_root}/snapshot-after.txt"
|
|
|
|
touch "$sampler_stop"
|
|
if ! wait "$sampler_pid"; then
|
|
echo "Resource sampler failed during the benchmark." >&2
|
|
exit 1
|
|
fi
|
|
unset sampler_pid
|
|
|
|
expected_results="$(( ${#names[@]} * ${#concurrencies[@]} * repeat_count ))"
|
|
actual_results="$(find "$output_root" -maxdepth 1 -type f -name '*-c*-r*.json' | wc -l)"
|
|
if [[ "$actual_results" -ne "$expected_results" ]]; then
|
|
echo "Expected ${expected_results} benchmark result files, found ${actual_results}." >&2
|
|
exit 1
|
|
fi
|
|
if ! awk -F, 'NR > 1 && $2 != "" { found = 1 } END { exit(found ? 0 : 1) }' \
|
|
"${output_root}/resources-during-load.csv"; then
|
|
echo "Resource sampler did not capture any phase-tagged rows." >&2
|
|
exit 1
|
|
fi
|
|
|
|
python3 "${repo_root}/benchmark/summarize-business-results.py" "$output_root" \
|
|
--markdown-output "${output_root}/summary.md" \
|
|
--json-output "${output_root}/summary.json" \
|
|
--require-concurrency-one-success
|
|
|
|
echo "Authenticated business benchmark completed: $output_root"
|