first commit
This commit is contained in:
Beyhan Oğur
497
tests/e2e/api/runners/individual/run-newman-anthropic-integration.sh
Executable file
497
tests/e2e/api/runners/individual/run-newman-anthropic-integration.sh
Executable file
@@ -0,0 +1,497 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Bifrost Anthropic Integration API Newman Test Runner
|
||||
# This script runs the Anthropic integration API test suite using Newman
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
API_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
cd "$API_DIR"
|
||||
|
||||
# Configuration
|
||||
COLLECTION="collections/bifrost-anthropic-integration.postman_collection.json"
|
||||
ENVIRONMENT="bifrost-v1.postman_environment.json"
|
||||
REPORT_DIR="newman-reports/anthropic-integration"
|
||||
PROVIDER_CONFIG_DIR="provider_config"
|
||||
|
||||
# Colors for output
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Detect if --env was passed (so we run single provider vs all providers)
|
||||
PROVIDER_ENV_FILE=""
|
||||
ARGS=()
|
||||
while [[ $# -gt 0 ]]; do
|
||||
if [[ "$1" == "--env" ]]; then
|
||||
if [[ -z "${2:-}" || "${2:-}" == --* ]]; then
|
||||
echo -e "${RED}Error: --env requires a value${NC}"
|
||||
exit 1
|
||||
fi
|
||||
PROVIDER_ENV_FILE="$2"
|
||||
shift 2
|
||||
else
|
||||
ARGS+=("$1")
|
||||
shift
|
||||
fi
|
||||
done
|
||||
set -- "${ARGS[@]}"
|
||||
|
||||
# Normalize CI for retry logic (accept 1 or true, case-insensitive)
|
||||
ci_normalized="$(printf '%s' "${CI:-}" | tr '[:upper:]' '[:lower:]')"
|
||||
|
||||
# Print banner
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
if [ "$ci_normalized" = "1" ] || [ "$ci_normalized" = "true" ]; then
|
||||
echo -e "${GREEN}Bifrost Anthropic Integration API Test Runner with retries: 10${NC}"
|
||||
else
|
||||
echo -e "${GREEN}Bifrost Anthropic Integration API Test Runner${NC}"
|
||||
fi
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
echo ""
|
||||
|
||||
# Check if Newman is installed
|
||||
if ! command -v newman &> /dev/null; then
|
||||
echo -e "${RED}Error: Newman is not installed${NC}"
|
||||
echo "Install it with: npm install -g newman"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if collection exists
|
||||
if [ ! -f "$COLLECTION" ]; then
|
||||
echo -e "${RED}Error: Collection file not found: $COLLECTION${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if environment exists
|
||||
if [ ! -f "$ENVIRONMENT" ]; then
|
||||
echo -e "${YELLOW}Warning: Environment file not found: $ENVIRONMENT${NC}"
|
||||
echo "Using collection variables only"
|
||||
ENV_FLAG=""
|
||||
else
|
||||
ENV_FLAG="-e $ENVIRONMENT"
|
||||
fi
|
||||
|
||||
# Create report directory
|
||||
mkdir -p "$REPORT_DIR"
|
||||
|
||||
# When no --env: resolve list of provider Postman env .json files (sorted), excluding sgl and ollama
|
||||
EXCLUDED_PROVIDERS="sgl ollama"
|
||||
if [ -z "$PROVIDER_ENV_FILE" ] && [ -d "$PROVIDER_CONFIG_DIR" ]; then
|
||||
PROVIDER_JSON_FILES=()
|
||||
while IFS= read -r -d '' f; do
|
||||
# basename: bifrost-v1-openai.postman_environment.json -> openai
|
||||
name="${f##*/}"
|
||||
name="${name#bifrost-v1-}"
|
||||
name="${name%.postman_environment.json}"
|
||||
skip=""
|
||||
for ex in $EXCLUDED_PROVIDERS; do
|
||||
if [ "$name" = "$ex" ]; then skip=1; break; fi
|
||||
done
|
||||
[ -z "$skip" ] && PROVIDER_JSON_FILES+=("$f")
|
||||
done < <(find "$PROVIDER_CONFIG_DIR" -maxdepth 1 -name "bifrost-v1-*.postman_environment.json" -print0 2>/dev/null | sort -z)
|
||||
fi
|
||||
|
||||
# Parse command line arguments
|
||||
FOLDER=""
|
||||
VERBOSE="--verbose" # Enable verbose by default to capture console.log statements
|
||||
REPORTERS="cli"
|
||||
BAIL=""
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--folder)
|
||||
if [[ -z "${2:-}" ]]; then
|
||||
echo -e "${RED}Error: --folder requires a value${NC}"
|
||||
exit 1
|
||||
fi
|
||||
FOLDER="--folder \"$2\""
|
||||
shift 2
|
||||
;;
|
||||
--verbose)
|
||||
VERBOSE="--verbose"
|
||||
shift
|
||||
;;
|
||||
--html)
|
||||
REPORTERS="cli,html"
|
||||
shift
|
||||
;;
|
||||
--json)
|
||||
REPORTERS="cli,json"
|
||||
shift
|
||||
;;
|
||||
--all-reports)
|
||||
REPORTERS="cli,html,json"
|
||||
shift
|
||||
;;
|
||||
--bail)
|
||||
BAIL="--bail"
|
||||
shift
|
||||
;;
|
||||
--help)
|
||||
echo "Usage: $0 [OPTIONS]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --folder <name> Run only tests in specified folder"
|
||||
echo " --verbose Show detailed output"
|
||||
echo " --html Generate HTML report"
|
||||
echo " --json Generate JSON report"
|
||||
echo " --all-reports Generate all report types"
|
||||
echo " --bail Stop on first failure"
|
||||
echo " --env <path> Postman env .json path or provider name (e.g. provider_config/bifrost-v1-openai.postman_environment.json or openai)"
|
||||
echo " --help Show this help message"
|
||||
echo ""
|
||||
echo "Environment Variables:"
|
||||
echo " BIFROST_BASE_URL Override base URL (default: http://localhost:8080)"
|
||||
echo " BIFROST_PROVIDER Override provider (default: openai)"
|
||||
echo " BIFROST_MODEL Override model name (default: gpt-4o)"
|
||||
echo " BIFROST_EMBEDDING_MODEL Override embedding model (default: text-embedding-3-small)"
|
||||
echo " BIFROST_SPEECH_MODEL Override speech model (default: tts-1)"
|
||||
echo " BIFROST_TRANSCRIPTION_MODEL Override transcription model (default: whisper-1)"
|
||||
echo " BIFROST_IMAGE_MODEL Override image model (default: dall-e-3)"
|
||||
echo ""
|
||||
echo "Examples:"
|
||||
echo " $0 # Run collection for all providers (each provider_config/bifrost-v1-*.postman_environment.json)"
|
||||
echo " $0 --env openai # Run once with OpenAI provider only"
|
||||
echo " $0 --folder \"Chat Completions\" # Run specific folder"
|
||||
echo " $0 --html --verbose # Verbose with HTML report"
|
||||
echo " BIFROST_BASE_URL=http://api:8080 $0 # Custom base URL"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo -e "${RED}Unknown option: $1${NC}"
|
||||
echo "Use --help for usage information"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Build and run Newman once.
|
||||
# Optional second arg: path to Postman env .json file (e.g. provider_config/bifrost-v1-openai.postman_environment.json).
|
||||
# When given, uses only that env file; otherwise uses default env and BIFROST_* overrides.
|
||||
run_newman() {
|
||||
local -a cmd=(newman run "$COLLECTION")
|
||||
if [ -n "${2:-}" ] && [ -f "${2}" ]; then
|
||||
cmd+=(-e "${2}")
|
||||
else
|
||||
local base_url="${BIFROST_BASE_URL:-http://localhost:8080}"
|
||||
local provider="${BIFROST_PROVIDER:-openai}"
|
||||
local model="${BIFROST_MODEL:-gpt-4o}"
|
||||
local embedding_model="${BIFROST_EMBEDDING_MODEL:-text-embedding-3-small}"
|
||||
local speech_model="${BIFROST_SPEECH_MODEL:-tts-1}"
|
||||
local transcription_model="${BIFROST_TRANSCRIPTION_MODEL:-whisper-1}"
|
||||
local image_model="${BIFROST_IMAGE_MODEL:-dall-e-3}"
|
||||
if [ -n "$ENV_FLAG" ]; then
|
||||
cmd+=(-e "$ENVIRONMENT")
|
||||
fi
|
||||
cmd+=(--env-var "base_url=$base_url" --env-var "provider=$provider" --env-var "model=$model" --env-var "embedding_model=$embedding_model" --env-var "speech_model=$speech_model" --env-var "transcription_model=$transcription_model" --env-var "image_model=$image_model")
|
||||
fi
|
||||
[ -n "$FOLDER" ] && cmd+=(--folder "$FOLDER")
|
||||
cmd+=(--timeout-script 120000 --timeout 900000)
|
||||
cmd+=(-r "$REPORTERS")
|
||||
if [[ "$REPORTERS" == *"html"* ]]; then
|
||||
cmd+=(--reporter-html-export "$REPORT_DIR/report_${1:-run}.html")
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"json"* ]]; then
|
||||
cmd+=(--reporter-json-export "$REPORT_DIR/report_${1:-run}.json")
|
||||
fi
|
||||
[ -n "$VERBOSE" ] && cmd+=("$VERBOSE")
|
||||
[ -n "$BAIL" ] && cmd+=("$BAIL")
|
||||
if [ "$ci_normalized" = "1" ] || [ "$ci_normalized" = "true" ]; then
|
||||
cmd+=(--env-var "CI=1")
|
||||
fi
|
||||
|
||||
"${cmd[@]}"
|
||||
}
|
||||
|
||||
# Post-process log file to pretty-print JSON blocks using jq
|
||||
post_process_log() {
|
||||
local input_file="$1"
|
||||
local output_file="$2"
|
||||
|
||||
if [ ! -f "$input_file" ]; then
|
||||
return 1
|
||||
fi
|
||||
if ! command -v python3 >/dev/null 2>&1; then
|
||||
cp "$input_file" "$output_file"
|
||||
return 0
|
||||
fi
|
||||
|
||||
python3 - "$input_file" "$output_file" << 'PYTHON_SCRIPT'
|
||||
import sys
|
||||
import json
|
||||
import subprocess
|
||||
import shutil
|
||||
|
||||
def format_json_with_jq(json_text):
|
||||
"""Format JSON using jq if available, otherwise use Python's json module"""
|
||||
if shutil.which('jq'):
|
||||
try:
|
||||
process = subprocess.Popen(
|
||||
['jq', '.'],
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True
|
||||
)
|
||||
stdout, stderr = process.communicate(input=json_text)
|
||||
if process.returncode == 0:
|
||||
return stdout
|
||||
except Exception:
|
||||
pass
|
||||
# Fallback to Python's json module
|
||||
try:
|
||||
parsed = json.loads(json_text)
|
||||
return json.dumps(parsed, indent=2)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
return json_text
|
||||
|
||||
def process_log_file(input_file, output_file):
|
||||
"""Process log file and format JSON blocks"""
|
||||
with open(input_file, 'r', encoding='utf-8', errors='ignore') as f_in:
|
||||
with open(output_file, 'w', encoding='utf-8') as f_out:
|
||||
in_json_block = False
|
||||
json_lines = []
|
||||
|
||||
for line in f_in:
|
||||
# Check if we're entering a JSON block
|
||||
if 'REQUEST BODY:' in line or 'RESPONSE BODY:' in line:
|
||||
if in_json_block and json_lines:
|
||||
# Format previous JSON block
|
||||
json_text = ''.join(json_lines).strip()
|
||||
formatted = format_json_with_jq(json_text)
|
||||
f_out.write(formatted)
|
||||
if not formatted.endswith('\n'):
|
||||
f_out.write('\n')
|
||||
json_lines = []
|
||||
in_json_block = True
|
||||
f_out.write(line)
|
||||
continue
|
||||
|
||||
# Check if we're exiting a JSON block
|
||||
if in_json_block:
|
||||
stripped = line.strip()
|
||||
if not stripped or stripped.startswith('=') or line.startswith('REQUEST:') or line.startswith('RESPONSE:'):
|
||||
# End of JSON block, format and write
|
||||
if json_lines:
|
||||
json_text = ''.join(json_lines).strip()
|
||||
formatted = format_json_with_jq(json_text)
|
||||
f_out.write(formatted)
|
||||
if not formatted.endswith('\n'):
|
||||
f_out.write('\n')
|
||||
json_lines = []
|
||||
in_json_block = False
|
||||
else:
|
||||
json_lines.append(line)
|
||||
continue
|
||||
|
||||
f_out.write(line)
|
||||
|
||||
# Handle case where file ends in a JSON block
|
||||
if json_lines:
|
||||
json_text = ''.join(json_lines).strip()
|
||||
formatted = format_json_with_jq(json_text)
|
||||
f_out.write(formatted)
|
||||
if not formatted.endswith('\n'):
|
||||
f_out.write('\n')
|
||||
|
||||
if __name__ == '__main__':
|
||||
if len(sys.argv) < 3:
|
||||
print(f"Error: Expected 2 arguments, got {len(sys.argv) - 1}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
try:
|
||||
process_log_file(sys.argv[1], sys.argv[2])
|
||||
except Exception as e:
|
||||
print(f"Error processing log file: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
PYTHON_SCRIPT
|
||||
}
|
||||
|
||||
# Run for a single provider (--env was passed: path to .json env or provider name)
|
||||
if [ -n "$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV=""
|
||||
if [ -f "$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json"
|
||||
fi
|
||||
if [ -z "$SINGLE_JSON_ENV" ]; then
|
||||
echo -e "${RED}Error: Env file not found: $PROVIDER_ENV_FILE${NC}"
|
||||
echo "Use a path to a .json env (e.g. provider_config/bifrost-v1-openai.postman_environment.json) or provider name (e.g. openai)"
|
||||
exit 1
|
||||
fi
|
||||
SINGLE_PROVIDER_NAME="${SINGLE_JSON_ENV##*/}"
|
||||
SINGLE_PROVIDER_NAME="${SINGLE_PROVIDER_NAME#bifrost-v1-}"
|
||||
SINGLE_PROVIDER_NAME="${SINGLE_PROVIDER_NAME%.postman_environment.json}"
|
||||
echo -e "Configuration: ${YELLOW}$SINGLE_JSON_ENV${NC}"
|
||||
echo -e " Reports: ${YELLOW}$REPORT_DIR${NC}"
|
||||
echo ""
|
||||
echo -e "${GREEN}Running tests...${NC}"
|
||||
echo ""
|
||||
set +e
|
||||
TEMP_LOG="$REPORT_DIR/${SINGLE_PROVIDER_NAME}.log.tmp"
|
||||
run_newman "$SINGLE_PROVIDER_NAME" "$SINGLE_JSON_ENV" > "$TEMP_LOG" 2>&1
|
||||
EXIT_CODE=$?
|
||||
set -e
|
||||
LOG_FILE="$REPORT_DIR/${SINGLE_PROVIDER_NAME}.log"
|
||||
post_process_log "$TEMP_LOG" "$LOG_FILE"
|
||||
rm -f "$TEMP_LOG"
|
||||
echo ""
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All tests passed!${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Some tests failed${NC}"
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"html"* ]] || [[ "$REPORTERS" == *"json"* ]]; then
|
||||
echo ""
|
||||
echo -e "Reports saved to: ${YELLOW}$REPORT_DIR${NC}"
|
||||
ls -lh "$REPORT_DIR" 2>/dev/null | tail -n +2
|
||||
fi
|
||||
exit $EXIT_CODE
|
||||
fi
|
||||
|
||||
# Run for all providers (no --env)
|
||||
if [ -z "${PROVIDER_JSON_FILES+x}" ] || [ ${#PROVIDER_JSON_FILES[@]} -eq 0 ]; then
|
||||
echo -e "${YELLOW}No provider env .json files found in $PROVIDER_CONFIG_DIR/. Using default (openai).${NC}"
|
||||
echo -e "Configuration:"
|
||||
echo -e " Base URL: ${YELLOW}${BIFROST_BASE_URL:-http://localhost:8080}${NC}"
|
||||
echo -e " Provider: ${YELLOW}${BIFROST_PROVIDER:-openai}${NC}"
|
||||
echo -e " Reports: ${YELLOW}$REPORT_DIR${NC}"
|
||||
echo ""
|
||||
echo -e "${GREEN}Running tests...${NC}"
|
||||
echo ""
|
||||
set +e
|
||||
TEMP_LOG="$REPORT_DIR/default.log.tmp"
|
||||
run_newman > "$TEMP_LOG" 2>&1
|
||||
EXIT_CODE=$?
|
||||
set -e
|
||||
LOG_FILE="$REPORT_DIR/default.log"
|
||||
post_process_log "$TEMP_LOG" "$LOG_FILE"
|
||||
rm -f "$TEMP_LOG"
|
||||
echo ""
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All tests passed!${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Some tests failed${NC}"
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"html"* ]] || [[ "$REPORTERS" == *"json"* ]]; then
|
||||
echo ""
|
||||
echo -e "Reports saved to: ${YELLOW}$REPORT_DIR${NC}"
|
||||
ls -lh "$REPORT_DIR" 2>/dev/null | tail -n +2
|
||||
fi
|
||||
exit $EXIT_CODE
|
||||
fi
|
||||
|
||||
PARALLEL_LOGS_DIR="$REPORT_DIR/parallel_logs"
|
||||
mkdir -p "$PARALLEL_LOGS_DIR"
|
||||
|
||||
# Print a one-line report for a provider from its Newman log and exit code
|
||||
print_provider_report() {
|
||||
local name="$1"
|
||||
local logfile="$2"
|
||||
local exitcode="$3"
|
||||
local failed_count=""
|
||||
local failed_tests=""
|
||||
if [ -f "$logfile" ]; then
|
||||
# Parse Newman summary table: assertions row, third column = failed count
|
||||
failed_count=$(grep "assertions" "$logfile" 2>/dev/null | awk -F'│' '{gsub(/^ *| *$/,"",$4); print $4}' | head -1)
|
||||
# Lines with " ✗ " are failed assertions; strip to get test name
|
||||
failed_tests=$(grep " ✗ " "$logfile" 2>/dev/null | sed 's/.*✗ */ - /' | sed 's/^ *//' | tr '\n' ' ' | sed 's/ $//')
|
||||
fi
|
||||
if [ "$exitcode" -eq 0 ]; then
|
||||
echo -e "${GREEN} ✓ $name: PASS${NC}"
|
||||
else
|
||||
echo -e "${RED} ✗ $name: FAIL${NC}"
|
||||
if [ -n "$failed_count" ] && [ "$failed_count" -gt 0 ] 2>/dev/null; then
|
||||
echo -e " ${RED}${failed_count} assertion(s) failed${NC}"
|
||||
fi
|
||||
if [ -n "$failed_tests" ]; then
|
||||
echo -e " Failed: $failed_tests"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
# Draw the provider status table (TABLE_LINES lines). Use after moving cursor up TABLE_LINES to refresh.
|
||||
draw_table() {
|
||||
printf '\033[2K%-16s %s\n' "Provider" "Status"
|
||||
for i in "${!NAMES[@]}"; do
|
||||
printf '\033[2K%-16s %b\n' "${NAMES[$i]}" "${STATUS[$i]}"
|
||||
done
|
||||
}
|
||||
|
||||
echo -e "Running tests for ${#PROVIDER_JSON_FILES[@]} provider(s) ${GREEN}in parallel${NC}. Reports: ${YELLOW}$REPORT_DIR${NC}"
|
||||
echo ""
|
||||
|
||||
# Run each provider in a background subshell; capture PID and log path per provider
|
||||
PIDS=()
|
||||
NAMES=()
|
||||
LOG_FILES=()
|
||||
for jsonfile in "${PROVIDER_JSON_FILES[@]}"; do
|
||||
name="${jsonfile##*/}"
|
||||
name="${name#bifrost-v1-}"
|
||||
name="${name%.postman_environment.json}"
|
||||
logfile="$PARALLEL_LOGS_DIR/${name}.log"
|
||||
temp_logfile="${logfile}.tmp"
|
||||
LOG_FILES+=("$logfile")
|
||||
NAMES+=("$name")
|
||||
( set +e; run_newman "$name" "$jsonfile" > "$temp_logfile" 2>&1; ec=$?; set -e; post_process_log "$temp_logfile" "$logfile"; rm -f "$temp_logfile"; exit $ec ) &
|
||||
PIDS+=($!)
|
||||
done
|
||||
|
||||
# Status for each provider: Pending, ✓ PASS, or ✗ FAIL (with color)
|
||||
STATUS=()
|
||||
for i in "${!PIDS[@]}"; do STATUS[$i]="${YELLOW}Pending${NC}"; done
|
||||
TABLE_LINES=$((${#NAMES[@]} + 1))
|
||||
|
||||
# Initial table
|
||||
draw_table
|
||||
|
||||
# Track which we've reaped (0 = pending, 1 = done)
|
||||
REAPED=()
|
||||
for i in "${!PIDS[@]}"; do REAPED[$i]=0; done
|
||||
|
||||
OVERALL_FAILED=0
|
||||
FAILED_NAMES=()
|
||||
|
||||
# As each provider finishes, update status and redraw table
|
||||
while true; do
|
||||
all_done=1
|
||||
for i in "${!PIDS[@]}"; do
|
||||
[ "${REAPED[$i]:-0}" -eq 1 ] && continue
|
||||
all_done=0
|
||||
if ! kill -0 "${PIDS[$i]}" 2>/dev/null; then
|
||||
exitcode=0; wait "${PIDS[$i]}" || exitcode=$?
|
||||
REAPED[$i]=1
|
||||
if [ "$exitcode" -eq 0 ]; then
|
||||
STATUS[$i]="${GREEN}✓ PASS${NC}"
|
||||
else
|
||||
OVERALL_FAILED=1
|
||||
FAILED_NAMES+=("${NAMES[$i]}")
|
||||
STATUS[$i]="${RED}✗ FAIL${NC}"
|
||||
fi
|
||||
# Move cursor up and redraw table
|
||||
printf '\033[%dA' "$TABLE_LINES"
|
||||
draw_table
|
||||
fi
|
||||
done
|
||||
[ "$all_done" -eq 1 ] && break
|
||||
sleep 0.3
|
||||
done
|
||||
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
if [ $OVERALL_FAILED -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All providers passed!${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ One or more providers had failures: ${FAILED_NAMES[*]}${NC}"
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"html"* ]] || [[ "$REPORTERS" == *"json"* ]]; then
|
||||
echo ""
|
||||
echo -e "Reports saved to: ${YELLOW}$REPORT_DIR${NC}"
|
||||
ls -lh "$REPORT_DIR" 2>/dev/null | tail -n +2
|
||||
fi
|
||||
# Parallel logs persist in $PARALLEL_LOGS_DIR (overwritten per provider on each run)
|
||||
exit $OVERALL_FAILED
|
||||
161
tests/e2e/api/runners/individual/run-newman-async-tests.sh
Executable file
161
tests/e2e/api/runners/individual/run-newman-async-tests.sh
Executable file
@@ -0,0 +1,161 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Bifrost V1 Async Inference Newman Test Runner
|
||||
# Runs async submit/poll tests. Requires LogsStore and governance plugin.
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
API_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
cd "$API_DIR"
|
||||
|
||||
COLLECTION="collections/bifrost-v1-async.postman_collection.json"
|
||||
REPORT_DIR="newman-reports/async"
|
||||
PROVIDER_CONFIG_DIR="provider_config"
|
||||
PROVIDER_CAPABILITIES_JSON="provider-capabilities.json"
|
||||
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
PROVIDER_ENV_FILE=""
|
||||
ARGS=()
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--env)
|
||||
if [[ -z "${2:-}" || "${2:-}" == --* ]]; then
|
||||
echo -e "${RED}Error: --env requires a value${NC}"
|
||||
exit 1
|
||||
fi
|
||||
PROVIDER_ENV_FILE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--help)
|
||||
echo "Usage: $0 [OPTIONS]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --env <provider> Postman env path or provider name"
|
||||
echo " --verbose Show detailed output"
|
||||
echo " --html Generate HTML report"
|
||||
echo " --json Generate JSON report"
|
||||
echo " --bail Stop on first failure"
|
||||
echo " --help Show this help message"
|
||||
echo ""
|
||||
echo "Prerequisites: LogsStore and governance plugin must be configured."
|
||||
echo "Environment Variables:"
|
||||
echo " BIFROST_BASE_URL Override base URL (default: http://localhost:8080)"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
ARGS+=("$1")
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
set -- "${ARGS[@]}"
|
||||
|
||||
echo -e "${GREEN}==============================================${NC}"
|
||||
echo -e "${GREEN}Bifrost V1 Async Inference Test Runner${NC}"
|
||||
echo -e "${GREEN}==============================================${NC}"
|
||||
echo ""
|
||||
|
||||
if ! command -v newman &> /dev/null; then
|
||||
echo -e "${RED}Error: Newman is not installed${NC}"
|
||||
echo "Install it with: npm install -g newman"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -f "$COLLECTION" ]; then
|
||||
echo -e "${RED}Error: Collection file not found: $COLLECTION${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$REPORT_DIR"
|
||||
|
||||
GLOBALS_TMP=""
|
||||
if [ -f "$PROVIDER_CAPABILITIES_JSON" ] && command -v jq &>/dev/null; then
|
||||
GLOBALS_TMP=$(mktemp)
|
||||
trap 'rm -f "$GLOBALS_TMP"' EXIT
|
||||
jq -n --rawfile cap "$PROVIDER_CAPABILITIES_JSON" '{id: "bifrost-provider-capabilities", name: "Provider capabilities", values: [{key: "provider_capabilities", value: $cap, type: "default", enabled: true}]}' > "$GLOBALS_TMP"
|
||||
fi
|
||||
|
||||
VERBOSE=""
|
||||
REPORTERS="cli"
|
||||
BAIL=""
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--verbose) VERBOSE="--verbose"; shift ;;
|
||||
--html) REPORTERS="${REPORTERS},html"; shift ;;
|
||||
--json) REPORTERS="${REPORTERS},json"; shift ;;
|
||||
--bail) BAIL="--bail"; shift ;;
|
||||
*) echo -e "${RED}Unknown option: $1${NC}"; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
SINGLE_JSON_ENV=""
|
||||
if [ -n "$PROVIDER_ENV_FILE" ]; then
|
||||
if [ -f "$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json"
|
||||
else
|
||||
echo -e "${RED}Error: Could not find environment file for: $PROVIDER_ENV_FILE${NC}"
|
||||
echo "Searched:"
|
||||
echo " - $PROVIDER_ENV_FILE"
|
||||
echo " - $PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE"
|
||||
echo " - $PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -z "$SINGLE_JSON_ENV" ]; then
|
||||
if [ -f "$PROVIDER_CONFIG_DIR/bifrost-v1-openai.postman_environment.json" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/bifrost-v1-openai.postman_environment.json"
|
||||
echo -e "${YELLOW}No --env specified, using openai${NC}"
|
||||
fi
|
||||
fi
|
||||
|
||||
cmd=(newman run "$COLLECTION")
|
||||
[ -n "$GLOBALS_TMP" ] && [ -f "$GLOBALS_TMP" ] && cmd+=(-g "$GLOBALS_TMP")
|
||||
[ -n "$SINGLE_JSON_ENV" ] && [ -f "$SINGLE_JSON_ENV" ] && cmd+=(-e "$SINGLE_JSON_ENV")
|
||||
base_url="${BIFROST_BASE_URL:-http://localhost:8080}"
|
||||
cmd+=(--env-var "base_url=$base_url")
|
||||
cmd+=(--timeout-script 120000 --timeout 900000)
|
||||
cmd+=(-r "$REPORTERS")
|
||||
[[ "$REPORTERS" == *"html"* ]] && cmd+=(--reporter-html-export "$REPORT_DIR/report.html")
|
||||
[[ "$REPORTERS" == *"json"* ]] && cmd+=(--reporter-json-export "$REPORT_DIR/report.json")
|
||||
[ -n "$VERBOSE" ] && cmd+=("$VERBOSE")
|
||||
[ -n "$BAIL" ] && cmd+=("$BAIL")
|
||||
|
||||
echo -e "Configuration:"
|
||||
echo -e " Collection: ${YELLOW}$COLLECTION${NC}"
|
||||
echo -e " Base URL: ${YELLOW}$base_url${NC}"
|
||||
if [ -n "$SINGLE_JSON_ENV" ]; then
|
||||
echo -e " Env: ${YELLOW}$SINGLE_JSON_ENV${NC}"
|
||||
fi
|
||||
echo -e " Reports: ${YELLOW}$REPORT_DIR${NC}"
|
||||
echo ""
|
||||
echo -e "${GREEN}Running tests...${NC}"
|
||||
echo ""
|
||||
|
||||
set +e
|
||||
"${cmd[@]}"
|
||||
EXIT_CODE=$?
|
||||
set -e
|
||||
|
||||
echo ""
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All async tests passed!${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Some tests failed${NC}"
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"html"* ]] || [[ "$REPORTERS" == *"json"* ]]; then
|
||||
echo ""
|
||||
echo -e "Reports saved to: ${YELLOW}$REPORT_DIR${NC}"
|
||||
ls -lh "$REPORT_DIR" 2>/dev/null | tail -n +2
|
||||
fi
|
||||
|
||||
exit $EXIT_CODE
|
||||
511
tests/e2e/api/runners/individual/run-newman-bedrock-integration.sh
Executable file
511
tests/e2e/api/runners/individual/run-newman-bedrock-integration.sh
Executable file
@@ -0,0 +1,511 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Bifrost Bedrock Integration API Newman Test Runner
|
||||
# This script runs the Bedrock integration API test suite using Newman
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
API_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
cd "$API_DIR"
|
||||
|
||||
# Configuration
|
||||
COLLECTION="collections/bifrost-bedrock-integration.postman_collection.json"
|
||||
ENVIRONMENT="bifrost-v1.postman_environment.json"
|
||||
REPORT_DIR="newman-reports/bedrock-integration"
|
||||
PROVIDER_CONFIG_DIR="provider_config"
|
||||
|
||||
# Colors for output
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Detect if --env was passed (so we run single provider vs all providers)
|
||||
PROVIDER_ENV_FILE=""
|
||||
ARGS=()
|
||||
while [[ $# -gt 0 ]]; do
|
||||
if [[ "$1" == "--env" ]]; then
|
||||
if [[ -z "${2:-}" || "${2:-}" == --* ]]; then
|
||||
echo -e "${RED}Error: --env requires a value${NC}"
|
||||
exit 1
|
||||
fi
|
||||
PROVIDER_ENV_FILE="$2"
|
||||
shift 2
|
||||
else
|
||||
ARGS+=("$1")
|
||||
shift
|
||||
fi
|
||||
done
|
||||
set -- "${ARGS[@]}"
|
||||
|
||||
# Normalize CI for retry logic (accept 1 or true, case-insensitive)
|
||||
ci_normalized="$(printf '%s' "${CI:-}" | tr '[:upper:]' '[:lower:]')"
|
||||
|
||||
# Print banner
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
if [ "$ci_normalized" = "1" ] || [ "$ci_normalized" = "true" ]; then
|
||||
echo -e "${GREEN}Bifrost Bedrock Integration API Test Runner with retries: 10${NC}"
|
||||
else
|
||||
echo -e "${GREEN}Bifrost Bedrock Integration API Test Runner${NC}"
|
||||
fi
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
echo ""
|
||||
|
||||
# Check if Newman is installed
|
||||
if ! command -v newman &> /dev/null; then
|
||||
echo -e "${RED}Error: Newman is not installed${NC}"
|
||||
echo "Install it with: npm install -g newman"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if collection exists
|
||||
if [ ! -f "$COLLECTION" ]; then
|
||||
echo -e "${RED}Error: Collection file not found: $COLLECTION${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if environment exists
|
||||
USE_DEFAULT_ENV=0
|
||||
if [ ! -f "$ENVIRONMENT" ]; then
|
||||
echo -e "${YELLOW}Warning: Environment file not found: $ENVIRONMENT${NC}"
|
||||
echo "Using collection variables only"
|
||||
else
|
||||
USE_DEFAULT_ENV=1
|
||||
fi
|
||||
|
||||
# Create report directory
|
||||
mkdir -p "$REPORT_DIR"
|
||||
|
||||
# When no --env: resolve list of provider Postman env .json files (sorted), excluding sgl and ollama
|
||||
EXCLUDED_PROVIDERS="sgl ollama"
|
||||
if [ -z "$PROVIDER_ENV_FILE" ] && [ -d "$PROVIDER_CONFIG_DIR" ]; then
|
||||
PROVIDER_JSON_FILES=()
|
||||
while IFS= read -r -d '' f; do
|
||||
# basename: bifrost-v1-openai.postman_environment.json -> openai
|
||||
name="${f##*/}"
|
||||
name="${name#bifrost-v1-}"
|
||||
name="${name%.postman_environment.json}"
|
||||
skip=""
|
||||
for ex in $EXCLUDED_PROVIDERS; do
|
||||
if [ "$name" = "$ex" ]; then skip=1; break; fi
|
||||
done
|
||||
[ -z "$skip" ] && PROVIDER_JSON_FILES+=("$f")
|
||||
done < <(find "$PROVIDER_CONFIG_DIR" -maxdepth 1 -name "bifrost-v1-*.postman_environment.json" -print0 2>/dev/null | sort -z)
|
||||
fi
|
||||
|
||||
# Parse command line arguments
|
||||
FOLDER=""
|
||||
VERBOSE="--verbose" # Enable verbose by default to capture console.log statements
|
||||
REPORTERS="cli"
|
||||
BAIL=""
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--folder)
|
||||
if [[ -z "${2:-}" || "${2:-}" == --* ]]; then
|
||||
echo -e "${RED}Error: --folder requires a value${NC}"
|
||||
exit 1
|
||||
fi
|
||||
FOLDER="$2"
|
||||
shift 2
|
||||
;;
|
||||
--verbose)
|
||||
VERBOSE="--verbose"
|
||||
shift
|
||||
;;
|
||||
--html)
|
||||
REPORTERS="cli,html"
|
||||
shift
|
||||
;;
|
||||
--json)
|
||||
REPORTERS="cli,json"
|
||||
shift
|
||||
;;
|
||||
--all-reports)
|
||||
REPORTERS="cli,html,json"
|
||||
shift
|
||||
;;
|
||||
--bail)
|
||||
BAIL="--bail"
|
||||
shift
|
||||
;;
|
||||
--help)
|
||||
echo "Usage: $0 [OPTIONS]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --folder <name> Run only tests in specified folder"
|
||||
echo " --verbose Show detailed output"
|
||||
echo " --html Generate HTML report"
|
||||
echo " --json Generate JSON report"
|
||||
echo " --all-reports Generate all report types"
|
||||
echo " --bail Stop on first failure"
|
||||
echo " --env <path> Postman env .json path or provider name (e.g. provider_config/bifrost-v1-openai.postman_environment.json or openai)"
|
||||
echo " --help Show this help message"
|
||||
echo ""
|
||||
echo "Environment Variables:"
|
||||
echo " BIFROST_BASE_URL Override base URL (default: http://localhost:8080)"
|
||||
echo " BIFROST_PROVIDER Override provider (default: openai)"
|
||||
echo " BIFROST_MODEL Override model name (default: gpt-4o)"
|
||||
echo " BIFROST_MODEL_INVOKE Override model for Bedrock invoke/invoke-stream (default: amazon.titan-text-express-v1)"
|
||||
echo " BIFROST_EMBEDDING_MODEL Override embedding model (default: text-embedding-3-small)"
|
||||
echo " BIFROST_SPEECH_MODEL Override speech model (default: tts-1)"
|
||||
echo " BIFROST_TRANSCRIPTION_MODEL Override transcription model (default: whisper-1)"
|
||||
echo " BIFROST_IMAGE_MODEL Override image model (default: dall-e-3)"
|
||||
echo " BIFROST_BEDROCK_API_KEY Bedrock API key (when --env bedrock)"
|
||||
echo " BIFROST_BEDROCK_ACCESS_KEY Bedrock AWS access key (when --env bedrock)"
|
||||
echo " BIFROST_BEDROCK_SECRET_KEY Bedrock AWS secret key (when --env bedrock)"
|
||||
echo " BIFROST_BEDROCK_REGION Bedrock region (default: us-east-1)"
|
||||
echo ""
|
||||
echo "Examples:"
|
||||
echo " $0 # Run collection for all providers (each provider_config/bifrost-v1-*.postman_environment.json)"
|
||||
echo " $0 --env openai # Run once with OpenAI provider only"
|
||||
echo " $0 --folder \"Chat Completions\" # Run specific folder"
|
||||
echo " $0 --html --verbose # Verbose with HTML report"
|
||||
echo " BIFROST_BASE_URL=http://api:8080 $0 # Custom base URL"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo -e "${RED}Unknown option: $1${NC}"
|
||||
echo "Use --help for usage information"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Build and run Newman once.
|
||||
# Optional second arg: path to Postman env .json file (e.g. provider_config/bifrost-v1-openai.postman_environment.json).
|
||||
# When given, uses only that env file; otherwise uses default env and BIFROST_* overrides.
|
||||
run_newman() {
|
||||
local -a cmd=(newman run "$COLLECTION")
|
||||
if [ -n "${2:-}" ] && [ -f "${2}" ]; then
|
||||
cmd+=(-e "${2}")
|
||||
# Pass Bedrock credentials from env when using bedrock provider
|
||||
if [[ "${1:-}" == "bedrock" ]]; then
|
||||
[ -n "${BIFROST_BEDROCK_API_KEY:-}" ] && cmd+=(--env-var "bedrock_api_key=$BIFROST_BEDROCK_API_KEY")
|
||||
[ -n "${BIFROST_BEDROCK_ACCESS_KEY:-}" ] && cmd+=(--env-var "bedrock_access_key=$BIFROST_BEDROCK_ACCESS_KEY")
|
||||
[ -n "${BIFROST_BEDROCK_SECRET_KEY:-}" ] && cmd+=(--env-var "bedrock_secret_key=$BIFROST_BEDROCK_SECRET_KEY")
|
||||
[ -n "${BIFROST_BEDROCK_REGION:-}" ] && cmd+=(--env-var "bedrock_region=$BIFROST_BEDROCK_REGION")
|
||||
[ -n "${BIFROST_BEDROCK_SESSION_TOKEN:-}" ] && cmd+=(--env-var "bedrock_session_token=$BIFROST_BEDROCK_SESSION_TOKEN")
|
||||
fi
|
||||
else
|
||||
local base_url="${BIFROST_BASE_URL:-http://localhost:8080}"
|
||||
local provider="${BIFROST_PROVIDER:-openai}"
|
||||
local model="${BIFROST_MODEL:-gpt-4o}"
|
||||
local model_invoke="${BIFROST_MODEL_INVOKE:-amazon.titan-text-express-v1}"
|
||||
local embedding_model="${BIFROST_EMBEDDING_MODEL:-text-embedding-3-small}"
|
||||
local speech_model="${BIFROST_SPEECH_MODEL:-tts-1}"
|
||||
local transcription_model="${BIFROST_TRANSCRIPTION_MODEL:-whisper-1}"
|
||||
local image_model="${BIFROST_IMAGE_MODEL:-dall-e-3}"
|
||||
if [ "${USE_DEFAULT_ENV:-0}" -eq 1 ]; then
|
||||
cmd+=(-e "$ENVIRONMENT")
|
||||
fi
|
||||
cmd+=(--env-var "base_url=$base_url" --env-var "provider=$provider" --env-var "model=$model" --env-var "model_invoke=$model_invoke" --env-var "embedding_model=$embedding_model" --env-var "speech_model=$speech_model" --env-var "transcription_model=$transcription_model" --env-var "image_model=$image_model")
|
||||
fi
|
||||
[ -n "$FOLDER" ] && cmd+=(--folder "$FOLDER")
|
||||
cmd+=(--timeout-script 120000 --timeout 900000)
|
||||
cmd+=(-r "$REPORTERS")
|
||||
if [[ "$REPORTERS" == *"html"* ]]; then
|
||||
cmd+=(--reporter-html-export "$REPORT_DIR/report_${1:-run}.html")
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"json"* ]]; then
|
||||
cmd+=(--reporter-json-export "$REPORT_DIR/report_${1:-run}.json")
|
||||
fi
|
||||
[ -n "$VERBOSE" ] && cmd+=("$VERBOSE")
|
||||
[ -n "$BAIL" ] && cmd+=("$BAIL")
|
||||
if [ "$ci_normalized" = "1" ] || [ "$ci_normalized" = "true" ]; then
|
||||
cmd+=(--env-var "CI=1")
|
||||
fi
|
||||
|
||||
"${cmd[@]}"
|
||||
}
|
||||
|
||||
# Post-process log file to pretty-print JSON blocks using jq
|
||||
post_process_log() {
|
||||
local input_file="$1"
|
||||
local output_file="$2"
|
||||
|
||||
if [ ! -f "$input_file" ]; then
|
||||
return 1
|
||||
fi
|
||||
if ! command -v python3 >/dev/null 2>&1; then
|
||||
cp "$input_file" "$output_file"
|
||||
return 0
|
||||
fi
|
||||
|
||||
python3 - "$input_file" "$output_file" << 'PYTHON_SCRIPT'
|
||||
import sys
|
||||
import json
|
||||
import subprocess
|
||||
import shutil
|
||||
|
||||
def format_json_with_jq(json_text):
|
||||
"""Format JSON using jq if available, otherwise use Python's json module"""
|
||||
if shutil.which('jq'):
|
||||
try:
|
||||
process = subprocess.Popen(
|
||||
['jq', '.'],
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True
|
||||
)
|
||||
stdout, stderr = process.communicate(input=json_text)
|
||||
if process.returncode == 0:
|
||||
return stdout
|
||||
except Exception:
|
||||
pass
|
||||
# Fallback to Python's json module
|
||||
try:
|
||||
parsed = json.loads(json_text)
|
||||
return json.dumps(parsed, indent=2)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
return json_text
|
||||
|
||||
def process_log_file(input_file, output_file):
|
||||
"""Process log file and format JSON blocks"""
|
||||
with open(input_file, 'r', encoding='utf-8', errors='ignore') as f_in:
|
||||
with open(output_file, 'w', encoding='utf-8') as f_out:
|
||||
in_json_block = False
|
||||
json_lines = []
|
||||
|
||||
for line in f_in:
|
||||
# Check if we're entering a JSON block
|
||||
if 'REQUEST BODY:' in line or 'RESPONSE BODY:' in line:
|
||||
if in_json_block and json_lines:
|
||||
# Format previous JSON block
|
||||
json_text = ''.join(json_lines).strip()
|
||||
formatted = format_json_with_jq(json_text)
|
||||
f_out.write(formatted)
|
||||
if not formatted.endswith('\n'):
|
||||
f_out.write('\n')
|
||||
json_lines = []
|
||||
in_json_block = True
|
||||
f_out.write(line)
|
||||
continue
|
||||
|
||||
# Check if we're exiting a JSON block
|
||||
if in_json_block:
|
||||
stripped = line.strip()
|
||||
if not stripped or stripped.startswith('=') or line.startswith('REQUEST:') or line.startswith('RESPONSE:'):
|
||||
# End of JSON block, format and write
|
||||
if json_lines:
|
||||
json_text = ''.join(json_lines).strip()
|
||||
formatted = format_json_with_jq(json_text)
|
||||
f_out.write(formatted)
|
||||
if not formatted.endswith('\n'):
|
||||
f_out.write('\n')
|
||||
json_lines = []
|
||||
in_json_block = False
|
||||
else:
|
||||
json_lines.append(line)
|
||||
continue
|
||||
|
||||
f_out.write(line)
|
||||
|
||||
# Handle case where file ends in a JSON block
|
||||
if json_lines:
|
||||
json_text = ''.join(json_lines).strip()
|
||||
formatted = format_json_with_jq(json_text)
|
||||
f_out.write(formatted)
|
||||
if not formatted.endswith('\n'):
|
||||
f_out.write('\n')
|
||||
|
||||
if __name__ == '__main__':
|
||||
if len(sys.argv) < 3:
|
||||
print(f"Error: Expected 2 arguments, got {len(sys.argv) - 1}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
try:
|
||||
process_log_file(sys.argv[1], sys.argv[2])
|
||||
except Exception as e:
|
||||
print(f"Error processing log file: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
PYTHON_SCRIPT
|
||||
}
|
||||
|
||||
# Run for a single provider (--env was passed: path to .json env or provider name)
|
||||
if [ -n "$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV=""
|
||||
if [ -f "$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json"
|
||||
fi
|
||||
if [ -z "$SINGLE_JSON_ENV" ]; then
|
||||
echo -e "${RED}Error: Env file not found: $PROVIDER_ENV_FILE${NC}"
|
||||
echo "Use a path to a .json env (e.g. provider_config/bifrost-v1-openai.postman_environment.json) or provider name (e.g. openai)"
|
||||
exit 1
|
||||
fi
|
||||
SINGLE_PROVIDER_NAME="${SINGLE_JSON_ENV##*/}"
|
||||
SINGLE_PROVIDER_NAME="${SINGLE_PROVIDER_NAME#bifrost-v1-}"
|
||||
SINGLE_PROVIDER_NAME="${SINGLE_PROVIDER_NAME%.postman_environment.json}"
|
||||
echo -e "Configuration: ${YELLOW}$SINGLE_JSON_ENV${NC}"
|
||||
echo -e " Reports: ${YELLOW}$REPORT_DIR${NC}"
|
||||
echo ""
|
||||
echo -e "${GREEN}Running tests...${NC}"
|
||||
echo ""
|
||||
TEMP_LOG="$REPORT_DIR/${SINGLE_PROVIDER_NAME}.log.tmp"
|
||||
set +e
|
||||
run_newman "$SINGLE_PROVIDER_NAME" "$SINGLE_JSON_ENV" > "$TEMP_LOG" 2>&1
|
||||
EXIT_CODE=$?
|
||||
set -e
|
||||
LOG_FILE="$REPORT_DIR/${SINGLE_PROVIDER_NAME}.log"
|
||||
post_process_log "$TEMP_LOG" "$LOG_FILE"
|
||||
rm -f "$TEMP_LOG"
|
||||
echo ""
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All tests passed!${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Some tests failed${NC}"
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"html"* ]] || [[ "$REPORTERS" == *"json"* ]]; then
|
||||
echo ""
|
||||
echo -e "Reports saved to: ${YELLOW}$REPORT_DIR${NC}"
|
||||
ls -lh "$REPORT_DIR" 2>/dev/null | tail -n +2
|
||||
fi
|
||||
exit $EXIT_CODE
|
||||
fi
|
||||
|
||||
# Run for all providers (no --env)
|
||||
if [ -z "${PROVIDER_JSON_FILES+x}" ] || [ ${#PROVIDER_JSON_FILES[@]} -eq 0 ]; then
|
||||
echo -e "${YELLOW}No provider env .json files found in $PROVIDER_CONFIG_DIR/. Using default (openai).${NC}"
|
||||
echo -e "Configuration:"
|
||||
echo -e " Base URL: ${YELLOW}${BIFROST_BASE_URL:-http://localhost:8080}${NC}"
|
||||
echo -e " Provider: ${YELLOW}${BIFROST_PROVIDER:-openai}${NC}"
|
||||
echo -e " Reports: ${YELLOW}$REPORT_DIR${NC}"
|
||||
echo ""
|
||||
echo -e "${GREEN}Running tests...${NC}"
|
||||
echo ""
|
||||
TEMP_LOG="$REPORT_DIR/default.log.tmp"
|
||||
set +e
|
||||
run_newman > "$TEMP_LOG" 2>&1
|
||||
EXIT_CODE=$?
|
||||
set -e
|
||||
LOG_FILE="$REPORT_DIR/default.log"
|
||||
post_process_log "$TEMP_LOG" "$LOG_FILE"
|
||||
rm -f "$TEMP_LOG"
|
||||
echo ""
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All tests passed!${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Some tests failed${NC}"
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"html"* ]] || [[ "$REPORTERS" == *"json"* ]]; then
|
||||
echo ""
|
||||
echo -e "Reports saved to: ${YELLOW}$REPORT_DIR${NC}"
|
||||
ls -lh "$REPORT_DIR" 2>/dev/null | tail -n +2
|
||||
fi
|
||||
exit $EXIT_CODE
|
||||
fi
|
||||
|
||||
PARALLEL_LOGS_DIR="$REPORT_DIR/parallel_logs"
|
||||
mkdir -p "$PARALLEL_LOGS_DIR"
|
||||
|
||||
# Print a one-line report for a provider from its Newman log and exit code
|
||||
print_provider_report() {
|
||||
local name="$1"
|
||||
local logfile="$2"
|
||||
local exitcode="$3"
|
||||
local failed_count=""
|
||||
local failed_tests=""
|
||||
if [ -f "$logfile" ]; then
|
||||
# Parse Newman summary table: assertions row, third column = failed count
|
||||
failed_count=$(grep "assertions" "$logfile" 2>/dev/null | awk -F'│' '{gsub(/^ *| *$/,"",$4); print $4}' | head -1)
|
||||
# Lines with " ✗ " are failed assertions; strip to get test name
|
||||
failed_tests=$(grep " ✗ " "$logfile" 2>/dev/null | sed 's/.*✗ */ - /' | sed 's/^ *//' | tr '\n' ' ' | sed 's/ $//')
|
||||
fi
|
||||
if [ "$exitcode" -eq 0 ]; then
|
||||
echo -e "${GREEN} ✓ $name: PASS${NC}"
|
||||
else
|
||||
echo -e "${RED} ✗ $name: FAIL${NC}"
|
||||
if [ -n "$failed_count" ] && [ "$failed_count" -gt 0 ] 2>/dev/null; then
|
||||
echo -e " ${RED}${failed_count} assertion(s) failed${NC}"
|
||||
fi
|
||||
if [ -n "$failed_tests" ]; then
|
||||
echo -e " Failed: $failed_tests"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
# Draw the provider status table (TABLE_LINES lines). Use after moving cursor up TABLE_LINES to refresh.
|
||||
draw_table() {
|
||||
printf '\033[2K%-16s %s\n' "Provider" "Status"
|
||||
for i in "${!NAMES[@]}"; do
|
||||
printf '\033[2K%-16s %b\n' "${NAMES[$i]}" "${STATUS[$i]}"
|
||||
done
|
||||
}
|
||||
|
||||
echo -e "Running tests for ${#PROVIDER_JSON_FILES[@]} provider(s) ${GREEN}in parallel${NC}. Reports: ${YELLOW}$REPORT_DIR${NC}"
|
||||
echo ""
|
||||
|
||||
# Run each provider in a background subshell; capture PID and log path per provider
|
||||
PIDS=()
|
||||
NAMES=()
|
||||
LOG_FILES=()
|
||||
for jsonfile in "${PROVIDER_JSON_FILES[@]}"; do
|
||||
name="${jsonfile##*/}"
|
||||
name="${name#bifrost-v1-}"
|
||||
name="${name%.postman_environment.json}"
|
||||
logfile="$PARALLEL_LOGS_DIR/${name}.log"
|
||||
temp_logfile="${logfile}.tmp"
|
||||
LOG_FILES+=("$logfile")
|
||||
NAMES+=("$name")
|
||||
( set +e; run_newman "$name" "$jsonfile" > "$temp_logfile" 2>&1; rc=$?; set -e; post_process_log "$temp_logfile" "$logfile" || cp "$temp_logfile" "$logfile"; rm -f "$temp_logfile"; exit $rc ) &
|
||||
PIDS+=($!)
|
||||
done
|
||||
|
||||
# Status for each provider: Pending, ✓ PASS, or ✗ FAIL (with color)
|
||||
STATUS=()
|
||||
for i in "${!PIDS[@]}"; do STATUS[$i]="${YELLOW}Pending${NC}"; done
|
||||
TABLE_LINES=$((${#NAMES[@]} + 1))
|
||||
|
||||
# Initial table
|
||||
draw_table
|
||||
|
||||
# Track which we've reaped (0 = pending, 1 = done)
|
||||
REAPED=()
|
||||
for i in "${!PIDS[@]}"; do REAPED[$i]=0; done
|
||||
|
||||
OVERALL_FAILED=0
|
||||
FAILED_NAMES=()
|
||||
|
||||
# As each provider finishes, update status and redraw table
|
||||
while true; do
|
||||
all_done=1
|
||||
for i in "${!PIDS[@]}"; do
|
||||
[ "${REAPED[$i]:-0}" -eq 1 ] && continue
|
||||
all_done=0
|
||||
if ! kill -0 "${PIDS[$i]}" 2>/dev/null; then
|
||||
exitcode=0; wait "${PIDS[$i]}" || exitcode=$?
|
||||
REAPED[$i]=1
|
||||
if [ "$exitcode" -eq 0 ]; then
|
||||
STATUS[$i]="${GREEN}✓ PASS${NC}"
|
||||
else
|
||||
OVERALL_FAILED=1
|
||||
FAILED_NAMES+=("${NAMES[$i]}")
|
||||
STATUS[$i]="${RED}✗ FAIL${NC}"
|
||||
fi
|
||||
# Move cursor up and redraw table
|
||||
printf '\033[%dA' "$TABLE_LINES"
|
||||
draw_table
|
||||
fi
|
||||
done
|
||||
[ "$all_done" -eq 1 ] && break
|
||||
sleep 0.3
|
||||
done
|
||||
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
if [ $OVERALL_FAILED -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All providers passed!${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ One or more providers had failures: ${FAILED_NAMES[*]}${NC}"
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"html"* ]] || [[ "$REPORTERS" == *"json"* ]]; then
|
||||
echo ""
|
||||
echo -e "Reports saved to: ${YELLOW}$REPORT_DIR${NC}"
|
||||
ls -lh "$REPORT_DIR" 2>/dev/null | tail -n +2
|
||||
fi
|
||||
# Parallel logs persist in $PARALLEL_LOGS_DIR (overwritten per provider on each run)
|
||||
exit $OVERALL_FAILED
|
||||
497
tests/e2e/api/runners/individual/run-newman-composite-integration.sh
Executable file
497
tests/e2e/api/runners/individual/run-newman-composite-integration.sh
Executable file
@@ -0,0 +1,497 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Bifrost Composite Integrations API Newman Test Runner
|
||||
# This script runs the Composite Integrations API test suite using Newman
|
||||
|
||||
set -e
|
||||
|
||||
# Run from script directory so paths to collection and provider-config work
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
API_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
cd "$API_DIR"
|
||||
|
||||
# Configuration
|
||||
COLLECTION="collections/bifrost-composite-integrations.postman_collection.json"
|
||||
ENVIRONMENT="bifrost-v1.postman_environment.json"
|
||||
REPORT_DIR="newman-reports/composite-integration"
|
||||
PROVIDER_CONFIG_DIR="provider_config"
|
||||
|
||||
# Colors for output
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Detect if --env was passed (so we run single provider vs all providers)
|
||||
PROVIDER_ENV_FILE=""
|
||||
ARGS=()
|
||||
while [[ $# -gt 0 ]]; do
|
||||
if [[ "$1" == "--env" ]]; then
|
||||
if [[ -z "${2:-}" || "${2:-}" == --* ]]; then
|
||||
echo -e "${RED}Error: --env requires a value${NC}"
|
||||
exit 1
|
||||
fi
|
||||
PROVIDER_ENV_FILE="$2"
|
||||
shift 2
|
||||
else
|
||||
ARGS+=("$1")
|
||||
shift
|
||||
fi
|
||||
done
|
||||
set -- "${ARGS[@]}"
|
||||
|
||||
# Normalize CI for retry logic (accept 1 or true, case-insensitive)
|
||||
ci_normalized="$(printf '%s' "${CI:-}" | tr '[:upper:]' '[:lower:]')"
|
||||
|
||||
# Print banner
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
if [ "$ci_normalized" = "1" ] || [ "$ci_normalized" = "true" ]; then
|
||||
echo -e "${GREEN}Bifrost Composite Integrations API Test Runner with retries: 10${NC}"
|
||||
else
|
||||
echo -e "${GREEN}Bifrost Composite Integrations API Test Runner${NC}"
|
||||
fi
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
echo ""
|
||||
|
||||
# Check if Newman is installed
|
||||
if ! command -v newman &> /dev/null; then
|
||||
echo -e "${RED}Error: Newman is not installed${NC}"
|
||||
echo "Install it with: npm install -g newman"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if collection exists
|
||||
if [ ! -f "$COLLECTION" ]; then
|
||||
echo -e "${RED}Error: Collection file not found: $COLLECTION${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if environment exists
|
||||
if [ ! -f "$ENVIRONMENT" ]; then
|
||||
echo -e "${YELLOW}Warning: Environment file not found: $ENVIRONMENT${NC}"
|
||||
echo "Using collection variables only"
|
||||
ENV_FLAG=""
|
||||
else
|
||||
ENV_FLAG="-e $ENVIRONMENT"
|
||||
fi
|
||||
|
||||
# Create report directory
|
||||
mkdir -p "$REPORT_DIR"
|
||||
|
||||
# When no --env: resolve list of provider Postman env .json files (sorted), excluding sgl and ollama
|
||||
EXCLUDED_PROVIDERS="sgl ollama"
|
||||
if [ -z "$PROVIDER_ENV_FILE" ] && [ -d "$PROVIDER_CONFIG_DIR" ]; then
|
||||
PROVIDER_JSON_FILES=()
|
||||
while IFS= read -r -d '' f; do
|
||||
# basename: bifrost-v1-openai.postman_environment.json -> openai
|
||||
name="${f##*/}"
|
||||
name="${name#bifrost-v1-}"
|
||||
name="${name%.postman_environment.json}"
|
||||
skip=""
|
||||
for ex in $EXCLUDED_PROVIDERS; do
|
||||
if [ "$name" = "$ex" ]; then skip=1; break; fi
|
||||
done
|
||||
[ -z "$skip" ] && PROVIDER_JSON_FILES+=("$f")
|
||||
done < <(find "$PROVIDER_CONFIG_DIR" -maxdepth 1 -name "bifrost-v1-*.postman_environment.json" -print0 2>/dev/null | sort -z)
|
||||
fi
|
||||
|
||||
# Parse command line arguments
|
||||
FOLDER=""
|
||||
VERBOSE="--verbose" # Enable verbose by default to capture console.log statements
|
||||
REPORTERS="cli"
|
||||
BAIL=""
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--folder)
|
||||
if [[ -z "${2:-}" || "${2:-}" == --* ]]; then
|
||||
echo -e "${RED}Error: --folder requires a value${NC}"
|
||||
exit 1
|
||||
fi
|
||||
FOLDER="--folder \"$2\""
|
||||
shift 2
|
||||
;;
|
||||
--verbose)
|
||||
VERBOSE="--verbose"
|
||||
shift
|
||||
;;
|
||||
--html)
|
||||
REPORTERS="cli,html"
|
||||
shift
|
||||
;;
|
||||
--json)
|
||||
REPORTERS="cli,json"
|
||||
shift
|
||||
;;
|
||||
--all-reports)
|
||||
REPORTERS="cli,html,json"
|
||||
shift
|
||||
;;
|
||||
--bail)
|
||||
BAIL="--bail"
|
||||
shift
|
||||
;;
|
||||
--help)
|
||||
echo "Usage: $0 [OPTIONS]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --folder <name> Run only tests in specified folder"
|
||||
echo " --verbose Show detailed output"
|
||||
echo " --html Generate HTML report"
|
||||
echo " --json Generate JSON report"
|
||||
echo " --all-reports Generate all report types"
|
||||
echo " --bail Stop on first failure"
|
||||
echo " --env <path> Postman env .json path or provider name (e.g. provider_config/bifrost-v1-openai.postman_environment.json or openai)"
|
||||
echo " --help Show this help message"
|
||||
echo ""
|
||||
echo "Environment Variables:"
|
||||
echo " BIFROST_BASE_URL Override base URL (default: http://localhost:8080)"
|
||||
echo " BIFROST_PROVIDER Override provider (default: openai)"
|
||||
echo " BIFROST_MODEL Override model name (default: gpt-4o)"
|
||||
echo " BIFROST_EMBEDDING_MODEL Override embedding model (default: text-embedding-3-small)"
|
||||
echo " BIFROST_SPEECH_MODEL Override speech model (default: tts-1)"
|
||||
echo " BIFROST_TRANSCRIPTION_MODEL Override transcription model (default: whisper-1)"
|
||||
echo " BIFROST_IMAGE_MODEL Override image model (default: dall-e-3)"
|
||||
echo ""
|
||||
echo "Examples:"
|
||||
echo " $0 # Run collection for all providers (each provider_config/bifrost-v1-*.postman_environment.json)"
|
||||
echo " $0 --env openai # Run once with OpenAI provider only"
|
||||
echo " $0 --folder \"Chat Completions\" # Run specific folder"
|
||||
echo " $0 --html --verbose # Verbose with HTML report"
|
||||
echo " BIFROST_BASE_URL=http://api:8080 $0 # Custom base URL"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo -e "${RED}Unknown option: $1${NC}"
|
||||
echo "Use --help for usage information"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Build and run Newman once.
|
||||
# Optional second arg: path to Postman env .json file (e.g. provider_config/bifrost-v1-openai.postman_environment.json).
|
||||
# When given, uses only that env file; otherwise uses default env and BIFROST_* overrides.
|
||||
run_newman() {
|
||||
local cmd="newman run $COLLECTION"
|
||||
if [ -n "${2:-}" ] && [ -f "${2}" ]; then
|
||||
cmd="$cmd -e ${2}"
|
||||
else
|
||||
local base_url="${BIFROST_BASE_URL:-http://localhost:8080}"
|
||||
local provider="${BIFROST_PROVIDER:-openai}"
|
||||
local model="${BIFROST_MODEL:-gpt-4o}"
|
||||
local embedding_model="${BIFROST_EMBEDDING_MODEL:-text-embedding-3-small}"
|
||||
local speech_model="${BIFROST_SPEECH_MODEL:-tts-1}"
|
||||
local transcription_model="${BIFROST_TRANSCRIPTION_MODEL:-whisper-1}"
|
||||
local image_model="${BIFROST_IMAGE_MODEL:-dall-e-3}"
|
||||
if [ -n "$ENV_FLAG" ]; then
|
||||
cmd="$cmd $ENV_FLAG"
|
||||
fi
|
||||
cmd="$cmd --env-var \"base_url=$base_url\" --env-var \"provider=$provider\" --env-var \"model=$model\" --env-var \"embedding_model=$embedding_model\" --env-var \"speech_model=$speech_model\" --env-var \"transcription_model=$transcription_model\" --env-var \"image_model=$image_model\""
|
||||
fi
|
||||
[ -n "$FOLDER" ] && cmd="$cmd $FOLDER"
|
||||
cmd="$cmd --timeout-script 120000 --timeout 900000 -r $REPORTERS"
|
||||
if [[ "$REPORTERS" == *"html"* ]]; then
|
||||
cmd="$cmd --reporter-html-export $REPORT_DIR/report_${1:-run}.html"
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"json"* ]]; then
|
||||
cmd="$cmd --reporter-json-export $REPORT_DIR/report_${1:-run}.json"
|
||||
fi
|
||||
[ -n "$VERBOSE" ] && cmd="$cmd $VERBOSE"
|
||||
[ -n "$BAIL" ] && cmd="$cmd $BAIL"
|
||||
if [ "$ci_normalized" = "1" ] || [ "$ci_normalized" = "true" ]; then
|
||||
cmd="$cmd --env-var \"CI=1\""
|
||||
fi
|
||||
|
||||
eval $cmd
|
||||
}
|
||||
|
||||
# Post-process log file to pretty-print JSON blocks using jq
|
||||
post_process_log() {
|
||||
local input_file="$1"
|
||||
local output_file="$2"
|
||||
|
||||
if [ ! -f "$input_file" ]; then
|
||||
return 1
|
||||
fi
|
||||
if ! command -v python3 >/dev/null 2>&1; then
|
||||
cp "$input_file" "$output_file"
|
||||
return 0
|
||||
fi
|
||||
|
||||
python3 - "$input_file" "$output_file" << 'PYTHON_SCRIPT'
|
||||
import sys
|
||||
import json
|
||||
import subprocess
|
||||
import shutil
|
||||
|
||||
def format_json_with_jq(json_text):
|
||||
"""Format JSON using jq if available, otherwise use Python's json module"""
|
||||
if shutil.which('jq'):
|
||||
try:
|
||||
process = subprocess.Popen(
|
||||
['jq', '.'],
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True
|
||||
)
|
||||
stdout, stderr = process.communicate(input=json_text)
|
||||
if process.returncode == 0:
|
||||
return stdout
|
||||
except Exception:
|
||||
pass
|
||||
# Fallback to Python's json module
|
||||
try:
|
||||
parsed = json.loads(json_text)
|
||||
return json.dumps(parsed, indent=2)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
return json_text
|
||||
|
||||
def process_log_file(input_file, output_file):
|
||||
"""Process log file and format JSON blocks"""
|
||||
with open(input_file, 'r', encoding='utf-8', errors='ignore') as f_in:
|
||||
with open(output_file, 'w', encoding='utf-8') as f_out:
|
||||
in_json_block = False
|
||||
json_lines = []
|
||||
|
||||
for line in f_in:
|
||||
# Check if we're entering a JSON block
|
||||
if 'REQUEST BODY:' in line or 'RESPONSE BODY:' in line:
|
||||
if in_json_block and json_lines:
|
||||
# Format previous JSON block
|
||||
json_text = ''.join(json_lines).strip()
|
||||
formatted = format_json_with_jq(json_text)
|
||||
f_out.write(formatted)
|
||||
if not formatted.endswith('\n'):
|
||||
f_out.write('\n')
|
||||
json_lines = []
|
||||
in_json_block = True
|
||||
f_out.write(line)
|
||||
continue
|
||||
|
||||
# Check if we're exiting a JSON block
|
||||
if in_json_block:
|
||||
stripped = line.strip()
|
||||
if not stripped or stripped.startswith('=') or line.startswith('REQUEST:') or line.startswith('RESPONSE:'):
|
||||
# End of JSON block, format and write
|
||||
if json_lines:
|
||||
json_text = ''.join(json_lines).strip()
|
||||
formatted = format_json_with_jq(json_text)
|
||||
f_out.write(formatted)
|
||||
if not formatted.endswith('\n'):
|
||||
f_out.write('\n')
|
||||
json_lines = []
|
||||
in_json_block = False
|
||||
else:
|
||||
json_lines.append(line)
|
||||
continue
|
||||
|
||||
f_out.write(line)
|
||||
|
||||
# Handle case where file ends in a JSON block
|
||||
if json_lines:
|
||||
json_text = ''.join(json_lines).strip()
|
||||
formatted = format_json_with_jq(json_text)
|
||||
f_out.write(formatted)
|
||||
if not formatted.endswith('\n'):
|
||||
f_out.write('\n')
|
||||
|
||||
if __name__ == '__main__':
|
||||
if len(sys.argv) < 3:
|
||||
print(f"Error: Expected 2 arguments, got {len(sys.argv) - 1}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
try:
|
||||
process_log_file(sys.argv[1], sys.argv[2])
|
||||
except Exception as e:
|
||||
print(f"Error processing log file: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
PYTHON_SCRIPT
|
||||
}
|
||||
|
||||
# Run for a single provider (--env was passed: path to .json env or provider name)
|
||||
if [ -n "$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV=""
|
||||
if [ -f "$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json"
|
||||
fi
|
||||
if [ -z "$SINGLE_JSON_ENV" ]; then
|
||||
echo -e "${RED}Error: Env file not found: $PROVIDER_ENV_FILE${NC}"
|
||||
echo "Use a path to a .json env (e.g. provider_config/bifrost-v1-openai.postman_environment.json) or provider name (e.g. openai)"
|
||||
exit 1
|
||||
fi
|
||||
SINGLE_PROVIDER_NAME="${SINGLE_JSON_ENV##*/}"
|
||||
SINGLE_PROVIDER_NAME="${SINGLE_PROVIDER_NAME#bifrost-v1-}"
|
||||
SINGLE_PROVIDER_NAME="${SINGLE_PROVIDER_NAME%.postman_environment.json}"
|
||||
echo -e "Configuration: ${YELLOW}$SINGLE_JSON_ENV${NC}"
|
||||
echo -e " Reports: ${YELLOW}$REPORT_DIR${NC}"
|
||||
echo ""
|
||||
echo -e "${GREEN}Running tests...${NC}"
|
||||
echo ""
|
||||
TEMP_LOG="$REPORT_DIR/${SINGLE_PROVIDER_NAME}.log.tmp"
|
||||
set +e
|
||||
run_newman "$SINGLE_PROVIDER_NAME" "$SINGLE_JSON_ENV" > "$TEMP_LOG" 2>&1
|
||||
EXIT_CODE=$?
|
||||
set -e
|
||||
LOG_FILE="$REPORT_DIR/${SINGLE_PROVIDER_NAME}.log"
|
||||
post_process_log "$TEMP_LOG" "$LOG_FILE" || cp "$TEMP_LOG" "$LOG_FILE"
|
||||
rm -f "$TEMP_LOG"
|
||||
echo ""
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All tests passed!${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Some tests failed${NC}"
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"html"* ]] || [[ "$REPORTERS" == *"json"* ]]; then
|
||||
echo ""
|
||||
echo -e "Reports saved to: ${YELLOW}$REPORT_DIR${NC}"
|
||||
ls -lh "$REPORT_DIR" 2>/dev/null | tail -n +2
|
||||
fi
|
||||
exit $EXIT_CODE
|
||||
fi
|
||||
|
||||
# Run for all providers (no --env)
|
||||
if [ -z "${PROVIDER_JSON_FILES+x}" ] || [ ${#PROVIDER_JSON_FILES[@]} -eq 0 ]; then
|
||||
echo -e "${YELLOW}No provider env .json files found in $PROVIDER_CONFIG_DIR/. Using default (openai).${NC}"
|
||||
echo -e "Configuration:"
|
||||
echo -e " Base URL: ${YELLOW}${BIFROST_BASE_URL:-http://localhost:8080}${NC}"
|
||||
echo -e " Provider: ${YELLOW}${BIFROST_PROVIDER:-openai}${NC}"
|
||||
echo -e " Reports: ${YELLOW}$REPORT_DIR${NC}"
|
||||
echo ""
|
||||
echo -e "${GREEN}Running tests...${NC}"
|
||||
echo ""
|
||||
TEMP_LOG="$REPORT_DIR/default.log.tmp"
|
||||
set +e
|
||||
run_newman > "$TEMP_LOG" 2>&1
|
||||
EXIT_CODE=$?
|
||||
set -e
|
||||
LOG_FILE="$REPORT_DIR/default.log"
|
||||
post_process_log "$TEMP_LOG" "$LOG_FILE" || cp "$TEMP_LOG" "$LOG_FILE"
|
||||
rm -f "$TEMP_LOG"
|
||||
echo ""
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All tests passed!${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Some tests failed${NC}"
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"html"* ]] || [[ "$REPORTERS" == *"json"* ]]; then
|
||||
echo ""
|
||||
echo -e "Reports saved to: ${YELLOW}$REPORT_DIR${NC}"
|
||||
ls -lh "$REPORT_DIR" 2>/dev/null | tail -n +2
|
||||
fi
|
||||
exit $EXIT_CODE
|
||||
fi
|
||||
|
||||
PARALLEL_LOGS_DIR="$REPORT_DIR/parallel_logs"
|
||||
mkdir -p "$PARALLEL_LOGS_DIR"
|
||||
|
||||
# Print a one-line report for a provider from its Newman log and exit code
|
||||
print_provider_report() {
|
||||
local name="$1"
|
||||
local logfile="$2"
|
||||
local exitcode="$3"
|
||||
local failed_count=""
|
||||
local failed_tests=""
|
||||
if [ -f "$logfile" ]; then
|
||||
# Parse Newman summary table: assertions row, third column = failed count
|
||||
failed_count=$(grep "assertions" "$logfile" 2>/dev/null | awk -F'│' '{gsub(/^ *| *$/,"",$4); print $4}' | head -1)
|
||||
# Lines with " ✗ " are failed assertions; strip to get test name
|
||||
failed_tests=$(grep " ✗ " "$logfile" 2>/dev/null | sed 's/.*✗ */ - /' | sed 's/^ *//' | tr '\n' ' ' | sed 's/ $//')
|
||||
fi
|
||||
if [ "$exitcode" -eq 0 ]; then
|
||||
echo -e "${GREEN} ✓ $name: PASS${NC}"
|
||||
else
|
||||
echo -e "${RED} ✗ $name: FAIL${NC}"
|
||||
if [ -n "$failed_count" ] && [ "$failed_count" -gt 0 ] 2>/dev/null; then
|
||||
echo -e " ${RED}${failed_count} assertion(s) failed${NC}"
|
||||
fi
|
||||
if [ -n "$failed_tests" ]; then
|
||||
echo -e " Failed: $failed_tests"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
# Draw the provider status table (TABLE_LINES lines). Use after moving cursor up TABLE_LINES to refresh.
|
||||
draw_table() {
|
||||
printf '\033[2K%-16s %s\n' "Provider" "Status"
|
||||
for i in "${!NAMES[@]}"; do
|
||||
printf '\033[2K%-16s %b\n' "${NAMES[$i]}" "${STATUS[$i]}"
|
||||
done
|
||||
}
|
||||
|
||||
echo -e "Running tests for ${#PROVIDER_JSON_FILES[@]} provider(s) ${GREEN}in parallel${NC}. Reports: ${YELLOW}$REPORT_DIR${NC}"
|
||||
echo ""
|
||||
|
||||
# Run each provider in a background subshell; capture PID and log path per provider
|
||||
PIDS=()
|
||||
NAMES=()
|
||||
LOG_FILES=()
|
||||
for jsonfile in "${PROVIDER_JSON_FILES[@]}"; do
|
||||
name="${jsonfile##*/}"
|
||||
name="${name#bifrost-v1-}"
|
||||
name="${name%.postman_environment.json}"
|
||||
logfile="$PARALLEL_LOGS_DIR/${name}.log"
|
||||
temp_logfile="${logfile}.tmp"
|
||||
LOG_FILES+=("$logfile")
|
||||
NAMES+=("$name")
|
||||
( set +e; run_newman "$name" "$jsonfile" > "$temp_logfile" 2>&1; ec=$?; set -e; post_process_log "$temp_logfile" "$logfile" || cp "$temp_logfile" "$logfile"; rm -f "$temp_logfile"; exit $ec ) &
|
||||
PIDS+=($!)
|
||||
done
|
||||
|
||||
# Status for each provider: Pending, ✓ PASS, or ✗ FAIL (with color)
|
||||
STATUS=()
|
||||
for i in "${!PIDS[@]}"; do STATUS[$i]="${YELLOW}Pending${NC}"; done
|
||||
TABLE_LINES=$((${#NAMES[@]} + 1))
|
||||
|
||||
# Initial table
|
||||
draw_table
|
||||
|
||||
# Track which we've reaped (0 = pending, 1 = done)
|
||||
REAPED=()
|
||||
for i in "${!PIDS[@]}"; do REAPED[$i]=0; done
|
||||
|
||||
OVERALL_FAILED=0
|
||||
FAILED_NAMES=()
|
||||
|
||||
# As each provider finishes, update status and redraw table
|
||||
while true; do
|
||||
all_done=1
|
||||
for i in "${!PIDS[@]}"; do
|
||||
[ "${REAPED[$i]:-0}" -eq 1 ] && continue
|
||||
all_done=0
|
||||
if ! kill -0 "${PIDS[$i]}" 2>/dev/null; then
|
||||
exitcode=0; wait "${PIDS[$i]}" || exitcode=$?
|
||||
REAPED[$i]=1
|
||||
if [ "$exitcode" -eq 0 ]; then
|
||||
STATUS[$i]="${GREEN}✓ PASS${NC}"
|
||||
else
|
||||
OVERALL_FAILED=1
|
||||
FAILED_NAMES+=("${NAMES[$i]}")
|
||||
STATUS[$i]="${RED}✗ FAIL${NC}"
|
||||
fi
|
||||
# Move cursor up and redraw table
|
||||
printf '\033[%dA' "$TABLE_LINES"
|
||||
draw_table
|
||||
fi
|
||||
done
|
||||
[ "$all_done" -eq 1 ] && break
|
||||
sleep 0.3
|
||||
done
|
||||
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
if [ $OVERALL_FAILED -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All providers passed!${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ One or more providers had failures: ${FAILED_NAMES[*]}${NC}"
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"html"* ]] || [[ "$REPORTERS" == *"json"* ]]; then
|
||||
echo ""
|
||||
echo -e "Reports saved to: ${YELLOW}$REPORT_DIR${NC}"
|
||||
ls -lh "$REPORT_DIR" 2>/dev/null | tail -n +2
|
||||
fi
|
||||
# Parallel logs persist in $PARALLEL_LOGS_DIR (overwritten per provider on each run)
|
||||
exit $OVERALL_FAILED
|
||||
160
tests/e2e/api/runners/individual/run-newman-fallbacks-tests.sh
Executable file
160
tests/e2e/api/runners/individual/run-newman-fallbacks-tests.sh
Executable file
@@ -0,0 +1,160 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Bifrost V1 Fallbacks Newman Test Runner
|
||||
# Runs fallback failover tests.
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
API_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
cd "$API_DIR"
|
||||
|
||||
COLLECTION="collections/bifrost-v1-fallbacks.postman_collection.json"
|
||||
REPORT_DIR="newman-reports/fallbacks"
|
||||
PROVIDER_CONFIG_DIR="provider_config"
|
||||
PROVIDER_CAPABILITIES_JSON="provider-capabilities.json"
|
||||
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
PROVIDER_ENV_FILE=""
|
||||
ARGS=()
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--env)
|
||||
if [[ -z "${2:-}" || "${2:-}" == --* ]]; then
|
||||
echo -e "${RED}Error: --env requires a value${NC}"
|
||||
exit 1
|
||||
fi
|
||||
PROVIDER_ENV_FILE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--help)
|
||||
echo "Usage: $0 [OPTIONS]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --env <provider> Postman env path or provider name"
|
||||
echo " --verbose Show detailed output"
|
||||
echo " --html Generate HTML report"
|
||||
echo " --json Generate JSON report"
|
||||
echo " --bail Stop on first failure"
|
||||
echo " --help Show this help message"
|
||||
echo ""
|
||||
echo "Environment Variables:"
|
||||
echo " BIFROST_BASE_URL Override base URL (default: http://localhost:8080)"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
ARGS+=("$1")
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
set -- "${ARGS[@]}"
|
||||
|
||||
echo -e "${GREEN}==============================================${NC}"
|
||||
echo -e "${GREEN}Bifrost V1 Fallbacks Test Runner${NC}"
|
||||
echo -e "${GREEN}==============================================${NC}"
|
||||
echo ""
|
||||
|
||||
if ! command -v newman &> /dev/null; then
|
||||
echo -e "${RED}Error: Newman is not installed${NC}"
|
||||
echo "Install it with: npm install -g newman"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -f "$COLLECTION" ]; then
|
||||
echo -e "${RED}Error: Collection file not found: $COLLECTION${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$REPORT_DIR"
|
||||
|
||||
GLOBALS_TMP=""
|
||||
if [ -f "$PROVIDER_CAPABILITIES_JSON" ] && command -v jq &>/dev/null; then
|
||||
GLOBALS_TMP=$(mktemp)
|
||||
trap 'rm -f "$GLOBALS_TMP"' EXIT
|
||||
jq -n --rawfile cap "$PROVIDER_CAPABILITIES_JSON" '{id: "bifrost-provider-capabilities", name: "Provider capabilities", values: [{key: "provider_capabilities", value: $cap, type: "default", enabled: true}]}' > "$GLOBALS_TMP"
|
||||
fi
|
||||
|
||||
VERBOSE=""
|
||||
REPORTERS="cli"
|
||||
BAIL=""
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--verbose) VERBOSE="--verbose"; shift ;;
|
||||
--html) REPORTERS="${REPORTERS},html"; shift ;;
|
||||
--json) REPORTERS="${REPORTERS},json"; shift ;;
|
||||
--bail) BAIL="--bail"; shift ;;
|
||||
*) echo -e "${RED}Unknown option: $1${NC}"; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
SINGLE_JSON_ENV=""
|
||||
if [ -n "$PROVIDER_ENV_FILE" ]; then
|
||||
if [ -f "$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json"
|
||||
else
|
||||
echo -e "${RED}Error: Could not find environment file for: $PROVIDER_ENV_FILE${NC}"
|
||||
echo "Searched:"
|
||||
echo " - $PROVIDER_ENV_FILE"
|
||||
echo " - $PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE"
|
||||
echo " - $PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -z "$SINGLE_JSON_ENV" ]; then
|
||||
if [ -f "$PROVIDER_CONFIG_DIR/bifrost-v1-openai.postman_environment.json" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/bifrost-v1-openai.postman_environment.json"
|
||||
echo -e "${YELLOW}No --env specified, using openai${NC}"
|
||||
fi
|
||||
fi
|
||||
|
||||
cmd=(newman run "$COLLECTION")
|
||||
[ -n "$GLOBALS_TMP" ] && [ -f "$GLOBALS_TMP" ] && cmd+=(-g "$GLOBALS_TMP")
|
||||
[ -n "$SINGLE_JSON_ENV" ] && [ -f "$SINGLE_JSON_ENV" ] && cmd+=(-e "$SINGLE_JSON_ENV")
|
||||
base_url="${BIFROST_BASE_URL:-http://localhost:8080}"
|
||||
cmd+=(--env-var "base_url=$base_url")
|
||||
cmd+=(--timeout-script 120000 --timeout 900000)
|
||||
cmd+=(-r "$REPORTERS")
|
||||
[[ "$REPORTERS" == *"html"* ]] && cmd+=(--reporter-html-export "$REPORT_DIR/report.html")
|
||||
[[ "$REPORTERS" == *"json"* ]] && cmd+=(--reporter-json-export "$REPORT_DIR/report.json")
|
||||
[ -n "$VERBOSE" ] && cmd+=("$VERBOSE")
|
||||
[ -n "$BAIL" ] && cmd+=("$BAIL")
|
||||
|
||||
echo -e "Configuration:"
|
||||
echo -e " Collection: ${YELLOW}$COLLECTION${NC}"
|
||||
echo -e " Base URL: ${YELLOW}$base_url${NC}"
|
||||
if [ -n "$SINGLE_JSON_ENV" ]; then
|
||||
echo -e " Env: ${YELLOW}$SINGLE_JSON_ENV${NC}"
|
||||
fi
|
||||
echo -e " Reports: ${YELLOW}$REPORT_DIR${NC}"
|
||||
echo ""
|
||||
echo -e "${GREEN}Running tests...${NC}"
|
||||
echo ""
|
||||
|
||||
set +e
|
||||
"${cmd[@]}"
|
||||
EXIT_CODE=$?
|
||||
set -e
|
||||
|
||||
echo ""
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All fallbacks tests passed!${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Some tests failed${NC}"
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"html"* ]] || [[ "$REPORTERS" == *"json"* ]]; then
|
||||
echo ""
|
||||
echo -e "Reports saved to: ${YELLOW}$REPORT_DIR${NC}"
|
||||
ls -lh "$REPORT_DIR" 2>/dev/null | tail -n +2
|
||||
fi
|
||||
|
||||
exit $EXIT_CODE
|
||||
161
tests/e2e/api/runners/individual/run-newman-mgmt-flows-tests.sh
Executable file
161
tests/e2e/api/runners/individual/run-newman-mgmt-flows-tests.sh
Executable file
@@ -0,0 +1,161 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Bifrost V1 Management E2E Flows Newman Test Runner
|
||||
# Runs full lifecycle flows: Provider+Key+Inference, Customer+Team+VK+Inference, VK lifecycle.
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
API_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
cd "$API_DIR"
|
||||
|
||||
COLLECTION="collections/bifrost-v1-mgmt-flows.postman_collection.json"
|
||||
REPORT_DIR="newman-reports/mgmt-flows"
|
||||
PROVIDER_CONFIG_DIR="provider_config"
|
||||
PROVIDER_CAPABILITIES_JSON="provider-capabilities.json"
|
||||
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
PROVIDER_ENV_FILE=""
|
||||
ARGS=()
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--env)
|
||||
if [[ -z "${2:-}" || "${2:-}" == --* ]]; then
|
||||
echo -e "${RED}Error: --env requires a value${NC}"
|
||||
exit 1
|
||||
fi
|
||||
PROVIDER_ENV_FILE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--help)
|
||||
echo "Usage: $0 [OPTIONS]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --env <provider> Postman env path or provider name"
|
||||
echo " --verbose Show detailed output"
|
||||
echo " --html Generate HTML report"
|
||||
echo " --json Generate JSON report"
|
||||
echo " --bail Stop on first failure"
|
||||
echo " --help Show this help message"
|
||||
echo ""
|
||||
echo "Prerequisites: Governance plugin must be configured."
|
||||
echo "Environment Variables:"
|
||||
echo " BIFROST_BASE_URL Override base URL (default: http://localhost:8080)"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
ARGS+=("$1")
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
set -- "${ARGS[@]}"
|
||||
|
||||
echo -e "${GREEN}==============================================${NC}"
|
||||
echo -e "${GREEN}Bifrost V1 Management E2E Flows Test Runner${NC}"
|
||||
echo -e "${GREEN}==============================================${NC}"
|
||||
echo ""
|
||||
|
||||
if ! command -v newman &> /dev/null; then
|
||||
echo -e "${RED}Error: Newman is not installed${NC}"
|
||||
echo "Install it with: npm install -g newman"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -f "$COLLECTION" ]; then
|
||||
echo -e "${RED}Error: Collection file not found: $COLLECTION${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$REPORT_DIR"
|
||||
|
||||
GLOBALS_TMP=""
|
||||
if [ -f "$PROVIDER_CAPABILITIES_JSON" ] && command -v jq &>/dev/null; then
|
||||
GLOBALS_TMP=$(mktemp)
|
||||
trap 'rm -f "$GLOBALS_TMP"' EXIT
|
||||
jq -n --rawfile cap "$PROVIDER_CAPABILITIES_JSON" '{id: "bifrost-provider-capabilities", name: "Provider capabilities", values: [{key: "provider_capabilities", value: $cap, type: "default", enabled: true}]}' > "$GLOBALS_TMP"
|
||||
fi
|
||||
|
||||
VERBOSE=""
|
||||
REPORTERS="cli"
|
||||
BAIL=""
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--verbose) VERBOSE="--verbose"; shift ;;
|
||||
--html) REPORTERS="${REPORTERS},html"; shift ;;
|
||||
--json) REPORTERS="${REPORTERS},json"; shift ;;
|
||||
--bail) BAIL="--bail"; shift ;;
|
||||
*) echo -e "${RED}Unknown option: $1${NC}"; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
SINGLE_JSON_ENV=""
|
||||
if [ -n "$PROVIDER_ENV_FILE" ]; then
|
||||
if [ -f "$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json"
|
||||
else
|
||||
echo -e "${RED}Error: Could not find environment file for: $PROVIDER_ENV_FILE${NC}"
|
||||
echo "Searched:"
|
||||
echo " - $PROVIDER_ENV_FILE"
|
||||
echo " - $PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE"
|
||||
echo " - $PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -z "$SINGLE_JSON_ENV" ]; then
|
||||
if [ -f "$PROVIDER_CONFIG_DIR/bifrost-v1-openai.postman_environment.json" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/bifrost-v1-openai.postman_environment.json"
|
||||
echo -e "${YELLOW}No --env specified, using openai${NC}"
|
||||
fi
|
||||
fi
|
||||
|
||||
cmd=(newman run "$COLLECTION")
|
||||
[ -n "$GLOBALS_TMP" ] && [ -f "$GLOBALS_TMP" ] && cmd+=(-g "$GLOBALS_TMP")
|
||||
[ -n "$SINGLE_JSON_ENV" ] && [ -f "$SINGLE_JSON_ENV" ] && cmd+=(-e "$SINGLE_JSON_ENV")
|
||||
base_url="${BIFROST_BASE_URL:-http://localhost:8080}"
|
||||
cmd+=(--env-var "base_url=$base_url")
|
||||
cmd+=(--timeout-script 120000 --timeout 900000)
|
||||
cmd+=(-r "$REPORTERS")
|
||||
[[ "$REPORTERS" == *"html"* ]] && cmd+=(--reporter-html-export "$REPORT_DIR/report.html")
|
||||
[[ "$REPORTERS" == *"json"* ]] && cmd+=(--reporter-json-export "$REPORT_DIR/report.json")
|
||||
[ -n "$VERBOSE" ] && cmd+=("$VERBOSE")
|
||||
[ -n "$BAIL" ] && cmd+=("$BAIL")
|
||||
|
||||
echo -e "Configuration:"
|
||||
echo -e " Collection: ${YELLOW}$COLLECTION${NC}"
|
||||
echo -e " Base URL: ${YELLOW}$base_url${NC}"
|
||||
if [ -n "$SINGLE_JSON_ENV" ]; then
|
||||
echo -e " Env: ${YELLOW}$SINGLE_JSON_ENV${NC}"
|
||||
fi
|
||||
echo -e " Reports: ${YELLOW}$REPORT_DIR${NC}"
|
||||
echo ""
|
||||
echo -e "${GREEN}Running tests...${NC}"
|
||||
echo ""
|
||||
|
||||
set +e
|
||||
"${cmd[@]}"
|
||||
EXIT_CODE=$?
|
||||
set -e
|
||||
|
||||
echo ""
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All management flow tests passed!${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Some tests failed${NC}"
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"html"* ]] || [[ "$REPORTERS" == *"json"* ]]; then
|
||||
echo ""
|
||||
echo -e "Reports saved to: ${YELLOW}$REPORT_DIR${NC}"
|
||||
ls -lh "$REPORT_DIR" 2>/dev/null | tail -n +2
|
||||
fi
|
||||
|
||||
exit $EXIT_CODE
|
||||
514
tests/e2e/api/runners/individual/run-newman-openai-integration.sh
Executable file
514
tests/e2e/api/runners/individual/run-newman-openai-integration.sh
Executable file
@@ -0,0 +1,514 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Bifrost OpenAI Integration API Newman Test Runner
|
||||
# This script runs the OpenAI integration API test suite using Newman
|
||||
|
||||
set -e
|
||||
|
||||
# Run from script directory so paths to collection and provider-capabilities.json work
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
API_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
cd "$API_DIR"
|
||||
|
||||
# Configuration
|
||||
COLLECTION="collections/bifrost-openai-integration.postman_collection.json"
|
||||
ENVIRONMENT="bifrost-v1.postman_environment.json"
|
||||
REPORT_DIR="newman-reports/openai-integration"
|
||||
PROVIDER_CONFIG_DIR="provider_config"
|
||||
PROVIDER_CAPABILITIES_JSON="provider-capabilities.json"
|
||||
|
||||
# Colors for output
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Detect if --env was passed (so we run single provider vs all providers)
|
||||
PROVIDER_ENV_FILE=""
|
||||
ARGS=()
|
||||
while [[ $# -gt 0 ]]; do
|
||||
if [[ "$1" == "--env" ]]; then
|
||||
if [[ -z "${2:-}" || "${2:-}" == --* ]]; then
|
||||
echo -e "${RED}Error: --env requires a value${NC}"
|
||||
exit 1
|
||||
fi
|
||||
PROVIDER_ENV_FILE="$2"
|
||||
shift 2
|
||||
else
|
||||
ARGS+=("$1")
|
||||
shift
|
||||
fi
|
||||
done
|
||||
set -- "${ARGS[@]}"
|
||||
|
||||
# Normalize CI for retry logic (accept 1 or true, case-insensitive)
|
||||
ci_normalized="$(printf '%s' "${CI:-}" | tr '[:upper:]' '[:lower:]')"
|
||||
|
||||
# Print banner
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
if [ "$ci_normalized" = "1" ] || [ "$ci_normalized" = "true" ]; then
|
||||
echo -e "${GREEN}Bifrost OpenAI Integration API Test Runner with retries: 10${NC}"
|
||||
else
|
||||
echo -e "${GREEN}Bifrost OpenAI Integration API Test Runner${NC}"
|
||||
fi
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
echo ""
|
||||
|
||||
# Check if Newman is installed
|
||||
if ! command -v newman &> /dev/null; then
|
||||
echo -e "${RED}Error: Newman is not installed${NC}"
|
||||
echo "Install it with: npm install -g newman"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if collection exists
|
||||
if [ ! -f "$COLLECTION" ]; then
|
||||
echo -e "${RED}Error: Collection file not found: $COLLECTION${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if environment exists
|
||||
if [ ! -f "$ENVIRONMENT" ]; then
|
||||
echo -e "${YELLOW}Warning: Environment file not found: $ENVIRONMENT${NC}"
|
||||
echo "Using collection variables only"
|
||||
ENV_FLAG=""
|
||||
else
|
||||
ENV_FLAG="-e $ENVIRONMENT"
|
||||
fi
|
||||
|
||||
# Create report directory
|
||||
mkdir -p "$REPORT_DIR"
|
||||
|
||||
# Load provider capabilities from provider-capabilities.json (single source of truth) into a Newman globals file
|
||||
if [ ! -f "$PROVIDER_CAPABILITIES_JSON" ]; then
|
||||
echo -e "${RED}Error: $PROVIDER_CAPABILITIES_JSON not found${NC}"
|
||||
exit 1
|
||||
fi
|
||||
if ! command -v jq &>/dev/null; then
|
||||
echo -e "${RED}Error: jq is required to load $PROVIDER_CAPABILITIES_JSON${NC}"
|
||||
exit 1
|
||||
fi
|
||||
GLOBALS_TMP=$(mktemp)
|
||||
trap 'rm -f "$GLOBALS_TMP"' EXIT
|
||||
jq -n --rawfile cap "$PROVIDER_CAPABILITIES_JSON" '{id: "bifrost-provider-capabilities", name: "Provider capabilities", values: [{key: "provider_capabilities", value: $cap, type: "default", enabled: true}]}' > "$GLOBALS_TMP"
|
||||
|
||||
# When no --env: resolve list of provider Postman env .json files (sorted), excluding sgl and ollama
|
||||
EXCLUDED_PROVIDERS="sgl ollama"
|
||||
if [ -z "$PROVIDER_ENV_FILE" ] && [ -d "$PROVIDER_CONFIG_DIR" ]; then
|
||||
PROVIDER_JSON_FILES=()
|
||||
while IFS= read -r -d '' f; do
|
||||
# basename: bifrost-v1-openai.postman_environment.json -> openai
|
||||
name="${f##*/}"
|
||||
name="${name#bifrost-v1-}"
|
||||
name="${name%.postman_environment.json}"
|
||||
skip=""
|
||||
for ex in $EXCLUDED_PROVIDERS; do
|
||||
if [ "$name" = "$ex" ]; then skip=1; break; fi
|
||||
done
|
||||
[ -z "$skip" ] && PROVIDER_JSON_FILES+=("$f")
|
||||
done < <(find "$PROVIDER_CONFIG_DIR" -maxdepth 1 -name "bifrost-v1-*.postman_environment.json" -print0 2>/dev/null | sort -z)
|
||||
fi
|
||||
|
||||
# Parse command line arguments
|
||||
FOLDER=""
|
||||
VERBOSE="--verbose" # Enable verbose by default to capture console.log statements
|
||||
REPORTERS="cli"
|
||||
BAIL=""
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--folder)
|
||||
if [[ -z "${2:-}" || "${2:-}" == --* ]]; then
|
||||
echo -e "${RED}Error: --folder requires a value${NC}"
|
||||
exit 1
|
||||
fi
|
||||
FOLDER="$2"
|
||||
shift 2
|
||||
;;
|
||||
--verbose)
|
||||
VERBOSE="--verbose"
|
||||
shift
|
||||
;;
|
||||
--html)
|
||||
REPORTERS="cli,html"
|
||||
shift
|
||||
;;
|
||||
--json)
|
||||
REPORTERS="cli,json"
|
||||
shift
|
||||
;;
|
||||
--all-reports)
|
||||
REPORTERS="cli,html,json"
|
||||
shift
|
||||
;;
|
||||
--bail)
|
||||
BAIL="--bail"
|
||||
shift
|
||||
;;
|
||||
--help)
|
||||
echo "Usage: $0 [OPTIONS]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --folder <name> Run only tests in specified folder"
|
||||
echo " --verbose Show detailed output"
|
||||
echo " --html Generate HTML report"
|
||||
echo " --json Generate JSON report"
|
||||
echo " --all-reports Generate all report types"
|
||||
echo " --bail Stop on first failure"
|
||||
echo " --env <path> Postman env .json path or provider name (e.g. provider_config/bifrost-v1-openai.postman_environment.json or openai)"
|
||||
echo " --help Show this help message"
|
||||
echo ""
|
||||
echo "Environment Variables:"
|
||||
echo " BIFROST_BASE_URL Override base URL (default: http://localhost:8080)"
|
||||
echo " BIFROST_PROVIDER Override provider (default: openai)"
|
||||
echo " BIFROST_MODEL Override model name (default: gpt-4o)"
|
||||
echo " BIFROST_RESPONSES_MODEL Override Responses API model (default: BIFROST_MODEL)"
|
||||
echo " BIFROST_EMBEDDING_MODEL Override embedding model (default: text-embedding-3-small)"
|
||||
echo " BIFROST_SPEECH_MODEL Override speech model (default: tts-1)"
|
||||
echo " BIFROST_TRANSCRIPTION_MODEL Override transcription model (default: whisper-1)"
|
||||
echo " BIFROST_IMAGE_MODEL Override image model (default: dall-e-3)"
|
||||
echo ""
|
||||
echo "Examples:"
|
||||
echo " $0 # Run collection for all providers (each provider_config/bifrost-v1-*.postman_environment.json)"
|
||||
echo " $0 --env openai # Run once with OpenAI provider only"
|
||||
echo " $0 --folder \"Chat Completions\" # Run specific folder"
|
||||
echo " $0 --html --verbose # Verbose with HTML report"
|
||||
echo " BIFROST_BASE_URL=http://api:8080 $0 # Custom base URL"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo -e "${RED}Unknown option: $1${NC}"
|
||||
echo "Use --help for usage information"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Build and run Newman once.
|
||||
# Optional second arg: path to Postman env .json file (e.g. provider_config/bifrost-v1-openai.postman_environment.json).
|
||||
# When given, uses only that env file; otherwise uses default env and BIFROST_* overrides.
|
||||
run_newman() {
|
||||
local -a cmd=(newman run "$COLLECTION" -g "$GLOBALS_TMP")
|
||||
if [ -n "${2:-}" ] && [ -f "${2}" ]; then
|
||||
cmd+=(-e "${2}")
|
||||
else
|
||||
local base_url="${BIFROST_BASE_URL:-http://localhost:8080}"
|
||||
local provider="${BIFROST_PROVIDER:-openai}"
|
||||
local model="${BIFROST_MODEL:-gpt-4o}"
|
||||
local responses_model="${BIFROST_RESPONSES_MODEL:-$model}"
|
||||
local embedding_model="${BIFROST_EMBEDDING_MODEL:-text-embedding-3-small}"
|
||||
local speech_model="${BIFROST_SPEECH_MODEL:-tts-1}"
|
||||
local transcription_model="${BIFROST_TRANSCRIPTION_MODEL:-whisper-1}"
|
||||
local image_model="${BIFROST_IMAGE_MODEL:-dall-e-3}"
|
||||
if [ -n "$ENV_FLAG" ]; then
|
||||
cmd+=(-e "$ENVIRONMENT")
|
||||
fi
|
||||
cmd+=(--env-var "base_url=$base_url" --env-var "provider=$provider" --env-var "model=$model" --env-var "responses_model=$responses_model" --env-var "embedding_model=$embedding_model" --env-var "speech_model=$speech_model" --env-var "transcription_model=$transcription_model" --env-var "image_model=$image_model")
|
||||
fi
|
||||
[ -n "$FOLDER" ] && cmd+=(--folder "$FOLDER")
|
||||
cmd+=(--timeout-script 120000 --timeout 900000)
|
||||
cmd+=(-r "$REPORTERS")
|
||||
if [[ "$REPORTERS" == *"html"* ]]; then
|
||||
cmd+=(--reporter-html-export "$REPORT_DIR/report_${1:-run}.html")
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"json"* ]]; then
|
||||
cmd+=(--reporter-json-export "$REPORT_DIR/report_${1:-run}.json")
|
||||
fi
|
||||
[ -n "$VERBOSE" ] && cmd+=("$VERBOSE")
|
||||
[ -n "$BAIL" ] && cmd+=("$BAIL")
|
||||
if [ "$ci_normalized" = "1" ] || [ "$ci_normalized" = "true" ]; then
|
||||
cmd+=(--env-var "CI=1")
|
||||
fi
|
||||
|
||||
"${cmd[@]}"
|
||||
}
|
||||
|
||||
# Post-process log file to pretty-print JSON blocks using jq
|
||||
post_process_log() {
|
||||
local input_file="$1"
|
||||
local output_file="$2"
|
||||
|
||||
if [ ! -f "$input_file" ]; then
|
||||
return 1
|
||||
fi
|
||||
if ! command -v python3 >/dev/null 2>&1; then
|
||||
cp "$input_file" "$output_file"
|
||||
return 0
|
||||
fi
|
||||
|
||||
python3 - "$input_file" "$output_file" << 'PYTHON_SCRIPT'
|
||||
import sys
|
||||
import json
|
||||
import subprocess
|
||||
import shutil
|
||||
|
||||
def format_json_with_jq(json_text):
|
||||
"""Format JSON using jq if available, otherwise use Python's json module"""
|
||||
if shutil.which('jq'):
|
||||
try:
|
||||
process = subprocess.Popen(
|
||||
['jq', '.'],
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True
|
||||
)
|
||||
stdout, stderr = process.communicate(input=json_text)
|
||||
if process.returncode == 0:
|
||||
return stdout
|
||||
except Exception:
|
||||
pass
|
||||
# Fallback to Python's json module
|
||||
try:
|
||||
parsed = json.loads(json_text)
|
||||
return json.dumps(parsed, indent=2)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
return json_text
|
||||
|
||||
def process_log_file(input_file, output_file):
|
||||
"""Process log file and format JSON blocks"""
|
||||
with open(input_file, 'r', encoding='utf-8', errors='ignore') as f_in:
|
||||
with open(output_file, 'w', encoding='utf-8') as f_out:
|
||||
in_json_block = False
|
||||
json_lines = []
|
||||
|
||||
for line in f_in:
|
||||
# Check if we're entering a JSON block
|
||||
if 'REQUEST BODY:' in line or 'RESPONSE BODY:' in line:
|
||||
if in_json_block and json_lines:
|
||||
# Format previous JSON block
|
||||
json_text = ''.join(json_lines).strip()
|
||||
formatted = format_json_with_jq(json_text)
|
||||
f_out.write(formatted)
|
||||
if not formatted.endswith('\n'):
|
||||
f_out.write('\n')
|
||||
json_lines = []
|
||||
in_json_block = True
|
||||
f_out.write(line)
|
||||
continue
|
||||
|
||||
# Check if we're exiting a JSON block
|
||||
if in_json_block:
|
||||
stripped = line.strip()
|
||||
if not stripped or stripped.startswith('=') or line.startswith('REQUEST:') or line.startswith('RESPONSE:'):
|
||||
# End of JSON block, format and write
|
||||
if json_lines:
|
||||
json_text = ''.join(json_lines).strip()
|
||||
formatted = format_json_with_jq(json_text)
|
||||
f_out.write(formatted)
|
||||
if not formatted.endswith('\n'):
|
||||
f_out.write('\n')
|
||||
json_lines = []
|
||||
in_json_block = False
|
||||
else:
|
||||
json_lines.append(line)
|
||||
continue
|
||||
|
||||
f_out.write(line)
|
||||
|
||||
# Handle case where file ends in a JSON block
|
||||
if json_lines:
|
||||
json_text = ''.join(json_lines).strip()
|
||||
formatted = format_json_with_jq(json_text)
|
||||
f_out.write(formatted)
|
||||
if not formatted.endswith('\n'):
|
||||
f_out.write('\n')
|
||||
|
||||
if __name__ == '__main__':
|
||||
if len(sys.argv) < 3:
|
||||
print(f"Error: Expected 2 arguments, got {len(sys.argv) - 1}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
try:
|
||||
process_log_file(sys.argv[1], sys.argv[2])
|
||||
except Exception as e:
|
||||
print(f"Error processing log file: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
PYTHON_SCRIPT
|
||||
}
|
||||
|
||||
# Run for a single provider (--env was passed: path to .json env or provider name)
|
||||
if [ -n "$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV=""
|
||||
if [ -f "$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json"
|
||||
fi
|
||||
if [ -z "$SINGLE_JSON_ENV" ]; then
|
||||
echo -e "${RED}Error: Env file not found: $PROVIDER_ENV_FILE${NC}"
|
||||
echo "Use a path to a .json env (e.g. provider_config/bifrost-v1-openai.postman_environment.json) or provider name (e.g. openai)"
|
||||
exit 1
|
||||
fi
|
||||
SINGLE_PROVIDER_NAME="${SINGLE_JSON_ENV##*/}"
|
||||
SINGLE_PROVIDER_NAME="${SINGLE_PROVIDER_NAME#bifrost-v1-}"
|
||||
SINGLE_PROVIDER_NAME="${SINGLE_PROVIDER_NAME%.postman_environment.json}"
|
||||
echo -e "Configuration: ${YELLOW}$SINGLE_JSON_ENV${NC}"
|
||||
echo -e " Reports: ${YELLOW}$REPORT_DIR${NC}"
|
||||
echo ""
|
||||
echo -e "${GREEN}Running tests...${NC}"
|
||||
echo ""
|
||||
TEMP_LOG="$REPORT_DIR/${SINGLE_PROVIDER_NAME}.log.tmp"
|
||||
set +e
|
||||
run_newman "$SINGLE_PROVIDER_NAME" "$SINGLE_JSON_ENV" > "$TEMP_LOG" 2>&1
|
||||
EXIT_CODE=$?
|
||||
set -e
|
||||
LOG_FILE="$REPORT_DIR/${SINGLE_PROVIDER_NAME}.log"
|
||||
post_process_log "$TEMP_LOG" "$LOG_FILE" || cp "$TEMP_LOG" "$LOG_FILE"
|
||||
rm -f "$TEMP_LOG"
|
||||
echo ""
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All tests passed!${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Some tests failed${NC}"
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"html"* ]] || [[ "$REPORTERS" == *"json"* ]]; then
|
||||
echo ""
|
||||
echo -e "Reports saved to: ${YELLOW}$REPORT_DIR${NC}"
|
||||
ls -lh "$REPORT_DIR" 2>/dev/null | tail -n +2
|
||||
fi
|
||||
exit $EXIT_CODE
|
||||
fi
|
||||
|
||||
# Run for all providers (no --env)
|
||||
if [ -z "${PROVIDER_JSON_FILES+x}" ] || [ ${#PROVIDER_JSON_FILES[@]} -eq 0 ]; then
|
||||
echo -e "${YELLOW}No provider env .json files found in $PROVIDER_CONFIG_DIR/. Using default (openai).${NC}"
|
||||
echo -e "Configuration:"
|
||||
echo -e " Base URL: ${YELLOW}${BIFROST_BASE_URL:-http://localhost:8080}${NC}"
|
||||
echo -e " Provider: ${YELLOW}${BIFROST_PROVIDER:-openai}${NC}"
|
||||
echo -e " Reports: ${YELLOW}$REPORT_DIR${NC}"
|
||||
echo ""
|
||||
echo -e "${GREEN}Running tests...${NC}"
|
||||
echo ""
|
||||
TEMP_LOG="$REPORT_DIR/default.log.tmp"
|
||||
set +e
|
||||
run_newman > "$TEMP_LOG" 2>&1
|
||||
EXIT_CODE=$?
|
||||
set -e
|
||||
LOG_FILE="$REPORT_DIR/default.log"
|
||||
post_process_log "$TEMP_LOG" "$LOG_FILE" || cp "$TEMP_LOG" "$LOG_FILE"
|
||||
rm -f "$TEMP_LOG"
|
||||
echo ""
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All tests passed!${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Some tests failed${NC}"
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"html"* ]] || [[ "$REPORTERS" == *"json"* ]]; then
|
||||
echo ""
|
||||
echo -e "Reports saved to: ${YELLOW}$REPORT_DIR${NC}"
|
||||
ls -lh "$REPORT_DIR" 2>/dev/null | tail -n +2
|
||||
fi
|
||||
exit $EXIT_CODE
|
||||
fi
|
||||
|
||||
PARALLEL_LOGS_DIR="$REPORT_DIR/parallel_logs"
|
||||
mkdir -p "$PARALLEL_LOGS_DIR"
|
||||
|
||||
# Print a one-line report for a provider from its Newman log and exit code
|
||||
print_provider_report() {
|
||||
local name="$1"
|
||||
local logfile="$2"
|
||||
local exitcode="$3"
|
||||
local failed_count=""
|
||||
local failed_tests=""
|
||||
if [ -f "$logfile" ]; then
|
||||
# Parse Newman summary table: assertions row, third column = failed count
|
||||
failed_count=$(grep "assertions" "$logfile" 2>/dev/null | awk -F'│' '{gsub(/^ *| *$/,"",$4); print $4}' | head -1)
|
||||
# Lines with " ✗ " are failed assertions; strip to get test name
|
||||
failed_tests=$(grep " ✗ " "$logfile" 2>/dev/null | sed 's/.*✗ */ - /' | sed 's/^ *//' | tr '\n' ' ' | sed 's/ $//')
|
||||
fi
|
||||
if [ "$exitcode" -eq 0 ]; then
|
||||
echo -e "${GREEN} ✓ $name: PASS${NC}"
|
||||
else
|
||||
echo -e "${RED} ✗ $name: FAIL${NC}"
|
||||
if [ -n "$failed_count" ] && [ "$failed_count" -gt 0 ] 2>/dev/null; then
|
||||
echo -e " ${RED}${failed_count} assertion(s) failed${NC}"
|
||||
fi
|
||||
if [ -n "$failed_tests" ]; then
|
||||
echo -e " Failed: $failed_tests"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
# Draw the provider status table (TABLE_LINES lines). Use after moving cursor up TABLE_LINES to refresh.
|
||||
draw_table() {
|
||||
printf '\033[2K%-16s %s\n' "Provider" "Status"
|
||||
for i in "${!NAMES[@]}"; do
|
||||
printf '\033[2K%-16s %b\n' "${NAMES[$i]}" "${STATUS[$i]}"
|
||||
done
|
||||
}
|
||||
|
||||
echo -e "Running tests for ${#PROVIDER_JSON_FILES[@]} provider(s) ${GREEN}in parallel${NC}. Reports: ${YELLOW}$REPORT_DIR${NC}"
|
||||
echo ""
|
||||
|
||||
# Run each provider in a background subshell; capture PID and log path per provider
|
||||
PIDS=()
|
||||
NAMES=()
|
||||
LOG_FILES=()
|
||||
for jsonfile in "${PROVIDER_JSON_FILES[@]}"; do
|
||||
name="${jsonfile##*/}"
|
||||
name="${name#bifrost-v1-}"
|
||||
name="${name%.postman_environment.json}"
|
||||
logfile="$PARALLEL_LOGS_DIR/${name}.log"
|
||||
temp_logfile="${logfile}.tmp"
|
||||
LOG_FILES+=("$logfile")
|
||||
NAMES+=("$name")
|
||||
( set +e; run_newman "$name" "$jsonfile" > "$temp_logfile" 2>&1; ec=$?; set -e; post_process_log "$temp_logfile" "$logfile" || cp "$temp_logfile" "$logfile"; rm -f "$temp_logfile"; exit $ec ) &
|
||||
PIDS+=($!)
|
||||
done
|
||||
|
||||
# Status for each provider: Pending, ✓ PASS, or ✗ FAIL (with color)
|
||||
STATUS=()
|
||||
for i in "${!PIDS[@]}"; do STATUS[$i]="${YELLOW}Pending${NC}"; done
|
||||
TABLE_LINES=$((${#NAMES[@]} + 1))
|
||||
|
||||
# Initial table
|
||||
draw_table
|
||||
|
||||
# Track which we've reaped (0 = pending, 1 = done)
|
||||
REAPED=()
|
||||
for i in "${!PIDS[@]}"; do REAPED[$i]=0; done
|
||||
|
||||
OVERALL_FAILED=0
|
||||
FAILED_NAMES=()
|
||||
|
||||
# As each provider finishes, update status and redraw table
|
||||
while true; do
|
||||
all_done=1
|
||||
for i in "${!PIDS[@]}"; do
|
||||
[ "${REAPED[$i]:-0}" -eq 1 ] && continue
|
||||
all_done=0
|
||||
if ! kill -0 "${PIDS[$i]}" 2>/dev/null; then
|
||||
exitcode=0; wait "${PIDS[$i]}" || exitcode=$?
|
||||
REAPED[$i]=1
|
||||
if [ "$exitcode" -eq 0 ]; then
|
||||
STATUS[$i]="${GREEN}✓ PASS${NC}"
|
||||
else
|
||||
OVERALL_FAILED=1
|
||||
FAILED_NAMES+=("${NAMES[$i]}")
|
||||
STATUS[$i]="${RED}✗ FAIL${NC}"
|
||||
fi
|
||||
# Move cursor up and redraw table
|
||||
printf '\033[%dA' "$TABLE_LINES"
|
||||
draw_table
|
||||
fi
|
||||
done
|
||||
[ "$all_done" -eq 1 ] && break
|
||||
sleep 0.3
|
||||
done
|
||||
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
if [ $OVERALL_FAILED -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All providers passed!${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ One or more providers had failures: ${FAILED_NAMES[*]}${NC}"
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"html"* ]] || [[ "$REPORTERS" == *"json"* ]]; then
|
||||
echo ""
|
||||
echo -e "Reports saved to: ${YELLOW}$REPORT_DIR${NC}"
|
||||
ls -lh "$REPORT_DIR" 2>/dev/null | tail -n +2
|
||||
fi
|
||||
# Parallel logs persist in $PARALLEL_LOGS_DIR (overwritten per provider on each run)
|
||||
exit $OVERALL_FAILED
|
||||
161
tests/e2e/api/runners/individual/run-newman-rate-limit-tests.sh
Executable file
161
tests/e2e/api/runners/individual/run-newman-rate-limit-tests.sh
Executable file
@@ -0,0 +1,161 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Bifrost V1 Rate Limit Newman Test Runner
|
||||
# Runs rate limit enforcement tests. Requires governance plugin.
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
API_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
cd "$API_DIR"
|
||||
|
||||
COLLECTION="collections/bifrost-v1-rate-limit.postman_collection.json"
|
||||
REPORT_DIR="newman-reports/rate-limit"
|
||||
PROVIDER_CONFIG_DIR="provider_config"
|
||||
PROVIDER_CAPABILITIES_JSON="provider-capabilities.json"
|
||||
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
PROVIDER_ENV_FILE=""
|
||||
ARGS=()
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--env)
|
||||
if [[ -z "${2:-}" || "${2:-}" == --* ]]; then
|
||||
echo -e "${RED}Error: --env requires a value${NC}"
|
||||
exit 1
|
||||
fi
|
||||
PROVIDER_ENV_FILE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--help)
|
||||
echo "Usage: $0 [OPTIONS]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --env <provider> Postman env path or provider name"
|
||||
echo " --verbose Show detailed output"
|
||||
echo " --html Generate HTML report"
|
||||
echo " --json Generate JSON report"
|
||||
echo " --bail Stop on first failure"
|
||||
echo " --help Show this help message"
|
||||
echo ""
|
||||
echo "Prerequisites: Governance plugin must be configured."
|
||||
echo "Environment Variables:"
|
||||
echo " BIFROST_BASE_URL Override base URL (default: http://localhost:8080)"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
ARGS+=("$1")
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
set -- "${ARGS[@]}"
|
||||
|
||||
echo -e "${GREEN}==============================================${NC}"
|
||||
echo -e "${GREEN}Bifrost V1 Rate Limit Test Runner${NC}"
|
||||
echo -e "${GREEN}==============================================${NC}"
|
||||
echo ""
|
||||
|
||||
if ! command -v newman &> /dev/null; then
|
||||
echo -e "${RED}Error: Newman is not installed${NC}"
|
||||
echo "Install it with: npm install -g newman"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -f "$COLLECTION" ]; then
|
||||
echo -e "${RED}Error: Collection file not found: $COLLECTION${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$REPORT_DIR"
|
||||
|
||||
GLOBALS_TMP=""
|
||||
if [ -f "$PROVIDER_CAPABILITIES_JSON" ] && command -v jq &>/dev/null; then
|
||||
GLOBALS_TMP=$(mktemp)
|
||||
trap 'rm -f "$GLOBALS_TMP"' EXIT
|
||||
jq -n --rawfile cap "$PROVIDER_CAPABILITIES_JSON" '{id: "bifrost-provider-capabilities", name: "Provider capabilities", values: [{key: "provider_capabilities", value: $cap, type: "default", enabled: true}]}' > "$GLOBALS_TMP"
|
||||
fi
|
||||
|
||||
VERBOSE=""
|
||||
REPORTERS="cli"
|
||||
BAIL=""
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--verbose) VERBOSE="--verbose"; shift ;;
|
||||
--html) REPORTERS="${REPORTERS},html"; shift ;;
|
||||
--json) REPORTERS="${REPORTERS},json"; shift ;;
|
||||
--bail) BAIL="--bail"; shift ;;
|
||||
*) echo -e "${RED}Unknown option: $1${NC}"; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
SINGLE_JSON_ENV=""
|
||||
if [ -n "$PROVIDER_ENV_FILE" ]; then
|
||||
if [ -f "$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json"
|
||||
else
|
||||
echo -e "${RED}Error: Could not find environment file for: $PROVIDER_ENV_FILE${NC}"
|
||||
echo "Searched:"
|
||||
echo " - $PROVIDER_ENV_FILE"
|
||||
echo " - $PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE"
|
||||
echo " - $PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -z "$SINGLE_JSON_ENV" ]; then
|
||||
if [ -f "$PROVIDER_CONFIG_DIR/bifrost-v1-openai.postman_environment.json" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/bifrost-v1-openai.postman_environment.json"
|
||||
echo -e "${YELLOW}No --env specified, using openai${NC}"
|
||||
fi
|
||||
fi
|
||||
|
||||
cmd=(newman run "$COLLECTION")
|
||||
[ -n "$GLOBALS_TMP" ] && [ -f "$GLOBALS_TMP" ] && cmd+=(-g "$GLOBALS_TMP")
|
||||
[ -n "$SINGLE_JSON_ENV" ] && [ -f "$SINGLE_JSON_ENV" ] && cmd+=(-e "$SINGLE_JSON_ENV")
|
||||
base_url="${BIFROST_BASE_URL:-http://localhost:8080}"
|
||||
cmd+=(--env-var "base_url=$base_url")
|
||||
cmd+=(--timeout-script 120000 --timeout 900000)
|
||||
cmd+=(-r "$REPORTERS")
|
||||
[[ "$REPORTERS" == *"html"* ]] && cmd+=(--reporter-html-export "$REPORT_DIR/report.html")
|
||||
[[ "$REPORTERS" == *"json"* ]] && cmd+=(--reporter-json-export "$REPORT_DIR/report.json")
|
||||
[ -n "$VERBOSE" ] && cmd+=("$VERBOSE")
|
||||
[ -n "$BAIL" ] && cmd+=("$BAIL")
|
||||
|
||||
echo -e "Configuration:"
|
||||
echo -e " Collection: ${YELLOW}$COLLECTION${NC}"
|
||||
echo -e " Base URL: ${YELLOW}$base_url${NC}"
|
||||
if [ -n "$SINGLE_JSON_ENV" ]; then
|
||||
echo -e " Env: ${YELLOW}$SINGLE_JSON_ENV${NC}"
|
||||
fi
|
||||
echo -e " Reports: ${YELLOW}$REPORT_DIR${NC}"
|
||||
echo ""
|
||||
echo -e "${GREEN}Running tests...${NC}"
|
||||
echo ""
|
||||
|
||||
set +e
|
||||
"${cmd[@]}"
|
||||
EXIT_CODE=$?
|
||||
set -e
|
||||
|
||||
echo ""
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All rate limit tests passed!${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Some tests failed${NC}"
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"html"* ]] || [[ "$REPORTERS" == *"json"* ]]; then
|
||||
echo ""
|
||||
echo -e "Reports saved to: ${YELLOW}$REPORT_DIR${NC}"
|
||||
ls -lh "$REPORT_DIR" 2>/dev/null | tail -n +2
|
||||
fi
|
||||
|
||||
exit $EXIT_CODE
|
||||
166
tests/e2e/api/runners/individual/run-newman-session-tests.sh
Executable file
166
tests/e2e/api/runners/individual/run-newman-session-tests.sh
Executable file
@@ -0,0 +1,166 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Bifrost V1 Session Stickiness Newman Test Runner
|
||||
# Runs session stickiness tests (x-bf-session-id, x-bf-session-ttl).
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
API_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
cd "$API_DIR"
|
||||
|
||||
COLLECTION="collections/bifrost-v1-session.postman_collection.json"
|
||||
REPORT_DIR="newman-reports/session"
|
||||
PROVIDER_CONFIG_DIR="provider_config"
|
||||
PROVIDER_CAPABILITIES_JSON="provider-capabilities.json"
|
||||
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
PROVIDER_ENV_FILE=""
|
||||
ARGS=()
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--env)
|
||||
if [[ -z "${2:-}" || "${2:-}" == --* ]]; then
|
||||
echo -e "${RED}Error: --env requires a value${NC}"
|
||||
exit 1
|
||||
fi
|
||||
PROVIDER_ENV_FILE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--help)
|
||||
echo "Usage: $0 [OPTIONS]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --env <provider> Postman env path or provider name"
|
||||
echo " --verbose Show detailed output"
|
||||
echo " --html Generate HTML report"
|
||||
echo " --json Generate JSON report"
|
||||
echo " --bail Stop on first failure"
|
||||
echo " --help Show this help message"
|
||||
echo ""
|
||||
echo "Environment Variables:"
|
||||
echo " BIFROST_BASE_URL Override base URL (default: http://localhost:8080)"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
ARGS+=("$1")
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
set -- "${ARGS[@]}"
|
||||
|
||||
echo -e "${GREEN}==============================================${NC}"
|
||||
echo -e "${GREEN}Bifrost V1 Session Stickiness Test Runner${NC}"
|
||||
echo -e "${GREEN}==============================================${NC}"
|
||||
echo ""
|
||||
|
||||
if ! command -v newman &> /dev/null; then
|
||||
echo -e "${RED}Error: Newman is not installed${NC}"
|
||||
echo "Install it with: npm install -g newman"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -f "$COLLECTION" ]; then
|
||||
echo -e "${RED}Error: Collection file not found: $COLLECTION${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$REPORT_DIR"
|
||||
|
||||
GLOBALS_TMP=""
|
||||
if [ -f "$PROVIDER_CAPABILITIES_JSON" ] && command -v jq &>/dev/null; then
|
||||
GLOBALS_TMP=$(mktemp)
|
||||
trap 'rm -f "$GLOBALS_TMP"' EXIT
|
||||
jq -n --rawfile cap "$PROVIDER_CAPABILITIES_JSON" '{id: "bifrost-provider-capabilities", name: "Provider capabilities", values: [{key: "provider_capabilities", value: $cap, type: "default", enabled: true}]}' > "$GLOBALS_TMP"
|
||||
fi
|
||||
|
||||
VERBOSE=""
|
||||
BAIL=""
|
||||
HTML_REPORT=""
|
||||
JSON_REPORT=""
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--verbose) VERBOSE="--verbose"; shift ;;
|
||||
--html) HTML_REPORT="yes"; shift ;;
|
||||
--json) JSON_REPORT="yes"; shift ;;
|
||||
--bail) BAIL="--bail"; shift ;;
|
||||
*) echo -e "${RED}Unknown option: $1${NC}"; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Build reporters string
|
||||
REPORTERS="cli"
|
||||
[ -n "$HTML_REPORT" ] && REPORTERS="$REPORTERS,html"
|
||||
[ -n "$JSON_REPORT" ] && REPORTERS="$REPORTERS,json"
|
||||
|
||||
SINGLE_JSON_ENV=""
|
||||
if [ -n "$PROVIDER_ENV_FILE" ]; then
|
||||
if [ -f "$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json"
|
||||
else
|
||||
echo -e "${RED}Error: Could not find environment file for: $PROVIDER_ENV_FILE${NC}"
|
||||
echo "Searched:"
|
||||
echo " - $PROVIDER_ENV_FILE"
|
||||
echo " - $PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE"
|
||||
echo " - $PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -z "$SINGLE_JSON_ENV" ]; then
|
||||
if [ -f "$PROVIDER_CONFIG_DIR/bifrost-v1-openai.postman_environment.json" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/bifrost-v1-openai.postman_environment.json"
|
||||
echo -e "${YELLOW}No --env specified, using openai${NC}"
|
||||
fi
|
||||
fi
|
||||
|
||||
cmd=(newman run "$COLLECTION")
|
||||
[ -n "$GLOBALS_TMP" ] && [ -f "$GLOBALS_TMP" ] && cmd+=(-g "$GLOBALS_TMP")
|
||||
[ -n "$SINGLE_JSON_ENV" ] && [ -f "$SINGLE_JSON_ENV" ] && cmd+=(-e "$SINGLE_JSON_ENV")
|
||||
base_url="${BIFROST_BASE_URL:-http://localhost:8080}"
|
||||
cmd+=(--env-var "base_url=$base_url")
|
||||
cmd+=(--timeout-script 120000 --timeout 900000)
|
||||
cmd+=(-r "$REPORTERS")
|
||||
[[ "$REPORTERS" == *"html"* ]] && cmd+=(--reporter-html-export "$REPORT_DIR/report.html")
|
||||
[[ "$REPORTERS" == *"json"* ]] && cmd+=(--reporter-json-export "$REPORT_DIR/report.json")
|
||||
[ -n "$VERBOSE" ] && cmd+=("$VERBOSE")
|
||||
[ -n "$BAIL" ] && cmd+=("$BAIL")
|
||||
|
||||
echo -e "Configuration:"
|
||||
echo -e " Collection: ${YELLOW}$COLLECTION${NC}"
|
||||
echo -e " Base URL: ${YELLOW}$base_url${NC}"
|
||||
if [ -n "$SINGLE_JSON_ENV" ]; then
|
||||
echo -e " Env: ${YELLOW}$SINGLE_JSON_ENV${NC}"
|
||||
fi
|
||||
echo -e " Reports: ${YELLOW}$REPORT_DIR${NC}"
|
||||
echo ""
|
||||
echo -e "${GREEN}Running tests...${NC}"
|
||||
echo ""
|
||||
|
||||
set +e
|
||||
"${cmd[@]}"
|
||||
EXIT_CODE=$?
|
||||
set -e
|
||||
|
||||
echo ""
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All session tests passed!${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Some tests failed${NC}"
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"html"* ]] || [[ "$REPORTERS" == *"json"* ]]; then
|
||||
echo ""
|
||||
echo -e "Reports saved to: ${YELLOW}$REPORT_DIR${NC}"
|
||||
ls -lh "$REPORT_DIR" 2>/dev/null | tail -n +2
|
||||
fi
|
||||
|
||||
exit $EXIT_CODE
|
||||
160
tests/e2e/api/runners/individual/run-newman-streaming-tests.sh
Executable file
160
tests/e2e/api/runners/individual/run-newman-streaming-tests.sh
Executable file
@@ -0,0 +1,160 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Bifrost V1 Streaming Newman Test Runner
|
||||
# Runs streaming SSE tests for chat completions and responses.
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
API_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
cd "$API_DIR"
|
||||
|
||||
COLLECTION="collections/bifrost-v1-streaming.postman_collection.json"
|
||||
REPORT_DIR="newman-reports/streaming"
|
||||
PROVIDER_CONFIG_DIR="provider_config"
|
||||
PROVIDER_CAPABILITIES_JSON="provider-capabilities.json"
|
||||
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
PROVIDER_ENV_FILE=""
|
||||
ARGS=()
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--env)
|
||||
if [[ -z "${2:-}" || "${2:-}" == --* ]]; then
|
||||
echo -e "${RED}Error: --env requires a value${NC}"
|
||||
exit 1
|
||||
fi
|
||||
PROVIDER_ENV_FILE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--help)
|
||||
echo "Usage: $0 [OPTIONS]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --env <provider> Postman env path or provider name"
|
||||
echo " --verbose Show detailed output"
|
||||
echo " --html Generate HTML report"
|
||||
echo " --json Generate JSON report"
|
||||
echo " --bail Stop on first failure"
|
||||
echo " --help Show this help message"
|
||||
echo ""
|
||||
echo "Environment Variables:"
|
||||
echo " BIFROST_BASE_URL Override base URL (default: http://localhost:8080)"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
ARGS+=("$1")
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
set -- "${ARGS[@]}"
|
||||
|
||||
echo -e "${GREEN}==============================================${NC}"
|
||||
echo -e "${GREEN}Bifrost V1 Streaming Test Runner${NC}"
|
||||
echo -e "${GREEN}==============================================${NC}"
|
||||
echo ""
|
||||
|
||||
if ! command -v newman &> /dev/null; then
|
||||
echo -e "${RED}Error: Newman is not installed${NC}"
|
||||
echo "Install it with: npm install -g newman"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -f "$COLLECTION" ]; then
|
||||
echo -e "${RED}Error: Collection file not found: $COLLECTION${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$REPORT_DIR"
|
||||
|
||||
GLOBALS_TMP=""
|
||||
if [ -f "$PROVIDER_CAPABILITIES_JSON" ] && command -v jq &>/dev/null; then
|
||||
GLOBALS_TMP=$(mktemp)
|
||||
trap 'rm -f "$GLOBALS_TMP"' EXIT
|
||||
jq -n --rawfile cap "$PROVIDER_CAPABILITIES_JSON" '{id: "bifrost-provider-capabilities", name: "Provider capabilities", values: [{key: "provider_capabilities", value: $cap, type: "default", enabled: true}]}' > "$GLOBALS_TMP"
|
||||
fi
|
||||
|
||||
VERBOSE=""
|
||||
REPORTERS="cli"
|
||||
BAIL=""
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--verbose) VERBOSE="--verbose"; shift ;;
|
||||
--html) REPORTERS="${REPORTERS},html"; shift ;;
|
||||
--json) REPORTERS="${REPORTERS},json"; shift ;;
|
||||
--bail) BAIL="--bail"; shift ;;
|
||||
*) echo -e "${RED}Unknown option: $1${NC}"; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
SINGLE_JSON_ENV=""
|
||||
if [ -n "$PROVIDER_ENV_FILE" ]; then
|
||||
if [ -f "$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json"
|
||||
else
|
||||
echo -e "${RED}Error: Could not find environment file for: $PROVIDER_ENV_FILE${NC}"
|
||||
echo "Searched:"
|
||||
echo " - $PROVIDER_ENV_FILE"
|
||||
echo " - $PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE"
|
||||
echo " - $PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -z "$SINGLE_JSON_ENV" ]; then
|
||||
if [ -f "$PROVIDER_CONFIG_DIR/bifrost-v1-openai.postman_environment.json" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/bifrost-v1-openai.postman_environment.json"
|
||||
echo -e "${YELLOW}No --env specified, using openai${NC}"
|
||||
fi
|
||||
fi
|
||||
|
||||
cmd=(newman run "$COLLECTION")
|
||||
[ -n "$GLOBALS_TMP" ] && [ -f "$GLOBALS_TMP" ] && cmd+=(-g "$GLOBALS_TMP")
|
||||
[ -n "$SINGLE_JSON_ENV" ] && [ -f "$SINGLE_JSON_ENV" ] && cmd+=(-e "$SINGLE_JSON_ENV")
|
||||
base_url="${BIFROST_BASE_URL:-http://localhost:8080}"
|
||||
cmd+=(--env-var "base_url=$base_url")
|
||||
cmd+=(--timeout-script 120000 --timeout 900000)
|
||||
cmd+=(-r "$REPORTERS")
|
||||
[[ "$REPORTERS" == *"html"* ]] && cmd+=(--reporter-html-export "$REPORT_DIR/report.html")
|
||||
[[ "$REPORTERS" == *"json"* ]] && cmd+=(--reporter-json-export "$REPORT_DIR/report.json")
|
||||
[ -n "$VERBOSE" ] && cmd+=("$VERBOSE")
|
||||
[ -n "$BAIL" ] && cmd+=("$BAIL")
|
||||
|
||||
echo -e "Configuration:"
|
||||
echo -e " Collection: ${YELLOW}$COLLECTION${NC}"
|
||||
echo -e " Base URL: ${YELLOW}$base_url${NC}"
|
||||
if [ -n "$SINGLE_JSON_ENV" ]; then
|
||||
echo -e " Env: ${YELLOW}$SINGLE_JSON_ENV${NC}"
|
||||
fi
|
||||
echo -e " Reports: ${YELLOW}$REPORT_DIR${NC}"
|
||||
echo ""
|
||||
echo -e "${GREEN}Running tests...${NC}"
|
||||
echo ""
|
||||
|
||||
set +e
|
||||
"${cmd[@]}"
|
||||
EXIT_CODE=$?
|
||||
set -e
|
||||
|
||||
echo ""
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All streaming tests passed!${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Some tests failed${NC}"
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"html"* ]] || [[ "$REPORTERS" == *"json"* ]]; then
|
||||
echo ""
|
||||
echo -e "Reports saved to: ${YELLOW}$REPORT_DIR${NC}"
|
||||
ls -lh "$REPORT_DIR" 2>/dev/null | tail -n +2
|
||||
fi
|
||||
|
||||
exit $EXIT_CODE
|
||||
211
tests/e2e/api/runners/individual/run-newman-vk-auth-tests.sh
Executable file
211
tests/e2e/api/runners/individual/run-newman-vk-auth-tests.sh
Executable file
@@ -0,0 +1,211 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Bifrost V1 Virtual Key Auth Newman Test Runner
|
||||
# Runs VK auth tests: creates VK, runs inference with/without VK, tests rejection cases, cleans up.
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
API_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
cd "$API_DIR"
|
||||
|
||||
# Configuration
|
||||
COLLECTION="collections/bifrost-v1-vk-auth.postman_collection.json"
|
||||
REPORT_DIR="newman-reports/vk-auth"
|
||||
PROVIDER_CONFIG_DIR="provider_config"
|
||||
PROVIDER_CAPABILITIES_JSON="provider-capabilities.json"
|
||||
|
||||
# Colors for output
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Parse arguments
|
||||
PROVIDER_ENV_FILE=""
|
||||
ENFORCE_AUTH=""
|
||||
ARGS=()
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--env)
|
||||
if [[ -z "${2:-}" || "${2:-}" == --* ]]; then
|
||||
echo -e "${RED}Error: --env requires a value${NC}"
|
||||
exit 1
|
||||
fi
|
||||
PROVIDER_ENV_FILE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--enforce-auth)
|
||||
ENFORCE_AUTH="1"
|
||||
shift
|
||||
;;
|
||||
--help)
|
||||
echo "Usage: $0 [OPTIONS]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --env <provider> Postman env path or provider name (e.g. openai or provider_config/bifrost-v1-openai.postman_environment.json)"
|
||||
echo " --enforce-auth Enable auth enforcement mode (without-VK requests expect 401)"
|
||||
echo " --verbose Show detailed output"
|
||||
echo " --html Generate HTML report"
|
||||
echo " --json Generate JSON report"
|
||||
echo " --bail Stop on first failure"
|
||||
echo " --help Show this help message"
|
||||
echo ""
|
||||
echo "Environment Variables:"
|
||||
echo " BIFROST_BASE_URL Override base URL (default: http://localhost:8080)"
|
||||
echo ""
|
||||
echo "Examples:"
|
||||
echo " $0 --env openai # Run with OpenAI provider"
|
||||
echo " $0 --env openai --enforce-auth # Run with auth enforcement"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
ARGS+=("$1")
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
set -- "${ARGS[@]}"
|
||||
|
||||
# Print banner
|
||||
echo -e "${GREEN}==============================================${NC}"
|
||||
echo -e "${GREEN}Bifrost V1 Virtual Key Auth Test Runner${NC}"
|
||||
echo -e "${GREEN}==============================================${NC}"
|
||||
echo ""
|
||||
|
||||
# Check if Newman is installed
|
||||
if ! command -v newman &> /dev/null; then
|
||||
echo -e "${RED}Error: Newman is not installed${NC}"
|
||||
echo "Install it with: npm install -g newman"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if collection exists
|
||||
if [ ! -f "$COLLECTION" ]; then
|
||||
echo -e "${RED}Error: Collection file not found: $COLLECTION${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Create report directory
|
||||
mkdir -p "$REPORT_DIR"
|
||||
|
||||
# Load provider capabilities into globals (for consistency with v1 runner)
|
||||
GLOBALS_TMP=""
|
||||
if [ -f "$PROVIDER_CAPABILITIES_JSON" ] && command -v jq &>/dev/null; then
|
||||
GLOBALS_TMP=$(mktemp)
|
||||
trap 'rm -f "$GLOBALS_TMP"' EXIT
|
||||
jq -n --rawfile cap "$PROVIDER_CAPABILITIES_JSON" '{id: "bifrost-provider-capabilities", name: "Provider capabilities", values: [{key: "provider_capabilities", value: $cap, type: "default", enabled: true}]}' > "$GLOBALS_TMP"
|
||||
fi
|
||||
|
||||
# Parse remaining options
|
||||
VERBOSE=""
|
||||
REPORTERS="cli"
|
||||
BAIL=""
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--verbose)
|
||||
VERBOSE="--verbose"
|
||||
shift
|
||||
;;
|
||||
--html)
|
||||
REPORTERS="${REPORTERS},html"
|
||||
shift
|
||||
;;
|
||||
--json)
|
||||
REPORTERS="${REPORTERS},json"
|
||||
shift
|
||||
;;
|
||||
--bail)
|
||||
BAIL="--bail"
|
||||
shift
|
||||
;;
|
||||
*)
|
||||
echo -e "${RED}Unknown option: $1${NC}"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Resolve provider env file
|
||||
SINGLE_JSON_ENV=""
|
||||
if [ -n "$PROVIDER_ENV_FILE" ]; then
|
||||
if [ -f "$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json"
|
||||
else
|
||||
echo -e "${RED}Error: Could not find environment file for: $PROVIDER_ENV_FILE${NC}"
|
||||
echo "Searched:"
|
||||
echo " - $PROVIDER_ENV_FILE"
|
||||
echo " - $PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE"
|
||||
echo " - $PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Default to openai if no env specified
|
||||
if [ -z "$SINGLE_JSON_ENV" ]; then
|
||||
if [ -f "$PROVIDER_CONFIG_DIR/bifrost-v1-openai.postman_environment.json" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/bifrost-v1-openai.postman_environment.json"
|
||||
echo -e "${YELLOW}No --env specified, using openai${NC}"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Build Newman command
|
||||
cmd=(newman run "$COLLECTION")
|
||||
[ -n "$GLOBALS_TMP" ] && [ -f "$GLOBALS_TMP" ] && cmd+=(-g "$GLOBALS_TMP")
|
||||
[ -n "$SINGLE_JSON_ENV" ] && [ -f "$SINGLE_JSON_ENV" ] && cmd+=(-e "$SINGLE_JSON_ENV")
|
||||
|
||||
# Pass enforce_auth when --enforce-auth was set
|
||||
if [ -n "$ENFORCE_AUTH" ]; then
|
||||
cmd+=(--env-var "enforce_auth=1")
|
||||
echo -e "Mode: ${YELLOW}enforce_auth=1${NC} (without-VK requests expect 401)"
|
||||
fi
|
||||
|
||||
# Base URL override
|
||||
base_url="${BIFROST_BASE_URL:-http://localhost:8080}"
|
||||
cmd+=(--env-var "base_url=$base_url")
|
||||
|
||||
cmd+=(--timeout-script 120000 --timeout 900000)
|
||||
cmd+=(-r "$REPORTERS")
|
||||
|
||||
if [[ "$REPORTERS" == *"html"* ]]; then
|
||||
cmd+=(--reporter-html-export "$REPORT_DIR/report.html")
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"json"* ]]; then
|
||||
cmd+=(--reporter-json-export "$REPORT_DIR/report.json")
|
||||
fi
|
||||
[ -n "$VERBOSE" ] && cmd+=("$VERBOSE")
|
||||
[ -n "$BAIL" ] && cmd+=("$BAIL")
|
||||
|
||||
echo -e "Configuration:"
|
||||
echo -e " Collection: ${YELLOW}$COLLECTION${NC}"
|
||||
echo -e " Base URL: ${YELLOW}$base_url${NC}"
|
||||
if [ -n "$SINGLE_JSON_ENV" ]; then
|
||||
echo -e " Env: ${YELLOW}$SINGLE_JSON_ENV${NC}"
|
||||
fi
|
||||
echo -e " Reports: ${YELLOW}$REPORT_DIR${NC}"
|
||||
echo ""
|
||||
echo -e "${GREEN}Running tests...${NC}"
|
||||
echo ""
|
||||
|
||||
set +e
|
||||
"${cmd[@]}"
|
||||
EXIT_CODE=$?
|
||||
set -e
|
||||
|
||||
echo ""
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All VK auth tests passed!${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Some tests failed${NC}"
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"html"* ]] || [[ "$REPORTERS" == *"json"* ]]; then
|
||||
echo ""
|
||||
echo -e "Reports saved to: ${YELLOW}$REPORT_DIR${NC}"
|
||||
ls -lh "$REPORT_DIR" 2>/dev/null | tail -n +2
|
||||
fi
|
||||
|
||||
exit $EXIT_CODE
|
||||
161
tests/e2e/api/runners/individual/run-newman-vk-routing-tests.sh
Executable file
161
tests/e2e/api/runners/individual/run-newman-vk-routing-tests.sh
Executable file
@@ -0,0 +1,161 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Bifrost V1 VK Routing Newman Test Runner
|
||||
# Runs governance routing tests. Requires governance plugin.
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
API_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
cd "$API_DIR"
|
||||
|
||||
COLLECTION="collections/bifrost-v1-vk-routing.postman_collection.json"
|
||||
REPORT_DIR="newman-reports/vk-routing"
|
||||
PROVIDER_CONFIG_DIR="provider_config"
|
||||
PROVIDER_CAPABILITIES_JSON="provider-capabilities.json"
|
||||
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
PROVIDER_ENV_FILE=""
|
||||
ARGS=()
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--env)
|
||||
if [[ -z "${2:-}" || "${2:-}" == --* ]]; then
|
||||
echo -e "${RED}Error: --env requires a value${NC}"
|
||||
exit 1
|
||||
fi
|
||||
PROVIDER_ENV_FILE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--help)
|
||||
echo "Usage: $0 [OPTIONS]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --env <provider> Postman env path or provider name"
|
||||
echo " --verbose Show detailed output"
|
||||
echo " --html Generate HTML report"
|
||||
echo " --json Generate JSON report"
|
||||
echo " --bail Stop on first failure"
|
||||
echo " --help Show this help message"
|
||||
echo ""
|
||||
echo "Prerequisites: Governance plugin must be configured."
|
||||
echo "Environment Variables:"
|
||||
echo " BIFROST_BASE_URL Override base URL (default: http://localhost:8080)"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
ARGS+=("$1")
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
set -- "${ARGS[@]}"
|
||||
|
||||
echo -e "${GREEN}==============================================${NC}"
|
||||
echo -e "${GREEN}Bifrost V1 VK Routing Test Runner${NC}"
|
||||
echo -e "${GREEN}==============================================${NC}"
|
||||
echo ""
|
||||
|
||||
if ! command -v newman &> /dev/null; then
|
||||
echo -e "${RED}Error: Newman is not installed${NC}"
|
||||
echo "Install it with: npm install -g newman"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -f "$COLLECTION" ]; then
|
||||
echo -e "${RED}Error: Collection file not found: $COLLECTION${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$REPORT_DIR"
|
||||
|
||||
GLOBALS_TMP=""
|
||||
if [ -f "$PROVIDER_CAPABILITIES_JSON" ] && command -v jq &>/dev/null; then
|
||||
GLOBALS_TMP=$(mktemp)
|
||||
trap 'rm -f "$GLOBALS_TMP"' EXIT
|
||||
jq -n --rawfile cap "$PROVIDER_CAPABILITIES_JSON" '{id: "bifrost-provider-capabilities", name: "Provider capabilities", values: [{key: "provider_capabilities", value: $cap, type: "default", enabled: true}]}' > "$GLOBALS_TMP"
|
||||
fi
|
||||
|
||||
VERBOSE=""
|
||||
REPORTERS="cli"
|
||||
BAIL=""
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--verbose) VERBOSE="--verbose"; shift ;;
|
||||
--html) REPORTERS="${REPORTERS},html"; shift ;;
|
||||
--json) REPORTERS="${REPORTERS},json"; shift ;;
|
||||
--bail) BAIL="--bail"; shift ;;
|
||||
*) echo -e "${RED}Unknown option: $1${NC}"; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
SINGLE_JSON_ENV=""
|
||||
if [ -n "$PROVIDER_ENV_FILE" ]; then
|
||||
if [ -f "$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json"
|
||||
else
|
||||
echo -e "${RED}Error: Could not find environment file for: $PROVIDER_ENV_FILE${NC}"
|
||||
echo "Searched:"
|
||||
echo " - $PROVIDER_ENV_FILE"
|
||||
echo " - $PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE"
|
||||
echo " - $PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -z "$SINGLE_JSON_ENV" ]; then
|
||||
if [ -f "$PROVIDER_CONFIG_DIR/bifrost-v1-openai.postman_environment.json" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/bifrost-v1-openai.postman_environment.json"
|
||||
echo -e "${YELLOW}No --env specified, using openai${NC}"
|
||||
fi
|
||||
fi
|
||||
|
||||
cmd=(newman run "$COLLECTION")
|
||||
[ -n "$GLOBALS_TMP" ] && [ -f "$GLOBALS_TMP" ] && cmd+=(-g "$GLOBALS_TMP")
|
||||
[ -n "$SINGLE_JSON_ENV" ] && [ -f "$SINGLE_JSON_ENV" ] && cmd+=(-e "$SINGLE_JSON_ENV")
|
||||
base_url="${BIFROST_BASE_URL:-http://localhost:8080}"
|
||||
cmd+=(--env-var "base_url=$base_url")
|
||||
cmd+=(--timeout-script 120000 --timeout 900000)
|
||||
cmd+=(-r "$REPORTERS")
|
||||
[[ "$REPORTERS" == *"html"* ]] && cmd+=(--reporter-html-export "$REPORT_DIR/report.html")
|
||||
[[ "$REPORTERS" == *"json"* ]] && cmd+=(--reporter-json-export "$REPORT_DIR/report.json")
|
||||
[ -n "$VERBOSE" ] && cmd+=("$VERBOSE")
|
||||
[ -n "$BAIL" ] && cmd+=("$BAIL")
|
||||
|
||||
echo -e "Configuration:"
|
||||
echo -e " Collection: ${YELLOW}$COLLECTION${NC}"
|
||||
echo -e " Base URL: ${YELLOW}$base_url${NC}"
|
||||
if [ -n "$SINGLE_JSON_ENV" ]; then
|
||||
echo -e " Env: ${YELLOW}$SINGLE_JSON_ENV${NC}"
|
||||
fi
|
||||
echo -e " Reports: ${YELLOW}$REPORT_DIR${NC}"
|
||||
echo ""
|
||||
echo -e "${GREEN}Running tests...${NC}"
|
||||
echo ""
|
||||
|
||||
set +e
|
||||
"${cmd[@]}"
|
||||
EXIT_CODE=$?
|
||||
set -e
|
||||
|
||||
echo ""
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All VK routing tests passed!${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Some tests failed${NC}"
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"html"* ]] || [[ "$REPORTERS" == *"json"* ]]; then
|
||||
echo ""
|
||||
echo -e "Reports saved to: ${YELLOW}$REPORT_DIR${NC}"
|
||||
ls -lh "$REPORT_DIR" 2>/dev/null | tail -n +2
|
||||
fi
|
||||
|
||||
exit $EXIT_CODE
|
||||
133
tests/e2e/api/runners/run-all-integration-tests.sh
Executable file
133
tests/e2e/api/runners/run-all-integration-tests.sh
Executable file
@@ -0,0 +1,133 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Bifrost All Integration Tests Runner
|
||||
# This script runs all integration test suites sequentially and aggregates results
|
||||
|
||||
set -e
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
# Colors for output
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Print banner
|
||||
echo -e "${BLUE}========================================${NC}"
|
||||
echo -e "${BLUE}Bifrost All Integration Tests Runner${NC}"
|
||||
echo -e "${BLUE}========================================${NC}"
|
||||
echo ""
|
||||
|
||||
|
||||
# Parse command line arguments
|
||||
ARGS=()
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--help)
|
||||
echo "Usage: $0 [OPTIONS]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --verbose Show detailed output"
|
||||
echo " --html Generate HTML reports"
|
||||
echo " --json Generate JSON reports"
|
||||
echo " --all-reports Generate all report types"
|
||||
echo " --env <provider> Run tests with specific provider only"
|
||||
echo " --help Show this help message"
|
||||
echo ""
|
||||
echo "This script runs all integration test collections:"
|
||||
echo " 1. OpenAI Integration"
|
||||
echo " 2. Anthropic Integration"
|
||||
echo " 3. Bedrock Integration"
|
||||
echo " 4. Composite Integrations (GenAI, Cohere, LiteLLM, LangChain, PydanticAI)"
|
||||
echo ""
|
||||
echo "Examples:"
|
||||
echo " $0 # Run all tests for all providers"
|
||||
echo " $0 --env openai # Run all tests with OpenAI provider only"
|
||||
echo " $0 --html --verbose # Verbose with HTML reports"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
ARGS+=("$1")
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Test scripts
|
||||
TEST_SCRIPTS=(
|
||||
"run-newman-openai-integration.sh"
|
||||
"run-newman-anthropic-integration.sh"
|
||||
"run-newman-bedrock-integration.sh"
|
||||
"run-newman-composite-integration.sh"
|
||||
)
|
||||
|
||||
# Test names for display
|
||||
TEST_NAMES=(
|
||||
"OpenAI Integration"
|
||||
"Anthropic Integration"
|
||||
"Bedrock Integration"
|
||||
"Composite Integrations"
|
||||
)
|
||||
|
||||
# Track results
|
||||
FAILED_TESTS=()
|
||||
PASSED_COUNT=0
|
||||
FAILED_COUNT=0
|
||||
|
||||
echo -e "${GREEN}Running ${#TEST_SCRIPTS[@]} integration test suites...${NC}"
|
||||
echo ""
|
||||
|
||||
# Run each test suite
|
||||
for i in "${!TEST_SCRIPTS[@]}"; do
|
||||
script="${TEST_SCRIPTS[$i]}"
|
||||
name="${TEST_NAMES[$i]}"
|
||||
|
||||
echo -e "${BLUE}========================================${NC}"
|
||||
echo -e "${BLUE}[$((i+1))/${#TEST_SCRIPTS[@]}] Running ${name}${NC}"
|
||||
echo -e "${BLUE}========================================${NC}"
|
||||
echo ""
|
||||
|
||||
script_path="$SCRIPT_DIR/individual/$script"
|
||||
if [ -f "$script_path" ]; then
|
||||
if (cd "$SCRIPT_DIR/individual" && "./$script" "${ARGS[@]}"); then
|
||||
echo ""
|
||||
echo -e "${GREEN}✓ ${name} PASSED${NC}"
|
||||
PASSED_COUNT=$((PASSED_COUNT + 1))
|
||||
else
|
||||
echo ""
|
||||
echo -e "${RED}✗ ${name} FAILED${NC}"
|
||||
FAILED_TESTS+=("$name")
|
||||
FAILED_COUNT=$((FAILED_COUNT + 1))
|
||||
fi
|
||||
else
|
||||
echo -e "${RED}Error: Test script not found: $script_path${NC}"
|
||||
FAILED_TESTS+=("$name (script not found)")
|
||||
FAILED_COUNT=$((FAILED_COUNT + 1))
|
||||
fi
|
||||
|
||||
echo ""
|
||||
done
|
||||
|
||||
# Print summary
|
||||
echo -e "${BLUE}========================================${NC}"
|
||||
echo -e "${BLUE}Test Summary${NC}"
|
||||
echo -e "${BLUE}========================================${NC}"
|
||||
echo ""
|
||||
echo -e "Total test suites: ${#TEST_SCRIPTS[@]}"
|
||||
echo -e "${GREEN}Passed: ${PASSED_COUNT}${NC}"
|
||||
echo -e "${RED}Failed: ${FAILED_COUNT}${NC}"
|
||||
echo ""
|
||||
|
||||
if [ ${FAILED_COUNT} -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All integration test suites passed!${NC}"
|
||||
exit 0
|
||||
else
|
||||
echo -e "${RED}✗ The following test suites failed:${NC}"
|
||||
for test in "${FAILED_TESTS[@]}"; do
|
||||
echo -e " ${RED}- ${test}${NC}"
|
||||
done
|
||||
echo ""
|
||||
echo -e "${YELLOW}Check individual test reports in newman-reports/ directories${NC}"
|
||||
exit 1
|
||||
fi
|
||||
293
tests/e2e/api/runners/run-newman-api-tests.sh
Executable file
293
tests/e2e/api/runners/run-newman-api-tests.sh
Executable file
@@ -0,0 +1,293 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Bifrost API Management & Health Tests
|
||||
# This script runs tests for /api/* and /health endpoints
|
||||
|
||||
set -e
|
||||
set -o pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
API_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
|
||||
# Configuration
|
||||
COLLECTION="$API_DIR/collections/bifrost-api-management.postman_collection.json"
|
||||
REPORT_DIR="$API_DIR/newman-reports/api-management"
|
||||
|
||||
# Colors for output
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Print banner
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
echo -e "${GREEN}Bifrost API Management & Health Tests${NC}"
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
echo ""
|
||||
|
||||
# Check if Newman is installed
|
||||
if ! command -v newman &> /dev/null; then
|
||||
echo -e "${RED}Error: Newman is not installed${NC}"
|
||||
echo "Install it with: npm install -g newman"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if collection exists
|
||||
if [ ! -f "$COLLECTION" ]; then
|
||||
echo -e "${RED}Error: Collection file not found: $COLLECTION${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Create report directory and log directory
|
||||
mkdir -p "$REPORT_DIR"
|
||||
LOG_DIR="$REPORT_DIR/parallel_logs"
|
||||
mkdir -p "$LOG_DIR"
|
||||
|
||||
# Parse command line arguments
|
||||
VERBOSE="--verbose"
|
||||
REPORTERS="cli"
|
||||
BAIL=""
|
||||
DB_VERIFY=""
|
||||
DB_URL="${BIFROST_DB_URL:-}"
|
||||
LOGS_DB_URL="${BIFROST_LOGS_DB_URL:-}"
|
||||
DB_CONFIG_PATH=""
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--verbose)
|
||||
VERBOSE="--verbose"
|
||||
shift
|
||||
;;
|
||||
--no-verbose)
|
||||
VERBOSE=""
|
||||
shift
|
||||
;;
|
||||
--html)
|
||||
REPORTERS="${REPORTERS},html"
|
||||
shift
|
||||
;;
|
||||
--json)
|
||||
REPORTERS="${REPORTERS},json"
|
||||
shift
|
||||
;;
|
||||
--all-reports)
|
||||
REPORTERS="cli,html,json"
|
||||
shift
|
||||
;;
|
||||
--bail)
|
||||
BAIL="--bail"
|
||||
shift
|
||||
;;
|
||||
--db-verify)
|
||||
DB_VERIFY="1"
|
||||
shift
|
||||
;;
|
||||
--db-url)
|
||||
DB_URL="$2"
|
||||
shift 2
|
||||
;;
|
||||
--logs-db-url)
|
||||
LOGS_DB_URL="$2"
|
||||
shift 2
|
||||
;;
|
||||
--config-path)
|
||||
DB_CONFIG_PATH="$2"
|
||||
shift 2
|
||||
;;
|
||||
--help)
|
||||
echo "Usage: $0 [OPTIONS]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --verbose Show detailed output (enabled by default)"
|
||||
echo " --no-verbose Disable verbose output"
|
||||
echo " --html Generate HTML report"
|
||||
echo " --json Generate JSON report"
|
||||
echo " --all-reports Generate all report types"
|
||||
echo " --bail Stop on first failure"
|
||||
echo " --db-verify Enable DB verification reporter (PostgreSQL or SQLite)"
|
||||
echo " --db-url <dsn> Explicit main DB connection string (overrides auto-detection)"
|
||||
echo " --logs-db-url <dsn> Explicit logs DB url (also reads BIFROST_LOGS_DB_URL; auto-detected)"
|
||||
echo " PostgreSQL: postgresql://user:pass@host:port/db"
|
||||
echo " SQLite: sqlite:///path/to/file.db"
|
||||
echo " --config-path <p> Path to Bifrost config.json for auto DB detection"
|
||||
echo " (default: ./config.json; also reads BIFROST_CONFIG_PATH env)"
|
||||
echo " --help Show this help message"
|
||||
echo ""
|
||||
echo "Examples:"
|
||||
echo " $0 # Run API management tests"
|
||||
echo " $0 --html # Run with HTML report"
|
||||
echo " $0 --verbose # Run with verbose output"
|
||||
echo " $0 --db-verify # Run with DB verification"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo -e "${RED}Unknown option: $1${NC}"
|
||||
echo "Use --help for usage information"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
echo -e "Configuration:"
|
||||
echo -e " Collection: ${YELLOW}$COLLECTION${NC}"
|
||||
echo -e " Reports: ${YELLOW}$REPORT_DIR${NC}"
|
||||
echo -e " Verbose: ${YELLOW}$([ -n "$VERBOSE" ] && echo "enabled" || echo "disabled")${NC}"
|
||||
if [ -n "$DB_VERIFY" ]; then
|
||||
if [ -n "$DB_URL" ]; then
|
||||
echo -e " DB Verify: ${YELLOW}enabled (url: $DB_URL)${NC}"
|
||||
elif [ -n "$DB_CONFIG_PATH" ]; then
|
||||
echo -e " DB Verify: ${YELLOW}enabled (config: $DB_CONFIG_PATH)${NC}"
|
||||
else
|
||||
echo -e " DB Verify: ${YELLOW}enabled (auto-detect from ./config.json)${NC}"
|
||||
fi
|
||||
else
|
||||
echo -e " DB Verify: ${YELLOW}disabled${NC}"
|
||||
fi
|
||||
# Repo root (tests/e2e/api -> ../../..)
|
||||
BIFROST_ROOT="$(cd "$API_DIR/../../.." && pwd)"
|
||||
PLUGIN_DIR="$BIFROST_ROOT/examples/plugins/hello-world"
|
||||
PLUGIN_SO="$PLUGIN_DIR/build/hello-world.so"
|
||||
|
||||
# Build hello-world plugin and resolve absolute path for plugin_path (before any test infra)
|
||||
if [ -d "$PLUGIN_DIR" ] && [ -f "$PLUGIN_DIR/Makefile" ]; then
|
||||
echo "Building hello-world plugin..."
|
||||
(cd "$PLUGIN_DIR" && make build) 2>/dev/null || (cd "$PLUGIN_DIR" && make dev) 2>/dev/null || true
|
||||
if [ -f "$PLUGIN_SO" ]; then
|
||||
PLUGIN_PATH_ABS="$(cd "$(dirname "$PLUGIN_SO")" && pwd)/$(basename "$PLUGIN_SO")"
|
||||
echo " Plugin: $PLUGIN_PATH_ABS"
|
||||
else
|
||||
PLUGIN_PATH_ABS=""
|
||||
fi
|
||||
else
|
||||
PLUGIN_PATH_ABS=""
|
||||
fi
|
||||
|
||||
# ── http-no-ping-server (MCP HTTP server on :3001) ───────────────────────────
|
||||
HTTP_SERVER_DIR="$BIFROST_ROOT/examples/mcps/http-no-ping-server"
|
||||
HTTP_SERVER_BIN="$HTTP_SERVER_DIR/http-server"
|
||||
HTTP_SERVER_PID=""
|
||||
|
||||
start_http_mcp_server() {
|
||||
# Skip if something is already listening on 3001
|
||||
if lsof -ti tcp:3001 &>/dev/null 2>&1; then
|
||||
echo " http-no-ping-server: port 3001 already in use, skipping start"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [ ! -d "$HTTP_SERVER_DIR" ]; then
|
||||
echo " http-no-ping-server: directory not found ($HTTP_SERVER_DIR), skipping"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Build binary if missing
|
||||
if [ ! -f "$HTTP_SERVER_BIN" ]; then
|
||||
echo " Building http-no-ping-server..."
|
||||
(cd "$HTTP_SERVER_DIR" && CGO_ENABLED=0 go build -o http-server main.go) || {
|
||||
echo " http-no-ping-server: build failed, skipping"
|
||||
return 0
|
||||
}
|
||||
fi
|
||||
|
||||
echo " Starting http-no-ping-server on port 3001..."
|
||||
"$HTTP_SERVER_BIN" &
|
||||
HTTP_SERVER_PID=$!
|
||||
|
||||
# Wait up to 10 s for it to accept connections
|
||||
for i in $(seq 1 10); do
|
||||
sleep 1
|
||||
if lsof -ti tcp:3001 &>/dev/null 2>&1; then
|
||||
echo " http-no-ping-server ready (PID $HTTP_SERVER_PID)"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
|
||||
echo " WARNING: http-no-ping-server did not become ready in time"
|
||||
}
|
||||
|
||||
stop_http_mcp_server() {
|
||||
if [ -n "$HTTP_SERVER_PID" ] && kill -0 "$HTTP_SERVER_PID" 2>/dev/null; then
|
||||
echo "Stopping http-no-ping-server (PID $HTTP_SERVER_PID)..."
|
||||
kill "$HTTP_SERVER_PID" 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
|
||||
# Register teardown so the server is stopped even if the script exits early
|
||||
trap stop_http_mcp_server EXIT
|
||||
|
||||
echo "Setting up MCP test servers..."
|
||||
start_http_mcp_server
|
||||
echo ""
|
||||
echo ""
|
||||
echo -e "${GREEN}Running tests...${NC}"
|
||||
echo ""
|
||||
|
||||
# Add dbverify reporter if requested
|
||||
if [ -n "$DB_VERIFY" ]; then
|
||||
REPORTERS="$REPORTERS,dbverify"
|
||||
# Install dependencies for the dbverify reporter if not already present
|
||||
if [ ! -d "$API_DIR/node_modules" ]; then
|
||||
echo "Installing DB verify reporter dependencies..."
|
||||
(cd "$API_DIR" && npm install --silent)
|
||||
fi
|
||||
# Newman (global) resolves reporters via Node's module search. Prepend the
|
||||
# local node_modules so it can find newman-reporter-dbverify without a
|
||||
# global install.
|
||||
export NODE_PATH="$API_DIR/node_modules${NODE_PATH:+:$NODE_PATH}"
|
||||
fi
|
||||
|
||||
# Build Newman command
|
||||
cmd=(newman run "$COLLECTION" --timeout-script 120000 --timeout 900000 -r "$REPORTERS")
|
||||
|
||||
# Override plugin_path with resolved absolute path so Create Plugin / Get Plugin use the built .so
|
||||
# env-var takes precedence over collection variables in Newman's resolution order
|
||||
if [ -n "$PLUGIN_PATH_ABS" ]; then
|
||||
cmd+=(--env-var "plugin_path=$PLUGIN_PATH_ABS")
|
||||
fi
|
||||
|
||||
if [[ "$REPORTERS" == *"html"* ]]; then
|
||||
cmd+=(--reporter-html-export "$REPORT_DIR/report.html")
|
||||
fi
|
||||
|
||||
if [[ "$REPORTERS" == *"json"* ]]; then
|
||||
cmd+=(--reporter-json-export "$REPORT_DIR/report.json")
|
||||
fi
|
||||
|
||||
if [ -n "$DB_VERIFY" ]; then
|
||||
[ -n "$DB_URL" ] && cmd+=(--reporter-dbverify-db-url "$DB_URL")
|
||||
[ -n "$LOGS_DB_URL" ] && cmd+=(--reporter-dbverify-logs-db-url "$LOGS_DB_URL")
|
||||
[ -n "$DB_CONFIG_PATH" ] && cmd+=(--reporter-dbverify-config "$DB_CONFIG_PATH")
|
||||
fi
|
||||
|
||||
[ -n "$VERBOSE" ] && cmd+=("$VERBOSE")
|
||||
[ -n "$BAIL" ] && cmd+=("$BAIL")
|
||||
|
||||
# Run Newman and save output to log file while displaying to console (using tee)
|
||||
LOG_FILE="$LOG_DIR/api-management.log"
|
||||
|
||||
# Write resolved plugin path to log before running tests
|
||||
if [ -n "$PLUGIN_PATH_ABS" ]; then
|
||||
echo "[setup] plugin_path resolved to: $PLUGIN_PATH_ABS" | tee "$LOG_FILE"
|
||||
else
|
||||
echo "[setup] plugin_path not resolved (build may have failed)" | tee "$LOG_FILE"
|
||||
fi
|
||||
|
||||
set +e
|
||||
"${cmd[@]}" 2>&1 | tee -a "$LOG_FILE"
|
||||
EXIT_CODE=${PIPESTATUS[0]}
|
||||
set -e
|
||||
|
||||
echo ""
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All tests passed!${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Some tests failed${NC}"
|
||||
fi
|
||||
|
||||
if [[ "$REPORTERS" == *"html"* ]] || [[ "$REPORTERS" == *"json"* ]]; then
|
||||
echo ""
|
||||
echo -e "Reports saved to: ${YELLOW}$REPORT_DIR${NC}"
|
||||
ls -lh "$REPORT_DIR" 2>/dev/null | tail -n +2
|
||||
fi
|
||||
echo -e "Log saved to: ${YELLOW}$LOG_FILE${NC}"
|
||||
|
||||
exit $EXIT_CODE
|
||||
162
tests/e2e/api/runners/run-newman-inference-features-tests.sh
Executable file
162
tests/e2e/api/runners/run-newman-inference-features-tests.sh
Executable file
@@ -0,0 +1,162 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Bifrost V1 Inference with Bifrost Features Newman Test Runner
|
||||
# Runs combined: Async Inference, Fallbacks, Management Flows, Rate Limit, Session Stickiness, VK Routing.
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
API_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
cd "$API_DIR"
|
||||
|
||||
COLLECTION="collections/bifrost-v1-inference-features.postman_collection.json"
|
||||
REPORT_DIR="newman-reports/inference-features"
|
||||
PROVIDER_CONFIG_DIR="provider_config"
|
||||
PROVIDER_CAPABILITIES_JSON="provider-capabilities.json"
|
||||
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
PROVIDER_ENV_FILE=""
|
||||
ARGS=()
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--env)
|
||||
if [[ -z "${2:-}" || "${2:-}" == --* ]]; then
|
||||
echo -e "${RED}Error: --env requires a value${NC}"
|
||||
exit 1
|
||||
fi
|
||||
PROVIDER_ENV_FILE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--help)
|
||||
echo "Usage: $0 [OPTIONS]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --env <provider> Postman env path or provider name"
|
||||
echo " --verbose Show detailed output"
|
||||
echo " --html Generate HTML report"
|
||||
echo " --json Generate JSON report"
|
||||
echo " --bail Stop on first failure"
|
||||
echo " --help Show this help message"
|
||||
echo ""
|
||||
echo "Suites: Async Inference, Fallbacks, Management Flows, Rate Limit, Session Stickiness, VK Routing"
|
||||
echo "Prerequisites: governance plugin must be configured for management/rate-limit/routing suites."
|
||||
echo "Environment Variables:"
|
||||
echo " BIFROST_BASE_URL Override base URL (default: http://localhost:8080)"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
ARGS+=("$1")
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
set -- "${ARGS[@]}"
|
||||
|
||||
echo -e "${GREEN}==============================================${NC}"
|
||||
echo -e "${GREEN}Bifrost V1 Inference with Bifrost Features Test Runner${NC}"
|
||||
echo -e "${GREEN}==============================================${NC}"
|
||||
echo ""
|
||||
|
||||
if ! command -v newman &> /dev/null; then
|
||||
echo -e "${RED}Error: Newman is not installed${NC}"
|
||||
echo "Install it with: npm install -g newman"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -f "$COLLECTION" ]; then
|
||||
echo -e "${RED}Error: Collection file not found: $COLLECTION${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$REPORT_DIR"
|
||||
|
||||
GLOBALS_TMP=""
|
||||
if [ -f "$PROVIDER_CAPABILITIES_JSON" ] && command -v jq &>/dev/null; then
|
||||
GLOBALS_TMP=$(mktemp)
|
||||
trap 'rm -f "$GLOBALS_TMP"' EXIT
|
||||
jq -n --rawfile cap "$PROVIDER_CAPABILITIES_JSON" '{id: "bifrost-provider-capabilities", name: "Provider capabilities", values: [{key: "provider_capabilities", value: $cap, type: "default", enabled: true}]}' > "$GLOBALS_TMP"
|
||||
fi
|
||||
|
||||
VERBOSE=""
|
||||
REPORTERS="cli"
|
||||
BAIL=""
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--verbose) VERBOSE="--verbose"; shift ;;
|
||||
--html) REPORTERS="${REPORTERS},html"; shift ;;
|
||||
--json) REPORTERS="${REPORTERS},json"; shift ;;
|
||||
--bail) BAIL="--bail"; shift ;;
|
||||
*) echo -e "${RED}Unknown option: $1${NC}"; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
SINGLE_JSON_ENV=""
|
||||
if [ -n "$PROVIDER_ENV_FILE" ]; then
|
||||
if [ -f "$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json"
|
||||
else
|
||||
echo -e "${RED}Error: Could not find environment file for: $PROVIDER_ENV_FILE${NC}"
|
||||
echo "Searched:"
|
||||
echo " - $PROVIDER_ENV_FILE"
|
||||
echo " - $PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE"
|
||||
echo " - $PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -z "$SINGLE_JSON_ENV" ]; then
|
||||
if [ -f "$PROVIDER_CONFIG_DIR/bifrost-v1-openai.postman_environment.json" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/bifrost-v1-openai.postman_environment.json"
|
||||
echo -e "${YELLOW}No --env specified, using openai${NC}"
|
||||
fi
|
||||
fi
|
||||
|
||||
cmd=(newman run "$COLLECTION")
|
||||
[ -n "$GLOBALS_TMP" ] && [ -f "$GLOBALS_TMP" ] && cmd+=(-g "$GLOBALS_TMP")
|
||||
[ -n "$SINGLE_JSON_ENV" ] && [ -f "$SINGLE_JSON_ENV" ] && cmd+=(-e "$SINGLE_JSON_ENV")
|
||||
base_url="${BIFROST_BASE_URL:-http://localhost:8080}"
|
||||
cmd+=(--env-var "base_url=$base_url")
|
||||
cmd+=(--timeout-script 120000 --timeout 900000)
|
||||
cmd+=(-r "$REPORTERS")
|
||||
[[ "$REPORTERS" == *"html"* ]] && cmd+=(--reporter-html-export "$REPORT_DIR/report.html")
|
||||
[[ "$REPORTERS" == *"json"* ]] && cmd+=(--reporter-json-export "$REPORT_DIR/report.json")
|
||||
[ -n "$VERBOSE" ] && cmd+=("$VERBOSE")
|
||||
[ -n "$BAIL" ] && cmd+=("$BAIL")
|
||||
|
||||
echo -e "Configuration:"
|
||||
echo -e " Collection: ${YELLOW}$COLLECTION${NC}"
|
||||
echo -e " Base URL: ${YELLOW}$base_url${NC}"
|
||||
if [ -n "$SINGLE_JSON_ENV" ]; then
|
||||
echo -e " Env: ${YELLOW}$SINGLE_JSON_ENV${NC}"
|
||||
fi
|
||||
echo -e " Reports: ${YELLOW}$REPORT_DIR${NC}"
|
||||
echo ""
|
||||
echo -e "${GREEN}Running tests...${NC}"
|
||||
echo ""
|
||||
|
||||
set +e
|
||||
"${cmd[@]}"
|
||||
EXIT_CODE=$?
|
||||
set -e
|
||||
|
||||
echo ""
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All inference features tests passed!${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Some tests failed${NC}"
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"html"* ]] || [[ "$REPORTERS" == *"json"* ]]; then
|
||||
echo ""
|
||||
echo -e "Reports saved to: ${YELLOW}$REPORT_DIR${NC}"
|
||||
ls -lh "$REPORT_DIR" 2>/dev/null | tail -n +2
|
||||
fi
|
||||
|
||||
exit $EXIT_CODE
|
||||
406
tests/e2e/api/runners/run-newman-inference-tests.sh
Executable file
406
tests/e2e/api/runners/run-newman-inference-tests.sh
Executable file
@@ -0,0 +1,406 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Bifrost V1 API Newman Test Runner
|
||||
# This script runs the complete Bifrost V1 API test suite using Newman
|
||||
|
||||
set -e
|
||||
|
||||
# Run from script directory so paths to collection and provider-capabilities.json work
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
API_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
cd "$API_DIR"
|
||||
|
||||
# Configuration
|
||||
COLLECTION="collections/bifrost-v1-complete.postman_collection.json"
|
||||
REPORT_DIR="newman-reports/v1"
|
||||
PROVIDER_CONFIG_DIR="provider_config"
|
||||
PROVIDER_CAPABILITIES_JSON="provider-capabilities.json"
|
||||
|
||||
# Colors for output
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Detect if --env was passed (so we run single provider vs all providers)
|
||||
PROVIDER_ENV_FILE=""
|
||||
ARGS=()
|
||||
while [[ $# -gt 0 ]]; do
|
||||
if [[ "$1" == "--env" ]]; then
|
||||
if [[ -z "${2:-}" || "${2:-}" == --* ]]; then
|
||||
echo -e "${RED}Error: --env requires a value${NC}"
|
||||
exit 1
|
||||
fi
|
||||
PROVIDER_ENV_FILE="$2"
|
||||
shift 2
|
||||
else
|
||||
ARGS+=("$1")
|
||||
shift
|
||||
fi
|
||||
done
|
||||
set -- "${ARGS[@]}"
|
||||
|
||||
# Normalize CI for retry logic (accept 1 or true, case-insensitive)
|
||||
ci_normalized="$(printf '%s' "${CI:-}" | tr '[:upper:]' '[:lower:]')"
|
||||
|
||||
# Print banner
|
||||
echo -e "${GREEN}==============================================${NC}"
|
||||
if [ "$ci_normalized" = "1" ] || [ "$ci_normalized" = "true" ]; then
|
||||
echo -e "${GREEN}Bifrost V1 API Test Runner with retries: 10${NC}"
|
||||
else
|
||||
echo -e "${GREEN}Bifrost V1 API Test Runner${NC}"
|
||||
fi
|
||||
echo -e "${GREEN}==============================================${NC}"
|
||||
echo ""
|
||||
|
||||
# Check if Newman is installed
|
||||
if ! command -v newman &> /dev/null; then
|
||||
echo -e "${RED}Error: Newman is not installed${NC}"
|
||||
echo "Install it with: npm install -g newman"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if collection exists
|
||||
if [ ! -f "$COLLECTION" ]; then
|
||||
echo -e "${RED}Error: Collection file not found: $COLLECTION${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Create report directory
|
||||
mkdir -p "$REPORT_DIR"
|
||||
|
||||
# Load provider capabilities from provider-capabilities.json (single source of truth) into a Newman globals file
|
||||
if [ ! -f "$PROVIDER_CAPABILITIES_JSON" ]; then
|
||||
echo -e "${RED}Error: $PROVIDER_CAPABILITIES_JSON not found${NC}"
|
||||
exit 1
|
||||
fi
|
||||
if ! command -v jq &>/dev/null; then
|
||||
echo -e "${RED}Error: jq is required to load $PROVIDER_CAPABILITIES_JSON${NC}"
|
||||
exit 1
|
||||
fi
|
||||
GLOBALS_TMP=$(mktemp)
|
||||
trap 'rm -f "$GLOBALS_TMP"' EXIT
|
||||
jq -n --rawfile cap "$PROVIDER_CAPABILITIES_JSON" '{id: "bifrost-provider-capabilities", name: "Provider capabilities", values: [{key: "provider_capabilities", value: $cap, type: "default", enabled: true}]}' > "$GLOBALS_TMP"
|
||||
|
||||
# When no --env: resolve list of provider Postman env .json files (sorted), excluding sgl and ollama
|
||||
EXCLUDED_PROVIDERS="sgl ollama"
|
||||
if [ -z "$PROVIDER_ENV_FILE" ] && [ -d "$PROVIDER_CONFIG_DIR" ]; then
|
||||
PROVIDER_JSON_FILES=()
|
||||
while IFS= read -r -d '' f; do
|
||||
# basename: bifrost-v1-openai.postman_environment.json -> openai
|
||||
name="${f##*/}"
|
||||
name="${name#bifrost-v1-}"
|
||||
name="${name%.postman_environment.json}"
|
||||
skip=""
|
||||
for ex in $EXCLUDED_PROVIDERS; do
|
||||
if [ "$name" = "$ex" ]; then skip=1; break; fi
|
||||
done
|
||||
[ -z "$skip" ] && PROVIDER_JSON_FILES+=("$f")
|
||||
done < <(find "$PROVIDER_CONFIG_DIR" -maxdepth 1 -name "bifrost-v1-*.postman_environment.json" -print0 2>/dev/null | sort -z)
|
||||
fi
|
||||
|
||||
# Parse command line arguments
|
||||
FOLDER=""
|
||||
VERBOSE=""
|
||||
REPORTERS="cli"
|
||||
BAIL=""
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--folder)
|
||||
if [[ -z "${2:-}" || "${2:-}" == --* ]]; then
|
||||
echo -e "${RED}Error: --folder requires a value${NC}"
|
||||
exit 1
|
||||
fi
|
||||
FOLDER="$2"
|
||||
shift 2
|
||||
;;
|
||||
--verbose)
|
||||
VERBOSE="--verbose"
|
||||
shift
|
||||
;;
|
||||
--html)
|
||||
if [[ "$REPORTERS" == *"json"* ]]; then
|
||||
REPORTERS="cli,html,json"
|
||||
else
|
||||
REPORTERS="cli,html"
|
||||
fi
|
||||
shift
|
||||
;;
|
||||
--json)
|
||||
if [[ "$REPORTERS" == *"html"* ]]; then
|
||||
REPORTERS="cli,html,json"
|
||||
else
|
||||
REPORTERS="cli,json"
|
||||
fi
|
||||
shift
|
||||
;;
|
||||
--all-reports)
|
||||
REPORTERS="cli,html,json"
|
||||
shift
|
||||
;;
|
||||
--bail)
|
||||
BAIL="--bail"
|
||||
shift
|
||||
;;
|
||||
--help)
|
||||
echo "Usage: $0 [OPTIONS]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --folder <name> Run only tests in specified folder"
|
||||
echo " --verbose Show detailed output"
|
||||
echo " --html Generate HTML report"
|
||||
echo " --json Generate JSON report"
|
||||
echo " --all-reports Generate all report types"
|
||||
echo " --bail Stop on first failure"
|
||||
echo " --env <path> Postman env .json path or provider name (e.g. provider_config/bifrost-v1-openai.postman_environment.json or openai)"
|
||||
echo " --help Show this help message"
|
||||
echo ""
|
||||
echo "Environment Variables:"
|
||||
echo " CI=1 When set, each failing request is retried up to 3 times"
|
||||
echo " BIFROST_BASE_URL Override base URL (default: http://localhost:8080)"
|
||||
echo " BIFROST_PROVIDER Override provider (default: openai)"
|
||||
echo " BIFROST_MODEL Override model name (default: gpt-4o)"
|
||||
echo " BIFROST_CHAT_MODEL Override chat completions model (default: BIFROST_MODEL)"
|
||||
echo " BIFROST_TEXT_COMPLETION_MODEL Override text completions model (default: BIFROST_MODEL)"
|
||||
echo " BIFROST_RESPONSES_MODEL Override Responses API model (default: BIFROST_MODEL)"
|
||||
echo " BIFROST_EMBEDDING_MODEL Override embedding model (default: text-embedding-3-small)"
|
||||
echo " BIFROST_SPEECH_MODEL Override speech model (default: tts-1)"
|
||||
echo " BIFROST_TRANSCRIPTION_MODEL Override transcription model (default: whisper-1)"
|
||||
echo " BIFROST_IMAGE_MODEL Override image model (default: dall-e-3)"
|
||||
echo " AWS_S3_BUCKET For Bedrock: S3 bucket for file/batch (same as core tests)"
|
||||
echo " AWS_BEDROCK_ROLE_ARN For Bedrock: IAM role ARN for batch (same as core tests)"
|
||||
echo ""
|
||||
echo "Examples:"
|
||||
echo " $0 # Run collection for all providers (each provider_config/bifrost-v1-*.postman_environment.json)"
|
||||
echo " $0 --env openai # Run once with OpenAI provider only"
|
||||
echo " $0 --folder \"Chat Completions\" # Run specific folder"
|
||||
echo " $0 --html --verbose # Verbose with HTML report"
|
||||
echo " BIFROST_BASE_URL=http://api:8080 $0 # Custom base URL"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo -e "${RED}Unknown option: $1${NC}"
|
||||
echo "Use --help for usage information"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Build and run Newman once.
|
||||
# Optional second arg: path to Postman env .json file (e.g. provider_config/bifrost-v1-openai.postman_environment.json).
|
||||
# When given, uses only that env file; otherwise uses default env and BIFROST_* overrides.
|
||||
run_newman() {
|
||||
local -a cmd=(newman run "$COLLECTION" -g "$GLOBALS_TMP")
|
||||
if [ -n "${2:-}" ] && [ -f "${2}" ]; then
|
||||
cmd+=(-e "${2}")
|
||||
# Align with core Bedrock tests: pass AWS_S3_BUCKET / AWS_BEDROCK_ROLE_ARN when running with Bedrock env
|
||||
if [[ "${1:-}" == "bedrock" ]]; then
|
||||
[ -n "${AWS_S3_BUCKET:-}" ] && cmd+=(--env-var "s3_bucket=$AWS_S3_BUCKET" --env-var "s3_output_bucket=$AWS_S3_BUCKET" --env-var "output_s3_uri=s3://$AWS_S3_BUCKET/batch-output/")
|
||||
[ -n "${AWS_BEDROCK_ROLE_ARN:-}" ] && cmd+=(--env-var "role_arn=$AWS_BEDROCK_ROLE_ARN")
|
||||
fi
|
||||
else
|
||||
local base_url="${BIFROST_BASE_URL:-http://localhost:8080}"
|
||||
local provider="${BIFROST_PROVIDER:-openai}"
|
||||
local model="${BIFROST_MODEL:-gpt-4o}"
|
||||
local chat_model="${BIFROST_CHAT_MODEL:-$model}"
|
||||
local text_completion_model="${BIFROST_TEXT_COMPLETION_MODEL:-$model}"
|
||||
local responses_model="${BIFROST_RESPONSES_MODEL:-$model}"
|
||||
local embedding_model="${BIFROST_EMBEDDING_MODEL:-text-embedding-3-small}"
|
||||
local speech_model="${BIFROST_SPEECH_MODEL:-tts-1}"
|
||||
local transcription_model="${BIFROST_TRANSCRIPTION_MODEL:-whisper-1}"
|
||||
local image_model="${BIFROST_IMAGE_MODEL:-dall-e-3}"
|
||||
cmd+=(--env-var "base_url=$base_url" --env-var "provider=$provider" --env-var "model=$model" --env-var "chat_model=$chat_model" --env-var "text_completion_model=$text_completion_model" --env-var "responses_model=$responses_model" --env-var "embedding_model=$embedding_model" --env-var "speech_model=$speech_model" --env-var "transcription_model=$transcription_model" --env-var "image_model=$image_model")
|
||||
fi
|
||||
if [ "$ci_normalized" = "1" ] || [ "$ci_normalized" = "true" ]; then
|
||||
cmd+=(--env-var "CI=1")
|
||||
fi
|
||||
[ -n "$FOLDER" ] && cmd+=(--folder "$FOLDER")
|
||||
cmd+=(--timeout-script 120000 --timeout 900000)
|
||||
cmd+=(-r "$REPORTERS")
|
||||
if [[ "$REPORTERS" == *"html"* ]]; then
|
||||
cmd+=(--reporter-html-export "$REPORT_DIR/report_${1:-run}.html")
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"json"* ]]; then
|
||||
cmd+=(--reporter-json-export "$REPORT_DIR/report_${1:-run}.json")
|
||||
fi
|
||||
[ -n "$VERBOSE" ] && cmd+=("$VERBOSE")
|
||||
[ -n "$BAIL" ] && cmd+=("$BAIL")
|
||||
|
||||
"${cmd[@]}"
|
||||
}
|
||||
|
||||
# Run for a single provider (--env was passed: path to .json env or provider name)
|
||||
if [ -n "$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV=""
|
||||
if [ -f "$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/$PROVIDER_ENV_FILE"
|
||||
elif [ -f "$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json" ]; then
|
||||
SINGLE_JSON_ENV="$PROVIDER_CONFIG_DIR/bifrost-v1-${PROVIDER_ENV_FILE}.postman_environment.json"
|
||||
fi
|
||||
if [ -z "$SINGLE_JSON_ENV" ]; then
|
||||
echo -e "${RED}Error: Env file not found: $PROVIDER_ENV_FILE${NC}"
|
||||
echo "Use a path to a .json env (e.g. provider_config/bifrost-v1-openai.postman_environment.json) or provider name (e.g. openai)"
|
||||
exit 1
|
||||
fi
|
||||
SINGLE_PROVIDER_NAME="${SINGLE_JSON_ENV##*/}"
|
||||
SINGLE_PROVIDER_NAME="${SINGLE_PROVIDER_NAME#bifrost-v1-}"
|
||||
SINGLE_PROVIDER_NAME="${SINGLE_PROVIDER_NAME%.postman_environment.json}"
|
||||
echo -e "Configuration: ${YELLOW}$SINGLE_JSON_ENV${NC}"
|
||||
echo -e " Reports: ${YELLOW}$REPORT_DIR${NC}"
|
||||
echo ""
|
||||
echo -e "${GREEN}Running tests...${NC}"
|
||||
echo ""
|
||||
run_newman "$SINGLE_PROVIDER_NAME" "$SINGLE_JSON_ENV" && EXIT_CODE=0 || EXIT_CODE=$?
|
||||
echo ""
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All tests passed!${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Some tests failed${NC}"
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"html"* ]] || [[ "$REPORTERS" == *"json"* ]]; then
|
||||
echo ""
|
||||
echo -e "Reports saved to: ${YELLOW}$REPORT_DIR${NC}"
|
||||
ls -lh "$REPORT_DIR" 2>/dev/null | tail -n +2
|
||||
fi
|
||||
exit $EXIT_CODE
|
||||
fi
|
||||
|
||||
# Run for all providers (no --env)
|
||||
if [ -z "${PROVIDER_JSON_FILES+x}" ] || [ ${#PROVIDER_JSON_FILES[@]} -eq 0 ]; then
|
||||
echo -e "${YELLOW}No provider env .json files found in $PROVIDER_CONFIG_DIR/. Using default (openai).${NC}"
|
||||
echo -e "Configuration:"
|
||||
echo -e " Base URL: ${YELLOW}${BIFROST_BASE_URL:-http://localhost:8080}${NC}"
|
||||
echo -e " Provider: ${YELLOW}${BIFROST_PROVIDER:-openai}${NC}"
|
||||
echo -e " Reports: ${YELLOW}$REPORT_DIR${NC}"
|
||||
echo ""
|
||||
echo -e "${GREEN}Running tests...${NC}"
|
||||
echo ""
|
||||
set +e
|
||||
run_newman
|
||||
EXIT_CODE=$?
|
||||
set -e
|
||||
echo ""
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All tests passed!${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Some tests failed${NC}"
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"html"* ]] || [[ "$REPORTERS" == *"json"* ]]; then
|
||||
echo ""
|
||||
echo -e "Reports saved to: ${YELLOW}$REPORT_DIR${NC}"
|
||||
ls -lh "$REPORT_DIR" 2>/dev/null | tail -n +2
|
||||
fi
|
||||
exit $EXIT_CODE
|
||||
fi
|
||||
|
||||
PARALLEL_LOGS_DIR="$REPORT_DIR/parallel_logs"
|
||||
mkdir -p "$PARALLEL_LOGS_DIR"
|
||||
|
||||
# Print a one-line report for a provider from its Newman log and exit code
|
||||
print_provider_report() {
|
||||
local name="$1"
|
||||
local logfile="$2"
|
||||
local exitcode="$3"
|
||||
local failed_count=""
|
||||
local failed_tests=""
|
||||
if [ -f "$logfile" ]; then
|
||||
# Parse Newman summary table: assertions row, third column = failed count
|
||||
failed_count=$(grep "assertions" "$logfile" 2>/dev/null | awk -F'│' '{gsub(/^ *| *$/,"",$4); print $4}' | head -1)
|
||||
# Lines with " ✗ " are failed assertions; strip to get test name
|
||||
failed_tests=$(grep " ✗ " "$logfile" 2>/dev/null | sed 's/.*✗ */ - /' | sed 's/^ *//' | tr '\n' ' ' | sed 's/ $//')
|
||||
fi
|
||||
if [ "$exitcode" -eq 0 ]; then
|
||||
echo -e "${GREEN} ✓ $name: PASS${NC}"
|
||||
else
|
||||
echo -e "${RED} ✗ $name: FAIL${NC}"
|
||||
if [ -n "$failed_count" ] && [ "$failed_count" -gt 0 ] 2>/dev/null; then
|
||||
echo -e " ${RED}${failed_count} assertion(s) failed${NC}"
|
||||
fi
|
||||
if [ -n "$failed_tests" ]; then
|
||||
echo -e " Failed: $failed_tests"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
# Draw the provider status table (TABLE_LINES lines). Use after moving cursor up TABLE_LINES to refresh.
|
||||
draw_table() {
|
||||
printf '\033[2K%-16s %s\n' "Provider" "Status"
|
||||
for i in "${!NAMES[@]}"; do
|
||||
printf '\033[2K%-16s %b\n' "${NAMES[$i]}" "${STATUS[$i]}"
|
||||
done
|
||||
}
|
||||
|
||||
echo -e "Running tests for ${#PROVIDER_JSON_FILES[@]} provider(s) ${GREEN}in parallel${NC}. Reports: ${YELLOW}$REPORT_DIR${NC}"
|
||||
echo ""
|
||||
|
||||
# Run each provider in a background subshell; capture PID and log path per provider
|
||||
PIDS=()
|
||||
NAMES=()
|
||||
LOG_FILES=()
|
||||
for jsonfile in "${PROVIDER_JSON_FILES[@]}"; do
|
||||
name="${jsonfile##*/}"
|
||||
name="${name#bifrost-v1-}"
|
||||
name="${name%.postman_environment.json}"
|
||||
logfile="$PARALLEL_LOGS_DIR/${name}.log"
|
||||
LOG_FILES+=("$logfile")
|
||||
NAMES+=("$name")
|
||||
( run_newman "$name" "$jsonfile" ) > "$logfile" 2>&1 &
|
||||
PIDS+=($!)
|
||||
done
|
||||
|
||||
# Status for each provider: Pending, ✓ PASS, or ✗ FAIL (with color)
|
||||
STATUS=()
|
||||
for i in "${!PIDS[@]}"; do STATUS[$i]="${YELLOW}Pending${NC}"; done
|
||||
TABLE_LINES=$((${#NAMES[@]} + 1))
|
||||
|
||||
# Initial table
|
||||
draw_table
|
||||
|
||||
# Track which we've reaped (0 = pending, 1 = done)
|
||||
REAPED=()
|
||||
for i in "${!PIDS[@]}"; do REAPED[$i]=0; done
|
||||
|
||||
OVERALL_FAILED=0
|
||||
FAILED_NAMES=()
|
||||
|
||||
# As each provider finishes, update status and redraw table
|
||||
while true; do
|
||||
all_done=1
|
||||
for i in "${!PIDS[@]}"; do
|
||||
[ "${REAPED[$i]:-0}" -eq 1 ] && continue
|
||||
all_done=0
|
||||
if ! kill -0 "${PIDS[$i]}" 2>/dev/null; then
|
||||
exitcode=0; wait "${PIDS[$i]}" || exitcode=$?
|
||||
REAPED[$i]=1
|
||||
if [ "$exitcode" -eq 0 ]; then
|
||||
STATUS[$i]="${GREEN}✓ PASS${NC}"
|
||||
else
|
||||
OVERALL_FAILED=1
|
||||
FAILED_NAMES+=("${NAMES[$i]}")
|
||||
STATUS[$i]="${RED}✗ FAIL${NC}"
|
||||
fi
|
||||
# Move cursor up and redraw table
|
||||
printf '\033[%dA' "$TABLE_LINES"
|
||||
draw_table
|
||||
fi
|
||||
done
|
||||
[ "$all_done" -eq 1 ] && break
|
||||
sleep 0.3
|
||||
done
|
||||
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
if [ $OVERALL_FAILED -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All providers passed!${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ One or more providers had failures: ${FAILED_NAMES[*]}${NC}"
|
||||
fi
|
||||
if [[ "$REPORTERS" == *"html"* ]] || [[ "$REPORTERS" == *"json"* ]]; then
|
||||
echo ""
|
||||
echo -e "Reports saved to: ${YELLOW}$REPORT_DIR${NC}"
|
||||
ls -lh "$REPORT_DIR" 2>/dev/null | tail -n +2
|
||||
fi
|
||||
# Parallel logs persist in $PARALLEL_LOGS_DIR (overwritten per provider on each run)
|
||||
exit $OVERALL_FAILED
|
||||
Reference in New Issue
Block a user