#!/bin/bash
exec 1>/home/ubuntu/run_all_output.txt 2>&1
set -x

echo "STARTING at $(date)"

# Kill old GPU processes
nvidia-smi --query-compute-apps=pid --format=csv,noheader 2>/dev/null | xargs -r kill -9 2>/dev/null
pkill -9 -f "vllm" 2>/dev/null
pkill -9 -f "python3.*launch" 2>/dev/null
sleep 3

echo "=== GPU AFTER CLEANUP ==="
nvidia-smi

echo "=== LAUNCHING SERVER ==="
rm -f /home/ubuntu/vllm_server_log.txt
export VLLM_ATTENTION_BACKEND=TORCH_SDPA
/home/ubuntu/vllm_env/bin/python3 /home/ubuntu/launch_server.py > /home/ubuntu/vllm_server_log.txt 2>&1 &
SERVER_PID=$!
echo "Server PID: $SERVER_PID"

echo "=== WAITING 180s ==="
for i in $(seq 1 36); do
    sleep 5
    if ! kill -0 $SERVER_PID 2>/dev/null; then
        echo "Server process died at iteration $i"
        break
    fi
    echo "Alive at ${i}x5s = $((i*5))s"
done

echo "=== SERVER LOG ==="
cat /home/ubuntu/vllm_server_log.txt

echo "=== GPU STATUS ==="
nvidia-smi

echo "=== TEST ENDPOINT ==="
curl -s -m 10 http://localhost:8091/v1/audio/voices 2>&1 || echo "NOT RESPONDING"

echo "=== ALL DONE at $(date) ==="