예제 소스

run_vlm_eval_day.sh

#!/usr/bin/env bash
# Detached day-long text VLM self-play eval (until 19:00 local).
set -u
cd ~/project
export HF_HOME="$PWD/vlm_models"
export HF_HUB_CACHE="$PWD/vlm_models"
export TOKENIZERS_PARALLELISM=false
MODEL="********"
OUT=logs/vlm_eval_day
mkdir -p "$OUT" logs/vlm_servers
echo "[$(date)] runner start" >> "$OUT/watchdog.log"

start_server() {
  if lsof -tiTCP:9319 -sTCP:LISTEN >/dev/null 2>&1; then
    return 0
  fi
  echo "[$(date)] starting server" >> "$OUT/watchdog.log"
  .venv/bin/python -m vlm_server serve \
    --host 127.0.0.1 --port 9319 \
    --model "$MODEL" --lora-dir vlm_models/lora_baduk_v2 \
    --text-only-move --board-repr compact \
    --temperature 0.35 --max-new-tokens 64 --illegal-retry-max 3 \
    --log-jsonl logs/vlm_servers/selfplay_v2.jsonl \
    >> logs/vlm_servers/black_v2.log 2>&1 &
  for _ in $(seq 1 90); do
    if lsof -tiTCP:9319 -sTCP:LISTEN >/dev/null 2>&1; then
      echo "[$(date)] server up" >> "$OUT/watchdog.log"
      return 0
    fi
    sleep 2
  done
  echo "[$(date)] server failed to listen" >> "$OUT/watchdog.log"
  return 1
}

start_eval() {
  if pgrep -f 'vlm_server.eval_vlm_vs_vlm' >/dev/null 2>&1; then
    return 0
  fi
  echo "[$(date)] starting eval" >> "$OUT/watchdog.log"
  .venv/bin/python -m vlm_server.eval_vlm_vs_vlm \
    --black-port 9319 --white-port 9319 \
    --black-label lora_v2_compact --white-label lora_v2_compact \
    --until 19:00 --max-moves 160 --timeout-s 300 \
    --out-dir "$OUT" \
    >> "$OUT/eval_runner.log" 2>&1 &
}

# kill stale
pkill -f 'vlm_server.eval_vlm_vs_vlm' 2>/dev/null || true
for p in 9319 9320; do
  pid=$(lsof -tiTCP:$p -sTCP:LISTEN 2>/dev/null || true)
  [ -n "${pid:-}" ] && kill -9 $pid 2>/dev/null || true
done
sleep 1

start_server || exit 1
start_eval

while [ "$(date +%H%M)" -lt 1900 ]; do
  sleep 45
  if ! lsof -tiTCP:9319 -sTCP:LISTEN >/dev/null 2>&1; then
    echo "[$(date)] server dead → restart" >> "$OUT/watchdog.log"
    pkill -f 'vlm_server.eval_vlm_vs_vlm' 2>/dev/null || true
    start_server || continue
    start_eval
  elif ! pgrep -f 'vlm_server.eval_vlm_vs_vlm' >/dev/null 2>&1; then
    echo "[$(date)] eval dead → restart" >> "$OUT/watchdog.log"
    start_eval
  fi
done

echo "[$(date)] deadline reached, stopping" >> "$OUT/watchdog.log"
pkill -f 'vlm_server.eval_vlm_vs_vlm' 2>/dev/null || true
# leave server running for inspection
exit 0