예제 소스

daemon_core_harden_watchdog.py

#!/usr/bin/env python3
"""혼자 두는 동안 serve + 텍스트 코어 harden 가 죽으면 다시 살린다.

모델 재학습이 아니라 클라이언트 코어 강화(VLM vs VLM) 감시.

  .venv/bin/python scripts/daemon_core_harden_watchdog.py --foreground
"""

from __future__ import annotations

import os
import socket
import subprocess
import sys
import time
from datetime import datetime, timezone
from pathlib import Path
from zoneinfo import ZoneInfo

ROOT = Path(__file__).resolve().parents[1]
PY = str(ROOT / ".venv" / "bin" / "python")
OUT = ROOT / "logs" / "text_client_harden"
HEART = OUT / "HEARTBEAT.md"
LOG = OUT / "watchdog.log"
KST = ZoneInfo("Asia/Seoul")
PORT = 9319
SERVE_CMD = [
    PY,
    "-m",
    "vlm_server",
    "serve",
    "--host",
    "127.0.0.1",
    "--port",
    str(PORT),
    "--model",
    "********",
    "--lora-dir",
    "vlm_models/lora_mix_qwen_r5",
    "--text-only-move",
    "--board-repr",
    "aligned",
    "--score-guided",
    "--score-min-line",
    "3",
    "--illegal-retry-max",
    "16",
    "--illegal-retry-budget-s",
    "240",
    "--log-jsonl",
    "logs/vlm_eval_day/play_r5.jsonl",
]
HARDEN_CMD = [
    PY,
    "scripts/daemon_text_client_harden.py",
    "--foreground",
    "--port",
    str(PORT),
    "--max-games",
    "0",
    "--max-moves",
    "80",
    "--sleep-s",
    "0.5",
    "--timeout-s",
    "240",
    "--out-dir",
    "logs/text_client_harden",
]
SERVE_LOG = OUT / "serve.watchdog.log"
HARDEN_LOG = OUT / "harden.watchdog.log"


def now_kst() -> datetime:
    return datetime.now(KST)


def iso() -> str:
    return datetime.now(timezone.utc).isoformat()


def log(msg: str) -> None:
    line = f"[{now_kst():%Y-%m-%d %H:%M:%S}] {msg}"
    print(line, flush=True)
    OUT.mkdir(parents=True, exist_ok=True)
    with LOG.open("a", encoding="utf-8") as f:
        f.write(line + "\n")


def port_open(port: int = PORT) -> bool:
    try:
        with socket.create_connection(("127.0.0.1", port), timeout=1.5):
            return True
    except OSError:
        return False


def pgrep(pattern: str) -> list[int]:
    try:
        out = subprocess.check_output(["pgrep", "-f", pattern], text=True)
    except subprocess.CalledProcessError:
        return []
    pids = []
    for tok in out.split():
        try:
            pids.append(int(tok))
        except ValueError:
            pass
    me = os.getpid()
    return [p for p in pids if p != me]


def spawn(cmd: list[str], logfile: Path) -> subprocess.Popen:
    env = os.environ.copy()
    env["PYTHONUNBUFFERED"] = "1"
    env["HF_HOME"] = str(ROOT / "vlm_models")
    env["HF_HUB_CACHE"] = str(ROOT / "vlm_models")
    env["TOKENIZERS_PARALLELISM"] = "********"
    logfile.parent.mkdir(parents=True, exist_ok=True)
    fp = open(logfile, "a", encoding="utf-8")
    fp.write(f"\n===== spawn {now_kst():%Y-%m-%d %H:%M:%S} =====\n")
    fp.flush()
    return subprocess.Popen(
        cmd,
        cwd=str(ROOT),
        env=env,
        stdout=fp,
        stderr=subprocess.STDOUT,
        start_new_session=True,
    )


def games_count() -> int:
    p = OUT / "games.jsonl"
    if not p.exists():
        return 0
    n = 0
    with p.open(encoding="utf-8") as f:
        for line in f:
            if line.strip():
                n += 1
    return n


CORE_VERIFY_SCRIPT = """
from vlm_server.text_client_core import ClientCore, RejectReason, GameEndReason, CoreConfig
c=ClientCore()
c.board.play(3,3,1); c.to_play=1
assert not c.evaluate_move(3,3,is_pass=False).accept
e=None
for _ in range(3):
  e=c.on_reject(3,3,RejectReason.OCCUPIED)
assert e==GameEndReason.REJECT_LOOP
assert ClientCore().evaluate_move(0,0,is_pass=False).reason==RejectReason.LOW_LINE
assert CoreConfig().max_forbid==48 and CoreConfig().max_comm_retries==3
print('core_verify OK')
"""


def run_core_verify() -> bool:
    try:
        subprocess.check_call(
            [PY, "-c", CORE_VERIFY_SCRIPT],
            cwd=str(ROOT),
            stdout=subprocess.DEVNULL,
            stderr=subprocess.PIPE,
            text=True,
        )
        return True
    except subprocess.CalledProcessError as e:
        log(f"core_verify FAIL: {(e.stderr or '')[:200]}")
        return False


def write_heartbeat(*, serve_ok: bool, harden_ok: bool, note: str) -> None:
    n = games_count()
    issues = (OUT / "CORE_ISSUES.md").read_text(encoding="utf-8") if (OUT / "CORE_ISSUES.md").exists() else ""
    lines = [
        f"# Harden watchdog heartbeat — {now_kst():%Y-%m-%d %H:%M:%S KST}",
        "",
        f"- **serve :{PORT}**: `{'up' if serve_ok else 'DOWN'}`",
        f"- **harden daemon**: `{'up' if harden_ok else 'DOWN'}`",
        f"- **games.jsonl**: `{n}`",
        f"- **note**: {note}",
        "",
        "루프: **코어 검증 → VLM vs VLM 대국 → errors 집계 → 코어 수정** (LoRA 재학습 아님).",
        "",
        "## CORE_ISSUES (최근)",
        "",
        issues.strip() or "_없음_",
        "",
    ]
    HEART.write_text("\n".join(lines), encoding="utf-8")


def main() -> int:
    OUT.mkdir(parents=True, exist_ok=True)
    log("watchdog start — verify → harden loop")
    if run_core_verify():
        log("core_verify OK")
    else:
        log("core_verify FAILED (harden continues; fix text_client_core.py)")
    serve_restarts = 0
    harden_restarts = 0
    last_verify_games = games_count()
    while True:
        serve_ok = port_open()
        harden_ok = bool(pgrep("daemon_text_client_harden.py"))

        if not serve_ok:
            log("serve DOWN → restart")
            spawn(SERVE_CMD, SERVE_LOG)
            serve_restarts += 1
            deadline = time.time() + 240
            while time.time() < deadline and not port_open():
                time.sleep(2)
            serve_ok = port_open()
            log(f"serve {'UP' if serve_ok else 'STILL DOWN'} after restart #{serve_restarts}")

        if serve_ok and not harden_ok:
            log("harden DOWN → restart")
            spawn(HARDEN_CMD, HARDEN_LOG)
            harden_restarts += 1
            time.sleep(2)
            harden_ok = bool(pgrep("daemon_text_client_harden.py"))
            log(f"harden {'UP' if harden_ok else 'STILL DOWN'} after restart #{harden_restarts}")

        write_heartbeat(
            serve_ok=serve_ok,
            harden_ok=harden_ok,
            note=f"serve_restarts={serve_restarts} harden_restarts={harden_restarts}",
        )
        n = games_count()
        if n - last_verify_games >= 20:
            last_verify_games = n
            if run_core_verify():
                log(f"core_verify OK (after {n} games)")
            else:
                log(f"core_verify FAIL at {n} games — check CORE_ISSUES.md")
        time.sleep(30)


if __name__ == "__main__":
    try:
        raise SystemExit(main())
    except KeyboardInterrupt:
        log("watchdog interrupted")
        raise SystemExit(0)