예제 소스
바둑 on-policy 학습: 대국 로그를 KataGo 라벨로 바꾸기 · 파일 169개
daemon_mariago_web_watchdog.py
#!/usr/bin/env python3
"""MariaGO 웹 스택 감시: Next(:3000) + WS bridge(:9320) 죽으면 재기동.
서울 등 자리 비울 때 Node가 내려가지 않게 유지.
.venv/bin/python scripts/daemon_mariago_web_watchdog.py
"""
from __future__ import annotations
import os
import signal
import socket
import subprocess
import sys
import time
import urllib.error
import urllib.request
from datetime import datetime, timezone
from pathlib import Path
from zoneinfo import ZoneInfo
ROOT = Path(__file__).resolve().parents[1]
WEB = ROOT / "mariago-web"
PY = str(ROOT / ".venv" / "bin" / "python")
OUT = ROOT / "logs" / "mariago_web_watch"
HEART = OUT / "HEARTBEAT.md"
LOG = OUT / "watchdog.log"
NEXT_LOG = OUT / "next.watchdog.log"
BRIDGE_LOG = OUT / "bridge.watchdog.log"
RESTART_FLAG = OUT / "RESTART_REQUESTED"
KST = ZoneInfo("Asia/Seoul")
NEXT_PORT = 3000
BRIDGE_PORT = 9320
VLM_PORT = 9319
GEMMA_PORT = 9341
GEMMA_LOG = OUT / "gemma.watchdog.log"
GEMMA_MODEL = "********"
GEMMA_LORA = "********"
GEMMA_HOLD = OUT / "GEMMA_HOLD"
GEMMA_LORA_FILE = OUT / "gemma_lora.txt"
def gemma_lora_path() -> str:
try:
text = GEMMA_LORA_FILE.read_text(encoding="utf-8").strip()
except OSError:
text = ""
return text or GEMMA_LORA
def gemma_board_repr(lora: str) -> str:
"""어댑터 폴더의 serve_board_repr.txt (학습 때 쓴 보드 프롬프트 형식). 없으면 aligned."""
try:
text = (ROOT / lora / "serve_board_repr.txt").read_text(encoding="utf-8").strip()
except OSError:
text = ""
return text or "aligned"
POLL_S = 20
HTTP_TIMEOUT_S = 3.0
RESTART_WAIT_S = 90
# nvm node (절대 경로 — 셸 없이 재기동)
NODE_BIN = Path.home() / ".nvm" / "versions" / "node" / "v24.14.1" / "bin"
NPM = str(NODE_BIN / "npm")
def now_kst() -> datetime:
return datetime.now(KST)
def log(msg: str) -> None:
line = f"[{now_kst():%Y-%m-%d %H:%M:%S}] {msg}"
print(line, flush=True)
OUT.mkdir(parents=True, exist_ok=True)
with LOG.open("a", encoding="utf-8") as f:
f.write(line + "\n")
def port_open(port: int, host: str = "127.0.0.1") -> bool:
try:
with socket.create_connection((host, port), timeout=1.5):
return True
except OSError:
return False
def http_ok(url: str, expect: int = 200) -> bool:
try:
req = urllib.request.Request(url, method="GET")
with urllib.request.urlopen(req, timeout=HTTP_TIMEOUT_S) as resp:
return int(resp.status) == expect
except (urllib.error.URLError, TimeoutError, OSError, ValueError):
return False
def next_healthy() -> bool:
if not port_open(NEXT_PORT, host="127.0.0.1") and not port_open(NEXT_PORT, host="::1"):
# *:3000 은 IPv6 only일 수 있음 — HTTP로 최종 판정
pass
return http_ok(f"http://127.0.0.1:{NEXT_PORT}/login", 200) or http_ok(
f"http://127.0.0.1:{NEXT_PORT}/play", 200
)
def bridge_healthy() -> bool:
return port_open(BRIDGE_PORT)
def pgrep(pattern: str) -> list[int]:
try:
out = subprocess.check_output(["pgrep", "-f", pattern], text=True)
except subprocess.CalledProcessError:
return []
me = os.getpid()
pids = []
for tok in out.split():
try:
p = int(tok)
except ValueError:
continue
if p != me:
pids.append(p)
return pids
def kill_pids(pids: list[int], label: str) -> None:
if not pids:
return
log(f"kill {label}: {pids}")
for p in pids:
try:
os.kill(p, signal.SIGTERM)
except ProcessLookupError:
pass
time.sleep(2)
for p in pids:
try:
os.kill(p, signal.SIGKILL)
except ProcessLookupError:
pass
def free_port(port: int) -> None:
try:
out = subprocess.check_output(["lsof", "-nP", f"-iTCP:{port}", "-sTCP:LISTEN", "-t"], text=True)
except subprocess.CalledProcessError:
return
pids = []
for tok in out.split():
try:
pids.append(int(tok))
except ValueError:
pass
kill_pids(pids, f"port:{port}")
def spawn(cmd: list[str], cwd: Path, logfile: Path, extra_env: dict | None = None) -> subprocess.Popen:
env = os.environ.copy()
env["PYTHONUNBUFFERED"] = "1"
# nvm node PATH
path_parts = [str(NODE_BIN)]
if env.get("PATH"):
path_parts.append(env["PATH"])
env["PATH"] = ":".join(path_parts)
if extra_env:
env.update(extra_env)
logfile.parent.mkdir(parents=True, exist_ok=True)
fp = open(logfile, "a", encoding="utf-8")
fp.write(f"\n===== spawn {now_kst():%Y-%m-%d %H:%M:%S} =====\n")
fp.write(" ".join(cmd) + "\n")
fp.flush()
return subprocess.Popen(
cmd,
cwd=str(cwd),
env=env,
stdout=fp,
stderr=subprocess.STDOUT,
start_new_session=True,
)
def ensure_caffeinate() -> None:
me = str(os.getpid())
# 이미 이 워치독 pid를 감시 중이면 스킵
if pgrep(f"caffeinate -dims -w {me}"):
return
# idle/display/system sleep 방지 (뚜껑 닫으면 별도 — 전원 연결 권장)
spawn(
["caffeinate", "-dims", "-w", me],
ROOT,
OUT / "caffeinate.log",
)
log("caffeinate started (tied to this watchdog pid)")
def restart_next(n: int) -> bool:
log(f"Next DOWN → restart #{n}")
# mariago-web 만 정리 (omook/stoku/othello next-server 보호)
kill_pids(pgrep(r"/go/mariago-web|/mariago-web.*(next|npm|pnpm)"), "baduk-next")
free_port(NEXT_PORT)
if not Path(NPM).exists():
log(f"npm missing at {NPM}")
return False
spawn([NPM, "run", "dev"], WEB, NEXT_LOG)
deadline = time.time() + RESTART_WAIT_S
while time.time() < deadline:
if next_healthy():
log(f"Next UP after restart #{n}")
return True
time.sleep(2)
log(f"Next STILL DOWN after restart #{n}")
return False
def restart_bridge(n: int) -> bool:
log(f"bridge DOWN → restart #{n}")
kill_pids(pgrep("mariago_ws_bridge.py"), "bridge")
free_port(BRIDGE_PORT)
spawn(
[PY, "scripts/mariago_ws_bridge.py", "--ws-port", str(BRIDGE_PORT)],
ROOT,
BRIDGE_LOG,
)
deadline = time.time() + 30
while time.time() < deadline:
if bridge_healthy():
log(f"bridge UP after restart #{n}")
return True
time.sleep(1)
log(f"bridge STILL DOWN after restart #{n}")
return False
MONITOR_LOG = OUT / "monitor.watchdog.log"
MONITOR_OUT = ROOT / "logs" / "mariago_web_monitor"
MONITOR_CMD = [
PY,
"scripts/monitor_mariago_web_play.py",
"--ws-url",
"ws://127.0.0.1:9320",
"--games",
"0",
"--max-moves",
"150",
"--timeout-s",
"240",
"--sleep-s",
"8",
"--out-dir",
str(MONITOR_OUT),
]
def gemma_proc_alive() -> bool:
return bool(pgrep(r"vlm_server serve.*--port 9341"))
def restart_gemma(n: int) -> bool:
if gemma_proc_alive():
log(f"gemma process exists, waiting for :{GEMMA_PORT}")
return port_open(GEMMA_PORT)
log(f"gemma DOWN → restart #{n}")
lora = gemma_lora_path()
spawn(
[
PY, "-m", "vlm_server", "serve",
"--host", "127.0.0.1", "--port", str(GEMMA_PORT),
"--model", GEMMA_MODEL, "--lora-dir", lora,
"--text-only-move", "--board-repr", gemma_board_repr(lora), "--no-score-guided",
"--illegal-retry-max", "16", "--illegal-retry-budget-s", "240",
"--max-new-tokens", "192",
"--log-jsonl", "logs/mariago_web_watch/gemma.jsonl",
],
ROOT,
GEMMA_LOG,
)
deadline = time.time() + 240
while time.time() < deadline:
if port_open(GEMMA_PORT):
log(f"gemma UP after restart #{n}")
return True
time.sleep(5)
log(f"gemma STILL DOWN after restart #{n}")
return False
def monitor_alive() -> bool:
return bool(pgrep("monitor_mariago_web_play.py"))
def restart_monitor(n: int) -> bool:
log(f"web-monitor DOWN → restart #{n}")
kill_pids(pgrep("monitor_mariago_web_play.py"), "web-monitor")
kill_pids(pgrep("daemon_mariago_web_monitor.py"), "web-monitor-wrap")
spawn(MONITOR_CMD, ROOT, MONITOR_LOG)
time.sleep(3)
ok = monitor_alive()
log(f"web-monitor {'UP' if ok else 'STILL DOWN'} after restart #{n}")
return ok
def write_heartbeat(
*,
next_ok: bool,
bridge_ok: bool,
vlm_ok: bool,
gemma_ok: bool,
monitor_ok: bool,
next_restarts: int,
bridge_restarts: int,
monitor_restarts: int,
note: str,
) -> None:
mon_heart = MONITOR_OUT / "HEARTBEAT.md"
mon_snip = mon_heart.read_text(encoding="utf-8") if mon_heart.exists() else "_없음_"
lines = [
f"# MariaGO web watchdog — {now_kst():%Y-%m-%d %H:%M:%S KST}",
"",
f"- **Next :{NEXT_PORT}**: `{'up' if next_ok else 'DOWN'}`",
f"- **bridge :{BRIDGE_PORT}**: `{'up' if bridge_ok else 'DOWN'}`",
f"- **VLM :{VLM_PORT}**: `{'up' if vlm_ok else 'DOWN'}` (harden 워치독)",
f"- **Gemma :{GEMMA_PORT}**: `{'up' if gemma_ok else 'DOWN'}`",
f"- **web-monitor**: `{'up' if monitor_ok else 'DOWN'}` (조기 PASS·불법수)",
f"- **next_restarts**: `{next_restarts}`",
f"- **bridge_restarts**: `{bridge_restarts}`",
f"- **monitor_restarts**: `{monitor_restarts}`",
f"- **note**: {note}",
"",
"## monitor heartbeat",
"",
mon_snip.strip(),
"",
"로그: `logs/mariago_web_watch/watchdog.log`",
"",
]
HEART.write_text("\n".join(lines), encoding="utf-8")
def main() -> int:
OUT.mkdir(parents=True, exist_ok=True)
MONITOR_OUT.mkdir(parents=True, exist_ok=True)
log("mariago web watchdog start (Next + bridge + web-monitor)")
ensure_caffeinate()
next_restarts = 0
bridge_restarts = 0
gemma_restarts = 0
monitor_restarts = 0
while True:
if RESTART_FLAG.exists():
reason = RESTART_FLAG.read_text(encoding="utf-8").strip()
try:
RESTART_FLAG.unlink()
except OSError:
pass
log(f"ops requested restart: {reason}")
next_restarts += 1
nxt = restart_next(next_restarts)
nxt = next_healthy()
br = bridge_healthy()
vlm = port_open(VLM_PORT)
if GEMMA_HOLD.exists():
gemma = port_open(GEMMA_PORT)
else:
gemma = port_open(GEMMA_PORT) or gemma_proc_alive()
if not port_open(GEMMA_PORT) and not gemma_proc_alive():
gemma_restarts += 1
gemma = restart_gemma(gemma_restarts)
mon = monitor_alive()
if not nxt:
next_restarts += 1
nxt = restart_next(next_restarts)
if not br:
bridge_restarts += 1
br = restart_bridge(bridge_restarts)
# 150수 상한 대국은 매일 예약한 작업이 아니라 상시 재시작이었다.
# 그 대국은 멈춘다. Next·브리지 감시는 유지한다.
write_heartbeat(
next_ok=nxt,
bridge_ok=br,
vlm_ok=vlm,
gemma_ok=gemma,
monitor_ok=mon,
next_restarts=next_restarts,
bridge_restarts=bridge_restarts,
monitor_restarts=monitor_restarts,
note="polling",
)
if not vlm:
log("WARN VLM :9319 DOWN (web play / monitor paused until serve recovers)")
time.sleep(POLL_S)
if __name__ == "__main__":
try:
raise SystemExit(main())
except KeyboardInterrupt:
log("watchdog interrupted")
raise SystemExit(0)