예제 소스

run_heuristic_gemma4_r3.py

#!/usr/bin/env python3
"""휴리스틱과 80수 제한 40대국으로 Gemma 4를 다시 학습한다.

이전 혼합 학습은 긴 대국의 수를 기존 수 데이터와 섞어, 초반에도 1선에 두었다.
이번엔 그 40국의 국면을 휴리스틱 수로 다시 라벨하고, 기존 수 혼합은 넣지 않는다.
좌표 데이터만 조금 남겨 좌표를 잊지 않게 한다.
"""

from __future__ import annotations

import json
import os
import signal
import socket
import sqlite3
import subprocess
import time
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
PY = ROOT / ".venv" / "bin" / "python"
OUT = ROOT / "logs" / "heuristic_gemma4_r3_20261010"
HOLD = ROOT / "logs" / "mariago_web_watch" / "GEMMA_HOLD"
LORA_FILE = ROOT / "logs" / "mariago_web_watch" / "gemma_lora.txt"
MODEL = "********"
BASE = "********"
LORA = "********"
DATA = "sft_data/heuristic_gemma4_r3"
PORT = 9341
GAMES = 40
MAX_MOVES = 80


def log(msg: str) -> None:
    print(msg, flush=True)


def run(cmd: list[str], log_path: Path) -> int:
    log("[runner] " + " ".join(cmd))
    log_path.parent.mkdir(parents=True, exist_ok=True)
    with log_path.open("ab") as fh:
        return subprocess.call(cmd, cwd=ROOT, stdout=fh, stderr=subprocess.STDOUT)


def port_open(port: int) -> bool:
    sock = socket.socket()
    sock.settimeout(1)
    try:
        sock.connect(("127.0.0.1", port))
        return True
    except OSError:
        return False
    finally:
        sock.close()


def wait_port(port: int, timeout_s: float) -> bool:
    deadline = time.time() + timeout_s
    while time.time() < deadline:
        if port_open(port):
            return True
        time.sleep(5)
    return False


def gemma_pids() -> list[int]:
    try:
        out = subprocess.check_output(["pgrep", "-f", r"vlm_server serve.*--port 9341"], text=True)
    except subprocess.CalledProcessError:
        return []
    return [int(x) for x in out.split() if x.strip()]


def stop_gemma() -> None:
    pids = gemma_pids()
    if not pids:
        log("[runner] gemma already down")
        return
    log(f"[runner] stop gemma {pids}")
    for pid in pids:
        try:
            os.kill(pid, signal.SIGTERM)
        except ProcessLookupError:
            pass
    deadline = time.time() + 40
    while time.time() < deadline and (port_open(PORT) or gemma_pids()):
        time.sleep(1)
    for pid in gemma_pids():
        try:
            os.kill(pid, signal.SIGKILL)
        except ProcessLookupError:
            pass


def start_gemma(lora: str) -> subprocess.Popen:
    cmd = [
        str(PY), "-m", "vlm_server", "serve",
        "--host", "127.0.0.1", "--port", str(PORT),
        "--model", MODEL, "--lora-dir", lora,
        "--text-only-move", "--board-repr", "aligned", "--no-score-guided",
        "--illegal-retry-max", "16", "--illegal-retry-budget-s", "240",
        "--log-jsonl", str(OUT / "gemma4.jsonl"),
    ]
    OUT.mkdir(parents=True, exist_ok=True)
    logf = open(OUT / "gemma_serve.stdout", "ab")
    log(f"[runner] start gemma4 {lora}")
    proc = subprocess.Popen(cmd, cwd=ROOT, stdout=logf, stderr=subprocess.STDOUT, start_new_session=True)
    if not wait_port(PORT, 600):
        proc.terminate()
        raise SystemExit("gemma4 did not listen")
    log(f"[runner] gemma4 listening pid={proc.pid}")
    return proc


def edge_line(row: int, col: int) -> int:
    return min(row, col, 18 - row, 18 - col) + 1


def white_line_report(db_path: Path) -> dict:
    con = sqlite3.connect(db_path)
    rows = con.execute(
        "select ply, row, col from moves where color='W' and kind='move' and row is not null"
    ).fetchall()
    con.close()
    lines: dict[str, int] = {}
    early: dict[str, int] = {}
    for ply, row, col in rows:
        key = str(edge_line(int(row), int(col)))
        lines[key] = lines.get(key, 0) + 1
        if int(ply) < 40:
            early[key] = early.get(key, 0) + 1
    n = sum(lines.values())
    early_n = sum(early.values())
    return {
        "white_moves": n,
        "lines": lines,
        "line1_rate": (lines.get("1", 0) / n) if n else None,
        "early40_moves": early_n,
        "early40_lines": early,
        "early40_line1_rate": (early.get("1", 0) / early_n) if early_n else None,
    }


def write_report(name: str, report: dict) -> None:
    path = OUT / name
    path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
    log(f"[runner] {name} line1={report.get('line1_rate')} early40={report.get('early40_line1_rate')}")


def completed_games(log_path: Path) -> int:
    if not log_path.exists():
        return 0
    n = 0
    for line in log_path.read_text(encoding="utf-8", errors="replace").splitlines():
        if line.startswith("[match] game ") and " plies=" in line:
            n += 1
    return n


def play_games(
    lora_name: str,
    games: int,
    max_moves: int,
    out_dir: str,
    log_name: str,
    *,
    start_id: int = 1,
) -> int:
    if not port_open(PORT):
        start_gemma(BASE if lora_name.endswith("r2") else LORA)
    return run(
        [
            str(PY), "scripts/play_heuristic_vs_model.py",
            "--port", str(PORT),
            "--model-name", lora_name,
            "--games", str(games),
            "--start-id", str(start_id),
            "--max-moves", str(max_moves),
            "--out-dir", out_dir,
        ],
        OUT / log_name,
    )


def update_registry() -> None:
    LORA_FILE.write_text(LORA + "\n", encoding="utf-8")
    reg_path = ROOT / "scripts" / "models.json"
    reg = json.loads(reg_path.read_text(encoding="utf-8"))
    reg.setdefault("models", {})["gemma4_heuristic"] = {
        "port": PORT,
        "model": MODEL,
        "lora": LORA,
        "note": "휴리스틱 80수 40대국으로 이어서 학습. 대국 때는 --no-score-guided.",
    }
    reg_path.write_text(json.dumps(reg, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")


def main() -> int:
    OUT.mkdir(parents=True, exist_ok=True)
    HOLD.parent.mkdir(parents=True, exist_ok=True)
    play_mark = OUT / "PLAY_DONE"
    train_mark = OUT / "TRAIN_DONE"
    if not play_mark.exists():
        done = completed_games(OUT / "play.log")
        left = GAMES - done
        log(f"[runner] teacher games done={done} left={left}")
        if left > 0:
            rc = play_games(
                "gemma4_heuristic_r2",
                left,
                MAX_MOVES,
                str(OUT.relative_to(ROOT)),
                "play.log",
                start_id=done + 1,
            )
            log(f"[runner] play rc={rc}")
            if rc != 0:
                return rc
            if completed_games(OUT / "play.log") < GAMES:
                log("[runner] play stopped before 40 games")
                return 1
        db = OUT / "matches.sqlite"
        write_report("white_lines_before.json", white_line_report(db))
        play_mark.write_text("ok\n", encoding="utf-8")

    HOLD.write_text("r3 train\n", encoding="utf-8")
    stop_gemma()
    trained = False
    try:
        summary = ROOT / DATA / "summary.json"
        if not summary.exists():
            rc = run(
                [
                    str(PY), "scripts/dataset_heuristic_teacher.py",
                    "--relabel",
                    "--source", f"{OUT.relative_to(ROOT)}/matches.sqlite:BW",
                    "--out", DATA,
                ],
                OUT / "dataset.log",
            )
            log(f"[runner] dataset rc={rc}")
            if rc != 0:
                return rc
        if not train_mark.exists():
            rc = run(
                [
                    str(PY), "scripts/train_ladder_lora.py",
                    "--shape", "onpolicy",
                    "--data-name", "heuristic_gemma4_r3",
                    "--log-name", "gemma4_heuristic_r3",
                    "--model", MODEL,
                    "--base", BASE,
                    "--out", LORA,
                    "--shape-n", "8000",
                    "--move-n", "0",
                    "--coord-n", "200",
                    "--skip-base-eval",
                ],
                OUT / "train.log",
            )
            log(f"[runner] train rc={rc}")
            if rc != 0:
                return rc
            train_mark.write_text("ok\n", encoding="utf-8")
        update_registry()
        start_gemma(LORA)
        trained = True
    finally:
        if HOLD.exists():
            HOLD.unlink()
        if not port_open(PORT):
            try:
                start_gemma(LORA if train_mark.exists() else BASE)
            except SystemExit as exc:
                log(f"[runner] gemma restart failed: {exc}")
    if not trained:
        return 1
    probe_mark = OUT / "PROBE_DONE"
    if probe_mark.exists():
        return 0
    probe = ROOT / "logs" / "heuristic_gemma4_150x5"
    rc = play_games(
        "gemma4_heuristic_r3",
        5,
        150,
        str(probe.relative_to(ROOT)),
        "play_150.log",
    )
    log(f"[runner] probe rc={rc}")
    probe_db = probe / "matches.sqlite"
    if probe_db.exists():
        write_report("white_lines_after.json", white_line_report(probe_db))
    if rc == 0:
        probe_mark.write_text("ok\n", encoding="utf-8")
    return rc


if __name__ == "__main__":
    raise SystemExit(main())