예제 소스

daemon_coord_verify_after_smoke.py

#!/usr/bin/env python3
"""Wait for smoke train to finish, then run Qwen base + smoke LoRA coord verify."""
from __future__ import annotations

import os
import subprocess
import time
from pathlib import Path

ROOT = Path("~/project")
LOG = ROOT / "logs" / "vlm_eval_day" / "coord_verify_run.log"
PY = str(ROOT / ".venv" / "bin" / "python")
ADAPTER = ROOT / "vlm_models" / "lora_baduk_v3_qwen_smoke" / "adapters.safetensors"
TRAIN_LOG = ROOT / "logs" / "vlm_eval_day" / "qwen_smoke_train.log"


def log(msg: str) -> None:
    line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}"
    print(line, flush=True)
    with LOG.open("a", encoding="utf-8") as f:
        f.write(line + "\n")


def train_running() -> bool:
    r = subprocess.run(
        ["pgrep", "-f", "mlx_vlm.lora.*lora_baduk_v3_qwen_smoke"],
        capture_output=True,
        text=True,
    )
    return r.returncode == 0


def wait_train(max_wait_s: int = 7200) -> bool:
    t0 = time.time()
    while time.time() - t0 < max_wait_s:
        if not train_running():
            # ensure final adapters exist and log mentions completion-ish
            if ADAPTER.exists() and ADAPTER.stat().st_size > 1_000_000:
                # wait a few seconds for clean exit
                time.sleep(5)
                if not train_running():
                    return True
        time.sleep(20)
    return ADAPTER.exists()


def run_verify(tag: str, extra: list[str]) -> int:
    out_jsonl = ROOT / "logs" / "vlm_eval_day" / f"coord_{tag}.jsonl"
    out_sum = ROOT / "logs" / "vlm_eval_day" / f"coord_{tag}_summary.json"
    cmd = [
        PY,
        "-m",
        "vlm_server.verify_coords",
        "--model",
        "********",
        "--board-format",
        "aligned",
        "--trials",
        "48",
        "--out-jsonl",
        str(out_jsonl),
        "--out-summary",
        str(out_sum),
        *extra,
    ]
    log("RUN " + " ".join(cmd))
    env = os.environ.copy()
    env["HF_HOME"] = str(ROOT / "vlm_models")
    env["HF_HUB_CACHE"] = str(ROOT / "vlm_models")
    env["PYTHONUNBUFFERED"] = "1"
    with LOG.open("a", encoding="utf-8") as f:
        p = subprocess.run(cmd, cwd=str(ROOT), env=env, stdout=f, stderr=subprocess.STDOUT)
    log(f"DONE {tag} exit={p.returncode}")
    return int(p.returncode)


def main() -> int:
    LOG.parent.mkdir(parents=True, exist_ok=True)
    if os.fork() > 0:
        return 0
    os.setsid()
    if os.fork() > 0:
        os._exit(0)
    os.chdir(ROOT)
    # redirect
    with LOG.open("a", encoding="utf-8") as f:
        os.dup2(f.fileno(), 1)
        os.dup2(f.fileno(), 2)
    dn = os.open(os.devnull, os.O_RDONLY)
    os.dup2(dn, 0)
    os.close(dn)

    log("waiting for smoke train to finish...")
    if not wait_train():
        log("TIMEOUT waiting for train")
        os._exit(1)
    log("smoke train finished; starting coord verify (base then lora)")
    # Base first
    rc1 = run_verify("qwen_base", [])
    rc2 = run_verify("qwen_smoke", ["--lora-dir", "********"])
    log(f"all done base_rc={rc1} smoke_rc={rc2}")
    os._exit(0 if rc1 == 0 and rc2 == 0 else 3)


if __name__ == "__main__":
    raise SystemExit(main())