예제 소스

daemon_janggy_stack_watchdog.py

#!/usr/bin/env python3
"""장기(janggy) 웹 감시: Next(:3007) + IDC 터널(:18088).

죽으면 재기동. Cursor 터미널과 무관하게 :3007 과 IDC :18088 터널을 유지한다.

  python3 scripts/daemon_janggy_stack_watchdog.py
"""

from __future__ import annotations

import os
import signal
import socket
import subprocess
import sys
import time
import urllib.error
import urllib.request
from datetime import datetime
from pathlib import Path
from zoneinfo import ZoneInfo

ROOT = Path(os.environ.get("JANGGY_ROOT", Path(__file__).resolve().parents[1]))
WEB = ROOT / "web"
OUT = Path.home() / "Library" / "Logs" / "Janggy"
HEART = OUT / "HEARTBEAT.md"
LOG = OUT / "watchdog.log"
NEXT_LOG = OUT / "next.watchdog.log"
TUNNEL_LOG = OUT / "janggy.tunnel.log"
RESTART_FLAG = OUT / "RESTART_REQUESTED"
# 있으면 next start 대신 next dev 유지 (옛 빌드 회피용 임시 모드)
DEV_FLAG = OUT / "PREFER_DEV"
KST = ZoneInfo("Asia/Seoul")

NEXT_PORT = 3007
REMOTE_PORT = 18088
SSH_HOST = "[REDACTED]"
POLL_S = 20
HTTP_TIMEOUT_S = 3.0
NEXT_RESTART_WAIT_S = 90

NODE_BIN = Path.home() / ".nvm" / "versions" / "node" / "v24.14.1" / "bin"
NPM = str(NODE_BIN / "npm")
PNPM = str(NODE_BIN / "pnpm")


def now_kst() -> datetime:
    return datetime.now(KST)


def log(msg: str) -> None:
    line = f"[{now_kst():%Y-%m-%d %H:%M:%S}] {msg}"
    print(line, flush=True)
    OUT.mkdir(parents=True, exist_ok=True)
    with LOG.open("a", encoding="utf-8") as f:
        f.write(line + "\n")


def port_open(port: int, host: str = "127.0.0.1") -> bool:
    try:
        with socket.create_connection((host, port), timeout=1.5):
            return True
    except OSError:
        return False


def http_ok(url: str, expect: int = 200) -> bool:
    try:
        req = urllib.request.Request(url, method="GET")
        with urllib.request.urlopen(req, timeout=HTTP_TIMEOUT_S) as resp:
            return int(resp.status) == expect
    except (urllib.error.URLError, TimeoutError, OSError, ValueError):
        return False


def listeners_on(port: int) -> list[int]:
    try:
        out = subprocess.check_output(
            ["lsof", "-nP", f"-iTCP:{port}", "-sTCP:LISTEN", "-t"],
            text=True,
        )
    except (subprocess.CalledProcessError, FileNotFoundError, PermissionError, OSError):
        return []
    pids: list[int] = []
    for tok in out.split():
        try:
            pids.append(int(tok))
        except ValueError:
            pass
    return pids


def next_healthy() -> bool:
    return http_ok(f"http://127.0.0.1:{NEXT_PORT}/", 200) or http_ok(
        f"http://127.0.0.1:{NEXT_PORT}/play", 200
    )


def pgrep(pattern: str) -> list[int]:
    try:
        out = subprocess.check_output(["pgrep", "-f", pattern], text=True)
    except (subprocess.CalledProcessError, FileNotFoundError, PermissionError, OSError):
        return []
    me = os.getpid()
    pids: list[int] = []
    for tok in out.split():
        try:
            p = int(tok)
        except ValueError:
            continue
        if p != me:
            pids.append(p)
    return pids


def kill_pids(pids: list[int], label: str) -> None:
    if not pids:
        return
    log(f"kill {label}: {pids}")
    for p in pids:
        try:
            os.kill(p, signal.SIGTERM)
        except (ProcessLookupError, PermissionError, OSError):
            pass
    time.sleep(2)
    for p in pids:
        try:
            os.kill(p, signal.SIGKILL)
        except (ProcessLookupError, PermissionError, OSError):
            pass


def free_port(port: int) -> None:
    kill_pids(listeners_on(port), f"port:{port}")


def spawn(cmd: list[str], cwd: Path, logfile: Path) -> subprocess.Popen:
    env = os.environ.copy()
    env["PYTHONUNBUFFERED"] = "1"
    path_parts = [str(NODE_BIN), "/opt/homebrew/bin", "/usr/local/bin"]
    if env.get("PATH"):
        path_parts.append(env["PATH"])
    env["PATH"] = ":".join(path_parts)
    env["PORT"] = str(NEXT_PORT)
    logfile.parent.mkdir(parents=True, exist_ok=True)
    fp = open(logfile, "a", encoding="utf-8")
    fp.write(f"\n===== spawn {now_kst():%Y-%m-%d %H:%M:%S} =====\n")
    fp.write(" ".join(cmd) + "\n")
    fp.flush()
    return subprocess.Popen(
        cmd,
        cwd=str(cwd),
        env=env,
        stdout=fp,
        stderr=subprocess.STDOUT,
        start_new_session=True,
    )


def ensure_caffeinate() -> None:
    me = str(os.getpid())
    try:
        if pgrep(f"caffeinate -dims -w {me}"):
            return
        spawn(["caffeinate", "-dims", "-w", me], Path.home(), OUT / "caffeinate.log")
        log("caffeinate started (tied to this watchdog pid)")
    except (PermissionError, OSError) as e:
        log(f"caffeinate skip: {e}")


def proc_cmdline(pid: int) -> str:
    try:
        return subprocess.check_output(
            ["ps", "-p", str(pid), "-o", "command="], text=True
        )
    except (subprocess.CalledProcessError, FileNotFoundError, PermissionError, OSError):
        return ""


def proc_ppid(pid: int) -> int | None:
    try:
        out = subprocess.check_output(
            ["ps", "-p", str(pid), "-o", "ppid="], text=True
        ).strip()
        return int(out)
    except (subprocess.CalledProcessError, FileNotFoundError, PermissionError, OSError, ValueError):
        return None


def cmdline_is_dev(cmd: str) -> bool:
    low = cmd.lower()
    return "next dev" in low or "dev-with-ts.mjs dev" in low


def next_is_dev() -> bool:
    # listen 소켓은 next-server 자식일 수 있어 조상 cmdline 까지 본다.
    for pid in listeners_on(NEXT_PORT):
        cur: int | None = pid
        for _ in range(8):
            if cur is None or cur <= 1:
                break
            if cmdline_is_dev(proc_cmdline(cur)):
                return True
            cur = proc_ppid(cur)
    for pid in pgrep(r"/janggy/web|/janggy-web"):
        if cmdline_is_dev(proc_cmdline(pid)):
            return True
    return False


def prefer_dev() -> bool:
    return DEV_FLAG.exists() or os.environ.get("JANGGY_DEV", "").strip() in {
        "1",
        "true",
        "yes",
    }


def tunnel_up() -> bool:
    needle = f"{REMOTE_PORT}:127.0.0.1:{NEXT_PORT}"
    for pid in pgrep("ssh "):
        if needle in proc_cmdline(pid):
            return True
    return False


def ensure_tunnel() -> bool:
    if tunnel_up():
        return True
    log(f"janggy tunnel IDC :{REMOTE_PORT}→:{NEXT_PORT} DOWN → spawn")
    # 잔여 불량 포워드 정리
    kill_pids(
        [p for p in pgrep("ssh ") if f"{REMOTE_PORT}:127.0.0.1:{NEXT_PORT}" in proc_cmdline(p)],
        "janggy-tunnel",
    )
    spawn(
        [
            "ssh",
            "-N",
            "-o",
            "BatchMode=yes",
            "-o",
            "ExitOnForwardFailure=yes",
            "-o",
            "ServerAliveInterval=30",
            "-o",
            "ServerAliveCountMax=3",
            "-o",
            "StrictHostKeyChecking=accept-new",
            "-o",
            "TCPKeepAlive=yes",
            "-R",
            f"127.0.0.1:{REMOTE_PORT}:127.0.0.1:{NEXT_PORT}",
            SSH_HOST,
        ],
        Path.home(),
        TUNNEL_LOG,
    )
    deadline = time.time() + 20
    while time.time() < deadline:
        if tunnel_up():
            log("janggy tunnel UP")
            return True
        time.sleep(1)
    log("janggy tunnel STILL DOWN")
    return False


def pkg_runner() -> list[str] | None:
    if Path(PNPM).exists() and (WEB / "pnpm-lock.yaml").exists():
        return [PNPM]
    if Path(NPM).exists():
        return [NPM]
    return None


def ensure_next_build() -> bool:
    marker = WEB / ".next" / "BUILD_ID"
    if marker.exists():
        return True
    runner = pkg_runner()
    if not runner:
        log(f"npm/pnpm missing (npm={NPM})")
        return False
    if not (WEB / "node_modules").is_dir():
        log("node_modules missing → skip build")
        return False
    log("Next .next missing → run build")
    try:
        subprocess.run(
            [*runner, "run", "build"],
            cwd=str(WEB),
            check=True,
            env={
                **os.environ,
                "PATH": f"{NODE_BIN}:{os.environ.get('PATH', '')}",
                "PORT": str(NEXT_PORT),
            },
        )
    except subprocess.CalledProcessError as e:
        log(f"build failed: {e}")
        return False
    return marker.exists()


def restart_next(n: int, *, as_dev: bool | None = None) -> bool:
    use_dev = prefer_dev() if as_dev is None else as_dev
    mode = "dev" if use_dev else "service"
    log(f"Next :{NEXT_PORT} → {mode} restart #{n}")
    kill_pids(
        pgrep(r"/janggy/web|/janggy-web.*(next|npm|pnpm)"),
        "janggy-next",
    )
    free_port(NEXT_PORT)
    if not use_dev and not ensure_next_build():
        return False
    runner = pkg_runner()
    if not runner:
        log("npm/pnpm missing")
        return False
    spawn([*runner, "run", "dev" if use_dev else "start"], WEB, NEXT_LOG)
    deadline = time.time() + NEXT_RESTART_WAIT_S
    while time.time() < deadline:
        if use_dev:
            if next_healthy() and next_is_dev():
                log(f"Next dev UP after restart #{n}")
                return True
        elif next_healthy() and not next_is_dev():
            log(f"Next service UP after restart #{n}")
            return True
        time.sleep(2)
    log(f"Next STILL DOWN after restart #{n} ({mode})")
    return False


def write_heartbeat(
    *,
    next_ok: bool,
    tunnel_ok: bool,
    next_restarts: int,
    note: str,
) -> None:
    mode = "service" if next_ok and not next_is_dev() else ("dev" if next_ok else "down")
    prefer = "on" if prefer_dev() else "off"
    lines = [
        f"# Janggy stack watchdog — {now_kst():%Y-%m-%d %H:%M:%S KST}",
        "",
        f"- **Next :{NEXT_PORT}**: `{'up' if next_ok else 'DOWN'}` ({mode})",
        f"- **prefer_dev**: `{prefer}` (`{DEV_FLAG}`)",
        f"- **IDC tunnel :{REMOTE_PORT}→:{NEXT_PORT}**: `{'up' if tunnel_ok else 'DOWN'}`",
        f"- **public**: `https://janggy.mariago.kr/`",
        f"- **next_restarts**: `{next_restarts}`",
        f"- **note**: {note}",
        "",
        "로그: `~/Library/Logs/Janggy/watchdog.log`",
        "",
    ]
    HEART.write_text("\n".join(lines), encoding="utf-8")


def acquire_singleton() -> object:
    OUT.mkdir(parents=True, exist_ok=True)
    lock_path = OUT / "watchdog.lock"
    fh = lock_path.open("a+", encoding="utf-8")
    try:
        import fcntl

        fcntl.flock(fh.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
    except OSError as e:
        fh.close()
        raise SystemExit(f"another janggy watchdog holds {lock_path}: {e}") from e
    fh.seek(0)
    fh.truncate()
    fh.write(f"{os.getpid()}\n")
    fh.flush()
    return fh


def main() -> int:
    OUT.mkdir(parents=True, exist_ok=True)
    lock_fh = acquire_singleton()
    log(f"janggy stack watchdog start (Next:{NEXT_PORT} tunnel:{REMOTE_PORT})")
    ensure_caffeinate()
    next_restarts = 0
    migrated_to_service = False

    while True:
        want_dev = prefer_dev()
        if RESTART_FLAG.exists():
            reason = RESTART_FLAG.read_text(encoding="utf-8").strip()
            try:
                RESTART_FLAG.unlink()
            except OSError:
                pass
            log(f"ops requested restart: {reason}")
            next_restarts += 1
            restart_next(next_restarts, as_dev=want_dev)

        nxt = next_healthy()
        if want_dev:
            if not nxt or not next_is_dev():
                if nxt and not next_is_dev():
                    log("prefer_dev on → migrate service → next dev")
                next_restarts += 1
                nxt = restart_next(next_restarts, as_dev=True)
            migrated_to_service = False
        elif nxt and next_is_dev() and not migrated_to_service:
            log("Next is dev mode → migrate to service (next start)")
            next_restarts += 1
            nxt = restart_next(next_restarts, as_dev=False)
            migrated_to_service = True
        elif not nxt:
            next_restarts += 1
            nxt = restart_next(next_restarts, as_dev=False)
            migrated_to_service = True

        tun = ensure_tunnel()
        write_heartbeat(
            next_ok=nxt,
            tunnel_ok=tun,
            next_restarts=next_restarts,
            note="polling",
        )
        time.sleep(POLL_S)


if __name__ == "__main__":
    try:
        raise SystemExit(main())
    except KeyboardInterrupt:
        log("watchdog interrupted")
        raise SystemExit(0)
    except Exception:
        log("watchdog crashed")
        import traceback

        OUT.mkdir(parents=True, exist_ok=True)
        with LOG.open("a", encoding="utf-8") as f:
            f.write(traceback.format_exc() + "\n")
        raise