예제 소스
SGF 기보로 학습하는 바둑 LLM 인공지능 · 파일 169개
daemon_ssh_reverse_tunnel.py
#!/usr/bin/env python3
"""MariaGO / Stoku / Omook / Othello / OPS → IDC reverse SSH tunnel watchdog.
집 Mac의 Next(바둑:3000, 스토쿠:3001, 오목:3002, OPS:3003, 오델로:3004)
+ ops agent(:3040) + WS bridge(:9320) 를 IDC(ethtrade) localhost 로 역터널.
.venv/bin/python scripts/daemon_ssh_reverse_tunnel.py
터널:
IDC 127.0.0.1:18080 → 집 127.0.0.1:3000 (baduk.mariago.kr)
IDC 127.0.0.1:18081 → 집 127.0.0.1:3001 (stoku.mariago.kr)
IDC 127.0.0.1:18082 → 집 127.0.0.1:3002 (omook.mariago.kr)
IDC 127.0.0.1:18083 → 집 127.0.0.1:3003 (ops.mariago.kr UI)
IDC 127.0.0.1:18084 → 집 127.0.0.1:3040 (ops.mariago.kr API)
IDC 127.0.0.1:18085 → 집 127.0.0.1:3004 (othello.mariago.kr)
IDC 127.0.0.1:19320 → 집 127.0.0.1:9320 (baduk VLM WS)
"""
from __future__ import annotations
import os
import signal
import socket
import subprocess
import sys
import time
from datetime import datetime
from pathlib import Path
from zoneinfo import ZoneInfo
ROOT = Path(__file__).resolve().parents[1]
PY = str(ROOT / ".venv" / "bin" / "python")
OUT = ROOT / "logs" / "mariago_ssh_tunnel"
HEART = OUT / "HEARTBEAT.md"
LOG = OUT / "tunnel.watchdog.log"
SSH_LOG = OUT / "ssh.tunnel.log"
BRIDGE_LOG = OUT / "bridge.tunnel-watchdog.log"
RESTART_FLAG = OUT / "RESTART_REQUESTED"
KST = ZoneInfo("Asia/Seoul")
SSH_HOST = "[REDACTED]"
REMOTE_HTTP = 18080
REMOTE_STOKU_HTTP = 18081
REMOTE_OMOOK_HTTP = 18082
REMOTE_OPS_HTTP = 18083
REMOTE_OPS_API = 18084
REMOTE_OTHELLO_HTTP = 18085
REMOTE_WS = 19320
LOCAL_HTTP = 3000
LOCAL_STOKU_HTTP = 3001
LOCAL_OMOOK_HTTP = 3002
LOCAL_OPS_HTTP = 3003
LOCAL_OPS_API = 3040
LOCAL_OTHELLO_HTTP = 3004
LOCAL_WS = 9320
BRIDGE_PORT = LOCAL_WS
POLL_S = 15
RESTART_WAIT_S = 25
REMOTE_PORTS = (
REMOTE_HTTP,
REMOTE_STOKU_HTTP,
REMOTE_OMOOK_HTTP,
REMOTE_OPS_HTTP,
REMOTE_OPS_API,
REMOTE_OTHELLO_HTTP,
REMOTE_WS,
)
def now_kst() -> datetime:
return datetime.now(KST)
def log(msg: str) -> None:
line = f"[{now_kst():%Y-%m-%d %H:%M:%S}] {msg}"
print(line, flush=True)
OUT.mkdir(parents=True, exist_ok=True)
with LOG.open("a", encoding="utf-8") as f:
f.write(line + "\n")
def pgrep(pattern: str) -> list[int]:
try:
out = subprocess.check_output(["pgrep", "-f", pattern], text=True)
except subprocess.CalledProcessError:
return []
me = os.getpid()
pids: list[int] = []
for tok in out.split():
try:
p = int(tok)
except ValueError:
continue
if p != me:
pids.append(p)
return pids
def port_open(port: int, host: str = "127.0.0.1") -> bool:
try:
with socket.create_connection((host, port), timeout=1.5):
return True
except OSError:
try:
with socket.create_connection(("::1", port), timeout=1.5):
return True
except OSError:
return False
def spawn(cmd: list[str], cwd: Path, logfile: Path) -> subprocess.Popen:
env = os.environ.copy()
env["PYTHONUNBUFFERED"] = "1"
logfile.parent.mkdir(parents=True, exist_ok=True)
fp = open(logfile, "a", encoding="utf-8")
fp.write(f"\n===== spawn {now_kst():%Y-%m-%d %H:%M:%S} =====\n")
fp.write(" ".join(cmd) + "\n")
fp.flush()
return subprocess.Popen(
cmd,
cwd=str(cwd),
stdout=fp,
stderr=subprocess.STDOUT,
env=env,
start_new_session=True,
)
def kill_pids(pids: list[int], label: str) -> None:
if not pids:
return
log(f"kill {label}: {pids}")
for p in pids:
try:
os.kill(p, signal.SIGTERM)
except ProcessLookupError:
pass
time.sleep(2)
for p in pids:
try:
os.kill(p, signal.SIGKILL)
except ProcessLookupError:
pass
def tunnel_pids() -> list[int]:
try:
out = subprocess.check_output(["pgrep", "-f", f"ssh .*{SSH_HOST}"], text=True)
except subprocess.CalledProcessError:
return []
me = os.getpid()
pids: list[int] = []
for tok in out.split():
try:
p = int(tok)
except ValueError:
continue
if p == me:
continue
try:
cmd = Path(f"/proc/{p}/cmdline").read_bytes().replace(b"\x00", b" ").decode()
except Exception:
try:
cmd = subprocess.check_output(
["ps", "-p", str(p), "-o", "command="], text=True
)
except subprocess.CalledProcessError:
continue
# 바둑 HTTP 포워드가 있으면 MariaGO 터널로 간주 (구버전·신버전 모두)
if f"127.0.0.1:{REMOTE_HTTP}:127.0.0.1:{LOCAL_HTTP}" in cmd:
pids.append(p)
return pids
def tunnel_has_stoku(cmd: str) -> bool:
return f"127.0.0.1:{REMOTE_STOKU_HTTP}:127.0.0.1:{LOCAL_STOKU_HTTP}" in cmd
def tunnel_has_omook(cmd: str) -> bool:
return f"127.0.0.1:{REMOTE_OMOOK_HTTP}:127.0.0.1:{LOCAL_OMOOK_HTTP}" in cmd
def tunnel_has_othello(cmd: str) -> bool:
return f"127.0.0.1:{REMOTE_OTHELLO_HTTP}:127.0.0.1:{LOCAL_OTHELLO_HTTP}" in cmd
def tunnel_has_ops(cmd: str) -> bool:
return (
f"127.0.0.1:{REMOTE_OPS_HTTP}:127.0.0.1:{LOCAL_OPS_HTTP}" in cmd
and f"127.0.0.1:{REMOTE_OPS_API}:127.0.0.1:{LOCAL_OPS_API}" in cmd
)
def current_tunnel_cmd() -> str | None:
for p in tunnel_pids():
try:
return subprocess.check_output(
["ps", "-p", str(p), "-o", "command="], text=True
).strip()
except subprocess.CalledProcessError:
continue
return None
def remote_ports_ok() -> bool:
"""IDC에서 reverse listen 포트가 열려 있는지 확인."""
ports = ",".join(str(p) for p in REMOTE_PORTS)
probe = (
"python3 - <<'PY'\n"
"import socket\n"
f"ports=({ports})\n"
"ok=True\n"
"for port in ports:\n"
" s=socket.socket(); s.settimeout(2)\n"
" try:\n"
" s.connect(('127.0.0.1', port))\n"
" except Exception:\n"
" ok=False\n"
" finally:\n"
" s.close()\n"
"print('OK' if ok else 'DOWN')\n"
"PY"
)
try:
out = subprocess.check_output(
[
"ssh",
"-o",
"BatchMode=yes",
"-o",
"ConnectTimeout=8",
"-o",
"********",
SSH_HOST,
probe,
],
text=True,
stderr=subprocess.DEVNULL,
timeout=20,
)
return "OK" in out
except (subprocess.CalledProcessError, subprocess.TimeoutExpired, OSError):
return False
def local_backends_ok() -> tuple[bool, bool, bool, bool, bool, bool, bool]:
return (
port_open(LOCAL_HTTP),
port_open(LOCAL_STOKU_HTTP),
port_open(LOCAL_OMOOK_HTTP),
port_open(LOCAL_WS),
port_open(LOCAL_OPS_HTTP),
port_open(LOCAL_OPS_API),
port_open(LOCAL_OTHELLO_HTTP),
)
def ensure_caffeinate() -> None:
me = str(os.getpid())
if pgrep(f"caffeinate -dims -w {me}"):
return
OUT.mkdir(parents=True, exist_ok=True)
with (OUT / "caffeinate.log").open("a", encoding="utf-8") as fp:
fp.write(f"\n===== spawn {now_kst():%Y-%m-%d %H:%M:%S} =====\n")
fp.flush()
subprocess.Popen(
["caffeinate", "-dims", "-w", me],
cwd=str(ROOT),
stdout=fp,
stderr=subprocess.STDOUT,
start_new_session=True,
)
log("caffeinate started (tied to this tunnel watchdog pid)")
def start_tunnel() -> subprocess.Popen:
OUT.mkdir(parents=True, exist_ok=True)
cmd = [
"ssh",
"-N",
"-o",
"BatchMode=yes",
"-o",
"ExitOnForwardFailure=yes",
"-o",
"ServerAliveInterval=30",
"-o",
"ServerAliveCountMax=3",
"-o",
"********",
"-o",
"TCPKeepAlive=yes",
"-R",
f"127.0.0.1:{REMOTE_HTTP}:127.0.0.1:{LOCAL_HTTP}",
"-R",
f"127.0.0.1:{REMOTE_STOKU_HTTP}:127.0.0.1:{LOCAL_STOKU_HTTP}",
"-R",
f"127.0.0.1:{REMOTE_OMOOK_HTTP}:127.0.0.1:{LOCAL_OMOOK_HTTP}",
"-R",
f"127.0.0.1:{REMOTE_OPS_HTTP}:127.0.0.1:{LOCAL_OPS_HTTP}",
"-R",
f"127.0.0.1:{REMOTE_OPS_API}:127.0.0.1:{LOCAL_OPS_API}",
"-R",
f"127.0.0.1:{REMOTE_OTHELLO_HTTP}:127.0.0.1:{LOCAL_OTHELLO_HTTP}",
"-R",
f"127.0.0.1:{REMOTE_WS}:127.0.0.1:{LOCAL_WS}",
SSH_HOST,
]
fp = open(SSH_LOG, "a", encoding="utf-8")
fp.write(f"\n===== spawn {now_kst():%Y-%m-%d %H:%M:%S} =====\n")
fp.write(" ".join(cmd) + "\n")
fp.flush()
return subprocess.Popen(
cmd,
cwd=str(ROOT),
stdout=fp,
stderr=subprocess.STDOUT,
start_new_session=True,
)
def restart_tunnel(n: int) -> bool:
log(f"tunnel DOWN → restart #{n}")
kill_pids(tunnel_pids(), "ssh-tunnel")
kill_pids(pgrep(f"ssh .*{SSH_HOST}"), "ssh-ethtrade")
start_tunnel()
deadline = time.time() + RESTART_WAIT_S
while time.time() < deadline:
if tunnel_pids() and remote_ports_ok():
log(f"tunnel UP after restart #{n}")
return True
time.sleep(2)
if tunnel_pids():
log(f"tunnel process up but remote ports not confirmed after restart #{n}")
return True
log(f"tunnel STILL DOWN after restart #{n}")
return False
def write_heartbeat(
*,
tunnel_ok: bool,
remote_ok: bool,
next_ok: bool,
stoku_ok: bool,
omook_ok: bool,
bridge_ok: bool,
ops_ui_ok: bool,
ops_api_ok: bool,
othello_ok: bool,
restarts: int,
note: str,
) -> None:
lines = [
f"# MariaGO SSH reverse tunnel — {now_kst():%Y-%m-%d %H:%M:%S KST}",
"",
f"- **ssh tunnel process**: `{'up' if tunnel_ok else 'DOWN'}`",
f"- **IDC :{REMOTE_HTTP}/:{REMOTE_STOKU_HTTP}/:{REMOTE_OMOOK_HTTP}/:{REMOTE_OPS_HTTP}/:{REMOTE_OPS_API}/:{REMOTE_OTHELLO_HTTP}/:{REMOTE_WS}**: `{'up' if remote_ok else 'DOWN'}`",
f"- **local Next(baduk) :{LOCAL_HTTP}**: `{'up' if next_ok else 'DOWN'}`",
f"- **local Next(stoku) :{LOCAL_STOKU_HTTP}**: `{'up' if stoku_ok else 'DOWN'}`",
f"- **local Next(omook) :{LOCAL_OMOOK_HTTP}**: `{'up' if omook_ok else 'DOWN'}`",
f"- **local Next(othello) :{LOCAL_OTHELLO_HTTP}**: `{'up' if othello_ok else 'DOWN'}`",
f"- **local Next(ops) :{LOCAL_OPS_HTTP}**: `{'up' if ops_ui_ok else 'DOWN'}`",
f"- **local ops agent :{LOCAL_OPS_API}**: `{'up' if ops_api_ok else 'DOWN'}`",
f"- **local bridge :{LOCAL_WS}**: `{'up' if bridge_ok else 'DOWN'}`",
f"- **tunnel_restarts**: `{restarts}`",
f"- **note**: {note}",
"",
f"IDC bind: `127.0.0.1:{REMOTE_HTTP}` → home `:{LOCAL_HTTP}` (baduk)",
f"IDC bind: `127.0.0.1:{REMOTE_STOKU_HTTP}` → home `:{LOCAL_STOKU_HTTP}` (stoku)",
f"IDC bind: `127.0.0.1:{REMOTE_OMOOK_HTTP}` → home `:{LOCAL_OMOOK_HTTP}` (omook)",
f"IDC bind: `127.0.0.1:{REMOTE_OTHELLO_HTTP}` → home `:{LOCAL_OTHELLO_HTTP}` (othello)",
f"IDC bind: `127.0.0.1:{REMOTE_OPS_HTTP}` → home `:{LOCAL_OPS_HTTP}` (ops ui)",
f"IDC bind: `127.0.0.1:{REMOTE_OPS_API}` → home `:{LOCAL_OPS_API}` (ops api)",
f"IDC bind: `127.0.0.1:{REMOTE_WS}` → home `:{LOCAL_WS}` (ws)",
"",
"로그: `logs/mariago_ssh_tunnel/tunnel.watchdog.log`",
"",
]
HEART.write_text("\n".join(lines), encoding="utf-8")
def main() -> int:
OUT.mkdir(parents=True, exist_ok=True)
log(
"ssh reverse tunnel watchdog start "
f"({SSH_HOST} R:{REMOTE_HTTP}->{LOCAL_HTTP}, "
f"R:{REMOTE_STOKU_HTTP}->{LOCAL_STOKU_HTTP}, "
f"R:{REMOTE_OMOOK_HTTP}->{LOCAL_OMOOK_HTTP}, "
f"R:{REMOTE_OTHELLO_HTTP}->{LOCAL_OTHELLO_HTTP}, "
f"R:{REMOTE_OPS_HTTP}->{LOCAL_OPS_HTTP}, "
f"R:{REMOTE_OPS_API}->{LOCAL_OPS_API}, "
f"R:{REMOTE_WS}->{LOCAL_WS})"
)
ensure_caffeinate()
restarts = 0
while True:
if RESTART_FLAG.exists():
reason = RESTART_FLAG.read_text(encoding="utf-8").strip()
try:
RESTART_FLAG.unlink()
except OSError:
pass
log(f"external probe requested tunnel restart: {reason}")
restarts += 1
restart_tunnel(restarts)
http_ok, stoku_ok, omook_ok, ws_ok, ops_ui_ok, ops_api_ok, othello_ok = (
local_backends_ok()
)
t_pids = tunnel_pids()
tunnel_ok = bool(t_pids)
remote_ok = False
# 구버전 터널(스토쿠/오목/오델로/OPS 포워드 없음)이면 강제 재시작
cmd = current_tunnel_cmd()
if tunnel_ok and cmd and not tunnel_has_stoku(cmd):
log("tunnel missing stoku forward (:18081→:3001) → force restart")
tunnel_ok = False
if tunnel_ok and cmd and not tunnel_has_omook(cmd):
log("tunnel missing omook forward (:18082→:3002) → force restart")
tunnel_ok = False
if tunnel_ok and cmd and not tunnel_has_othello(cmd):
log("tunnel missing othello forward (:18085→:3004) → force restart")
tunnel_ok = False
if tunnel_ok and cmd and not tunnel_has_ops(cmd):
log("tunnel missing ops forward (:18083→:3003, :18084→:3040) → force restart")
tunnel_ok = False
if tunnel_ok:
remote_ok = remote_ports_ok()
if not remote_ok:
log("tunnel process alive but remote ports DOWN → force restart")
tunnel_ok = False
if not tunnel_ok:
restarts += 1
tunnel_ok = restart_tunnel(restarts)
remote_ok = remote_ports_ok() if tunnel_ok else False
note = "polling"
if not http_ok:
note = "WARN local Next(baduk) :3000 DOWN (tunnel may 502)"
log(note)
if not stoku_ok:
note = "WARN local Next(stoku) :3001 DOWN (stoku.mariago.kr may 502)"
log(note)
if not omook_ok:
note = "WARN local Next(omook) :3002 DOWN (omook.mariago.kr may 502)"
log(note)
if not othello_ok:
note = "WARN local Next(othello) :3004 DOWN (othello.mariago.kr may 502)"
log(note)
if not ws_ok:
note = "WARN local bridge :9320 DOWN (WS may 502)"
log(note)
kill_pids(pgrep("mariago_ws_bridge.py"), "bridge")
spawn(
[PY, "scripts/mariago_ws_bridge.py", "--ws-port", str(BRIDGE_PORT)],
ROOT,
BRIDGE_LOG,
)
time.sleep(2)
ws_ok = port_open(BRIDGE_PORT)
if not ops_api_ok:
note = "WARN local ops agent :3040 DOWN (ops.mariago.kr API 502)"
log(note)
write_heartbeat(
tunnel_ok=tunnel_ok,
remote_ok=remote_ok,
next_ok=http_ok,
stoku_ok=stoku_ok,
omook_ok=omook_ok,
bridge_ok=ws_ok,
ops_ui_ok=ops_ui_ok,
ops_api_ok=ops_api_ok,
othello_ok=othello_ok,
restarts=restarts,
note=note,
)
time.sleep(POLL_S)
if __name__ == "__main__":
try:
raise SystemExit(main())
except KeyboardInterrupt:
log("tunnel watchdog interrupted")
raise SystemExit(0)