예제 소스

main_llm_db_rl_viewer_legacy.py

"""
DB 캔들 → RL(REINFORCE)로 매수·매도 시점 학습(매도 승률·수익률 보상) 후,
viewer.py 와 동일한 Tk 화면으로 재생 · LLM은 참고(비전)만 표시.

설정은 argparse 없이 다음 파일만 수정합니다.
  - config/llim_rl_backend.py       백엔드·모델 경로
  - config/llim_rl_model_download.py HF 스냅샷 다운로드 여부
  - config/llim_rl_train.py          학습·뷰어·LABELS_CSV_PATH

극단 라벨: 먼저 ``python build_market_extreme_labels.py`` 로 CSV 생성 후
``LABELS_CSV_PATH`` 가 가리키는 파일이 있으면 RL 상태(17차원)에 반영됩니다.

실행 (프로젝트 루트에서):
  python main_llim_db_rl_viewer.py
"""

from __future__ import annotations

import json
import logging.config
import warnings
from pathlib import Path

import numpy as np

warnings.filterwarnings("ignore")
np.seterr(divide="ignore", invalid="ignore")

_cfg = Path(__file__).resolve().parent / "config" / "logging.json"
if _cfg.exists():
    with open(_cfg, "r", encoding="utf-8") as f:
        logging.config.dictConfig(json.load(f))

import logging

logger = logging.getLogger(__name__)

from config import llm_rl_backend as be  # noqa: E402
from config import llm_rl_model_download as dl  # noqa: E402
from config import llm_rl_train as tr  # noqa: E402
from lib.rl_chart_trainer import load_policy, train_reinforce_episodes  # noqa: E402
from lib.rl_data_labels import merge_extreme_labels_from_csv  # noqa: E402
from lib.rl_policy_teacher import load_policy_markdown  # noqa: E402
from lib.rl_vision_viewer import RlVisionBacktestViewer  # noqa: E402
from Market import Market  # noqa: E402
import main_llm_db_test_ollama as ollama_bt  # noqa: E402
from viewer import (  # noqa: E402
    DEFAULT_MLX_MODEL_DIR,
    DEFAULT_TORCH_MODEL_DIR,
    download_hf_snapshot,
    make_local_vision_callable,
    make_ollama_vision_callable,
    resolve_model_path,
)


def _maybe_download_hf() -> None:
    if not dl.ENABLE_SNAPSHOT:
        return
    models_dir = Path(be.MODELS_DIR).resolve()
    models_dir.mkdir(parents=True, exist_ok=True)
    logger.info("HF 스냅샷 다운로드: %s", dl.REPO_ID)
    download_hf_snapshot(dl.REPO_ID.strip(), models_dir)


def _build_vision_callable():
    if be.BACKEND == "ollama":
        vision = make_ollama_vision_callable(
            host=str(be.OLLAMA_HOST).rstrip("/"),
            model=str(be.OLLAMA_MODEL).strip(),
        )
        v_retry = max(1, int(ollama_bt.OLLAMA_RETRY))
        v_backoff = float(ollama_bt.OLLAMA_RETRY_BACKOFF_SEC)
        status_lbl = f"ollama · {be.OLLAMA_MODEL}"
        return vision, v_retry, v_backoff, status_lbl

    models_dir = Path(be.MODELS_DIR).resolve()
    default_name = (
        DEFAULT_TORCH_MODEL_DIR if be.BACKEND == "torch" else DEFAULT_MLX_MODEL_DIR
    )
    folder = be.TORCH_MODEL_FOLDER if be.BACKEND == "torch" else be.MLX_MODEL_FOLDER
    mp = resolve_model_path(models_dir, folder.strip())
    if not mp.exists():
        raise SystemExit(
            f"모델 경로 없음: {mp}\n"
            "  config/llim_rl_model_download.py 에서 ENABLE_SNAPSHOT = True 로 받거나 폴더를 채우세요."
        )
    vision = make_local_vision_callable(
        be.BACKEND,
        str(mp),
        max_new_tokens_torch=int(be.MAX_NEW_TOKENS_TORCH),
        max_tokens_mlx=int(be.MAX_TOKENS_MLX),
    )
    import os

    v_retry = int(os.environ.get("LOCAL_LLM_RETRY", "2"))
    v_backoff = float(os.environ.get("LOCAL_LLM_RETRY_BACKOFF_SEC", "1.0"))
    status_lbl = f"{be.BACKEND} · {mp.name}"
    return vision, v_retry, v_backoff, status_lbl


def main() -> None:
    _maybe_download_hf()

    vision, v_retry, v_backoff, status_lbl = _build_vision_callable()

    logger.info("Market.get_data() …")
    market = Market()
    df = market.get_data()
    if df is None or len(df) < tr.MIN_BARS + tr.BAR_STEP:
        raise SystemExit("데이터가 부족합니다.")

    n_full = len(df)
    cap = getattr(tr, "RL_MAX_BARS", None)
    bar_offset = 0
    if cap is not None and n_full > int(cap):
        bar_offset = n_full - int(cap)
        df = df.iloc[-int(cap) :].reset_index(drop=True)
        logger.info("RL_MAX_BARS=%s 적용: tail %s봉 (전체 %s봉, bar_offset=%s)", cap, len(df), n_full, bar_offset)

    df = merge_extreme_labels_from_csv(df, tr.LABELS_CSV_PATH, source_bar_offset=bar_offset)

    policy_doc_path = getattr(tr, "POLICY_DOC_PATH", None)
    vision_policy_doc = (
        load_policy_markdown(policy_doc_path)
        if getattr(tr, "********", False)
        else ""
    )

    if not getattr(tr, "ONLINE_TRAIN", False):
        logger.info("강화학습 시작 … rows=%s", len(df))
        policy = train_reinforce_episodes(
            df,
            episodes=tr.EPISODES,
            lr=tr.LEARNING_RATE,
            hidden_dim=tr.HIDDEN_DIM,
            weight_decay=tr.WEIGHT_DECAY,
            min_bars=max(50, tr.MIN_BARS),
            bar_step=max(1, tr.BAR_STEP),
            initial_krw=float(tr.INITIAL_KRW),
            min_trade_bars=max(0, tr.MIN_TRADE_BARS),
            checkpoint_path=tr.RL_CHECKPOINT_PATH,
            reward_win_weight=tr.REWARD_WIN_RATE_WEIGHT,
            reward_profit_scale=tr.REWARD_PROFIT_SCALE,
            load_if_exists=True,
            policy_doc_path=policy_doc_path,
            teacher_kl_coef=float(getattr(tr, "RL_TEACHER_KL_COEF", 0.0)),
            teacher_temperature=float(getattr(tr, "RL_TEACHER_TEMPERATURE", 1.0)),
            teacher_pretrain_episodes=int(getattr(tr, "RL_TEACHER_PRETRAIN_EPISODES", 0)),
            save_train_meta=bool(getattr(tr, "********", True)),
        )

        if not tr.OPEN_VIEWER_AFTER_TRAIN:
            logger.info("OPEN_VIEWER_AFTER_TRAIN=False 이므로 뷰어를 띄우지 않고 종료합니다.")
            return
    else:
        # 온라인 모드: 뷰어 먼저 띄우고, 뷰어 진행으로 무한 학습
        policy = load_policy(tr.RL_CHECKPOINT_PATH, hidden_dim=tr.HIDDEN_DIM)

    port = ollama_bt.PaperLlmPortfolio(float(tr.INITIAL_KRW))
    port.min_trade_bars = max(0, int(tr.MIN_TRADE_BARS))

    app = RlVisionBacktestViewer(
        df=df,
        portfolio=port,
        vision_call=vision,
        rl_policy=policy,
        online_train=bool(getattr(tr, "ONLINE_TRAIN", False)),
        online_sample_action=bool(getattr(tr, "ONLINE_SAMPLE_ACTION", True)),
        online_max_episodes=getattr(tr, "ONLINE_MAX_EPISODES", None),
        online_save_every_ep=int(getattr(tr, "ONLINE_SAVE_EVERY_EP", 1)),
        online_clear_text_each_ep=bool(getattr(tr, "ONLINE_CLEAR_TEXT_EACH_EP", False)),
        checkpoint_path=str(tr.RL_CHECKPOINT_PATH),
        train_lr=float(tr.LEARNING_RATE),
        train_weight_decay=float(tr.WEIGHT_DECAY),
        reward_win_weight=float(tr.REWARD_WIN_RATE_WEIGHT),
        reward_profit_scale=float(tr.REWARD_PROFIT_SCALE),
        initial_krw=float(tr.INITIAL_KRW),
        min_trade_bars=int(tr.MIN_TRADE_BARS),
        min_bars=max(50, tr.MIN_BARS),
        bar_step=max(1, tr.BAR_STEP),
        dry_run=tr.DRY_RUN,
        auto_interval_sec=tr.AUTO_INTERVAL_SEC,
        chart_max_width=tr.CHART_MAX_WIDTH,
        vision_retries=max(1, int(be.VISION_RETRIES)),
        vision_backoff_sec=float(be.VISION_BACKOFF_SEC),
        window_title=tr.WINDOW_TITLE,
        status_backend_label=status_lbl,
        teacher_kl_coef=float(getattr(tr, "RL_TEACHER_KL_COEF", 0.0)),
        teacher_temperature=float(getattr(tr, "RL_TEACHER_TEMPERATURE", 1.0)),
        llm_align_weight=float(getattr(tr, "RL_LLM_ALIGN_WEIGHT", 0.0)),
        vision_policy_doc=vision_policy_doc or None,
        policy_doc_path_for_meta=str(policy_doc_path) if policy_doc_path else None,
        save_train_meta_with_checkpoint=bool(
            getattr(tr, "********", False)
        ),
    )
    if bool(getattr(tr, "ONLINE_TRAIN", False)) and bool(getattr(tr, "ONLINE_AUTOSTART", False)):
        app.auto_mode.set(True)
        app.btn_stop.configure(state="normal")
        app._start_step()
    app.mainloop()


if __name__ == "__main__":
    main()