gymnasium          # 環境インターフェースの標準仕様
stable-baselines3  # PPO・DQN・SACなどの実装済みアルゴリズム
pandas-ta          # テクニカル指標の計算
ccxt               # 取引所API接続（本番運用時）

pip3 install gymnasium stable-baselines3 pandas-ta ccxt

import numpy as np
import pandas as pd
import pandas_ta as ta
import gymnasium as gym
from gymnasium import spaces


class CryptoTradingEnv(gym.Env):
    """
    BTC/USDT 仮想通貨トレード環境
    行動空間: 0=保有, 1=買い, 2=売り（離散型）
    状態空間: テクニカル指標 + ポジション情報
    """
    metadata = {"render_modes": ["human"]}

    def __init__(self, df: pd.DataFrame, initial_balance: float = 10_000.0,
                 fee_rate: float = 0.001, window_size: int = 20):
        super().__init__()

        self.df = df.reset_index(drop=True)
        self.initial_balance = initial_balance
        self.fee_rate = fee_rate        # 取引手数料（0.1%）
        self.window_size = window_size

        # --- テクニカル指標の事前計算 ---
        self.df["rsi"] = ta.rsi(self.df["close"], length=14)
        macd = ta.macd(self.df["close"])
        self.df["macd"] = macd["MACD_12_26_9"]
        self.df["macd_signal"] = macd["MACDs_12_26_9"]
        bb = ta.bbands(self.df["close"], length=20)
        self.df["bb_upper"] = bb["BBU_20_2.0"]
        self.df["bb_lower"] = bb["BBL_20_2.0"]
        self.df.dropna(inplace=True)
        self.df.reset_index(drop=True, inplace=True)

        # 特徴量カラム
        self.feature_cols = ["close", "volume", "rsi", "macd",
                             "macd_signal", "bb_upper", "bb_lower"]
        n_features = len(self.feature_cols) + 2  # +2: ポジション・残高比率

        # --- 行動空間（離散: 保有/買い/売り） ---
        self.action_space = spaces.Discrete(3)

        # --- 状態空間 ---
        self.observation_space = spaces.Box(
            low=-np.inf, high=np.inf,
            shape=(self.window_size, n_features),
            dtype=np.float32
        )

    def reset(self, seed=None, options=None):
        super().reset(seed=seed)
        self.current_step = self.window_size
        self.balance = self.initial_balance
        self.position = 0.0      # 保有BTC量
        self.entry_price = 0.0
        self.total_pnl = 0.0
        self.returns = []

        obs = self._get_observation()
        return obs, {}

    def step(self, action: int):
        current_price = self.df.loc[self.current_step, "close"]
        prev_portfolio = self._get_portfolio_value(current_price)

        # --- 行動の実行 ---
        if action == 1 and self.position == 0:   # 買い
            amount = self.balance / current_price
            fee = amount * current_price * self.fee_rate
            self.position = amount
            self.balance -= (amount * current_price + fee)
            self.entry_price = current_price

        elif action == 2 and self.position > 0:  # 売り
            proceeds = self.position * current_price
            fee = proceeds * self.fee_rate
            self.balance += proceeds - fee
            self.position = 0.0
            self.entry_price = 0.0

        # --- 次のステップへ ---
        self.current_step += 1
        terminated = self.current_step >= len(self.df) - 1
        truncated = self.balance <= self.initial_balance * 0.1  # 残高90%損失で強制終了

        new_price = self.df.loc[self.current_step, "close"]
        new_portfolio = self._get_portfolio_value(new_price)

        # --- 報酬の計算（後述の第3章で詳述） ---
        step_return = (new_portfolio - prev_portfolio) / prev_portfolio
        self.returns.append(step_return)
        reward = self._compute_reward(step_return, truncated)

        obs = self._get_observation()
        info = {"portfolio_value": new_portfolio, "balance": self.balance,
                "position": self.position}

        return obs, reward, terminated, truncated, info

    def _get_observation(self):
        """直近 window_size ステップの特徴量を取得"""
        start = self.current_step - self.window_size
        end = self.current_step
        frame = self.df.loc[start:end - 1, self.feature_cols].copy()

        # 正規化（各カラムをウィンドウ内の最大値でスケーリング）
        frame = (frame - frame.mean()) / (frame.std() + 1e-8)

        # ポジション情報を付加
        current_price = self.df.loc[self.current_step - 1, "close"]
        portfolio_value = self._get_portfolio_value(current_price)
        position_flag = np.full((self.window_size, 1), float(self.position > 0))
        balance_ratio = np.full((self.window_size, 1),
                                self.balance / portfolio_value if portfolio_value > 0 else 1.0)

        obs = np.hstack([frame.values, position_flag, balance_ratio])
        return obs.astype(np.float32)

    def _get_portfolio_value(self, price: float) -> float:
        return self.balance + self.position * price

    def _compute_reward(self, step_return: float, truncated: bool) -> float:
        # 破産ペナルティ
        if truncated:
            return -10.0

        # シャープレシオベースの報酬（直近50ステップ）
        if len(self.returns) >= 50:
            recent = np.array(self.returns[-50:])
            sharpe = np.mean(recent) / (np.std(recent) + 1e-8) * np.sqrt(252 * 1440)
            return float(sharpe)
        return step_return * 100

def _compute_reward(self, step_return: float, truncated: bool) -> float:
    """
    複合報酬関数：
    1. シャープレシオ（リスク調整済みリターン）
    2. ドローダウン罰則
    3. 取引コスト反映（step内で費用控除済み）
    """
    if truncated:
        return -10.0  # 破産ペナルティ

    reward = step_return * 100  # ベース報酬

    # シャープレシオ成分（直近50ステップ）
    if len(self.returns) >= 50:
        recent = np.array(self.returns[-50:])
        sharpe_component = (
            np.mean(recent) / (np.std(recent) + 1e-8) * np.sqrt(252 * 1440)
        )
        reward += sharpe_component * 0.5

    # 最大ドローダウン罰則
    if len(self.returns) >= 2:
        cumulative = np.cumprod(1 + np.array(self.returns))
        peak = np.maximum.accumulate(cumulative)
        drawdown = (peak - cumulative) / (peak + 1e-8)
        max_dd = drawdown[-1]
        if max_dd > 0.1:   # 10%以上のドローダウンにペナルティ
            reward -= max_dd * 5.0

    return float(reward)

import pandas as pd
from stable_baselines3 import PPO
from stable_baselines3.common.env_util import make_vec_env
from stable_baselines3.common.callbacks import EvalCallback, StopTrainingOnNoModelImprovement
from stable_baselines3.common.monitor import Monitor

# --- データ読み込み（例：Binance BTC/USDT 1分足） ---
df = pd.read_csv("btcusdt_1m.csv")  # columns: open, high, low, close, volume
train_df = df.iloc[:int(len(df) * 0.7)].reset_index(drop=True)
eval_df  = df.iloc[int(len(df) * 0.7):int(len(df) * 0.85)].reset_index(drop=True)

# --- 環境の作成 ---
train_env = make_vec_env(
    lambda: Monitor(CryptoTradingEnv(train_df)),
    n_envs=4  # 並列環境数（学習速度向上）
)
eval_env = Monitor(CryptoTradingEnv(eval_df))

# --- コールバック設定 ---
stop_callback = StopTrainingOnNoModelImprovement(
    max_no_improvement_evals=10, min_evals=20, verbose=1
)
eval_callback = EvalCallback(
    eval_env,
    best_model_save_path="./models/",
    log_path="./logs/",
    eval_freq=5000,
    callback_after_eval=stop_callback,
    verbose=1
)

# --- PPOモデルの定義 ---
model = PPO(
    policy="MlpPolicy",
    env=train_env,
    learning_rate=3e-4,
    n_steps=2048,        # ロールアウト長
    batch_size=64,
    n_epochs=10,         # 各ロールアウトの学習回数
    gamma=0.99,          # 割引率
    gae_lambda=0.95,     # GAEパラメータ
    clip_range=0.2,      # PPOクリッピング
    ent_coef=0.01,       # エントロピーボーナス（探索促進）
    verbose=1,
    tensorboard_log="./tb_logs/"
)

# --- 学習の実行 ---
model.learn(total_timesteps=500_000, callback=eval_callback)
model.save("ppo_crypto_trader")

print("学習完了。最良モデルを ./models/ に保存しました。")

from stable_baselines3 import DQN

# DQNエージェント（離散行動向け）
dqn_model = DQN(
    policy="MlpPolicy",
    env=Monitor(CryptoTradingEnv(train_df)),
    learning_rate=1e-4,
    buffer_size=100_000,    # リプレイバッファ
    learning_starts=10_000,
    batch_size=32,
    gamma=0.99,
    target_update_interval=1000,
    exploration_fraction=0.1,
    verbose=1
)
dqn_model.learn(total_timesteps=500_000)

import numpy as np
from stable_baselines3 import PPO, DQN

def ensemble_predict(models, obs):
    """複数モデルの多数決アンサンブル"""
    actions = []
    for model in models:
        action, _ = model.predict(obs, deterministic=True)
        actions.append(int(action))
    # 多数決
    return max(set(actions), key=actions.count)

# 使用例
models = [
    PPO.load("models/ppo_v1"),
    PPO.load("models/ppo_v2"),
    DQN.load("models/dqn_v1"),
]

def walk_forward_validation(df, model_class, n_splits=5, train_ratio=0.6, val_ratio=0.2):
    """
    ウォークフォワード検証
    各分割で: 訓練→検証→評価 の順序を厳守
    """
    results = []
    split_size = len(df) // n_splits

    for i in range(n_splits):
        start = i * split_size
        train_end = start + int(split_size * train_ratio)
        val_end   = train_end + int(split_size * val_ratio)
        test_end  = start + split_size

        train_df = df.iloc[start:train_end].reset_index(drop=True)
        val_df   = df.iloc[train_end:val_end].reset_index(drop=True)
        test_df  = df.iloc[val_end:test_end].reset_index(drop=True)

        # 学習
        env = make_vec_env(lambda: Monitor(CryptoTradingEnv(train_df)), n_envs=2)
        model = model_class("MlpPolicy", env, verbose=0)
        model.learn(total_timesteps=200_000)

        # テスト期間で評価
        test_env = CryptoTradingEnv(test_df)
        obs, _ = test_env.reset()
        portfolio_values = []

        while True:
            action, _ = model.predict(obs, deterministic=True)
            obs, reward, terminated, truncated, info = test_env.step(action)
            portfolio_values.append(info["portfolio_value"])
            if terminated or truncated:
                break

        # シャープレシオ算出
        returns = np.diff(portfolio_values) / np.array(portfolio_values[:-1])
        sharpe = (np.mean(returns) / (np.std(returns) + 1e-8)) * np.sqrt(252 * 1440)
        max_dd = _calc_max_drawdown(portfolio_values)

        results.append({
            "fold": i + 1,
            "sharpe_ratio": round(sharpe, 3),
            "max_drawdown": round(max_dd, 3),
            "final_value": round(portfolio_values[-1], 2)
        })
        print(f"Fold {i+1}: Sharpe={sharpe:.3f}, MaxDD={max_dd:.1%}")

    return results


def _calc_max_drawdown(values):
    arr = np.array(values)
    peak = np.maximum.accumulate(arr)
    drawdown = (peak - arr) / (peak + 1e-8)
    return float(np.max(drawdown))

import ccxt
import time

class LiveTradingBot:
    def __init__(self, exchange_id: str, api_key: str, secret: str,
                 model_path: str, max_position_usd: float = 100.0):
        self.exchange = getattr(ccxt, exchange_id)({
            "apiKey": api_key,
            "secret": secret,
            "options": {"defaultType": "future"},
        })
        self.model = PPO.load(model_path)
        self.max_position_usd = max_position_usd
        self.max_drawdown_threshold = 0.15  # 15%で緊急停止

    def get_state(self, symbol: str, window: int = 20) -> np.ndarray:
        """最新のOHLCVデータから状態を構築"""
        ohlcv = self.exchange.fetch_ohlcv(symbol, "1m", limit=window + 50)
        df = pd.DataFrame(ohlcv, columns=["timestamp", "open", "high", "low", "close", "volume"])
        # テクニカル指標の計算は CryptoTradingEnv と同じロジックを使用
        # （省略）
        return state

    def execute_trade(self, symbol: str, action: int, current_price: float):
        """行動に基づいて注文を発行"""
        if action == 1:  # 買い
            amount = self.max_position_usd / current_price
            self.exchange.create_market_buy_order(symbol, amount)
        elif action == 2:  # 売り
            # 保有ポジションを全決済
            position = self.exchange.fetch_positions([symbol])
            if position and float(position[0]["contracts"]) > 0:
                self.exchange.create_market_sell_order(
                    symbol, float(position[0]["contracts"])
                )

    def run(self, symbol: str = "BTC/USDT"):
        """メインループ（1分足）"""
        initial_balance = float(self.exchange.fetch_balance()["USDT"]["total"])
        print(f"運用開始: 初期残高 ${initial_balance:,.2f}")

        while True:
            balance = float(self.exchange.fetch_balance()["USDT"]["total"])
            drawdown = (initial_balance - balance) / initial_balance

            # 緊急停止チェック
            if drawdown > self.max_drawdown_threshold:
                print(f"緊急停止: ドローダウン {drawdown:.1%} が閾値を超えました")
                break

            state = self.get_state(symbol)
            action, _ = self.model.predict(state, deterministic=True)
            ticker = self.exchange.fetch_ticker(symbol)
            self.execute_trade(symbol, int(action), ticker["last"])

            time.sleep(60)  # 1分待機

# pip install mamba-ssm（GPU環境が必要）
# stable-baselines3のカスタムポリシーとして組み込む例（概念コード）
from stable_baselines3.common.policies import ActorCriticPolicy
import torch.nn as nn

class MambaPolicy(ActorCriticPolicy):
    """Mambaアーキテクチャを使ったカスタムポリシー（概念実装）"""
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs,
                         net_arch=dict(pi=[256, 128], vf=[256, 128]))
    # Mamba層の組み込みは別途実装が必要