import gymnasium as gym
import numpy as np
from gymnasium import spaces

class CryptoTradingEnv(gym.Env):
    def __init__(self, df, initial_balance=100000, transaction_cost=0.001):
        super(CryptoTradingEnv, self).__init__()
        self.df = df.reset_index(drop=True)
        self.initial_balance = initial_balance
        self.transaction_cost = transaction_cost
        self.n_steps = len(df)

        # 状態空間：OHLCV + テクニカル指標 + 残高比率 + ポジション量
        n_features = df.shape[1] + 2
        self.observation_space = spaces.Box(
            low=-np.inf, high=np.inf,
            shape=(n_features,), dtype=np.float32
        )

        # 行動空間：0=ホールド, 1=買い, 2=売り
        self.action_space = spaces.Discrete(3)

    def reset(self, seed=None):
        super().reset(seed=seed)
        self.current_step = 0
        self.balance = self.initial_balance
        self.position = 0.0
        self.total_profit = 0.0
        self.returns = []
        return self._get_obs(), {}

    def _get_obs(self):
        row = self.df.iloc[self.current_step].values.astype(np.float32)
        balance_ratio = np.array([self.balance / self.initial_balance], dtype=np.float32)
        position = np.array([self.position], dtype=np.float32)
        return np.concatenate([row, balance_ratio, position])

    def step(self, action):
        price = self.df.iloc[self.current_step]["close"]
        prev_balance = self.balance + self.position * price

        # 行動の実行
        if action == 1 and self.balance > 0:    # 買い
            cost = self.balance * (1 - self.transaction_cost)
            self.position += cost / price
            self.balance = 0.0
        elif action == 2 and self.position > 0: # 売り
            revenue = self.position * price * (1 - self.transaction_cost)
            self.balance += revenue
            self.position = 0.0

        self.current_step += 1
        done = self.current_step >= self.n_steps - 1

        current_value = self.balance + self.position * price
        step_return = (current_value - prev_balance) / (prev_balance + 1e-8)
        self.returns.append(step_return)

        # シャープレシオベース報酬（学習の要）
        if len(self.returns) > 1:
            mean_r = np.mean(self.returns)
            std_r  = np.std(self.returns) + 1e-8
            reward = mean_r / std_r
        else:
            reward = step_return

        return self._get_obs(), reward, done, False, {}

python -m pip install stable-baselines3 tensorboard optuna

from stable_baselines3 import PPO
from stable_baselines3.common.env_util import make_vec_env
from stable_baselines3.common.callbacks import EvalCallback, StopTrainingOnNoModelImprovement
import pandas as pd

# 第1・2部のデータパイプラインを再利用
df_train = pd.read_csv("features_train.csv")
df_val   = pd.read_csv("features_val.csv")

train_env = make_vec_env(lambda: CryptoTradingEnv(df_train), n_envs=4)
val_env   = CryptoTradingEnv(df_val)

# 早期停止コールバック
stop_callback = StopTrainingOnNoModelImprovement(
    max_no_improvement_evals=10, min_evals=20, verbose=1
)
eval_callback = EvalCallback(
    val_env,
    best_model_save_path="./logs/best_model",
    eval_freq=5000,
    callback_after_eval=stop_callback,
    verbose=1
)

# PPOエージェントの定義
model = PPO(
    "MlpPolicy",
    train_env,
    learning_rate=3e-4,
    n_steps=2048,
    batch_size=64,
    clip_range=0.2,
    verbose=1,
    tensorboard_log="./logs/tensorboard/"
)

# 学習実行
model.learn(total_timesteps=500_000, callback=eval_callback)
model.save("ppo_crypto_trader")

tensorboard --logdir ./logs/tensorboard/

from stable_baselines3 import PPO
from stable_baselines3.common.env_util import make_vec_env
from stable_baselines3.common.callbacks import EvalCallback, StopTrainingOnNoModelImprovement
import numpy as np
import pandas as pd

def calculate_max_drawdown(returns: np.ndarray) -> float:
    cumulative  = np.cumprod(1 + returns)
    running_max = np.maximum.accumulate(cumulative)
    drawdown    = (cumulative - running_max) / (running_max + 1e-8)
    return float(drawdown.min())

def walk_forward_rl(df, train_size=365, val_size=90, test_size=90):
    results = []
    start   = 0

    while start + train_size + val_size + test_size <= len(df):
        df_tr   = df.iloc[start : start + train_size]
        df_val_ = df.iloc[start + train_size : start + train_size + val_size]
        df_te   = df.iloc[start + train_size + val_size :
                          start + train_size + val_size + test_size]

        # 訓練フェーズ
        train_env = make_vec_env(lambda: CryptoTradingEnv(df_tr), n_envs=4)
        val_env   = CryptoTradingEnv(df_val_)

        stop_cb = StopTrainingOnNoModelImprovement(
            max_no_improvement_evals=10, min_evals=10, verbose=0
        )
        eval_cb = EvalCallback(
            val_env, eval_freq=5000,
            callback_after_eval=stop_cb, verbose=0
        )

        model = PPO("MlpPolicy", train_env,
                    learning_rate=3e-4, n_steps=2048, verbose=0)
        model.learn(total_timesteps=200_000, callback=eval_cb)

        # テスト評価フェーズ
        test_env = CryptoTradingEnv(df_te)
        obs, _   = test_env.reset()
        done     = False
        while not done:
            action, _ = model.predict(obs, deterministic=True)
            obs, _, done, _, _ = test_env.step(action)

        # メトリクス計算
        returns      = np.array(test_env.returns)
        final_value  = (test_env.balance
                        + test_env.position * df_te.iloc[-1]["close"])
        total_return = final_value / test_env.initial_balance - 1
        sharpe       = (np.mean(returns) / (np.std(returns) + 1e-8)
                        * np.sqrt(365))
        max_dd       = calculate_max_drawdown(returns)

        results.append({
            "period_start" : df_te.index[0],
            "total_return" : round(total_return, 4),
            "sharpe_ratio" : round(sharpe, 4),
            "max_drawdown" : round(max_dd, 4),
        })

        start += test_size

    return pd.DataFrame(results)

# 実行例
df_all     = pd.read_csv("features_all.csv",
                         index_col="timestamp", parse_dates=True)
wf_results = walk_forward_rl(df_all)
print(wf_results)