import ccxt
import pandas as pd
import time

def fetch_ohlcv_all(
    symbol: str = 'BTC/USDT',
    timeframe: str = '1d',
    start_date: str = '2022-01-01',
    exchange_name: str = 'binance'
) -> pd.DataFrame:
    """ページネーションを使った全件 OHLCV 取得"""
    exchange = getattr(ccxt, exchange_name)({
        'rateLimit': 1200,
        'enableRateLimit': True,
    })

    since = exchange.parse8601(f'{start_date}T00:00:00Z')
    all_ohlcv = []
    limit = 1000  # Binance の 1 リクエスト上限

    while True:
        try:
            ohlcv = exchange.fetch_ohlcv(symbol, timeframe, since=since, limit=limit)
            if not ohlcv:
                break
            all_ohlcv.extend(ohlcv)
            since = ohlcv[-1][0] + 1
            exchange.sleep(1000)  # レート制限対策
            if since > exchange.milliseconds():
                break
        except ccxt.NetworkError as e:
            print(f"ネットワークエラー: {e} → リトライ")
            time.sleep(3)

    df = pd.DataFrame(all_ohlcv, columns=['timestamp', 'open', 'high', 'low', 'close', 'volume'])
    df['timestamp'] = pd.to_datetime(df['timestamp'], unit='ms', utc=True)
    df = df.set_index('timestamp').sort_index()
    df = df[~df.index.duplicated(keep='first')]
    return df.astype(float)

# ① 欠損値の確認
print(df.isnull().sum())

# ② タイムスタンプの連続性チェック
expected_freq = pd.tseries.frequencies.to_offset('1D')
gap = df.index.to_series().diff().dropna()
print(gap[gap > expected_freq])  # 欠損している時間帯を検出

# ③ 異常値（スパイク）の検出
z_score = (df['close'] - df['close'].mean()) / df['close'].std()
print(df[z_score.abs() > 4])  # 4σ 以上の外れ値

# 前後値補完（Forward Fill → Backward Fill）
df = df.fillna(method='ffill').fillna(method='bfill')

# 短期の欠損には線形補完が自然な近似となる
df = df.interpolate(method='linear')

from statsmodels.tsa.stattools import adfuller
import numpy as np

# ADF 検定で定常性を確認
result = adfuller(df['close'].dropna())
print(f'ADF 統計量: {result[0]:.4f}, p値: {result[1]:.4f}')
# p値 > 0.05 → 非定常（変換が必要）

# 対数変換でスケールを安定化
df['log_close'] = np.log(df['close'])

# 差分変換でトレンドを除去
df['diff_close'] = df['close'].diff()

from sklearn.preprocessing import MinMaxScaler

scaler = MinMaxScaler(feature_range=(0, 1))

# ⚠️ 重要：fit はトレーニングデータのみで行う
train_size = int(len(df) * 0.7)
scaler.fit(df[:train_size][feature_cols])

df_scaled = scaler.transform(df[feature_cols])

import numpy as np

def create_sequences(data: np.ndarray, window_size: int = 10):
    """スライドウィンドウで入力系列とターゲットを生成"""
    X, y = [], []
    for i in range(window_size, len(data)):
        X.append(data[i - window_size:i])   # 過去 window_size ステップ
        y.append(data[i, 3])                 # 翌ステップの終値（Close = index 3）
    return np.array(X), np.array(y)

# LSTM 用 3D テンソル: (samples, timesteps, features)
X, y = create_sequences(df_scaled, window_size=10)
print(X.shape)  # → (samples, 10, n_features)

total = len(X)
train_end = int(total * 0.70)
val_end   = int(total * 0.85)

X_train, y_train = X[:train_end],       y[:train_end]
X_val,   y_val   = X[train_end:val_end], y[train_end:val_end]
X_test,  y_test  = X[val_end:],          y[val_end:]

# 単純移動平均（SMA）
df['MA_25']  = df['close'].rolling(window=25).mean()
df['MA_75']  = df['close'].rolling(window=75).mean()
df['MA_200'] = df['close'].rolling(window=200).mean()

# 指数移動平均（EMA）：直近の価格に高いウェイト
df['EMA_12'] = df['close'].ewm(span=12, adjust=False).mean()
df['EMA_26'] = df['close'].ewm(span=26, adjust=False).mean()

# MACD：EMA の差でトレンドの強さと方向を計測
df['MACD']        = df['EMA_12'] - df['EMA_26']
df['MACD_signal'] = df['MACD'].ewm(span=9, adjust=False).mean()
df['MACD_hist']   = df['MACD'] - df['MACD_signal']

# RSI（14期間が標準）
def compute_rsi(series: pd.Series, period: int = 14) -> pd.Series:
    delta = series.diff()
    gain  = delta.clip(lower=0).rolling(period).mean()
    loss  = (-delta.clip(upper=0)).rolling(period).mean()
    rs    = gain / loss
    return 100 - (100 / (1 + rs))

df['RSI'] = compute_rsi(df['close'])

# ストキャスティクス（%K・%D）
low_14  = df['low'].rolling(14).min()
high_14 = df['high'].rolling(14).max()
df['stoch_k'] = 100 * (df['close'] - low_14) / (high_14 - low_14)
df['stoch_d'] = df['stoch_k'].rolling(3).mean()

# ボリンジャーバンド（20期間・2σ）
rolling_mean = df['close'].rolling(20).mean()
rolling_std  = df['close'].rolling(20).std()
df['BB_upper']  = rolling_mean + (2 * rolling_std)
df['BB_middle'] = rolling_mean
df['BB_lower']  = rolling_mean - (2 * rolling_std)
df['BB_width']  = (df['BB_upper'] - df['BB_lower']) / df['BB_middle']  # バンド幅

# ATR（Average True Range）：ボラティリティの絶対値
df['TR'] = pd.concat([
    df['high'] - df['low'],
    (df['high'] - df['close'].shift()).abs(),
    (df['low']  - df['close'].shift()).abs()
], axis=1).max(axis=1)
df['ATR'] = df['TR'].rolling(14).mean()

# 価格内部特徴量
df['HL_range']   = df['high'] - df['low']          # 値幅
df['CO_diff']    = df['close'] - df['open']         # 終値-始値
df['HC_diff']    = df['high'] - df['close']         # 高値-終値（上ひげ）

# 出来高変化率
df['vol_change'] = df['volume'].pct_change()

# 相関分析で重要特徴量を確認
corr = df[feature_cols + ['close']].corr()['close'].sort_values(ascending=False)
print(corr.head(10))

from ta import add_all_ta_features

df = add_all_ta_features(
    df, open='open', high='high', low='low', close='close', volume='volume',
    fillna=True
)

# ✅ 正しい設計：t〜t+window-1 で t+window を予測
X.append(data[i - window_size:i])  # 過去データ
y.append(data[i])                  # 翌ステップのターゲット（1つ先）

# 相関フィルタリング：閾値以上の相関を持つ特徴量を除去
corr_matrix = df[feature_cols].corr().abs()
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1).astype(bool))
to_drop = [col for col in upper.columns if any(upper[col] > 0.95)]
df = df.drop(columns=to_drop)