import numpy as np
from typing import Tuple, Dict, Any

class TradingEnvironment:
    """取引環境の基本構造"""

    def __init__(self):
        # 状態空間（State Space）
        self.state_space = {
            'price': None,           # 現在価格
            'position': None,        # ポジション（-1: ショート, 0: なし, 1: ロング）
            'balance': None,         # 残高
            'features': None         # テクニカル指標など
        }

        # 行動空間（Action Space）
        self.action_space = {
            0: 'HOLD',      # 何もしない
            1: 'BUY',       # 買い
            2: 'SELL',      # 売り
            3: 'CLOSE'      # ポジションクローズ
        }

        # 報酬関数（Reward Function）
        self.reward_function = self.calculate_reward

    def step(self, action: int) -> Tuple[Dict, float, bool, Dict]:
        """
        環境の1ステップ実行

        Returns:
            next_state: 次の状態
            reward: 報酬
            done: エピソード終了フラグ
            info: 追加情報
        """
        # 行動を実行
        self._execute_action(action)

        # 新しい状態を観測
        next_state = self._get_observation()

        # 報酬を計算
        reward = self.calculate_reward(action)

        # 終了判定
        done = self._is_done()

        return next_state, reward, done, {}

    def calculate_reward(self, action: int) -> float:
        """
        報酬関数の実装

        考慮すべき要素：
        - 実現損益
        - 未実現損益
        - 取引コスト
        - リスク調整済みリターン
        """
        reward = 0.0

        # 基本的な損益
        if self.state_space['position'] != 0:
            pnl = self._calculate_pnl()
            reward += pnl

        # 取引コストのペナルティ
        if action in [1, 2]:  # BUY or SELL
            reward -= self.transaction_cost

        # リスクペナルティ（ドローダウンなど）
        risk_penalty = self._calculate_risk_penalty()
        reward -= risk_penalty

        return reward

class ValueFunction:
    """価値関数の概念"""

    def state_value(self, state, policy):
        """
        状態価値関数 V(s)
        ある状態から方策πに従って行動した時の期待収益
        """
        V_s = 0
        for action in self.actions:
            # 方策に従った行動確率
            action_prob = policy(state, action)

            # 期待報酬
            expected_reward = 0
            for next_state, transition_prob in self.transitions(state, action):
                reward = self.reward(state, action, next_state)
                future_value = self.gamma * self.state_value(next_state, policy)
                expected_reward += transition_prob * (reward + future_value)

            V_s += action_prob * expected_reward

        return V_s

    def action_value(self, state, action, policy):
        """
        行動価値関数 Q(s,a)
        状態sで行動aを取った時の期待収益
        """
        Q_sa = 0
        for next_state, transition_prob in self.transitions(state, action):
            reward = self.reward(state, action, next_state)
            future_value = self.gamma * self.state_value(next_state, policy)
            Q_sa += transition_prob * (reward + future_value)

        return Q_sa

import torch
import torch.nn as nn
import torch.optim as optim
from collections import deque
import random

class DQN(nn.Module):
    """Deep Q-Network"""

    def __init__(self, state_size: int, action_size: int, hidden_sizes: list = [256, 256]):
        super(DQN, self).__init__()

        # ネットワーク構造
        layers = []
        input_size = state_size

        for hidden_size in hidden_sizes:
            layers.extend([
                nn.Linear(input_size, hidden_size),
                nn.ReLU(),
                nn.BatchNorm1d(hidden_size),
                nn.Dropout(0.2)
            ])
            input_size = hidden_size

        layers.append(nn.Linear(input_size, action_size))

        self.network = nn.Sequential(*layers)

    def forward(self, state):
        """Q値の計算"""
        return self.network(state)

class DQNAgent:
    """DQNエージェント"""

    def __init__(self, state_size: int, action_size: int, learning_rate: float = 0.001):
        self.state_size = state_size
        self.action_size = action_size
        self.memory = deque(maxlen=10000)
        self.epsilon = 1.0  # 探索率
        self.epsilon_min = 0.01
        self.epsilon_decay = 0.995
        self.gamma = 0.95  # 割引率

        # メインネットワークとターゲットネットワーク
        self.q_network = DQN(state_size, action_size)
        self.target_network = DQN(state_size, action_size)
        self.optimizer = optim.Adam(self.q_network.parameters(), lr=learning_rate)

        # ターゲットネットワークの初期化
        self.update_target_network()

    def remember(self, state, action, reward, next_state, done):
        """経験の記憶"""
        self.memory.append((state, action, reward, next_state, done))

    def act(self, state):
        """ε-greedy方策による行動選択"""
        if random.random() <= self.epsilon:
            return random.randrange(self.action_size)

        state_tensor = torch.FloatTensor(state).unsqueeze(0)
        q_values = self.q_network(state_tensor)
        return np.argmax(q_values.detach().numpy())

    def replay(self, batch_size: int = 32):
        """経験再生による学習"""
        if len(self.memory) < batch_size:
            return

        batch = random.sample(self.memory, batch_size)
        states = torch.FloatTensor([e[0] for e in batch])
        actions = torch.LongTensor([e[1] for e in batch])
        rewards = torch.FloatTensor([e[2] for e in batch])
        next_states = torch.FloatTensor([e[3] for e in batch])
        dones = torch.FloatTensor([e[4] for e in batch])

        current_q_values = self.q_network(states).gather(1, actions.unsqueeze(1))
        next_q_values = self.target_network(next_states).max(1)[0].detach()
        target_q_values = rewards + (self.gamma * next_q_values * (1 - dones))

        loss = nn.MSELoss()(current_q_values.squeeze(), target_q_values)

        self.optimizer.zero_grad()
        loss.backward()
        self.optimizer.step()

        # ε減衰
        if self.epsilon > self.epsilon_min:
            self.epsilon *= self.epsilon_decay

    def update_target_network(self):
        """ターゲットネットワークの更新"""
        self.target_network.load_state_dict(self.q_network.state_dict())

import pandas as pd
import numpy as np
from gym import Env, spaces

class CryptoTradingEnv(Env):
    """暗号資産取引環境"""

    def __init__(self, df: pd.DataFrame, initial_balance: float = 10000, 
                 transaction_fee: float = 0.001):
        super(CryptoTradingEnv, self).__init__()

        self.df = df
        self.initial_balance = initial_balance
        self.transaction_fee = transaction_fee

        # 行動空間: 0=Hold, 1=Buy, 2=Sell
        self.action_space = spaces.Discrete(3)

        # 状態空間: 価格、テクニカル指標、ポジション情報など
        self.observation_space = spaces.Box(
            low=-np.inf, high=np.inf, 
            shape=(self._get_observation_size(),), 
            dtype=np.float32
        )

        self.reset()

    def _get_observation_size(self):
        """観測空間のサイズを計算"""
        # 価格情報 + テクニカル指標 + ポジション情報
        return 20  # 例: OHLCV(5) + MA(3) + RSI(1) + MACD(3) + Position(3) + PnL(2) + Volume(3)

    def _get_observation(self):
        """現在の観測を取得"""
        obs = []

        # 価格情報（正規化）
        current_data = self.df.iloc[self.current_step]
        price_features = [
            current_data['open'] / current_data['close'] - 1,
            current_data['high'] / current_data['close'] - 1,
            current_data['low'] / current_data['close'] - 1,
            current_data['volume'] / self.df['volume'].rolling(20).mean().iloc[self.current_step],
            np.log(current_data['close'] / self.df['close'].iloc[self.current_step - 1])
        ]
        obs.extend(price_features)

        # テクニカル指標
        obs.extend([
            current_data['ma_5'] / current_data['close'] - 1,
            current_data['ma_20'] / current_data['close'] - 1,
            current_data['ma_50'] / current_data['close'] - 1,
            current_data['rsi'] / 100,
            current_data['macd'],
            current_data['macd_signal'],
            current_data['macd_diff']
        ])

        # ポジション情報
        obs.extend([
            self.position,  # -1, 0, 1
            self.shares,
            self.balance / self.initial_balance
        ])

        # 損益情報
        obs.extend([
            self.total_profit / self.initial_balance,
            self.unrealized_pnl / self.initial_balance
        ])

        # ボリューム情報
        obs.extend([
            current_data['buy_volume'] / current_data['volume'],
            current_data['sell_volume'] / current_data['volume'],
            current_data['trade_count'] / self.df['trade_count'].rolling(20).mean().iloc[self.current_step]
        ])

        return np.array(obs, dtype=np.float32)

    def step(self, action):
        """環境の1ステップ実行"""
        # 現在の価格
        current_price = self.df['close'].iloc[self.current_step]

        # 行動の実行
        if action == 1:  # Buy
            if self.position <= 0:  # ショートポジションをクローズしてロング
                if self.position < 0:
                    # ショートクローズ
                    self._close_position(current_price)
                # ロングポジション
                self._open_long(current_price)

        elif action == 2:  # Sell
            if self.position >= 0:  # ロングポジションをクローズしてショート
                if self.position > 0:
                    # ロングクローズ
                    self._close_position(current_price)
                # ショートポジション
                self._open_short(current_price)

        # 時間を進める
        self.current_step += 1

        # 報酬の計算
        reward = self._calculate_reward()

        # 終了判定
        done = self.current_step >= len(self.df) - 1 or self.balance <= 0

        # 次の観測
        obs = self._get_observation()

        # 追加情報
        info = {
            'balance': self.balance,
            'position': self.position,
            'total_profit': self.total_profit
        }

        return obs, reward, done, info

    def _calculate_reward(self):
        """報酬の計算"""
        # シャープレシオベースの報酬
        returns = []
        for i in range(max(0, self.current_step - 20), self.current_step):
            returns.append(self.balance_history[i] / self.balance_history[i-1] - 1)

        if returns:
            sharpe = np.mean(returns) / (np.std(returns) + 1e-8) * np.sqrt(252)
        else:
            sharpe = 0

        # ポジション保有時間のペナルティ（過度な保有を防ぐ）
        holding_penalty = -0.001 * abs(self.position) * self.holding_time

        # 取引頻度のペナルティ（過度な取引を防ぐ）
        if self.last_action in [1, 2]:
            trade_penalty = -0.001
        else:
            trade_penalty = 0

        reward = sharpe + holding_penalty + trade_penalty

        return reward

    def reset(self):
        """環境のリセット"""
        self.current_step = 20  # テクニカル指標の計算に必要な期間
        self.balance = self.initial_balance
        self.position = 0  # -1: ショート, 0: なし, 1: ロング
        self.shares = 0
        self.entry_price = 0
        self.total_profit = 0
        self.unrealized_pnl = 0
        self.balance_history = [self.initial_balance] * 20
        self.holding_time = 0
        self.last_action = 0

        return self._get_observation()

import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.distributions import Categorical

class ActorCritic(nn.Module):
    """Actor-Criticネットワーク"""

    def __init__(self, state_size: int, action_size: int, hidden_size: int = 256):
        super(ActorCritic, self).__init__()

        # 共通層
        self.fc1 = nn.Linear(state_size, hidden_size)
        self.fc2 = nn.Linear(hidden_size, hidden_size)

        # Actor（方策）
        self.actor = nn.Linear(hidden_size, action_size)

        # Critic（価値関数）
        self.critic = nn.Linear(hidden_size, 1)

    def forward(self, state):
        x = F.relu(self.fc1(state))
        x = F.relu(self.fc2(x))

        # 行動確率
        action_probs = F.softmax(self.actor(x), dim=-1)

        # 状態価値
        state_value = self.critic(x)

        return action_probs, state_value

class PPOAgent:
    """PPOエージェント"""

    def __init__(self, state_size: int, action_size: int, lr: float = 3e-4):
        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

        self.actor_critic = ActorCritic(state_size, action_size).to(self.device)
        self.optimizer = optim.Adam(self.actor_critic.parameters(), lr=lr)

        # PPOハイパーパラメータ
        self.clip_param = 0.2
        self.ppo_epochs = 10
        self.mini_batch_size = 64
        self.gamma = 0.99
        self.gae_lambda = 0.95

    def get_action(self, state):
        """行動の選択"""
        state = torch.FloatTensor(state).unsqueeze(0).to(self.device)

        with torch.no_grad():
            action_probs, _ = self.actor_critic(state)
            dist = Categorical(action_probs)
            action = dist.sample()
            action_log_prob = dist.log_prob(action)

        return action.item(), action_log_prob.item()

    def compute_gae(self, rewards, values, dones):
        """Generalized Advantage Estimation"""
        advantages = []
        gae = 0

        for t in reversed(range(len(rewards))):
            if t == len(rewards) - 1:
                next_value = 0
            else:
                next_value = values[t + 1]

            delta = rewards[t] + self.gamma * next_value * (1 - dones[t]) - values[t]
            gae = delta + self.gamma * self.gae_lambda * (1 - dones[t]) * gae
            advantages.insert(0, gae)

        return torch.FloatTensor(advantages)

    def update(self, states, actions, old_log_probs, rewards, dones):
        """PPO更新"""
        states = torch.FloatTensor(states).to(self.device)
        actions = torch.LongTensor(actions).to(self.device)
        old_log_probs = torch.FloatTensor(old_log_probs).to(self.device)

        # 価値の計算
        _, values = self.actor_critic(states)
        values = values.squeeze().detach().cpu().numpy()

        # GAEの計算
        advantages = self.compute_gae(rewards, values, dones)
        returns = advantages + torch.FloatTensor(values)

        # 正規化
        advantages = (advantages - advantages.mean()) / (advantages.std() + 1e-8)

        # PPO更新
        for _ in range(self.ppo_epochs):
            # ミニバッチ処理
            for indices in self._get_batches(len(states), self.mini_batch_size):
                batch_states = states[indices]
                batch_actions = actions[indices]
                batch_old_log_probs = old_log_probs[indices]
                batch_advantages = advantages[indices].to(self.device)
                batch_returns = returns[indices].to(self.device)

                # 現在の方策での確率計算
                action_probs, values = self.actor_critic(batch_states)
                dist = Categorical(action_probs)
                log_probs = dist.log_prob(batch_actions)

                # 確率比
                ratio = torch.exp(log_probs - batch_old_log_probs)

                # クリップされた目的関数
                surr1 = ratio * batch_advantages
                surr2 = torch.clamp(ratio, 1 - self.clip_param, 1 + self.clip_param) * batch_advantages

                # 損失関数
                actor_loss = -torch.min(surr1, surr2).mean()
                critic_loss = F.mse_loss(values.squeeze(), batch_returns)
                entropy_loss = -dist.entropy().mean()

                loss = actor_loss + 0.5 * critic_loss - 0.01 * entropy_loss

                # 勾配更新
                self.optimizer.zero_grad()
                loss.backward()
                torch.nn.utils.clip_grad_norm_(self.actor_critic.parameters(), 0.5)
                self.optimizer.step()

import torch.multiprocessing as mp
from torch.distributions import Normal

class A3CNetwork(nn.Module):
    """A3C用の連続行動空間ネットワーク"""

    def __init__(self, state_size: int, action_size: int = 1):
        super(A3CNetwork, self).__init__()

        # 特徴抽出層
        self.features = nn.Sequential(
            nn.Linear(state_size, 256),
            nn.ReLU(),
            nn.Linear(256, 256),
            nn.ReLU()
        )

        # Actor: 平均と標準偏差を出力
        self.actor_mean = nn.Linear(256, action_size)
        self.actor_log_std = nn.Linear(256, action_size)

        # Critic
        self.critic = nn.Linear(256, 1)

    def forward(self, state):
        features = self.features(state)

        # 行動の平均と標準偏差
        action_mean = torch.tanh(self.actor_mean(features))  # -1 to 1
        action_log_std = self.actor_log_std(features)
        action_std = torch.exp(torch.clamp(action_log_std, -20, 2))

        # 状態価値
        value = self.critic(features)

        return action_mean, action_std, value

class A3CWorker(mp.Process):
    """A3Cワーカープロセス"""

    def __init__(self, global_net, optimizer, global_ep, global_ep_r, res_queue, name):
        super(A3CWorker, self).__init__()
        self.name = f'worker_{name}'
        self.global_net = global_net
        self.optimizer = optimizer
        self.global_ep = global_ep
        self.global_ep_r = global_ep_r
        self.res_queue = res_queue

        # ローカルネットワーク
        self.local_net = A3CNetwork(state_size, action_size)

        # 環境
        self.env = CryptoTradingEnv(df)

    def run(self):
        """ワーカーの実行"""
        total_step = 1

        while self.global_ep.value < MAX_EP:
            state = self.env.reset()
            buffer_s, buffer_a, buffer_r = [], [], []
            ep_r = 0.

            while True:
                # 行動選択
                action = self._choose_action(state)
                next_state, reward, done, _ = self.env.step(action)

                ep_r += reward
                buffer_s.append(state)
                buffer_a.append(action)
                buffer_r.append(reward)

                # 更新またはエピソード終了
                if total_step % UPDATE_GLOBAL_ITER == 0 or done:
                    # グローバルネットワークに勾配を送信
                    self._push_and_pull(done, next_state, buffer_s, buffer_a, buffer_r)
                    buffer_s, buffer_a, buffer_r = [], [], []

                    if done:
                        self._record(ep_r)
                        break

                state = next_state
                total_step += 1

    def _choose_action(self, state):
        """行動選択（連続値）"""
        state = torch.FloatTensor(state).unsqueeze(0)

        with torch.no_grad():
            mean, std, _ = self.local_net(state)
            dist = Normal(mean, std)
            action = dist.sample()
            action = torch.clamp(action, -1, 1)  # 行動を-1から1に制限

        return action.numpy()[0]

class TransformerRL(nn.Module):
    """Transformerベースの強化学習モデル"""

    def __init__(self, state_size: int, action_size: int, seq_length: int = 100):
        super(TransformerRL, self).__init__()

        self.seq_length = seq_length
        hidden_size = 256

        # 状態エンコーダー
        self.state_encoder = nn.Linear(state_size, hidden_size)

        # 位置エンコーディング
        self.position_encoding = nn.Parameter(
            torch.randn(1, seq_length, hidden_size)
        )

        # Transformer
        self.transformer = nn.TransformerEncoder(
            nn.TransformerEncoderLayer(
                d_model=hidden_size,
                nhead=8,
                dim_feedforward=1024,
                dropout=0.1
            ),
            num_layers=6
        )

        # 出力ヘッド
        self.value_head = nn.Linear(hidden_size, 1)
        self.policy_head = nn.Linear(hidden_size, action_size)

    def forward(self, states_sequence):
        """
        Args:
            states_sequence: (batch_size, seq_length, state_size)
        """
        # エンコード
        encoded = self.state_encoder(states_sequence)
        encoded = encoded + self.position_encoding[:, :states_sequence.size(1), :]

        # Transformer処理
        # (seq_length, batch_size, hidden_size)に転置
        encoded = encoded.transpose(0, 1)
        transformed = self.transformer(encoded)

        # 最後の時点の出力を使用
        last_output = transformed[-1]

        # 価値と方策を出力
        value = self.value_head(last_output)
        policy_logits = self.policy_head(last_output)

        return F.softmax(policy_logits, dim=-1), value

class MultiAgentTradingSystem:
    """マルチエージェント取引システム"""

    def __init__(self, n_agents: int = 5):
        self.n_agents = n_agents
        self.agents = []

        # 異なる戦略を持つエージェントを作成
        strategies = ['scalping', 'swing', 'trend', 'mean_reversion', 'arbitrage']

        for i in range(n_agents):
            agent = {
                'model': DQN(state_size=50, action_size=3),
                'strategy': strategies[i % len(strategies)],
                'risk_tolerance': np.random.uniform(0.5, 2.0),
                'time_horizon': np.random.choice([1, 5, 20, 100])
            }
            self.agents.append(agent)

    def collective_decision(self, state):
        """集団的意思決定"""
        votes = np.zeros(3)  # Hold, Buy, Sell
        confidences = []

        for agent in self.agents:
            # 各エージェントの予測
            q_values = agent['model'](torch.FloatTensor(state))
            action = torch.argmax(q_values).item()
            confidence = torch.max(q_values).item()

            # 重み付き投票
            weight = confidence * agent['risk_tolerance']
            votes[action] += weight
            confidences.append(confidence)

        # 最終決定
        if np.max(confidences) < 0.5:  # 信頼度が低い場合はHold
            return 0

        return np.argmax(votes)

    def train_with_shared_experience(self, env, episodes: int = 1000):
        """共有経験による学習"""
        shared_memory = deque(maxlen=50000)

        for episode in range(episodes):
            state = env.reset()
            episode_rewards = []

            while True:
                # 集団決定
                action = self.collective_decision(state)
                next_state, reward, done, _ = env.step(action)

                # 全エージェントで経験を共有
                shared_memory.append((state, action, reward, next_state, done))

                # 各エージェントの学習
                if len(shared_memory) > 1000:
                    for agent in self.agents:
                        # 戦略に応じたサンプリング
                        if agent['strategy'] == 'scalping':
                            # 最近のデータを重視
                            recent_memory = list(shared_memory)[-5000:]
                            batch = random.sample(recent_memory, 32)
                        else:
                            batch = random.sample(shared_memory, 32)

                        # 学習（省略）
                        # agent['model'].train(batch)

                episode_rewards.append(reward)
                state = next_state

                if done:
                    break

class RiskManagedRLAgent:
    """リスク管理機能を持つRLエージェント"""

    def __init__(self, base_agent, max_drawdown: float = 0.2, var_limit: float = 0.05):
        self.base_agent = base_agent
        self.max_drawdown = max_drawdown
        self.var_limit = var_limit

        # リスク指標の追跡
        self.peak_balance = 0
        self.current_drawdown = 0
        self.position_history = []
        self.return_history = []

    def get_risk_adjusted_action(self, state, current_balance):
        """リスク調整済み行動"""
        # 基本エージェントの推奨行動
        base_action = self.base_agent.act(state)

        # 現在のドローダウンを計算
        if current_balance > self.peak_balance:
            self.peak_balance = current_balance

        self.current_drawdown = (self.peak_balance - current_balance) / self.peak_balance

        # リスクチェック
        if self.current_drawdown > self.max_drawdown * 0.8:
            # ドローダウンが限界に近い場合はポジションクローズ
            return 3  # CLOSE action

        # VaRチェック
        if len(self.return_history) > 20:
            var_95 = np.percentile(self.return_history, 5)
            if var_95 < -self.var_limit:
                # VaRが限界を超えている場合は新規ポジションを避ける
                if base_action in [1, 2]:  # BUY or SELL
                    return 0  # HOLD

        # ポジションサイジング
        if base_action in [1, 2]:
            # Kelly基準に基づくポジションサイズ調整
            position_size = self._calculate_kelly_position_size()
            # 実装は省略（実際にはposition_sizeを環境に渡す）

        return base_action

    def _calculate_kelly_position_size(self):
        """Kelly基準によるポジションサイズ計算"""
        if len(self.return_history) < 50:
            return 0.1  # デフォルトの小さいサイズ

        returns = np.array(self.return_history)
        win_rate = len(returns[returns > 0]) / len(returns)

        if win_rate == 0 or win_rate == 1:
            return 0.1

        avg_win = returns[returns > 0].mean() if len(returns[returns > 0]) > 0 else 0
        avg_loss = abs(returns[returns < 0].mean()) if len(returns[returns < 0]) > 0 else 1

        # Kelly %
        kelly_percent = (win_rate * avg_win - (1 - win_rate) * avg_loss) / avg_win

        # 安全のため25%に制限
        return min(kelly_percent * 0.25, 0.25)

class RLBacktester:
    """強化学習エージェントのバックテスター"""

    def __init__(self, agent, test_env):
        self.agent = agent
        self.test_env = test_env

    def run_backtest(self, n_episodes: int = 100):
        """バックテストの実行"""
        results = {
            'returns': [],
            'sharpe_ratios': [],
            'max_drawdowns': [],
            'win_rates': [],
            'trade_counts': []
        }

        for episode in range(n_episodes):
            episode_result = self._run_single_episode()

            results['returns'].append(episode_result['total_return'])
            results['sharpe_ratios'].append(episode_result['sharpe_ratio'])
            results['max_drawdowns'].append(episode_result['max_drawdown'])
            results['win_rates'].append(episode_result['win_rate'])
            results['trade_counts'].append(episode_result['trade_count'])

        # 統計サマリー
        summary = {
            'mean_return': np.mean(results['returns']),
            'std_return': np.std(results['returns']),
            'mean_sharpe': np.mean(results['sharpe_ratios']),
            'mean_max_dd': np.mean(results['max_drawdowns']),
            'mean_win_rate': np.mean(results['win_rates']),
            'mean_trades': np.mean(results['trade_counts'])
        }

        return results, summary

    def _run_single_episode(self):
        """単一エピソードの実行"""
        state = self.test_env.reset()
        done = False

        balance_history = [self.test_env.balance]
        trades = []

        while not done:
            action = self.agent.act(state)
            next_state, reward, done, info = self.test_env.step(action)

            balance_history.append(info['balance'])

            if action in [1, 2]:  # Trade executed
                trades.append({
                    'action': action,
                    'price': self.test_env.df.iloc[self.test_env.current_step]['close'],
                    'balance': info['balance']
                })

            state = next_state

        # パフォーマンス計算
        returns = np.diff(balance_history) / balance_history[:-1]

        return {
            'total_return': (balance_history[-1] - balance_history[0]) / balance_history[0],
            'sharpe_ratio': np.mean(returns) / (np.std(returns) + 1e-8) * np.sqrt(252),
            'max_drawdown': self._calculate_max_drawdown(balance_history),
            'win_rate': self._calculate_win_rate(trades),
            'trade_count': len(trades)
        }

class CryptoRLTradingSystem:
    """暗号資産強化学習取引システム"""

    def __init__(self, config):
        self.config = config

        # データ準備
        self.data_processor = DataProcessor()
        self.feature_engineer = FeatureEngineer()

        # 環境
        self.train_env = CryptoTradingEnv(
            df=self.prepare_training_data(),
            **config['env_params']
        )

        # エージェント
        if config['algorithm'] == 'dqn':
            self.agent = DQNAgent(**config['agent_params'])
        elif config['algorithm'] == 'ppo':
            self.agent = PPOAgent(**config['agent_params'])

        # リスク管理
        self.risk_manager = RiskManagedRLAgent(
            self.agent,
            **config['risk_params']
        )

    def prepare_training_data(self):
        """訓練データの準備"""
        # データ取得
        raw_data = self.data_processor.fetch_data(
            symbol=self.config['symbol'],
            start_date=self.config['start_date'],
            end_date=self.config['end_date']
        )

        # 特徴量生成
        features = self.feature_engineer.generate_features(raw_data)

        return features

    def train(self, episodes: int = 1000):
        """エージェントの訓練"""
        best_reward = -np.inf

        for episode in range(episodes):
            state = self.train_env.reset()
            episode_reward = 0

            while True:
                # リスク調整済み行動
                action = self.risk_manager.get_risk_adjusted_action(
                    state, 
                    self.train_env.balance
                )

                next_state, reward, done, info = self.train_env.step(action)

                # 経験の記憶
                self.agent.remember(state, action, reward, next_state, done)

                # 学習
                if len(self.agent.memory) > self.config['batch_size']:
                    self.agent.replay(self.config['batch_size'])

                episode_reward += reward
                state = next_state

                if done:
                    break

            # ベストモデルの保存
            if episode_reward > best_reward:
                best_reward = episode_reward
                self.save_model(f'best_model_ep{episode}.pth')

            # ログ
            if episode % 10 == 0:
                print(f"Episode {episode}, Reward: {episode_reward:.2f}, "
                      f"Epsilon: {self.agent.epsilon:.3f}")

    def live_trading(self):
        """ライブ取引"""
        print("Starting live trading...")

        while True:
            try:
                # 最新データの取得
                current_state = self._get_current_market_state()

                # 行動決定
                action = self.risk_manager.get_risk_adjusted_action(
                    current_state,
                    self._get_current_balance()
                )

                # 取引実行
                if action != 0:  # Not HOLD
                    self._execute_trade(action)

                # 待機
                time.sleep(60)  # 1分ごと

            except KeyboardInterrupt:
                print("Stopping live trading...")
                break
            except Exception as e:
                print(f"Error in live trading: {e}")
                time.sleep(300)  # エラー時は5分待機

from optuna import create_study

def optimize_rl_hyperparameters(trial):
    """Optunaによるハイパーパラメータ最適化"""

    # ハイパーパラメータの提案
    config = {
        'algorithm': trial.suggest_categorical('algorithm', ['dqn', 'ppo']),
        'agent_params': {
            'learning_rate': trial.suggest_loguniform('lr', 1e-5, 1e-2),
            'hidden_sizes': [
                trial.suggest_int('hidden_size', 64, 512),
                trial.suggest_int('hidden_size', 64, 512)
            ]
        },
        'env_params': {
            'transaction_fee': trial.suggest_uniform('fee', 0.0001, 0.01)
        },
        'risk_params': {
            'max_drawdown': trial.suggest_uniform('max_dd', 0.1, 0.3),
            'var_limit': trial.suggest_uniform('var', 0.01, 0.1)
        }
    }

    # システムの構築と訓練
    system = CryptoRLTradingSystem(config)
    system.train(episodes=100)

    # バックテスト
    backtester = RLBacktester(system.agent, system.train_env)
    _, summary = backtester.run_backtest(n_episodes=10)

    # 目的関数（シャープレシオを最大化）
    return summary['mean_sharpe']

# 最適化の実行
study = create_study(direction='maximize')
study.optimize(optimize_rl_hyperparameters, n_trials=100)

print(f"Best parameters: {study.best_params}")
print(f"Best Sharpe ratio: {study.best_value}")