{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyNK8+RE4/qVk7vLLrwnVNKn",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/AntonDozhdikov/Demography_migration/blob/main/MADDPG_vs_EVO_vs_DGM_EN.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Experiment block with standard MADDPG, evolutionary optimizer MADDPG-EVO\n",
        "# Simplified Darwin-Gödel Machine MADDPG-EVO-DGM\n",
        "\n",
        "# Import libraries for numerical computations and data processing\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "# Import PyTorch libraries for constructing and training neural networks\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "import torch.nn.functional as F\n",
        "\n",
        "# Import data structures and random number generator module\n",
        "from collections import defaultdict, deque\n",
        "import random\n",
        "\n",
        "# Import modules for JSON handling and visualization\n",
        "import json\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "# Import module for date and time handling\n",
        "from datetime import datetime\n",
        "\n",
        "# Import module for managing warnings\n",
        "import warnings\n",
        "warnings.filterwarnings('ignore')\n",
        "\n",
        "# Configure random seeds for experimental reproducibility\n",
        "np.random.seed(42)\n",
        "torch.manual_seed(42)\n",
        "random.seed(42)\n",
        "\n",
        "# Configure matplotlib for correct display\n",
        "plt.rcParams['font.family'] = 'DejaVu Sans'\n",
        "plt.rcParams['axes.unicode_minus'] = False"
      ],
      "metadata": {
        "id": "P9xpvYrx7QDz"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# --- BLOCK 1: File Processing and Demographic Data Generation ---\n",
        "class DemographicDataProcessor:\n",
        "    \"\"\"Processor for demographic data based on actual regional statistics.\"\"\"\n",
        "\n",
        "    def __init__(self, real_data_path, crisis_scenarios_path):\n",
        "        \"\"\"Initialize the data processor with file paths.\"\"\"\n",
        "        self.df = pd.read_csv(real_data_path)\n",
        "        # Load crisis scenarios from a JSON configuration file\n",
        "        with open(crisis_scenarios_path, 'r') as f:\n",
        "            self.crisis_scenarios = json.load(f)\n",
        "        self.preprocess_data()\n",
        "\n",
        "    def preprocess_data(self):\n",
        "        \"\"\"Preprocess raw data to ensure consistency and completeness for modeling.\"\"\"\n",
        "        # Forward-fill and backward-fill missing values\n",
        "        self.df = self.df.fillna(method='ffill').fillna(method='bfill')\n",
        "        # Compute derived indicators where data is missing\n",
        "        self.df.loc[self.df['natural_increase_rate'].isna(), 'natural_increase_rate'] = \\\n",
        "            self.df['birth_rate'] - self.df['death_rate']\n",
        "        # Compute and store regional statistical profiles\n",
        "        self.region_stats = {}\n",
        "        for region in self.df['region_name'].unique():\n",
        "            region_data = self.df[self.df['region_name'] == region]\n",
        "            self.region_stats[region] = {\n",
        "                'birth_rate_mean': region_data['birth_rate'].mean(),\n",
        "                'birth_rate_std': region_data['birth_rate'].std(),\n",
        "                'death_rate_mean': region_data['death_rate'].mean(),\n",
        "                'death_rate_std': region_data['death_rate'].std(),\n",
        "                'migration_mean': region_data['migration_balance'].mean(),\n",
        "                'migration_std': region_data['migration_balance'].std(),\n",
        "                'gdp_mean': region_data['gdp_per_capita'].mean(),\n",
        "                'gdp_std': region_data['gdp_per_capita'].std(),\n",
        "                'unemployment_mean': region_data['unemployment_rate'].mean(),\n",
        "                'unemployment_std': region_data['unemployment_rate'].std(),\n",
        "                'population_trend': self._calculate_trend(region_data['population']),\n",
        "                'region_id': region_data['region_id'].iloc[0]\n",
        "            }\n",
        "\n",
        "    def _calculate_trend(self, series):\n",
        "        \"\"\"Calculate the linear trend of a time series using least squares regression.\"\"\"\n",
        "        series = series.dropna()\n",
        "        if len(series) < 2:\n",
        "            return 0\n",
        "        x = np.arange(len(series))\n",
        "        z = np.polyfit(x, series, 1)\n",
        "        return z[0]\n",
        "\n",
        "    def apply_crisis_impact(self, base_values, crisis_scenario, year, crisis_start_year):\n",
        "        \"\"\"Apply the demographic and economic impact of a crisis scenario, with decaying intensity over time.\"\"\"\n",
        "        crisis_duration = year - crisis_start_year + 1\n",
        "        max_duration = crisis_scenario['end_year'] - crisis_scenario['start_year'] + 1\n",
        "        intensity = max(0, 1 - (crisis_duration - 1) / max_duration)\n",
        "        impacts = crisis_scenario['demographic_impacts']\n",
        "\n",
        "        birth_rate_impact = impacts['birth_rate_change'] * intensity\n",
        "        death_rate_impact = impacts['death_rate_change'] * intensity\n",
        "        migration_impact = impacts['migration_change'] * intensity\n",
        "        economic_impact = impacts['economic_impact'] * intensity\n",
        "\n",
        "        modified_values = base_values.copy()\n",
        "        modified_values['birth_rate'] *= (1 + birth_rate_impact)\n",
        "        modified_values['death_rate'] *= (1 + death_rate_impact)\n",
        "        modified_values['migration_balance'] *= (1 + migration_impact)\n",
        "        modified_values['gdp_per_capita'] *= (1 + economic_impact)\n",
        "        modified_values['unemployment_rate'] *= (1 - economic_impact * 0.5)\n",
        "        return modified_values\n",
        "\n",
        "    def generate_training_data(self, years, regions, apply_crisis=True):\n",
        "        \"\"\"Generate a synthetic training dataset based on real-world statistics and optional crisis scenarios.\"\"\"\n",
        "        training_data = []\n",
        "        for year in years:\n",
        "            for region in regions:\n",
        "                if region not in self.region_stats:\n",
        "                    continue\n",
        "                real_data = self.df[(self.df['region_name'] == region) & (self.df['year'] == year)]\n",
        "                if len(real_data) > 0:\n",
        "                    base_values = {\n",
        "                        'region_id': real_data['region_id'].iloc[0],\n",
        "                        'region_name': region,\n",
        "                        'year': year,\n",
        "                        'birth_rate': real_data['birth_rate'].iloc[0],\n",
        "                        'death_rate': real_data['death_rate'].iloc[0],\n",
        "                        'migration_balance': real_data['migration_balance'].iloc[0],\n",
        "                        'gdp_per_capita': real_data['gdp_per_capita'].iloc[0],\n",
        "                        'unemployment_rate': real_data['unemployment_rate'].iloc[0],\n",
        "                        'population': real_data['population'].iloc[0],\n",
        "                        'average_wage': real_data['average_wage'].iloc[0]\n",
        "                    }\n",
        "                else:\n",
        "                    stats = self.region_stats[region]\n",
        "                    base_values = {\n",
        "                        'region_id': stats['region_id'],\n",
        "                        'region_name': region,\n",
        "                        'year': year,\n",
        "                        'birth_rate': max(0, np.random.normal(stats['birth_rate_mean'], stats['birth_rate_std'])),\n",
        "                        'death_rate': max(0, np.random.normal(stats['death_rate_mean'], stats['death_rate_std'])),\n",
        "                        'migration_balance': np.random.normal(stats['migration_mean'], stats['migration_std']),\n",
        "                        'gdp_per_capita': max(0, np.random.normal(stats['gdp_mean'], stats['gdp_std'])),\n",
        "                        'unemployment_rate': max(0, min(100, np.random.normal(stats['unemployment_mean'], stats['unemployment_std'])))\n",
        "                    }\n",
        "                if apply_crisis:\n",
        "                    for scenario in self.crisis_scenarios:\n",
        "                        if scenario['start_year'] <= year <= scenario['end_year']:\n",
        "                            base_values = self.apply_crisis_impact(\n",
        "                                base_values, scenario, year, scenario['start_year']\n",
        "                            )\n",
        "                base_values['natural_increase_rate'] = base_values['birth_rate'] - base_values['death_rate']\n",
        "                if 'population' not in base_values:\n",
        "                    base_population = 1000000 + stats['population_trend'] * (year - 2010)\n",
        "                    base_values['population'] = max(0, int(base_population +\n",
        "                        (base_values['natural_increase_rate'] + base_values['migration_balance']/1000) * 1000))\n",
        "                if 'average_wage' not in base_values or pd.isna(base_values['average_wage']):\n",
        "                    base_values['average_wage'] = max(0, base_values['gdp_per_capita'] * 0.03 *\n",
        "                        (1 - base_values['unemployment_rate']/100))\n",
        "                training_data.append(base_values)\n",
        "        return pd.DataFrame(training_data)"
      ],
      "metadata": {
        "id": "wCpewdUb7QGr"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# --- BLOCK 2: Definition of Learning Environment (DemographicEnvironment) ---\n",
        "class DemographicEnvironment:\n",
        "    \"\"\"A multi-agent reinforcement learning environment for simulating regional demographic dynamics.\"\"\"\n",
        "\n",
        "    def __init__(self, data, n_regions=8, max_steps=50):\n",
        "        self.data = data\n",
        "        self.regions = data['region_name'].unique()[:n_regions]\n",
        "        self.n_regions = len(self.regions)\n",
        "        self.max_steps = max_steps\n",
        "        self.current_step = 0\n",
        "        self.state_dim = 8  # Core demographic and economic indicators\n",
        "        self.action_dim = 4  # Policy intervention dimensions\n",
        "        self.stability_history = {region: deque(maxlen=20) for region in self.regions}\n",
        "        self.reset()\n",
        "\n",
        "    def reset(self):\n",
        "        \"\"\"Reset the environment to its initial state at the beginning of an episode.\"\"\"\n",
        "        self.current_step = 0\n",
        "        self.states = {}\n",
        "        self.histories = {region: [] for region in self.regions}\n",
        "        for i, region in enumerate(self.regions):\n",
        "            region_data = self.data[self.data['region_name'] == region].iloc[0]\n",
        "            self.states[region] = self._normalize_state(region_data)\n",
        "        return self.get_observations()\n",
        "\n",
        "    def _normalize_state(self, region_data):\n",
        "        \"\"\"Normalize raw demographic data into a bounded state vector for neural network input.\"\"\"\n",
        "        return np.array([\n",
        "            region_data['birth_rate'] / 20.0,\n",
        "            region_data['death_rate'] / 30.0,\n",
        "            region_data['natural_increase_rate'] / 10.0,\n",
        "            min(region_data['migration_balance'] / 100000.0, 1.0),\n",
        "            region_data['gdp_per_capita'] / 2000000.0,\n",
        "            region_data['unemployment_rate'] / 100.0,\n",
        "            region_data['population'] / 10000000.0,\n",
        "            region_data['average_wage'] / 200000.0\n",
        "        ])\n",
        "\n",
        "    def get_observations(self):\n",
        "        \"\"\"Generate observations for all agents, including local state and aggregated global information.\"\"\"\n",
        "        observations = {}\n",
        "        for region in self.regions:\n",
        "            obs = self.states[region].copy()\n",
        "            other_states = [self.states[r] for r in self.regions if r != region]\n",
        "            if other_states:\n",
        "                avg_other = np.mean(other_states, axis=0)\n",
        "                obs = np.concatenate([obs, avg_other])\n",
        "            else:\n",
        "                obs = np.concatenate([obs, np.zeros(self.state_dim)])\n",
        "            observations[region] = obs\n",
        "        return observations\n",
        "\n",
        "    def step(self, actions):\n",
        "        \"\"\"Execute one timestep of the environment's dynamics based on agents' actions.\"\"\"\n",
        "        rewards = {}\n",
        "        for region in self.regions:\n",
        "            if region in actions:\n",
        "                action = actions[region]\n",
        "                old_state = self.states[region].copy()\n",
        "                self.states[region] = self._apply_policy_action(self.states[region], action)\n",
        "                rewards[region] = self._calculate_reward(old_state, self.states[region])\n",
        "                self.histories[region].append({\n",
        "                    'step': self.current_step,\n",
        "                    'state': old_state.copy(),\n",
        "                    'action': action.copy(),\n",
        "                    'reward': rewards[region]\n",
        "                })\n",
        "        self.current_step += 1\n",
        "        done = self.current_step >= self.max_steps\n",
        "        return self.get_observations(), rewards, done, {}\n",
        "\n",
        "    def _apply_policy_action(self, state, action):\n",
        "        \"\"\"Apply a policy intervention vector to modify the region's demographic state.\"\"\"\n",
        "        new_state = state.copy()\n",
        "        birth_rate_change = action[0] * 0.1\n",
        "        death_rate_change = -action[1] * 0.05\n",
        "        migration_change = action[2] * 0.05\n",
        "        economic_change = action[3] * 0.02\n",
        "\n",
        "        new_state[0] = max(0, new_state[0] + birth_rate_change)\n",
        "        new_state[1] = max(0, new_state[1] + death_rate_change)\n",
        "        new_state[2] = new_state[0] - new_state[1]\n",
        "        new_state[3] = new_state[3] + migration_change\n",
        "        new_state[4] = max(0, new_state[4] + economic_change)\n",
        "        new_state[5] = max(0, min(1, new_state[5] - economic_change * 0.5))\n",
        "\n",
        "        population_change = (new_state[2] + new_state[3]) * 0.01\n",
        "        new_state[6] = max(0, new_state[6] + population_change)\n",
        "        new_state[7] = new_state[4] * 0.1 * (1 - new_state[5])\n",
        "        return new_state\n",
        "\n",
        "    def _calculate_reward(self, old_state, new_state):\n",
        "        \"\"\"Compute a scalar reward signal based on the improvement in key demographic and economic indicators.\"\"\"\n",
        "        birth_rate_improvement = (new_state[0] - old_state[0]) * 10\n",
        "        death_rate_improvement = (old_state[1] - new_state[1]) * 10\n",
        "        natural_increase_improvement = (new_state[2] - old_state[2]) * 15\n",
        "        migration_improvement = (new_state[3] - old_state[3]) * 5\n",
        "        gdp_improvement = (new_state[4] - old_state[4]) * 5\n",
        "        unemployment_improvement = (old_state[5] - new_state[5]) * 5\n",
        "        population_stability = -abs(new_state[6] - old_state[6]) * 5\n",
        "\n",
        "        total_reward = (birth_rate_improvement + death_rate_improvement +\n",
        "                       natural_increase_improvement + migration_improvement +\n",
        "                       gdp_improvement + unemployment_improvement + population_stability)\n",
        "        return total_reward"
      ],
      "metadata": {
        "id": "kqpctXj77QJE"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# --- BLOCK 3: Definition of Neural Networks for Agents ---\n",
        "class ActorNetwork(nn.Module):\n",
        "    \"\"\"Actor network for the MADDPG algorithm, mapping states to deterministic actions.\"\"\"\n",
        "\n",
        "    def __init__(self, input_dim, action_dim, hidden_dim=256):\n",
        "        super(ActorNetwork, self).__init__()\n",
        "        # Fully connected layers\n",
        "        self.fc1 = nn.Linear(input_dim, hidden_dim)\n",
        "        self.fc2 = nn.Linear(hidden_dim, hidden_dim)\n",
        "        self.fc3 = nn.Linear(hidden_dim, action_dim)\n",
        "        self.dropout = nn.Dropout(0.1)\n",
        "\n",
        "    def forward(self, state):\n",
        "        x = F.relu(self.fc1(state))\n",
        "        x = self.dropout(x)\n",
        "        x = F.relu(self.fc2(x))\n",
        "        x = torch.tanh(self.fc3(x)) # Actions bounded between -1 and 1\n",
        "        return x\n",
        "\n",
        "class CriticNetwork(nn.Module):\n",
        "    \"\"\"Critic network for the MADDPG algorithm, estimating the Q-value of joint state-action pairs.\"\"\"\n",
        "\n",
        "    def __init__(self, state_dim, action_dim, n_agents, hidden_dim=256):\n",
        "        super(CriticNetwork, self).__init__()\n",
        "        # Calculate total input size (states and actions of all agents)\n",
        "        total_input_dim = state_dim * n_agents + action_dim * n_agents\n",
        "        self.fc1 = nn.Linear(total_input_dim, hidden_dim)\n",
        "        self.fc2 = nn.Linear(hidden_dim, hidden_dim)\n",
        "        self.fc3 = nn.Linear(hidden_dim, 1)\n",
        "        self.dropout = nn.Dropout(0.1)\n",
        "\n",
        "    def forward(self, states, actions):\n",
        "        x = torch.cat([states, actions], dim=1)\n",
        "        x = F.relu(self.fc1(x))\n",
        "        x = self.dropout(x)\n",
        "        x = F.relu(self.fc2(x))\n",
        "        return self.fc3(x)"
      ],
      "metadata": {
        "id": "zwNmTCkB7QLN"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# --- BLOCK 4: Definition of Agents (MADDPG) ---\n",
        "class MADDPGAgent:\n",
        "    \"\"\"A Multi-Agent Deep Deterministic Policy Gradient (MADDPG) agent.\"\"\"\n",
        "\n",
        "    def __init__(self, agent_id, state_dim, action_dim, n_agents, lr_actor=1e-4, lr_critic=1e-3, device=None):\n",
        "        self.agent_id = agent_id\n",
        "        self.state_dim = state_dim\n",
        "        self.action_dim = action_dim\n",
        "        self.n_agents = n_agents\n",
        "        # Add device support (CPU/GPU)\n",
        "        self.device = device or torch.device(\"cpu\")\n",
        "        # Actor networks (main and target)\n",
        "        self.actor = ActorNetwork(state_dim, action_dim).to(self.device)\n",
        "        self.actor_target = ActorNetwork(state_dim, action_dim).to(self.device)\n",
        "        self.actor_optimizer = optim.Adam(self.actor.parameters(), lr=lr_actor)\n",
        "        # Critic networks (main and target)\n",
        "        self.critic = CriticNetwork(state_dim, action_dim, n_agents).to(self.device)\n",
        "        self.critic_target = CriticNetwork(state_dim, action_dim, n_agents).to(self.device)\n",
        "        self.critic_optimizer = optim.Adam(self.critic.parameters(), lr=lr_critic)\n",
        "        # Initialize target networks by copying weights from main networks\n",
        "        self.hard_update(self.actor_target, self.actor)\n",
        "        self.hard_update(self.critic_target, self.critic)\n",
        "        # Training parameters\n",
        "        self.gamma = 0.95 # Discount factor\n",
        "        self.tau = 0.02   # Soft update parameter\n",
        "\n",
        "    def act(self, state, noise_scale=0.1):\n",
        "        \"\"\"Select an action with added noise for exploration.\"\"\"\n",
        "        # Convert state to tensor and move to device\n",
        "        state_tensor = torch.FloatTensor(state).unsqueeze(0).to(self.device)\n",
        "        action = self.actor(state_tensor).cpu().squeeze(0).detach().numpy()\n",
        "        # Add noise for exploration\n",
        "        noise = np.random.normal(0, noise_scale, size=action.shape)\n",
        "        action = np.clip(action + noise, -1, 1)\n",
        "        return action\n",
        "\n",
        "    def hard_update(self, target, source):\n",
        "        \"\"\"Perform a hard update (copy) of target network parameters.\"\"\"\n",
        "        for target_param, param in zip(target.parameters(), source.parameters()):\n",
        "            target_param.data.copy_(param.data)\n",
        "\n",
        "    def soft_update(self, target, source):\n",
        "        \"\"\"Perform a soft (Polyak averaging) update of target network parameters.\"\"\"\n",
        "        for target_param, param in zip(target.parameters(), source.parameters()):\n",
        "            target_param.data.copy_(target_param.data * (1.0 - self.tau) + param.data * self.tau)"
      ],
      "metadata": {
        "id": "KrXysCyl7QN2"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# --- BLOCK 5: Evolutionary Booster (EvolutionaryBooster) ---\n",
        "class EvolutionaryBooster:\n",
        "    \"\"\"An evolutionary algorithm module to optimize and diversify a population of MARL agents.\"\"\"\n",
        "\n",
        "    def __init__(self, population_size=16, mutation_rate=0.1, crossover_rate=0.7):\n",
        "        self.population_size = population_size\n",
        "        self.mutation_rate = mutation_rate\n",
        "        self.crossover_rate = crossover_rate\n",
        "        self.population = []\n",
        "        self.fitness_history = []\n",
        "\n",
        "    def initialize_population(self, agent_template):\n",
        "        \"\"\"Initialize a diverse population of agents by cloning and mutating a template.\"\"\"\n",
        "        self.population = []\n",
        "        for _ in range(self.population_size):\n",
        "            agent_copy = self._copy_agent(agent_template)\n",
        "            self._mutate_agent(agent_copy, mutation_strength=0.3)\n",
        "            self.population.append(agent_copy)\n",
        "\n",
        "    def _copy_agent(self, agent):\n",
        "        \"\"\"Create a deep copy of an agent, preserving its architecture and device.\"\"\"\n",
        "        # Determine agent type and create a new instance of the corresponding class\n",
        "        # Pass the device argument when creating the copy\n",
        "        if isinstance(agent, MADDPGAgent):\n",
        "            new_agent = MADDPGAgent(\n",
        "                agent.agent_id,\n",
        "                agent.state_dim,\n",
        "                agent.action_dim,\n",
        "                agent.n_agents,\n",
        "                device=agent.device # Pass device\n",
        "            )\n",
        "        else:\n",
        "            raise ValueError(f\"Unsupported agent type: {type(agent)}\")\n",
        "        # Copy network states\n",
        "        if hasattr(agent, 'actor'):\n",
        "            new_agent.actor.load_state_dict(agent.actor.state_dict())\n",
        "            # Ensure networks of the new copy are on the same device\n",
        "            new_agent.actor.to(agent.device)\n",
        "            new_agent.actor_target.to(agent.device)\n",
        "        if hasattr(agent, 'critic'):\n",
        "            new_agent.critic.load_state_dict(agent.critic.state_dict())\n",
        "            new_agent.critic.to(agent.device)\n",
        "            new_agent.critic_target.to(agent.device)\n",
        "        if hasattr(agent, 'actor_target'):\n",
        "            new_agent.actor_target.load_state_dict(agent.actor_target.state_dict())\n",
        "        if hasattr(agent, 'critic_target'):\n",
        "            new_agent.critic_target.load_state_dict(agent.critic_target.state_dict())\n",
        "        return new_agent\n",
        "\n",
        "    def _mutate_agent(self, agent, mutation_strength=0.1):\n",
        "        \"\"\"Apply Gaussian mutation to an agent's neural network parameters.\"\"\"\n",
        "        # Mutate actor network parameters\n",
        "        if hasattr(agent, 'actor'):\n",
        "            for param in agent.actor.parameters():\n",
        "                if np.random.random() < self.mutation_rate:\n",
        "                    noise = torch.randn_like(param) * mutation_strength\n",
        "                    param.data += noise\n",
        "        # Mutate critic network parameters\n",
        "        if hasattr(agent, 'critic'):\n",
        "            for param in agent.critic.parameters():\n",
        "                if np.random.random() < self.mutation_rate:\n",
        "                    noise = torch.randn_like(param) * mutation_strength\n",
        "                    param.data += noise\n",
        "\n",
        "    def _crossover_agents(self, parent1, parent2):\n",
        "        \"\"\"Perform uniform crossover between two parent agents to produce offspring.\"\"\"\n",
        "        child1 = self._copy_agent(parent1)\n",
        "        child2 = self._copy_agent(parent2)\n",
        "        # Crossover actor parameters\n",
        "        if hasattr(parent1, 'actor') and hasattr(parent2, 'actor'):\n",
        "            for p1, p2, c1, c2 in zip(parent1.actor.parameters(), parent2.actor.parameters(),\n",
        "                                     child1.actor.parameters(), child2.actor.parameters()):\n",
        "                if np.random.random() < self.crossover_rate:\n",
        "                    mask = torch.rand_like(p1) > 0.5\n",
        "                    c1.data = torch.where(mask, p1.data, p2.data)\n",
        "                    c2.data = torch.where(mask, p2.data, p1.data)\n",
        "        # Crossover critic parameters\n",
        "        if hasattr(parent1, 'critic') and hasattr(parent2, 'critic'):\n",
        "            for p1, p2, c1, c2 in zip(parent1.critic.parameters(), parent2.critic.parameters(),\n",
        "                                     child1.critic.parameters(), child2.critic.parameters()):\n",
        "                if np.random.random() < self.crossover_rate:\n",
        "                    mask = torch.rand_like(p1) > 0.5\n",
        "                    c1.data = torch.where(mask, p1.data, p2.data)\n",
        "                    c2.data = torch.where(mask, p2.data, p1.data)\n",
        "        return child1, child2\n",
        "\n",
        "    def evolve_population(self, fitness_scores):\n",
        "        \"\"\"Evolve the population using elitism, selection, crossover, and mutation.\"\"\"\n",
        "        # Combine agents and fitness, sort by fitness\n",
        "        population_fitness = list(zip(self.population, fitness_scores))\n",
        "        population_fitness.sort(key=lambda x: x[1], reverse=True)\n",
        "        # Select elite individuals\n",
        "        elite_size = self.population_size // 4\n",
        "        elite = [agent for agent, _ in population_fitness[:elite_size]]\n",
        "        # Create new generation\n",
        "        new_population = elite.copy()\n",
        "        # Fill population to required size\n",
        "        while len(new_population) < self.population_size:\n",
        "            parent1 = self._tournament_selection(population_fitness)\n",
        "            parent2 = self._tournament_selection(population_fitness)\n",
        "            if np.random.random() < self.crossover_rate:\n",
        "                child1, child2 = self._crossover_agents(parent1, parent2)\n",
        "                self._mutate_agent(child1)\n",
        "                self._mutate_agent(child2)\n",
        "                new_population.extend([child1, child2])\n",
        "            else:\n",
        "                child = self._copy_agent(parent1)\n",
        "                self._mutate_agent(child)\n",
        "                new_population.append(child)\n",
        "        # Trim population to required size and update\n",
        "        self.population = new_population[:self.population_size]\n",
        "        # Save fitness history\n",
        "        self.fitness_history.append(max(fitness_scores))\n",
        "        return self.population[0] # Return the best agent\n",
        "\n",
        "    def _tournament_selection(self, population_fitness, tournament_size=3):\n",
        "        \"\"\"Select a parent agent via tournament selection based on fitness.\"\"\"\n",
        "        tournament = random.sample(population_fitness, min(tournament_size, len(population_fitness)))\n",
        "        return max(tournament, key=lambda x: x[1])[0]"
      ],
      "metadata": {
        "id": "wco2itql7QQw"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# --- BLOCK 6: Experiment Logger (ExperimentLogger) ---\n",
        "class ExperimentLogger:\n",
        "    \"\"\"A logger for recording, storing, and visualizing the results of experiments.\"\"\"\n",
        "\n",
        "    def __init__(self):\n",
        "        self.metrics = defaultdict(list)\n",
        "        self.episode_rewards = defaultdict(list)\n",
        "        self.evolution_history = []\n",
        "\n",
        "    def log_episode(self, episode, agent_rewards, avg_reward, stability_metric):\n",
        "        \"\"\"Log the results of a single training episode.\"\"\"\n",
        "        self.metrics['episode'].append(episode)\n",
        "        self.metrics['avg_reward'].append(avg_reward)\n",
        "        self.metrics['stability'].append(stability_metric)\n",
        "        # Save rewards for each individual agent\n",
        "        for agent_id, reward in agent_rewards.items():\n",
        "            self.episode_rewards[agent_id].append(reward)\n",
        "\n",
        "    def log_evolution(self, generation, best_fitness, avg_fitness):\n",
        "        \"\"\"Log the results of an evolutionary generation.\"\"\"\n",
        "        self.evolution_history.append({\n",
        "            'generation': generation,\n",
        "            'best_fitness': best_fitness,\n",
        "            'avg_fitness': avg_fitness\n",
        "        })\n",
        "\n",
        "    def plot_results(self):\n",
        "        \"\"\"Visualize the experiment results using matplotlib.\"\"\"\n",
        "        plt.figure(figsize=(15, 10))\n",
        "        # Plot 1: Average reward over episodes\n",
        "        plt.subplot(2, 3, 1)\n",
        "        plt.plot(self.metrics['episode'], self.metrics['avg_reward'])\n",
        "        plt.title('Average Reward per Episode')\n",
        "        plt.xlabel('Episode')\n",
        "        plt.ylabel('Average Reward')\n",
        "        plt.grid(True)\n",
        "        # Plot 2: System stability metric\n",
        "        plt.subplot(2, 3, 2)\n",
        "        plt.plot(self.metrics['episode'], self.metrics['stability'])\n",
        "        plt.title('System Stability Metric')\n",
        "        plt.xlabel('Episode')\n",
        "        plt.ylabel('Stability')\n",
        "        plt.grid(True)\n",
        "        # Plot 3: Rewards for individual agents (first 50 episodes)\n",
        "        plt.subplot(2, 3, 3)\n",
        "        for agent_id, rewards in self.episode_rewards.items():\n",
        "            plt.plot(rewards[:50], label=f'Agent {agent_id}', alpha=0.7)\n",
        "        plt.title('Agent Rewards (First 50 Episodes)')\n",
        "        plt.xlabel('Episode')\n",
        "        plt.ylabel('Reward')\n",
        "        plt.legend()\n",
        "        plt.grid(True)\n",
        "        # Plot 4: Population evolution (if data is available)\n",
        "        if self.evolution_history:\n",
        "            plt.subplot(2, 3, 4)\n",
        "            generations = [x['generation'] for x in self.evolution_history]\n",
        "            best_fitness = [x['best_fitness'] for x in self.evolution_history]\n",
        "            avg_fitness = [x['avg_fitness'] for x in self.evolution_history]\n",
        "            plt.plot(generations, best_fitness, label='Best Fitness', linewidth=2)\n",
        "            plt.plot(generations, avg_fitness, label='Average Fitness', alpha=0.7)\n",
        "            plt.title('Population Evolution')\n",
        "            plt.xlabel('Generation')\n",
        "            plt.ylabel('Fitness')\n",
        "            plt.legend()\n",
        "            plt.grid(True)\n",
        "        # Plot 5: Distribution of rewards\n",
        "        plt.subplot(2, 3, 5)\n",
        "        all_rewards = []\n",
        "        for rewards in self.episode_rewards.values():\n",
        "            all_rewards.extend(rewards)\n",
        "        plt.hist(all_rewards, bins=30, alpha=0.7, edgecolor='black')\n",
        "        plt.title('Reward Distribution')\n",
        "        plt.xlabel('Reward')\n",
        "        plt.ylabel('Frequency')\n",
        "        plt.grid(True)\n",
        "        # Plot 6: Moving average of rewards\n",
        "        plt.subplot(2, 3, 6)\n",
        "        window_size = 10\n",
        "        if len(self.metrics['avg_reward']) >= window_size:\n",
        "            moving_avg = pd.Series(self.metrics['avg_reward']).rolling(window=window_size).mean()\n",
        "            plt.plot(self.metrics['episode'], moving_avg)\n",
        "            plt.title(f'Moving Average Reward (Window {window_size})')\n",
        "            plt.xlabel('Episode')\n",
        "            plt.ylabel('Moving Average')\n",
        "            plt.grid(True)\n",
        "        plt.tight_layout()\n",
        "        plt.show()\n",
        "\n",
        "    def save_results(self, filename):\n",
        "        \"\"\"Save the logged metrics to a CSV file.\"\"\"\n",
        "        results_df = pd.DataFrame(self.metrics)\n",
        "        results_df.to_csv(filename, index=False)\n",
        "        print(f\"Results saved to {filename}\")"
      ],
      "metadata": {
        "id": "Rd4MgzU-9azg"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# --- BLOCK 7: Agent Training Functions (_train_agent_*) ---\n",
        "def _train_agent_maddpg(agent, replay_buffer, batch_size=32):\n",
        "    \"\"\"Train a single MADDPG agent using experience replay.\"\"\"\n",
        "    if len(replay_buffer) < batch_size:\n",
        "        return\n",
        "    batch = random.sample(replay_buffer, batch_size)\n",
        "    # Move data to the agent's designated device (CPU/GPU)\n",
        "    states = torch.FloatTensor([x[0] for x in batch]).to(agent.device)\n",
        "    actions = torch.FloatTensor([x[1] for x in batch]).to(agent.device)\n",
        "    rewards = torch.FloatTensor([x[2] for x in batch]).unsqueeze(1).to(agent.device)\n",
        "    next_states = torch.FloatTensor([x[3] for x in batch]).to(agent.device)\n",
        "    dones = torch.BoolTensor([x[4] for x in batch]).unsqueeze(1).to(agent.device)\n",
        "    # Update the Critic network\n",
        "    with torch.no_grad():\n",
        "        next_actions = agent.actor_target(next_states)\n",
        "        # Construct global state-action vectors for the critic\n",
        "        global_next_states = next_states.repeat(1, agent.n_agents).view(batch_size, -1)\n",
        "        global_next_actions = next_actions.repeat(1, agent.n_agents).view(batch_size, -1)\n",
        "        target_q = agent.critic_target(global_next_states, global_next_actions)\n",
        "        target_q = rewards + agent.gamma * target_q * (~dones)\n",
        "    global_states = states.repeat(1, agent.n_agents).view(batch_size, -1)\n",
        "    global_actions = actions.repeat(1, agent.n_agents).view(batch_size, -1)\n",
        "    current_q = agent.critic(global_states, global_actions)\n",
        "    critic_loss = F.mse_loss(current_q, target_q)\n",
        "    agent.critic_optimizer.zero_grad()\n",
        "    critic_loss.backward()\n",
        "    # Clip gradients to prevent explosion\n",
        "    torch.nn.utils.clip_grad_norm_(agent.critic.parameters(), 0.5)\n",
        "    agent.critic_optimizer.step()\n",
        "    # Update the Actor network\n",
        "    predicted_actions = agent.actor(states)\n",
        "    global_predicted_actions = predicted_actions.repeat(1, agent.n_agents).view(batch_size, -1)\n",
        "    actor_loss = -agent.critic(global_states, global_predicted_actions).mean()\n",
        "    agent.actor_optimizer.zero_grad()\n",
        "    actor_loss.backward()\n",
        "    # Clip gradients to prevent explosion\n",
        "    torch.nn.utils.clip_grad_norm_(agent.actor.parameters(), 0.5)\n",
        "    agent.actor_optimizer.step()\n",
        "    # Perform a soft update of the target networks\n",
        "    agent.soft_update(agent.actor_target, agent.actor)\n",
        "    agent.soft_update(agent.critic_target, agent.critic)"
      ],
      "metadata": {
        "id": "mwAsBy4H9a2a"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# --- BLOCK 8: Auxiliary Evaluation and Metric Functions ---\n",
        "def _evaluate_agent_fitness(agent, env, region, n_eval_episodes=3):\n",
        "    \"\"\"Evaluate the fitness of an agent by running it in the environment without exploration noise.\"\"\"\n",
        "    total_reward = 0\n",
        "    for _ in range(n_eval_episodes):\n",
        "        observations = env.reset() # observations is a dict {region: obs}\n",
        "        episode_reward = 0\n",
        "        for _ in range(env.max_steps):\n",
        "            # Use zero noise for a deterministic evaluation\n",
        "            action = agent.act(observations[region], noise_scale=0.0)\n",
        "            actions = {region: action}\n",
        "            # Provide random actions for other agents to simulate a multi-agent environment\n",
        "            for other_region in env.regions:\n",
        "                if other_region != region:\n",
        "                    actions[other_region] = np.random.uniform(-1, 1, env.action_dim)\n",
        "            next_observations, rewards, done, _ = env.step(actions)\n",
        "            episode_reward += rewards.get(region, 0)\n",
        "            observations = next_observations\n",
        "            if done:\n",
        "                break\n",
        "        total_reward += episode_reward\n",
        "    return total_reward / n_eval_episodes\n",
        "\n",
        "def _calculate_stability_metric(env):\n",
        "    \"\"\"Calculate an enhanced metric for the stability of the multi-agent demographic system.\"\"\"\n",
        "    stability_scores = []\n",
        "    for region in env.regions:\n",
        "        state = env.states[region]\n",
        "        # Component 1: Birth-Death balance (penalize only negative natural increase)\n",
        "        natural_balance = max(0, -state[2]) * 2\n",
        "        # Component 2: Economic stability (more sensitive to unemployment)\n",
        "        economic_stability = 1 - (state[5] ** 2) * 1.5\n",
        "        # Component 3: Migration stability (stricter penalty for volatility)\n",
        "        migration_stability = 1 / (1 + abs(state[3]) * 20)\n",
        "        # Component 4: Population size stability (assuming normalized pop target ~1.0)\n",
        "        population_stability = 1 - min(1, abs(state[6] - 1.0))\n",
        "        # Calculate current stability for the region\n",
        "        current_stability = (natural_balance + economic_stability +\n",
        "                           migration_stability + population_stability) / 4\n",
        "        # Save to history for long-term analysis\n",
        "        env.stability_history[region].append(current_stability)\n",
        "        # Calculate long-term stability (less fluctuation implies higher stability)\n",
        "        if len(env.stability_history[region]) > 5:\n",
        "            long_term_stability = 1 - np.std(env.stability_history[region]) * 2\n",
        "        else:\n",
        "            long_term_stability = 1.0 # Insufficient data for calculation\n",
        "        # Final metric is a weighted average of current and long-term stability\n",
        "        region_stability = 0.4 * current_stability + 0.6 * long_term_stability\n",
        "        stability_scores.append(region_stability)\n",
        "    return np.mean(stability_scores)"
      ],
      "metadata": {
        "id": "0Qgr3SuV9a5C"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# --- BLOCK 9: Main Training Functions (Experiments) ---\n",
        "# EXPERIMENT: MADDPG-EVO (Enhanced)\n",
        "def train_maddpg_with_evolution(env, n_episodes=1000, min_improvement=0.1,\n",
        "                               population_size=16, save_results=True, experiment_name=\"MADDPG-EVO\"):\n",
        "    \"\"\"Train MADDPG agents with adaptive evolutionary optimization triggered by performance plateaus.\"\"\"\n",
        "    print(f\"=== Initiating Experiment: {experiment_name} ===\")\n",
        "    print(f\"Training for {n_episodes} episodes. Adaptive evolution activated upon performance stagnation (min_improvement={min_improvement}).\")\n",
        "    # Initialize one agent per region\n",
        "    agents = {}\n",
        "    for i, region in enumerate(env.regions):\n",
        "        agents[region] = MADDPGAgent(\n",
        "            agent_id=i,\n",
        "            state_dim=env.state_dim * 2, # local + global state\n",
        "            action_dim=env.action_dim,\n",
        "            n_agents=env.n_regions,\n",
        "            lr_actor=5e-5, # Reduced learning rate\n",
        "            lr_critic=1e-4\n",
        "        )\n",
        "    # Initialize evolutionary boosters for each region\n",
        "    evolution_boosters = {}\n",
        "    for region in env.regions:\n",
        "        booster = EvolutionaryBooster(population_size=population_size, mutation_rate=0.05) # Reduced mutation\n",
        "        booster.initialize_population(agents[region])\n",
        "        evolution_boosters[region] = booster\n",
        "    # Create experience replay buffers\n",
        "    replay_buffers = {region: deque(maxlen=10000) for region in env.regions}\n",
        "    # Initialize experiment logger\n",
        "    logger = ExperimentLogger()\n",
        "    print(f\"Commencing training for {n_episodes} episodes...\")\n",
        "    # Tracking variables for adaptive evolution\n",
        "    reward_history = []\n",
        "    last_evolution_episode = 0\n",
        "    min_episodes_between_evolutions = 20 # Increased interval between evolutions\n",
        "    # Main training loop\n",
        "    for episode in range(n_episodes):\n",
        "        observations = env.reset()\n",
        "        episode_rewards = {region: 0 for region in env.regions}\n",
        "        # Adaptive exploration noise\n",
        "        noise_scale = max(0.05, 0.2 * (1 - episode / n_episodes))\n",
        "        # Episode step loop\n",
        "        for step in range(env.max_steps):\n",
        "            # Select actions for all agents\n",
        "            actions = {}\n",
        "            for region in env.regions:\n",
        "                action = agents[region].act(observations[region], noise_scale=noise_scale)\n",
        "                actions[region] = action\n",
        "            # Execute step in environment\n",
        "            next_observations, rewards, done, _ = env.step(actions)\n",
        "            # Store experience\n",
        "            for region in env.regions:\n",
        "                replay_buffers[region].append((\n",
        "                    observations[region], actions[region], rewards[region],\n",
        "                    next_observations[region], done\n",
        "                ))\n",
        "                episode_rewards[region] += rewards[region]\n",
        "            observations = next_observations\n",
        "            if done:\n",
        "                break\n",
        "        # Train agents periodically\n",
        "        if episode % 10 == 0 and episode > 0:\n",
        "            for region in env.regions:\n",
        "                if len(replay_buffers[region]) > 200:\n",
        "                    _train_agent_maddpg(agents[region], replay_buffers[region], batch_size=64)\n",
        "        # Log metrics\n",
        "        avg_reward = np.mean(list(episode_rewards.values()))\n",
        "        stability_metric = _calculate_stability_metric(env)\n",
        "        logger.log_episode(episode, episode_rewards, avg_reward, stability_metric)\n",
        "        reward_history.append(avg_reward)\n",
        "        # Trigger adaptive evolution\n",
        "        if len(reward_history) > 30 and episode - last_evolution_episode >= min_episodes_between_evolutions:\n",
        "            recent_rewards = reward_history[-30:]\n",
        "            improvement_rate = (recent_rewards[-1] - recent_rewards[0]) / 30\n",
        "            if improvement_rate < min_improvement:\n",
        "                print(f\"{experiment_name} - Adaptive evolutionary optimization triggered at episode {episode} due to performance plateau\")\n",
        "                for region in env.regions:\n",
        "                    # Evaluate fitness of the entire population\n",
        "                    fitness_scores = []\n",
        "                    for agent in evolution_boosters[region].population:\n",
        "                        fitness = _evaluate_agent_fitness(agent, env, region)\n",
        "                        fitness_scores.append(fitness)\n",
        "                    # Evolve the population\n",
        "                    best_agent = evolution_boosters[region].evolve_population(fitness_scores)\n",
        "                    agents[region] = best_agent\n",
        "                    # Log evolution results\n",
        "                    logger.log_evolution(\n",
        "                        episode,\n",
        "                        max(fitness_scores),\n",
        "                        np.mean(fitness_scores)\n",
        "                    )\n",
        "                last_evolution_episode = episode # Update last evolution counter\n",
        "        # Progress report\n",
        "        if episode % 20 == 0:\n",
        "            print(f\"{experiment_name} - Episode {episode}, Average Reward: {avg_reward:.3f}, \"\n",
        "                  f\"Stability Metric: {stability_metric:.3f}\")\n",
        "    print(f\"{experiment_name} - Training successfully completed!\")\n",
        "    # Visualize and save results\n",
        "    logger.plot_results()\n",
        "    if save_results:\n",
        "        timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S\")\n",
        "        logger.save_results(f\"{experiment_name.lower()}_results_{timestamp}.csv\")\n",
        "    return agents, logger"
      ],
      "metadata": {
        "id": "ogz15bwn9a7m"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# === BLOCK 10: Standard MADDPG (Baseline) ===\n",
        "def train_maddpg_baseline(env, n_episodes=1000, save_results=True, experiment_name=\"MADDPG-BASELINE\"):\n",
        "    \"\"\"Train standard MADDPG agents without any evolutionary components.\"\"\"\n",
        "    print(f\"=== Initiating Experiment: {experiment_name} ===\")\n",
        "    print(f\"Training for {n_episodes} episodes. Pure MADDPG baseline (no evolutionary optimization).\")\n",
        "    # Initialize agents\n",
        "    agents = {}\n",
        "    for i, region in enumerate(env.regions):\n",
        "        agents[region] = MADDPGAgent(\n",
        "            agent_id=i,\n",
        "            state_dim=env.state_dim * 2, # local + global state\n",
        "            action_dim=env.action_dim,\n",
        "            n_agents=env.n_regions,\n",
        "            lr_actor=5e-5, # Actor learning rate\n",
        "            lr_critic=1e-4 # Critic learning rate\n",
        "        )\n",
        "    # Create replay buffers\n",
        "    replay_buffers = {region: deque(maxlen=10000) for region in env.regions}\n",
        "    # Initialize logger\n",
        "    logger = ExperimentLogger()\n",
        "    print(f\"Commencing training for {n_episodes} episodes...\")\n",
        "    # Main training loop\n",
        "    for episode in range(n_episodes):\n",
        "        observations = env.reset()\n",
        "        episode_rewards = {region: 0 for region in env.regions}\n",
        "        # Adaptive noise for exploration\n",
        "        noise_scale = max(0.05, 0.3 * (1 - episode / n_episodes)) # Slightly higher initial noise\n",
        "        # Step loop\n",
        "        for step in range(env.max_steps):\n",
        "            # Select actions for all agents\n",
        "            actions = {}\n",
        "            for region in env.regions:\n",
        "                action = agents[region].act(observations[region], noise_scale=noise_scale)\n",
        "                actions[region] = action\n",
        "            # Execute step in environment\n",
        "            next_observations, rewards, done, _ = env.step(actions)\n",
        "            # Store experience\n",
        "            for region in env.regions:\n",
        "                replay_buffers[region].append((\n",
        "                    observations[region], actions[region], rewards[region],\n",
        "                    next_observations[region], done\n",
        "                ))\n",
        "                episode_rewards[region] += rewards[region]\n",
        "            observations = next_observations\n",
        "            if done:\n",
        "                break\n",
        "        # Train agents\n",
        "        if episode % 5 == 0 and episode > 5:\n",
        "            for region in env.regions:\n",
        "                if len(replay_buffers[region]) > 200: # Minimum buffer size\n",
        "                    _train_agent_maddpg(agents[region], replay_buffers[region], batch_size=64)\n",
        "        # Log and report\n",
        "        avg_reward = np.mean(list(episode_rewards.values()))\n",
        "        stability_metric = _calculate_stability_metric(env)\n",
        "        logger.log_episode(episode, episode_rewards, avg_reward, stability_metric)\n",
        "        if episode % 50 == 0:\n",
        "            print(f\"{experiment_name} - Episode {episode}, Average Reward: {avg_reward:.3f}, \"\n",
        "                  f\"Stability Metric: {stability_metric:.3f}\")\n",
        "    print(f\"{experiment_name} - Training successfully completed!\")\n",
        "    # Visualize and save results\n",
        "    logger.plot_results()\n",
        "    if save_results:\n",
        "        timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S\")\n",
        "        logger.save_results(f\"{experiment_name.lower()}_results_{timestamp}.csv\")\n",
        "    return agents, logger"
      ],
      "metadata": {
        "id": "y2vtT2kR9a-7"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# === BLOCK 11: Main Training Function (MADDPG-EVO-DGM) ===\n",
        "def train_maddpg_with_evolution_dgm(env, n_episodes=1000, min_improvement=0.1,\n",
        "                               initial_population_size=16, save_results=True, experiment_name=\"MADDPG-EVO-DGM\"):\n",
        "    \"\"\"Train MADDPG agents with a simplified Darwin-Gödel Machine evolutionary optimizer.\"\"\"\n",
        "    print(f\"=== Initiating Experiment: {experiment_name} ===\")\n",
        "    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "    print(f\"Utilized computing device: {device}\")\n",
        "    # Initialize agents\n",
        "    agents = {}\n",
        "    for i, region in enumerate(env.regions):\n",
        "        agents[region] = MADDPGAgent(i, env.state_dim * 2, env.action_dim, env.n_regions, lr_actor=5e-5, lr_critic=1e-4, device=device)\n",
        "    # Initialize DGM-style evolutionary boosters\n",
        "    evolution_boosters = {}\n",
        "    for region in env.regions:\n",
        "        booster = EvolutionaryBoosterDGM(population_size=initial_population_size, initial_mutation_rate=0.05, initial_crossover_rate=0.7)\n",
        "        booster.initialize_population(agents[region])\n",
        "        # Ensure all agents in the initial population are on the correct device\n",
        "        for pop_agent in booster.population:\n",
        "             pop_agent.actor.to(device); pop_agent.actor_target.to(device)\n",
        "             pop_agent.critic.to(device); pop_agent.critic_target.to(device)\n",
        "        evolution_boosters[region] = booster\n",
        "    # Create replay buffers\n",
        "    replay_buffers = {region: deque(maxlen=10000) for region in env.regions}\n",
        "    # Initialize logger\n",
        "    logger = ExperimentLogger()\n",
        "    print(f\"Commencing training for {n_episodes} episodes...\")\n",
        "    # Tracking variables\n",
        "    reward_history, stability_history = [], []\n",
        "    last_evolution_episode, min_episodes_between_evolutions = 0, 15\n",
        "    # Main training loop\n",
        "    for episode in range(n_episodes):\n",
        "        observations = env.reset()\n",
        "        episode_rewards = {region: 0 for region in env.regions}\n",
        "        # Adaptive noise\n",
        "        noise_scale = max(0.05, 0.2 * (1 - episode / n_episodes))\n",
        "        # Episode step loop\n",
        "        for step in range(env.max_steps):\n",
        "            actions = {}\n",
        "            for region in env.regions:\n",
        "                actions[region] = agents[region].act(observations[region], noise_scale=noise_scale)\n",
        "            next_observations, rewards, done, _ = env.step(actions)\n",
        "            # Store experience\n",
        "            for region in env.regions:\n",
        "                replay_buffers[region].append((observations[region], actions[region], rewards[region], next_observations[region], done))\n",
        "                episode_rewards[region] += rewards[region]\n",
        "            observations = next_observations\n",
        "            if done: break\n",
        "        # Train agents more frequently\n",
        "        if episode % 5 == 0 and episode > 5:\n",
        "            for region in env.regions:\n",
        "                if len(replay_buffers[region]) > 500: # Larger buffer required\n",
        "                    _train_agent_maddpg_dgm(agents[region], replay_buffers[region], batch_size=128)\n",
        "        # Log metrics\n",
        "        avg_reward = np.mean(list(episode_rewards.values()))\n",
        "        stability_metric = _calculate_stability_metric(env)\n",
        "        logger.log_episode(episode, episode_rewards, avg_reward, stability_metric)\n",
        "        reward_history.append(avg_reward)\n",
        "        stability_history.append(stability_metric)\n",
        "        # Trigger DGM-style adaptive evolution\n",
        "        if len(reward_history) > 20 and episode - last_evolution_episode >= min_episodes_between_evolutions:\n",
        "            recent_window = min(25, len(reward_history))\n",
        "            recent_rewards = reward_history[-recent_window:]\n",
        "            improvement_rate = (recent_rewards[-1] - recent_rewards[0]) / recent_window\n",
        "            if improvement_rate < min_improvement:\n",
        "                print(f\"{experiment_name} - DGM-style adaptive evolutionary optimization triggered at episode {episode} due to performance plateau\")\n",
        "                # Snapshot of other agents for fair fitness evaluation\n",
        "                other_agents_snapshot = {r: a for r, a in agents.items() if r in env.regions}\n",
        "                for region in env.regions:\n",
        "                    # Evaluate fitness\n",
        "                    fitness_scores = []\n",
        "                    for agent in evolution_boosters[region].population:\n",
        "                        fitness = _evaluate_agent_fitness_improved(agent, env, region, other_agents_snapshot, n_eval_episodes=5)\n",
        "                        fitness_scores.append(fitness)\n",
        "                    current_avg_fitness = np.mean(fitness_scores)\n",
        "                    # Adapt GA parameters based on recent performance\n",
        "                    evolution_boosters[region].adapt_parameters(current_avg_fitness)\n",
        "                    # Evolve population\n",
        "                    best_agent = evolution_boosters[region].evolve_population(fitness_scores)\n",
        "                    # Ensure the new best agent is on the correct device\n",
        "                    best_agent.actor.to(device); best_agent.actor_target.to(device)\n",
        "                    best_agent.critic.to(device); best_agent.critic_target.to(device)\n",
        "                    agents[region] = best_agent\n",
        "                    logger.log_evolution(episode, max(fitness_scores), current_avg_fitness)\n",
        "                last_evolution_episode = episode\n",
        "        # Progress report\n",
        "        if episode % 20 == 0:\n",
        "            print(f\"{experiment_name} - Episode {episode}, Average Reward: {avg_reward:.3f}, Stability Metric: {stability_metric:.3f}\")\n",
        "    print(f\"{experiment_name} - Training successfully completed!\")\n",
        "    # Visualize and save results\n",
        "    logger.plot_results()\n",
        "    if save_results:\n",
        "        timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S\")\n",
        "        logger.save_results(f\"{experiment_name.lower()}_results_{timestamp}.csv\")\n",
        "    return agents, logger"
      ],
      "metadata": {
        "id": "BVdzEpAW-NqH"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# --- Auxiliary Functions for MADDPG-EVO-DGM ---\n",
        "class EvolutionaryBoosterDGM:\n",
        "    \"\"\"A simplified Darwin-Gödel Machine style evolutionary booster with adaptive GA parameters.\"\"\"\n",
        "    def __init__(self, population_size=16, initial_mutation_rate=0.1, initial_crossover_rate=0.7):\n",
        "        self.population_size = population_size\n",
        "        self.mutation_rate = initial_mutation_rate\n",
        "        self.crossover_rate = initial_crossover_rate\n",
        "        self.population = []\n",
        "        self.fitness_history = []\n",
        "        self.last_fitness_improvement = -np.inf\n",
        "        self.generations_since_improvement = 0\n",
        "\n",
        "    def initialize_population(self, agent_template):\n",
        "        \"\"\"Initialize the population of agents.\"\"\"\n",
        "        self.population = []\n",
        "        for _ in range(self.population_size):\n",
        "            agent_copy = self._copy_agent(agent_template)\n",
        "            self._mutate_agent(agent_copy, mutation_strength=0.3)\n",
        "            self.population.append(agent_copy)\n",
        "\n",
        "    def _copy_agent(self, agent):\n",
        "        \"\"\"Create a deep copy of an agent.\"\"\"\n",
        "        new_agent = MADDPGAgent(agent.agent_id, agent.state_dim, agent.action_dim, agent.n_agents, device=agent.device)\n",
        "        if hasattr(agent, 'actor'): new_agent.actor.load_state_dict(agent.actor.state_dict())\n",
        "        if hasattr(agent, 'critic'): new_agent.critic.load_state_dict(agent.critic.state_dict())\n",
        "        if hasattr(agent, 'actor_target'): new_agent.actor_target.load_state_dict(agent.actor_target.state_dict())\n",
        "        if hasattr(agent, 'critic_target'): new_agent.critic_target.load_state_dict(agent.critic_target.state_dict())\n",
        "        return new_agent\n",
        "\n",
        "    def _mutate_agent(self, agent, mutation_strength=0.1):\n",
        "        \"\"\"Apply mutation to an agent's parameters.\"\"\"\n",
        "        if hasattr(agent, 'actor'):\n",
        "            for param in agent.actor.parameters():\n",
        "                if np.random.random() < self.mutation_rate:\n",
        "                    noise = torch.randn_like(param) * mutation_strength\n",
        "                    param.data.add_(noise) # In-place addition\n",
        "        if hasattr(agent, 'critic'):\n",
        "            for param in agent.critic.parameters():\n",
        "                if np.random.random() < self.mutation_rate:\n",
        "                    noise = torch.randn_like(param) * mutation_strength\n",
        "                    param.data.add_(noise) # In-place addition\n",
        "\n",
        "    def _crossover_agents(self, parent1, parent2):\n",
        "        \"\"\"Perform crossover between two parent agents.\"\"\"\n",
        "        child1, child2 = self._copy_agent(parent1), self._copy_agent(parent2)\n",
        "        if hasattr(parent1, 'actor') and hasattr(parent2, 'actor'):\n",
        "            for p1, p2, c1, c2 in zip(parent1.actor.parameters(), parent2.actor.parameters(), child1.actor.parameters(), child2.actor.parameters()):\n",
        "                if np.random.random() < self.crossover_rate:\n",
        "                    mask = torch.rand_like(p1) > 0.5\n",
        "                    c1.data.copy_(torch.where(mask, p1.data, p2.data)) # In-place copy\n",
        "                    c2.data.copy_(torch.where(mask, p2.data, p1.data)) # In-place copy\n",
        "        if hasattr(parent1, 'critic') and hasattr(parent2, 'critic'):\n",
        "            for p1, p2, c1, c2 in zip(parent1.critic.parameters(), parent2.critic.parameters(), child1.critic.parameters(), child2.critic.parameters()):\n",
        "                if np.random.random() < self.crossover_rate:\n",
        "                    mask = torch.rand_like(p1) > 0.5\n",
        "                    c1.data.copy_(torch.where(mask, p1.data, p2.data)) # In-place copy\n",
        "                    c2.data.copy_(torch.where(mask, p2.data, p1.data)) # In-place copy\n",
        "        return child1, child2\n",
        "\n",
        "    def adapt_parameters(self, current_fitness, fitness_threshold=0.01):\n",
        "        \"\"\"Adapt mutation and crossover rates based on recent fitness improvement.\"\"\"\n",
        "        improvement = current_fitness - self.last_fitness_improvement\n",
        "        if improvement < fitness_threshold:\n",
        "            # No significant improvement, increase exploration\n",
        "            self.generations_since_improvement += 1\n",
        "            self.mutation_rate = min(0.5, self.mutation_rate * 1.1) # Increase mutation\n",
        "            self.crossover_rate = max(0.3, self.crossover_rate * 0.95) # Decrease crossover\n",
        "        else:\n",
        "            # Improvement seen, exploit current strategies\n",
        "            self.generations_since_improvement = max(0, self.generations_since_improvement - 1)\n",
        "            self.mutation_rate = max(0.01, self.mutation_rate * 0.95) # Decrease mutation\n",
        "            self.crossover_rate = min(0.9, self.crossover_rate * 1.02) # Increase crossover\n",
        "        self.last_fitness_improvement = current_fitness\n",
        "\n",
        "    def evolve_population(self, fitness_scores):\n",
        "        \"\"\"Evolve the population using selection, crossover, and mutation.\"\"\"\n",
        "        population_fitness = list(zip(self.population, fitness_scores))\n",
        "        population_fitness.sort(key=lambda x: x[1], reverse=True)\n",
        "        elite_size = self.population_size // 4\n",
        "        elite = [agent for agent, _ in population_fitness[:elite_size]]\n",
        "        new_population = elite.copy()\n",
        "        while len(new_population) < self.population_size:\n",
        "            parent1 = self._tournament_selection(population_fitness)\n",
        "            parent2 = self._tournament_selection(population_fitness)\n",
        "            if np.random.random() < self.crossover_rate:\n",
        "                child1, child2 = self._crossover_agents(parent1, parent2)\n",
        "                self._mutate_agent(child1); self._mutate_agent(child2)\n",
        "                new_population.extend([child1, child2])\n",
        "            else:\n",
        "                child = self._copy_agent(parent1)\n",
        "                self._mutate_agent(child)\n",
        "                new_population.append(child)\n",
        "        self.population = new_population[:self.population_size]\n",
        "        self.fitness_history.append(max(fitness_scores))\n",
        "        return self.population[0] # Return the best agent\n",
        "\n",
        "    def _tournament_selection(self, population_fitness, tournament_size=3):\n",
        "        \"\"\"Select a parent via tournament selection.\"\"\"\n",
        "        tournament = random.sample(population_fitness, min(tournament_size, len(population_fitness)))\n",
        "        return max(tournament, key=lambda x: x[1])[0]\n",
        "\n",
        "def _train_agent_maddpg_dgm(agent, replay_buffer, batch_size=32):\n",
        "    \"\"\"Training function for a single MADDPG agent (DGM variant).\"\"\"\n",
        "    if len(replay_buffer) < batch_size: return\n",
        "    batch = random.sample(replay_buffer, batch_size)\n",
        "    states = torch.FloatTensor([x[0] for x in batch]).to(agent.device)\n",
        "    actions = torch.FloatTensor([x[1] for x in batch]).to(agent.device)\n",
        "    rewards = torch.FloatTensor([x[2] for x in batch]).unsqueeze(1).to(agent.device)\n",
        "    next_states = torch.FloatTensor([x[3] for x in batch]).to(agent.device)\n",
        "    dones = torch.BoolTensor([x[4] for x in batch]).unsqueeze(1).to(agent.device)\n",
        "    # Update Critic\n",
        "    with torch.no_grad():\n",
        "        next_actions = agent.actor_target(next_states)\n",
        "        global_next_states = next_states.repeat(1, agent.n_agents).view(batch_size, -1)\n",
        "        global_next_actions = next_actions.repeat(1, agent.n_agents).view(batch_size, -1)\n",
        "        target_q = agent.critic_target(global_next_states, global_next_actions)\n",
        "        target_q = rewards + agent.gamma * target_q * (~dones)\n",
        "    global_states = states.repeat(1, agent.n_agents).view(batch_size, -1)\n",
        "    global_actions = actions.repeat(1, agent.n_agents).view(batch_size, -1)\n",
        "    current_q = agent.critic(global_states, global_actions)\n",
        "    critic_loss = F.mse_loss(current_q, target_q)\n",
        "    agent.critic_optimizer.zero_grad()\n",
        "    critic_loss.backward()\n",
        "    torch.nn.utils.clip_grad_norm_(agent.critic.parameters(), 0.5)\n",
        "    agent.critic_optimizer.step()\n",
        "    # Update Actor\n",
        "    predicted_actions = agent.actor(states)\n",
        "    global_predicted_actions = predicted_actions.repeat(1, agent.n_agents).view(batch_size, -1)\n",
        "    actor_loss = -agent.critic(global_states, global_predicted_actions).mean()\n",
        "    agent.actor_optimizer.zero_grad()\n",
        "    actor_loss.backward()\n",
        "    torch.nn.utils.clip_grad_norm_(agent.actor.parameters(), 0.5)\n",
        "    agent.actor_optimizer.step()\n",
        "    # Soft update target networks\n",
        "    agent.soft_update(agent.actor_target, agent.actor)\n",
        "    agent.soft_update(agent.critic_target, agent.critic)\n",
        "\n",
        "def _evaluate_agent_fitness_improved(agent, env, region, other_agents_dict, n_eval_episodes=5):\n",
        "    \"\"\"Improved fitness evaluation using a snapshot of other agents for a fair comparison.\"\"\"\n",
        "    total_reward = 0\n",
        "    for _ in range(n_eval_episodes):\n",
        "        observations = env.reset()\n",
        "        episode_reward = 0\n",
        "        for _ in range(env.max_steps):\n",
        "            # Get action from the agent being evaluated\n",
        "            state_tensor = torch.FloatTensor(observations[region]).unsqueeze(0).to(agent.device)\n",
        "            with torch.no_grad():\n",
        "                action = agent.actor(state_tensor).cpu().numpy().squeeze(0)\n",
        "            actions = {region: np.clip(action, -1, 1)}\n",
        "            # Get actions from the snapshot of other agents\n",
        "            for other_region, other_agent in other_agents_dict.items():\n",
        "                if other_region != region:\n",
        "                    other_state_tensor = torch.FloatTensor(observations[other_region]).unsqueeze(0).to(agent.device)\n",
        "                    with torch.no_grad():\n",
        "                        other_action = other_agent.actor(other_state_tensor).cpu().numpy().squeeze(0)\n",
        "                    actions[other_region] = np.clip(other_action, -1, 1)\n",
        "            next_observations, rewards, done, _ = env.step(actions)\n",
        "            episode_reward += rewards.get(region, 0)\n",
        "            observations = next_observations\n",
        "            if done: break\n",
        "        total_reward += episode_reward\n",
        "    return total_reward / n_eval_episodes"
      ],
      "metadata": {
        "id": "WpBofS0o-NtP"
      },
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# === BLOCK 13: Results Comparison ===\n",
        "if __name__ == \"__main__\":\n",
        "    print(\"=== INITIATING EXPERIMENTS ===\")\n",
        "    # --- Data Preparation and Environment Setup ---\n",
        "    print(\"Loading and processing real-world demographic data...\")\n",
        "    # REAL FILES\n",
        "    data_processor = DemographicDataProcessor(\n",
        "        'regions_data_selective.csv',  # path to actual regional statistics\n",
        "        'crisis.txt'  # path to crisis scenario definitions\n",
        "    )\n",
        "    # MODIFIED: Utilize data from the year 2000 onwards\n",
        "    years = list(range(2000, 2025))\n",
        "    target_regions = ['Moscow', 'St. Petersburg', 'Tatarstan', 'Krasnodar Krai',\n",
        "                     'Sverdlovsk Oblast', 'Novosibirsk Oblast', 'Samara Oblast', 'Rostov Oblast']\n",
        "    training_data = data_processor.generate_training_data(years, target_regions, apply_crisis=True)\n",
        "    print(f\"Processed {len(training_data)} data records for {len(target_regions)} regions.\")\n",
        "\n",
        "    # --- EXPERIMENT 1: MADDPG-BASELINE ---\n",
        "    print(\"\\n--- EXPERIMENT 1: MADDPG-BASELINE ---\")\n",
        "    # Create a new, independent environment instance for this experiment\n",
        "    env_baseline = DemographicEnvironment(training_data.copy(), n_regions=8, max_steps=50)\n",
        "    agents_maddpg_baseline, logger_maddpg_baseline = train_maddpg_baseline(\n",
        "        env_baseline, n_episodes=1000, experiment_name=\"MADDPG-BASELINE\"\n",
        "    )\n",
        "    print(\"=== MADDPG-BASELINE Experiment Concluded ===\")\n",
        "\n",
        "    # --- EXPERIMENT 2: MADDPG-EVO ---\n",
        "    print(\"\\n--- EXPERIMENT 2: MADDPG-EVO ---\")\n",
        "    # Create a new, independent environment instance for this experiment\n",
        "    env_evo = DemographicEnvironment(training_data.copy(), n_regions=8, max_steps=50)\n",
        "    agents_maddpg_evo, logger_maddpg_evo = train_maddpg_with_evolution(\n",
        "        env_evo, n_episodes=1000, min_improvement=0.1, population_size=16, experiment_name=\"MADDPG-EVO\"\n",
        "    )\n",
        "    print(\"=== MADDPG-EVO Experiment Concluded ===\")\n",
        "\n",
        "    # --- EXPERIMENT 3: MADDPG-EVO-DGM ---\n",
        "    print(\"\\n--- EXPERIMENT 3: MADDPG-EVO-DGM ---\")\n",
        "    # Create a new, independent environment instance for this experiment\n",
        "    env_dgm = DemographicEnvironment(training_data.copy(), n_regions=8, max_steps=50)\n",
        "    agents_maddpg_evo_dgm, logger_maddpg_evo_dgm = train_maddpg_with_evolution_dgm(\n",
        "        env_dgm, n_episodes=1000, min_improvement=0.1, initial_population_size=16, experiment_name=\"MADDPG-EVO-DGM\"\n",
        "    )\n",
        "    print(\"=== MADDPG-EVO-DGM Experiment Concluded ===\")\n",
        "\n",
        "    # --- COMPARATIVE ANALYSIS OF RESULTS ---\n",
        "    print(\"\\n=== COMPARATIVE ANALYSIS OF ALL EXPERIMENTS ===\")\n",
        "    # Extract performance data from the experiment loggers\n",
        "    # --- MADDPG-BASELINE ---\n",
        "    baseline_episodes = np.array(logger_maddpg_baseline.metrics['episode'])\n",
        "    baseline_avg_rewards = np.array(logger_maddpg_baseline.metrics['avg_reward'])\n",
        "    baseline_stability = np.array(logger_maddpg_baseline.metrics['stability'])\n",
        "    # --- MADDPG-EVO ---\n",
        "    evo_episodes = np.array(logger_maddpg_evo.metrics['episode'])\n",
        "    evo_avg_rewards = np.array(logger_maddpg_evo.metrics['avg_reward'])\n",
        "    evo_stability = np.array(logger_maddpg_evo.metrics['stability'])\n",
        "    # --- MADDPG-EVO-DGM ---\n",
        "    dgm_episodes = np.array(logger_maddpg_evo_dgm.metrics['episode'])\n",
        "    dgm_avg_rewards = np.array(logger_maddpg_evo_dgm.metrics['avg_reward'])\n",
        "    dgm_stability = np.array(logger_maddpg_evo_dgm.metrics['stability'])\n",
        "\n",
        "    # --- Generate Comparative Performance Plots ---\n",
        "    plt.figure(figsize=(15, 6))\n",
        "    # Plot 1: Comparison of Average Episode Reward\n",
        "    plt.subplot(1, 2, 1)\n",
        "    plt.plot(baseline_episodes, baseline_avg_rewards, label='MADDPG-BASELINE', alpha=0.8)\n",
        "    plt.plot(evo_episodes, evo_avg_rewards, label='MADDPG-EVO', alpha=0.8)\n",
        "    plt.plot(dgm_episodes, dgm_avg_rewards, label='MADDPG-EVO-DGM', alpha=0.8)\n",
        "    plt.title('Comparison: Average Reward per Episode')\n",
        "    plt.xlabel('Episode')\n",
        "    plt.ylabel('Average Reward')\n",
        "    plt.legend()\n",
        "    plt.grid(True)\n",
        "    # Plot 2: Comparison of System Stability Metric\n",
        "    plt.subplot(1, 2, 2)\n",
        "    plt.plot(baseline_episodes, baseline_stability, label='MADDPG-BASELINE', alpha=0.8)\n",
        "    plt.plot(evo_episodes, evo_stability, label='MADDPG-EVO', alpha=0.8)\n",
        "    plt.plot(dgm_episodes, dgm_stability, label='MADDPG-EVO-DGM', alpha=0.8)\n",
        "    plt.title('Comparison: System Stability Metric')\n",
        "    plt.xlabel('Episode')\n",
        "    plt.ylabel('Stability Metric')\n",
        "    plt.legend()\n",
        "    plt.grid(True)\n",
        "    plt.tight_layout()\n",
        "    plt.show()\n",
        "\n",
        "    # --- Calculate and Report Aggregated Performance Metrics ---\n",
        "    # Determine a common length for fair comparison (minimum length across all experiments)\n",
        "    min_length = min(len(baseline_avg_rewards), len(evo_avg_rewards), len(dgm_avg_rewards))\n",
        "    if min_length == 0:\n",
        "        print(\"Error: Insufficient data available for comparative analysis.\")\n",
        "    else:\n",
        "        # Aggregated metrics for MADDPG-BASELINE\n",
        "        baseline_metrics_summary = {\n",
        "            'Mean Reward': np.mean(baseline_avg_rewards[-min_length:]),\n",
        "            'Max Reward': np.max(baseline_avg_rewards),\n",
        "            'Mean Stability': np.mean(baseline_stability[-min_length:]),\n",
        "            'Max Stability': np.max(baseline_stability)\n",
        "        }\n",
        "        # Aggregated metrics for MADDPG-EVO\n",
        "        evo_metrics_summary = {\n",
        "            'Mean Reward': np.mean(evo_avg_rewards[-min_length:]),\n",
        "            'Max Reward': np.max(evo_avg_rewards),\n",
        "            'Mean Stability': np.mean(evo_stability[-min_length:]),\n",
        "            'Max Stability': np.max(evo_stability)\n",
        "        }\n",
        "        # Aggregated metrics for MADDPG-EVO-DGM\n",
        "        dgm_metrics_summary = {\n",
        "            'Mean Reward': np.mean(dgm_avg_rewards[-min_length:]),\n",
        "            'Max Reward': np.max(dgm_avg_rewards),\n",
        "            'Mean Stability': np.mean(dgm_stability[-min_length:]),\n",
        "            'Max Stability': np.max(dgm_stability)\n",
        "        }\n",
        "        # --- Construct and Display the Comparative Metrics Table ---\n",
        "        import pandas as pd\n",
        "        # Assemble the data into a DataFrame\n",
        "        comparison_data = {\n",
        "            'Metric': list(baseline_metrics_summary.keys()),\n",
        "            'MADDPG-BASELINE': list(baseline_metrics_summary.values()),\n",
        "            'MADDPG-EVO': list(evo_metrics_summary.values()),\n",
        "            'MADDPG-EVO-DGM': list(dgm_metrics_summary.values())\n",
        "        }\n",
        "        comparison_df = pd.DataFrame(comparison_data)\n",
        "        # Calculate performance differences relative to the BASELINE\n",
        "        comparison_df['Difference (EVO - BASE)'] = comparison_df['MADDPG-EVO'] - comparison_df['MADDPG-BASELINE']\n",
        "        comparison_df['Difference (DGM - BASE)'] = comparison_df['MADDPG-EVO-DGM'] - comparison_df['MADDPG-BASELINE']\n",
        "        # Formatting for clean numerical display\n",
        "        pd.options.display.float_format = '{:.4f}'.format\n",
        "        print(\"\\n--- Comparative Table of Aggregated Metrics ---\")\n",
        "        print(comparison_df.to_string(index=False))\n",
        "        print(\"----------------------------------------------\")\n",
        "    print(\"=== COMPARATIVE ANALYSIS OF ALL EXPERIMENTS CONCLUDED ===\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "j--Q_TfD-Nv2",
        "outputId": "a93db36c-30dc-4770-d88e-e85ac57e94b8"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "=== INITIATING EXPERIMENTS ===\n",
            "Loading and processing real-world demographic data...\n",
            "Processed 200 data records for 8 regions.\n",
            "\n",
            "--- EXPERIMENT 1: MADDPG-BASELINE ---\n",
            "=== Initiating Experiment: MADDPG-BASELINE ===\n",
            "Training for 1000 episodes. Pure MADDPG baseline (no evolutionary optimization).\n",
            "Commencing training for 1000 episodes...\n",
            "MADDPG-BASELINE - Episode 0, Average Reward: 69.251, Stability Metric: 1.118\n",
            "MADDPG-BASELINE - Episode 50, Average Reward: 71.158, Stability Metric: 0.995\n",
            "MADDPG-BASELINE - Episode 100, Average Reward: 78.126, Stability Metric: 0.942\n",
            "MADDPG-BASELINE - Episode 150, Average Reward: 88.818, Stability Metric: 0.862\n",
            "MADDPG-BASELINE - Episode 200, Average Reward: 108.392, Stability Metric: 0.751\n",
            "MADDPG-BASELINE - Episode 250, Average Reward: 117.017, Stability Metric: 0.752\n",
            "MADDPG-BASELINE - Episode 300, Average Reward: 138.903, Stability Metric: 0.719\n",
            "MADDPG-BASELINE - Episode 350, Average Reward: 155.203, Stability Metric: 0.746\n",
            "MADDPG-BASELINE - Episode 400, Average Reward: 173.259, Stability Metric: 0.761\n",
            "MADDPG-BASELINE - Episode 450, Average Reward: 195.974, Stability Metric: 0.770\n",
            "MADDPG-BASELINE - Episode 500, Average Reward: 207.166, Stability Metric: 0.757\n",
            "MADDPG-BASELINE - Episode 550, Average Reward: 219.734, Stability Metric: 0.743\n",
            "MADDPG-BASELINE - Episode 600, Average Reward: 227.428, Stability Metric: 0.733\n",
            "MADDPG-BASELINE - Episode 650, Average Reward: 232.332, Stability Metric: 0.724\n",
            "MADDPG-BASELINE - Episode 700, Average Reward: 236.177, Stability Metric: 0.719\n",
            "MADDPG-BASELINE - Episode 750, Average Reward: 238.579, Stability Metric: 0.714\n",
            "MADDPG-BASELINE - Episode 800, Average Reward: 240.534, Stability Metric: 0.710\n",
            "MADDPG-BASELINE - Episode 850, Average Reward: 241.872, Stability Metric: 0.708\n",
            "MADDPG-BASELINE - Episode 900, Average Reward: 242.719, Stability Metric: 0.707\n",
            "MADDPG-BASELINE - Episode 950, Average Reward: 243.095, Stability Metric: 0.706\n",
            "MADDPG-BASELINE - Training successfully completed!\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 1500x1000 with 5 Axes>"
            ],
            "image/png": "[encoded data, 294668 characters]\n"
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Results saved to maddpg-baseline_results_20250912_081827.csv\n",
            "=== MADDPG-BASELINE Experiment Concluded ===\n",
            "\n",
            "--- EXPERIMENT 2: MADDPG-EVO ---\n",
            "=== Initiating Experiment: MADDPG-EVO ===\n",
            "Training for 1000 episodes. Adaptive evolution activated upon performance stagnation (min_improvement=0.1).\n",
            "Commencing training for 1000 episodes...\n",
            "MADDPG-EVO - Episode 0, Average Reward: 69.970, Stability Metric: 1.115\n",
            "MADDPG-EVO - Episode 20, Average Reward: 69.670, Stability Metric: 1.039\n",
            "MADDPG-EVO - Adaptive evolutionary optimization triggered at episode 30 due to performance plateau\n",
            "MADDPG-EVO - Episode 40, Average Reward: 113.524, Stability Metric: 0.493\n",
            "MADDPG-EVO - Adaptive evolutionary optimization triggered at episode 60 due to performance plateau\n",
            "MADDPG-EVO - Episode 60, Average Reward: 116.585, Stability Metric: 0.831\n",
            "MADDPG-EVO - Episode 80, Average Reward: 121.687, Stability Metric: 0.802\n",
            "MADDPG-EVO - Adaptive evolutionary optimization triggered at episode 82 due to performance plateau\n",
            "MADDPG-EVO - Episode 100, Average Reward: 156.254, Stability Metric: 0.763\n",
            "MADDPG-EVO - Adaptive evolutionary optimization triggered at episode 118 due to performance plateau\n",
            "MADDPG-EVO - Episode 120, Average Reward: 162.469, Stability Metric: 0.781\n",
            "MADDPG-EVO - Episode 140, Average Reward: 168.299, Stability Metric: 0.778\n",
            "MADDPG-EVO - Adaptive evolutionary optimization triggered at episode 141 due to performance plateau\n",
            "MADDPG-EVO - Episode 160, Average Reward: 171.718, Stability Metric: 0.713\n",
            "MADDPG-EVO - Adaptive evolutionary optimization triggered at episode 179 due to performance plateau\n",
            "MADDPG-EVO - Episode 180, Average Reward: 180.358, Stability Metric: 0.735\n",
            "MADDPG-EVO - Adaptive evolutionary optimization triggered at episode 200 due to performance plateau\n",
            "MADDPG-EVO - Episode 200, Average Reward: 180.879, Stability Metric: 0.728\n",
            "MADDPG-EVO - Episode 220, Average Reward: 188.315, Stability Metric: 0.715\n",
            "MADDPG-EVO - Episode 240, Average Reward: 191.687, Stability Metric: 0.725\n",
            "MADDPG-EVO - Adaptive evolutionary optimization triggered at episode 260 due to performance plateau\n",
            "MADDPG-EVO - Episode 260, Average Reward: 194.802, Stability Metric: 0.726\n",
            "MADDPG-EVO - Episode 280, Average Reward: 201.756, Stability Metric: 0.733\n",
            "MADDPG-EVO - Adaptive evolutionary optimization triggered at episode 287 due to performance plateau\n",
            "MADDPG-EVO - Episode 300, Average Reward: 193.968, Stability Metric: 0.718\n",
            "MADDPG-EVO - Adaptive evolutionary optimization triggered at episode 307 due to performance plateau\n",
            "MADDPG-EVO - Episode 320, Average Reward: 198.205, Stability Metric: 0.724\n",
            "MADDPG-EVO - Adaptive evolutionary optimization triggered at episode 338 due to performance plateau\n",
            "MADDPG-EVO - Episode 340, Average Reward: 196.837, Stability Metric: 0.732\n",
            "MADDPG-EVO - Adaptive evolutionary optimization triggered at episode 360 due to performance plateau\n",
            "MADDPG-EVO - Episode 360, Average Reward: 202.915, Stability Metric: 0.734\n",
            "MADDPG-EVO - Episode 380, Average Reward: 207.127, Stability Metric: 0.734\n",
            "MADDPG-EVO - Adaptive evolutionary optimization triggered at episode 384 due to performance plateau\n",
            "MADDPG-EVO - Episode 400, Average Reward: 204.382, Stability Metric: 0.733\n",
            "MADDPG-EVO - Adaptive evolutionary optimization triggered at episode 405 due to performance plateau\n",
            "MADDPG-EVO - Episode 420, Average Reward: 209.395, Stability Metric: 0.743\n",
            "MADDPG-EVO - Adaptive evolutionary optimization triggered at episode 426 due to performance plateau\n",
            "MADDPG-EVO - Episode 440, Average Reward: 211.409, Stability Metric: 0.722\n",
            "MADDPG-EVO - Adaptive evolutionary optimization triggered at episode 447 due to performance plateau\n",
            "MADDPG-EVO - Episode 460, Average Reward: 212.544, Stability Metric: 0.728\n",
            "MADDPG-EVO - Adaptive evolutionary optimization triggered at episode 467 due to performance plateau\n",
            "MADDPG-EVO - Episode 480, Average Reward: 214.515, Stability Metric: 0.716\n",
            "MADDPG-EVO - Adaptive evolutionary optimization triggered at episode 488 due to performance plateau\n",
            "MADDPG-EVO - Episode 500, Average Reward: 221.920, Stability Metric: 0.710\n",
            "MADDPG-EVO - Episode 520, Average Reward: 224.523, Stability Metric: 0.715\n",
            "MADDPG-EVO - Adaptive evolutionary optimization triggered at episode 530 due to performance plateau\n",
            "MADDPG-EVO - Episode 540, Average Reward: 218.232, Stability Metric: 0.712\n",
            "MADDPG-EVO - Adaptive evolutionary optimization triggered at episode 550 due to performance plateau\n",
            "MADDPG-EVO - Episode 560, Average Reward: 224.232, Stability Metric: 0.715\n",
            "MADDPG-EVO - Adaptive evolutionary optimization triggered at episode 570 due to performance plateau\n",
            "MADDPG-EVO - Episode 580, Average Reward: 228.314, Stability Metric: 0.712\n",
            "MADDPG-EVO - Adaptive evolutionary optimization triggered at episode 590 due to performance plateau\n",
            "MADDPG-EVO - Episode 600, Average Reward: 224.472, Stability Metric: 0.718\n",
            "MADDPG-EVO - Adaptive evolutionary optimization triggered at episode 610 due to performance plateau\n",
            "MADDPG-EVO - Episode 620, Average Reward: 226.362, Stability Metric: 0.715\n",
            "MADDPG-EVO - Adaptive evolutionary optimization triggered at episode 630 due to performance plateau\n",
            "MADDPG-EVO - Episode 640, Average Reward: 230.542, Stability Metric: 0.716\n",
            "MADDPG-EVO - Adaptive evolutionary optimization triggered at episode 650 due to performance plateau\n",
            "MADDPG-EVO - Episode 660, Average Reward: 228.683, Stability Metric: 0.717\n",
            "MADDPG-EVO - Adaptive evolutionary optimization triggered at episode 670 due to performance plateau\n",
            "MADDPG-EVO - Episode 680, Average Reward: 229.843, Stability Metric: 0.714\n",
            "MADDPG-EVO - Adaptive evolutionary optimization triggered at episode 690 due to performance plateau\n",
            "MADDPG-EVO - Episode 700, Average Reward: 236.132, Stability Metric: 0.707\n",
            "MADDPG-EVO - Episode 720, Average Reward: 238.262, Stability Metric: 0.712\n",
            "MADDPG-EVO - Adaptive evolutionary optimization triggered at episode 727 due to performance plateau\n",
            "MADDPG-EVO - Episode 740, Average Reward: 234.405, Stability Metric: 0.714\n",
            "MADDPG-EVO - Adaptive evolutionary optimization triggered at episode 747 due to performance plateau\n",
            "MADDPG-EVO - Episode 760, Average Reward: 238.238, Stability Metric: 0.715\n",
            "MADDPG-EVO - Adaptive evolutionary optimization triggered at episode 771 due to performance plateau\n",
            "MADDPG-EVO - Episode 780, Average Reward: 241.406, Stability Metric: 0.709\n",
            "MADDPG-EVO - Adaptive evolutionary optimization triggered at episode 791 due to performance plateau\n",
            "MADDPG-EVO - Episode 800, Average Reward: 232.173, Stability Metric: 0.716\n",
            "MADDPG-EVO - Adaptive evolutionary optimization triggered at episode 811 due to performance plateau\n",
            "MADDPG-EVO - Episode 820, Average Reward: 232.930, Stability Metric: 0.723\n",
            "MADDPG-EVO - Adaptive evolutionary optimization triggered at episode 833 due to performance plateau\n",
            "MADDPG-EVO - Episode 840, Average Reward: 241.501, Stability Metric: 0.715\n",
            "MADDPG-EVO - Episode 860, Average Reward: 242.163, Stability Metric: 0.720\n",
            "MADDPG-EVO - Adaptive evolutionary optimization triggered at episode 863 due to performance plateau\n",
            "MADDPG-EVO - Episode 880, Average Reward: 242.131, Stability Metric: 0.718\n",
            "MADDPG-EVO - Adaptive evolutionary optimization triggered at episode 883 due to performance plateau\n",
            "MADDPG-EVO - Episode 900, Average Reward: 236.890, Stability Metric: 0.721\n",
            "MADDPG-EVO - Adaptive evolutionary optimization triggered at episode 903 due to performance plateau\n",
            "MADDPG-EVO - Episode 920, Average Reward: 239.361, Stability Metric: 0.718\n",
            "MADDPG-EVO - Adaptive evolutionary optimization triggered at episode 923 due to performance plateau\n",
            "MADDPG-EVO - Episode 940, Average Reward: 243.147, Stability Metric: 0.715\n",
            "MADDPG-EVO - Adaptive evolutionary optimization triggered at episode 943 due to performance plateau\n",
            "MADDPG-EVO - Episode 960, Average Reward: 244.567, Stability Metric: 0.702\n",
            "MADDPG-EVO - Adaptive evolutionary optimization triggered at episode 963 due to performance plateau\n",
            "MADDPG-EVO - Episode 980, Average Reward: 247.410, Stability Metric: 0.708\n",
            "MADDPG-EVO - Adaptive evolutionary optimization triggered at episode 986 due to performance plateau\n",
            "MADDPG-EVO - Training successfully completed!\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 1500x1000 with 6 Axes>"
            ],
            "image/png": "[encoded data, 326996 characters]\n"
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Results saved to maddpg-evo_results_20250912_083503.csv\n",
            "=== MADDPG-EVO Experiment Concluded ===\n",
            "\n",
            "--- EXPERIMENT 3: MADDPG-EVO-DGM ---\n",
            "=== Initiating Experiment: MADDPG-EVO-DGM ===\n",
            "Utilized computing device: cpu\n",
            "Commencing training for 1000 episodes...\n",
            "MADDPG-EVO-DGM - Episode 0, Average Reward: 61.586, Stability Metric: 1.166\n",
            "MADDPG-EVO-DGM - DGM-style adaptive evolutionary optimization triggered at episode 20 due to performance plateau\n",
            "MADDPG-EVO-DGM - Episode 20, Average Reward: 62.414, Stability Metric: 1.082\n",
            "MADDPG-EVO-DGM - Episode 40, Average Reward: 118.904, Stability Metric: 0.775\n",
            "MADDPG-EVO-DGM - Episode 60, Average Reward: 133.824, Stability Metric: 0.696\n",
            "MADDPG-EVO-DGM - Episode 80, Average Reward: 145.978, Stability Metric: 0.710\n",
            "MADDPG-EVO-DGM - Episode 100, Average Reward: 154.469, Stability Metric: 0.707\n",
            "MADDPG-EVO-DGM - Episode 120, Average Reward: 166.287, Stability Metric: 0.719\n",
            "MADDPG-EVO-DGM - Episode 140, Average Reward: 173.909, Stability Metric: 0.735\n",
            "MADDPG-EVO-DGM - Episode 160, Average Reward: 181.357, Stability Metric: 0.739\n",
            "MADDPG-EVO-DGM - Episode 180, Average Reward: 188.783, Stability Metric: 0.741\n",
            "MADDPG-EVO-DGM - Episode 200, Average Reward: 194.318, Stability Metric: 0.740\n",
            "MADDPG-EVO-DGM - Episode 220, Average Reward: 197.193, Stability Metric: 0.734\n",
            "MADDPG-EVO-DGM - DGM-style adaptive evolutionary optimization triggered at episode 230 due to performance plateau\n",
            "MADDPG-EVO-DGM - Episode 240, Average Reward: 207.758, Stability Metric: 0.722\n",
            "MADDPG-EVO-DGM - Episode 260, Average Reward: 212.227, Stability Metric: 0.722\n",
            "MADDPG-EVO-DGM - DGM-style adaptive evolutionary optimization triggered at episode 265 due to performance plateau\n",
            "MADDPG-EVO-DGM - Episode 280, Average Reward: 216.238, Stability Metric: 0.708\n",
            "MADDPG-EVO-DGM - DGM-style adaptive evolutionary optimization triggered at episode 283 due to performance plateau\n",
            "MADDPG-EVO-DGM - Episode 300, Average Reward: 228.172, Stability Metric: 0.702\n",
            "MADDPG-EVO-DGM - DGM-style adaptive evolutionary optimization triggered at episode 308 due to performance plateau\n",
            "MADDPG-EVO-DGM - Episode 320, Average Reward: 231.132, Stability Metric: 0.707\n",
            "MADDPG-EVO-DGM - DGM-style adaptive evolutionary optimization triggered at episode 330 due to performance plateau\n",
            "MADDPG-EVO-DGM - Episode 340, Average Reward: 234.387, Stability Metric: 0.716\n",
            "MADDPG-EVO-DGM - DGM-style adaptive evolutionary optimization triggered at episode 345 due to performance plateau\n",
            "MADDPG-EVO-DGM - DGM-style adaptive evolutionary optimization triggered at episode 360 due to performance plateau\n",
            "MADDPG-EVO-DGM - Episode 360, Average Reward: 236.406, Stability Metric: 0.717\n",
            "MADDPG-EVO-DGM - DGM-style adaptive evolutionary optimization triggered at episode 375 due to performance plateau\n",
            "MADDPG-EVO-DGM - Episode 380, Average Reward: 241.326, Stability Metric: 0.715\n",
            "MADDPG-EVO-DGM - DGM-style adaptive evolutionary optimization triggered at episode 390 due to performance plateau\n",
            "MADDPG-EVO-DGM - Episode 400, Average Reward: 241.265, Stability Metric: 0.722\n",
            "MADDPG-EVO-DGM - DGM-style adaptive evolutionary optimization triggered at episode 405 due to performance plateau\n",
            "MADDPG-EVO-DGM - DGM-style adaptive evolutionary optimization triggered at episode 420 due to performance plateau\n",
            "MADDPG-EVO-DGM - Episode 420, Average Reward: 243.815, Stability Metric: 0.717\n",
            "MADDPG-EVO-DGM - DGM-style adaptive evolutionary optimization triggered at episode 435 due to performance plateau\n",
            "MADDPG-EVO-DGM - Episode 440, Average Reward: 245.306, Stability Metric: 0.712\n",
            "MADDPG-EVO-DGM - DGM-style adaptive evolutionary optimization triggered at episode 450 due to performance plateau\n",
            "MADDPG-EVO-DGM - Episode 460, Average Reward: 246.842, Stability Metric: 0.703\n",
            "MADDPG-EVO-DGM - DGM-style adaptive evolutionary optimization triggered at episode 465 due to performance plateau\n",
            "MADDPG-EVO-DGM - DGM-style adaptive evolutionary optimization triggered at episode 480 due to performance plateau\n",
            "MADDPG-EVO-DGM - Episode 480, Average Reward: 246.825, Stability Metric: 0.707\n",
            "MADDPG-EVO-DGM - DGM-style adaptive evolutionary optimization triggered at episode 495 due to performance plateau\n",
            "MADDPG-EVO-DGM - Episode 500, Average Reward: 246.439, Stability Metric: 0.708\n",
            "MADDPG-EVO-DGM - DGM-style adaptive evolutionary optimization triggered at episode 510 due to performance plateau\n",
            "MADDPG-EVO-DGM - Episode 520, Average Reward: 247.813, Stability Metric: 0.701\n",
            "MADDPG-EVO-DGM - DGM-style adaptive evolutionary optimization triggered at episode 525 due to performance plateau\n",
            "MADDPG-EVO-DGM - DGM-style adaptive evolutionary optimization triggered at episode 540 due to performance plateau\n",
            "MADDPG-EVO-DGM - Episode 540, Average Reward: 247.997, Stability Metric: 0.703\n",
            "MADDPG-EVO-DGM - DGM-style adaptive evolutionary optimization triggered at episode 555 due to performance plateau\n",
            "MADDPG-EVO-DGM - Episode 560, Average Reward: 248.819, Stability Metric: 0.708\n",
            "MADDPG-EVO-DGM - DGM-style adaptive evolutionary optimization triggered at episode 570 due to performance plateau\n",
            "MADDPG-EVO-DGM - Episode 580, Average Reward: 248.354, Stability Metric: 0.707\n",
            "MADDPG-EVO-DGM - DGM-style adaptive evolutionary optimization triggered at episode 585 due to performance plateau\n",
            "MADDPG-EVO-DGM - DGM-style adaptive evolutionary optimization triggered at episode 600 due to performance plateau\n",
            "MADDPG-EVO-DGM - Episode 600, Average Reward: 249.103, Stability Metric: 0.707\n",
            "MADDPG-EVO-DGM - DGM-style adaptive evolutionary optimization triggered at episode 615 due to performance plateau\n",
            "MADDPG-EVO-DGM - Episode 620, Average Reward: 250.539, Stability Metric: 0.708\n",
            "MADDPG-EVO-DGM - DGM-style adaptive evolutionary optimization triggered at episode 630 due to performance plateau\n",
            "MADDPG-EVO-DGM - Episode 640, Average Reward: 250.326, Stability Metric: 0.708\n",
            "MADDPG-EVO-DGM - DGM-style adaptive evolutionary optimization triggered at episode 645 due to performance plateau\n",
            "MADDPG-EVO-DGM - DGM-style adaptive evolutionary optimization triggered at episode 660 due to performance plateau\n",
            "MADDPG-EVO-DGM - Episode 660, Average Reward: 250.739, Stability Metric: 0.707\n",
            "MADDPG-EVO-DGM - DGM-style adaptive evolutionary optimization triggered at episode 675 due to performance plateau\n",
            "MADDPG-EVO-DGM - Episode 680, Average Reward: 251.001, Stability Metric: 0.707\n",
            "MADDPG-EVO-DGM - DGM-style adaptive evolutionary optimization triggered at episode 690 due to performance plateau\n",
            "MADDPG-EVO-DGM - Episode 700, Average Reward: 251.144, Stability Metric: 0.710\n",
            "MADDPG-EVO-DGM - DGM-style adaptive evolutionary optimization triggered at episode 705 due to performance plateau\n",
            "MADDPG-EVO-DGM - DGM-style adaptive evolutionary optimization triggered at episode 720 due to performance plateau\n",
            "MADDPG-EVO-DGM - Episode 720, Average Reward: 251.346, Stability Metric: 0.708\n",
            "MADDPG-EVO-DGM - DGM-style adaptive evolutionary optimization triggered at episode 735 due to performance plateau\n",
            "MADDPG-EVO-DGM - Episode 740, Average Reward: 251.921, Stability Metric: 0.710\n",
            "MADDPG-EVO-DGM - DGM-style adaptive evolutionary optimization triggered at episode 750 due to performance plateau\n",
            "MADDPG-EVO-DGM - Episode 760, Average Reward: 252.238, Stability Metric: 0.712\n",
            "MADDPG-EVO-DGM - DGM-style adaptive evolutionary optimization triggered at episode 765 due to performance plateau\n",
            "MADDPG-EVO-DGM - DGM-style adaptive evolutionary optimization triggered at episode 780 due to performance plateau\n",
            "MADDPG-EVO-DGM - Episode 780, Average Reward: 252.459, Stability Metric: 0.711\n",
            "MADDPG-EVO-DGM - DGM-style adaptive evolutionary optimization triggered at episode 795 due to performance plateau\n",
            "MADDPG-EVO-DGM - Episode 800, Average Reward: 252.239, Stability Metric: 0.713\n",
            "MADDPG-EVO-DGM - DGM-style adaptive evolutionary optimization triggered at episode 810 due to performance plateau\n",
            "MADDPG-EVO-DGM - Episode 820, Average Reward: 252.700, Stability Metric: 0.711\n",
            "MADDPG-EVO-DGM - DGM-style adaptive evolutionary optimization triggered at episode 825 due to performance plateau\n",
            "MADDPG-EVO-DGM - DGM-style adaptive evolutionary optimization triggered at episode 840 due to performance plateau\n",
            "MADDPG-EVO-DGM - Episode 840, Average Reward: 252.394, Stability Metric: 0.711\n",
            "MADDPG-EVO-DGM - DGM-style adaptive evolutionary optimization triggered at episode 855 due to performance plateau\n",
            "MADDPG-EVO-DGM - Episode 860, Average Reward: 252.746, Stability Metric: 0.711\n",
            "MADDPG-EVO-DGM - DGM-style adaptive evolutionary optimization triggered at episode 870 due to performance plateau\n",
            "MADDPG-EVO-DGM - Episode 880, Average Reward: 252.389, Stability Metric: 0.711\n",
            "MADDPG-EVO-DGM - DGM-style adaptive evolutionary optimization triggered at episode 885 due to performance plateau\n",
            "MADDPG-EVO-DGM - DGM-style adaptive evolutionary optimization triggered at episode 900 due to performance plateau\n",
            "MADDPG-EVO-DGM - Episode 900, Average Reward: 252.589, Stability Metric: 0.712\n",
            "MADDPG-EVO-DGM - DGM-style adaptive evolutionary optimization triggered at episode 915 due to performance plateau\n",
            "MADDPG-EVO-DGM - Episode 920, Average Reward: 252.505, Stability Metric: 0.712\n",
            "MADDPG-EVO-DGM - DGM-style adaptive evolutionary optimization triggered at episode 930 due to performance plateau\n",
            "MADDPG-EVO-DGM - Episode 940, Average Reward: 252.437, Stability Metric: 0.712\n",
            "MADDPG-EVO-DGM - DGM-style adaptive evolutionary optimization triggered at episode 945 due to performance plateau\n",
            "MADDPG-EVO-DGM - DGM-style adaptive evolutionary optimization triggered at episode 960 due to performance plateau\n",
            "MADDPG-EVO-DGM - Episode 960, Average Reward: 252.068, Stability Metric: 0.711\n",
            "MADDPG-EVO-DGM - DGM-style adaptive evolutionary optimization triggered at episode 975 due to performance plateau\n",
            "MADDPG-EVO-DGM - Episode 980, Average Reward: 252.146, Stability Metric: 0.712\n",
            "MADDPG-EVO-DGM - DGM-style adaptive evolutionary optimization triggered at episode 990 due to performance plateau\n",
            "MADDPG-EVO-DGM - Training successfully completed!\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 1500x1000 with 6 Axes>"
            ],
            "image/png": "[encoded data, 295748 characters]\n"
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Results saved to maddpg-evo-dgm_results_20250912_095440.csv\n",
            "=== MADDPG-EVO-DGM Experiment Concluded ===\n",
            "\n",
            "=== COMPARATIVE ANALYSIS OF ALL EXPERIMENTS ===\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 1500x600 with 2 Axes>"
            ],
            "image/png": "[encoded data, 162972 characters]\n"
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "--- Comparative Table of Aggregated Metrics ---\n",
            "        Metric  MADDPG-BASELINE  MADDPG-EVO  MADDPG-EVO-DGM  Difference (EVO - BASE)  Difference (DGM - BASE)\n",
            "   Mean Reward         180.7376    206.3377        224.2435                  25.6001                  43.5060\n",
            "    Max Reward         243.7199    250.1268        253.0780                   6.4068                   9.3581\n",
            "Mean Stability           0.7669      0.7326          0.7189                  -0.0344                  -0.0481\n",
            " Max Stability           1.1322      1.1329          1.1749                   0.0007                   0.0428\n",
            "----------------------------------------------\n",
            "=== COMPARATIVE ANALYSIS OF ALL EXPERIMENTS CONCLUDED ===\n"
          ]
        }
      ]
    }
  ]
}