#!/usr/bin/env python3
"""
Generate an HTML dashboard from experiment results and visualizations.
Creates a self-contained dashboard that displays all evaluation results and probe score histograms.
Also creates a downloadable zip file for sharing.
"""

import pandas as pd
import json
from pathlib import Path
import argparse
from datetime import datetime
import glob
import shutil
import os
import zipfile
from typing import Optional

from deception_detection.experiment import ExperimentConfig


def find_experiment_folders(search_dirs: Optional[list[str]] = None) -> list[Path]:
    """
    Find all experiment folders that contain cfg.yaml files.
    
    Args:
        search_dirs: List of directories to search in. Defaults to ['results', 'example_results']
    
    Returns:
        List of paths to experiment folders
    """
    if search_dirs is None:
        search_dirs = ['results', 'example_results']
    
    experiment_folders: list[Path] = []
    
    for search_dir in search_dirs:
        if not os.path.exists(search_dir):
            continue
            
        # Look for cfg.yaml files in subdirectories
        pattern = os.path.join(search_dir, '*', 'cfg.yaml')
        cfg_files = glob.glob(pattern)
        
        for cfg_file in cfg_files:
            exp_folder = Path(cfg_file).parent
            experiment_folders.append(exp_folder)
    
    return sorted(experiment_folders, key=lambda p: p.stat().st_mtime)


def get_latest_experiment_folder(search_dirs: Optional[list[str]] = None) -> Path:
    """
    Get the most recently created experiment folder.
    
    Args:
        search_dirs: List of directories to search in
        
    Returns:
        Path to the latest experiment folder
    """
    experiment_folders = find_experiment_folders(search_dirs)
    
    if not experiment_folders:
        raise ValueError("No experiment folders found with cfg.yaml files")
    
    # Sort by modification time (most recent first)
    experiment_folders.sort(key=lambda p: p.stat().st_mtime, reverse=True)
    
    return experiment_folders[0]


def load_experiment_results(exp_path: Path) -> pd.DataFrame:
    """Load experiment results from the results table CSV."""
    results_file = exp_path / "results_table.csv"
    
    if not results_file.exists():
        raise FileNotFoundError(f"Results table not found: {results_file}")
    
    # Load CSV with first column as index (dataset names)
    df = pd.read_csv(results_file, index_col=0)
    
    # Reset index to make dataset a regular column
    df = df.reset_index()
    df = df.rename(columns={df.columns[0]: 'eval_dataset'})
    
    # Load experiment config to get additional metadata
    cfg = ExperimentConfig.from_path(exp_path)
    
    # Add experiment metadata to each row
    df['model'] = cfg.model_name.value
    df['exp_path'] = str(exp_path)
    df['exp_name'] = exp_path.name
    df['train_dataset'] = cfg.train_data
    df['method'] = cfg.method
    
    # Keep old 'dataset' column for backward compatibility with visualizations
    df['dataset'] = df['eval_dataset']
    
    return df


def find_visualization_files(exp_path: Path) -> dict:
    """Find all visualization files in the experiment plots folder."""
    plots_dir = exp_path / "plots"
    viz_mapping = {}
    
    if not plots_dir.exists():
        print(f"Warning: No plots directory found at {plots_dir}")
        return viz_mapping
    
    # Look for histogram plots in mean/ and max/ subdirectories
    for token_type in ['mean', 'max']:
        token_dir = plots_dir / token_type
        if token_dir.exists():
            for img_file in token_dir.glob("probe_scores_*.png"):
                # Extract dataset name from filename
                # Format: probe_scores_{model}_layer_{layer}_{token}_{dataset}.png
                filename = img_file.name
                parts = filename.replace('.png', '').split('_')
                
                # Find dataset name (everything after the last occurrence of token_type)
                try:
                    token_idx = -1
                    for i, part in enumerate(parts):
                        if part == token_type:
                            token_idx = i
                    
                    if token_idx >= 0 and token_idx < len(parts) - 1:
                        dataset = '_'.join(parts[token_idx + 1:])
                        key = f"{dataset}|{token_type}"
                        viz_mapping[key] = str(img_file.relative_to(exp_path))
                except Exception as e:
                    print(f"Warning: Could not parse filename {filename}: {e}")
    
    # Also look for other plots in the main plots directory
    for img_file in plots_dir.glob("*.png"):
        key = f"overview|{img_file.stem}"
        viz_mapping[key] = str(img_file.relative_to(exp_path))
    
    return viz_mapping


def generate_html_dashboard(df: pd.DataFrame, exp_path: Path, viz_mapping: dict) -> str:
    """Generate HTML dashboard content."""
    
    # Get experiment metadata
    cfg = ExperimentConfig.from_path(exp_path)
    model_name = cfg.model_name.value
    layers = cfg.detect_layers
    exp_name = exp_path.name
    
    # Get unique values for filters
    datasets = sorted(df['dataset'].unique()) if 'dataset' in df.columns else []
    methods = ['mean', 'max']  # Token reduction methods
    
    # Get all columns except metadata columns
    display_columns = [col for col in df.columns if col not in ['model', 'exp_path', 'exp_name']]
    
    # Calculate summary stats
    best_auroc = df.loc[df['auroc'].idxmax()] if 'auroc' in df.columns else None
    best_recall_1 = df.loc[df['recall 1.0%'].idxmax()] if 'recall 1.0%' in df.columns else None
    
    html_content = f"""
<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>Deception Detection Experiment Dashboard</title>
    <style>
        body {{
            font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
            margin: 0;
            padding: 20px;
            background-color: #f5f5f5;
        }}
        
        .header {{
            background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
            color: white;
            padding: 30px;
            border-radius: 10px;
            margin-bottom: 30px;
            text-align: center;
        }}
        
        .header h1 {{
            margin: 0;
            font-size: 2.5em;
            font-weight: 300;
        }}
        
        .header p {{
            margin: 10px 0 0 0;
            font-size: 1.1em;
            opacity: 0.9;
        }}
        
        .summary-stats {{
            display: grid;
            grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
            gap: 20px;
            margin-bottom: 30px;
        }}
        
        .stat-card {{
            background: white;
            padding: 20px;
            border-radius: 8px;
            box-shadow: 0 2px 10px rgba(0,0,0,0.1);
            text-align: center;
        }}
        
        .stat-number {{
            font-size: 2em;
            font-weight: bold;
            color: #667eea;
        }}
        
        .stat-label {{
            color: #666;
            margin-top: 5px;
        }}
        
        .filters {{
            background: white;
            padding: 20px;
            border-radius: 8px;
            box-shadow: 0 2px 10px rgba(0,0,0,0.1);
            margin-bottom: 30px;
        }}
        
        .filter-group {{
            display: inline-block;
            margin-right: 20px;
            margin-bottom: 10px;
        }}
        
        .filter-group label {{
            display: block;
            margin-bottom: 5px;
            font-weight: bold;
            color: #333;
        }}
        
        .filter-group select {{
            padding: 8px 12px;
            border: 1px solid #ddd;
            border-radius: 4px;
            background: white;
        }}
        
        .download-btn {{
            background: #28a745;
            color: white;
            border: none;
            padding: 10px 20px;
            border-radius: 5px;
            cursor: pointer;
            font-size: 1em;
            margin-left: 20px;
        }}
        
        .download-btn:hover {{
            background: #218838;
        }}
        
        table {{
            width: 100%;
            border-collapse: collapse;
            background: white;
            border-radius: 8px;
            overflow: hidden;
            box-shadow: 0 2px 10px rgba(0,0,0,0.1);
            margin-bottom: 30px;
        }}
        
        th, td {{
            padding: 12px 8px;
            text-align: left;
            border-bottom: 1px solid #eee;
            font-size: 0.9em;
        }}
        
        th {{
            background: #f8f9fa;
            font-weight: 600;
            color: #333;
            cursor: pointer;
            user-select: none;
            position: sticky;
            top: 0;
            z-index: 10;
        }}
        
        th:hover {{
            background: #e9ecef;
        }}
        
        tr:hover {{
            background-color: #f8f9fa;
        }}
        
        .metric-cell {{
            font-weight: bold;
        }}
        
        .auroc-high {{ color: #28a745; }}
        .auroc-medium {{ color: #ffc107; }}
        .auroc-low {{ color: #dc3545; }}
        
        .section {{
            background: white;
            padding: 25px;
            border-radius: 8px;
            box-shadow: 0 2px 10px rgba(0,0,0,0.1);
            margin-bottom: 30px;
        }}
        
        .section h2 {{
            margin-top: 0;
            color: #333;
            border-bottom: 2px solid #667eea;
            padding-bottom: 10px;
        }}
        
        /* Modal styles for visualizations */
        .modal {{
            display: none;
            position: fixed;
            z-index: 1000;
            left: 0;
            top: 0;
            width: 100%;
            height: 100%;
            background-color: rgba(0,0,0,0.8);
        }}
        
        .modal-content {{
            background-color: white;
            margin: 2% auto;
            padding: 20px;
            border-radius: 10px;
            width: 95%;
            max-width: 1200px;
            max-height: 90vh;
            overflow-y: auto;
            position: relative;
        }}
        
        .modal-header {{
            display: flex;
            justify-content: space-between;
            align-items: center;
            margin-bottom: 20px;
            padding-bottom: 10px;
            border-bottom: 2px solid #667eea;
        }}
        
        .modal-title {{
            font-size: 1.5em;
            font-weight: bold;
            color: #333;
        }}
        
        .close {{
            color: #aaa;
            font-size: 28px;
            font-weight: bold;
            cursor: pointer;
            line-height: 1;
        }}
        
        .close:hover,
        .close:focus {{
            color: #667eea;
        }}
        
        .visualization-container {{
            margin: 20px 0;
        }}
        
        .visualization-container img {{
            max-width: 100%;
            height: auto;
            border-radius: 8px;
            box-shadow: 0 4px 8px rgba(0,0,0,0.1);
            margin: 10px;
        }}
        
        .viz-error {{
            color: #dc3545;
            background: #f8f9fa;
            padding: 20px;
            border-radius: 8px;
            border-left: 4px solid #dc3545;
        }}
        
        .row-clickable {{
            cursor: pointer;
        }}
        
        .row-clickable:hover {{
            background-color: #e3f2fd !important;
        }}
        
        .experiment-info {{
            background: #e8f5e8;
            color: #2e7d32;
            padding: 15px;
            border-radius: 8px;
            margin-bottom: 20px;
            border-left: 4px solid #4caf50;
        }}
        
        .viz-grid {{
            display: grid;
            grid-template-columns: repeat(auto-fit, minmax(400px, 1fr));
            gap: 20px;
            margin: 20px 0;
        }}
        
        .viz-item {{
            text-align: center;
        }}
        
        .viz-item h4 {{
            margin: 10px 0;
            color: #333;
        }}
        
        .overview-plots {{
            margin-top: 30px;
        }}
        
        .overview-plots h3 {{
            color: #333;
            margin-bottom: 15px;
        }}
        
        .table-container {{
            overflow-x: auto;
            max-width: 100%;
        }}
        
        .dataset-name {{
            font-weight: bold;
            color: #667eea;
        }}
    </style>
</head>
<body>
    <div class="header">
        <h1>🔍 Deception Detection Experiment Dashboard</h1>
        <p>Experiment: {exp_name}</p>
        <p>Generated on {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}</p>
    </div>
    
    <div class="experiment-info">
        📊 <strong>Experiment Details:</strong><br>
        <strong>Model:</strong> {model_name}<br>
        <strong>Layers:</strong> {', '.join(map(str, layers))}<br>
        <strong>Path:</strong> {exp_path}<br>
        <strong>Visualizations:</strong> {len(viz_mapping)} files available
    </div>
    
    <div class="summary-stats">
        <div class="stat-card">
            <div class="stat-number">{len(datasets)}</div>
            <div class="stat-label">Evaluation Datasets</div>
        </div>
        <div class="stat-card">
            <div class="stat-number">{len(df)}</div>
            <div class="stat-label">Total Results</div>
        </div>
        <div class="stat-card">
            <div class="stat-number">{len(viz_mapping)}</div>
            <div class="stat-label">Visualizations</div>
        </div>
        <div class="stat-card">
            <div class="stat-number">{len(methods)}</div>
            <div class="stat-label">Token Methods</div>
        </div>
    </div>
    
    <div class="filters">
        <h3>🔧 Filters & Downloads</h3>
        <div class="filter-group">
            <label for="datasetFilter">Dataset:</label>
            <select id="datasetFilter">
                <option value="">All Datasets</option>
                {''.join([f'<option value="{dataset}">{dataset}</option>' for dataset in datasets])}
            </select>
        </div>
        
        <button class="download-btn" onclick="downloadDashboard()">📦 Download Dashboard ZIP</button>
        <button class="download-btn" onclick="downloadCSV()">📊 Download Results CSV</button>
    </div>
    
    <div class="section">
        <h2>📊 Evaluation Results</h2>
        <p><strong>💡 Tip:</strong> Click on any row to view all visualizations for that dataset (probe score histograms and overview plots).</p>
        <div class="table-container">
            <table id="resultsTable">
                <thead>
                    <tr>
"""

    # Add table headers for all columns
    for col in display_columns:
        display_name = col.replace('_', ' ').replace('%', ' %').title()
        html_content += f'                        <th>{display_name}</th>\n'
    
    html_content += """
                    </tr>
                </thead>
                <tbody>
"""

    # Add table rows with all data
    for _, row in df.iterrows():
        dataset_name = row.get('dataset', 'unknown')
        html_content += f'                    <tr class="row-clickable" data-dataset="{dataset_name}" onclick="showVisualization(this)">\n'
        
        for col in display_columns:
            value = row.get(col, 'N/A')
            cell_class = "metric-cell"
            
            # Special formatting for different column types
            if col == 'dataset':
                cell_class += " dataset-name"
                html_content += f'                        <td class="{cell_class}">{value}</td>\n'
            elif col == 'auroc':
                auroc_class = "auroc-high" if value >= 0.8 else "auroc-medium" if value >= 0.6 else "auroc-low"
                cell_class += f" {auroc_class}"
                html_content += f'                        <td class="{cell_class}">{value:.4f}</td>\n'
            elif isinstance(value, float):
                html_content += f'                        <td class="{cell_class}">{value:.4f}</td>\n'
            elif isinstance(value, int):
                html_content += f'                        <td class="{cell_class}">{value:,}</td>\n'
            else:
                html_content += f'                        <td class="{cell_class}">{value}</td>\n'
        
        html_content += '                    </tr>\n'

    # Best results section
    best_results_html = ""
    if best_auroc is not None and 'auroc' in best_auroc:
        best_results_html += f"""
            <div class="stat-card">
                <h4>🎯 Best AUROC</h4>
                <div class="stat-number">{best_auroc['auroc']:.4f}</div>
                <div class="stat-label">{best_auroc.get('dataset', 'Unknown Dataset')}</div>
            </div>
        """
    
    if best_recall_1 is not None and 'recall 1.0%' in best_recall_1:
        best_results_html += f"""
            <div class="stat-card">
                <h4>📈 Best Recall @ 1% FPR</h4>
                <div class="stat-number">{best_recall_1['recall 1.0%']:.4f}</div>
                <div class="stat-label">{best_recall_1.get('dataset', 'Unknown Dataset')}</div>
            </div>
        """

    # Get overview plots
    overview_plots = [key for key in viz_mapping.keys() if key.startswith('overview|')]

    html_content += f"""
                </tbody>
            </table>
        </div>
    </div>
    
    <div class="section">
        <h2>🏆 Best Results</h2>
        <div style="display: grid; grid-template-columns: repeat(auto-fit, minmax(300px, 1fr)); gap: 20px;">
            {best_results_html}
        </div>
    </div>

    <!-- Modal for visualizations -->
    <div id="vizModal" class="modal">
        <div class="modal-content">
            <div class="modal-header">
                <div class="modal-title" id="modalTitle">Visualizations</div>
                <span class="close" onclick="closeModal()">&times;</span>
            </div>
            <div id="modalBody">
                <!-- Visualizations will be loaded here -->
            </div>
        </div>
    </div>

    <script>
        // Store data and visualization mapping
        const originalData = {json.dumps(df.to_dict('records'))};
        const vizMapping = {json.dumps(viz_mapping)};
        const overviewPlots = {json.dumps(overview_plots)};
        
        function filterTable() {{
            const datasetFilter = document.getElementById('datasetFilter').value;
            
            const rows = document.querySelectorAll('#resultsTable tbody tr');
            
            rows.forEach(row => {{
                const dataset = row.getAttribute('data-dataset');
                
                const showRow = (datasetFilter === '' || dataset === datasetFilter);
                
                row.style.display = showRow ? '' : 'none';
            }});
        }}
        
        function showVisualization(row) {{
            const dataset = row.getAttribute('data-dataset');
            
            // Update modal title
            document.getElementById('modalTitle').textContent = 
                `Visualizations: ${{dataset}}`;
            
            // Build visualization content
            const methods = ['mean', 'max'];
            let modalBody = '<div class="visualization-container">';
            
            modalBody += `
                <p><strong>Dataset:</strong> ${{dataset}}</p>
                <p>These visualizations show the distribution of probe scores for honest (blue) vs. deceptive (red) statements using different token reduction methods.</p>
                
                <h3>📊 Probe Score Histograms</h3>
                <div class="viz-grid">
            `;
            
            // Add probe score histograms
            methods.forEach(method => {{
                const key = `${{dataset}}|${{method}}`;
                const imgPath = vizMapping[key];
                
                modalBody += '<div class="viz-item">';
                if (imgPath) {{
                    modalBody += `
                        <h4>Token Method: ${{method}}</h4>
                        <img src="${{imgPath}}" alt="Probe scores for ${{method}} method" 
                             onclick="window.open(this.src, '_blank')" style="cursor: pointer;">
                    `;
                }} else {{
                    modalBody += `
                        <h4>Token Method: ${{method}}</h4>
                        <div class="viz-error">
                            ⚠️ Visualization not found for ${{method}} method
                        </div>
                    `;
                }}
                modalBody += '</div>';
            }});
            
            modalBody += '</div>';
            
            // Add overview plots if available
            if (overviewPlots.length > 0) {{
                modalBody += `
                    <div class="overview-plots">
                        <h3>📈 Overview Plots</h3>
                        <div class="viz-grid">
                `;
                
                overviewPlots.forEach(plotKey => {{
                    const imgPath = vizMapping[plotKey];
                    const plotName = plotKey.split('|')[1];
                    
                    modalBody += `
                        <div class="viz-item">
                            <h4>${{plotName.replace('_', ' ').replace(/\\b\\w/g, l => l.toUpperCase())}}</h4>
                            <img src="${{imgPath}}" alt="${{plotName}}" 
                                 onclick="window.open(this.src, '_blank')" style="cursor: pointer;">
                        </div>
                    `;
                }});
                
                modalBody += '</div></div>';
            }}
            
            modalBody += '</div>';
            
            document.getElementById('modalBody').innerHTML = modalBody;
            document.getElementById('vizModal').style.display = 'block';
        }}
        
        function closeModal() {{
            document.getElementById('vizModal').style.display = 'none';
        }}
        
        // Close modal when clicking outside of it
        window.onclick = function(event) {{
            const modal = document.getElementById('vizModal');
            if (event.target === modal) {{
                modal.style.display = 'none';
            }}
        }}
        
        // Attach event listeners
        document.getElementById('datasetFilter').addEventListener('change', filterTable);
        
        function downloadCSV() {{
            const csv = convertToCSV(originalData);
            const blob = new Blob([csv], {{ type: 'text/csv' }});
            const url = window.URL.createObjectURL(blob);
            const a = document.createElement('a');
            a.setAttribute('hidden', '');
            a.setAttribute('href', url);
            a.setAttribute('download', 'experiment_results.csv');
            document.body.appendChild(a);
            a.click();
            document.body.removeChild(a);
        }}
        
        function convertToCSV(data) {{
            if (data.length === 0) return '';
            const headers = Object.keys(data[0]);
            const csvContent = [
                headers.join(','),
                ...data.map(row => headers.map(header => {{
                    const value = row[header];
                    return typeof value === 'string' && value.includes(',') ? `"${{value}}"` : value;
                }}).join(','))
            ].join('\\n');
            return csvContent;
        }}
        
        function downloadDashboard() {{
            // This would trigger a server-side zip creation in a real implementation
            alert('Dashboard ZIP download functionality requires server-side implementation. The dashboard is already saved in the experiment folder.');
        }}
    </script>
</body>
</html>
"""

    return html_content


def create_dashboard_zip(exp_path: Path, output_filename: str = "dashboard_portable.zip"):
    """Create a portable zip file with dashboard and all visualizations."""
    
    # Create temporary directory for building the package
    temp_dir = Path("temp_dashboard_build")
    if temp_dir.exists():
        shutil.rmtree(temp_dir)
    temp_dir.mkdir()
    
    try:
        print(f"Creating portable dashboard zip...")
        
        # Load experiment results
        df = load_experiment_results(exp_path)
        viz_mapping = find_visualization_files(exp_path)
        
        # Generate HTML dashboard
        html_content = generate_html_dashboard(df, exp_path, viz_mapping)
        
        # Save dashboard HTML
        dashboard_path = temp_dir / "dashboard.html"
        with open(dashboard_path, 'w', encoding='utf-8') as f:
            f.write(html_content)
        
        # Copy all visualization files
        plots_dir = temp_dir / "plots"
        plots_dir.mkdir(exist_ok=True)
        
        source_plots_dir = exp_path / "plots"
        if source_plots_dir.exists():
            # Copy entire plots directory structure
            shutil.copytree(source_plots_dir, plots_dir, dirs_exist_ok=True)
        
        # Copy results CSV
        results_csv = exp_path / "results_table.csv"
        if results_csv.exists():
            shutil.copy2(results_csv, temp_dir / "results_table.csv")
        
        # Create README
        readme_content = f"""# Deception Detection Experiment Dashboard

This package contains a self-contained dashboard for viewing experiment results.

## Experiment Details:
- **Name:** {exp_path.name}
- **Model:** {df.iloc[0]['model'] if len(df) > 0 else 'Unknown'}
- **Generated:** {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}

## Contents:
- `dashboard.html` - Interactive dashboard (open this in your browser)
- `plots/` - All visualization images
- `results_table.csv` - Raw results data
- `README.md` - This file

## Usage:
1. Extract this zip file
2. Open `dashboard.html` in any web browser
3. Click on table rows to view visualizations
4. Use filters to explore different datasets

## Statistics:
- {len(df)} evaluation results
- {len(viz_mapping)} visualization files
- {len(df['dataset'].unique()) if len(df) > 0 else 0} datasets evaluated

## Features:
- Interactive filtering and sorting
- Click-to-view visualizations
- Downloadable CSV data
- Responsive design for mobile/desktop
"""
        
        with open(temp_dir / "README.md", 'w') as f:
            f.write(readme_content)
        
        # Create the zip file
        zip_path = exp_path / output_filename
        print(f"Creating zip file: {zip_path}")
        
        with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf:
            for file_path in temp_dir.rglob('*'):
                if file_path.is_file():
                    arcname = file_path.relative_to(temp_dir)
                    zipf.write(file_path, arcname)
        
        # Get zip file size
        zip_size_mb = zip_path.stat().st_size / (1024 * 1024)
        
        print(f"\n✅ Portable dashboard ZIP created successfully!")
        print(f"📁 File: {zip_path}")
        print(f"💾 Size: {zip_size_mb:.1f} MB")
        print(f"📊 Contents: {len(df)} results, {len(viz_mapping)} visualizations")
        print(f"📖 Instructions: Extract the zip and open dashboard.html in a browser")
        
        return zip_path
    
    finally:
        # Clean up temporary directory
        if temp_dir.exists():
            shutil.rmtree(temp_dir)


def generate_dashboard(exp_path: Optional[str] = None, output_filename: str = "dashboard.html", create_zip: bool = True):
    """
    Generate HTML dashboard for an experiment.
    
    Args:
        exp_path: Path to the experiment folder (if None, uses latest experiment)
        output_filename: Name of the output HTML file
        create_zip: Whether to create a portable zip file
    """
    # Determine experiment path
    if exp_path is None:
        try:
            exp_path = get_latest_experiment_folder()
            print(f"Auto-detected latest experiment: {exp_path}")
        except ValueError as e:
            print(f"Error: {e}")
            print("Use --list_experiments to see available experiments")
            return
    else:
        exp_path = Path(exp_path)
        if not exp_path.exists():
            raise ValueError(f"Experiment path does not exist: {exp_path}")
        
        config_file = exp_path / "cfg.yaml"
        if not config_file.exists():
            raise ValueError(f"Config file not found: {config_file}")
    
    print(f"Processing experiment: {exp_path}")
    
    # Load experiment results
    try:
        df = load_experiment_results(exp_path)
        print(f"Loaded {len(df)} result rows")
    except FileNotFoundError as e:
        print(f"Error: {e}")
        return
    
    # Find visualization files
    viz_mapping = find_visualization_files(exp_path)
    print(f"Found {len(viz_mapping)} visualization files")
    
    # Generate HTML dashboard
    html_content = generate_html_dashboard(df, exp_path, viz_mapping)
    
    # Save dashboard in experiment folder
    output_path = exp_path / output_filename
    with open(output_path, 'w', encoding='utf-8') as f:
        f.write(html_content)
    
    print(f"✅ HTML dashboard generated: {output_path}")
    print(f"📊 Dashboard includes {len(df)} results and {len(viz_mapping)} visualizations")
    
    # Create portable zip if requested
    if create_zip:
        zip_path = create_dashboard_zip(exp_path)
        print(f"📦 Portable ZIP created: {zip_path}")
    
    print(f"🌐 Open {output_path} in your browser to view the dashboard")


def load_all_experiment_results(search_dirs: Optional[list[str]] = None) -> pd.DataFrame:
    """Load and combine experiment results from all experiment folders."""
    experiment_folders = find_experiment_folders(search_dirs)
    
    if not experiment_folders:
        raise ValueError("No experiment folders found with cfg.yaml files")
    
    print(f"Found {len(experiment_folders)} experiment folders")
    
    all_dfs = []
    
    for exp_path in experiment_folders:
        try:
            print(f"Loading results from: {exp_path}")
            df = load_experiment_results(exp_path)
            all_dfs.append(df)
        except FileNotFoundError as e:
            print(f"Warning: Skipping {exp_path} - {e}")
            continue
        except Exception as e:
            print(f"Error loading {exp_path}: {e}")
            continue
    
    if not all_dfs:
        raise ValueError("No valid experiment results found")
    
    # Combine all dataframes
    combined_df = pd.concat(all_dfs, ignore_index=True)
    
    print(f"Combined {len(all_dfs)} experiments with {len(combined_df)} total result rows")
    
    return combined_df


def find_all_visualization_files(search_dirs: Optional[list[str]] = None) -> dict:
    """Find all visualization files from all experiment folders."""
    experiment_folders = find_experiment_folders(search_dirs)
    
    all_viz_mapping = {}
    
    for exp_path in experiment_folders:
        exp_viz = find_visualization_files(exp_path)
        
        # Prefix each visualization key with experiment name to avoid conflicts
        for key, path in exp_viz.items():
            # Convert relative path to include experiment folder
            full_path = exp_path / path
            if full_path.exists():
                new_key = f"{exp_path.name}|{key}"
                # Store relative path from the search directory
                all_viz_mapping[new_key] = str(full_path)
    
    return all_viz_mapping


def generate_multi_experiment_dashboard(search_dirs: Optional[list[str]] = None, output_filename: str = "multi_experiment_dashboard.html", output_dir: Optional[str] = None, create_zip: bool = True):
    """
    Generate HTML dashboard for ALL experiments in the results folder.
    
    Args:
        search_dirs: List of directories to search for experiments (defaults to ['results', 'example_results'])
        output_filename: Name of the output HTML file
        output_dir: Directory to save the dashboard (if None, saves in results directory with timestamp)
        create_zip: Whether to create a portable zip file with all resources
    """
    if search_dirs is None:
        search_dirs = ['results', 'example_results']
    
    print(f"Searching for experiments in: {search_dirs}")
    
    # Load combined results from all experiments
    try:
        combined_df = load_all_experiment_results(search_dirs)
        print(f"Loaded {len(combined_df)} total result rows from all experiments")
    except ValueError as e:
        print(f"Error: {e}")
        return
    
    # Find all visualization files
    all_viz_mapping = find_all_visualization_files(search_dirs)
    print(f"Found {len(all_viz_mapping)} total visualization files")
    
    # Generate timestamp for multi-experiment outputs
    timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
    
    # Set output directory - for multi-experiment, default to results folder with timestamp
    if output_dir is None:
        # Use the first search directory (usually 'results') as the output location
        results_dir = Path(search_dirs[0]) if search_dirs else Path('results')
        results_dir.mkdir(exist_ok=True)
        
        # Add timestamp to filename
        base_name = Path(output_filename).stem
        extension = Path(output_filename).suffix
        timestamped_filename = f"{base_name}_{timestamp}{extension}"
        output_path = results_dir / timestamped_filename
    else:
        output_path = Path(output_dir) / output_filename
    
    if create_zip:
        # Create portable zip version with all resources
        # Add timestamp to zip filename too
        zip_base_name = output_path.stem
        zip_path = output_path.parent / f"{zip_base_name}_{timestamp}.zip"
        create_multi_experiment_zip_dashboard(combined_df, all_viz_mapping, zip_path.with_suffix(''))
    else:
        # Generate HTML dashboard with combined data
        html_content = generate_multi_experiment_html_dashboard(combined_df, all_viz_mapping)
        
        # Save dashboard
        output_path.parent.mkdir(parents=True, exist_ok=True)
        with open(output_path, 'w', encoding='utf-8') as f:
            f.write(html_content)
        
        print(f"✅ Multi-experiment HTML dashboard generated: {output_path}")
    
    print(f"📊 Dashboard includes:")
    print(f"   - {len(combined_df)} total results")
    print(f"   - {len(combined_df['exp_name'].unique())} experiments")
    print(f"   - {len(combined_df['dataset'].unique())} unique datasets")
    print(f"   - {len(all_viz_mapping)} visualizations")
    
    # Also create a downloadable summary CSV with timestamp
    csv_base_name = output_path.stem.replace('_dashboard', '_results')  # Remove '_dashboard' if present
    csv_filename = f"{csv_base_name}_{timestamp}.csv"
    csv_path = output_path.parent / csv_filename
    combined_df.to_csv(csv_path, index=False)
    print(f"📁 Combined results CSV saved: {csv_path}")


def generate_multi_experiment_html_dashboard(df: pd.DataFrame, viz_mapping: dict) -> str:
    """Generate HTML dashboard content for multiple experiments."""
    
    # Get experiment statistics
    experiments = df['exp_name'].unique()
    datasets = df['dataset'].unique()
    models = df['model'].unique()
    
    # Find best results across all experiments
    best_auroc = None
    best_recall_1 = None
    best_balanced_accuracy = None
    
    if 'auroc' in df.columns:
        best_auroc_idx = df['auroc'].idxmax()
        best_auroc = df.loc[best_auroc_idx]
    
    if 'recall 1.0%' in df.columns:
        best_recall_idx = df['recall 1.0%'].idxmax()
        best_recall_1 = df.loc[best_recall_idx]
    
    if 'balanced_accuracy' in df.columns:
        best_balanced_accuracy_idx = df['balanced_accuracy'].idxmax()
        best_balanced_accuracy = df.loc[best_balanced_accuracy_idx]
    
    # Define columns to display in table with optimal ordering
    important_columns = ['exp_name', 'method', 'model', 'train_dataset', 'eval_dataset']
    metric_columns = [col for col in df.columns if col not in important_columns + ['exp_path', 'dataset']]
    display_columns = important_columns + metric_columns
    
    html_content = f"""<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>Multi-Experiment Deception Detection Dashboard</title>
    <style>
        body {{
            font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, sans-serif;
            margin: 0;
            padding: 20px;
            background-color: #f5f5f5;
            line-height: 1.6;
        }}
        .header {{
            background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
            color: white;
            padding: 30px;
            border-radius: 10px;
            text-align: center;
            margin-bottom: 30px;
            box-shadow: 0 4px 6px rgba(0,0,0,0.1);
        }}
        .header h1 {{
            margin: 0;
            font-size: 2.5em;
            font-weight: 600;
        }}
        .header p {{
            margin: 10px 0 0 0;
            font-size: 1.1em;
            opacity: 0.9;
        }}
        .stats-grid {{
            display: grid;
            grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
            gap: 20px;
            margin-bottom: 30px;
        }}
        .stat-card {{
            background: white;
            padding: 25px;
            border-radius: 8px;
            box-shadow: 0 2px 4px rgba(0,0,0,0.1);
            text-align: center;
            border-top: 4px solid #667eea;
        }}
        .stat-number {{
            font-size: 2.5em;
            font-weight: bold;
            color: #333;
            margin-bottom: 5px;
        }}
        .stat-label {{
            color: #666;
            font-size: 1.1em;
            text-transform: uppercase;
            letter-spacing: 0.5px;
        }}
        .filters {{
            background: white;
            padding: 20px;
            border-radius: 8px;
            margin-bottom: 20px;
            box-shadow: 0 2px 4px rgba(0,0,0,0.1);
        }}
        .filter-group {{
            display: inline-block;
            margin-right: 20px;
            margin-bottom: 10px;
        }}
        .filter-group label {{
            display: block;
            margin-bottom: 5px;
            font-weight: 600;
            color: #333;
        }}
        .filter-group select {{
            padding: 8px 12px;
            border: 1px solid #ddd;
            border-radius: 4px;
            font-size: 14px;
            min-width: 150px;
        }}
        .download-btn {{
            background: #667eea;
            color: white;
            border: none;
            padding: 10px 20px;
            border-radius: 5px;
            cursor: pointer;
            font-size: 14px;
            margin: 5px;
            transition: background-color 0.3s;
        }}
        .download-btn:hover {{
            background: #5a67d8;
        }}
        .section {{
            background: white;
            margin-bottom: 20px;
            border-radius: 8px;
            overflow: hidden;
            box-shadow: 0 2px 4px rgba(0,0,0,0.1);
        }}
        .section h2 {{
            margin: 0;
            padding: 20px;
            background: #f8f9fa;
            border-bottom: 1px solid #e9ecef;
            color: #333;
        }}
        .table-container {{
            overflow-x: auto;
            max-height: 600px;
            overflow-y: auto;
        }}
        table {{
            width: 100%;
            border-collapse: collapse;
            font-size: 14px;
        }}
        th, td {{
            padding: 12px 8px;
            text-align: left;
            border-bottom: 1px solid #e9ecef;
        }}
        th {{
            background: #f8f9fa;
            font-weight: 600;
            position: sticky;
            top: 0;
            z-index: 10;
        }}
        .row-clickable {{
            cursor: pointer;
            transition: background-color 0.2s;
        }}
        .row-clickable:hover {{
            background-color: #f8f9fa;
        }}
        .metric-cell {{
            font-family: 'SF Mono', Monaco, 'Cascadia Code', 'Roboto Mono', monospace;
        }}
        .dataset-name {{
            font-weight: 600;
            color: #333;
        }}
        .exp-name {{
            font-weight: 500;
            color: #667eea;
        }}
        .auroc-high {{ background-color: #d4edda; color: #155724; }}
        .auroc-medium {{ background-color: #fff3cd; color: #856404; }}
        .auroc-low {{ background-color: #f8d7da; color: #721c24; }}
        .modal {{
            display: none;
            position: fixed;
            z-index: 1000;
            left: 0;
            top: 0;
            width: 100%;
            height: 100%;
            background-color: rgba(0,0,0,0.5);
        }}
        .modal-content {{
            background-color: #fefefe;
            margin: 2% auto;
            padding: 0;
            border-radius: 8px;
            width: 95%;
            max-width: 1200px;
            max-height: 90vh;
            overflow-y: auto;
        }}
        .modal-header {{
            padding: 20px;
            background: #f8f9fa;
            border-bottom: 1px solid #e9ecef;
            display: flex;
            justify-content: space-between;
            align-items: center;
        }}
        .modal-title {{
            font-size: 1.5em;
            font-weight: 600;
            color: #333;
        }}
        .close {{
            color: #aaa;
            float: right;
            font-size: 28px;
            font-weight: bold;
            cursor: pointer;
        }}
        .close:hover {{
            color: #000;
        }}
        .visualization-container {{
            padding: 20px;
        }}
        .viz-grid {{
            display: grid;
            grid-template-columns: repeat(auto-fit, minmax(400px, 1fr));
            gap: 20px;
            margin-top: 20px;
        }}
        .viz-item {{
            text-align: center;
        }}
        .viz-item h4 {{
            margin-top: 0;
            color: #333;
        }}
        .viz-item img {{
            max-width: 100%;
            height: auto;
            border: 1px solid #ddd;
            border-radius: 4px;
        }}
        .viz-error {{
            padding: 20px;
            background: #f8d7da;
            color: #721c24;
            border-radius: 4px;
            font-style: italic;
        }}
    </style>
</head>
<body>
    <div class="header">
        <h1>🔍 Multi-Experiment Deception Detection Dashboard</h1>
        <p>Comprehensive analysis across all experiment results • Generated on {datetime.now().strftime('%B %d, %Y at %I:%M %p')}</p>
    </div>
    
    <div class="stats-grid">
        <div class="stat-card">
            <div class="stat-number">{len(experiments)}</div>
            <div class="stat-label">Experiments</div>
        </div>
        <div class="stat-card">
            <div class="stat-number">{len(datasets)}</div>
            <div class="stat-label">Unique Datasets</div>
        </div>
        <div class="stat-card">
            <div class="stat-number">{len(df)}</div>
            <div class="stat-label">Total Results</div>
        </div>
        <div class="stat-card">
            <div class="stat-number">{len(models)}</div>
            <div class="stat-label">Models</div>
        </div>
    </div>
    
    <div class="filters">
        <h3>🔧 Filters & Downloads</h3>
        <div class="filter-group">
            <label for="experimentFilter">Experiment:</label>
            <select id="experimentFilter" onchange="filterTable()">
                <option value="">All Experiments</option>
                {''.join([f'<option value="{exp}">{exp}</option>' for exp in sorted(experiments)])}
            </select>
        </div>
        <div class="filter-group">
            <label for="datasetFilter">Dataset:</label>
            <select id="datasetFilter" onchange="filterTable()">
                <option value="">All Datasets</option>
                {''.join([f'<option value="{dataset}">{dataset}</option>' for dataset in sorted(datasets)])}
            </select>
        </div>
        <div class="filter-group">
            <label for="modelFilter">Model:</label>
            <select id="modelFilter" onchange="filterTable()">
                <option value="">All Models</option>
                {''.join([f'<option value="{model}">{model}</option>' for model in sorted(models)])}
            </select>
        </div>
        
        <button class="download-btn" onclick="downloadCSV()">📊 Download Combined Results CSV</button>
    </div>
    
    <div class="section">
        <h2>📊 All Experiment Results</h2>
        <p><strong>💡 Tip:</strong> Click on any row to view visualizations for that dataset/experiment combination.</p>
        <div class="table-container">
            <table id="resultsTable">
                <thead>
                    <tr>
"""

    # Add table headers for all columns
    for col in display_columns:
        display_name = col.replace('_', ' ').replace('%', ' %').title()
        html_content += f'                        <th>{display_name}</th>\n'
    
    html_content += """
                    </tr>
                </thead>
                <tbody>
"""

    # Add table rows with all data
    for _, row in df.iterrows():
        dataset_name = row.get('dataset', 'unknown')
        exp_name = row.get('exp_name', 'unknown')
        html_content += f'                    <tr class="row-clickable" data-dataset="{dataset_name}" data-experiment="{exp_name}" onclick="showVisualization(this)">\n'
        
        for col in display_columns:
            value = row.get(col, 'N/A')
            cell_class = "metric-cell"
            
            # Special formatting for different column types
            if col == 'dataset':
                cell_class += " dataset-name"
                html_content += f'                        <td class="{cell_class}">{value}</td>\n'
            elif col == 'exp_name':
                cell_class += " exp-name"
                html_content += f'                        <td class="{cell_class}">{value}</td>\n'
            elif col == 'auroc':
                auroc_class = "auroc-high" if value >= 0.8 else "auroc-medium" if value >= 0.6 else "auroc-low"
                cell_class += f" {auroc_class}"
                html_content += f'                        <td class="{cell_class}">{value:.4f}</td>\n'
            elif isinstance(value, float):
                html_content += f'                        <td class="{cell_class}">{value:.4f}</td>\n'
            elif isinstance(value, int):
                html_content += f'                        <td class="{cell_class}">{value:,}</td>\n'
            else:
                html_content += f'                        <td class="{cell_class}">{value}</td>\n'
        
        html_content += '                    </tr>\n'

    # Best results section
    best_results_html = ""
    if best_auroc is not None and 'auroc' in best_auroc:
        best_results_html += f"""
            <div class="stat-card">
                <h4>🎯 Best AUROC</h4>
                <div class="stat-number">{best_auroc['auroc']:.4f}</div>
                <div class="stat-label">{best_auroc.get('dataset', 'Unknown Dataset')}<br>
                <small>({best_auroc.get('exp_name', 'Unknown Experiment')})</small></div>
            </div>
        """
    
    if best_recall_1 is not None and 'recall 1.0%' in best_recall_1:
        best_results_html += f"""
            <div class="stat-card">
                <h4>📈 Best Recall @ 1% FPR</h4>
                <div class="stat-number">{best_recall_1['recall 1.0%']:.4f}</div>
                <div class="stat-label">{best_recall_1.get('dataset', 'Unknown Dataset')}<br>
                <small>({best_recall_1.get('exp_name', 'Unknown Experiment')})</small></div>
            </div>
        """

    html_content += f"""
                </tbody>
            </table>
        </div>
    </div>
    
    <div class="section">
        <h2>🏆 Best Results Across All Experiments</h2>
        <div style="display: grid; grid-template-columns: repeat(auto-fit, minmax(300px, 1fr)); gap: 20px;">
            {best_results_html}
        </div>
    </div>

    <!-- Modal for visualizations -->
    <div id="vizModal" class="modal">
        <div class="modal-content">
            <div class="modal-header">
                <div class="modal-title" id="modalTitle">Visualizations</div>
                <span class="close" onclick="closeModal()">&times;</span>
            </div>
            <div id="modalBody">
                <!-- Visualizations will be loaded here -->
            </div>
        </div>
    </div>

    <script>
        // Store data and visualization mapping
        const originalData = {json.dumps(df.to_dict('records'))};
        const vizMapping = {json.dumps(viz_mapping)};
        
        function filterTable() {{
            const experimentFilter = document.getElementById('experimentFilter').value;
            const datasetFilter = document.getElementById('datasetFilter').value;
            const modelFilter = document.getElementById('modelFilter').value;
            
            const rows = document.querySelectorAll('#resultsTable tbody tr');
            
            rows.forEach(row => {{
                const experiment = row.getAttribute('data-experiment');
                const dataset = row.getAttribute('data-dataset');
                const modelCell = row.querySelector('td:nth-child(3)'); // Assuming model is 3rd column
                const model = modelCell ? modelCell.textContent.trim() : '';
                
                const showRow = (experimentFilter === '' || experiment === experimentFilter) &&
                              (datasetFilter === '' || dataset === datasetFilter) &&
                              (modelFilter === '' || model === modelFilter);
                
                row.style.display = showRow ? '' : 'none';
            }});
        }}
        
        function showVisualization(row) {{
            const dataset = row.getAttribute('data-dataset');
            const experiment = row.getAttribute('data-experiment');
            
            // Update modal title
            document.getElementById('modalTitle').textContent = 
                `Visualizations: ${{dataset}} (Experiment: ${{experiment}})`;
            
            // Find visualizations for this experiment and dataset
            const expDatasetKey = `${{experiment}}|${{dataset}}`;
            const methods = ['mean', 'max'];
            let modalBody = '<div class="visualization-container">';
            
            modalBody += `
                <p><strong>Dataset:</strong> ${{dataset}}</p>
                <p><strong>Experiment:</strong> ${{experiment}}</p>
                <p>These visualizations show the distribution of probe scores for honest (blue) vs. deceptive (red) statements using different token reduction methods.</p>
                
                <h3>📊 Probe Score Histograms</h3>
                <div class="viz-grid">
            `;
            
            let foundViz = false;
            
            // Add probe score histograms
            methods.forEach(method => {{
                const key = `${{experiment}}|${{dataset}}|${{method}}`;
                const imgPath = vizMapping[key];
                
                modalBody += '<div class="viz-item">';
                if (imgPath) {{
                    modalBody += `
                        <h4>Token Method: ${{method}}</h4>
                        <img src="${{imgPath}}" alt="Probe scores for ${{method}}" 
                             onclick="window.open('${{imgPath}}', '_blank')" style="cursor: pointer;">
                    `;
                    foundViz = true;
                }} else {{
                    modalBody += `
                        <h4>Token Method: ${{method}}</h4>
                        <div class="viz-error">
                            ⚠️ Visualization not found for ${{method}} method
                        </div>
                    `;
                }}
                modalBody += '</div>';
            }});
            
            modalBody += '</div>';
            
            // Add overview plots (violin plots, ROC curves, etc.) for this experiment
            const overviewPlots = Object.keys(vizMapping).filter(key => 
                key.startsWith(`${{experiment}}|overview|`)
            );
            
            if (overviewPlots.length > 0) {{
                modalBody += `
                    <div class="overview-plots">
                        <h3>📈 Standard Experiment Plots</h3>
                        <div class="viz-grid">
                `;
                
                overviewPlots.forEach(plotKey => {{
                    const imgPath = vizMapping[plotKey];
                    const plotName = plotKey.split('|')[2]; // experiment|overview|plotname
                    const displayName = plotName.replace(/_/g, ' ').replace(/\\b\\w/g, l => l.toUpperCase());
                    
                    modalBody += `
                        <div class="viz-item">
                            <h4>${{displayName}}</h4>
                            <img src="${{imgPath}}" alt="${{plotName}}" 
                                 onclick="window.open('${{imgPath}}', '_blank')" style="cursor: pointer;">
                        </div>
                    `;
                    foundViz = true;
                }});
                
                modalBody += '</div></div>';
            }}
            
            if (!foundViz) {{
                modalBody += '<div class="viz-error">No visualizations found for this experiment/dataset combination.</div>';
            }}
            
            modalBody += '</div>';
            
            document.getElementById('modalBody').innerHTML = modalBody;
            document.getElementById('vizModal').style.display = 'block';
        }}
        
        function closeModal() {{
            document.getElementById('vizModal').style.display = 'none';
        }}
        
        function downloadCSV() {{
            // Create CSV content from the original data
            const headers = Object.keys(originalData[0]);
            const csvContent = [
                headers.join(','),
                ...originalData.map(row => headers.map(h => `"${{row[h] || ''}}"`).join(','))
            ].join('\\n');
            
            // Create download link
            const blob = new Blob([csvContent], {{ type: 'text/csv' }});
            const url = window.URL.createObjectURL(blob);
            const a = document.createElement('a');
            a.href = url;
            a.download = 'combined_experiment_results.csv';
            document.body.appendChild(a);
            a.click();
            document.body.removeChild(a);
            window.URL.revokeObjectURL(url);
        }}
        
        // Close modal when clicking outside of it
        window.onclick = function(event) {{
            const modal = document.getElementById('vizModal');
            if (event.target === modal) {{
                closeModal();
            }}
        }}
    </script>
</body>
</html>
"""
    
    return html_content


def generate_multi_experiment_html_dashboard_for_zip(df: pd.DataFrame, viz_mapping: dict) -> str:
    """Generate HTML dashboard content for multiple experiments with relative paths for zip package."""
    
    # Get experiment statistics
    experiments = df['exp_name'].unique()
    datasets = df['dataset'].unique()
    models = df['model'].unique()
    
    # Find best results across all experiments
    best_auroc = None
    best_recall_1 = None
    best_balanced_accuracy = None
    
    if 'auroc' in df.columns:
        best_auroc_idx = df['auroc'].idxmax()
        best_auroc = df.loc[best_auroc_idx]
    
    if 'recall 1.0%' in df.columns:
        best_recall_idx = df['recall 1.0%'].idxmax()
        best_recall_1 = df.loc[best_recall_idx]
    
    if 'balanced_accuracy' in df.columns:
        best_balanced_accuracy_idx = df['balanced_accuracy'].idxmax()
        best_balanced_accuracy = df.loc[best_balanced_accuracy_idx]
    
    # Define columns to display in table with optimal ordering
    important_columns = ['exp_name', 'method', 'model', 'train_dataset', 'eval_dataset']
    metric_columns = [col for col in df.columns if col not in important_columns + ['exp_path', 'dataset']]
    display_columns = important_columns + metric_columns
    
    # Convert viz_mapping to use relative paths for zip package (paths already converted by zip function)
    relative_viz_mapping = viz_mapping
    
    html_content = f"""<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>Multi-Experiment Deception Detection Dashboard</title>
    <style>
        body {{
            font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, sans-serif;
            margin: 0;
            padding: 20px;
            background-color: #f5f5f5;
            line-height: 1.6;
        }}
        .header {{
            background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
            color: white;
            padding: 30px;
            border-radius: 10px;
            text-align: center;
            margin-bottom: 30px;
            box-shadow: 0 4px 6px rgba(0,0,0,0.1);
        }}
        .header h1 {{
            margin: 0;
            font-size: 2.5em;
            font-weight: 600;
        }}
        .header p {{
            margin: 10px 0 0 0;
            font-size: 1.1em;
            opacity: 0.9;
        }}
        .stats-grid {{
            display: grid;
            grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
            gap: 20px;
            margin-bottom: 30px;
        }}
        .stat-card {{
            background: white;
            padding: 25px;
            border-radius: 8px;
            box-shadow: 0 2px 4px rgba(0,0,0,0.1);
            text-align: center;
            border-top: 4px solid #667eea;
        }}
        .stat-number {{
            font-size: 2.5em;
            font-weight: bold;
            color: #333;
            margin-bottom: 5px;
        }}
        .stat-label {{
            color: #666;
            font-size: 1.1em;
            text-transform: uppercase;
            letter-spacing: 0.5px;
        }}
        .filters {{
            background: white;
            padding: 20px;
            border-radius: 8px;
            margin-bottom: 20px;
            box-shadow: 0 2px 4px rgba(0,0,0,0.1);
        }}
        .filter-group {{
            display: inline-block;
            margin-right: 20px;
            margin-bottom: 10px;
        }}
        .filter-group label {{
            display: block;
            margin-bottom: 5px;
            font-weight: 600;
            color: #333;
        }}
        .filter-group select {{
            padding: 8px 12px;
            border: 1px solid #ddd;
            border-radius: 4px;
            font-size: 14px;
            min-width: 150px;
        }}
        .download-btn {{
            background: #667eea;
            color: white;
            border: none;
            padding: 10px 20px;
            border-radius: 5px;
            cursor: pointer;
            font-size: 14px;
            margin: 5px;
            transition: background-color 0.3s;
        }}
        .download-btn:hover {{
            background: #5a67d8;
        }}
        .section {{
            background: white;
            margin-bottom: 20px;
            border-radius: 8px;
            overflow: hidden;
            box-shadow: 0 2px 4px rgba(0,0,0,0.1);
        }}
        .section h2 {{
            margin: 0;
            padding: 20px;
            background: #f8f9fa;
            border-bottom: 1px solid #e9ecef;
            color: #333;
        }}
        .table-container {{
            overflow-x: auto;
            max-height: 600px;
            overflow-y: auto;
        }}
        table {{
            width: 100%;
            border-collapse: collapse;
            font-size: 14px;
        }}
        th, td {{
            padding: 12px 8px;
            text-align: left;
            border-bottom: 1px solid #e9ecef;
        }}
        th {{
            background: #f8f9fa;
            font-weight: 600;
            position: sticky;
            top: 0;
            z-index: 10;
        }}
        .row-clickable {{
            cursor: pointer;
            transition: background-color 0.2s;
        }}
        .row-clickable:hover {{
            background-color: #f8f9fa;
        }}
        .metric-cell {{
            font-family: 'SF Mono', Monaco, 'Cascadia Code', 'Roboto Mono', monospace;
        }}
        .dataset-name {{
            font-weight: 600;
            color: #333;
        }}
        .exp-name {{
            font-weight: 500;
            color: #667eea;
        }}
        .auroc-high {{ background-color: #d4edda; color: #155724; }}
        .auroc-medium {{ background-color: #fff3cd; color: #856404; }}
        .auroc-low {{ background-color: #f8d7da; color: #721c24; }}
        .modal {{
            display: none;
            position: fixed;
            z-index: 1000;
            left: 0;
            top: 0;
            width: 100%;
            height: 100%;
            background-color: rgba(0,0,0,0.5);
        }}
        .modal-content {{
            background-color: #fefefe;
            margin: 2% auto;
            padding: 0;
            border-radius: 8px;
            width: 95%;
            max-width: 1200px;
            max-height: 90vh;
            overflow-y: auto;
        }}
        .modal-header {{
            padding: 20px;
            background: #f8f9fa;
            border-bottom: 1px solid #e9ecef;
            display: flex;
            justify-content: space-between;
            align-items: center;
        }}
        .modal-title {{
            font-size: 1.5em;
            font-weight: 600;
            color: #333;
        }}
        .close {{
            color: #aaa;
            float: right;
            font-size: 28px;
            font-weight: bold;
            cursor: pointer;
        }}
        .close:hover {{
            color: #000;
        }}
        .visualization-container {{
            padding: 20px;
        }}
        .viz-grid {{
            display: grid;
            grid-template-columns: repeat(auto-fit, minmax(400px, 1fr));
            gap: 20px;
            margin-top: 20px;
        }}
        .viz-item {{
            text-align: center;
        }}
        .viz-item h4 {{
            margin-top: 0;
            color: #333;
        }}
        .viz-item img {{
            max-width: 100%;
            height: auto;
            border: 1px solid #ddd;
            border-radius: 4px;
        }}
        .viz-error {{
            padding: 20px;
            background: #f8d7da;
            color: #721c24;
            border-radius: 4px;
            font-style: italic;
        }}
        .package-info {{
            background: #e7f3ff;
            color: #0066cc;
            padding: 15px;
            border-radius: 8px;
            margin-bottom: 20px;
            border-left: 4px solid #0066cc;
        }}
    </style>
</head>
<body>
    <div class="package-info">
        <strong>📦 Portable Dashboard Package</strong> - This is a self-contained dashboard with all visualizations included. Generated on {datetime.now().strftime('%B %d, %Y at %I:%M %p')}.
    </div>

    <div class="header">
        <h1>🔍 Multi-Experiment Deception Detection Dashboard</h1>
        <p>Comprehensive analysis across all experiment results</p>
    </div>
    
    <div class="stats-grid">
        <div class="stat-card">
            <div class="stat-number">{len(experiments)}</div>
            <div class="stat-label">Experiments</div>
        </div>
        <div class="stat-card">
            <div class="stat-number">{len(datasets)}</div>
            <div class="stat-label">Unique Datasets</div>
        </div>
        <div class="stat-card">
            <div class="stat-number">{len(df)}</div>
            <div class="stat-label">Total Results</div>
        </div>
        <div class="stat-card">
            <div class="stat-number">{len(models)}</div>
            <div class="stat-label">Models</div>
        </div>
    </div>
    
    <div class="filters">
        <h3>🔧 Filters & Downloads</h3>
        <div class="filter-group">
            <label for="experimentFilter">Experiment:</label>
            <select id="experimentFilter" onchange="filterTable()">
                <option value="">All Experiments</option>
                {''.join([f'<option value="{exp}">{exp}</option>' for exp in sorted(experiments)])}
            </select>
        </div>
        <div class="filter-group">
            <label for="datasetFilter">Dataset:</label>
            <select id="datasetFilter" onchange="filterTable()">
                <option value="">All Datasets</option>
                {''.join([f'<option value="{dataset}">{dataset}</option>' for dataset in sorted(datasets)])}
            </select>
        </div>
        <div class="filter-group">
            <label for="modelFilter">Model:</label>
            <select id="modelFilter" onchange="filterTable()">
                <option value="">All Models</option>
                {''.join([f'<option value="{model}">{model}</option>' for model in sorted(models)])}
            </select>
        </div>
        
        <button class="download-btn" onclick="downloadCSV()">📊 Download Combined Results CSV</button>
    </div>
    
    <div class="section">
        <h2>📊 All Experiment Results</h2>
        <p><strong>💡 Tip:</strong> Click on any row to view visualizations for that dataset/experiment combination.</p>
        <div class="table-container">
            <table id="resultsTable">
                <thead>
                    <tr>
"""

    # Add table headers for all columns
    for col in display_columns:
        display_name = col.replace('_', ' ').replace('%', ' %').title()
        html_content += f'                        <th>{display_name}</th>\n'
    
    html_content += """
                    </tr>
                </thead>
                <tbody>
"""

    # Add table rows with all data
    for _, row in df.iterrows():
        dataset_name = row.get('dataset', 'unknown')
        exp_name = row.get('exp_name', 'unknown')
        html_content += f'                    <tr class="row-clickable" data-dataset="{dataset_name}" data-experiment="{exp_name}" onclick="showVisualization(this)">\n'
        
        for col in display_columns:
            value = row.get(col, 'N/A')
            cell_class = "metric-cell"
            
            # Special formatting for different column types
            if col == 'dataset':
                cell_class += " dataset-name"
                html_content += f'                        <td class="{cell_class}">{value}</td>\n'
            elif col == 'exp_name':
                cell_class += " exp-name"
                html_content += f'                        <td class="{cell_class}">{value}</td>\n'
            elif col == 'auroc':
                auroc_class = "auroc-high" if value >= 0.8 else "auroc-medium" if value >= 0.6 else "auroc-low"
                cell_class += f" {auroc_class}"
                html_content += f'                        <td class="{cell_class}">{value:.4f}</td>\n'
            elif isinstance(value, float):
                html_content += f'                        <td class="{cell_class}">{value:.4f}</td>\n'
            elif isinstance(value, int):
                html_content += f'                        <td class="{cell_class}">{value:,}</td>\n'
            else:
                html_content += f'                        <td class="{cell_class}">{value}</td>\n'
        
        html_content += '                    </tr>\n'

    # Best results section
    best_results_html = ""
    if best_auroc is not None and 'auroc' in best_auroc:
        best_results_html += f"""
            <div class="stat-card">
                <h4>🎯 Best AUROC</h4>
                <div class="stat-number">{best_auroc['auroc']:.4f}</div>
                <div class="stat-label">{best_auroc.get('dataset', 'Unknown Dataset')}<br>
                <small>({best_auroc.get('exp_name', 'Unknown Experiment')})</small></div>
            </div>
        """
    
    if best_recall_1 is not None and 'recall 1.0%' in best_recall_1:
        best_results_html += f"""
            <div class="stat-card">
                <h4>📈 Best Recall @ 1% FPR</h4>
                <div class="stat-number">{best_recall_1['recall 1.0%']:.4f}</div>
                <div class="stat-label">{best_recall_1.get('dataset', 'Unknown Dataset')}<br>
                <small>({best_recall_1.get('exp_name', 'Unknown Experiment')})</small></div>
            </div>
        """

    html_content += f"""
                </tbody>
            </table>
        </div>
    </div>
    
    <div class="section">
        <h2>🏆 Best Results Across All Experiments</h2>
        <div style="display: grid; grid-template-columns: repeat(auto-fit, minmax(300px, 1fr)); gap: 20px;">
            {best_results_html}
        </div>
    </div>

    <!-- Modal for visualizations -->
    <div id="vizModal" class="modal">
        <div class="modal-content">
            <div class="modal-header">
                <div class="modal-title" id="modalTitle">Visualizations</div>
                <span class="close" onclick="closeModal()">&times;</span>
            </div>
            <div id="modalBody">
                <!-- Visualizations will be loaded here -->
            </div>
        </div>
    </div>

    <script>
        // Store data and visualization mapping
        const originalData = {json.dumps(df.to_dict('records'))};
        const vizMapping = {json.dumps(relative_viz_mapping)};
        
        function filterTable() {{
            const experimentFilter = document.getElementById('experimentFilter').value;
            const datasetFilter = document.getElementById('datasetFilter').value;
            const modelFilter = document.getElementById('modelFilter').value;
            
            const rows = document.querySelectorAll('#resultsTable tbody tr');
            
            rows.forEach(row => {{
                const experiment = row.getAttribute('data-experiment');
                const dataset = row.getAttribute('data-dataset');
                const modelCell = row.querySelector('td:nth-child(3)'); // Assuming model is 3rd column
                const model = modelCell ? modelCell.textContent.trim() : '';
                
                const showRow = (experimentFilter === '' || experiment === experimentFilter) &&
                              (datasetFilter === '' || dataset === datasetFilter) &&
                              (modelFilter === '' || model === modelFilter);
                
                row.style.display = showRow ? '' : 'none';
            }});
        }}
        
        function showVisualization(row) {{
            const dataset = row.getAttribute('data-dataset');
            const experiment = row.getAttribute('data-experiment');
            
            // Update modal title
            document.getElementById('modalTitle').textContent = 
                `Visualizations: ${{dataset}} (Experiment: ${{experiment}})`;
            
            // Find visualizations for this experiment and dataset
            const expDatasetKey = `${{experiment}}|${{dataset}}`;
            const methods = ['mean', 'max'];
            let modalBody = '<div class="visualization-container">';
            
            modalBody += `
                <p><strong>Dataset:</strong> ${{dataset}}</p>
                <p><strong>Experiment:</strong> ${{experiment}}</p>
                <p>These visualizations show the distribution of probe scores for honest (blue) vs. deceptive (red) statements using different token reduction methods.</p>
                
                <h3>📊 Probe Score Histograms</h3>
                <div class="viz-grid">
            `;
            
            let foundViz = false;
            
            // Add probe score histograms
            methods.forEach(method => {{
                const key = `${{experiment}}|${{dataset}}|${{method}}`;
                const imgPath = vizMapping[key];
                
                modalBody += '<div class="viz-item">';
                if (imgPath) {{
                    modalBody += `
                        <h4>Token Method: ${{method}}</h4>
                        <img src="${{imgPath}}" alt="Probe scores for ${{method}}" 
                             onclick="window.open('${{imgPath}}', '_blank')" style="cursor: pointer;">
                    `;
                    foundViz = true;
                }} else {{
                    modalBody += `
                        <h4>Token Method: ${{method}}</h4>
                        <div class="viz-error">
                            ⚠️ Visualization not found for ${{method}} method
                        </div>
                    `;
                }}
                modalBody += '</div>';
            }});
            
            modalBody += '</div>';
            
            // Add overview plots (violin plots, ROC curves, etc.) for this experiment
            const overviewPlots = Object.keys(vizMapping).filter(key => 
                key.startsWith(`${{experiment}}|overview|`)
            );
            
            if (overviewPlots.length > 0) {{
                modalBody += `
                    <div class="overview-plots">
                        <h3>📈 Standard Experiment Plots</h3>
                        <div class="viz-grid">
                `;
                
                overviewPlots.forEach(plotKey => {{
                    const imgPath = vizMapping[plotKey];
                    const plotName = plotKey.split('|')[2]; // experiment|overview|plotname
                    const displayName = plotName.replace(/_/g, ' ').replace(/\\b\\w/g, l => l.toUpperCase());
                    
                    modalBody += `
                        <div class="viz-item">
                            <h4>${{displayName}}</h4>
                            <img src="${{imgPath}}" alt="${{plotName}}" 
                                 onclick="window.open('${{imgPath}}', '_blank')" style="cursor: pointer;">
                        </div>
                    `;
                    foundViz = true;
                }});
                
                modalBody += '</div></div>';
            }}
            
            if (!foundViz) {{
                modalBody += '<div class="viz-error">No visualizations found for this experiment/dataset combination.</div>';
            }}
            
            modalBody += '</div>';
            
            document.getElementById('modalBody').innerHTML = modalBody;
            document.getElementById('vizModal').style.display = 'block';
        }}
        
        function closeModal() {{
            document.getElementById('vizModal').style.display = 'none';
        }}
        
        function downloadCSV() {{
            // Create CSV content from the original data
            const headers = Object.keys(originalData[0]);
            const csvContent = [
                headers.join(','),
                ...originalData.map(row => headers.map(h => `"${{row[h] || ''}}"`).join(','))
            ].join('\\n');
            
            // Create download link
            const blob = new Blob([csvContent], {{ type: 'text/csv' }});
            const url = window.URL.createObjectURL(blob);
            const a = document.createElement('a');
            a.href = url;
            a.download = 'combined_experiment_results.csv';
            document.body.appendChild(a);
            a.click();
            document.body.removeChild(a);
            window.URL.revokeObjectURL(url);
        }}
        
        // Close modal when clicking outside of it
        window.onclick = function(event) {{
            const modal = document.getElementById('vizModal');
            if (event.target === modal) {{
                closeModal();
            }}
        }}
    </script>
</body>
</html>
"""
    
    return html_content


def create_multi_experiment_zip_dashboard(df: pd.DataFrame, viz_mapping: dict, output_path: Path):
    """Create a portable zip file containing the dashboard and all visualization resources."""
    import tempfile
    import shutil
    
    # Create temporary directory
    temp_dir = Path(tempfile.mkdtemp())
    
    try:
        print(f"📦 Creating portable zip dashboard...")
        
        # Create subdirectories in temp folder
        viz_dir = temp_dir / "visualizations"
        viz_dir.mkdir(parents=True, exist_ok=True)
        
        # Copy all visualization files to temp directory
        copied_viz_mapping = {}
        copied_files = set()
        
        for key, original_path in viz_mapping.items():
            source_path = Path(original_path)
            if source_path.exists():
                # Create unique filename to avoid conflicts
                filename = source_path.name
                counter = 1
                while filename in copied_files:
                    stem = source_path.stem
                    suffix = source_path.suffix
                    filename = f"{stem}_{counter}{suffix}"
                    counter += 1
                
                dest_path = viz_dir / filename
                shutil.copy2(source_path, dest_path)
                copied_viz_mapping[key] = f"visualizations/{filename}"
                copied_files.add(filename)
                
        print(f"📁 Copied {len(copied_viz_mapping)} visualization files")
        
        # Generate HTML with relative paths
        html_content = generate_multi_experiment_html_dashboard_for_zip(df, copied_viz_mapping)
        
        # Save HTML to temp directory
        html_file = temp_dir / "dashboard.html"
        with open(html_file, 'w', encoding='utf-8') as f:
            f.write(html_content)
        
        # Save CSV to temp directory
        csv_file = temp_dir / "combined_results.csv" 
        df.to_csv(csv_file, index=False)
        
        # Create README file
        readme_content = f"""# Multi-Experiment Deception Detection Dashboard

This portable package contains:

## Files
- `dashboard.html` - Interactive dashboard (open this in your browser)
- `combined_results.csv` - All experiment results in CSV format
- `visualizations/` - All probe score histogram images
- `README.txt` - This file

## Usage
1. Extract this zip file to any location
2. Open `dashboard.html` in any modern web browser
3. The dashboard is fully self-contained and will work offline

## Features
- Compare results across {len(df['exp_name'].unique())} experiments
- View {len(df['dataset'].unique())} unique datasets
- Interactive filtering and sorting
- Click any row to view visualizations
- Download results as CSV

## Generated
- Date: {datetime.now().strftime('%B %d, %Y at %I:%M %p')}
- Total Results: {len(df)}
- Visualizations: {len(copied_viz_mapping)}

Enjoy your analysis! 🔍
"""
        
        readme_file = temp_dir / "README.txt"
        with open(readme_file, 'w', encoding='utf-8') as f:
            f.write(readme_content)
        
        # Create zip file (output_path already includes timestamp if needed)
        if output_path.suffix != '.zip':
            zip_path = output_path.with_suffix('.zip')
        else:
            zip_path = output_path
        shutil.make_archive(str(zip_path.with_suffix('')), 'zip', temp_dir)
        
        print(f"✅ Portable dashboard zip created: {zip_path}")
        print(f"📦 Package contents:")
        print(f"   - Interactive HTML dashboard")
        print(f"   - {len(copied_viz_mapping)} visualization images") 
        print(f"   - Combined results CSV")
        print(f"   - README with instructions")
        print(f"🌐 Extract the zip and open 'dashboard.html' in a browser!")
        
    finally:
        # Clean up temporary directory
        if temp_dir.exists():
            shutil.rmtree(temp_dir)


def main():
    parser = argparse.ArgumentParser(description="Generate HTML dashboard from experiment results")
    parser.add_argument("--exp_path", type=str, default=None,
                       help="Path to the experiment folder (if not provided, will use the latest experiment)")
    parser.add_argument("--output", type=str, default="dashboard.html",
                       help="Output HTML filename")
    parser.add_argument("--no_zip", action="store_true",
                       help="Don't create a portable zip file", default=False)
    parser.add_argument("--multi", action="store_true",
                       help="Generate dashboard for all experiments (multi-experiment mode)")
    parser.add_argument("--search_dirs", type=str, nargs="+", default=["results", "example_results"],
                       help="Directories to search for experiments (default: results example_results)")
    parser.add_argument("--list_experiments", action="store_true",
                       help="List all available experiment folders and exit")
    
    args = parser.parse_args()
    
    # List experiments if requested
    if args.list_experiments:
        experiment_folders = find_experiment_folders(args.search_dirs)
        print("Available experiment folders:")
        for i, folder in enumerate(experiment_folders, 1):
            print(f"  {i}. {folder}")
        return
    
    # Generate dashboard
    if args.multi:
        # Generate multi-experiment dashboard (will save to results folder with timestamp by default)
        output_filename = args.output if args.output != "dashboard.html" else "multi_experiment_dashboard.html"
        generate_multi_experiment_dashboard(
            search_dirs=args.search_dirs,
            output_filename=output_filename,
            output_dir=None,  # Will default to results folder with timestamp
            create_zip=not args.no_zip
        )
    else:
        # Generate single-experiment dashboard
        generate_dashboard(args.exp_path, args.output, create_zip=not args.no_zip)


if __name__ == "__main__":
    main() 