import os
import pandas as pd
from datasets import Dataset
from pathlib import Path

# Directory containing the CSV files
csv_dir = Path("results/subsets")
files = [
    'alpaca.csv',
    'insider-trading.csv',
    'soft-trigger.csv',
    'harm-pressure-choice.csv',
    'harm-pressure-knowledge-report.csv',
    'mask.csv',
    'instructed-deception.csv',
    'convincing-game.csv'
]

# Load each CSV file and push as a separate configuration
for csv_file in files:
    # Get the name without extension
    name = csv_file.split(".")[0]
    
    print(f"Processing {name}...")
    
    # Read the CSV file
    df = pd.read_csv(csv_dir / csv_file)
    
    # Evaluate each message in the messages column
    if "messages" in df.columns:
        df["messages"] = df["messages"].apply(eval)
    
    # Create a Hugging Face dataset from the pandas DataFrame
    dataset = Dataset.from_pandas(df)
    
    # Push to Hugging Face Hub
    # Note: You'll need to be logged in to Hugging Face first
    # You can do this by running: huggingface-cli login
    dataset.push_to_hub(
        "XXXX-1-Labs/dishonesty-bench",
        config_name=name,  # Use the CSV name as the configuration name
        split="test",  # Explicitly set the split to "test"
        private=True  # Set to False if you want it to be public
    )
    
    print(f"Successfully pushed {name} to the hub!")

print("Successfully pushed all datasets to the hub!")