import os
from datasets import load_dataset

# Set the Hugging Face cache directory
os.environ["HF_HOME"] = "/is/cluster/fast/XXXX-11/hfcache/huggingface"
os.environ["TRANSFORMERS_CACHE"] = "/is/cluster/fast/XXXX-11/hfcache/huggingface"
os.environ["HF_DATASETS_CACHE"] = "/is/cluster/fast/XXXX-11/hfcache/huggingface"

# Download the dataset
dataset = load_dataset("llamafactory/OpenThoughts-114k")
print(f"Dataset loaded: {dataset}")
print(f"Dataset info: {dataset.info}")

#/fast/XXXX-3/forecasting/evals/custom/manifold/zeroshot/deepseek-r1_distill_curated_results.json

#/fast/XXXX-3/forecasting/evals/custom/metaculus/zeroshot/deepseek-r1_train_curated_results.json