#!/bin/bash
MAX_RETRIES=-1
RETRY_INTERVAL=1
download_dataset(){
huggingface-cli download --repo-type dataset --resume-download MiniLLM/dolly --local-dir /opt/dpcvol/datasets/8625883998351850434/datasets/llm/
# huggingface-cli download MiniLLM/MiniLLM-OPT-6.7B --local-dir /opt/dpcvol/datasets/8625883998351850434/ckpt/minillm/minillm_official/opt-6.7B
# modelscope download --model Qwen/Qwen2.5-7B-Instruct --local_dir /opt/dpcvol/datasets/8625883998351850434/ckpt/minillm/minillm_official/qwen2.5-7b/
}
retry_count=0
while true; do
    # 尝试下载
    if download_dataset; then
        echo "Done!"
        break
    else
        retry_count=$((retry_count + 1))
        
        # 检查是否达到最大重试次数
        if [ $MAX_RETRIES -ne -1 ] && [ $retry_count -gt $MAX_RETRIES ]; then
            echo "Reaching the max times for download ($MAX_RETRIES), exit."
            exit 1
        fi
        
        echo "Download failed, will retry in $RETRY_INTERVAL seconds... ($retry_count/$MAX_RETRIES)"
        sleep $RETRY_INTERVAL
    fi
done
