

# https://zenodo.org/record/4656032/files/wind_4_seconds_dataset.zip?download=1, wind_power_4s, standard
# https://zenodo.org/record/4656027/files/solar_4_seconds_dataset.zip?download=1, solar_power_4s, standard
# https://zenodo.org/record/4656049/files/us_births_dataset.zip?download=1, us_births, standard
# https://zenodo.org/record/4656058/files/saugeenday_dataset.zip?download=1, saugeen_river_flow, standard
# https://zenodo.org/record/4654722/files/sunspot_dataset_without_missing_values.zip?download=1, sunspot, no_missing_values
# https://zenodo.org/record/4654773/files/sunspot_dataset_with_missing_values.zip?download=1, sunspot, missing_values
# https://zenodo.org/record/5129091/files/temperature_rain_dataset_without_missing_values.zip?download=1, temperature_rain, no_missing_values
# https://zenodo.org/record/5129073/files/temperature_rain_dataset_with_missing_values.zip?download=1, temperature_rain, missing_values
# https://zenodo.org/record/4656009/files/covid_deaths_dataset.zip?download=1, covid_deaths, standard
# https://zenodo.org/record/4656014/files/hospital_dataset.zip?download=1, hospital, standard
# https://zenodo.org/record/5122232/files/rideshare_dataset_without_missing_values.zip?download=1, rideshare, no_missing_values
# https://zenodo.org/record/5122114/files/rideshare_dataset_with_missing_values.zip?download=1, rideshare, missing_values
# https://zenodo.org/record/4656132/files/traffic_hourly_dataset.zip?download=1, traffic_hourly, standard
# https://zenodo.org/record/4656135/files/traffic_weekly_dataset.zip?download=1, traffic_weekly, standard
# https://zenodo.org/record/4654833/files/fred_md_dataset.zip?download=1, fred_md, standard
# https://zenodo.org/record/4656022/files/car_parts_dataset_with_missing_values.zip?download=1, car_parts, missing_values
# https://zenodo.org/record/4656021/files/car_parts_dataset_without_missing_values.zip?download=1, car_parts, no_missing_values
# https://zenodo.org/record/4656140/files/electricity_hourly_dataset.zip?download=1, electricity_hourly, standard
# https://zenodo.org/record/4656141/files/electricity_weekly_dataset.zip?download=1, electricity_weekly, standard
# https://zenodo.org/record/4656144/files/solar_10_minutes_dataset.zip?download=1, solar_10_minutes, standard
# https://zenodo.org/record/4656151/files/solar_weekly_dataset.zip?download=1, solar_weekly, standard
# https://zenodo.org/record/4656080/files/kaggle_web_traffic_dataset_with_missing_values.zip?download=1, kaggle_web_traffic_daily, missing_values
# https://zenodo.org/record/4656075/files/kaggle_web_traffic_dataset_without_missing_values.zip?download=1, kaggle_web_traffic_daily, no_missing_values
# https://zenodo.org/record/4656664/files/kaggle_web_traffic_weekly_dataset.zip?download=1, kaggle_web_traffic_weekly, standard
# https://zenodo.org/record/4656110/files/nn5_daily_dataset_with_missing_values.zip?download=1, nn5_daily, missing_values
# https://zenodo.org/record/4656117/files/nn5_daily_dataset_without_missing_values.zip?download=1, nn5_daily, no_missing_values
# https://zenodo.org/record/4656125/files/nn5_weekly_dataset.zip?download=1, nn5_weekly, standard
# https://zenodo.org/record/4654822/files/weather_dataset.zip?download=1, weather, standard
# https://zenodo.org/record/4656719/files/kdd_cup_2018_dataset_with_missing_values.zip?download=1, kdd_cup, missing_values
# https://zenodo.org/record/4656756/files/kdd_cup_2018_dataset_without_missing_values.zip?download=1, kdd_cup, no_missing_values
# https://zenodo.org/record/5122535/files/vehicle_trips_dataset_with_missing_values.zip?download=1, vehicle_trips, missing_values
# https://zenodo.org/record/5122537/files/vehicle_trips_dataset_without_missing_values.zip?download=1, vehicle_trips, no_missing_values
# https://zenodo.org/record/4656626/files/pedestrian_counts_dataset.zip?download=1, melbourne_pedestrian_counts, standard
# https://zenodo.org/record/5121965/files/bitcoin_dataset_with_missing_values.zip?download=1, bitcoin, missing_values
# https://zenodo.org/record/5122101/files/bitcoin_dataset_without_missing_values.zip?download=1, bitcoin, no_missing_values
# https://zenodo.org/record/4654802/files/dominick_dataset.zip?download=1, dominick, standard
# https://zenodo.org/record/4654909/files/wind_farms_minutely_dataset_with_missing_values.zip?download=1, wind_farms_minutely, missing_values
# https://zenodo.org/record/4654858/files/wind_farms_minutely_dataset_without_missing_values.zip?download=1, wind_farms_minutely, no_missing_values
# https://zenodo.org/record/4659727/files/australian_electricity_demand_dataset.zip?download=1, australian_electricity, standard
# https://zenodo.org/record/4656091/files/london_smart_meters_dataset_without_missing_values.zip?download=1, london_smart_meters, no_missing_values
# https://zenodo.org/record/4656072/files/london_smart_meters_dataset_with_missing_values.zip?download=1, london_smart_meters, missing_values
# https://zenodo.org/record/4656042/files/cif_2016_dataset.zip?download=1, cif_2016, standard
# https://zenodo.org/record/4656096/files/tourism_monthly_dataset.zip?download=1, tourism_monthly, standard
# https://zenodo.org/record/4656093/files/tourism_quarterly_dataset.zip?download=1, tourism_quarterly, standard
# https://zenodo.org/record/4656103/files/tourism_yearly_dataset.zip?download=1, tourism_yearly, standard
# https://zenodo.org/record/4656589/files/m4_hourly_dataset.zip?download=1, m4_hourly, standard
# https://zenodo.org/record/4656548/files/m4_daily_dataset.zip?download=1, m4_daily, standard
# https://zenodo.org/record/4656522/files/m4_weekly_dataset.zip?download=1, m4_weekly, standard
# https://zenodo.org/record/4656480/files/m4_monthly_dataset.zip?download=1, m4_monthly, standard
# https://zenodo.org/record/4656410/files/m4_quarterly_dataset.zip?download=1, m4_quarterly, standard
# https://zenodo.org/record/4656379/files/m4_yearly_dataset.zip?download=1, m4_yearly, standard
# https://zenodo.org/record/4656222/files/m3_yearly_dataset.zip?download=1, m3_yearly, standard
# https://zenodo.org/record/4656262/files/m3_quarterly_dataset.zip?download=1, m3_quarterly, standard
# https://zenodo.org/record/4656298/files/m3_monthly_dataset.zip?download=1, m3_monthly, standard
# https://zenodo.org/record/4656335/files/m3_other_dataset.zip?download=1, m3_other, standard
# https://zenodo.org/record/4656193/files/m1_yearly_dataset.zip?download=1, m1_yearly, standard
# https://zenodo.org/record/4656154/files/m1_quarterly_dataset.zip?download=1, m1_quarterly, standard
# https://zenodo.org/record/4656159/files/m1_monthly_dataset.zip?download=1, m1_monthly, standard
# https://zenodo.org/record/5184708/files/oikolab_weather_dataset.zip?download=1, oikolab_weather, standard
# https://zenodo.org/record/4663762/files/covid_mobility_dataset_with_missing_values.zip?download=1, covid_mobility, missing_values
# https://zenodo.org/record/4663809/files/covid_mobility_dataset_without_missing_values.zip?download=1, covid_mobility, no_missing_values


# Organize all of the zenodo links above into a dictionary
URLS = {
    'wind_power_4s': {'standard': 'https://zenodo.org/record/4656032/files/wind_4_seconds_dataset.zip'},
    'solar_power_4s': {'standard': 'https://zenodo.org/record/4656027/files/solar_4_seconds_dataset.zip'},
    'us_births': {'standard': 'https://zenodo.org/record/4656049/files/us_births_dataset.zip'},
    'saugeen_river_flow': {'standard': 'https://zenodo.org/record/4656058/files/saugeenday_dataset.zip'},
    'sunspot': 
        {
            'no_missing_values': 'https://zenodo.org/record/4654722/files/sunspot_dataset_without_missing_values.zip', 
            'missing_values': 'https://zenodo.org/record/4654773/files/sunspot_dataset_with_missing_values.zip',
        },
    'temperature_rain': 
        {
            'no_missing_values': 'https://zenodo.org/record/5129091/files/temperature_rain_dataset_without_missing_values.zip', 
            'missing_values': 'https://zenodo.org/record/5129073/files/temperature_rain_dataset_with_missing_values.zip',
        },
    'covid_deaths': {'standard': 'https://zenodo.org/record/4656009/files/covid_deaths_dataset.zip'},
    'hospital': {'standard': 'https://zenodo.org/record/4656014/files/hospital_dataset.zip'},
    'rideshare': 
        {
            'no_missing_values': 'https://zenodo.org/record/5122232/files/rideshare_dataset_without_missing_values.zip', 
            'missing_values': 'https://zenodo.org/record/5122114/files/rideshare_dataset_with_missing_values.zip'
        },
    'traffic_hourly': {'standard': 'https://zenodo.org/record/4656132/files/traffic_hourly_dataset.zip'},
    'traffic_weekly': {'standard': 'https://zenodo.org/record/4656135/files/traffic_weekly_dataset.zip'},
    'fred_md': {'standard': 'https://zenodo.org/record/4654833/files/fred_md_dataset.zip'},
    'car_parts': 
        {
            'missing_values': 'https://zenodo.org/record/4656021/files/car_parts_dataset_without_missing_values.zip', 
            'no_missing_values': 'https://zenodo.org/record/4656022/files/car_parts_dataset_with_missing_values.zip'
        },
    'electricity_hourly': {'standard': 'https://zenodo.org/record/4656140/files/electricity_hourly_dataset.zip'},
    'electricity_weekly': {'standard': 'https://zenodo.org/record/4656141/files/electricity_weekly_dataset.zip'},
    'solar_10_minutes': {'standard': 'https://zenodo.org/record/4656144/files/solar_10_minutes_dataset.zip'},
    'solar_weekly': {'standard': 'https://zenodo.org/record/4656151/files/solar_weekly_dataset.zip'},
    'kaggle_web_traffic_daily':
        {
            'missing_values': 'https://zenodo.org/record/4656080/files/kaggle_web_traffic_dataset_with_missing_values.zip',
            'no_missing_values': 'https://zenodo.org/record/4656075/files/kaggle_web_traffic_dataset_without_missing_values.zip',
        },
    'kaggle_web_traffic_weekly': {'standard': 'https://zenodo.org/record/4656664/files/kaggle_web_traffic_weekly_dataset.zip'},
    'nn5_daily':
        {
            'missing_values': 'https://zenodo.org/record/4656110/files/nn5_daily_dataset_with_missing_values.zip',
            'no_missing_values': 'https://zenodo.org/record/4656117/files/nn5_daily_dataset_without_missing_values.zip',
        },
    'nn5_weekly': {'standard': 'https://zenodo.org/record/4656125/files/nn5_weekly_dataset.zip'},
    'weather': {'standard': 'https://zenodo.org/record/4654822/files/weather_dataset.zip'},
    'kdd_cup':
        {
            'missing_values': 'https://zenodo.org/record/4656719/files/kdd_cup_2018_dataset_with_missing_values.zip',
            'no_missing_values': 'https://zenodo.org/record/4656756/files/kdd_cup_2018_dataset_without_missing_values.zip',
        },
    'vehicle_trips':
        {
            'missing_values': 'https://zenodo.org/record/5122535/files/vehicle_trips_dataset_with_missing_values.zip',
            'no_missing_values': 'https://zenodo.org/record/5122537/files/vehicle_trips_dataset_without_missing_values.zip',
        },
    'melbourne_pedestrian_counts': {'standard': 'https://zenodo.org/record/4656626/files/pedestrian_counts_dataset.zip'},
    'bitcoin':
        {
            'missing_values': 'https://zenodo.org/record/5121965/files/bitcoin_dataset_with_missing_values.zip',
            'no_missing_values': 'https://zenodo.org/record/5122101/files/bitcoin_dataset_without_missing_values.zip',
        },
    'dominick': {'standard': 'https://zenodo.org/record/4654802/files/dominick_dataset.zip'},
    'wind_farms_minutely':
        {
            'missing_values': 'https://zenodo.org/record/4654909/files/wind_farms_minutely_dataset_with_missing_values.zip',
            'no_missing_values': 'https://zenodo.org/record/4654858/files/wind_farms_minutely_dataset_without_missing_values.zip',
        },
    'aus_elecdemand': {'standard': 'https://zenodo.org/record/4659727/files/australian_electricity_demand_dataset.zip'},
    'london_smart_meters':
        {
            'missing_values': 'https://zenodo.org/record/4656072/files/london_smart_meters_dataset_with_missing_values.zip',
            'no_missing_values': 'https://zenodo.org/record/4656091/files/london_smart_meters_dataset_without_missing_values.zip',
        },
    'cif_2016': {'standard': 'https://zenodo.org/record/4656042/files/cif_2016_dataset.zip'},
    'tourism_monthly': {'standard': 'https://zenodo.org/record/4656096/files/tourism_monthly_dataset.zip'},
    'tourism_quarterly': {'standard': 'https://zenodo.org/record/4656093/files/tourism_quarterly_dataset.zip'},
    'tourism_yearly': {'standard': 'https://zenodo.org/record/4656103/files/tourism_yearly_dataset.zip'},
    'm4_hourly': {'standard': 'https://zenodo.org/record/4656589/files/m4_hourly_dataset.zip'},
    'm4_daily': {'standard': 'https://zenodo.org/record/4656548/files/m4_daily_dataset.zip'},
    'm4_weekly': {'standard': 'https://zenodo.org/record/4656522/files/m4_weekly_dataset.zip'},
    'm4_monthly': {'standard': 'https://zenodo.org/record/4656480/files/m4_monthly_dataset.zip'},
    'm4_quarterly': {'standard': 'https://zenodo.org/record/4656410/files/m4_quarterly_dataset.zip'},
    'm4_yearly': {'standard': 'https://zenodo.org/record/4656379/files/m4_yearly_dataset.zip'},
    'm3_yearly': {'standard': 'https://zenodo.org/record/4656222/files/m3_yearly_dataset.zip'},
    'm3_quarterly': {'standard': 'https://zenodo.org/record/4656262/files/m3_quarterly_dataset.zip'},
    'm3_monthly': {'standard': 'https://zenodo.org/record/4656298/files/m3_monthly_dataset.zip'},
    'm3_other': {'standard': 'https://zenodo.org/record/4656335/files/m3_other_dataset.zip'},
    'm1_yearly': {'standard': 'https://zenodo.org/record/4656193/files/m1_yearly_dataset.zip'},
    'm1_quarterly': {'standard': 'https://zenodo.org/record/4656154/files/m1_quarterly_dataset.zip'},
    'm1_monthly': {'standard': 'https://zenodo.org/record/4656159/files/m1_monthly_dataset.zip'},
    'oikolab_weather': {'standard': 'https://zenodo.org/record/5184708/files/oikolab_weather_dataset.zip'},
    'covid_mobility':
        {
            'missing_values': 'https://zenodo.org/record/4663762/files/covid_mobility_dataset_with_missing_values.zip',
            'no_missing_values': 'https://zenodo.org/record/4663809/files/covid_mobility_dataset_without_missing_values.zip',
        },
}

# Download all links inside the URLS dictionary, and save them in a folder named 'monash', with a subfolder for each dataset
# The folder structure is:
#   - monash
#       - london_smart_meters
#           - missing_values
#               - london_smart_meters_dataset_with_missing_values.zip
#           - no_missing_values
#               - london_smart_meters_dataset_without_missing_values.zip
#       - australian_electricity
#           - standard
#               - australian_electricity_demand_dataset.zip
#      - ...
#
# The files are downloaded in the current working directory


import os
import requests
import zipfile
import shutil


def download_file(url, filename):
    """Download a file from a url and save it to a filename."""
    r = requests.get(url, stream=True)
    with open(filename, 'wb') as f:
        shutil.copyfileobj(r.raw, f)
    del r

    
def download_all_links(urls):
    """Download all links inside the URLS dictionary, and save them in a folder named 'monash', with a subfolder for each dataset."""
    # Create the 'monash' folder if it doesn't exist
    if not os.path.exists('monash'):
        os.mkdir('monash')
    
    # Iterate over the datasets
    for dataset_name, dataset_urls in urls.items():
        print('Downloading dataset {}...'.format(dataset_name))
        # Create the subfolder for the dataset
        if not os.path.exists(os.path.join('monash', dataset_name)):
            os.mkdir(os.path.join('monash', dataset_name))
        else:
            print('The dataset {} already exists. Skipping...'.format(dataset_name))
            continue
        
        # Iterate over the urls for the dataset
        for url_name, url in dataset_urls.items():
            # Download the file
            download_file(url, os.path.join('monash', dataset_name, url_name + '.zip'))
            
            # Unzip the file
            try:
                with zipfile.ZipFile(os.path.join('monash', dataset_name, url_name + '.zip'), 'r') as zip_ref:
                    zip_ref.extractall(os.path.join('monash', dataset_name))
            except zipfile.BadZipFile:
                print('The file {}.zip is not a valid zip file. Skipping...'.format(url_name))
                
            # Delete the zip file
            os.remove(os.path.join('monash', dataset_name, url_name + '.zip'))

    
if __name__ == '__main__':
    download_all_links(URLS)
    print('Done!')
    print('The files are saved in the current working directory.')
    print('You can find them in the folder "monash".')
    # Pretty print the folder structure using a for loop
    for root, dirs, files in os.walk('monash'):
        level = root.replace('monash', '').count(os.sep)
        indent = ' ' * 4 * (level)
        print('{}{}/'.format(indent, os.path.basename(root)))
        subindent = ' ' * 4 * (level + 1)
        for f in files:
            print('{}{}'.format(subindent, f))




    
    
    
