########################### Problem #################################

Checking:

~/exepriments/check_nips_2022_w8a_new_10/
check_nips_2022_w8a_new
check_nips_2022_w8a_new_10
check_nips_2022_w8a_new_10_noncovnex
check_nips_2022_w8a_new_10_nor_reg
check_nips_2022_w8a_new_10_nor_reg_seeds
check_nips_2022_w8a_new_600_nor_reg
Probably MA is better in some regimes when Liptchitz constant is small...
check_marina_nonconvex_num_coor_10_seeds_5

########################### NeuriPS 2022 #################################
# Gradient with mushrooms dataset: the proof of concept
PYTHONPATH=./code python3 ./code/distributed_optimization_library/experiments/dasha_partial_participation/config_libsvm_dasha_partial_particiaption.py --dumps_path /home/tyurina/exepriments/ --dataset_path ~/data --dataset mushrooms --experiments_name neurips_2022_gradient_mushrooms_nodes_20_coord_10_seeds_3  --num_nodes_list 20 --step_size_range -5 4 --number_of_seeds 3 --number_of_iterations 21000 --algorithm_names zero_marina marina frecon zero_marina_partial_participation --function nonconvex  --compressors rand_k  --number_of_coordinates 10 --quality_check_rate 10 --partial_participation_probability 0.2
PYTHONPATH=./code python3 ./code/distributed_optimization_library/experiments/dasha_partial_participation/config_libsvm_dasha_partial_particiaption.py --dumps_path /home/tyurina/exepriments/ --dataset_path ~/data --dataset mushrooms --experiments_name neurips_2022_gradient_mushrooms_nodes_100_coord_10_seeds_3  --num_nodes_list 100 --step_size_range -5 4 --number_of_seeds 3 --number_of_iterations 21000 --algorithm_names zero_marina marina frecon zero_marina_partial_participation --function nonconvex  --compressors rand_k  --number_of_coordinates 10 --quality_check_rate 10 --partial_participation_probability 0.05
PYTHONPATH=./code python3 ./code/distributed_optimization_library/experiments/dasha_partial_participation/config_libsvm_dasha_partial_particiaption.py --dumps_path /home/tyurina/exepriments/ --dataset_path ~/data --dataset mushrooms --experiments_name neurips_2022_gradient_mushrooms_nodes_100_coord_10_seeds_3_no_init  --num_nodes_list 100 --step_size_range -5 4 --number_of_seeds 3 --number_of_iterations 21000 --algorithm_names zero_marina zero_marina_no_init marina frecon zero_marina_partial_participation zero_marina_partial_participation_no_init --function nonconvex --compressors rand_k  --number_of_coordinates 10 --quality_check_rate 10 --partial_participation_probability 0.1
PYTHONPATH=./code python3 ./code/distributed_optimization_library/experiments/dasha_partial_participation/config_libsvm_dasha_partial_particiaption.py --dumps_path /home/tyurina/exepriments/ --dataset_path ~/data --dataset mushrooms --experiments_name neurips_2022_gradient_mushrooms_nodes_100_coord_10_prob_0_1_seeds_3_no_init  --num_nodes_list 100 --step_size_range -5 4 --number_of_seeds 3 --number_of_iterations 21000 --algorithm_names zero_marina zero_marina_no_init marina frecon zero_marina_partial_participation zero_marina_partial_participation_no_init --function nonconvex --compressors rand_k  --number_of_coordinates 10 --quality_check_rate 10 --partial_participation_probability 0.1

PYTHONPATH=./code python3 ./code/distributed_optimization_library/experiments/dasha_partial_participation/config_libsvm_dasha_partial_particiaption.py --dumps_path /home/tyurina/exepriments/ --dataset_path ~/data --dataset mushrooms --experiments_name neurips_2022_gradient_mushrooms_nodes_100_coord_10_prob_0_01_seeds_3_no_init  --num_nodes_list 100 --step_size_range -5 4 --number_of_seeds 3 --number_of_iterations 21000 --algorithm_names zero_marina zero_marina_no_init marina frecon zero_marina_partial_participation zero_marina_partial_participation_no_init --function nonconvex --compressors rand_k  --number_of_coordinates 10 --quality_check_rate 10 --partial_participation_probability 0.01
PYTHONPATH=./code python3 ./code/distributed_optimization_library/experiments/plots/dasha_partial_participation/plot_marina_libsvm_gradient_robust.py --dumps_paths ~/exepriments/neurips_2022_gradient_mushrooms_nodes_100_coord_10_prob_0_01_seeds_3_no_init --output_path ~/tmp_launch/tmp

PYTHONPATH=./code python3 ./code/distributed_optimization_library/experiments/dasha_partial_participation/config_libsvm_dasha_partial_particiaption.py --dumps_path /home/tyurina/exepriments/ --dataset_path ~/data --dataset mushrooms --experiments_name neurips_2022_gradient_mushrooms_nodes_100_coord_10_more_probs_seeds_3_no_init  --num_nodes_list 100 --step_size_range -5 4 --number_of_seeds 3 --number_of_iterations 21000 --algorithm_names zero_marina marina frecon zero_marina_partial_participation marina_partial_participation --function nonconvex --compressors rand_k  --number_of_coordinates 10 --quality_check_rate 100 --partial_participation_probabilities 0.01 0.1 0.5 1.0

# Finite-Sum
./code/distributed_optimization_library/experiments/dasha_partial_participation/config_libsvm_dasha_partial_particiaption.py --dumps_path /home/tyurina/exepriments --dataset_path /home/tyurina/data --dataset real-sim --experiments_name neurips_2022_finite_sum_real-sim_nof_500_numnodes_100_prob_0_01 --num_nodes_list 100 --step_size_range -10 4 --number_of_seeds 1 --number_of_iterations 50000 --algorithm_names vr_marina zero_marina_page zero_marina_partial_participation_page --cpus_per_task 11 --number_of_processes 10 --time 10 --parallel --compressors rand_k --number_of_coordinates 500 --quality_check_rate 100 --oracle minibatch --batch_size 1 --function nonconvex --partial_participation_probabilities 1.0 0.9 0.1 0.01
PYTHONPATH=./code python3 ./code/distributed_optimization_library/experiments/plots/dasha_partial_participation/plot_vr-marina_real-sim_mini-batch.py --dumps_paths ~/exepriments/neurips_2022_finite_sum_real-sim_nof_500_numnodes_100_prob_0_01/ --output_path ~/tmp_launch/tmp
## Batch size (REPEAT with larger step sizes)
PYTHONPATH=./code python3 ./code/distributed_optimization_library/experiments/dasha_partial_participation/config_libsvm_dasha_partial_particiaption.py --dumps_path /home/tyurina/exepriments --dataset_path /home/tyurina/data --dataset real-sim --experiments_name neurips_2022_finite_sum_real-sim_nof_500_numnodes_100_more_probs_batch_size_1 --num_nodes_list 100 --step_size_range -10 4 --number_of_seeds 1 --number_of_iterations 50000 --algorithm_names vr_marina zero_marina_page zero_marina_partial_participation_page --cpus_per_task 11 --number_of_processes 10 --time 10 --parallel --compressors rand_k --number_of_coordinates 500 --quality_check_rate 100 --oracle minibatch --batch_size 1 --function nonconvex --partial_participation_probabilities 1.0 0.9 0.5 0.1 0.01
PYTHONPATH=./code python3 ./code/distributed_optimization_library/experiments/dasha_partial_participation/config_libsvm_dasha_partial_particiaption.py --dumps_path /home/tyurina/exepriments --dataset_path /home/tyurina/data --dataset real-sim --experiments_name neurips_2022_finite_sum_real-sim_nof_500_numnodes_100_more_probs_batch_size_100 --num_nodes_list 100 --step_size_range -10 4 --number_of_seeds 1 --number_of_iterations 50000 --algorithm_names vr_marina zero_marina_page zero_marina_partial_participation_page --cpus_per_task 11 --number_of_processes 10 --time 10 --parallel --compressors rand_k --number_of_coordinates 500 --quality_check_rate 100 --oracle minibatch --batch_size 100 --function nonconvex --partial_participation_probabilities 1.0 0.9 0.5 0.1 0.01
PYTHONPATH=./code python3 ./code/distributed_optimization_library/experiments/dasha_partial_participation/config_libsvm_dasha_partial_particiaption.py --dumps_path /home/tyurina/exepriments --dataset_path /home/tyurina/data --dataset real-sim --experiments_name neurips_2022_finite_sum_real-sim_nof_500_numnodes_100_more_probs_batch_size_1000 --num_nodes_list 100 --step_size_range -10 4 --number_of_seeds 1 --number_of_iterations 50000 --algorithm_names vr_marina zero_marina_page zero_marina_partial_participation_page --cpus_per_task 11 --number_of_processes 10 --time 10 --parallel --compressors rand_k --number_of_coordinates 500 --quality_check_rate 100 --oracle minibatch --batch_size 1000 --function nonconvex --partial_participation_probabilities 1.0 0.9 0.5 0.1 0.01

neurips_2022_finite_sum_real-sim_nof_20000_numnodes_100_more_probs_batch_size_100_large_step_size
neurips_2022_finite_sum_real-sim_nof_20000_numnodes_100_more_probs_batch_size_1_large_step_size
neurips_2022_finite_sum_real-sim_nof_20000_numnodes_10_more_probs_batch_size_100_large_step_size

neurips_2022_finite_sum_real-sim_nof_20959_numnodes_10_more_probs_batch_size_100_large_step_size
neurips_2022_finite_sum_real-sim_nof_20959_numnodes_10_more_probs_batch_size_1_large_step_size

neurips_2022_finite_sum_mnist_nof_7850_numnodes_10_more_probs_batch_size_1000_split_by_labels_logistic
neurips_2022_finite_sum_mnist_nof_7850_numnodes_10_more_probs_batch_size_100_split_by_labels_logistic
neurips_2022_finite_sum_mnist_nof_7850_numnodes_10_more_probs_batch_size_10_split_by_labels_logistic
neurips_2022_finite_sum_mnist_nof_7850_numnodes_10_more_probs_batch_size_1_split_by_labels_logistic

neurips_2022_finite_sum_mnist_nof_7850_numnodes_100_more_probs_batch_size_1_split_by_labels_logistic
neurips_2022_finite_sum_mnist_nof_7850_numnodes_100_more_probs_batch_size_10_split_by_labels_logistic

# Stochastic
PYTHONPATH=./code python3 ./code/distributed_optimization_library/experiments/dasha_partial_participation/config_libsvm_dasha_partial_particiaption.py --dumps_path /home/tyurina/exepriments --dataset_path ~/data --dataset real-sim --experiments_name neurips_2022_stochastic_real-sim_nof_200_numnodes_10_probs_mega_batch_100000 --num_nodes_list 10 --step_size_range -10 0 --number_of_seeds 1 --number_of_iterations 500000 --algorithm_names zero_marina_stochastic zero_marina_sync_stochastic zero_marina_partial_participation_stochastic marina_stochastic frecon_stochastic --cpus_per_task 11 --number_of_processes 10 --time 10 --parallel --compressors rand_k --number_of_coordinates 200 --quality_check_rate 1000 --oracle stochastic --mega_batch 100000 --batch_size 1 --function stochastic_logistic_regression --logistic_regression_nonconvex 0.001 --partial_participation_probabilities 1.0 0.5 0.1
PYTHONPATH=./code python3 ./code/distributed_optimization_library/experiments/plots/dasha_partial_participation/plot_vr-marina_real-sim_stochastic.py --dumps_paths ~/exepriments/neurips_2022_stochastic_real-sim_nof_200_numnodes_10_probs_mega_batch_100000 --output_path ~/tmp_launch/tm
## Batch sizes
PYTHONPATH=./code python3 ./code/distributed_optimization_library/experiments/dasha_partial_participation/config_libsvm_dasha_partial_particiaption.py --dumps_path /home/tyurina/exepriments --dataset_path ~/data --dataset real-sim --experiments_name neurips_2022_stochastic_real-sim_nof_200_numnodes_10_probs_mega_batch_100000_batch_100 --num_nodes_list 10 --step_size_range -10 0 --number_of_seeds 1 --number_of_iterations 500000 --algorithm_names zero_marina_stochastic zero_marina_sync_stochastic zero_marina_partial_participation_stochastic marina_stochastic frecon_stochastic --cpus_per_task 11 --number_of_processes 10 --time 10 --parallel --compressors rand_k --number_of_coordinates 200 --quality_check_rate 1000 --oracle stochastic --mega_batch 100000 --batch_size 100 --function stochastic_logistic_regression --logistic_regression_nonconvex 0.001 --partial_participation_probabilities 1.0 0.1
PYTHONPATH=./code python3 ./code/distributed_optimization_library/experiments/dasha_partial_participation/config_libsvm_dasha_partial_particiaption.py --dumps_path /home/tyurina/exepriments --dataset_path ~/data --dataset real-sim --experiments_name neurips_2022_stochastic_real-sim_nof_200_numnodes_10_probs_mega_batch_100000_batch_10 --num_nodes_list 10 --step_size_range -10 0 --number_of_seeds 1 --number_of_iterations 500000 --algorithm_names zero_marina_stochastic zero_marina_sync_stochastic zero_marina_partial_participation_stochastic marina_stochastic frecon_stochastic --cpus_per_task 11 --number_of_processes 10 --time 10 --parallel --compressors rand_k --number_of_coordinates 200 --quality_check_rate 1000 --oracle stochastic --mega_batch 100000 --batch_size 10 --function stochastic_logistic_regression --logistic_regression_nonconvex 0.001 --partial_participation_probabilities 1.0 0.1
PYTHONPATH=./code python3 ./code/distributed_optimization_library/experiments/plots/dasha_partial_participation/plot_vr-marina_real-sim_stochastic.py --dumps_paths ~/exepriments/neurips_2022_stochastic_real-sim_nof_200_numnodes_10_probs_mega_batch_100000_batch_10 --output_path ~/tmp_launch/tmp
## Fix bug in noise momentum
PYTHONPATH=./code python3 ./code/distributed_optimization_library/experiments/dasha_partial_participation/config_libsvm_dasha_partial_particiaption.py --dumps_path /home/tyurina/exepriments --dataset_path ~/data --dataset real-sim --experiments_name neurips_2022_stochastic_real-sim_nof_200_numnodes_10_probs_mega_batch_100000_fix_nm_bug_longer --num_nodes_list 10 --step_size_range -10 0 --number_of_seeds 1 --number_of_iterations 5000000 --algorithm_names zero_marina_stochastic zero_marina_sync_stochastic zero_marina_partial_participation_stochastic marina_stochastic  --cpus_per_task 11 --number_of_processes 10 --time 10 --parallel --compressors rand_k --number_of_coordinates 200 --quality_check_rate 1000 --oracle stochastic --mega_batch 100000 --batch_size 1 --function stochastic_logistic_regression --logistic_regression_nonconvex 0.001 --partial_participation_probabilities 1.0 0.5 0.1


## Paper plots
# Finite-Sum
PYTHONPATH=./code python3 ./code/distributed_optimization_library/experiments/dasha_partial_participation/config_libsvm_dasha_partial_particiaption.py --dumps_path /home/tyurina/exepriments --dataset_path /home/tyurina/data --dataset real-sim --experiments_name neurips_2022_finite_sum_real-sim_nof_500_numnodes_100_more_probs_batch_size_1_longer --num_nodes_list 100 --step_size_range -10 4 --number_of_seeds 1 --number_of_iterations 500000 --algorithm_names vr_marina zero_marina_page zero_marina_partial_participation_page --cpus_per_task 11 --number_of_processes 10 --time 10 --parallel --compressors rand_k --number_of_coordinates 500 --quality_check_rate 100 --oracle minibatch --batch_size 1 --function nonconvex --partial_participation_probabilities 1.0 0.9 0.5 0.1 0.01
PYTHONPATH=./code python3 ./code/distributed_optimization_library/experiments/plots/dasha_partial_participation/plot_vr-marina_real-sim_mini-batch.py --dumps_paths ~/exepriments/neurips_2022_finite_sum_real-sim_nof_500_numnodes_100_more_probs_batch_size_1_longer --output_path ~/tmp_launch/neurips_2022_finite_sum_real-sim_nof_500_numnodes_100_more_probs_batch_size_1_longer --ignore_methods "VR-MARINA"
# Stochastic 
PYTHONPATH=./code python3 ./code/distributed_optimization_library/experiments/dasha_partial_participation/config_libsvm_dasha_partial_particiaption.py --dumps_path /home/tyurina/exepriments --dataset_path /home/tyurina/data --dataset real-sim --experiments_name neurips_2022_stochastic_real-sim_nof_200_numnodes_100_probs_mega_batch_10000_fix_nm_bug_longer --num_nodes_list 100 --step_size_range -10 0 --number_of_seeds 1 --number_of_iterations 5000000 --algorithm_names zero_marina_sync_stochastic zero_marina_partial_participation_stochastic --cpus_per_task 11 --number_of_processes 10 --time 10 --parallel --compressors rand_k --number_of_coordinates 200 --quality_check_rate 1000 --oracle stochastic --mega_batch 10000 --batch_size 1 --function stochastic_logistic_regression --logistic_regression_nonconvex 0.001 --partial_participation_probabilities 1.0 0.5 0.1 0.01
PYTHONPATH=./code python3 ./code/distributed_optimization_library/experiments/plots/dasha_partial_participation/plot_vr-marina_real-sim_stochastic.py --dumps_paths ~/exepriments/neurips_2022_stochastic_real-sim_nof_200_numnodes_100_probs_mega_batch_10000_fix_nm_bug_longer --output_path ~/tmp_launch/neurips_2022_stochastic_real-sim_nof_200_numnodes_100_probs_mega_batch_10000_fix_nm_bug_longer --ignore_methods "VR-MARINA (online)" "DASHA-MVR"

## In the previous experiments, I chose small step sizes, the following experiments checked that bigger step sizes don't help
neurips_2022_finite_sum_real-sim_nof_500_numnodes_100_more_probs_batch_size_1_longer_only_sync_large_step_size
neurips_2022_stochastic_real-sim_nof_200_numnodes_100_probs_mega_batch_10000_fix_nm_bug_longer_only_sync_large_step_size

#Batch size
neurips_2022_stochastic_real-sim_ident_numnodes_100_probs_mega_batch_10000_fix_nm_bug_longer_bz_1
neurips_2022_stochastic_real-sim_ident_numnodes_100_probs_mega_batch_10000_fix_nm_bug_longer_bz_100
neurips_2022_stochastic_real-sim_ident_numnodes_100_probs_mega_batch_10000_fix_nm_bug_longer_bz_1000
neurips_2022_stochastic_real-sim_ident_numnodes_100_probs_mega_batch_10000_fix_nm_bug_longer_bz_100_sort_labels