Put your plotting/figure-creating command lines here

### Comparing ensemble vs. no ensemble: run these commands under /om2/user/ellisk/ec
python bin/graphs.py --checkpoints `grep  -h 'it=20.*graph=True.pickle' jobs/list_ensemble*` `grep  -h 'it=20.*graph=True.pickle' jobs/list_ellisk_2019-01-25T*` `grep  -h 'it=20.*graph=True.pickle' jobs/list_no_dsl_ellisk*` `grep  -h 'it=20.*graph=True.pickle' jobs/list_no_dsl_no_ensemble_ellisk_2019-01-28T1*` `grep  -h 'it=14.*graph=True.pickle' jobs/list_no_dsl_no_ensemble_no_auxiliary_ellisk_2019-01-29T10.*` --title "List ensemble" --export figures/list.png --colors green green green green green blue blue blue blue red red red orange orange orange orange orange purple purple purple --testingTimeout 600
python bin/graphs.py --checkpoints `grep  -h 'it=20.*graph=True.pickle' jobs/text_ensemble*` `grep  -h 'it=20.*graph=True.pickle' jobs/text_ellisk_2019*` `grep  -h 'it=20.*graph=True.pickle' jobs/text_no_dsl_ellisk_2019*` `grep  -h 'it=20.*graph=True.pickle' jobs/text_no_dsl_no_ensemble_ellisk_2019-01-28T*` `grep  -h 'it=13.*graph=True.pickle' jobs/text_no_dsl_no_ensemble_no_auxiliary_ellisk_2019-01-29T10.*`  `grep  -h 'it=7.*graph=True.pickle' jobs/text_no_generative*` --title "Text ensemble" --export figures/text.png --colors green green green green green blue blue blue blue blue blue red red red orange orange orange orange orange purple purple purple cyan cyan cyan  --testingTimeout 600

### Text figures - time and hits split
python bin/graphs.py --x-label "" -i 20 --testingTimeout 600 --title "Text editing" --export figures/text_hits.png --checkpoints teal `grep  -h 'it=20.*graph=True.pickle' jobs/text_ellisk_2019-01-2*` purple `grep  -h 'it=20.*.pickle' jobs/text_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'` orange `grep  -h it=20.*graph jobs/text_no_generative_ellisk_2019-02-14*` cyan `grep -h  it=20 jobs/baseline_ec_text*|sed 's/Exported checkpoint to//g'` --noTime --baseline enum 31 NS 0
python bin/graphs.py -i 20 --testingTimeout 600 --export figures/text_time.png --checkpoints teal `grep  -h 'it=20.*graph=True.pickle' jobs/text_ellisk_2019-01-2*` purple `grep  -h 'it=20.*.pickle' jobs/text_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'` orange `grep  -h it=20.*graph jobs/text_no_generative_ellisk_2019-02-14*` cyan `grep -h  it=20 jobs/baseline_ec_text*|sed 's/Exported checkpoint to//g'` --title "Text editing" --onlyTime --averageColors -c 
python bin/graphs.py -i 20 --testingTimeout 600 --title "Text editing" --export figures/text_hits_ws.png --checkpoints teal `grep  -h 'it=20.*graph=True.pickle' jobs/text_ellisk_2019-01-2*` purple `grep  -h 'it=20.*.pickle' jobs/text_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'` orange `grep  -h it=20.*graph jobs/text_no_generative_ellisk_2019-02-14*`  cyan `grep -h  it=20 jobs/baseline_ec_text*|sed 's/Exported checkpoint to//g'` --noTime --baseline enum 31 NS 0
python bin/graphs.py -i 20 --testingTimeout 600 --title "Text editing" --export figures/text_talk_memorize_hits_ws.png --checkpoints teal `grep  -h 'it=20.*graph=True.pickle' jobs/text_ellisk_2019-01-2*` purple `grep  -h 'it=20.*.pickle' jobs/text_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'` orange `grep  -h it=20.*graph jobs/text_no_generative_ellisk_2019-02-14*`  cyan `grep -h  it=20.*graph=True jobs/text_memorize_ellisk*|sed 's/Exported checkpoint to//g'` --noTime --baseline enum 31 NS 0
python bin/graphs.py -i 20 --testingTimeout 600 --title "Text editing" --export figures/text_memorize.png --checkpoints teal `grep  -h 'it=20.*graph=True.pickle' jobs/text_ellisk_2019-01-2*` orange `grep  -h 'it=20.*graph=True.pickle' jobs/text_memorize_ellisk*` purple `grep -h  it=20 jobs/text_memorize_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'`   --noTime --averageColors -c 


### List figures - time and hits split
python bin/graphs.py --x-label "" -i 20 --testingTimeout 600 --title "List processing" --export figures/list_hits.png --checkpoints teal `grep  -h 'it=20.*graph=True.pickle' jobs/list_ellisk_2019-01-25T*|head  -n 5` purple `grep  -h 'it=20.*.pickle' jobs/list_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'` orange `grep  -h it=20.*graph jobs/list_no_dsl_no_ensemble_ellisk*`  --noTime
python bin/graphs.py -i 20 --testingTimeout 600 --export figures/list_time.png --checkpoints teal `grep  -h 'it=20.*graph=True.pickle' jobs/list_ellisk_2019-01-25T*|head  -n 5` purple `grep  -h 'it=20.*.pickle' jobs/list_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'` orange `grep  -h it=20.*graph jobs/list_no_dsl_no_ensemble_ellisk*`  --onlyTime

### List figures - hard testing tasks only
python bin/graphs.py -i 20 --testingTimeout 600 --title "List processing" --export figures/list_hard_hits_ws.png --checkpoints teal `grep  -h 'it=20.*graph=True.pickle' jobs/list_hard_test_ellisk*|head  -n 5` purple `grep  -h 'it=20.*.pickle' jobs/list_hard_test_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'` orange `grep  -h it=20.*graph jobs/list_hard_test_no_dsl_ellisk*`  cyan `grep -h  it=20 jobs/baseline_ec_list*|sed 's/Exported checkpoint to//g'`   --noTime --baseline enum 22 NS 15
python bin/graphs.py -i 20 --testingTimeout 600 --export figures/list_hard_time.png --checkpoints  teal `grep  -h 'it=20.*graph=True.pickle' jobs/list_hard_test_ellisk*|head  -n 5` purple `grep  -h 'it=20.*.pickle' jobs/list_hard_test_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'` orange `grep  -h it=20.*graph jobs/list_hard_test_no_dsl_ellisk*` cyan `grep -h  it=20 jobs/baseline_ec_list*|sed 's/Exported checkpoint to//g'` --title "List processing"  --onlyTime -c --averageColors
python bin/graphs.py -i 20 --testingTimeout 600 --title "List processing" --export figures/list_hard_hits_ws.png --checkpoints teal `grep  -h 'it=20.*graph=True.pickle' jobs/list_hard_test_ellisk*|head  -n 5` purple `grep  -h 'it=20.*.pickle' jobs/list_hard_test_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'` orange `grep  -h it=20.*graph jobs/list_hard_test_no_dsl_ellisk*`  --noTime 31 NS 15
python bin/graphs.py -i 20 --testingTimeout 600 --title "List processing" --export figures/list_talk_memorize_hard_hits_ws.png --checkpoints teal `grep  -h 'it=20.*graph=True.pickle' jobs/list_hard_test_ellisk*|head  -n 5` purple `grep  -h 'it=20.*.pickle' jobs/list_hard_test_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'` orange `grep  -h it=20.*graph jobs/list_hard_test_no_dsl_ellisk*`  cyan `grep -h  it=20.*graph jobs/list_memorize_ellisk*|sed 's/Exported checkpoint to//g'`   --noTime --baseline enum 22 NS 15
python bin/graphs.py  -i 20 --testingTimeout 600 --title "List processing" --export figures/list_memorize.png --checkpoints teal `grep  -h 'it=20.*graph=True.pickle' jobs/list_hard_test_ellisk*|head  -n 5` orange `grep  -h 'it=20.*graph=True.pickle' jobs/list_memorize_ellisk*`  purple `grep  -h 'it=20.*.pickle' jobs/list_memorize_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'` --noTime --averageColors -c

### LOGO figures - time and hits split
python bin/graphs.py --checkpoints teal `grep  -h it=20.*graph jobs/logo_batch_50_1h_ell*` orange `grep  -h it=20.*graph jobs/logo_no_dsl_auxiliary_batch_50_1h*` purple `grep  -h it=20.*pickle jobs/logo_no_recognition_batch_50_1h*|sed 's/Exported checkpoint to//g'` cyan `grep -h  it=20 jobs/baseline_ec_logo*|sed 's/Exported checkpoint to//g'`  --export figures/logo_hits.png --noTime -i 20 --x-label "" --title "LOGO Graphics" --baseline enum 25 NS 1.2
python bin/graphs.py --checkpoints teal `grep  -h it=20.*graph jobs/logo_batch_50_1h_ell*` orange `grep  -h it=20.*graph jobs/logo_no_dsl_auxiliary_batch_50_1h*` purple `grep  -h it=20.*pickle jobs/logo_no_recognition_batch_50_1h*|sed 's/Exported checkpoint to//g'` cyan `grep -h  it=20 jobs/baseline_ec_logo*|sed 's/Exported checkpoint to//g'` --export figures/logo_time.png --onlyTime -i 20   --testingTimeout 600 --title "LOGO graphics" -c  --averageColors
python bin/graphs.py --checkpoints teal `grep  -h it=20.*graph jobs/logo_batch_50_1h_ell*` orange `grep  -h it=20.*graph jobs/logo_no_dsl_auxiliary_batch_50_1h*` purple `grep  -h it=20.*pickle jobs/logo_no_recognition_batch_50_1h*|sed 's/Exported checkpoint to//g'` cyan `grep -h  it=20 jobs/baseline_ec_logo*|sed 's/Exported checkpoint to//g'` --export figures/logo_hits_ws.png --noTime -i 20 --title "LOGO Graphics" --baseline enum 25 NS 1.2
python bin/graphs.py --checkpoints teal `grep  -h it=20.*graph jobs/logo_batch_50_1h_ell*` orange `grep  -h it=20.*graph jobs/logo_no_dsl_auxiliary_batch_50_1h*` purple `grep  -h it=20.*pickle jobs/logo_no_recognition_batch_50_1h*|sed 's/Exported checkpoint to//g'` cyan `grep -h  it=20.*graph jobs/logo_memorize_ellisk*|sed 's/Exported checkpoint to//g'` --export figures/logo_memorize_talk_hits_ws.png --noTime -i 20 --title "LOGO Graphics" --baseline enum 25 NS 1.2
python bin/graphs.py --checkpoints teal `grep  -h it=20.*graph jobs/logo_batch_50_1h_ell*` orange `grep  -h it=20.*graph jobs/logo_memorize_ellisk*` purple `grep  -h it=20.*pickle jobs/logo_memorize_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'` --export figures/logo_memorize.png --noTime -i 20 --title "LOGO Graphics"  --averageColors -c 


### Rational figures - time and hits split
python bin/graphs.py --checkpoint teal `grep  -h it=20.*graph jobs/rational_batch*` orange `grep  -h it=20.*graph jobs/rational_no_dsl*` purple `grep  -h it=20.*pickle jobs/rational_noRecognition*|sed 's/Exported checkpoint to//g'` cyan `grep -h  it=20 jobs/rational_baseline_ec*|sed 's/Exported checkpoint to//g'` --noTime --x-label "" --title "Symbolic Regression" --export figures/rational_hits.png --baseline enum 65  NS 69 -i 20b
python bin/graphs.py --checkpoint teal `grep  -h it=20.*graph jobs/rational_batch*` orange `grep  -h it=20.*graph jobs/rational_no_dsl*` purple `grep  -h it=20.*pickle jobs/rational_noRecognition*|sed 's/Exported checkpoint to//g'` cyan `grep -h  it=20 jobs/rational_baseline_ec*|sed 's/Exported checkpoint to//g'`  --onlyTime --export figures/rational_time.png  --testingTimeout 600 -i 20 --title "Symbolic regression" -c  --averageColors
python bin/graphs.py --checkpoint teal `grep  -h it=20.*graph jobs/rational_batch*` orange `grep  -h it=20.*graph jobs/rational_no_dsl*` purple `grep  -h it=20.*pickle jobs/rational_noRecognition*|sed 's/Exported checkpoint to//g'` cyan `grep -h  it=20 jobs/rational_baseline_ec*|sed 's/Exported checkpoint to//g'` --noTime --title "Symbolic Regression" --export figures/rational_hits_ws.png --baseline enum 65  NS 69 -i 20
python bin/graphs.py --checkpoint teal `grep  -h it=20.*graph jobs/rational_batch*` orange `grep  -h it=20.*graph jobs/rational_no_dsl*` purple `grep  -h it=20.*pickle jobs/rational_noRecognition*|sed 's/Exported checkpoint to//g'` cyan `grep -h  it=20.*graph jobs/rational_memorize_*2020-04-14_*|sed 's/Exported checkpoint to//g'` --noTime --title "Symbolic Regression" --export figures/rational_memorize_talk_hits_ws.png --baseline enum 65  NS 69 -i 20
python bin/graphs.py --checkpoint teal `grep  -h it=20.*graph jobs/rational_batch*` orange `grep  -h it=20.*graph jobs/rational_memorize_*2020-04-14_*` purple `grep  -h it=20.*pickle jobs/rational_memorize_no_recognition*|sed 's/Exported checkpoint to//g'` --noTime --title "Symbolic Regression" --export figures/rational_memorize.png -i 20 --averageColors -c 

### Tower figures - time and hits split
# actual baseline results are both 9, separated for visibility
python bin/graphs.py --checkpoints teal `grep  -h it=20.*graph jobs/tower_batch_50_3600*` orange `grep  -h it=20.*graph jobs/tower_no_dsl_batch_50_3600*` purple `grep  -h it=20.*pickle jobs/tower_no_recognition_batch_50_3600*|sed 's/Exported checkpoint to//g'` cyan `grep -h  it=20 jobs/baseline_ec_tower*|sed 's/Exported checkpoint to//g'` --export figures/tower_hits.png --noTime --x-label "" --title "Tower building" --baseline enum 8 NS 10 -i 20
python bin/graphs.py --checkpoints teal `grep  -h it=20.*graph jobs/tower_batch_50_3600*` orange `grep  -h it=20.*graph jobs/tower_no_dsl_batch_50_3600*` purple `grep  -h it=20.*pickle jobs/tower_no_recognition_batch_50_3600*|sed 's/Exported checkpoint to//g'` cyan `grep -h  it=20 jobs/baseline_ec_tower*|sed 's/Exported checkpoint to//g'` --export figures/tower_time.png --onlyTime --testingTimeout 600 -i 20  --title "Tower building" --averageColors -c 
python bin/graphs.py --checkpoints teal `grep  -h it=20.*graph jobs/tower_batch_50_3600*` orange `grep  -h it=20.*graph jobs/tower_no_dsl_batch_50_3600*` purple `grep  -h it=20.*pickle jobs/tower_no_recognition_batch_50_3600*|sed 's/Exported checkpoint to//g'` cyan `grep -h  it=20 jobs/baseline_ec_tower*|sed 's/Exported checkpoint to//g'` --export figures/tower_hits_ws.png --noTime --title "Tower building" --baseline enum 8 NS 10  -i 20
python bin/graphs.py --checkpoints teal `grep  -h it=20.*graph jobs/tower_batch_50_3600*` orange `grep  -h it=20.*graph jobs/tower_no_dsl_batch_50_3600*` purple `grep  -h it=20.*pickle jobs/tower_no_recognition_batch_50_3600*|sed 's/Exported checkpoint to//g'` cyan `grep -h  it=20.*graph jobs/tower_memorize_ellisk*|sed 's/Exported checkpoint to//g'` --export figures/tower_memorize_talk_hits_ws.png --noTime --title "Tower building" --baseline enum 8 NS 10  -i 20
python bin/graphs.py --checkpoints teal `grep  -h it=20.*graph jobs/tower_batch_50_3600*` orange `grep  -h it=20.*graph jobs/tower_memorize_ellisk*` purple `grep  -h it=20.* jobs/tower_memorize_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'`  --export figures/tower_memorize.png --noTime --title "Tower building"  --noTime --averageColors -c  -i 20


### Regex figures - time and hits split

### Regex figures - likelihoods
python bin/graphs.py --checkpoints teal `grep  -h it=6.*graph jobs/regex_b40_strConst_7m_ellisk_2019-02-27*` --cutoff gt --noTime --export figures/regex_7m_gt.png
python bin/graphs.py --checkpoints teal `grep  -h it=6.*graph jobs/regex_b40_strConst_7m_ellisk_2019-02-27*` --cutoff bigram --noTime --export figures/regex_7m_bigram.png
python bin/graphs.py --checkpoints teal `grep  -h it=6.*graph jobs/regex_b40_strConst_7m_ellisk_2019-02-27*` --likelihood maximum --noTime --export figures/regex_7m_maximum.png -i 6
python bin/graphs.py --checkpoints teal `grep  -h it=6.*graph jobs/regex_b40_strConst_7m_ellisk_2019-02-27*` --likelihood task --noTime --export figures/regex_7m_marginal.png -i 6
python bin/graphs.py --checkpoints teal `grep  -h it=10.*graph jobs/regex_b40_strConst_2m_ellisk_2019-02-*` --cutoff gt --noTime --export figures/regex_2m_gt.png
python bin/graphs.py --checkpoints teal `grep  -h it=10.*graph jobs/regex_b40_strConst_2m_ellisk_2019-02-*` --cutoff bigram --noTime --export figures/regex_2m_bigram.png
python bin/graphs.py --checkpoints teal `grep  -h it=10.*graph jobs/regex_b40_strConst_2m_ellisk_2019-02-*` --likelihood maximum --noTime --export figures/regex_2m_maximum.png -i 10
python bin/graphs.py --checkpoints teal `grep  -h it=10.*graph jobs/regex_b40_strConst_2m_ellisk_2019-02-*` --likelihood task --noTime --export figures/regex_2m_marginal.png -i 10

python bin/graphs.py --checkpoints teal `grep  -h it=10.*graph jobs/regex_b10_strConst_2m_ellisk_*` orange `grep  -h it=10.*graph jobs/regex_b10_strConst_2m_no_dsl_*` --cutoff gt --noTime --export figures/regex_2m_b10_gt.png
python bin/graphs.py --checkpoints teal `grep  -h it=10.*graph jobs/regex_b10_strConst_2m_ellisk_*` orange `grep  -h it=10.*graph jobs/regex_b10_strConst_2m_no_dsl_*` --cutoff bigram --noTime --export figures/regex_2m_b10_bigram.png
python bin/graphs.py --checkpoints teal `grep  -h it=10.*graph jobs/regex_b10_strConst_2m_ellisk_*` orange `grep  -h it=10.*graph jobs/regex_b10_strConst_2m_no_dsl_*` --likelihood maximum --noTime --export figures/regex_2m_b10_maximum.png -i 10
python bin/graphs.py --checkpoints teal `grep  -h it=10.*graph jobs/regex_b10_strConst_2m_ellisk_*` orange `grep  -h it=10.*graph jobs/regex_b10_strConst_2m_no_dsl_*` --likelihood task --noTime --export figures/regex_2m_b10_marginal.png -i 10

python bin/graphs.py --checkpoints teal `grep  -h it=9.*graph jobs/regex_b10_strConst_7m_no_mask_ellisk*` orange `grep  -h it=5.*graph jobs/regex_b10_strConst_7m_no_mask_no_dsl_ellisk*` --cutoff gt --noTime --export figures/regex_7m_b10_no_mask_gt.png -i 10

python bin/graphs.py --checkpoints teal `grep  -h it=10.*graph jobs/regex_b10_strConst_7m_ellisk*` orange `grep  -h it=10.*graph jobs/regex_b10_strConst_7m_no_dsl*` --cutoff gt --noTime --export figures/regex_7m_b10_gt.png
python bin/graphs.py --checkpoints teal `grep  -h it=10.*graph jobs/regex_b10_strConst_7m_ellisk*` orange `grep  -h it=10.*graph jobs/regex_b10_strConst_7m_no_dsl*` --cutoff bigram --noTime --export figures/regex_7m_b10_bigram.png
python bin/graphs.py --checkpoints teal `grep  -h it=10.*graph jobs/regex_b10_strConst_7m_ellisk*` orange `grep  -h it=10.*graph jobs/regex_b10_strConst_7m_no_dsl*` --likelihood maximum --noTime --export figures/regex_7m_b10_maximum.png -i 10
python bin/graphs.py --checkpoints teal `grep  -h it=10.*graph jobs/regex_b10_strConst_7m_ellisk*` orange `grep  -h it=10.*graph jobs/regex_b10_strConst_7m_no_dsl*` --likelihood task --noTime --export figures/regex_7m_b10_marginal.png -i 10

### Comparing masked vs full recognition model
python bin/graphs.py --checkpoints `grep  -h 'it=20.*graph=True.pickle' jobs/list_no_generative_mask*` `grep  -h 'it=20.*graph=True.pickle' jobs/list_no_generative_ellisk*` `grep  -h 'it=20.*graph=True.pickle' jobs/list_ellisk_2019-01-25T**` `grep -h  it=20.*graph.*pickle jobs/list_mask_ellisk_2019-02-07T11.1*` --title "List, mask RM" --export figures/list_mask.png  --names "no dsl (mask),no dsl,dc,dc (mask)" --colors green green green orange orange orange blue blue blue blue blue  red red red --noTime
python bin/graphs.py --checkpoints  `grep  -h 'it=20.*graph=True.pickle' jobs/text_ellisk_2019-01-2*|head  -n 5` `grep  -h 'it=20.*graph=True.pickle' jobs/text_mask_ellisk_2019-02-07T11.*` `grep  -h 'it=20.*graph.*pickle'  jobs/text_no_generative_ellisk_2019-01-30T15.*`  --export figures/text_mask.png --colors orange orange orange orange orange  purple purple purple green green green --names "dc,dc+mask,dc-dsl"

### List w/ small amount of training data
python bin/graphs.py --testingTimeout 600 --title "List, 75/25 test/train" --export figures/list_small_train.png --checkpoints teal `grep  -h 'it=10.*graph.*pickle' jobs/list_small_train_ellisk*` orange `grep  -h 'it=10.*graph.*pickle' jobs/list_small_train_no_dsl_ellisk*` --names "dc,dc-dsl"
### List w/ tiny amount of training data
python bin/graphs.py --testingTimeout 600 --title "List, 85/15 test/train" --export figures/list_tiny_train.png --checkpoints teal `grep  -h 'it=10.*graph.*pickle' jobs/list_tiny_train_ellisk*` orange `grep  -h 'it=10.*graph.*pickle' jobs/list_tiny_train_no_dsl_ellisk*`  --names "dc,dc-dsl"
### List w/ large amount of training data
python bin/graphs.py --checkpoints `grep  -h 'it=20.*graph=True.pickle' jobs/list_ellisk_2019-01-25T*` `grep  -h 'it=20.*graph=True.pickle' jobs/list_no_dsl_no_ensemble_ellisk_2019-01-28T1*` `grep  -h 'it=14.*graph=True.pickle' jobs/list_no_dsl_no_ensemble_no_auxiliary_ellisk_2019-01-29T10.*` --title "List, 50/50 test/train" --export figures/list_big_train.png --colors orange orange orange orange orange purple purple purple purple purple --testingTimeout 600

### Regular expressions
python bin/graphs.py --title "Regular expressions" --export figures/regex.png --checkpoints `grep -h  it=20.*graph=True jobs/regex_batch_40_mask_ellisk_2019-02-06T14.*` `grep  -h it=20.*graph jobs/regex_no_dsl_batch_40_mask*` `grep  -h it=10.*graph jobs/regex_2h_mask*` --colors orange orange orange orange orange purple purple purple purple purple green --names "dc,dc (no dsl),dc (no batch)" --noTime --alpha 0.75

### Regex, for josh
python bin/graphs.py --title "Regular expressions" --export regex_baselines.png --checkpoints orange `grep -h 'it=10.*graph=True' ../ec_aws_logs/jobs/regex_batch_40_reduced_mask_mnye_2019-02-12T1*` purple `grep -h 'it=10.*graph=True' ../ec_aws_logs/jobs/regex_batch_40_reduced_bigram_mnye_2019-02-12T1*` green `grep -h 'it=10.*graph=True' ../ec_aws_logs/jobs/regex_batch_40_reduced_unigram_mnye_2019-02-12T1*` blue `grep -h 'it=10.*graph=True' ../ec_aws_logs/jobs/regex_batch_40_reduced_nodsl_mask_mnye_2019-02-12T1*` --noTime --alpha 0.75 #--names "mask (hybrid),bigram,unigram,nodsl (mask, hybrid)"

### Expertise figures
python ~/projects/ec/graphs.py --x-label "" -i 20 --testingTimeout 600 --title "List processing" --export figures/list_no_recognition_solved.png --checkpoints `grep  -h 'it=20.*graph=True.pickle' jobs/list_ellisk_2019-01-25T*|head  -n 5` `grep  -h 'it=20.*.pickle' jobs/list_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'`   --colors teal teal teal teal orange  orange  orange --noTime 
python ~/projects/ec/graphs.py --x-label "Iterations (learning time)" -i 20 --testingTimeout 600  --export figures/list_no_recognition_time.png --checkpoints `grep  -h 'it=20.*graph=True.pickle' jobs/list_ellisk_2019-01-25T*|head  -n 5` `grep  -h 'it=20.*.pickle' jobs/list_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'`   --colors teal teal teal teal orange  orange  orange --onlyTime 
python ~/projects/ec/graphs.py --x-label "" -i 20 --testingTimeout 600 --title "Text editing" --export figures/text_no_recognition_solved.png --checkpoints `grep  -h 'it=20.*graph=True.pickle' jobs/text_ellisk_2019-01-2*|head  -n 5` `grep  -h 'it=20.*.pickle' jobs/text_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'`   --colors teal teal teal teal teal orange  orange  orange  --noTime 
python ~/projects/ec/graphs.py --x-label "Iterations (learning time)" -i 20 --testingTimeout 600  --export figures/text_no_recognition_time.png --checkpoints `grep  -h 'it=20.*graph=True.pickle' jobs/text_ellisk_2019-01-2*|head  -n 5` `grep  -h 'it=20.*.pickle' jobs/text_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'`   --colors teal teal teal teal orange  orange  orange  --onlyTime 
python ~/projects/ec/graphs.py --x-label "" -i 12 --testingTimeout 600 --title "LOGO Graphics"  --epochFrequency 2 --colors teal orange --checkpoints /home/ellisk/om/ec/experimentOutputs/logo/2018-12-23T03:17:36.165981/logo_aic=1.0_arity=3_BO=True_CO=True_ET=3600_HR=0.5_it=12_MF=5_baseline=False_pc=30.0_RT=1800_RR=False_RW=False_storeTask=True_L=1.5_batch=40_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=LogoFeatureCNN_graph=True.pickle /home/ellisk/om/ec/experimentOutputs/logo/2019-01-01T00:44:56.828323/logo_aic=1.0_arity=3_ET=3600_it=20_MF=5_baseline=False_pc=30.0_RW=False_storeTask=False_L=1.5_batch=40_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=False.pickle --export figures/logo_no_recognition_solved.png --noTime
python ~/projects/ec/graphs.py --x-label "Iterations (learning time)" -i 12 --testingTimeout 600  --epochFrequency 2 --colors teal orange --checkpoints /home/ellisk/om/ec/experimentOutputs/logo/2018-12-23T03:17:36.165981/logo_aic=1.0_arity=3_BO=True_CO=True_ET=3600_HR=0.5_it=12_MF=5_baseline=False_pc=30.0_RT=1800_RR=False_RW=False_storeTask=True_L=1.5_batch=40_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=LogoFeatureCNN_graph=True.pickle /home/ellisk/om/ec/experimentOutputs/logo/2019-01-01T00:44:56.828323/logo_aic=1.0_arity=3_ET=3600_it=20_MF=5_baseline=False_pc=30.0_RW=False_storeTask=False_L=1.5_batch=40_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=False.pickle --export figures/logo_no_recognition_time.png --onlyTime 

### Text - we should add no DSL baseline to this
python bin/graphs.py -i 11 --testingTimeout 600 --title "Text editing" --export figures/text.png --checkpoints teal `grep  -h 'it=20.*graph=True.pickle' jobs/text_ellisk_2019-01-2*|head  -n 5` orange `grep  -h 'it=20.*.pickle' jobs/text_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'` purple `grep  -h it=20.*graph=True jobs/text_no_dsl_no_ensemble   --colors teal teal teal teal teal orange  orange  orange  --noTime

### Comparing full model to ablations
python bin/graphs.py -i 12 --testingTimeout 600 --title "LOGO Graphics"  --epochFrequency 2 --interval --failAsTimeout --checkpoints /home/ellisk/om/ec/experimentOutputs/logo/2018-12-23T03:17:36.165981/logo_aic=1.0_arity=3_BO=True_CO=True_ET=3600_HR=0.5_it=12_MF=5_baseline=False_pc=30.0_RT=1800_RR=False_RW=False_storeTask=True_L=1.5_batch=40_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=LogoFeatureCNN_graph=True.pickle /home/ellisk/om/ec/experimentOutputs/logo/2019-01-01T00:44:56.828323/logo_aic=1.0_arity=3_ET=3600_it=20_MF=5_baseline=False_pc=30.0_RW=False_storeTask=False_L=1.5_batch=40_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=False.pickle /home/ellisk/om/ec/experimentOutputs/logo/2019-01-01T21:58:32.845270/logo_aic=1000.0_arity=3_BO=True_CO=True_ET=3600_HR=0.5_it=12_MF=5_baseline=False_pc=30.0_RT=1800_RR=False_RW=False_storeTask=True_L=1.5_batch=40_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=LogoFeatureCNN_graph=True.pickle --export ../ecPaper/figures/logoLearningCurve_batch.png

python bin/graphs.py --testingTimeout 600 --title "Text editing"  --interval --failAsTimeout --checkpoints /home/ellisk/om/ec/experimentOutputs/text_aic=1.0_arity=3_BO=True_CO=True_ET=720_HR=0.5_it=20_MF=5_baseline=False_pc=30.0_RT=7200_RW=False_storeTask=True_L=1.0_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=LearnedFeatureExtractor_graph=True.pickle /home/ellisk/om/ec/experimentOutputs/text_aic=1.0_arity=3_ET=720_it=20_MF=5_baseline=False_pc=30.0_RW=False_storeTask=False_L=1.0_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=False.pickle  /home/ellisk/om/ec/experimentOutputs/text_aic=1000.0_arity=3_BO=True_CO=True_ET=720_HR=0.5_it=20_MF=5_baseline=False_pc=30.0_RT=7200_RR=False_RW=False_storeTask=True_L=1.0_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=LearnedFeatureExtractor_graph=True.pickle --export ../ecPaper/figures/textLearningCurve_batch.png

python bin/graphs.py --testingTimeout 600 --title "List processing"  --interval --failAsTimeout --checkpoints /home/ellisk/om/ec/experimentOutputs/list_aic=1.0_arity=3_BO=True_CO=True_ET=720_HR=0.5_it=20_MF=5_baseline=False_pc=30.0_RT=7200_RW=False_storeTask=True_L=1.0_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=LearnedFeatureExtractor_graph=True.pickle  /home/ellisk/om/ec/experimentOutputs/list_aic=1.0_arity=3_ET=720_it=20_MF=5_baseline=False_pc=30.0_RW=False_storeTask=False_L=1.0_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=False.pickle /home/ellisk/om/ec/experimentOutputs/list_aic=1000.0_arity=3_BO=True_CO=True_ET=720_HR=0.5_it=20_MF=5_baseline=False_pc=30.0_RT=7200_RR=False_RW=False_storeTask=True_L=1.0_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=LearnedFeatureExtractor_graph=True.pickle --export ../ecPaper/figures/listLearningCurve_batch.png

python bin/graphs.py --testingTimeout 600 --title "Tower building" --interval --failAsTimeout --checkpoints /home/ellisk/om/ec/experimentOutputs/tower_aic=1.0_arity=3_BO=True_CO=True_ET=120_HR=0.5_it=20_MF=5_baseline=False_pc=30.0_RT=300_RR=True_RW=False_storeTask=True_L=1.0_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=TowerCNN_graph=True.pickle /home/ellisk/om/ec/experimentOutputs/towers/2019-01-08T15:08:42.341376/tower_aic=1.0_arity=3_ET=120_it=20_MF=5_baseline=False_pc=30.0_RW=False_storeTask=True_L=1.0_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=False.pickle /home/ellisk/om/ec/experimentOutputs/towers/2019-01-08T15:06:41.057543/tower_aic=1000.0_arity=3_BO=True_CO=True_ET=120_HR=0.5_it=20_MF=5_baseline=False_pc=30.0_RT=3600_RR=False_RW=False_storeTask=True_L=1.0_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=TowerCNN_graph=True.pickle --export ../ecPaper/figures/towerLearningCurve_batch.png

python bin/graphs.py --testingTimeout 600 --title "Tower building, new primitives" --showEpochs --interval --failAsTimeout --checkpoints /home/ellisk/om/ec/experimentOutputs/towers/2019-01-05T19:28:09.239650/tower_aic=1.0_arity=3_BO=True_CO=True_ET=120_HR=0.5_it=19_MF=5_baseline=False_pc=30.0_RT=3600_RR=False_RW=False_storeTask=True_L=1.0_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=TowerCNN_graph=True.pickle /home/ellisk/om/ec/experimentOutputs/towers/2019-01-05T19:34:18.595339/tower_aic=1.0_arity=3_ET=120_it=20_MF=5_baseline=False_pc=30.0_RW=False_storeTask=True_L=1.0_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=False.pickle /home/ellisk/om/ec/experimentOutputs/towers/2019-01-05T19:36:21.603299/tower_aic=1000.0_arity=3_BO=True_CO=True_ET=120_HR=0.5_it=13_MF=5_baseline=False_pc=30.0_RT=3600_RR=False_RW=False_storeTask=True_L=1.0_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=TowerCNN_graph=True.pickle --export /tmp/tower.png

python bin/graphs.py --testingTimeout 600 --title "Symbolic regression" --interval --checkpoints  /home/ellisk/om/ec/experimentOutputs/rational_aic=1.0_arity=3_BO=True_CO=True_ET=120_HR=0.5_it=20_MF=5_baseline=False_pc=30.0_RT=3600_RR=False_RW=False_storeTask=True_L=1.0_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=FeatureExtractor_graph=True.pickle  /home/ellisk/om/ec/experimentOutputs/rational_aic=1.0_arity=3_ET=120_it=20_MF=5_baseline=False_pc=30.0_RW=False_STM=True_L=1.0_batch=10_TRR=randomShuffle_K=2_topkNotMAP=False_rec=False.pickle  /home/ellisk/om/ec/experimentOutputs/rational_aic=1000.0_arity=3_BO=True_CO=True_ET=120_HR=0.5_it=20_MF=5_baseline=False_pc=30.0_RT=3600_RR=False_RW=False_storeTask=True_L=30.0_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=FeatureExtractor_graph=True.pickle  --export ../ecPaper/figures/rationalLearningCurve_batch.png  -i 20 --epochFrequency 3

python bin/graphs.py --title "Generative Regexes" --checkpoints /home/ellisk/om/ec/experimentOutputs/regex/2019-01-11T21:52:14.607041/regex_aic=1.0_arity=3_BO=True_CO=True_ET=1800_HR=0.5_it=20_MR=32_MF=10_baseline=False_pc=30.0_RT=1800_RR=True_RW=False_STM=True_L=1.0_batch=40_TRR=randomShuffle_K=10_topkNotMAP=True_rec=True_feat=LearnedFeatureExtractor_graph=True.pickle ~/om/ec/experimentOutputs/regex/regex_aic=1.0_arity=3_BO=True_CO=True_ET=1800_HR=0.5_it=20_MF=10_baseline=False_pc=30.0_RT=1800_RR=True_RW=False_storeTask=False_L=1.0_batch=40_taskReranker=randomShuffle_K=10_topkNotMAP=True_rec=True_feat=LearnedFeatureExtractor_graph=True.pickle --names "bigram (rank 32),bigram" --export ../ecPaper/figures/regexLearningCurve_batch.png

#### Comparing old and new recognition models
python bin/graphs.py --testingTimeout 600 --title "List, recognition model only" --interval --checkpoints /home/ellisk/om/ec/experimentOutputs/list_aic=1.0_arity=3_ET=720_it=20_MF=5_baseline=False_pc=30.0_RW=False_storeTask=False_L=1.0_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=False.pickle /home/ellisk/om/ec/experimentOutputs/list_aic=1000.0_arity=3_BO=False_CO=False_ET=720_HR=0.5_it=20_MF=5_baseline=False_pc=30.0_RT=7200_RR=False_RW=False_storeTask=True_L=1.0_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=LearnedFeatureExtractor_graph=True.pickle -i 20 --export ../ecPaper/figures/list_recognition_comparison.png

python bin/graphs.py --testingTimeout 600 --title "Text, recognition model only" --interval --checkpoints /home/ellisk/om/ec/experimentOutputs/text_aic=1000.0_arity=3_BO=True_CO=True_ET=720_HR=0.5_it=20_MF=5_baseline=False_pc=30.0_RT=7200_RR=False_RW=False_storeTask=True_L=1.0_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=LearnedFeatureExtractor_graph=True.pickle /home/ellisk/om/ec/experimentOutputs/text_aic=1000.0_arity=3_BO=False_CO=False_ET=720_HR=0.5_it=20_MF=5_baseline=False_pc=30.0_RT=7200_RR=False_RW=False_storeTask=True_L=1.0_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=LearnedFeatureExtractor_graph=True.pickle -i 20 --export ../ecPaper/figures/text_recognition_comparison.png

### Batching figures
python bin/graphs.py --testingTimeout 600 --title "LOGO, varying batch size" --checkpoints /home/ellisk/om/ec/experimentOutputs/logo/2018-12-17T20:58:48.581674/logo_aic=1.0_arity=3_BO=True_CO=True_ET=7200_HR=0.5_it=7_MF=5_baseline=False_pc=30.0_RT=3600_RR=False_RW=False_storeTask=False_L=1.5_taskReranker=default_K=2_topkNotMAP=False_rec=True_feat=LogoFeatureCNN_graph=True.pickle /home/ellisk/om/ec/experimentOutputs/logo/2018-12-23T03:17:36.165981/logo_aic=1.0_arity=3_BO=True_CO=True_ET=3600_HR=0.5_it=12_MF=5_baseline=False_pc=30.0_RT=1800_RR=False_RW=False_storeTask=True_L=1.5_batch=40_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=LogoFeatureCNN_graph=True.pickle /home/ellisk/om/ec/experimentOutputs/logo/2018-12-20T16:34:58.729116/logo_aic=1.0_arity=3_BO=True_CO=True_ET=1800_HR=0.5_it=30_MF=5_baseline=False_pc=30.0_RT=1800_RR=False_RW=False_storeTask=True_L=1.5_batch=20_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=LogoFeatureCNN_graph=True.pickle  /home/ellisk/om/ec/experimentOutputs/logo/2018-12-20T03:32:41.459119/logo_aic=1.0_arity=3_BO=True_CO=True_ET=1440_HR=0.5_it=20_MF=5_baseline=False_pc=30.0_RT=1800_RR=False_RW=False_storeTask=True_L=1.5_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=LogoFeatureCNN_graph=True.pickle /home/ellisk/om/ec/experimentOutputs/logo/2019-01-07T16:31:32.509029/logo_aic=1.0_arity=3_BO=True_CO=True_ET=3600_HR=0.5_it=9_MF=5_baseline=False_pc=30.0_RT=1800_RR=False_RW=False_storeTask=True_L=1.0_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=LogoFeatureCNN_graph=True.pickle --names "no batch,40,20,10,10 (1h)" --export ../ecPaper/figures/logo_batch_size.png --noTime
python bin/graphs.py --testingTimeout 600 --showTraining --title "LOGO, varying batch size (training)" --checkpoints /home/ellisk/om/ec/experimentOutputs/logo/2018-12-17T20:58:48.581674/logo_aic=1.0_arity=3_BO=True_CO=True_ET=7200_HR=0.5_it=7_MF=5_baseline=False_pc=30.0_RT=3600_RR=False_RW=False_storeTask=False_L=1.5_taskReranker=default_K=2_topkNotMAP=False_rec=True_feat=LogoFeatureCNN_graph=True.pickle /home/ellisk/om/ec/experimentOutputs/logo/2018-12-23T03:17:36.165981/logo_aic=1.0_arity=3_BO=True_CO=True_ET=3600_HR=0.5_it=12_MF=5_baseline=False_pc=30.0_RT=1800_RR=False_RW=False_storeTask=True_L=1.5_batch=40_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=LogoFeatureCNN_graph=True.pickle /home/ellisk/om/ec/experimentOutputs/logo/2018-12-20T16:34:58.729116/logo_aic=1.0_arity=3_BO=True_CO=True_ET=1800_HR=0.5_it=30_MF=5_baseline=False_pc=30.0_RT=1800_RR=False_RW=False_storeTask=True_L=1.5_batch=20_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=LogoFeatureCNN_graph=True.pickle  /home/ellisk/om/ec/experimentOutputs/logo/2018-12-20T03:32:41.459119/logo_aic=1.0_arity=3_BO=True_CO=True_ET=1440_HR=0.5_it=20_MF=5_baseline=False_pc=30.0_RT=1800_RR=False_RW=False_storeTask=True_L=1.5_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=LogoFeatureCNN_graph=True.pickle /home/ellisk/om/ec/experimentOutputs/logo/2019-01-07T16:31:32.509029/logo_aic=1.0_arity=3_BO=True_CO=True_ET=3600_HR=0.5_it=9_MF=5_baseline=False_pc=30.0_RT=1800_RR=False_RW=False_storeTask=True_L=1.0_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=LogoFeatureCNN_graph=True.pickle --names "no batch,40,20,10,10 (1h)" --export ../ecPaper/figures/logo_batch_size_training.png --noTime

python bin/graphs.py --testingTimeout 600 --title "Text, varying batch size" --checkpoints /home/ellisk/om/ec/experimentOutputs/text_aic=1.0_arity=3_BO=True_CO=True_ET=7200_HR=0.5_it=7_MF=5_baseline=False_pc=30.0_RT=7200_RW=False_storeTask=True_L=1.0_taskReranker=default_K=2_topkNotMAP=False_rec=True_feat=LearnedFeatureExtractor_graph=True.pickle /home/ellisk/om/ec/experimentOutputs/text_aic=1.0_arity=3_BO=True_CO=True_ET=3600_HR=0.5_it=12_MF=5_baseline=False_pc=30.0_RT=7200_RR=False_RW=False_storeTask=True_L=1.0_batch=40_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=LearnedFeatureExtractor_graph=True.pickle /home/ellisk/om/ec/experimentOutputs/text_aic=1.0_arity=3_BO=True_CO=True_ET=720_HR=0.5_it=20_MF=5_baseline=False_pc=30.0_RT=7200_RW=False_storeTask=True_L=1.0_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=LearnedFeatureExtractor_graph=True.pickle --names "no batch,40,10" --export ../ecPaper/figures/text_batch_size.png


# Visualizing program embeddings
python bin/taskRankGraphs.py --checkpoints /home/ellisk/om/ec/experimentOutputs/tower_aic=1.0_arity=3_BO=True_CO=True_ET=120_HR=0.5_it=20_MF=5_baseline=False_pc=30.0_RT=300_RR=True_RW=False_storeTask=True_L=1.0_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=TowerCNN_graph=True.pickle --labelWithImages 1 --metricsToCluster expectedProductionUses

python bin/taskRankGraphs.py --checkpoints /home/ellisk/om/ec/experimentOutputs/tower_aic=1.0_arity=3_BO=True_CO=True_ET=120_HR=0.5_it=20_MF=5_baseline=False_pc=30.0_RT=300_RR=True_RW=False_storeTask=True_L=1.0_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=TowerCNN_graph=True.pickle --labelWithImages 1 --metricsToCluster expectedProductionUses

python bin/taskRankGraphs.py --checkpoints experimentOutputs/logo/2018-12-17T20:58:48.581674/logo_aic=1.0_arity=3_BO=True_CO=True_ET=7200_HR=0.5_it=7_MF=5_baseline=False_pc=30.0_RT=3600_RR=False_RW=False_storeTask=False_L=1.5_taskReranker=default_K=2_topkNotMAP=False_rec=True_feat=LogoFeatureCNN_graph=True.pickle --labelWithImages 1 --metricsToCluster expectedProductionUses
	# Cathy: try changing the tsne learning rate if you haven't already; recommend 350.0 for this one.

python bin/taskRankGraphs.py --checkpoints experimentOutputs/logo_aic=1.0_arity=3_BO=True_CO=True_ET=7200_HR=0.5_it=7_MF=5_baseline=False_pc=30.0_RT=3600_RR=False_RW=False_storeTask=False_L=1.5_taskReranker=default_K=2_topkNotMAP=False_rec=True_feat=LogoFeatureCNN_FTM=True_graph=True.pickle --labelWithImages 1 --metricsToCluster expectedProductionUses

Rational, Feature vectors.
python bin/taskRankGraphs.py --checkpoints /home/ellisk/om/ec/experimentOutputs/rational_aic=1.0_arity=3_BO=True_CO=True_ET=120_HR=0.5_it=20_MF=5_baseline=False_pc=30.0_RT=3600_RR=False_RW=False_storeTask=True_L=1.0_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=FeatureExtractor_graph=True.pickle --labelWithImages 1 --metricsToCluster contextualLogProductions --tsneLearningRate 300

# Visualizing confusion matrices
./container.img python synergyStatistics.py /home/ellisk/om/ec/experimentOutputs/rational_aic=1.0_arity=3_BO=True_CO=True_ET=120_HR=0.5_it=20_MF=5_baseline=False_pc=30.0_RT=3600_RR=False_RW=False_storeTask=True_L=1.0_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=FeatureExtractor_graph=True.pickle /home/ellisk/om/ec/experimentOutputs/rational_aic=1.0_arity=3_ET=120_it=20_MF=5_baseline=False_pc=30.0_RW=False_STM=True_L=1.0_batch=10_TRR=randomShuffle_K=2_topkNotMAP=False_rec=False.pickle /home/ellisk/om/ec/experimentOutputs/rational_aic=1000.0_arity=3_BO=True_CO=True_ET=120_HR=0.5_it=14_MF=5_baseline=False_pc=30.0_RT=3600_RR=False_RW=False_storeTask=True_L=30.0_batch=10_taskReranker=randomShuffle_K=2_topkNotMAP=False_rec=True_feat=FeatureExtractor_graph=True.pickle

### ~deep program learning~
python bin/tower.py  --primitive-graph `grep -h  it=20.*graph jobs/tower_batch_40_3600_ellisk_2019-0*` # batching
python bin/tower.py  --primitive-graph experimentOutputs/towers/2019-02-12T22:26:56.272150/tower_aic=1.0_arity=3_BO=True_CO=True_ES=1_ET=3600_HR=0.5_it=10_MF=5_pc=30.0_RS=5000_RT=3600_RR=False_RW=False_STM=True_L=1.5_TRR=default_K=2_topkNotMAP=False_graph=True.pickle # no batching

python bin/logo.py --primitive-graph `grep  -h it=20.*graph jobs/logo_batch_50*` # batching

python bin/list.py --primitive-graph `grep  -h it=20.*graph jobs/list_hard_test_ellisk*` # batching
python bin/list.py --primitive-graph experimentOutputs/list_aic=1.0_arity=4_ET=11640_expandFrontier=4.0_it=4_MF=5_baseline=False_pc=30.0_RW=False_storeTask=False_L=1.0_taskReranker=default_K=2_topkNotMAP=False_rec=False.pickle `grep  -h it=20.*graph jobs/list_hard_test_ellisk*` /om2/user/zyzzyva/ec/experimentOutputs/list_aic=1.0_arity=3_BO=True_CO=True_ET=7200_HR=0.5_it=7_MF=10_baseline=False_pc=10.0_RT=3600_RW=False_storeTask=True_L=2.0_taskReranker=default_K=2_topkNotMAP=False_rec=True_feat=LearnedFeatureExtractor_graph=True.pickle # both batching and not batching and then McCarthy

python bin/text.py --primitive-graph `grep  -h it=20.*graph jobs/text_ellisk_2019-01-2*`

python bin/rational.py --primitive-graph `grep  -h it=20.*graph jobs/rational_batch*`

python bin/regexes.py --primitive-graph `grep  -h it=20.*graph jobs/regex_b40_str_red_12m_mnye_2019-03*`

python bin/scientificLaws.py --primitive-graph `grep  -h it=.*graph jobs/scientific_unsolved_4h*`

python bin/list.py --primitive-graph experimentOutputs/list_aic=1.0_arity=4_ET=11640_expandFrontier=4.0_it=4_MF=5_baseline=False_pc=30.0_RW=False_storeTask=False_L=1.0_taskReranker=default_K=2_topkNotMAP=False_rec=False.pickle

# this will show you the correlation between depth and accuracy
python bin/analyzeDepth.py  --recognition `grep -h  it=20.*graph jobs/tower_batch_50_3600*` `grep  -h it=20.*graph jobs/logo_batch_50*` `grep  -h it=20.*graph jobs/list_hard_test_ellisk*` `grep  -h it=20.*graph jobs/text_ellisk_2019-01-2*` `grep  -h it=20.*graph jobs/rational_batch*`  --generative `grep  -h 'it=20.*.pickle' jobs/text_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'` `grep  -h 'it=20.*.pickle' jobs/list_hard_test_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'` `grep  -h it=20.*pickle jobs/logo_no_recognition_batch_50_1h*|sed 's/Exported checkpoint to//g'` `grep  -h it=20.*pickle jobs/rational_noRecognition*|sed 's/Exported checkpoint to//g'` `grep  -h it=20.*pickle jobs/tower_no_recognition_batch_50_3600*|sed 's/Exported checkpoint to//g'` --legend MEAN

python bin/analyzeDepth.py  --recognition `grep -h  it=20.*graph jobs/tower_batch_50_3600*` `grep  -h it=20.*graph jobs/logo_batch_50*` `grep  -h it=20.*graph jobs/list_hard_test_ellisk*` `grep  -h it=20.*graph jobs/text_ellisk_2019-01-2*` `grep  -h it=20.*graph jobs/rational_batch*`  --generative `grep  -h 'it=20.*.pickle' jobs/text_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'` `grep  -h 'it=20.*.pickle' jobs/list_hard_test_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'` `grep  -h it=20.*pickle jobs/logo_no_recognition_batch_50_1h*|sed 's/Exported checkpoint to//g'` `grep  -h it=20.*pickle jobs/rational_noRecognition*|sed 's/Exported checkpoint to//g'` `grep  -h it=20.*pickle jobs/tower_no_recognition_batch_50_3600*|sed 's/Exported checkpoint to//g'` --every