Environment:
	Python: 3.6.13
	PyTorch: 1.4.0
	Torchvision: 0.5.0
	CUDA: 10.1
	CUDNN: 7603
	NumPy: 1.19.5
	PIL: 8.1.0
Args:
	algorithm: IRM
	batch_size: 8
	data: /data/GQA/MetaDataset-subpopulation-shift
	hparams: None
	hparams_seed: 0
	log_prefix: 
	num_classes: 2
	num_domains: 2
	output_dir: train_output
	save_model_every_checkpoint: False
	seed: 0
	skip_model_save: False
	workers: 4
train_dataset.samples reverse: [('cat(indoor)', 750), ('dog(outdoor)', 750), ('cat(outdoor)', 100), ('dog(indoor)', 100)]
self.domain_to_groups {0: {'cat': ['cat(indoor)'], 'dog': ['dog(indoor)']}, 1: {'cat': ['cat(outdoor)'], 'dog': ['dog(outdoor)']}}
HParams:
	batch_size: 32
	class_balanced: False
	data_augmentation: True
	irm_lambda: 100.0
	irm_penalty_anneal_iters: 500
	lr: 5e-05
	nonlinear_classifier: False
	resnet18: True
	resnet_dropout: 0.0
	weight_decay: 0.0
step_vals {'loss': 0.8120059370994568, 'nll': 0.7828301191329956, 'penalty': 0.029175810515880585}
Iteration: 0
out-of-domain val
accuracy 0.578 	 roc_auc_score 0.677
confusion_matrix
[[ 65 223]
 [ 20 268]]
classification_report
              precision    recall  f1-score   support

           0       0.76      0.23      0.35       288
           1       0.55      0.93      0.69       288

    accuracy                           0.58       576
   macro avg       0.66      0.58      0.52       576
weighted avg       0.66      0.58      0.52       576

VAL * Acc@1 57.812
 * Acc@1 57.812 Acc@5 0.000
accuracy 0.931 	 size: 144 	 dog(indoor)
accuracy 0.931 	 size: 144 	 dog(outdoor)
accuracy 0.278 	 size: 144 	 cat(indoor)
accuracy 0.174 	 size: 144 	 cat(outdoor)
step_vals {'loss': 0.7039221525192261, 'nll': 0.6990245580673218, 'penalty': 0.004897603299468756}
step_vals {'loss': 0.6552651524543762, 'nll': 0.6567761301994324, 'penalty': -0.0015109960222616792}
step_vals {'loss': 0.48394858837127686, 'nll': 0.4632059931755066, 'penalty': 0.02074260078370571}
step_vals {'loss': 0.551364004611969, 'nll': 0.5444856882095337, 'penalty': 0.006878293585032225}
step_vals {'loss': 0.4928128719329834, 'nll': 0.4833628237247467, 'penalty': 0.009450063109397888}
step_vals {'loss': 0.45389044284820557, 'nll': 0.45264244079589844, 'penalty': 0.0012479908764362335}
step_vals {'loss': 0.5107627511024475, 'nll': 0.4942281246185303, 'penalty': 0.016534650698304176}
step_vals {'loss': 0.5247142910957336, 'nll': 0.5205217003822327, 'penalty': 0.004192613530904055}
step_vals {'loss': 0.5286956429481506, 'nll': 0.5090144276618958, 'penalty': 0.019681213423609734}
step_vals {'loss': 0.5242652893066406, 'nll': 0.5262757539749146, 'penalty': -0.002010437659919262}
step_vals {'loss': 0.3863545060157776, 'nll': 0.3651929497718811, 'penalty': 0.02116156369447708}
step_vals {'loss': 0.570439875125885, 'nll': 0.5613517761230469, 'penalty': 0.009088082239031792}
step_vals {'loss': 0.45016470551490784, 'nll': 0.44850873947143555, 'penalty': 0.0016559567302465439}
step_vals {'loss': 0.39689773321151733, 'nll': 0.39443933963775635, 'penalty': 0.002458407776430249}
step_vals {'loss': 0.4559664726257324, 'nll': 0.4742487668991089, 'penalty': -0.018282296136021614}
step_vals {'loss': 0.7417200207710266, 'nll': 0.7072091102600098, 'penalty': 0.034510932862758636}
step_vals {'loss': 0.4461424946784973, 'nll': 0.4362800419330597, 'penalty': 0.009862450882792473}
step_vals {'loss': 0.36099788546562195, 'nll': 0.3325475752353668, 'penalty': 0.028450313955545425}
step_vals {'loss': 0.5977588295936584, 'nll': 0.5898383259773254, 'penalty': 0.007920484989881516}
step_vals {'loss': 0.40518492460250854, 'nll': 0.4055653512477875, 'penalty': -0.000380428449716419}
Iteration: 20
out-of-domain val
accuracy 0.762 	 roc_auc_score 0.886
confusion_matrix
[[257  31]
 [106 182]]
classification_report
              precision    recall  f1-score   support

           0       0.71      0.89      0.79       288
           1       0.85      0.63      0.73       288

    accuracy                           0.76       576
   macro avg       0.78      0.76      0.76       576
weighted avg       0.78      0.76      0.76       576

VAL * Acc@1 76.215
 * Acc@1 76.215 Acc@5 0.000
accuracy 0.951 	 size: 144 	 cat(indoor)
accuracy 0.833 	 size: 144 	 cat(outdoor)
accuracy 0.785 	 size: 144 	 dog(outdoor)
accuracy 0.479 	 size: 144 	 dog(indoor)
step_vals {'loss': 0.4124167859554291, 'nll': 0.412641704082489, 'penalty': -0.0002249041572213173}
step_vals {'loss': 0.323260635137558, 'nll': 0.2902354598045349, 'penalty': 0.03302518650889397}
step_vals {'loss': 0.4790327847003937, 'nll': 0.4622421860694885, 'penalty': 0.016790607944130898}
step_vals {'loss': 0.4293856918811798, 'nll': 0.41879698634147644, 'penalty': 0.01058870367705822}
step_vals {'loss': 0.35495901107788086, 'nll': 0.33322200179100037, 'penalty': 0.021737009286880493}
step_vals {'loss': 0.5802249312400818, 'nll': 0.5965539216995239, 'penalty': -0.016329007223248482}
step_vals {'loss': 0.36918821930885315, 'nll': 0.3609113097190857, 'penalty': 0.008276915177702904}
step_vals {'loss': 0.33039653301239014, 'nll': 0.31876906752586365, 'penalty': 0.011627459898591042}
step_vals {'loss': 0.2836751341819763, 'nll': 0.26954033970832825, 'penalty': 0.014134795404970646}
step_vals {'loss': 0.5487698316574097, 'nll': 0.5359690189361572, 'penalty': 0.01280083879828453}
step_vals {'loss': 0.24400977790355682, 'nll': 0.22671549022197723, 'penalty': 0.017294282093644142}
step_vals {'loss': 0.3490923047065735, 'nll': 0.3422233462333679, 'penalty': 0.006868965458124876}
step_vals {'loss': 0.5111001133918762, 'nll': 0.5232506990432739, 'penalty': -0.012150607071816921}
step_vals {'loss': 0.3045438230037689, 'nll': 0.32303985953330994, 'penalty': -0.01849602349102497}
step_vals {'loss': 0.4489656388759613, 'nll': 0.4768804907798767, 'penalty': -0.02791486494243145}
step_vals {'loss': 0.38960063457489014, 'nll': 0.3785741925239563, 'penalty': 0.011026433669030666}
step_vals {'loss': 0.3092915415763855, 'nll': 0.3068642318248749, 'penalty': 0.0024273227900266647}
step_vals {'loss': 0.4141360819339752, 'nll': 0.42171400785446167, 'penalty': -0.00757793802767992}
step_vals {'loss': 0.4655948877334595, 'nll': 0.4833027422428131, 'penalty': -0.017707867547869682}
step_vals {'loss': 0.4294332265853882, 'nll': 0.4359581172466278, 'penalty': -0.006524900905787945}
Iteration: 40
out-of-domain val
accuracy 0.814 	 roc_auc_score 0.907
confusion_matrix
[[243  45]
 [ 62 226]]
classification_report
              precision    recall  f1-score   support

           0       0.80      0.84      0.82       288
           1       0.83      0.78      0.81       288

    accuracy                           0.81       576
   macro avg       0.82      0.81      0.81       576
weighted avg       0.82      0.81      0.81       576

VAL * Acc@1 81.424
 * Acc@1 81.424 Acc@5 0.000
accuracy 0.917 	 size: 144 	 dog(outdoor)
accuracy 0.910 	 size: 144 	 cat(indoor)
accuracy 0.778 	 size: 144 	 cat(outdoor)
accuracy 0.653 	 size: 144 	 dog(indoor)
step_vals {'loss': 0.2552010416984558, 'nll': 0.25096479058265686, 'penalty': 0.004236241336911917}
step_vals {'loss': 0.3589707911014557, 'nll': 0.3734935522079468, 'penalty': -0.014522750861942768}
step_vals {'loss': 0.3252408504486084, 'nll': 0.32311949133872986, 'penalty': 0.002121369354426861}
step_vals {'loss': 0.49592140316963196, 'nll': 0.49318039417266846, 'penalty': 0.00274099875241518}
step_vals {'loss': 0.3627893030643463, 'nll': 0.37053918838500977, 'penalty': -0.00774987880140543}
step_vals {'loss': 0.3197694718837738, 'nll': 0.33128973841667175, 'penalty': -0.011520267464220524}
step_vals {'loss': 0.3194872736930847, 'nll': 0.3035542368888855, 'penalty': 0.015933040529489517}
step_vals {'loss': 0.34213751554489136, 'nll': 0.3527294397354126, 'penalty': -0.010591918602585793}
step_vals {'loss': 0.3392382562160492, 'nll': 0.3564404249191284, 'penalty': -0.017202157527208328}
step_vals {'loss': 0.44447317719459534, 'nll': 0.42645201086997986, 'penalty': 0.01802116446197033}
step_vals {'loss': 0.3081142008304596, 'nll': 0.29752272367477417, 'penalty': 0.010591472499072552}
step_vals {'loss': 0.3158676326274872, 'nll': 0.30820953845977783, 'penalty': 0.007658100221306086}
step_vals {'loss': 0.3076166808605194, 'nll': 0.30207014083862305, 'penalty': 0.005546525586396456}
step_vals {'loss': 0.27082559466362, 'nll': 0.2676389813423157, 'penalty': 0.003186620306223631}
step_vals {'loss': 0.23748737573623657, 'nll': 0.21980679035186768, 'penalty': 0.017680585384368896}
step_vals {'loss': 0.255462110042572, 'nll': 0.23472970724105835, 'penalty': 0.020732415840029716}
step_vals {'loss': 0.3656509220600128, 'nll': 0.3628624677658081, 'penalty': 0.0027884449809789658}
step_vals {'loss': 0.2985539138317108, 'nll': 0.2983083724975586, 'penalty': 0.0002455266658216715}
step_vals {'loss': 0.3328286409378052, 'nll': 0.3397085964679718, 'penalty': -0.006879947613924742}
step_vals {'loss': 0.49449455738067627, 'nll': 0.478422611951828, 'penalty': 0.01607193984091282}
Iteration: 60
out-of-domain val
accuracy 0.809 	 roc_auc_score 0.906
confusion_matrix
[[244  44]
 [ 66 222]]
classification_report
              precision    recall  f1-score   support

           0       0.79      0.85      0.82       288
           1       0.83      0.77      0.80       288

    accuracy                           0.81       576
   macro avg       0.81      0.81      0.81       576
weighted avg       0.81      0.81      0.81       576

VAL * Acc@1 80.903
 * Acc@1 80.903 Acc@5 0.000
accuracy 0.889 	 size: 144 	 cat(indoor)
accuracy 0.847 	 size: 144 	 dog(outdoor)
accuracy 0.806 	 size: 144 	 cat(outdoor)
accuracy 0.694 	 size: 144 	 dog(indoor)
step_vals {'loss': 0.28252264857292175, 'nll': 0.2822471857070923, 'penalty': 0.0002754516899585724}
step_vals {'loss': 0.219722181558609, 'nll': 0.2118208408355713, 'penalty': 0.00790133886039257}
step_vals {'loss': 0.3005351722240448, 'nll': 0.3208245038986206, 'penalty': -0.020289327949285507}
step_vals {'loss': 0.4020223915576935, 'nll': 0.4096720516681671, 'penalty': -0.007649658247828484}
step_vals {'loss': 0.1754492223262787, 'nll': 0.17603915929794312, 'penalty': -0.0005899444222450256}
step_vals {'loss': 0.2902834117412567, 'nll': 0.28811192512512207, 'penalty': 0.0021714840549975634}
step_vals {'loss': 0.49672216176986694, 'nll': 0.491126149892807, 'penalty': 0.005596000701189041}
step_vals {'loss': 0.19642475247383118, 'nll': 0.19191060960292816, 'penalty': 0.004514146130532026}
step_vals {'loss': 0.29167282581329346, 'nll': 0.29790040850639343, 'penalty': -0.006227585952728987}
step_vals {'loss': 0.43564215302467346, 'nll': 0.4564887285232544, 'penalty': -0.020846566185355186}
step_vals {'loss': 0.19721189141273499, 'nll': 0.18807636201381683, 'penalty': 0.009135524742305279}
step_vals {'loss': 0.2613513767719269, 'nll': 0.26103711128234863, 'penalty': 0.0003142748028039932}
step_vals {'loss': 0.3622727394104004, 'nll': 0.36231309175491333, 'penalty': -4.0360871935263276e-05}
step_vals {'loss': 0.22185468673706055, 'nll': 0.20290690660476685, 'penalty': 0.018947776407003403}
step_vals {'loss': 0.2413606494665146, 'nll': 0.2365584671497345, 'penalty': 0.004802188370376825}
step_vals {'loss': 0.22447100281715393, 'nll': 0.22470757365226746, 'penalty': -0.00023656990379095078}
step_vals {'loss': 0.2992996871471405, 'nll': 0.2918134331703186, 'penalty': 0.007486263290047646}
step_vals {'loss': 0.21195665001869202, 'nll': 0.2097182422876358, 'penalty': 0.002238402608782053}
step_vals {'loss': 0.25658082962036133, 'nll': 0.26148372888565063, 'penalty': -0.004902899265289307}
step_vals {'loss': 0.20824311673641205, 'nll': 0.19791091978549957, 'penalty': 0.010332194156944752}
Iteration: 80
out-of-domain val
accuracy 0.825 	 roc_auc_score 0.917
confusion_matrix
[[227  61]
 [ 40 248]]
classification_report
              precision    recall  f1-score   support

           0       0.85      0.79      0.82       288
           1       0.80      0.86      0.83       288

    accuracy                           0.82       576
   macro avg       0.83      0.82      0.82       576
weighted avg       0.83      0.82      0.82       576

VAL * Acc@1 82.465
 * Acc@1 82.465 Acc@5 0.000
accuracy 0.938 	 size: 144 	 dog(outdoor)
accuracy 0.847 	 size: 144 	 cat(indoor)
accuracy 0.785 	 size: 144 	 dog(indoor)
accuracy 0.729 	 size: 144 	 cat(outdoor)
