Environment:
	Python: 3.6.13
	PyTorch: 1.4.0
	Torchvision: 0.5.0
	CUDA: 10.1
	CUDNN: 7603
	NumPy: 1.19.5
	PIL: 8.1.0
Args:
	algorithm: IRM
	batch_size: 8
	data: /data/GQA/MetaDataset-subpopulation-shift
	hparams: None
	hparams_seed: 0
	log_prefix: 
	num_classes: 2
	num_domains: 2
	output_dir: train_output
	save_model_every_checkpoint: False
	seed: 0
	skip_model_save: False
	workers: 4
train_dataset.samples reverse: [('cat(indoor)', 840), ('dog(outdoor)', 840), ('cat(outdoor)', 10), ('dog(indoor)', 10)]
self.domain_to_groups {0: {'cat': ['cat(indoor)'], 'dog': ['dog(indoor)']}, 1: {'cat': ['cat(outdoor)'], 'dog': ['dog(outdoor)']}}
HParams:
	batch_size: 32
	class_balanced: False
	data_augmentation: True
	irm_lambda: 100.0
	irm_penalty_anneal_iters: 500
	lr: 5e-05
	nonlinear_classifier: False
	resnet18: True
	resnet_dropout: 0.0
	weight_decay: 0.0
step_vals {'loss': 0.776574969291687, 'nll': 0.7696692943572998, 'penalty': 0.006905704271048307}
Iteration: 0
out-of-domain val
accuracy 0.573 	 roc_auc_score 0.659
confusion_matrix
[[ 67 221]
 [ 25 263]]
classification_report
              precision    recall  f1-score   support

           0       0.73      0.23      0.35       288
           1       0.54      0.91      0.68       288

    accuracy                           0.57       576
   macro avg       0.64      0.57      0.52       576
weighted avg       0.64      0.57      0.52       576

VAL * Acc@1 57.292
 * Acc@1 57.292 Acc@5 0.000
accuracy 0.917 	 size: 144 	 dog(indoor)
accuracy 0.910 	 size: 144 	 dog(outdoor)
accuracy 0.312 	 size: 144 	 cat(indoor)
accuracy 0.153 	 size: 144 	 cat(outdoor)
step_vals {'loss': 0.6333073973655701, 'nll': 0.642687201499939, 'penalty': -0.009379805065691471}
step_vals {'loss': 0.5438275337219238, 'nll': 0.5462986826896667, 'penalty': -0.0024711773730814457}
step_vals {'loss': 0.46606820821762085, 'nll': 0.45142972469329834, 'penalty': 0.014638472348451614}
step_vals {'loss': 0.35745859146118164, 'nll': 0.32737094163894653, 'penalty': 0.03008764237165451}
step_vals {'loss': 0.41641783714294434, 'nll': 0.40639975666999817, 'penalty': 0.010018090717494488}
step_vals {'loss': 0.38170039653778076, 'nll': 0.37908366322517395, 'penalty': 0.0026167393662035465}
step_vals {'loss': 0.28829655051231384, 'nll': 0.26096677780151367, 'penalty': 0.027329765260219574}
step_vals {'loss': 0.23491129279136658, 'nll': 0.22074094414710999, 'penalty': 0.014170356094837189}
step_vals {'loss': 0.2219347506761551, 'nll': 0.2113913595676422, 'penalty': 0.010543396696448326}
step_vals {'loss': 0.2528652250766754, 'nll': 0.2423919141292572, 'penalty': 0.010473314672708511}
step_vals {'loss': 0.11272615939378738, 'nll': 0.11219656467437744, 'penalty': 0.0005295923911035061}
step_vals {'loss': 0.15440236032009125, 'nll': 0.15881729125976562, 'penalty': -0.0044149374589324}
step_vals {'loss': 0.13785775005817413, 'nll': 0.1343999207019806, 'penalty': 0.0034578307531774044}
step_vals {'loss': 0.08766817301511765, 'nll': 0.09205614775419235, 'penalty': -0.004387974739074707}
step_vals {'loss': 0.30613377690315247, 'nll': 0.28645405173301697, 'penalty': 0.019679715856909752}
step_vals {'loss': 0.4223092198371887, 'nll': 0.38744235038757324, 'penalty': 0.03486686199903488}
step_vals {'loss': 0.5494765043258667, 'nll': 0.5322798490524292, 'penalty': 0.017196685075759888}
step_vals {'loss': 0.2061092108488083, 'nll': 0.22092565894126892, 'penalty': -0.01481644343584776}
step_vals {'loss': 0.2190764993429184, 'nll': 0.2333841174840927, 'penalty': -0.014307612553238869}
step_vals {'loss': 0.15574118494987488, 'nll': 0.16130155324935913, 'penalty': -0.005560369696468115}
Iteration: 20
out-of-domain val
accuracy 0.800 	 roc_auc_score 0.879
confusion_matrix
[[227  61]
 [ 54 234]]
classification_report
              precision    recall  f1-score   support

           0       0.81      0.79      0.80       288
           1       0.79      0.81      0.80       288

    accuracy                           0.80       576
   macro avg       0.80      0.80      0.80       576
weighted avg       0.80      0.80      0.80       576

VAL * Acc@1 80.035
 * Acc@1 80.035 Acc@5 0.000
accuracy 0.965 	 size: 144 	 dog(outdoor)
accuracy 0.931 	 size: 144 	 cat(indoor)
accuracy 0.660 	 size: 144 	 dog(indoor)
accuracy 0.646 	 size: 144 	 cat(outdoor)
step_vals {'loss': 0.08133503794670105, 'nll': 0.0685063898563385, 'penalty': 0.012828651815652847}
step_vals {'loss': 0.32275208830833435, 'nll': 0.29344773292541504, 'penalty': 0.02930436097085476}
step_vals {'loss': 0.47500064969062805, 'nll': 0.41489672660827637, 'penalty': 0.060103919357061386}
step_vals {'loss': 0.12475823611021042, 'nll': 0.11418575793504715, 'penalty': 0.010572480969130993}
step_vals {'loss': 0.26866933703422546, 'nll': 0.2774064540863037, 'penalty': -0.008737108670175076}
step_vals {'loss': 0.20553405582904816, 'nll': 0.19086122512817383, 'penalty': 0.01467282697558403}
step_vals {'loss': 0.2616015672683716, 'nll': 0.2558983862400055, 'penalty': 0.00570316705852747}
step_vals {'loss': 0.14908041059970856, 'nll': 0.14186786115169525, 'penalty': 0.007212549448013306}
step_vals {'loss': 0.16630113124847412, 'nll': 0.16233904659748077, 'penalty': 0.0039620776660740376}
step_vals {'loss': 0.13717596232891083, 'nll': 0.1347312033176422, 'penalty': 0.0024447659961879253}
step_vals {'loss': 0.42709824442863464, 'nll': 0.3822120130062103, 'penalty': 0.04488622769713402}
step_vals {'loss': 0.2533346116542816, 'nll': 0.26363083720207214, 'penalty': -0.010296227410435677}
step_vals {'loss': 0.31694531440734863, 'nll': 0.30333754420280457, 'penalty': 0.013607773929834366}
step_vals {'loss': 0.15539315342903137, 'nll': 0.13783428072929382, 'penalty': 0.017558874562382698}
step_vals {'loss': 0.24743463099002838, 'nll': 0.2508707046508789, 'penalty': -0.0034360759891569614}
step_vals {'loss': 0.18746429681777954, 'nll': 0.18499070405960083, 'penalty': 0.002473593456670642}
step_vals {'loss': 0.18211433291435242, 'nll': 0.17574232816696167, 'penalty': 0.006372012197971344}
step_vals {'loss': 0.07859142869710922, 'nll': 0.0739995539188385, 'penalty': 0.004591877106577158}
step_vals {'loss': 0.11731259524822235, 'nll': 0.11392898857593536, 'penalty': 0.003383609466254711}
step_vals {'loss': 0.12716135382652283, 'nll': 0.13768534362316132, 'penalty': -0.01052398793399334}
Iteration: 40
out-of-domain val
accuracy 0.780 	 roc_auc_score 0.878
confusion_matrix
[[189  99]
 [ 28 260]]
classification_report
              precision    recall  f1-score   support

           0       0.87      0.66      0.75       288
           1       0.72      0.90      0.80       288

    accuracy                           0.78       576
   macro avg       0.80      0.78      0.78       576
weighted avg       0.80      0.78      0.78       576

VAL * Acc@1 77.951
 * Acc@1 77.951 Acc@5 0.000
accuracy 0.986 	 size: 144 	 dog(outdoor)
accuracy 0.826 	 size: 144 	 cat(indoor)
accuracy 0.819 	 size: 144 	 dog(indoor)
accuracy 0.486 	 size: 144 	 cat(outdoor)
step_vals {'loss': 0.593590497970581, 'nll': 0.40523117780685425, 'penalty': 0.18835929036140442}
step_vals {'loss': 0.07912987470626831, 'nll': 0.072292260825634, 'penalty': 0.006837613880634308}
step_vals {'loss': 0.1780293732881546, 'nll': 0.16299796104431152, 'penalty': 0.015031406655907631}
step_vals {'loss': 0.2109355628490448, 'nll': 0.22219985723495483, 'penalty': -0.011264290660619736}
step_vals {'loss': 0.22404687106609344, 'nll': 0.2288091778755188, 'penalty': -0.004762308672070503}
step_vals {'loss': 0.08373688906431198, 'nll': 0.07102292031049728, 'penalty': 0.012713967822492123}
step_vals {'loss': 0.21080592274665833, 'nll': 0.2119482457637787, 'penalty': -0.001142330002039671}
step_vals {'loss': 0.24530518054962158, 'nll': 0.2452753484249115, 'penalty': 2.9828865081071854e-05}
step_vals {'loss': 0.1890098601579666, 'nll': 0.1799604892730713, 'penalty': 0.009049365296959877}
step_vals {'loss': 0.06978170573711395, 'nll': 0.059680938720703125, 'penalty': 0.010100769810378551}
step_vals {'loss': 0.12141766399145126, 'nll': 0.12261725962162018, 'penalty': -0.0011995926033705473}
step_vals {'loss': 0.22266873717308044, 'nll': 0.22435200214385986, 'penalty': -0.0016832581022754312}
step_vals {'loss': 0.21927621960639954, 'nll': 0.21868202090263367, 'penalty': 0.0005941931158304214}
step_vals {'loss': 0.11907465755939484, 'nll': 0.11552716791629791, 'penalty': 0.003547488246113062}
step_vals {'loss': 0.17482738196849823, 'nll': 0.17936363816261292, 'penalty': -0.0045362599194049835}
step_vals {'loss': 0.08175668865442276, 'nll': 0.07676815986633301, 'penalty': 0.004988532047718763}
step_vals {'loss': 0.1331447958946228, 'nll': 0.12433000653982162, 'penalty': 0.008814786560833454}
step_vals {'loss': 0.1192917749285698, 'nll': 0.1272895485162735, 'penalty': -0.007997771725058556}
step_vals {'loss': 0.06048283353447914, 'nll': 0.058831848204135895, 'penalty': 0.001650986261665821}
step_vals {'loss': 0.1395493447780609, 'nll': 0.1488371044397354, 'penalty': -0.009287764318287373}
Iteration: 60
out-of-domain val
accuracy 0.776 	 roc_auc_score 0.863
confusion_matrix
[[204  84]
 [ 45 243]]
classification_report
              precision    recall  f1-score   support

           0       0.82      0.71      0.76       288
           1       0.74      0.84      0.79       288

    accuracy                           0.78       576
   macro avg       0.78      0.78      0.78       576
weighted avg       0.78      0.78      0.78       576

VAL * Acc@1 77.604
 * Acc@1 77.604 Acc@5 0.000
accuracy 0.972 	 size: 144 	 dog(outdoor)
accuracy 0.875 	 size: 144 	 cat(indoor)
accuracy 0.715 	 size: 144 	 dog(indoor)
accuracy 0.542 	 size: 144 	 cat(outdoor)
step_vals {'loss': 0.07598010450601578, 'nll': 0.07410834729671478, 'penalty': 0.0018717574421316385}
step_vals {'loss': 0.240524560213089, 'nll': 0.2584455609321594, 'penalty': -0.017920993268489838}
step_vals {'loss': 0.05215747654438019, 'nll': 0.04777941852807999, 'penalty': 0.004378057550638914}
step_vals {'loss': 0.11576967686414719, 'nll': 0.12194925546646118, 'penalty': -0.006179578602313995}
step_vals {'loss': 0.09776967018842697, 'nll': 0.09885957837104797, 'penalty': -0.0010899062035605311}
step_vals {'loss': 0.12157447636127472, 'nll': 0.121413454413414, 'penalty': 0.00016102357767522335}
step_vals {'loss': 0.04339638352394104, 'nll': 0.04209702089428902, 'penalty': 0.0012993629788979888}
step_vals {'loss': 0.21901750564575195, 'nll': 0.2102596014738083, 'penalty': 0.008757898584008217}
step_vals {'loss': 0.07712286710739136, 'nll': 0.07505428791046143, 'penalty': 0.0020685812924057245}
step_vals {'loss': 0.15975269675254822, 'nll': 0.1749979555606842, 'penalty': -0.015245255082845688}
step_vals {'loss': 0.13088200986385345, 'nll': 0.12693409621715546, 'penalty': 0.003947910387068987}
step_vals {'loss': 0.21104735136032104, 'nll': 0.2114555835723877, 'penalty': -0.00040823296876624227}
step_vals {'loss': 0.04229964315891266, 'nll': 0.03877178579568863, 'penalty': 0.0035278555005788803}
step_vals {'loss': 0.07577283680438995, 'nll': 0.07244183868169785, 'penalty': 0.0033309985883533955}
step_vals {'loss': 0.13651859760284424, 'nll': 0.14028021693229675, 'penalty': -0.0037616239860653877}
step_vals {'loss': 0.08140354603528976, 'nll': 0.07796667516231537, 'penalty': 0.003436869941651821}
step_vals {'loss': 0.07216081768274307, 'nll': 0.06425409018993378, 'penalty': 0.007906725630164146}
step_vals {'loss': 0.13515660166740417, 'nll': 0.1334373503923416, 'penalty': 0.0017192584928125143}
step_vals {'loss': 0.03792612999677658, 'nll': 0.034798964858055115, 'penalty': 0.0031271646730601788}
step_vals {'loss': 0.16764165461063385, 'nll': 0.16611738502979279, 'penalty': 0.0015242710942402482}
Iteration: 80
out-of-domain val
accuracy 0.773 	 roc_auc_score 0.869
confusion_matrix
[[202  86]
 [ 45 243]]
classification_report
              precision    recall  f1-score   support

           0       0.82      0.70      0.76       288
           1       0.74      0.84      0.79       288

    accuracy                           0.77       576
   macro avg       0.78      0.77      0.77       576
weighted avg       0.78      0.77      0.77       576

VAL * Acc@1 77.257
 * Acc@1 77.257 Acc@5 0.000
accuracy 0.986 	 size: 144 	 dog(outdoor)
accuracy 0.875 	 size: 144 	 cat(indoor)
accuracy 0.701 	 size: 144 	 dog(indoor)
accuracy 0.528 	 size: 144 	 cat(outdoor)
