Environment:
	Python: 3.6.13
	PyTorch: 1.4.0
	Torchvision: 0.5.0
	CUDA: 10.1
	CUDNN: 7603
	NumPy: 1.19.5
	PIL: 8.1.0
Args:
	algorithm: ERM
	batch_size: 8
	data: /data/GQA/MetaDataset-subpopulation-shift
	hparams: None
	hparams_seed: 0
	log_prefix: 
	num_classes: 2
	num_domains: 2
	output_dir: train_output
	save_model_every_checkpoint: False
	seed: 0
	skip_model_save: False
	workers: 4
train_dataset.samples reverse: [('cat(indoor)', 800), ('dog(outdoor)', 800), ('cat(outdoor)', 50), ('dog(indoor)', 50)]
self.domain_to_groups {0: {'cat': ['cat(indoor)'], 'dog': ['dog(indoor)']}, 1: {'cat': ['cat(outdoor)'], 'dog': ['dog(outdoor)']}}
HParams:
	batch_size: 32
	class_balanced: False
	data_augmentation: True
	lr: 5e-05
	nonlinear_classifier: False
	resnet18: True
	resnet_dropout: 0.0
	weight_decay: 0.0
step_vals {'loss': 0.704740583896637}
Iteration: 0
out-of-domain val
accuracy 0.576 	 roc_auc_score 0.615
confusion_matrix
[[136 152]
 [ 92 196]]
classification_report
              precision    recall  f1-score   support

           0       0.60      0.47      0.53       288
           1       0.56      0.68      0.62       288

    accuracy                           0.58       576
   macro avg       0.58      0.58      0.57       576
weighted avg       0.58      0.58      0.57       576

VAL * Acc@1 57.639
 * Acc@1 57.639 Acc@5 0.000
accuracy 0.688 	 size: 144 	 dog(indoor)
accuracy 0.674 	 size: 144 	 dog(outdoor)
accuracy 0.542 	 size: 144 	 cat(indoor)
accuracy 0.403 	 size: 144 	 cat(outdoor)
step_vals {'loss': 0.7274962663650513}
step_vals {'loss': 0.6756082773208618}
step_vals {'loss': 0.5864546298980713}
step_vals {'loss': 0.6465474963188171}
step_vals {'loss': 0.530127763748169}
step_vals {'loss': 0.48950064182281494}
step_vals {'loss': 0.3678171634674072}
step_vals {'loss': 0.3853759169578552}
step_vals {'loss': 0.5825425386428833}
step_vals {'loss': 0.39939001202583313}
step_vals {'loss': 0.547408401966095}
step_vals {'loss': 0.36638712882995605}
step_vals {'loss': 0.3542309105396271}
step_vals {'loss': 0.39823633432388306}
step_vals {'loss': 0.5083070993423462}
step_vals {'loss': 0.582489013671875}
step_vals {'loss': 0.3694325089454651}
step_vals {'loss': 0.34102341532707214}
step_vals {'loss': 0.4192543923854828}
step_vals {'loss': 0.3929373025894165}
Iteration: 20
out-of-domain val
accuracy 0.764 	 roc_auc_score 0.872
confusion_matrix
[[254  34]
 [102 186]]
classification_report
              precision    recall  f1-score   support

           0       0.71      0.88      0.79       288
           1       0.85      0.65      0.73       288

    accuracy                           0.76       576
   macro avg       0.78      0.76      0.76       576
weighted avg       0.78      0.76      0.76       576

VAL * Acc@1 76.389
 * Acc@1 76.389 Acc@5 0.000
accuracy 0.951 	 size: 144 	 cat(indoor)
accuracy 0.812 	 size: 144 	 cat(outdoor)
accuracy 0.736 	 size: 144 	 dog(outdoor)
accuracy 0.556 	 size: 144 	 dog(indoor)
step_vals {'loss': 0.3348149359226227}
step_vals {'loss': 0.6147184371948242}
step_vals {'loss': 0.5217999219894409}
step_vals {'loss': 0.32603690028190613}
step_vals {'loss': 0.40451109409332275}
step_vals {'loss': 0.3352183997631073}
step_vals {'loss': 0.40368881821632385}
step_vals {'loss': 0.42782777547836304}
step_vals {'loss': 0.3180251121520996}
step_vals {'loss': 0.20276087522506714}
step_vals {'loss': 0.3293840289115906}
step_vals {'loss': 0.3190288841724396}
step_vals {'loss': 0.40049976110458374}
step_vals {'loss': 0.2376098781824112}
step_vals {'loss': 0.27500540018081665}
step_vals {'loss': 0.18174447119235992}
step_vals {'loss': 0.345970094203949}
step_vals {'loss': 0.3050304651260376}
step_vals {'loss': 0.30133938789367676}
step_vals {'loss': 0.4055979549884796}
Iteration: 40
out-of-domain val
accuracy 0.823 	 roc_auc_score 0.896
confusion_matrix
[[238  50]
 [ 52 236]]
classification_report
              precision    recall  f1-score   support

           0       0.82      0.83      0.82       288
           1       0.83      0.82      0.82       288

    accuracy                           0.82       576
   macro avg       0.82      0.82      0.82       576
weighted avg       0.82      0.82      0.82       576

VAL * Acc@1 82.292
 * Acc@1 82.292 Acc@5 0.000
accuracy 0.944 	 size: 144 	 dog(outdoor)
accuracy 0.931 	 size: 144 	 cat(indoor)
accuracy 0.722 	 size: 144 	 cat(outdoor)
accuracy 0.694 	 size: 144 	 dog(indoor)
step_vals {'loss': 0.35111990571022034}
step_vals {'loss': 0.2767699062824249}
step_vals {'loss': 0.301107794046402}
step_vals {'loss': 0.36023640632629395}
step_vals {'loss': 0.15112298727035522}
step_vals {'loss': 0.4380726218223572}
step_vals {'loss': 0.24336403608322144}
step_vals {'loss': 0.44242069125175476}
step_vals {'loss': 0.12227285653352737}
step_vals {'loss': 0.23652096092700958}
step_vals {'loss': 0.4517872631549835}
step_vals {'loss': 0.5716555714607239}
step_vals {'loss': 0.39043691754341125}
step_vals {'loss': 0.43339160084724426}
step_vals {'loss': 0.20237350463867188}
step_vals {'loss': 0.45739275217056274}
step_vals {'loss': 0.3579755127429962}
step_vals {'loss': 0.236223965883255}
step_vals {'loss': 0.23666910827159882}
step_vals {'loss': 0.24872352182865143}
Iteration: 60
out-of-domain val
accuracy 0.809 	 roc_auc_score 0.892
confusion_matrix
[[242  46]
 [ 64 224]]
classification_report
              precision    recall  f1-score   support

           0       0.79      0.84      0.81       288
           1       0.83      0.78      0.80       288

    accuracy                           0.81       576
   macro avg       0.81      0.81      0.81       576
weighted avg       0.81      0.81      0.81       576

VAL * Acc@1 80.903
 * Acc@1 80.903 Acc@5 0.000
accuracy 0.924 	 size: 144 	 cat(indoor)
accuracy 0.875 	 size: 144 	 dog(outdoor)
accuracy 0.757 	 size: 144 	 cat(outdoor)
accuracy 0.681 	 size: 144 	 dog(indoor)
step_vals {'loss': 0.24287614226341248}
step_vals {'loss': 0.3895385265350342}
step_vals {'loss': 0.22642096877098083}
step_vals {'loss': 0.14660154283046722}
step_vals {'loss': 0.3730027675628662}
step_vals {'loss': 0.2593017816543579}
step_vals {'loss': 0.228658527135849}
step_vals {'loss': 0.23216235637664795}
step_vals {'loss': 0.2521154582500458}
step_vals {'loss': 0.23261168599128723}
step_vals {'loss': 0.1610991209745407}
step_vals {'loss': 0.29062509536743164}
step_vals {'loss': 0.3498866856098175}
step_vals {'loss': 0.23990076780319214}
step_vals {'loss': 0.3048349916934967}
step_vals {'loss': 0.1668981909751892}
step_vals {'loss': 0.21912556886672974}
step_vals {'loss': 0.22062912583351135}
step_vals {'loss': 0.3313884139060974}
step_vals {'loss': 0.3356865346431732}
Iteration: 80
out-of-domain val
accuracy 0.823 	 roc_auc_score 0.900
confusion_matrix
[[232  56]
 [ 46 242]]
classification_report
              precision    recall  f1-score   support

           0       0.83      0.81      0.82       288
           1       0.81      0.84      0.83       288

    accuracy                           0.82       576
   macro avg       0.82      0.82      0.82       576
weighted avg       0.82      0.82      0.82       576

VAL * Acc@1 82.292
 * Acc@1 82.292 Acc@5 0.000
accuracy 0.938 	 size: 144 	 dog(outdoor)
accuracy 0.910 	 size: 144 	 cat(indoor)
accuracy 0.743 	 size: 144 	 dog(indoor)
accuracy 0.701 	 size: 144 	 cat(outdoor)
