Environment:
	Python: 3.6.13
	PyTorch: 1.4.0
	Torchvision: 0.5.0
	CUDA: 10.1
	CUDNN: 7603
	NumPy: 1.19.5
	PIL: 8.1.0
Args:
	algorithm: ERM
	batch_size: 8
	data: /data/GQA/MetaDataset-subpopulation-shift
	hparams: None
	hparams_seed: 0
	log_prefix: 
	num_classes: 2
	num_domains: 2
	output_dir: train_output
	save_model_every_checkpoint: False
	seed: 0
	skip_model_save: False
	workers: 4
train_dataset.samples reverse: [('cat(indoor)', 840), ('dog(outdoor)', 840), ('cat(outdoor)', 10), ('dog(indoor)', 10)]
self.domain_to_groups {0: {'cat': ['cat(indoor)'], 'dog': ['dog(indoor)']}, 1: {'cat': ['cat(outdoor)'], 'dog': ['dog(outdoor)']}}
HParams:
	batch_size: 32
	class_balanced: False
	data_augmentation: True
	lr: 5e-05
	nonlinear_classifier: False
	resnet18: True
	resnet_dropout: 0.0
	weight_decay: 0.0
step_vals {'loss': 0.7696692943572998}
Iteration: 0
out-of-domain val
accuracy 0.583 	 roc_auc_score 0.666
confusion_matrix
[[ 83 205]
 [ 35 253]]
classification_report
              precision    recall  f1-score   support

           0       0.70      0.29      0.41       288
           1       0.55      0.88      0.68       288

    accuracy                           0.58       576
   macro avg       0.63      0.58      0.54       576
weighted avg       0.63      0.58      0.54       576

VAL * Acc@1 58.333
 * Acc@1 58.333 Acc@5 0.000
accuracy 0.882 	 size: 144 	 dog(outdoor)
accuracy 0.875 	 size: 144 	 dog(indoor)
accuracy 0.375 	 size: 144 	 cat(indoor)
accuracy 0.201 	 size: 144 	 cat(outdoor)
step_vals {'loss': 0.620733916759491}
step_vals {'loss': 0.5357328057289124}
step_vals {'loss': 0.44054338335990906}
step_vals {'loss': 0.30290278792381287}
step_vals {'loss': 0.37801119685173035}
step_vals {'loss': 0.3574149012565613}
step_vals {'loss': 0.2317497432231903}
step_vals {'loss': 0.18288472294807434}
step_vals {'loss': 0.18889755010604858}
step_vals {'loss': 0.24801582098007202}
step_vals {'loss': 0.11877145618200302}
step_vals {'loss': 0.1338040679693222}
step_vals {'loss': 0.10373365879058838}
step_vals {'loss': 0.07947570085525513}
step_vals {'loss': 0.29134225845336914}
step_vals {'loss': 0.24709171056747437}
step_vals {'loss': 0.46633872389793396}
step_vals {'loss': 0.18060533702373505}
step_vals {'loss': 0.22116386890411377}
step_vals {'loss': 0.20075085759162903}
Iteration: 20
out-of-domain val
accuracy 0.790 	 roc_auc_score 0.872
confusion_matrix
[[229  59]
 [ 62 226]]
classification_report
              precision    recall  f1-score   support

           0       0.79      0.80      0.79       288
           1       0.79      0.78      0.79       288

    accuracy                           0.79       576
   macro avg       0.79      0.79      0.79       576
weighted avg       0.79      0.79      0.79       576

VAL * Acc@1 78.993
 * Acc@1 78.993 Acc@5 0.000
accuracy 0.958 	 size: 144 	 dog(outdoor)
accuracy 0.917 	 size: 144 	 cat(indoor)
accuracy 0.674 	 size: 144 	 cat(outdoor)
accuracy 0.611 	 size: 144 	 dog(indoor)
step_vals {'loss': 0.04934423416852951}
step_vals {'loss': 0.2715609669685364}
step_vals {'loss': 0.39890429377555847}
step_vals {'loss': 0.1222507655620575}
step_vals {'loss': 0.3440709114074707}
step_vals {'loss': 0.16368871927261353}
step_vals {'loss': 0.2083214819431305}
step_vals {'loss': 0.14012454450130463}
step_vals {'loss': 0.11395548284053802}
step_vals {'loss': 0.133932963013649}
step_vals {'loss': 0.28729283809661865}
step_vals {'loss': 0.28187429904937744}
step_vals {'loss': 0.2305234670639038}
step_vals {'loss': 0.21130773425102234}
step_vals {'loss': 0.25987064838409424}
step_vals {'loss': 0.1762685626745224}
step_vals {'loss': 0.16746950149536133}
step_vals {'loss': 0.049887869507074356}
step_vals {'loss': 0.10441930592060089}
step_vals {'loss': 0.15160739421844482}
Iteration: 40
out-of-domain val
accuracy 0.786 	 roc_auc_score 0.876
confusion_matrix
[[220  68]
 [ 55 233]]
classification_report
              precision    recall  f1-score   support

           0       0.80      0.76      0.78       288
           1       0.77      0.81      0.79       288

    accuracy                           0.79       576
   macro avg       0.79      0.79      0.79       576
weighted avg       0.79      0.79      0.79       576

VAL * Acc@1 78.646
 * Acc@1 78.646 Acc@5 0.000
accuracy 0.972 	 size: 144 	 dog(outdoor)
accuracy 0.903 	 size: 144 	 cat(indoor)
accuracy 0.646 	 size: 144 	 dog(indoor)
accuracy 0.625 	 size: 144 	 cat(outdoor)
step_vals {'loss': 0.33111584186553955}
step_vals {'loss': 0.03403155133128166}
step_vals {'loss': 0.2352449744939804}
step_vals {'loss': 0.24056123197078705}
step_vals {'loss': 0.14501294493675232}
step_vals {'loss': 0.10176555812358856}
step_vals {'loss': 0.12258568406105042}
step_vals {'loss': 0.39594173431396484}
step_vals {'loss': 0.20332464575767517}
step_vals {'loss': 0.06326358020305634}
step_vals {'loss': 0.07360406965017319}
step_vals {'loss': 0.1126624047756195}
step_vals {'loss': 0.22063207626342773}
step_vals {'loss': 0.16118976473808289}
step_vals {'loss': 0.14453206956386566}
step_vals {'loss': 0.12181604653596878}
step_vals {'loss': 0.36729860305786133}
step_vals {'loss': 0.12378645688295364}
step_vals {'loss': 0.030263040214776993}
step_vals {'loss': 0.14042101800441742}
Iteration: 60
out-of-domain val
accuracy 0.740 	 roc_auc_score 0.849
confusion_matrix
[[251  37]
 [113 175]]
classification_report
              precision    recall  f1-score   support

           0       0.69      0.87      0.77       288
           1       0.83      0.61      0.70       288

    accuracy                           0.74       576
   macro avg       0.76      0.74      0.73       576
weighted avg       0.76      0.74      0.73       576

VAL * Acc@1 73.958
 * Acc@1 73.958 Acc@5 0.000
accuracy 0.993 	 size: 144 	 cat(indoor)
accuracy 0.861 	 size: 144 	 dog(outdoor)
accuracy 0.750 	 size: 144 	 cat(outdoor)
accuracy 0.354 	 size: 144 	 dog(indoor)
step_vals {'loss': 0.12050803005695343}
step_vals {'loss': 0.23142464458942413}
step_vals {'loss': 0.13136596977710724}
step_vals {'loss': 0.09285122901201248}
step_vals {'loss': 0.03990187495946884}
step_vals {'loss': 0.08525905758142471}
step_vals {'loss': 0.09457617998123169}
step_vals {'loss': 0.15337570011615753}
step_vals {'loss': 0.10423896461725235}
step_vals {'loss': 0.22976569831371307}
step_vals {'loss': 0.1319151222705841}
step_vals {'loss': 0.07715663313865662}
step_vals {'loss': 0.054496921598911285}
step_vals {'loss': 0.11447712033987045}
step_vals {'loss': 0.08122992515563965}
step_vals {'loss': 0.06602347642183304}
step_vals {'loss': 0.03363034874200821}
step_vals {'loss': 0.07892630994319916}
step_vals {'loss': 0.04052416607737541}
step_vals {'loss': 0.2520119249820709}
Iteration: 80
out-of-domain val
accuracy 0.759 	 roc_auc_score 0.851
confusion_matrix
[[222  66]
 [ 73 215]]
classification_report
              precision    recall  f1-score   support

           0       0.75      0.77      0.76       288
           1       0.77      0.75      0.76       288

    accuracy                           0.76       576
   macro avg       0.76      0.76      0.76       576
weighted avg       0.76      0.76      0.76       576

VAL * Acc@1 75.868
 * Acc@1 75.868 Acc@5 0.000
accuracy 0.931 	 size: 144 	 dog(outdoor)
accuracy 0.924 	 size: 144 	 cat(indoor)
accuracy 0.618 	 size: 144 	 cat(outdoor)
accuracy 0.562 	 size: 144 	 dog(indoor)
