Environment:
	Python: 3.6.13
	PyTorch: 1.4.0
	Torchvision: 0.5.0
	CUDA: 10.1
	CUDNN: 7603
	NumPy: 1.19.5
	PIL: 8.1.0
Args:
	algorithm: GroupDRO
	batch_size: 8
	data: /data/GQA/MetaDataset-subpopulation-shift
	hparams: None
	hparams_seed: 0
	log_prefix: 
	num_classes: 2
	num_domains: 2
	output_dir: train_output
	save_model_every_checkpoint: False
	seed: 0
	skip_model_save: False
	workers: 4
train_dataset.samples reverse: [('cat(indoor)', 800), ('dog(outdoor)', 800), ('cat(outdoor)', 50), ('dog(indoor)', 50)]
self.domain_to_groups {0: {'cat': ['cat(indoor)'], 'dog': ['dog(indoor)']}, 1: {'cat': ['cat(outdoor)'], 'dog': ['dog(outdoor)']}}
HParams:
	batch_size: 32
	class_balanced: False
	data_augmentation: True
	groupdro_eta: 0.01
	lr: 5e-05
	nonlinear_classifier: False
	resnet18: True
	resnet_dropout: 0.0
	weight_decay: 0.0
step_vals {'loss': 0.704755425453186}
Iteration: 0
out-of-domain val
accuracy 0.576 	 roc_auc_score 0.615
confusion_matrix
[[136 152]
 [ 92 196]]
classification_report
              precision    recall  f1-score   support

           0       0.60      0.47      0.53       288
           1       0.56      0.68      0.62       288

    accuracy                           0.58       576
   macro avg       0.58      0.58      0.57       576
weighted avg       0.58      0.58      0.57       576

VAL * Acc@1 57.639
 * Acc@1 57.639 Acc@5 0.000
accuracy 0.688 	 size: 144 	 dog(indoor)
accuracy 0.674 	 size: 144 	 dog(outdoor)
accuracy 0.542 	 size: 144 	 cat(indoor)
accuracy 0.403 	 size: 144 	 cat(outdoor)
step_vals {'loss': 0.7275203466415405}
step_vals {'loss': 0.675646185874939}
step_vals {'loss': 0.586363673210144}
step_vals {'loss': 0.6463390588760376}
step_vals {'loss': 0.5304422378540039}
step_vals {'loss': 0.4895630478858948}
step_vals {'loss': 0.36818429827690125}
step_vals {'loss': 0.3854919373989105}
step_vals {'loss': 0.5806385278701782}
step_vals {'loss': 0.3977152705192566}
step_vals {'loss': 0.5485771298408508}
step_vals {'loss': 0.3633064925670624}
step_vals {'loss': 0.3534889817237854}
step_vals {'loss': 0.3985072374343872}
step_vals {'loss': 0.5099668502807617}
step_vals {'loss': 0.5806925296783447}
step_vals {'loss': 0.3705524802207947}
step_vals {'loss': 0.33551064133644104}
step_vals {'loss': 0.41679155826568604}
step_vals {'loss': 0.3874824047088623}
Iteration: 20
out-of-domain val
accuracy 0.764 	 roc_auc_score 0.874
confusion_matrix
[[253  35]
 [101 187]]
classification_report
              precision    recall  f1-score   support

           0       0.71      0.88      0.79       288
           1       0.84      0.65      0.73       288

    accuracy                           0.76       576
   macro avg       0.78      0.76      0.76       576
weighted avg       0.78      0.76      0.76       576

VAL * Acc@1 76.389
 * Acc@1 76.389 Acc@5 0.000
accuracy 0.944 	 size: 144 	 cat(indoor)
accuracy 0.812 	 size: 144 	 cat(outdoor)
accuracy 0.750 	 size: 144 	 dog(outdoor)
accuracy 0.549 	 size: 144 	 dog(indoor)
step_vals {'loss': 0.3370106816291809}
step_vals {'loss': 0.6133320331573486}
step_vals {'loss': 0.5176644325256348}
step_vals {'loss': 0.32782062888145447}
step_vals {'loss': 0.4000436067581177}
step_vals {'loss': 0.33548790216445923}
step_vals {'loss': 0.4049030542373657}
step_vals {'loss': 0.41700729727745056}
step_vals {'loss': 0.3136902451515198}
step_vals {'loss': 0.2023943066596985}
step_vals {'loss': 0.33011025190353394}
step_vals {'loss': 0.3167998492717743}
step_vals {'loss': 0.39191389083862305}
step_vals {'loss': 0.2326880246400833}
step_vals {'loss': 0.2792383134365082}
step_vals {'loss': 0.1793985366821289}
step_vals {'loss': 0.34362828731536865}
step_vals {'loss': 0.3102024495601654}
step_vals {'loss': 0.3073161840438843}
step_vals {'loss': 0.41063225269317627}
Iteration: 40
out-of-domain val
accuracy 0.818 	 roc_auc_score 0.896
confusion_matrix
[[238  50]
 [ 55 233]]
classification_report
              precision    recall  f1-score   support

           0       0.81      0.83      0.82       288
           1       0.82      0.81      0.82       288

    accuracy                           0.82       576
   macro avg       0.82      0.82      0.82       576
weighted avg       0.82      0.82      0.82       576

VAL * Acc@1 81.771
 * Acc@1 81.771 Acc@5 0.000
accuracy 0.938 	 size: 144 	 dog(outdoor)
accuracy 0.931 	 size: 144 	 cat(indoor)
accuracy 0.722 	 size: 144 	 cat(outdoor)
accuracy 0.681 	 size: 144 	 dog(indoor)
step_vals {'loss': 0.35627102851867676}
step_vals {'loss': 0.2805798053741455}
step_vals {'loss': 0.29425328969955444}
step_vals {'loss': 0.3462010622024536}
step_vals {'loss': 0.14742042124271393}
step_vals {'loss': 0.44217056035995483}
step_vals {'loss': 0.251234769821167}
step_vals {'loss': 0.4278506338596344}
step_vals {'loss': 0.11990723013877869}
step_vals {'loss': 0.23662860691547394}
step_vals {'loss': 0.43671900033950806}
step_vals {'loss': 0.5617944002151489}
step_vals {'loss': 0.3958263397216797}
step_vals {'loss': 0.41628313064575195}
step_vals {'loss': 0.205969899892807}
step_vals {'loss': 0.46023038029670715}
step_vals {'loss': 0.3580282926559448}
step_vals {'loss': 0.22721022367477417}
step_vals {'loss': 0.24233116209506989}
step_vals {'loss': 0.24122068285942078}
Iteration: 60
out-of-domain val
accuracy 0.809 	 roc_auc_score 0.894
confusion_matrix
[[245  43]
 [ 67 221]]
classification_report
              precision    recall  f1-score   support

           0       0.79      0.85      0.82       288
           1       0.84      0.77      0.80       288

    accuracy                           0.81       576
   macro avg       0.81      0.81      0.81       576
weighted avg       0.81      0.81      0.81       576

VAL * Acc@1 80.903
 * Acc@1 80.903 Acc@5 0.000
accuracy 0.924 	 size: 144 	 cat(indoor)
accuracy 0.861 	 size: 144 	 dog(outdoor)
accuracy 0.778 	 size: 144 	 cat(outdoor)
accuracy 0.674 	 size: 144 	 dog(indoor)
step_vals {'loss': 0.24289846420288086}
step_vals {'loss': 0.39243263006210327}
step_vals {'loss': 0.23594217002391815}
step_vals {'loss': 0.14519073069095612}
step_vals {'loss': 0.3706355690956116}
step_vals {'loss': 0.26145410537719727}
step_vals {'loss': 0.23506209254264832}
step_vals {'loss': 0.22716930508613586}
step_vals {'loss': 0.25248074531555176}
step_vals {'loss': 0.22390109300613403}
step_vals {'loss': 0.15648695826530457}
step_vals {'loss': 0.27985358238220215}
step_vals {'loss': 0.35744908452033997}
step_vals {'loss': 0.22697541117668152}
step_vals {'loss': 0.29386448860168457}
step_vals {'loss': 0.16582193970680237}
step_vals {'loss': 0.22903020679950714}
step_vals {'loss': 0.20763710141181946}
step_vals {'loss': 0.32108163833618164}
step_vals {'loss': 0.33586475253105164}
Iteration: 80
out-of-domain val
accuracy 0.818 	 roc_auc_score 0.900
confusion_matrix
[[235  53]
 [ 52 236]]
classification_report
              precision    recall  f1-score   support

           0       0.82      0.82      0.82       288
           1       0.82      0.82      0.82       288

    accuracy                           0.82       576
   macro avg       0.82      0.82      0.82       576
weighted avg       0.82      0.82      0.82       576

VAL * Acc@1 81.771
 * Acc@1 81.771 Acc@5 0.000
accuracy 0.924 	 size: 144 	 dog(outdoor)
accuracy 0.910 	 size: 144 	 cat(indoor)
accuracy 0.722 	 size: 144 	 cat(outdoor)
accuracy 0.715 	 size: 144 	 dog(indoor)
