Environment:
	Python: 3.6.13
	PyTorch: 1.4.0
	Torchvision: 0.5.0
	CUDA: 10.1
	CUDNN: 7603
	NumPy: 1.19.5
	PIL: 8.1.0
Args:
	algorithm: ERM
	batch_size: 8
	data: /data/GQA/MetaDataset-subpopulation-shift
	hparams: None
	hparams_seed: 0
	log_prefix: 
	num_classes: 2
	num_domains: 2
	output_dir: train_output
	save_model_every_checkpoint: False
	seed: 0
	skip_model_save: False
	workers: 4
train_dataset.samples reverse: [('cat(indoor)', 750), ('dog(outdoor)', 750), ('cat(outdoor)', 100), ('dog(indoor)', 100)]
self.domain_to_groups {0: {'cat': ['cat(indoor)'], 'dog': ['dog(indoor)']}, 1: {'cat': ['cat(outdoor)'], 'dog': ['dog(outdoor)']}}
HParams:
	batch_size: 32
	class_balanced: False
	data_augmentation: True
	lr: 5e-05
	nonlinear_classifier: False
	resnet18: True
	resnet_dropout: 0.0
	weight_decay: 0.0
step_vals {'loss': 0.7828301787376404}
Iteration: 0
out-of-domain val
accuracy 0.585 	 roc_auc_score 0.679
confusion_matrix
[[ 72 216]
 [ 23 265]]
classification_report
              precision    recall  f1-score   support

           0       0.76      0.25      0.38       288
           1       0.55      0.92      0.69       288

    accuracy                           0.59       576
   macro avg       0.65      0.59      0.53       576
weighted avg       0.65      0.59      0.53       576

VAL * Acc@1 58.507
 * Acc@1 58.507 Acc@5 0.000
accuracy 0.931 	 size: 144 	 dog(outdoor)
accuracy 0.910 	 size: 144 	 dog(indoor)
accuracy 0.312 	 size: 144 	 cat(indoor)
accuracy 0.188 	 size: 144 	 cat(outdoor)
step_vals {'loss': 0.6923855543136597}
step_vals {'loss': 0.6441269516944885}
step_vals {'loss': 0.45756641030311584}
step_vals {'loss': 0.5360032916069031}
step_vals {'loss': 0.4563385844230652}
step_vals {'loss': 0.4077763855457306}
step_vals {'loss': 0.5124871730804443}
step_vals {'loss': 0.5246487855911255}
step_vals {'loss': 0.46638554334640503}
step_vals {'loss': 0.5117183327674866}
step_vals {'loss': 0.3674734830856323}
step_vals {'loss': 0.5059324502944946}
step_vals {'loss': 0.4010550081729889}
step_vals {'loss': 0.49442028999328613}
step_vals {'loss': 0.4516168236732483}
step_vals {'loss': 0.581584632396698}
step_vals {'loss': 0.3639432489871979}
step_vals {'loss': 0.34656500816345215}
step_vals {'loss': 0.5973146557807922}
step_vals {'loss': 0.34184131026268005}
Iteration: 20
out-of-domain val
accuracy 0.814 	 roc_auc_score 0.895
confusion_matrix
[[216  72]
 [ 35 253]]
classification_report
              precision    recall  f1-score   support

           0       0.86      0.75      0.80       288
           1       0.78      0.88      0.83       288

    accuracy                           0.81       576
   macro avg       0.82      0.81      0.81       576
weighted avg       0.82      0.81      0.81       576

VAL * Acc@1 81.424
 * Acc@1 81.424 Acc@5 0.000
accuracy 0.972 	 size: 144 	 dog(outdoor)
accuracy 0.806 	 size: 144 	 cat(indoor)
accuracy 0.785 	 size: 144 	 dog(indoor)
accuracy 0.694 	 size: 144 	 cat(outdoor)
step_vals {'loss': 0.2938311994075775}
step_vals {'loss': 0.3306925296783447}
step_vals {'loss': 0.6190306544303894}
step_vals {'loss': 0.4305766820907593}
step_vals {'loss': 0.30728957056999207}
step_vals {'loss': 0.5197385549545288}
step_vals {'loss': 0.351116418838501}
step_vals {'loss': 0.37016764283180237}
step_vals {'loss': 0.2937968671321869}
step_vals {'loss': 0.49757224321365356}
step_vals {'loss': 0.22724702954292297}
step_vals {'loss': 0.3661227822303772}
step_vals {'loss': 0.5010581016540527}
step_vals {'loss': 0.3491334021091461}
step_vals {'loss': 0.4380115568637848}
step_vals {'loss': 0.33413344621658325}
step_vals {'loss': 0.263653427362442}
step_vals {'loss': 0.4072166681289673}
step_vals {'loss': 0.4881696105003357}
step_vals {'loss': 0.4275237023830414}
Iteration: 40
out-of-domain val
accuracy 0.818 	 roc_auc_score 0.917
confusion_matrix
[[236  52]
 [ 53 235]]
classification_report
              precision    recall  f1-score   support

           0       0.82      0.82      0.82       288
           1       0.82      0.82      0.82       288

    accuracy                           0.82       576
   macro avg       0.82      0.82      0.82       576
weighted avg       0.82      0.82      0.82       576

VAL * Acc@1 81.771
 * Acc@1 81.771 Acc@5 0.000
accuracy 0.931 	 size: 144 	 dog(outdoor)
accuracy 0.889 	 size: 144 	 cat(indoor)
accuracy 0.750 	 size: 144 	 cat(outdoor)
accuracy 0.701 	 size: 144 	 dog(indoor)
step_vals {'loss': 0.2272261083126068}
step_vals {'loss': 0.32689082622528076}
step_vals {'loss': 0.3327343761920929}
step_vals {'loss': 0.5045005679130554}
step_vals {'loss': 0.36366674304008484}
step_vals {'loss': 0.3060109317302704}
step_vals {'loss': 0.3066454827785492}
step_vals {'loss': 0.39386889338493347}
step_vals {'loss': 0.3649263381958008}
step_vals {'loss': 0.3786584734916687}
step_vals {'loss': 0.3321344554424286}
step_vals {'loss': 0.2739378809928894}
step_vals {'loss': 0.30840665102005005}
step_vals {'loss': 0.2791079878807068}
step_vals {'loss': 0.22688743472099304}
step_vals {'loss': 0.22827358543872833}
step_vals {'loss': 0.3811195492744446}
step_vals {'loss': 0.2798110246658325}
step_vals {'loss': 0.2907680571079254}
step_vals {'loss': 0.5064208507537842}
Iteration: 60
out-of-domain val
accuracy 0.816 	 roc_auc_score 0.901
confusion_matrix
[[242  46]
 [ 60 228]]
classification_report
              precision    recall  f1-score   support

           0       0.80      0.84      0.82       288
           1       0.83      0.79      0.81       288

    accuracy                           0.82       576
   macro avg       0.82      0.82      0.82       576
weighted avg       0.82      0.82      0.82       576

VAL * Acc@1 81.597
 * Acc@1 81.597 Acc@5 0.000
accuracy 0.889 	 size: 144 	 cat(indoor)
accuracy 0.868 	 size: 144 	 dog(outdoor)
accuracy 0.792 	 size: 144 	 cat(outdoor)
accuracy 0.715 	 size: 144 	 dog(indoor)
step_vals {'loss': 0.25972115993499756}
step_vals {'loss': 0.21116022765636444}
step_vals {'loss': 0.3152885138988495}
step_vals {'loss': 0.39407479763031006}
step_vals {'loss': 0.18115942180156708}
step_vals {'loss': 0.3854907155036926}
step_vals {'loss': 0.3825560212135315}
step_vals {'loss': 0.19034963846206665}
step_vals {'loss': 0.2594069838523865}
step_vals {'loss': 0.33268702030181885}
step_vals {'loss': 0.2046629637479782}
step_vals {'loss': 0.27302923798561096}
step_vals {'loss': 0.2960209548473358}
step_vals {'loss': 0.1956194043159485}
step_vals {'loss': 0.28958678245544434}
step_vals {'loss': 0.2103433609008789}
step_vals {'loss': 0.23207449913024902}
step_vals {'loss': 0.20523031055927277}
step_vals {'loss': 0.2502230107784271}
step_vals {'loss': 0.16515880823135376}
Iteration: 80
out-of-domain val
accuracy 0.840 	 roc_auc_score 0.913
confusion_matrix
[[249  39]
 [ 53 235]]
classification_report
              precision    recall  f1-score   support

           0       0.82      0.86      0.84       288
           1       0.86      0.82      0.84       288

    accuracy                           0.84       576
   macro avg       0.84      0.84      0.84       576
weighted avg       0.84      0.84      0.84       576

VAL * Acc@1 84.028
 * Acc@1 84.028 Acc@5 0.000
accuracy 0.931 	 size: 144 	 cat(indoor)
accuracy 0.917 	 size: 144 	 dog(outdoor)
accuracy 0.799 	 size: 144 	 cat(outdoor)
accuracy 0.715 	 size: 144 	 dog(indoor)
