Environment:
	Python: 3.6.13
	PyTorch: 1.4.0
	Torchvision: 0.5.0
	CUDA: 10.1
	CUDNN: 7603
	NumPy: 1.19.5
	PIL: 8.1.0
Args:
	algorithm: ERM
	batch_size: 8
	data: /data/GQA/MetaDataset-subpopulation-shift
	hparams: None
	hparams_seed: 0
	log_prefix: 
	num_classes: 2
	num_domains: 2
	output_dir: train_output
	save_model_every_checkpoint: False
	seed: 0
	skip_model_save: False
	workers: 4
train_dataset.samples reverse: [('cat(indoor)', 700), ('dog(outdoor)', 700), ('cat(outdoor)', 150), ('dog(indoor)', 150)]
self.domain_to_groups {0: {'cat': ['cat(indoor)'], 'dog': ['dog(indoor)']}, 1: {'cat': ['cat(outdoor)'], 'dog': ['dog(outdoor)']}}
HParams:
	batch_size: 32
	class_balanced: False
	data_augmentation: True
	lr: 5e-05
	nonlinear_classifier: False
	resnet18: True
	resnet_dropout: 0.0
	weight_decay: 0.0
step_vals {'loss': 0.708216667175293}
Iteration: 0
out-of-domain val
accuracy 0.556 	 roc_auc_score 0.668
confusion_matrix
[[ 45 243]
 [ 13 275]]
classification_report
              precision    recall  f1-score   support

           0       0.78      0.16      0.26       288
           1       0.53      0.95      0.68       288

    accuracy                           0.56       576
   macro avg       0.65      0.56      0.47       576
weighted avg       0.65      0.56      0.47       576

VAL * Acc@1 55.556
 * Acc@1 55.556 Acc@5 0.000
accuracy 0.958 	 size: 144 	 dog(indoor)
accuracy 0.951 	 size: 144 	 dog(outdoor)
accuracy 0.188 	 size: 144 	 cat(indoor)
accuracy 0.125 	 size: 144 	 cat(outdoor)
step_vals {'loss': 0.6392594575881958}
step_vals {'loss': 0.6702041029930115}
step_vals {'loss': 0.5334820747375488}
step_vals {'loss': 0.5399430990219116}
step_vals {'loss': 0.5571776628494263}
step_vals {'loss': 0.4512776732444763}
step_vals {'loss': 0.41051948070526123}
step_vals {'loss': 0.4904266893863678}
step_vals {'loss': 0.6708498001098633}
step_vals {'loss': 0.4310111999511719}
step_vals {'loss': 0.43866410851478577}
step_vals {'loss': 0.45120859146118164}
step_vals {'loss': 0.38525888323783875}
step_vals {'loss': 0.39994877576828003}
step_vals {'loss': 0.4373093843460083}
step_vals {'loss': 0.5182849764823914}
step_vals {'loss': 0.5755912661552429}
step_vals {'loss': 0.3451465368270874}
step_vals {'loss': 0.4101354777812958}
step_vals {'loss': 0.29530036449432373}
Iteration: 20
out-of-domain val
accuracy 0.790 	 roc_auc_score 0.898
confusion_matrix
[[243  45]
 [ 76 212]]
classification_report
              precision    recall  f1-score   support

           0       0.76      0.84      0.80       288
           1       0.82      0.74      0.78       288

    accuracy                           0.79       576
   macro avg       0.79      0.79      0.79       576
weighted avg       0.79      0.79      0.79       576

VAL * Acc@1 78.993
 * Acc@1 78.993 Acc@5 0.000
accuracy 0.896 	 size: 144 	 cat(indoor)
accuracy 0.840 	 size: 144 	 dog(outdoor)
accuracy 0.792 	 size: 144 	 cat(outdoor)
accuracy 0.632 	 size: 144 	 dog(indoor)
step_vals {'loss': 0.5442280173301697}
step_vals {'loss': 0.4136503338813782}
step_vals {'loss': 0.5831597447395325}
step_vals {'loss': 0.43482011556625366}
step_vals {'loss': 0.4926215708255768}
step_vals {'loss': 0.4346597492694855}
step_vals {'loss': 0.620734453201294}
step_vals {'loss': 0.4101578891277313}
step_vals {'loss': 0.3321970999240875}
step_vals {'loss': 0.3995245397090912}
step_vals {'loss': 0.4156452417373657}
step_vals {'loss': 0.2285323590040207}
step_vals {'loss': 0.3127326965332031}
step_vals {'loss': 0.4413031339645386}
step_vals {'loss': 0.438345730304718}
step_vals {'loss': 0.25989824533462524}
step_vals {'loss': 0.3844788074493408}
step_vals {'loss': 0.2753562033176422}
step_vals {'loss': 0.48362284898757935}
step_vals {'loss': 0.3165188133716583}
Iteration: 40
out-of-domain val
accuracy 0.847 	 roc_auc_score 0.925
confusion_matrix
[[228  60]
 [ 28 260]]
classification_report
              precision    recall  f1-score   support

           0       0.89      0.79      0.84       288
           1       0.81      0.90      0.86       288

    accuracy                           0.85       576
   macro avg       0.85      0.85      0.85       576
weighted avg       0.85      0.85      0.85       576

VAL * Acc@1 84.722
 * Acc@1 84.722 Acc@5 0.000
accuracy 0.965 	 size: 144 	 dog(outdoor)
accuracy 0.840 	 size: 144 	 cat(indoor)
accuracy 0.840 	 size: 144 	 dog(indoor)
accuracy 0.743 	 size: 144 	 cat(outdoor)
step_vals {'loss': 0.49539515376091003}
step_vals {'loss': 0.18555134534835815}
step_vals {'loss': 0.3710169792175293}
step_vals {'loss': 0.3674470782279968}
step_vals {'loss': 0.24255965650081635}
step_vals {'loss': 0.4863223135471344}
step_vals {'loss': 0.580361008644104}
step_vals {'loss': 0.3602485656738281}
step_vals {'loss': 0.2890634834766388}
step_vals {'loss': 0.4988305866718292}
step_vals {'loss': 0.2524101138114929}
step_vals {'loss': 0.2649160325527191}
step_vals {'loss': 0.3908706605434418}
step_vals {'loss': 0.37204253673553467}
step_vals {'loss': 0.30844050645828247}
step_vals {'loss': 0.37504902482032776}
step_vals {'loss': 0.3154239058494568}
step_vals {'loss': 0.29908624291419983}
step_vals {'loss': 0.27164751291275024}
step_vals {'loss': 0.2447838932275772}
Iteration: 60
out-of-domain val
accuracy 0.837 	 roc_auc_score 0.923
confusion_matrix
[[224  64]
 [ 30 258]]
classification_report
              precision    recall  f1-score   support

           0       0.88      0.78      0.83       288
           1       0.80      0.90      0.85       288

    accuracy                           0.84       576
   macro avg       0.84      0.84      0.84       576
weighted avg       0.84      0.84      0.84       576

VAL * Acc@1 83.681
 * Acc@1 83.681 Acc@5 0.000
accuracy 0.972 	 size: 144 	 dog(outdoor)
accuracy 0.833 	 size: 144 	 cat(indoor)
accuracy 0.819 	 size: 144 	 dog(indoor)
accuracy 0.722 	 size: 144 	 cat(outdoor)
step_vals {'loss': 0.3500238060951233}
step_vals {'loss': 0.3651812970638275}
step_vals {'loss': 0.25862517952919006}
step_vals {'loss': 0.23215818405151367}
step_vals {'loss': 0.3828955888748169}
step_vals {'loss': 0.252684623003006}
step_vals {'loss': 0.5391696691513062}
step_vals {'loss': 0.331351637840271}
step_vals {'loss': 0.2889059782028198}
step_vals {'loss': 0.29774585366249084}
step_vals {'loss': 0.5428783297538757}
step_vals {'loss': 0.35231515765190125}
step_vals {'loss': 0.4325850009918213}
step_vals {'loss': 0.2731376588344574}
step_vals {'loss': 0.3806893229484558}
step_vals {'loss': 0.2739350497722626}
step_vals {'loss': 0.2767668664455414}
step_vals {'loss': 0.322948694229126}
step_vals {'loss': 0.3338989317417145}
step_vals {'loss': 0.2652442753314972}
Iteration: 80
out-of-domain val
accuracy 0.809 	 roc_auc_score 0.914
confusion_matrix
[[262  26]
 [ 84 204]]
classification_report
              precision    recall  f1-score   support

           0       0.76      0.91      0.83       288
           1       0.89      0.71      0.79       288

    accuracy                           0.81       576
   macro avg       0.82      0.81      0.81       576
weighted avg       0.82      0.81      0.81       576

VAL * Acc@1 80.903
 * Acc@1 80.903 Acc@5 0.000
accuracy 0.938 	 size: 144 	 cat(indoor)
accuracy 0.882 	 size: 144 	 cat(outdoor)
accuracy 0.771 	 size: 144 	 dog(outdoor)
accuracy 0.646 	 size: 144 	 dog(indoor)
