{"agg_test_acc": 0.5609540633237632, "agg_val_acc": 0.43748960120969405, "args": {"algorithm": "Selective_KD", "checkpoint_freq": 300, "data_dir": "./domainbed/data", "dataset": "VLCS", "holdout_fraction": 0.2, "hparams": "{\r\n    \"resnet18\": false,\r\n    \"resnet_dropout\": 0,\r\n    \"nonlinear_classifier\": false,\r\n    \"data_augmentation\": true,\r\n    \"clip_backbone\": \"ViT-B/32\",\r\n    \"student_model\": \"resnet\",\r\n    \"SMA\": true,\r\n    \"batch_size\": 32\r\n}", "hparams_seed": 0, "output_dir": "sweep/ablation3/outputs/cd494624fcd6af94c03e17685074d9dd", "save_linear_probed_clip": false, "save_model_every_checkpoint": false, "seed": 1200988316, "skip_model_save": false, "steps": 5001, "sweep": true, "task": "domain_generalization", "test_envs": [0], "trial_seed": 1, "uda_holdout_fraction": 0, "visualize": false}, "env0_in_acc": 0.5742049469964664, "env0_out_acc": 0.5477031802120141, "env1_in_acc": 0.4395294117647059, "env1_out_acc": 0.4595103578154426, "env2_in_acc": 0.44478293983244477, "env2_out_acc": 0.4603658536585366, "env3_in_acc": 0.41540170307293595, "env3_out_acc": 0.3925925925925926, "epoch": 0.0, "hparams": {"SMA": true, "batch_size": 32, "class_balanced": false, "class_names": ["bird", "car", "chair", "dog", "person"], "clip_backbone": "ViT-B/32", "data_augmentation": true, "domain_names": ["Caltech101", "LabelMe", "SUN09", "VOC2007"], "lambda1": 0.5, "lambda2": 0.5, "last_k_epoch": 0.25, "lr": 5e-05, "nonlinear_classifier": false, "resnet18": false, "resnet_dropout": 0, "student_model": "resnet", "temperature": 3, "weight_decay": 0.0, "worst_case_p": 0.3333333333333333}, "loss": 3.442826271057129, "mem_gb": 2.333779811859131, "step": 0, "step_time": 1.6644890308380127}
{"agg_test_acc": 0.9814487627601589, "agg_val_acc": 0.8041036238190539, "args": {"algorithm": "Selective_KD", "checkpoint_freq": 300, "data_dir": "./domainbed/data", "dataset": "VLCS", "holdout_fraction": 0.2, "hparams": "{\r\n    \"resnet18\": false,\r\n    \"resnet_dropout\": 0,\r\n    \"nonlinear_classifier\": false,\r\n    \"data_augmentation\": true,\r\n    \"clip_backbone\": \"ViT-B/32\",\r\n    \"student_model\": \"resnet\",\r\n    \"SMA\": true,\r\n    \"batch_size\": 32\r\n}", "hparams_seed": 0, "output_dir": "sweep/ablation3/outputs/cd494624fcd6af94c03e17685074d9dd", "save_linear_probed_clip": false, "save_model_every_checkpoint": false, "seed": 1200988316, "skip_model_save": false, "steps": 5001, "sweep": true, "task": "domain_generalization", "test_envs": [0], "trial_seed": 1, "uda_holdout_fraction": 0, "visualize": false}, "env0_in_acc": 0.980565371024735, "env0_out_acc": 0.9823321554770318, "env1_in_acc": 0.7792941176470588, "env1_out_acc": 0.7175141242937854, "env2_in_acc": 0.8796648895658796, "env2_out_acc": 0.8414634146341463, "env3_in_acc": 0.9178082191780822, "env3_out_acc": 0.8533333333333334, "epoch": 8.480565371024735, "hparams": {"SMA": true, "batch_size": 32, "class_balanced": false, "class_names": ["bird", "car", "chair", "dog", "person"], "clip_backbone": "ViT-B/32", "data_augmentation": true, "domain_names": ["Caltech101", "LabelMe", "SUN09", "VOC2007"], "lambda1": 0.5, "lambda2": 0.5, "last_k_epoch": 0.25, "lr": 5e-05, "nonlinear_classifier": false, "resnet18": false, "resnet_dropout": 0, "student_model": "resnet", "temperature": 3, "weight_decay": 0.0, "worst_case_p": 0.3333333333333333}, "loss": 1.5184456094106038, "mem_gb": 2.594730854034424, "step": 300, "step_time": 0.6162695995966594}
{"agg_test_acc": 0.9854240277758392, "agg_val_acc": 0.818728047742034, "args": {"algorithm": "Selective_KD", "checkpoint_freq": 300, "data_dir": "./domainbed/data", "dataset": "VLCS", "holdout_fraction": 0.2, "hparams": "{\r\n    \"resnet18\": false,\r\n    \"resnet_dropout\": 0,\r\n    \"nonlinear_classifier\": false,\r\n    \"data_augmentation\": true,\r\n    \"clip_backbone\": \"ViT-B/32\",\r\n    \"student_model\": \"resnet\",\r\n    \"SMA\": true,\r\n    \"batch_size\": 32\r\n}", "hparams_seed": 0, "output_dir": "sweep/ablation3/outputs/cd494624fcd6af94c03e17685074d9dd", "save_linear_probed_clip": false, "save_model_every_checkpoint": false, "seed": 1200988316, "skip_model_save": false, "steps": 5001, "sweep": true, "task": "domain_generalization", "test_envs": [0], "trial_seed": 1, "uda_holdout_fraction": 0, "visualize": false}, "env0_in_acc": 0.984982332155477, "env0_out_acc": 0.9858657243816255, "env1_in_acc": 0.8150588235294117, "env1_out_acc": 0.7344632768361582, "env2_in_acc": 0.9238385376999239, "env2_out_acc": 0.850609756097561, "env3_in_acc": 0.9496482784154017, "env3_out_acc": 0.8711111111111111, "epoch": 16.96113074204947, "hparams": {"SMA": true, "batch_size": 32, "class_balanced": false, "class_names": ["bird", "car", "chair", "dog", "person"], "clip_backbone": "ViT-B/32", "data_augmentation": true, "domain_names": ["Caltech101", "LabelMe", "SUN09", "VOC2007"], "lambda1": 0.5, "lambda2": 0.5, "last_k_epoch": 0.25, "lr": 5e-05, "nonlinear_classifier": false, "resnet18": false, "resnet_dropout": 0, "student_model": "resnet", "temperature": 3, "weight_decay": 0.0, "worst_case_p": 0.3333333333333333}, "loss": 1.197332810163498, "mem_gb": 2.594730854034424, "step": 600, "step_time": 0.5957393050193787}
{"agg_test_acc": 0.988957596678666, "agg_val_acc": 0.8228809218494285, "args": {"algorithm": "Selective_KD", "checkpoint_freq": 300, "data_dir": "./domainbed/data", "dataset": "VLCS", "holdout_fraction": 0.2, "hparams": "{\r\n    \"resnet18\": false,\r\n    \"resnet_dropout\": 0,\r\n    \"nonlinear_classifier\": false,\r\n    \"data_augmentation\": true,\r\n    \"clip_backbone\": \"ViT-B/32\",\r\n    \"student_model\": \"resnet\",\r\n    \"SMA\": true,\r\n    \"batch_size\": 32\r\n}", "hparams_seed": 0, "output_dir": "sweep/ablation3/outputs/cd494624fcd6af94c03e17685074d9dd", "save_linear_probed_clip": false, "save_model_every_checkpoint": false, "seed": 1200988316, "skip_model_save": false, "steps": 5001, "sweep": true, "task": "domain_generalization", "test_envs": [0], "trial_seed": 1, "uda_holdout_fraction": 0, "visualize": false}, "env0_in_acc": 0.984982332155477, "env0_out_acc": 0.9929328621908127, "env1_in_acc": 0.8390588235294117, "env1_out_acc": 0.7589453860640302, "env2_in_acc": 0.9463061690784463, "env2_out_acc": 0.8445121951219512, "env3_in_acc": 0.9674194742687894, "env3_out_acc": 0.8651851851851852, "epoch": 25.441696113074205, "hparams": {"SMA": true, "batch_size": 32, "class_balanced": false, "class_names": ["bird", "car", "chair", "dog", "person"], "clip_backbone": "ViT-B/32", "data_augmentation": true, "domain_names": ["Caltech101", "LabelMe", "SUN09", "VOC2007"], "lambda1": 0.5, "lambda2": 0.5, "last_k_epoch": 0.25, "lr": 5e-05, "nonlinear_classifier": false, "resnet18": false, "resnet_dropout": 0, "student_model": "resnet", "temperature": 3, "weight_decay": 0.0, "worst_case_p": 0.3333333333333333}, "loss": 1.0390525784095128, "mem_gb": 2.594730854034424, "step": 900, "step_time": 0.6076529534657796}
{"agg_test_acc": 0.9893992927915193, "agg_val_acc": 0.8194732924352871, "args": {"algorithm": "Selective_KD", "checkpoint_freq": 300, "data_dir": "./domainbed/data", "dataset": "VLCS", "holdout_fraction": 0.2, "hparams": "{\r\n    \"resnet18\": false,\r\n    \"resnet_dropout\": 0,\r\n    \"nonlinear_classifier\": false,\r\n    \"data_augmentation\": true,\r\n    \"clip_backbone\": \"ViT-B/32\",\r\n    \"student_model\": \"resnet\",\r\n    \"SMA\": true,\r\n    \"batch_size\": 32\r\n}", "hparams_seed": 0, "output_dir": "sweep/ablation3/outputs/cd494624fcd6af94c03e17685074d9dd", "save_linear_probed_clip": false, "save_model_every_checkpoint": false, "seed": 1200988316, "skip_model_save": false, "steps": 5001, "sweep": true, "task": "domain_generalization", "test_envs": [0], "trial_seed": 1, "uda_holdout_fraction": 0, "visualize": false}, "env0_in_acc": 0.9858657243816255, "env0_out_acc": 0.9929328621908127, "env1_in_acc": 0.8757647058823529, "env1_out_acc": 0.7532956685499058, "env2_in_acc": 0.9668697638994669, "env2_out_acc": 0.8399390243902439, "env3_in_acc": 0.9796371714179933, "env3_out_acc": 0.8651851851851852, "epoch": 33.92226148409894, "hparams": {"SMA": true, "batch_size": 32, "class_balanced": false, "class_names": ["bird", "car", "chair", "dog", "person"], "clip_backbone": "ViT-B/32", "data_augmentation": true, "domain_names": ["Caltech101", "LabelMe", "SUN09", "VOC2007"], "lambda1": 0.5, "lambda2": 0.5, "last_k_epoch": 0.25, "lr": 5e-05, "nonlinear_classifier": false, "resnet18": false, "resnet_dropout": 0, "student_model": "resnet", "temperature": 3, "weight_decay": 0.0, "worst_case_p": 0.3333333333333333}, "loss": 0.8806401997804642, "mem_gb": 2.594730854034424, "step": 1200, "step_time": 0.6125505789120992}
{"agg_test_acc": 0.9893992927915193, "agg_val_acc": 0.8308331161521484, "args": {"algorithm": "Selective_KD", "checkpoint_freq": 300, "data_dir": "./domainbed/data", "dataset": "VLCS", "holdout_fraction": 0.2, "hparams": "{\r\n    \"resnet18\": false,\r\n    \"resnet_dropout\": 0,\r\n    \"nonlinear_classifier\": false,\r\n    \"data_augmentation\": true,\r\n    \"clip_backbone\": \"ViT-B/32\",\r\n    \"student_model\": \"resnet\",\r\n    \"SMA\": true,\r\n    \"batch_size\": 32\r\n}", "hparams_seed": 0, "output_dir": "sweep/ablation3/outputs/cd494624fcd6af94c03e17685074d9dd", "save_linear_probed_clip": false, "save_model_every_checkpoint": false, "seed": 1200988316, "skip_model_save": false, "steps": 5001, "sweep": true, "task": "domain_generalization", "test_envs": [0], "trial_seed": 1, "uda_holdout_fraction": 0, "visualize": false}, "env0_in_acc": 0.9858657243816255, "env0_out_acc": 0.9929328621908127, "env1_in_acc": 0.9030588235294118, "env1_out_acc": 0.7645951035781544, "env2_in_acc": 0.9782939832444782, "env2_out_acc": 0.8597560975609756, "env3_in_acc": 0.9863013698630136, "env3_out_acc": 0.8681481481481481, "epoch": 42.40282685512368, "hparams": {"SMA": true, "batch_size": 32, "class_balanced": false, "class_names": ["bird", "car", "chair", "dog", "person"], "clip_backbone": "ViT-B/32", "data_augmentation": true, "domain_names": ["Caltech101", "LabelMe", "SUN09", "VOC2007"], "lambda1": 0.5, "lambda2": 0.5, "last_k_epoch": 0.25, "lr": 5e-05, "nonlinear_classifier": false, "resnet18": false, "resnet_dropout": 0, "student_model": "resnet", "temperature": 3, "weight_decay": 0.0, "worst_case_p": 0.3333333333333333}, "loss": 0.8200278188784917, "mem_gb": 2.594730854034424, "step": 1500, "step_time": 0.6101728987693786}
{"agg_test_acc": 0.9885159005658126, "agg_val_acc": 0.8238696351731436, "args": {"algorithm": "Selective_KD", "checkpoint_freq": 300, "data_dir": "./domainbed/data", "dataset": "VLCS", "holdout_fraction": 0.2, "hparams": "{\r\n    \"resnet18\": false,\r\n    \"resnet_dropout\": 0,\r\n    \"nonlinear_classifier\": false,\r\n    \"data_augmentation\": true,\r\n    \"clip_backbone\": \"ViT-B/32\",\r\n    \"student_model\": \"resnet\",\r\n    \"SMA\": true,\r\n    \"batch_size\": 32\r\n}", "hparams_seed": 0, "output_dir": "sweep/ablation3/outputs/cd494624fcd6af94c03e17685074d9dd", "save_linear_probed_clip": false, "save_model_every_checkpoint": false, "seed": 1200988316, "skip_model_save": false, "steps": 5001, "sweep": true, "task": "domain_generalization", "test_envs": [0], "trial_seed": 1, "uda_holdout_fraction": 0, "visualize": false}, "env0_in_acc": 0.9840989399293286, "env0_out_acc": 0.9929328621908127, "env1_in_acc": 0.9237647058823529, "env1_out_acc": 0.751412429378531, "env2_in_acc": 0.9809596344249809, "env2_out_acc": 0.8490853658536586, "env3_in_acc": 0.9885227693446872, "env3_out_acc": 0.8711111111111111, "epoch": 50.88339222614841, "hparams": {"SMA": true, "batch_size": 32, "class_balanced": false, "class_names": ["bird", "car", "chair", "dog", "person"], "clip_backbone": "ViT-B/32", "data_augmentation": true, "domain_names": ["Caltech101", "LabelMe", "SUN09", "VOC2007"], "lambda1": 0.5, "lambda2": 0.5, "last_k_epoch": 0.25, "lr": 5e-05, "nonlinear_classifier": false, "resnet18": false, "resnet_dropout": 0, "student_model": "resnet", "temperature": 3, "weight_decay": 0.0, "worst_case_p": 0.3333333333333333}, "loss": 0.7813749577601751, "mem_gb": 2.594730854034424, "step": 1800, "step_time": 0.6268576828638712}
{"agg_test_acc": 0.9871908122272526, "agg_val_acc": 0.8213136228436276, "args": {"algorithm": "Selective_KD", "checkpoint_freq": 300, "data_dir": "./domainbed/data", "dataset": "VLCS", "holdout_fraction": 0.2, "hparams": "{\r\n    \"resnet18\": false,\r\n    \"resnet_dropout\": 0,\r\n    \"nonlinear_classifier\": false,\r\n    \"data_augmentation\": true,\r\n    \"clip_backbone\": \"ViT-B/32\",\r\n    \"student_model\": \"resnet\",\r\n    \"SMA\": true,\r\n    \"batch_size\": 32\r\n}", "hparams_seed": 0, "output_dir": "sweep/ablation3/outputs/cd494624fcd6af94c03e17685074d9dd", "save_linear_probed_clip": false, "save_model_every_checkpoint": false, "seed": 1200988316, "skip_model_save": false, "steps": 5001, "sweep": true, "task": "domain_generalization", "test_envs": [0], "trial_seed": 1, "uda_holdout_fraction": 0, "visualize": false}, "env0_in_acc": 0.984982332155477, "env0_out_acc": 0.9893992932862191, "env1_in_acc": 0.9449411764705883, "env1_out_acc": 0.7589453860640302, "env2_in_acc": 0.9874333587204874, "env2_out_acc": 0.8353658536585366, "env3_in_acc": 0.9911144020733061, "env3_out_acc": 0.8696296296296296, "epoch": 59.36395759717315, "hparams": {"SMA": true, "batch_size": 32, "class_balanced": false, "class_names": ["bird", "car", "chair", "dog", "person"], "clip_backbone": "ViT-B/32", "data_augmentation": true, "domain_names": ["Caltech101", "LabelMe", "SUN09", "VOC2007"], "lambda1": 0.5, "lambda2": 0.5, "last_k_epoch": 0.25, "lr": 5e-05, "nonlinear_classifier": false, "resnet18": false, "resnet_dropout": 0, "student_model": "resnet", "temperature": 3, "weight_decay": 0.0, "worst_case_p": 0.3333333333333333}, "loss": 0.7525746484597524, "mem_gb": 2.594730854034424, "step": 2100, "step_time": 0.6308759601910909}
{"agg_test_acc": 0.9867491161143993, "agg_val_acc": 0.8275021942847178, "args": {"algorithm": "Selective_KD", "checkpoint_freq": 300, "data_dir": "./domainbed/data", "dataset": "VLCS", "holdout_fraction": 0.2, "hparams": "{\r\n    \"resnet18\": false,\r\n    \"resnet_dropout\": 0,\r\n    \"nonlinear_classifier\": false,\r\n    \"data_augmentation\": true,\r\n    \"clip_backbone\": \"ViT-B/32\",\r\n    \"student_model\": \"resnet\",\r\n    \"SMA\": true,\r\n    \"batch_size\": 32\r\n}", "hparams_seed": 0, "output_dir": "sweep/ablation3/outputs/cd494624fcd6af94c03e17685074d9dd", "save_linear_probed_clip": false, "save_model_every_checkpoint": false, "seed": 1200988316, "skip_model_save": false, "steps": 5001, "sweep": true, "task": "domain_generalization", "test_envs": [0], "trial_seed": 1, "uda_holdout_fraction": 0, "visualize": false}, "env0_in_acc": 0.9840989399293286, "env0_out_acc": 0.9893992932862191, "env1_in_acc": 0.9505882352941176, "env1_out_acc": 0.7608286252354048, "env2_in_acc": 0.9897182025894897, "env2_out_acc": 0.8490853658536586, "env3_in_acc": 0.9937060348019252, "env3_out_acc": 0.8725925925925926, "epoch": 67.84452296819788, "hparams": {"SMA": true, "batch_size": 32, "class_balanced": false, "class_names": ["bird", "car", "chair", "dog", "person"], "clip_backbone": "ViT-B/32", "data_augmentation": true, "domain_names": ["Caltech101", "LabelMe", "SUN09", "VOC2007"], "lambda1": 0.5, "lambda2": 0.5, "last_k_epoch": 0.25, "lr": 5e-05, "nonlinear_classifier": false, "resnet18": false, "resnet_dropout": 0, "student_model": "resnet", "temperature": 3, "weight_decay": 0.0, "worst_case_p": 0.3333333333333333}, "loss": 0.7385782180229823, "mem_gb": 2.594730854034424, "step": 2400, "step_time": 0.619146712621053}
{"agg_test_acc": 0.9863074200015459, "agg_val_acc": 0.8235669389259431, "args": {"algorithm": "Selective_KD", "checkpoint_freq": 300, "data_dir": "./domainbed/data", "dataset": "VLCS", "holdout_fraction": 0.2, "hparams": "{\r\n    \"resnet18\": false,\r\n    \"resnet_dropout\": 0,\r\n    \"nonlinear_classifier\": false,\r\n    \"data_augmentation\": true,\r\n    \"clip_backbone\": \"ViT-B/32\",\r\n    \"student_model\": \"resnet\",\r\n    \"SMA\": true,\r\n    \"batch_size\": 32\r\n}", "hparams_seed": 0, "output_dir": "sweep/ablation3/outputs/cd494624fcd6af94c03e17685074d9dd", "save_linear_probed_clip": false, "save_model_every_checkpoint": false, "seed": 1200988316, "skip_model_save": false, "steps": 5001, "sweep": true, "task": "domain_generalization", "test_envs": [0], "trial_seed": 1, "uda_holdout_fraction": 0, "visualize": false}, "env0_in_acc": 0.9832155477031802, "env0_out_acc": 0.9893992932862191, "env1_in_acc": 0.9618823529411765, "env1_out_acc": 0.7532956685499058, "env2_in_acc": 0.9923838537699924, "env2_out_acc": 0.8551829268292683, "env3_in_acc": 0.9944465012958164, "env3_out_acc": 0.8622222222222222, "epoch": 76.32508833922262, "hparams": {"SMA": true, "batch_size": 32, "class_balanced": false, "class_names": ["bird", "car", "chair", "dog", "person"], "clip_backbone": "ViT-B/32", "data_augmentation": true, "domain_names": ["Caltech101", "LabelMe", "SUN09", "VOC2007"], "lambda1": 0.5, "lambda2": 0.5, "last_k_epoch": 0.25, "lr": 5e-05, "nonlinear_classifier": false, "resnet18": false, "resnet_dropout": 0, "student_model": "resnet", "temperature": 3, "weight_decay": 0.0, "worst_case_p": 0.3333333333333333}, "loss": 0.7285469025373459, "mem_gb": 2.594730854034424, "step": 2700, "step_time": 0.6198999373118083}
{"agg_test_acc": 0.9849823316629859, "agg_val_acc": 0.8274244277349503, "args": {"algorithm": "Selective_KD", "checkpoint_freq": 300, "data_dir": "./domainbed/data", "dataset": "VLCS", "holdout_fraction": 0.2, "hparams": "{\r\n    \"resnet18\": false,\r\n    \"resnet_dropout\": 0,\r\n    \"nonlinear_classifier\": false,\r\n    \"data_augmentation\": true,\r\n    \"clip_backbone\": \"ViT-B/32\",\r\n    \"student_model\": \"resnet\",\r\n    \"SMA\": true,\r\n    \"batch_size\": 32\r\n}", "hparams_seed": 0, "output_dir": "sweep/ablation3/outputs/cd494624fcd6af94c03e17685074d9dd", "save_linear_probed_clip": false, "save_model_every_checkpoint": false, "seed": 1200988316, "skip_model_save": false, "steps": 5001, "sweep": true, "task": "domain_generalization", "test_envs": [0], "trial_seed": 1, "uda_holdout_fraction": 0, "visualize": false}, "env0_in_acc": 0.980565371024735, "env0_out_acc": 0.9893992932862191, "env1_in_acc": 0.9703529411764706, "env1_out_acc": 0.7664783427495292, "env2_in_acc": 0.9958111195734958, "env2_out_acc": 0.850609756097561, "env3_in_acc": 0.9955572010366531, "env3_out_acc": 0.8651851851851852, "epoch": 84.80565371024736, "hparams": {"SMA": true, "batch_size": 32, "class_balanced": false, "class_names": ["bird", "car", "chair", "dog", "person"], "clip_backbone": "ViT-B/32", "data_augmentation": true, "domain_names": ["Caltech101", "LabelMe", "SUN09", "VOC2007"], "lambda1": 0.5, "lambda2": 0.5, "last_k_epoch": 0.25, "lr": 5e-05, "nonlinear_classifier": false, "resnet18": false, "resnet_dropout": 0, "student_model": "resnet", "temperature": 3, "weight_decay": 0.0, "worst_case_p": 0.3333333333333333}, "loss": 0.7324521619081498, "mem_gb": 2.594730854034424, "step": 3000, "step_time": 0.6271978346506755}
{"agg_test_acc": 0.9849823316629859, "agg_val_acc": 0.8227745494580565, "args": {"algorithm": "Selective_KD", "checkpoint_freq": 300, "data_dir": "./domainbed/data", "dataset": "VLCS", "holdout_fraction": 0.2, "hparams": "{\r\n    \"resnet18\": false,\r\n    \"resnet_dropout\": 0,\r\n    \"nonlinear_classifier\": false,\r\n    \"data_augmentation\": true,\r\n    \"clip_backbone\": \"ViT-B/32\",\r\n    \"student_model\": \"resnet\",\r\n    \"SMA\": true,\r\n    \"batch_size\": 32\r\n}", "hparams_seed": 0, "output_dir": "sweep/ablation3/outputs/cd494624fcd6af94c03e17685074d9dd", "save_linear_probed_clip": false, "save_model_every_checkpoint": false, "seed": 1200988316, "skip_model_save": false, "steps": 5001, "sweep": true, "task": "domain_generalization", "test_envs": [0], "trial_seed": 1, "uda_holdout_fraction": 0, "visualize": false}, "env0_in_acc": 0.980565371024735, "env0_out_acc": 0.9893992932862191, "env1_in_acc": 0.9741176470588235, "env1_out_acc": 0.7645951035781544, "env2_in_acc": 0.9954303122619954, "env2_out_acc": 0.8429878048780488, "env3_in_acc": 0.9970381340244354, "env3_out_acc": 0.8607407407407407, "epoch": 93.28621908127208, "hparams": {"SMA": true, "batch_size": 32, "class_balanced": false, "class_names": ["bird", "car", "chair", "dog", "person"], "clip_backbone": "ViT-B/32", "data_augmentation": true, "domain_names": ["Caltech101", "LabelMe", "SUN09", "VOC2007"], "lambda1": 0.5, "lambda2": 0.5, "last_k_epoch": 0.25, "lr": 5e-05, "nonlinear_classifier": false, "resnet18": false, "resnet_dropout": 0, "student_model": "resnet", "temperature": 3, "weight_decay": 0.0, "worst_case_p": 0.3333333333333333}, "loss": 0.7150592623154323, "mem_gb": 2.594730854034424, "step": 3300, "step_time": 0.6341155012448628}
{"agg_test_acc": 0.9849823316629859, "agg_val_acc": 0.8237214130224216, "args": {"algorithm": "Selective_KD", "checkpoint_freq": 300, "data_dir": "./domainbed/data", "dataset": "VLCS", "holdout_fraction": 0.2, "hparams": "{\r\n    \"resnet18\": false,\r\n    \"resnet_dropout\": 0,\r\n    \"nonlinear_classifier\": false,\r\n    \"data_augmentation\": true,\r\n    \"clip_backbone\": \"ViT-B/32\",\r\n    \"student_model\": \"resnet\",\r\n    \"SMA\": true,\r\n    \"batch_size\": 32\r\n}", "hparams_seed": 0, "output_dir": "sweep/ablation3/outputs/cd494624fcd6af94c03e17685074d9dd", "save_linear_probed_clip": false, "save_model_every_checkpoint": false, "seed": 1200988316, "skip_model_save": false, "steps": 5001, "sweep": true, "task": "domain_generalization", "test_envs": [0], "trial_seed": 1, "uda_holdout_fraction": 0, "visualize": false}, "env0_in_acc": 0.980565371024735, "env0_out_acc": 0.9893992932862191, "env1_in_acc": 0.9778823529411764, "env1_out_acc": 0.7495291902071564, "env2_in_acc": 0.9958111195734958, "env2_out_acc": 0.8475609756097561, "env3_in_acc": 0.9962976675305443, "env3_out_acc": 0.8740740740740741, "epoch": 101.76678445229682, "hparams": {"SMA": true, "batch_size": 32, "class_balanced": false, "class_names": ["bird", "car", "chair", "dog", "person"], "clip_backbone": "ViT-B/32", "data_augmentation": true, "domain_names": ["Caltech101", "LabelMe", "SUN09", "VOC2007"], "lambda1": 0.5, "lambda2": 0.5, "last_k_epoch": 0.25, "lr": 5e-05, "nonlinear_classifier": false, "resnet18": false, "resnet_dropout": 0, "student_model": "resnet", "temperature": 3, "weight_decay": 0.0, "worst_case_p": 0.3333333333333333}, "loss": 0.7105193551381429, "mem_gb": 2.594730854034424, "step": 3600, "step_time": 0.6297298248608907}
{"agg_test_acc": 0.9836572433244257, "agg_val_acc": 0.8267670165000401, "args": {"algorithm": "Selective_KD", "checkpoint_freq": 300, "data_dir": "./domainbed/data", "dataset": "VLCS", "holdout_fraction": 0.2, "hparams": "{\r\n    \"resnet18\": false,\r\n    \"resnet_dropout\": 0,\r\n    \"nonlinear_classifier\": false,\r\n    \"data_augmentation\": true,\r\n    \"clip_backbone\": \"ViT-B/32\",\r\n    \"student_model\": \"resnet\",\r\n    \"SMA\": true,\r\n    \"batch_size\": 32\r\n}", "hparams_seed": 0, "output_dir": "sweep/ablation3/outputs/cd494624fcd6af94c03e17685074d9dd", "save_linear_probed_clip": false, "save_model_every_checkpoint": false, "seed": 1200988316, "skip_model_save": false, "steps": 5001, "sweep": true, "task": "domain_generalization", "test_envs": [0], "trial_seed": 1, "uda_holdout_fraction": 0, "visualize": false}, "env0_in_acc": 0.9814487632508834, "env0_out_acc": 0.9858657243816255, "env1_in_acc": 0.9821176470588235, "env1_out_acc": 0.7721280602636534, "env2_in_acc": 0.9965727341964966, "env2_out_acc": 0.8429878048780488, "env3_in_acc": 0.9977786005183266, "env3_out_acc": 0.8651851851851852, "epoch": 110.24734982332156, "hparams": {"SMA": true, "batch_size": 32, "class_balanced": false, "class_names": ["bird", "car", "chair", "dog", "person"], "clip_backbone": "ViT-B/32", "data_augmentation": true, "domain_names": ["Caltech101", "LabelMe", "SUN09", "VOC2007"], "lambda1": 0.5, "lambda2": 0.5, "last_k_epoch": 0.25, "lr": 5e-05, "nonlinear_classifier": false, "resnet18": false, "resnet_dropout": 0, "student_model": "resnet", "temperature": 3, "weight_decay": 0.0, "worst_case_p": 0.3333333333333333}, "loss": 0.5915089570482572, "mem_gb": 5.391877174377441, "step": 3900, "step_time": 0.6350712958971659}
{"agg_test_acc": 0.9832155472115723, "agg_val_acc": 0.8288619414504745, "args": {"algorithm": "Selective_KD", "checkpoint_freq": 300, "data_dir": "./domainbed/data", "dataset": "VLCS", "holdout_fraction": 0.2, "hparams": "{\r\n    \"resnet18\": false,\r\n    \"resnet_dropout\": 0,\r\n    \"nonlinear_classifier\": false,\r\n    \"data_augmentation\": true,\r\n    \"clip_backbone\": \"ViT-B/32\",\r\n    \"student_model\": \"resnet\",\r\n    \"SMA\": true,\r\n    \"batch_size\": 32\r\n}", "hparams_seed": 0, "output_dir": "sweep/ablation3/outputs/cd494624fcd6af94c03e17685074d9dd", "save_linear_probed_clip": false, "save_model_every_checkpoint": false, "seed": 1200988316, "skip_model_save": false, "steps": 5001, "sweep": true, "task": "domain_generalization", "test_envs": [0], "trial_seed": 1, "uda_holdout_fraction": 0, "visualize": false}, "env0_in_acc": 0.980565371024735, "env0_out_acc": 0.9858657243816255, "env1_in_acc": 0.9835294117647059, "env1_out_acc": 0.7740112994350282, "env2_in_acc": 0.9973343488194973, "env2_out_acc": 0.8414634146341463, "env3_in_acc": 0.9985190670122177, "env3_out_acc": 0.8711111111111111, "epoch": 118.7279151943463, "hparams": {"SMA": true, "batch_size": 32, "class_balanced": false, "class_names": ["bird", "car", "chair", "dog", "person"], "clip_backbone": "ViT-B/32", "data_augmentation": true, "domain_names": ["Caltech101", "LabelMe", "SUN09", "VOC2007"], "lambda1": 0.5, "lambda2": 0.5, "last_k_epoch": 0.25, "lr": 5e-05, "nonlinear_classifier": false, "resnet18": false, "resnet_dropout": 0, "student_model": "resnet", "temperature": 3, "weight_decay": 0.0, "worst_case_p": 0.3333333333333333}, "loss": 0.4384502252936363, "mem_gb": 5.391877174377441, "step": 4200, "step_time": 0.6372417910893758}
{"agg_test_acc": 0.9849823316629859, "agg_val_acc": 0.8242417280182418, "args": {"algorithm": "Selective_KD", "checkpoint_freq": 300, "data_dir": "./domainbed/data", "dataset": "VLCS", "holdout_fraction": 0.2, "hparams": "{\r\n    \"resnet18\": false,\r\n    \"resnet_dropout\": 0,\r\n    \"nonlinear_classifier\": false,\r\n    \"data_augmentation\": true,\r\n    \"clip_backbone\": \"ViT-B/32\",\r\n    \"student_model\": \"resnet\",\r\n    \"SMA\": true,\r\n    \"batch_size\": 32\r\n}", "hparams_seed": 0, "output_dir": "sweep/ablation3/outputs/cd494624fcd6af94c03e17685074d9dd", "save_linear_probed_clip": false, "save_model_every_checkpoint": false, "seed": 1200988316, "skip_model_save": false, "steps": 5001, "sweep": true, "task": "domain_generalization", "test_envs": [0], "trial_seed": 1, "uda_holdout_fraction": 0, "visualize": false}, "env0_in_acc": 0.980565371024735, "env0_out_acc": 0.9893992932862191, "env1_in_acc": 0.9882352941176471, "env1_out_acc": 0.7645951035781544, "env2_in_acc": 0.9961919268849961, "env2_out_acc": 0.8414634146341463, "env3_in_acc": 0.9970381340244354, "env3_out_acc": 0.8666666666666667, "epoch": 127.20848056537102, "hparams": {"SMA": true, "batch_size": 32, "class_balanced": false, "class_names": ["bird", "car", "chair", "dog", "person"], "clip_backbone": "ViT-B/32", "data_augmentation": true, "domain_names": ["Caltech101", "LabelMe", "SUN09", "VOC2007"], "lambda1": 0.5, "lambda2": 0.5, "last_k_epoch": 0.25, "lr": 5e-05, "nonlinear_classifier": false, "resnet18": false, "resnet_dropout": 0, "student_model": "resnet", "temperature": 3, "weight_decay": 0.0, "worst_case_p": 0.3333333333333333}, "loss": 0.4245626196265221, "mem_gb": 5.391877174377441, "step": 4500, "step_time": 0.6350571958223978}
{"agg_test_acc": 0.9854240277758392, "agg_val_acc": 0.8228094072454175, "args": {"algorithm": "Selective_KD", "checkpoint_freq": 300, "data_dir": "./domainbed/data", "dataset": "VLCS", "holdout_fraction": 0.2, "hparams": "{\r\n    \"resnet18\": false,\r\n    \"resnet_dropout\": 0,\r\n    \"nonlinear_classifier\": false,\r\n    \"data_augmentation\": true,\r\n    \"clip_backbone\": \"ViT-B/32\",\r\n    \"student_model\": \"resnet\",\r\n    \"SMA\": true,\r\n    \"batch_size\": 32\r\n}", "hparams_seed": 0, "output_dir": "sweep/ablation3/outputs/cd494624fcd6af94c03e17685074d9dd", "save_linear_probed_clip": false, "save_model_every_checkpoint": false, "seed": 1200988316, "skip_model_save": false, "steps": 5001, "sweep": true, "task": "domain_generalization", "test_envs": [0], "trial_seed": 1, "uda_holdout_fraction": 0, "visualize": false}, "env0_in_acc": 0.9814487632508834, "env0_out_acc": 0.9893992932862191, "env1_in_acc": 0.9882352941176471, "env1_out_acc": 0.7589453860640302, "env2_in_acc": 0.9977151561309977, "env2_out_acc": 0.836890243902439, "env3_in_acc": 0.9996297667530544, "env3_out_acc": 0.8725925925925926, "epoch": 135.68904593639576, "hparams": {"SMA": true, "batch_size": 32, "class_balanced": false, "class_names": ["bird", "car", "chair", "dog", "person"], "clip_backbone": "ViT-B/32", "data_augmentation": true, "domain_names": ["Caltech101", "LabelMe", "SUN09", "VOC2007"], "lambda1": 0.5, "lambda2": 0.5, "last_k_epoch": 0.25, "lr": 5e-05, "nonlinear_classifier": false, "resnet18": false, "resnet_dropout": 0, "student_model": "resnet", "temperature": 3, "weight_decay": 0.0, "worst_case_p": 0.3333333333333333}, "loss": 0.4145470276474953, "mem_gb": 5.391877174377441, "step": 4800, "step_time": 0.645227936108907}
{"agg_test_acc": 0.9854240277758392, "agg_val_acc": 0.8307462396242783, "args": {"algorithm": "Selective_KD", "checkpoint_freq": 300, "data_dir": "./domainbed/data", "dataset": "VLCS", "holdout_fraction": 0.2, "hparams": "{\r\n    \"resnet18\": false,\r\n    \"resnet_dropout\": 0,\r\n    \"nonlinear_classifier\": false,\r\n    \"data_augmentation\": true,\r\n    \"clip_backbone\": \"ViT-B/32\",\r\n    \"student_model\": \"resnet\",\r\n    \"SMA\": true,\r\n    \"batch_size\": 32\r\n}", "hparams_seed": 0, "output_dir": "sweep/ablation3/outputs/cd494624fcd6af94c03e17685074d9dd", "save_linear_probed_clip": false, "save_model_every_checkpoint": false, "seed": 1200988316, "skip_model_save": false, "steps": 5001, "sweep": true, "task": "domain_generalization", "test_envs": [0], "trial_seed": 1, "uda_holdout_fraction": 0, "visualize": false}, "env0_in_acc": 0.9814487632508834, "env0_out_acc": 0.9893992932862191, "env1_in_acc": 0.9849411764705882, "env1_out_acc": 0.7721280602636534, "env2_in_acc": 0.9965727341964966, "env2_out_acc": 0.8460365853658537, "env3_in_acc": 0.9988893002591632, "env3_out_acc": 0.8740740740740741, "epoch": 141.3427561837456, "hparams": {"SMA": true, "batch_size": 32, "class_balanced": false, "class_names": ["bird", "car", "chair", "dog", "person"], "clip_backbone": "ViT-B/32", "data_augmentation": true, "domain_names": ["Caltech101", "LabelMe", "SUN09", "VOC2007"], "lambda1": 0.5, "lambda2": 0.5, "last_k_epoch": 0.25, "lr": 5e-05, "nonlinear_classifier": false, "resnet18": false, "resnet_dropout": 0, "student_model": "resnet", "temperature": 3, "weight_decay": 0.0, "worst_case_p": 0.3333333333333333}, "loss": 0.41012814626097677, "mem_gb": 5.391877174377441, "step": 5000, "step_time": 0.6236763155460358}
