Namespace(device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=65536, lr=0.01, epochs=20000, eval_steps=5, runs=10, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=0.1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.2, rw_step=3, ns_rate=5, hops=3, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=65536, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=65536, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=65536, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=65536, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=65536, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=65536, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
All runs:
Hits@10:
  Final val: 22.73 ± 3.06   Final Test: 12.11 ± 0.64   Final old_old: 13.18 ± 0.79   Final old_new: 7.76 ± 0.42   Final new_new: 2.73 ± 2.49
Hits@20:
  Final val: 32.63 ± 2.93   Final Test: 17.54 ± 1.37   Final old_old: 19.14 ± 1.57   Final old_new: 11.13 ± 0.63   Final new_new: 1.82 ± 1.02
Hits@30:
  Final val: 39.51 ± 3.41   Final Test: 24.30 ± 4.45   Final old_old: 26.31 ± 4.75   Final old_new: 16.35 ± 3.51   Final new_new: 3.64 ± 2.03
Hits@50:
  Final val: 50.93 ± 4.08   Final Test: 31.29 ± 5.09   Final old_old: 33.71 ± 5.38   Final old_new: 21.81 ± 4.12   Final new_new: 4.55 ± 1.61
AUC:
  Final val: 94.49 ± 0.52   Final Test: 93.61 ± 0.45   Final old_old: 94.45 ± 0.48   Final old_new: 90.40 ± 0.59   Final new_new: 82.76 ± 2.04
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
All runs:
Hits@10:
  Final val: 22.73 ± 3.06   Final Test: 12.11 ± 0.64   Final old_old: 13.18 ± 0.79   Final old_new: 7.76 ± 0.42   Final new_new: 2.73 ± 2.49
Hits@20:
  Final val: 32.63 ± 2.93   Final Test: 17.54 ± 1.37   Final old_old: 19.14 ± 1.57   Final old_new: 11.13 ± 0.63   Final new_new: 1.82 ± 1.02
Hits@30:
  Final val: 39.51 ± 3.41   Final Test: 24.30 ± 4.45   Final old_old: 26.31 ± 4.75   Final old_new: 16.35 ± 3.51   Final new_new: 3.64 ± 2.03
Hits@50:
  Final val: 50.93 ± 4.08   Final Test: 31.29 ± 5.09   Final old_old: 33.71 ± 5.38   Final old_new: 21.81 ± 4.12   Final new_new: 4.55 ± 1.61
AUC:
  Final val: 94.49 ± 0.52   Final Test: 93.61 ± 0.45   Final old_old: 94.45 ± 0.48   Final old_new: 90.40 ± 0.59   Final new_new: 82.76 ± 2.04
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
All runs:
Hits@10:
  Final val: 22.73 ± 3.06   Final Test: 12.11 ± 0.64   Final old_old: 13.18 ± 0.79   Final old_new: 7.76 ± 0.42   Final new_new: 2.73 ± 2.49
Hits@20:
  Final val: 32.63 ± 2.93   Final Test: 17.54 ± 1.37   Final old_old: 19.14 ± 1.57   Final old_new: 11.13 ± 0.63   Final new_new: 1.82 ± 1.02
Hits@30:
  Final val: 39.51 ± 3.41   Final Test: 24.30 ± 4.45   Final old_old: 26.31 ± 4.75   Final old_new: 16.35 ± 3.51   Final new_new: 3.64 ± 2.03
Hits@50:
  Final val: 50.93 ± 4.08   Final Test: 31.29 ± 5.09   Final old_old: 33.71 ± 5.38   Final old_new: 21.81 ± 4.12   Final new_new: 4.55 ± 1.61
AUC:
  Final val: 94.49 ± 0.52   Final Test: 93.61 ± 0.45   Final old_old: 94.45 ± 0.48   Final old_new: 90.40 ± 0.59   Final new_new: 82.76 ± 2.04
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
All runs:
Hits@10:
  Final val: 22.73 ± 3.06   Final Test: 12.11 ± 0.64   Final old_old: 13.18 ± 0.79   Final old_new: 7.76 ± 0.42   Final new_new: 2.73 ± 2.49
Hits@20:
  Final val: 32.63 ± 2.93   Final Test: 17.54 ± 1.37   Final old_old: 19.14 ± 1.57   Final old_new: 11.13 ± 0.63   Final new_new: 1.82 ± 1.02
Hits@30:
  Final val: 39.51 ± 3.41   Final Test: 24.30 ± 4.45   Final old_old: 26.31 ± 4.75   Final old_new: 16.35 ± 3.51   Final new_new: 3.64 ± 2.03
Hits@50:
  Final val: 50.93 ± 4.08   Final Test: 31.29 ± 5.09   Final old_old: 33.71 ± 5.38   Final old_new: 21.81 ± 4.12   Final new_new: 4.55 ± 1.61
AUC:
  Final val: 94.49 ± 0.52   Final Test: 93.61 ± 0.45   Final old_old: 94.45 ± 0.48   Final old_new: 90.40 ± 0.59   Final new_new: 82.76 ± 2.04
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
All runs:
Hits@10:
  Final val: 26.17 ± 2.56   Final Test: 15.41 ± 3.75   Final old_old: 16.96 ± 4.27   Final old_new: 9.22 ± 1.74   Final new_new: 0.91 ± 2.03
Hits@20:
  Final val: 38.01 ± 2.61   Final Test: 21.25 ± 3.48   Final old_old: 23.19 ± 3.88   Final old_new: 13.55 ± 1.92   Final new_new: 2.27 ± 2.78
Hits@30:
  Final val: 46.25 ± 3.46   Final Test: 28.38 ± 3.36   Final old_old: 30.87 ± 3.73   Final old_new: 18.51 ± 2.09   Final new_new: 3.64 ± 2.59
Hits@50:
  Final val: 57.84 ± 4.27   Final Test: 40.35 ± 3.33   Final old_old: 43.46 ± 3.47   Final old_new: 28.11 ± 3.08   Final new_new: 7.73 ± 3.80
AUC:
  Final val: 95.55 ± 0.38   Final Test: 94.78 ± 0.36   Final old_old: 95.55 ± 0.31   Final old_new: 91.78 ± 0.66   Final new_new: 86.43 ± 0.90
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
All runs:
Hits@10:
  Final val: 26.17 ± 2.56   Final Test: 15.41 ± 3.75   Final old_old: 16.96 ± 4.27   Final old_new: 9.22 ± 1.74   Final new_new: 0.91 ± 2.03
Hits@20:
  Final val: 38.01 ± 2.61   Final Test: 21.25 ± 3.48   Final old_old: 23.19 ± 3.88   Final old_new: 13.55 ± 1.92   Final new_new: 2.27 ± 2.78
Hits@30:
  Final val: 46.25 ± 3.46   Final Test: 28.38 ± 3.36   Final old_old: 30.87 ± 3.73   Final old_new: 18.51 ± 2.09   Final new_new: 3.64 ± 2.59
Hits@50:
  Final val: 57.84 ± 4.27   Final Test: 40.35 ± 3.33   Final old_old: 43.46 ± 3.47   Final old_new: 28.11 ± 3.08   Final new_new: 7.73 ± 3.80
AUC:
  Final val: 95.55 ± 0.38   Final Test: 94.78 ± 0.36   Final old_old: 95.55 ± 0.31   Final old_new: 91.78 ± 0.66   Final new_new: 86.43 ± 0.90
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
All runs:
Hits@10:
  Final val: 26.17 ± 2.56   Final Test: 15.41 ± 3.75   Final old_old: 16.96 ± 4.27   Final old_new: 9.22 ± 1.74   Final new_new: 0.91 ± 2.03
Hits@20:
  Final val: 38.01 ± 2.61   Final Test: 21.25 ± 3.48   Final old_old: 23.19 ± 3.88   Final old_new: 13.55 ± 1.92   Final new_new: 2.27 ± 2.78
Hits@30:
  Final val: 46.25 ± 3.46   Final Test: 28.38 ± 3.36   Final old_old: 30.87 ± 3.73   Final old_new: 18.51 ± 2.09   Final new_new: 3.64 ± 2.59
Hits@50:
  Final val: 57.84 ± 4.27   Final Test: 40.35 ± 3.33   Final old_old: 43.46 ± 3.47   Final old_new: 28.11 ± 3.08   Final new_new: 7.73 ± 3.80
AUC:
  Final val: 95.55 ± 0.38   Final Test: 94.78 ± 0.36   Final old_old: 95.55 ± 0.31   Final old_new: 91.78 ± 0.66   Final new_new: 86.43 ± 0.90
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
All runs:
Hits@10:
  Final val: 26.17 ± 2.56   Final Test: 15.41 ± 3.75   Final old_old: 16.96 ± 4.27   Final old_new: 9.22 ± 1.74   Final new_new: 0.91 ± 2.03
Hits@20:
  Final val: 38.01 ± 2.61   Final Test: 21.25 ± 3.48   Final old_old: 23.19 ± 3.88   Final old_new: 13.55 ± 1.92   Final new_new: 2.27 ± 2.78
Hits@30:
  Final val: 46.25 ± 3.46   Final Test: 28.38 ± 3.36   Final old_old: 30.87 ± 3.73   Final old_new: 18.51 ± 2.09   Final new_new: 3.64 ± 2.59
Hits@50:
  Final val: 57.84 ± 4.27   Final Test: 40.35 ± 3.33   Final old_old: 43.46 ± 3.47   Final old_new: 28.11 ± 3.08   Final new_new: 7.73 ± 3.80
AUC:
  Final val: 95.55 ± 0.38   Final Test: 94.78 ± 0.36   Final old_old: 95.55 ± 0.31   Final old_new: 91.78 ± 0.66   Final new_new: 86.43 ± 0.90
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
All runs:
Hits@10:
  Final val: 28.87 ± 3.17   Final Test: 17.71 ± 3.09   Final old_old: 19.53 ± 3.37   Final old_new: 10.28 ± 2.12   Final new_new: 2.73 ± 1.90
Hits@20:
  Final val: 39.74 ± 4.55   Final Test: 23.11 ± 4.49   Final old_old: 25.24 ± 4.89   Final old_new: 14.56 ± 3.00   Final new_new: 4.09 ± 2.96
Hits@30:
  Final val: 48.14 ± 4.42   Final Test: 27.54 ± 4.11   Final old_old: 29.96 ± 4.38   Final old_new: 17.86 ± 3.02   Final new_new: 5.00 ± 5.18
Hits@50:
  Final val: 57.91 ± 4.11   Final Test: 37.87 ± 4.87   Final old_old: 40.92 ± 5.39   Final old_new: 25.67 ± 2.74   Final new_new: 9.09 ± 5.57
AUC:
  Final val: 95.55 ± 0.53   Final Test: 94.91 ± 0.49   Final old_old: 95.61 ± 0.51   Final old_new: 92.14 ± 0.48   Final new_new: 87.62 ± 0.69
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 28.87 ± 3.17   Final Test: 17.71 ± 3.09   Final old_old: 19.53 ± 3.37   Final old_new: 10.28 ± 2.12   Final new_new: 2.73 ± 1.90
Hits@20:
  Final val: 39.74 ± 4.55   Final Test: 23.11 ± 4.49   Final old_old: 25.24 ± 4.89   Final old_new: 14.56 ± 3.00   Final new_new: 4.09 ± 2.96
Hits@30:
  Final val: 48.14 ± 4.42   Final Test: 27.54 ± 4.11   Final old_old: 29.96 ± 4.38   Final old_new: 17.86 ± 3.02   Final new_new: 5.00 ± 5.18
Hits@50:
  Final val: 57.91 ± 4.11   Final Test: 37.87 ± 4.87   Final old_old: 40.92 ± 5.39   Final old_new: 25.67 ± 2.74   Final new_new: 9.09 ± 5.57
AUC:
  Final val: 95.55 ± 0.53   Final Test: 94.91 ± 0.49   Final old_old: 95.61 ± 0.51   Final old_new: 92.14 ± 0.48   Final new_new: 87.62 ± 0.69
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 28.87 ± 3.17   Final Test: 17.71 ± 3.09   Final old_old: 19.53 ± 3.37   Final old_new: 10.28 ± 2.12   Final new_new: 2.73 ± 1.90
Hits@20:
  Final val: 39.74 ± 4.55   Final Test: 23.11 ± 4.49   Final old_old: 25.24 ± 4.89   Final old_new: 14.56 ± 3.00   Final new_new: 4.09 ± 2.96
Hits@30:
  Final val: 48.14 ± 4.42   Final Test: 27.54 ± 4.11   Final old_old: 29.96 ± 4.38   Final old_new: 17.86 ± 3.02   Final new_new: 5.00 ± 5.18
Hits@50:
  Final val: 57.91 ± 4.11   Final Test: 37.87 ± 4.87   Final old_old: 40.92 ± 5.39   Final old_new: 25.67 ± 2.74   Final new_new: 9.09 ± 5.57
AUC:
  Final val: 95.55 ± 0.53   Final Test: 94.91 ± 0.49   Final old_old: 95.61 ± 0.51   Final old_new: 92.14 ± 0.48   Final new_new: 87.62 ± 0.69
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 28.87 ± 3.17   Final Test: 17.71 ± 3.09   Final old_old: 19.53 ± 3.37   Final old_new: 10.28 ± 2.12   Final new_new: 2.73 ± 1.90
Hits@20:
  Final val: 39.74 ± 4.55   Final Test: 23.11 ± 4.49   Final old_old: 25.24 ± 4.89   Final old_new: 14.56 ± 3.00   Final new_new: 4.09 ± 2.96
Hits@30:
  Final val: 48.14 ± 4.42   Final Test: 27.54 ± 4.11   Final old_old: 29.96 ± 4.38   Final old_new: 17.86 ± 3.02   Final new_new: 5.00 ± 5.18
Hits@50:
  Final val: 57.91 ± 4.11   Final Test: 37.87 ± 4.87   Final old_old: 40.92 ± 5.39   Final old_new: 25.67 ± 2.74   Final new_new: 9.09 ± 5.57
AUC:
  Final val: 95.55 ± 0.53   Final Test: 94.91 ± 0.49   Final old_old: 95.61 ± 0.51   Final old_new: 92.14 ± 0.48   Final new_new: 87.62 ± 0.69
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 22.14 ± 2.19   Final Test: 11.86 ± 1.41   Final old_old: 13.01 ± 1.46   Final old_new: 7.23 ± 1.23   Final new_new: 1.36 ± 1.24
Hits@20:
  Final val: 32.09 ± 3.43   Final Test: 16.37 ± 1.42   Final old_old: 17.88 ± 1.46   Final old_new: 10.28 ± 1.51   Final new_new: 2.73 ± 1.02
Hits@30:
  Final val: 40.16 ± 4.42   Final Test: 22.45 ± 2.96   Final old_old: 24.41 ± 3.12   Final old_new: 14.66 ± 2.51   Final new_new: 2.73 ± 1.02
Hits@50:
  Final val: 49.72 ± 4.93   Final Test: 30.99 ± 3.24   Final old_old: 33.38 ± 3.38   Final old_new: 21.66 ± 2.81   Final new_new: 4.09 ± 1.02
AUC:
  Final val: 94.36 ± 0.65   Final Test: 93.52 ± 0.51   Final old_old: 94.36 ± 0.46   Final old_new: 90.33 ± 0.70   Final new_new: 82.55 ± 2.03
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 22.55 ± 1.93   Final Test: 10.67 ± 1.75   Final old_old: 11.61 ± 1.98   Final old_new: 6.95 ± 1.16   Final new_new: 1.36 ± 1.24
Hits@20:
  Final val: 31.47 ± 2.12   Final Test: 16.34 ± 2.14   Final old_old: 17.80 ± 2.34   Final old_new: 10.60 ± 1.37   Final new_new: 1.36 ± 1.24
Hits@30:
  Final val: 39.19 ± 3.25   Final Test: 22.27 ± 2.60   Final old_old: 24.24 ± 2.72   Final old_new: 14.48 ± 2.31   Final new_new: 2.27 ± 1.61
Hits@50:
  Final val: 49.23 ± 4.14   Final Test: 31.02 ± 5.72   Final old_old: 33.42 ± 6.02   Final old_new: 21.59 ± 4.66   Final new_new: 5.45 ± 3.05
AUC:
  Final val: 94.29 ± 0.59   Final Test: 93.46 ± 0.49   Final old_old: 94.30 ± 0.45   Final old_new: 90.29 ± 0.66   Final new_new: 82.52 ± 2.24
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 24.22 ± 3.00   Final Test: 15.39 ± 3.04   Final old_old: 16.79 ± 3.31   Final old_new: 9.82 ± 2.09   Final new_new: 1.36 ± 1.24
Hits@20:
  Final val: 37.29 ± 5.57   Final Test: 21.65 ± 4.34   Final old_old: 23.63 ± 4.91   Final old_new: 13.75 ± 2.15   Final new_new: 2.73 ± 1.90
Hits@30:
  Final val: 45.48 ± 6.63   Final Test: 27.88 ± 3.71   Final old_old: 30.25 ± 4.11   Final old_new: 18.46 ± 2.54   Final new_new: 4.09 ± 1.90
Hits@50:
  Final val: 55.19 ± 7.26   Final Test: 35.92 ± 5.19   Final old_old: 38.55 ± 5.67   Final old_new: 25.62 ± 3.65   Final new_new: 7.27 ± 1.02
AUC:
  Final val: 95.24 ± 1.13   Final Test: 94.35 ± 1.03   Final old_old: 95.14 ± 1.03   Final old_new: 91.31 ± 1.04   Final new_new: 84.15 ± 2.40
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 25.32 ± 2.08   Final Test: 15.49 ± 4.61   Final old_old: 16.99 ± 5.19   Final old_new: 9.32 ± 2.34   Final new_new: 3.64 ± 3.05
Hits@20:
  Final val: 39.26 ± 4.60   Final Test: 20.95 ± 4.10   Final old_old: 22.92 ± 4.57   Final old_new: 12.97 ± 2.24   Final new_new: 4.09 ± 2.49
Hits@30:
  Final val: 46.40 ± 4.87   Final Test: 27.48 ± 6.11   Final old_old: 29.92 ± 6.76   Final old_new: 17.63 ± 3.53   Final new_new: 6.36 ± 3.73
Hits@50:
  Final val: 57.04 ± 4.96   Final Test: 38.47 ± 5.54   Final old_old: 41.46 ± 5.94   Final old_new: 26.62 ± 4.14   Final new_new: 7.73 ± 3.05
AUC:
  Final val: 95.59 ± 0.64   Final Test: 94.81 ± 0.64   Final old_old: 95.58 ± 0.59   Final old_new: 91.79 ± 0.87   Final new_new: 86.65 ± 1.46
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 24.26 ± 5.19   Final Test: 14.09 ± 3.69   Final old_old: 15.36 ± 3.83   Final old_new: 9.07 ± 3.20   Final new_new: 0.91 ± 1.24
Hits@20:
  Final val: 36.25 ± 7.94   Final Test: 21.67 ± 5.91   Final old_old: 23.58 ± 6.31   Final old_new: 14.08 ± 4.44   Final new_new: 2.27 ± 1.61
Hits@30:
  Final val: 44.11 ± 6.92   Final Test: 27.68 ± 5.17   Final old_old: 29.97 ± 5.57   Final old_new: 18.69 ± 3.68   Final new_new: 3.18 ± 2.59
Hits@50:
  Final val: 53.35 ± 6.96   Final Test: 35.82 ± 7.19   Final old_old: 38.56 ± 7.66   Final old_new: 24.99 ± 5.55   Final new_new: 6.82 ± 2.78
AUC:
  Final val: 95.18 ± 1.00   Final Test: 94.30 ± 0.97   Final old_old: 95.13 ± 0.92   Final old_new: 91.11 ± 1.24   Final new_new: 84.01 ± 1.86
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 25.84 ± 2.24   Final Test: 15.96 ± 6.14   Final old_old: 17.65 ± 6.76   Final old_new: 9.12 ± 3.82   Final new_new: 0.91 ± 1.24
Hits@20:
  Final val: 40.06 ± 3.87   Final Test: 22.18 ± 6.05   Final old_old: 24.34 ± 6.64   Final old_new: 13.43 ± 3.52   Final new_new: 4.09 ± 5.66
Hits@30:
  Final val: 46.95 ± 3.75   Final Test: 29.33 ± 6.05   Final old_old: 31.91 ± 6.63   Final old_new: 18.99 ± 3.74   Final new_new: 5.00 ± 5.18
Hits@50:
  Final val: 58.36 ± 3.91   Final Test: 41.09 ± 4.15   Final old_old: 44.35 ± 4.52   Final old_new: 27.98 ± 2.96   Final new_new: 11.82 ± 5.18
AUC:
  Final val: 95.73 ± 0.51   Final Test: 94.96 ± 0.56   Final old_old: 95.71 ± 0.51   Final old_new: 91.99 ± 0.81   Final new_new: 87.00 ± 1.11
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 28.46 ± 5.73   Final Test: 17.48 ± 3.91   Final old_old: 19.23 ± 4.47   Final old_new: 10.33 ± 1.73   Final new_new: 3.64 ± 2.59
Hits@20:
  Final val: 40.31 ± 5.61   Final Test: 22.31 ± 4.39   Final old_old: 24.44 ± 4.84   Final old_new: 13.65 ± 2.79   Final new_new: 4.09 ± 2.49
Hits@30:
  Final val: 47.44 ± 5.28   Final Test: 30.87 ± 6.10   Final old_old: 33.54 ± 6.69   Final old_new: 20.18 ± 4.03   Final new_new: 6.36 ± 1.90
Hits@50:
  Final val: 58.89 ± 5.21   Final Test: 41.45 ± 5.80   Final old_old: 44.75 ± 6.27   Final old_new: 28.29 ± 4.13   Final new_new: 10.00 ± 2.03
AUC:
  Final val: 95.79 ± 0.64   Final Test: 95.01 ± 0.67   Final old_old: 95.78 ± 0.63   Final old_new: 91.96 ± 0.85   Final new_new: 86.85 ± 1.24
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 29.36 ± 2.77   Final Test: 18.06 ± 2.99   Final old_old: 19.97 ± 3.42   Final old_new: 10.35 ± 1.63   Final new_new: 1.36 ± 2.03
Hits@20:
  Final val: 40.60 ± 3.00   Final Test: 23.88 ± 2.71   Final old_old: 26.25 ± 3.10   Final old_new: 14.41 ± 1.46   Final new_new: 1.36 ± 2.03
Hits@30:
  Final val: 49.14 ± 3.49   Final Test: 30.26 ± 3.07   Final old_old: 33.08 ± 3.41   Final old_new: 19.04 ± 2.16   Final new_new: 2.73 ± 2.96
Hits@50:
  Final val: 60.21 ± 3.62   Final Test: 37.93 ± 3.28   Final old_old: 41.05 ± 3.60   Final old_new: 25.62 ± 2.43   Final new_new: 5.00 ± 1.90
AUC:
  Final val: 95.78 ± 0.46   Final Test: 95.12 ± 0.35   Final old_old: 95.84 ± 0.36   Final old_new: 92.26 ± 0.40   Final new_new: 87.65 ± 1.06
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 26.19 ± 1.55   Final Test: 15.31 ± 3.85   Final old_old: 16.78 ± 4.29   Final old_new: 9.42 ± 2.26   Final new_new: 1.82 ± 2.96
Hits@20:
  Final val: 40.51 ± 5.24   Final Test: 21.91 ± 5.14   Final old_old: 23.99 ± 5.66   Final old_new: 13.63 ± 3.11   Final new_new: 1.82 ± 1.02
Hits@30:
  Final val: 48.64 ± 3.63   Final Test: 29.55 ± 4.85   Final old_old: 32.25 ± 5.41   Final old_new: 18.79 ± 2.71   Final new_new: 3.18 ± 2.59
Hits@50:
  Final val: 59.23 ± 4.37   Final Test: 40.32 ± 4.82   Final old_old: 43.56 ± 5.21   Final old_new: 27.36 ± 3.52   Final new_new: 9.55 ± 1.02
AUC:
  Final val: 95.73 ± 0.60   Final Test: 94.97 ± 0.57   Final old_old: 95.71 ± 0.55   Final old_new: 92.04 ± 0.71   Final new_new: 87.03 ± 0.70
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 26.03 ± 3.90   Final Test: 14.56 ± 2.37   Final old_old: 16.05 ± 2.62   Final old_new: 8.54 ± 1.52   Final new_new: 1.36 ± 2.03
Hits@20:
  Final val: 35.00 ± 5.32   Final Test: 19.41 ± 1.26   Final old_old: 21.17 ± 1.58   Final old_new: 12.22 ± 0.50   Final new_new: 5.45 ± 4.71
Hits@30:
  Final val: 43.29 ± 4.12   Final Test: 25.87 ± 2.43   Final old_old: 28.04 ± 2.65   Final old_new: 17.18 ± 1.80   Final new_new: 5.45 ± 3.45
Hits@50:
  Final val: 53.61 ± 3.84   Final Test: 34.99 ± 2.50   Final old_old: 37.67 ± 2.59   Final old_new: 24.26 ± 2.51   Final new_new: 9.09 ± 3.59
AUC:
  Final val: 95.04 ± 0.47   Final Test: 94.37 ± 0.45   Final old_old: 95.08 ± 0.46   Final old_new: 91.55 ± 0.47   Final new_new: 87.04 ± 1.42
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 27.01 ± 2.21   Final Test: 16.40 ± 2.89   Final old_old: 18.25 ± 3.43   Final old_new: 8.92 ± 0.64   Final new_new: 0.91 ± 1.24
Hits@20:
  Final val: 38.16 ± 3.52   Final Test: 20.15 ± 2.70   Final old_old: 22.07 ± 3.10   Final old_new: 12.44 ± 1.21   Final new_new: 2.73 ± 2.96
Hits@30:
  Final val: 46.94 ± 5.03   Final Test: 28.84 ± 3.26   Final old_old: 31.38 ± 3.73   Final old_new: 18.79 ± 1.47   Final new_new: 2.27 ± 2.27
Hits@50:
  Final val: 56.76 ± 3.54   Final Test: 37.36 ± 3.91   Final old_old: 40.32 ± 4.24   Final old_new: 25.77 ± 2.84   Final new_new: 5.00 ± 1.90
AUC:
  Final val: 95.41 ± 0.49   Final Test: 94.71 ± 0.41   Final old_old: 95.44 ± 0.41   Final old_new: 91.85 ± 0.52   Final new_new: 87.40 ± 0.95
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 21.45 ± 2.36   Final Test: 11.23 ± 2.01   Final old_old: 12.32 ± 2.21   Final old_new: 6.85 ± 1.35   Final new_new: 0.91 ± 1.24
Hits@20:
  Final val: 31.64 ± 2.71   Final Test: 15.47 ± 0.87   Final old_old: 16.93 ± 0.96   Final old_new: 9.65 ± 1.09   Final new_new: 1.36 ± 1.24
Hits@30:
  Final val: 39.40 ± 4.10   Final Test: 21.19 ± 2.13   Final old_old: 23.07 ± 2.26   Final old_new: 13.73 ± 2.03   Final new_new: 2.27 ± 1.61
Hits@50:
  Final val: 49.90 ± 5.29   Final Test: 30.29 ± 3.44   Final old_old: 32.70 ± 3.59   Final old_new: 20.88 ± 3.18   Final new_new: 3.64 ± 2.59
AUC:
  Final val: 94.32 ± 0.51   Final Test: 93.45 ± 0.44   Final old_old: 94.28 ± 0.42   Final old_new: 90.30 ± 0.65   Final new_new: 82.26 ± 2.32
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 22.73 ± 2.95   Final Test: 11.59 ± 3.10   Final old_old: 12.64 ± 3.41   Final old_new: 7.38 ± 1.99   Final new_new: 1.82 ± 1.90
Hits@20:
  Final val: 31.70 ± 4.09   Final Test: 19.42 ± 2.50   Final old_old: 21.11 ± 2.84   Final old_new: 12.67 ± 1.47   Final new_new: 3.18 ± 2.03
Hits@30:
  Final val: 39.70 ± 4.82   Final Test: 24.66 ± 3.51   Final old_old: 26.80 ± 3.87   Final old_new: 16.17 ± 2.13   Final new_new: 3.18 ± 1.24
Hits@50:
  Final val: 51.16 ± 4.96   Final Test: 32.77 ± 4.33   Final old_old: 35.30 ± 4.69   Final old_new: 22.85 ± 2.93   Final new_new: 5.45 ± 2.03
AUC:
  Final val: 94.55 ± 0.74   Final Test: 93.70 ± 0.58   Final old_old: 94.52 ± 0.67   Final old_new: 90.58 ± 0.24   Final new_new: 82.98 ± 1.54
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 28.20 ± 3.96   Final Test: 16.67 ± 2.33   Final old_old: 18.42 ± 2.82   Final old_new: 9.57 ± 0.96   Final new_new: 1.82 ± 2.49
Hits@20:
  Final val: 41.93 ± 4.93   Final Test: 21.46 ± 3.56   Final old_old: 23.47 ± 4.08   Final old_new: 13.50 ± 1.59   Final new_new: 1.36 ± 2.03
Hits@30:
  Final val: 49.95 ± 5.29   Final Test: 28.93 ± 2.45   Final old_old: 31.52 ± 2.95   Final old_new: 18.64 ± 0.96   Final new_new: 2.73 ± 2.96
Hits@50:
  Final val: 59.89 ± 5.26   Final Test: 38.17 ± 5.21   Final old_old: 41.27 ± 5.87   Final old_new: 25.92 ± 2.60   Final new_new: 5.91 ± 1.24
AUC:
  Final val: 95.71 ± 0.66   Final Test: 95.03 ± 0.58   Final old_old: 95.73 ± 0.62   Final old_new: 92.31 ± 0.45   Final new_new: 87.61 ± 0.69
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=65536, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
All runs:
Hits@10:
  Final val: 22.62 ± 3.75   Final Test: 12.14 ± 3.60   Final old_old: 13.20 ± 3.81   Final old_new: 7.88 ± 2.80   Final new_new: 1.82 ± 1.02
Hits@20:
  Final val: 34.03 ± 4.36   Final Test: 18.05 ± 2.88   Final old_old: 19.63 ± 2.96   Final old_new: 11.69 ± 2.60   Final new_new: 3.64 ± 3.05
Hits@30:
  Final val: 41.45 ± 5.51   Final Test: 22.63 ± 3.14   Final old_old: 24.46 ± 3.15   Final old_new: 15.31 ± 3.15   Final new_new: 5.00 ± 2.49
Hits@50:
  Final val: 50.53 ± 5.50   Final Test: 30.83 ± 4.02   Final old_old: 33.26 ± 4.28   Final old_new: 21.18 ± 3.01   Final new_new: 5.91 ± 3.05
AUC:
  Final val: 94.58 ± 0.67   Final Test: 93.70 ± 0.64   Final old_old: 94.51 ± 0.66   Final old_new: 90.60 ± 0.51   Final new_new: 83.24 ± 2.61
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 26.79 ± 2.41   Final Test: 17.39 ± 6.51   Final old_old: 19.18 ± 7.18   Final old_new: 10.18 ± 3.80   Final new_new: 1.36 ± 1.24
Hits@20:
  Final val: 40.08 ± 3.91   Final Test: 23.24 ± 6.32   Final old_old: 25.51 ± 6.94   Final old_new: 14.03 ± 3.75   Final new_new: 4.55 ± 2.78
Hits@30:
  Final val: 46.49 ± 4.10   Final Test: 28.81 ± 6.50   Final old_old: 31.40 ± 7.07   Final old_new: 18.36 ± 4.17   Final new_new: 5.45 ± 4.43
Hits@50:
  Final val: 58.39 ± 4.89   Final Test: 39.78 ± 5.27   Final old_old: 42.93 ± 5.83   Final old_new: 27.25 ± 2.96   Final new_new: 8.64 ± 3.37
AUC:
  Final val: 95.77 ± 0.54   Final Test: 94.96 ± 0.51   Final old_old: 95.72 ± 0.48   Final old_new: 91.95 ± 0.74   Final new_new: 86.98 ± 1.10
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 23.29 ± 2.27   Final Test: 12.49 ± 3.26   Final old_old: 13.67 ± 3.38   Final old_new: 7.73 ± 2.97   Final new_new: 2.27 ± 1.61
Hits@20:
  Final val: 33.13 ± 3.93   Final Test: 19.27 ± 4.83   Final old_old: 20.97 ± 5.19   Final old_new: 12.49 ± 3.46   Final new_new: 2.73 ± 1.90
Hits@30:
  Final val: 43.14 ± 2.32   Final Test: 25.35 ± 3.68   Final old_old: 27.48 ± 3.91   Final old_new: 16.88 ± 3.02   Final new_new: 4.09 ± 1.02
Hits@50:
  Final val: 52.55 ± 3.56   Final Test: 34.12 ± 4.15   Final old_old: 36.82 ± 4.25   Final old_new: 23.45 ± 3.95   Final new_new: 5.91 ± 2.03
AUC:
  Final val: 95.07 ± 0.38   Final Test: 94.24 ± 0.35   Final old_old: 95.01 ± 0.36   Final old_new: 91.22 ± 0.36   Final new_new: 85.91 ± 0.96
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 22.73 ± 3.06   Final Test: 12.11 ± 0.64   Final old_old: 13.18 ± 0.79   Final old_new: 7.76 ± 0.42   Final new_new: 2.73 ± 2.49
Hits@20:
  Final val: 32.63 ± 2.93   Final Test: 17.54 ± 1.37   Final old_old: 19.14 ± 1.57   Final old_new: 11.13 ± 0.63   Final new_new: 1.82 ± 1.02
Hits@30:
  Final val: 39.51 ± 3.41   Final Test: 24.30 ± 4.45   Final old_old: 26.31 ± 4.75   Final old_new: 16.35 ± 3.51   Final new_new: 3.64 ± 2.03
Hits@50:
  Final val: 50.93 ± 4.08   Final Test: 31.29 ± 5.09   Final old_old: 33.71 ± 5.38   Final old_new: 21.81 ± 4.12   Final new_new: 4.55 ± 1.61
AUC:
  Final val: 94.49 ± 0.52   Final Test: 93.61 ± 0.45   Final old_old: 94.45 ± 0.48   Final old_new: 90.40 ± 0.59   Final new_new: 82.76 ± 2.04
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
All runs:
Hits@10:
  Final val: 27.24 ± 1.75   Final Test: 17.79 ± 4.81   Final old_old: 19.74 ± 5.29   Final old_new: 9.87 ± 2.97   Final new_new: 0.91 ± 2.03
Hits@20:
  Final val: 42.51 ± 3.87   Final Test: 23.59 ± 4.94   Final old_old: 25.89 ± 5.39   Final old_new: 14.33 ± 3.19   Final new_new: 2.73 ± 3.73
Hits@30:
  Final val: 49.71 ± 4.14   Final Test: 31.98 ± 5.47   Final old_old: 34.88 ± 6.03   Final old_new: 20.40 ± 3.25   Final new_new: 4.09 ± 4.93
Hits@50:
  Final val: 60.73 ± 4.50   Final Test: 40.62 ± 5.56   Final old_old: 43.77 ± 6.06   Final old_new: 28.09 ± 3.65   Final new_new: 9.09 ± 3.94
AUC:
  Final val: 95.97 ± 0.53   Final Test: 95.14 ± 0.46   Final old_old: 95.89 ± 0.46   Final old_new: 92.19 ± 0.59   Final new_new: 87.04 ± 1.09
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 25.87 ± 2.26   Final Test: 16.38 ± 5.41   Final old_old: 18.12 ± 5.86   Final old_new: 9.29 ± 3.47   Final new_new: 2.73 ± 4.07
Hits@20:
  Final val: 39.71 ± 3.49   Final Test: 20.92 ± 4.21   Final old_old: 23.01 ± 4.66   Final old_new: 12.39 ± 2.46   Final new_new: 4.09 ± 4.93
Hits@30:
  Final val: 46.28 ± 3.56   Final Test: 29.35 ± 5.91   Final old_old: 31.96 ± 6.57   Final old_new: 18.84 ± 3.26   Final new_new: 5.45 ± 3.80
Hits@50:
  Final val: 57.17 ± 4.27   Final Test: 38.93 ± 4.48   Final old_old: 41.98 ± 4.97   Final old_new: 26.70 ± 2.55   Final new_new: 10.00 ± 4.71
AUC:
  Final val: 95.62 ± 0.45   Final Test: 94.85 ± 0.46   Final old_old: 95.61 ± 0.40   Final old_new: 91.84 ± 0.74   Final new_new: 87.34 ± 1.19
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 26.17 ± 2.56   Final Test: 15.41 ± 3.75   Final old_old: 16.96 ± 4.27   Final old_new: 9.22 ± 1.74   Final new_new: 0.91 ± 2.03
Hits@20:
  Final val: 38.01 ± 2.61   Final Test: 21.25 ± 3.48   Final old_old: 23.19 ± 3.88   Final old_new: 13.55 ± 1.92   Final new_new: 2.27 ± 2.78
Hits@30:
  Final val: 46.25 ± 3.46   Final Test: 28.38 ± 3.36   Final old_old: 30.87 ± 3.73   Final old_new: 18.51 ± 2.09   Final new_new: 3.64 ± 2.59
Hits@50:
  Final val: 57.84 ± 4.27   Final Test: 40.35 ± 3.33   Final old_old: 43.46 ± 3.47   Final old_new: 28.11 ± 3.08   Final new_new: 7.73 ± 3.80
AUC:
  Final val: 95.55 ± 0.38   Final Test: 94.78 ± 0.36   Final old_old: 95.55 ± 0.31   Final old_new: 91.78 ± 0.66   Final new_new: 86.43 ± 0.90
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
All runs:
Hits@10:
  Final val: 30.42 ± 3.04   Final Test: 18.86 ± 4.03   Final old_old: 20.86 ± 4.49   Final old_new: 10.78 ± 2.43   Final new_new: 0.91 ± 1.24
Hits@20:
  Final val: 42.41 ± 4.24   Final Test: 25.07 ± 5.58   Final old_old: 27.52 ± 6.05   Final old_new: 15.29 ± 4.15   Final new_new: 1.82 ± 1.90
Hits@30:
  Final val: 50.29 ± 4.67   Final Test: 30.31 ± 5.16   Final old_old: 33.03 ± 5.65   Final old_new: 19.57 ± 3.65   Final new_new: 2.73 ± 1.90
Hits@50:
  Final val: 60.49 ± 5.11   Final Test: 40.39 ± 4.95   Final old_old: 43.65 ± 5.36   Final old_new: 27.58 ± 3.49   Final new_new: 5.45 ± 2.03
AUC:
  Final val: 95.82 ± 0.60   Final Test: 95.14 ± 0.57   Final old_old: 95.85 ± 0.59   Final old_new: 92.37 ± 0.50   Final new_new: 87.47 ± 0.91
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 31.82 ± 2.60   Final Test: 21.15 ± 2.04   Final old_old: 23.21 ± 2.20   Final old_new: 12.90 ± 1.56   Final new_new: 1.82 ± 2.96
Hits@20:
  Final val: 47.65 ± 3.28   Final Test: 29.43 ± 1.53   Final old_old: 32.07 ± 1.53   Final old_new: 18.84 ± 1.83   Final new_new: 5.45 ± 2.03
Hits@30:
  Final val: 57.31 ± 1.17   Final Test: 37.29 ± 1.83   Final old_old: 40.42 ± 1.96   Final old_new: 24.71 ± 1.18   Final new_new: 8.18 ± 4.71
Hits@50:
  Final val: 67.31 ± 2.00   Final Test: 50.24 ± 2.65   Final old_old: 53.86 ± 2.87   Final old_new: 35.82 ± 2.12   Final new_new: 14.55 ± 3.80
AUC:
  Final val: 96.75 ± 0.16   Final Test: 96.09 ± 0.10   Final old_old: 96.71 ± 0.10   Final old_new: 93.71 ± 0.10   Final new_new: 88.74 ± 0.87
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 27.85 ± 3.32   Final Test: 16.86 ± 4.77   Final old_old: 18.60 ± 5.53   Final old_new: 9.82 ± 1.74   Final new_new: 1.82 ± 1.90
Hits@20:
  Final val: 41.15 ± 4.23   Final Test: 22.18 ± 4.42   Final old_old: 24.31 ± 5.06   Final old_new: 13.53 ± 2.12   Final new_new: 3.64 ± 3.45
Hits@30:
  Final val: 49.37 ± 4.66   Final Test: 28.55 ± 5.92   Final old_old: 31.11 ± 6.70   Final old_new: 18.34 ± 2.87   Final new_new: 3.64 ± 2.03
Hits@50:
  Final val: 59.21 ± 4.86   Final Test: 39.08 ± 5.49   Final old_old: 42.21 ± 5.99   Final old_new: 26.75 ± 3.65   Final new_new: 6.36 ± 1.90
AUC:
  Final val: 95.64 ± 0.62   Final Test: 94.94 ± 0.55   Final old_old: 95.65 ± 0.59   Final old_new: 92.17 ± 0.45   Final new_new: 87.38 ± 0.81
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 28.23 ± 2.95   Final Test: 15.45 ± 3.80   Final old_old: 17.09 ± 4.34   Final old_new: 8.87 ± 1.83   Final new_new: 0.91 ± 2.03
Hits@20:
  Final val: 39.35 ± 5.50   Final Test: 22.02 ± 6.56   Final old_old: 24.10 ± 7.41   Final old_new: 13.53 ± 3.37   Final new_new: 5.45 ± 5.23
Hits@30:
  Final val: 47.67 ± 5.63   Final Test: 27.84 ± 6.73   Final old_old: 30.35 ± 7.43   Final old_new: 17.76 ± 4.37   Final new_new: 4.55 ± 3.59
Hits@50:
  Final val: 57.89 ± 6.10   Final Test: 37.63 ± 5.51   Final old_old: 40.71 ± 6.00   Final old_new: 25.37 ± 3.83   Final new_new: 7.73 ± 5.23
AUC:
  Final val: 95.42 ± 0.81   Final Test: 94.70 ± 0.91   Final old_old: 95.43 ± 0.88   Final old_new: 91.87 ± 1.00   Final new_new: 86.83 ± 1.73
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 28.87 ± 3.17   Final Test: 17.71 ± 3.09   Final old_old: 19.53 ± 3.37   Final old_new: 10.28 ± 2.12   Final new_new: 2.73 ± 1.90
Hits@20:
  Final val: 39.74 ± 4.55   Final Test: 23.11 ± 4.49   Final old_old: 25.24 ± 4.89   Final old_new: 14.56 ± 3.00   Final new_new: 4.09 ± 2.96
Hits@30:
  Final val: 48.14 ± 4.42   Final Test: 27.54 ± 4.11   Final old_old: 29.96 ± 4.38   Final old_new: 17.86 ± 3.02   Final new_new: 5.00 ± 5.18
Hits@50:
  Final val: 57.91 ± 4.11   Final Test: 37.87 ± 4.87   Final old_old: 40.92 ± 5.39   Final old_new: 25.67 ± 2.74   Final new_new: 9.09 ± 5.57
AUC:
  Final val: 95.55 ± 0.53   Final Test: 94.91 ± 0.49   Final old_old: 95.61 ± 0.51   Final old_new: 92.14 ± 0.48   Final new_new: 87.62 ± 0.69
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 23.11 ± 2.39   Final Test: 12.28 ± 2.26   Final old_old: 13.38 ± 2.39   Final old_new: 7.91 ± 1.83   Final new_new: 1.36 ± 1.24
Hits@20:
  Final val: 33.42 ± 5.48   Final Test: 19.25 ± 3.05   Final old_old: 20.86 ± 3.40   Final old_new: 12.77 ± 1.65   Final new_new: 4.55 ± 3.21
Hits@30:
  Final val: 42.02 ± 6.37   Final Test: 25.01 ± 4.93   Final old_old: 27.15 ± 5.40   Final old_new: 16.50 ± 3.00   Final new_new: 3.18 ± 2.59
Hits@50:
  Final val: 52.74 ± 6.30   Final Test: 32.67 ± 5.39   Final old_old: 35.20 ± 5.87   Final old_new: 22.75 ± 3.55   Final new_new: 5.00 ± 2.96
AUC:
  Final val: 94.72 ± 1.00   Final Test: 93.88 ± 0.87   Final old_old: 94.65 ± 0.93   Final old_new: 90.97 ± 0.72   Final new_new: 83.79 ± 2.03
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 22.28 ± 2.69   Final Test: 12.54 ± 2.40   Final old_old: 13.73 ± 2.62   Final old_new: 7.76 ± 1.43   Final new_new: 1.82 ± 1.90
Hits@20:
  Final val: 32.14 ± 3.34   Final Test: 16.17 ± 2.62   Final old_old: 17.65 ± 2.84   Final old_new: 10.28 ± 1.93   Final new_new: 1.82 ± 1.02
Hits@30:
  Final val: 39.64 ± 2.80   Final Test: 20.67 ± 1.84   Final old_old: 22.48 ± 2.03   Final old_new: 13.50 ± 1.40   Final new_new: 2.27 ± 1.61
Hits@50:
  Final val: 49.62 ± 3.98   Final Test: 28.25 ± 2.91   Final old_old: 30.56 ± 3.08   Final old_new: 19.14 ± 2.74   Final new_new: 3.18 ± 2.59
AUC:
  Final val: 94.36 ± 0.50   Final Test: 93.46 ± 0.40   Final old_old: 94.29 ± 0.38   Final old_new: 90.31 ± 0.57   Final new_new: 82.77 ± 1.65
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 22.19 ± 1.82   Final Test: 12.06 ± 4.42   Final old_old: 13.19 ± 4.75   Final old_new: 7.48 ± 3.09   Final new_new: 2.27 ± 1.61
Hits@20:
  Final val: 31.80 ± 3.86   Final Test: 18.31 ± 3.74   Final old_old: 19.87 ± 4.18   Final old_new: 11.96 ± 2.20   Final new_new: 5.45 ± 4.71
Hits@30:
  Final val: 40.06 ± 6.00   Final Test: 22.97 ± 5.21   Final old_old: 24.94 ± 5.75   Final old_new: 15.14 ± 2.94   Final new_new: 4.09 ± 3.37
Hits@50:
  Final val: 51.06 ± 8.59   Final Test: 32.04 ± 6.49   Final old_old: 34.42 ± 7.05   Final old_new: 22.57 ± 4.16   Final new_new: 9.09 ± 4.55
AUC:
  Final val: 94.43 ± 1.26   Final Test: 93.61 ± 1.00   Final old_old: 94.38 ± 1.01   Final old_new: 90.68 ± 0.94   Final new_new: 83.68 ± 1.74
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 31.38 ± 2.80   Final Test: 23.02 ± 0.90   Final old_old: 25.35 ± 1.07   Final old_new: 13.68 ± 0.34   Final new_new: 0.91 ± 2.03
Hits@20:
  Final val: 45.64 ± 2.20   Final Test: 27.58 ± 3.55   Final old_old: 30.19 ± 3.80   Final old_new: 17.20 ± 2.55   Final new_new: 1.82 ± 2.96
Hits@30:
  Final val: 54.62 ± 1.41   Final Test: 33.97 ± 1.19   Final old_old: 36.91 ± 1.14   Final old_new: 22.29 ± 1.73   Final new_new: 4.09 ± 2.96
Hits@50:
  Final val: 65.09 ± 1.33   Final Test: 46.92 ± 2.14   Final old_old: 50.43 ± 2.16   Final old_new: 33.05 ± 2.46   Final new_new: 10.45 ± 3.45
AUC:
  Final val: 96.55 ± 0.12   Final Test: 95.96 ± 0.14   Final old_old: 96.52 ± 0.13   Final old_new: 93.78 ± 0.22   Final new_new: 89.30 ± 0.80
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 21.25 ± 2.52   Final Test: 11.56 ± 1.19   Final old_old: 12.66 ± 1.41   Final old_new: 7.13 ± 0.45   Final new_new: 1.82 ± 1.90
Hits@20:
  Final val: 32.73 ± 3.22   Final Test: 17.41 ± 1.61   Final old_old: 19.08 ± 1.83   Final old_new: 10.73 ± 0.90   Final new_new: 1.82 ± 1.02
Hits@30:
  Final val: 40.49 ± 3.32   Final Test: 24.22 ± 2.21   Final old_old: 26.39 ± 2.51   Final old_new: 15.59 ± 1.21   Final new_new: 2.73 ± 1.90
Hits@50:
  Final val: 51.42 ± 3.52   Final Test: 31.78 ± 2.68   Final old_old: 34.43 ± 2.95   Final old_new: 21.21 ± 1.76   Final new_new: 5.45 ± 2.59
AUC:
  Final val: 94.73 ± 0.37   Final Test: 93.81 ± 0.28   Final old_old: 94.71 ± 0.25   Final old_new: 90.33 ± 0.46   Final new_new: 83.24 ± 1.31
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 26.79 ± 2.53   Final Test: 14.98 ± 6.05   Final old_old: 16.44 ± 6.73   Final old_new: 9.07 ± 3.39   Final new_new: 2.73 ± 2.96
Hits@20:
  Final val: 39.65 ± 5.53   Final Test: 20.26 ± 5.11   Final old_old: 22.10 ± 5.74   Final old_new: 12.82 ± 2.87   Final new_new: 4.09 ± 4.37
Hits@30:
  Final val: 46.41 ± 4.90   Final Test: 26.34 ± 5.63   Final old_old: 28.73 ± 6.31   Final old_new: 16.73 ± 3.12   Final new_new: 4.55 ± 3.59
Hits@50:
  Final val: 56.72 ± 4.54   Final Test: 38.38 ± 7.30   Final old_old: 41.27 ± 7.89   Final old_new: 26.83 ± 5.17   Final new_new: 10.91 ± 1.90
AUC:
  Final val: 95.46 ± 0.71   Final Test: 94.67 ± 0.69   Final old_old: 95.40 ± 0.68   Final old_new: 91.82 ± 0.78   Final new_new: 86.66 ± 1.08
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 25.46 ± 1.87   Final Test: 16.17 ± 2.71   Final old_old: 17.70 ± 3.16   Final old_new: 10.08 ± 1.09   Final new_new: 1.36 ± 3.05
Hits@20:
  Final val: 39.48 ± 3.83   Final Test: 20.98 ± 1.55   Final old_old: 22.96 ± 1.79   Final old_new: 13.00 ± 0.96   Final new_new: 3.18 ± 3.45
Hits@30:
  Final val: 47.26 ± 4.15   Final Test: 29.76 ± 3.61   Final old_old: 32.40 ± 4.11   Final old_new: 19.17 ± 2.08   Final new_new: 5.00 ± 1.90
Hits@50:
  Final val: 57.79 ± 3.50   Final Test: 39.09 ± 3.18   Final old_old: 42.15 ± 3.53   Final old_new: 26.93 ± 2.25   Final new_new: 9.09 ± 1.61
AUC:
  Final val: 95.68 ± 0.44   Final Test: 94.91 ± 0.38   Final old_old: 95.67 ± 0.34   Final old_new: 91.91 ± 0.60   Final new_new: 87.32 ± 1.12
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 25.85 ± 1.49   Final Test: 15.38 ± 4.43   Final old_old: 16.89 ± 5.03   Final old_new: 9.32 ± 2.14   Final new_new: 0.91 ± 1.24
Hits@20:
  Final val: 39.89 ± 4.77   Final Test: 20.42 ± 4.53   Final old_old: 22.38 ± 5.18   Final old_new: 12.54 ± 1.92   Final new_new: 2.27 ± 2.78
Hits@30:
  Final val: 47.82 ± 3.67   Final Test: 27.22 ± 5.88   Final old_old: 29.74 ± 6.60   Final old_new: 17.20 ± 2.96   Final new_new: 1.82 ± 1.90
Hits@50:
  Final val: 58.39 ± 4.00   Final Test: 37.62 ± 4.28   Final old_old: 40.75 ± 4.67   Final old_new: 25.14 ± 3.17   Final new_new: 7.73 ± 6.93
AUC:
  Final val: 95.59 ± 0.47   Final Test: 94.80 ± 0.47   Final old_old: 95.55 ± 0.45   Final old_new: 91.83 ± 0.58   Final new_new: 87.25 ± 1.39
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 28.71 ± 2.03   Final Test: 17.00 ± 4.85   Final old_old: 18.72 ± 5.38   Final old_new: 10.13 ± 2.85   Final new_new: 0.91 ± 1.24
Hits@20:
  Final val: 42.36 ± 3.09   Final Test: 24.89 ± 5.65   Final old_old: 27.23 ± 6.15   Final old_new: 15.39 ± 3.78   Final new_new: 5.00 ± 4.07
Hits@30:
  Final val: 50.14 ± 3.43   Final Test: 29.14 ± 6.80   Final old_old: 31.82 ± 7.25   Final old_new: 18.39 ± 5.11   Final new_new: 4.09 ± 2.96
Hits@50:
  Final val: 60.79 ± 3.84   Final Test: 40.22 ± 6.24   Final old_old: 43.36 ± 6.68   Final old_new: 27.71 ± 4.62   Final new_new: 9.55 ± 4.37
AUC:
  Final val: 95.98 ± 0.49   Final Test: 95.20 ± 0.49   Final old_old: 95.92 ± 0.50   Final old_new: 92.37 ± 0.52   Final new_new: 87.34 ± 0.87
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 30.67 ± 7.64   Final Test: 19.66 ± 3.39   Final old_old: 21.31 ± 3.64   Final old_new: 13.02 ± 2.53   Final new_new: 4.09 ± 1.02
Hits@20:
  Final val: 43.70 ± 4.81   Final Test: 28.40 ± 3.34   Final old_old: 30.64 ± 3.64   Final old_new: 19.42 ± 2.23   Final new_new: 7.73 ± 3.05
Hits@30:
  Final val: 51.96 ± 3.71   Final Test: 37.16 ± 3.00   Final old_old: 39.88 ± 3.15   Final old_new: 26.20 ± 2.72   Final new_new: 12.27 ± 2.59
Hits@50:
  Final val: 63.68 ± 4.01   Final Test: 48.63 ± 4.18   Final old_old: 51.90 ± 4.33   Final old_new: 35.57 ± 3.82   Final new_new: 17.73 ± 3.73
AUC:
  Final val: 96.60 ± 0.36   Final Test: 96.00 ± 0.27   Final old_old: 96.53 ± 0.30   Final old_new: 93.97 ± 0.23   Final new_new: 88.70 ± 1.23
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 25.83 ± 4.27   Final Test: 14.52 ± 2.59   Final old_old: 15.98 ± 2.77   Final old_new: 8.66 ± 2.20   Final new_new: 0.91 ± 2.03
Hits@20:
  Final val: 36.56 ± 4.12   Final Test: 20.94 ± 2.19   Final old_old: 22.74 ± 2.48   Final old_new: 13.75 ± 1.47   Final new_new: 3.64 ± 4.43
Hits@30:
  Final val: 45.27 ± 3.86   Final Test: 25.99 ± 2.23   Final old_old: 28.26 ± 2.44   Final old_new: 16.95 ± 1.44   Final new_new: 3.18 ± 2.59
Hits@50:
  Final val: 54.47 ± 2.96   Final Test: 34.04 ± 3.18   Final old_old: 36.72 ± 3.35   Final old_new: 23.45 ± 2.89   Final new_new: 6.36 ± 1.90
AUC:
  Final val: 95.07 ± 0.46   Final Test: 94.44 ± 0.47   Final old_old: 95.13 ± 0.47   Final old_new: 91.66 ± 0.46   Final new_new: 87.52 ± 1.47
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 26.83 ± 3.60   Final Test: 17.27 ± 4.04   Final old_old: 19.01 ± 4.62   Final old_new: 10.25 ± 2.07   Final new_new: 1.82 ± 2.96
Hits@20:
  Final val: 40.64 ± 5.16   Final Test: 23.15 ± 3.70   Final old_old: 25.36 ± 4.13   Final old_new: 14.31 ± 2.23   Final new_new: 2.27 ± 2.78
Hits@30:
  Final val: 48.43 ± 5.02   Final Test: 29.80 ± 4.04   Final old_old: 32.46 ± 4.40   Final old_new: 19.24 ± 2.77   Final new_new: 3.64 ± 2.03
Hits@50:
  Final val: 58.96 ± 5.76   Final Test: 38.50 ± 5.22   Final old_old: 41.65 ± 5.62   Final old_new: 26.05 ± 3.84   Final new_new: 5.91 ± 2.03
AUC:
  Final val: 95.68 ± 0.64   Final Test: 94.98 ± 0.62   Final old_old: 95.69 ± 0.61   Final old_new: 92.20 ± 0.71   Final new_new: 87.64 ± 0.65
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 22.60 ± 2.89   Final Test: 13.07 ± 1.27   Final old_old: 14.22 ± 1.32   Final old_new: 8.44 ± 1.23   Final new_new: 2.73 ± 1.90
Hits@20:
  Final val: 32.35 ± 2.82   Final Test: 17.13 ± 1.26   Final old_old: 18.61 ± 1.37   Final old_new: 11.21 ± 1.16   Final new_new: 2.73 ± 1.90
Hits@30:
  Final val: 39.66 ± 2.93   Final Test: 23.19 ± 2.71   Final old_old: 25.11 ± 3.13   Final old_new: 15.62 ± 1.19   Final new_new: 2.73 ± 1.90
Hits@50:
  Final val: 50.13 ± 4.51   Final Test: 31.54 ± 2.70   Final old_old: 33.95 ± 3.02   Final old_new: 22.14 ± 1.44   Final new_new: 3.64 ± 1.24
AUC:
  Final val: 94.47 ± 0.41   Final Test: 93.58 ± 0.35   Final old_old: 94.44 ± 0.37   Final old_new: 90.29 ± 0.56   Final new_new: 83.06 ± 1.67
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 28.09 ± 6.25   Final Test: 16.10 ± 4.38   Final old_old: 17.76 ± 5.01   Final old_new: 9.37 ± 2.15   Final new_new: 1.82 ± 2.49
Hits@20:
  Final val: 39.30 ± 6.69   Final Test: 22.31 ± 3.18   Final old_old: 24.36 ± 3.87   Final old_new: 14.08 ± 0.56   Final new_new: 3.64 ± 4.71
Hits@30:
  Final val: 47.88 ± 6.00   Final Test: 29.51 ± 4.79   Final old_old: 32.25 ± 5.37   Final old_new: 18.49 ± 2.58   Final new_new: 4.55 ± 2.27
Hits@50:
  Final val: 57.68 ± 5.78   Final Test: 37.52 ± 6.28   Final old_old: 40.57 ± 7.11   Final old_new: 25.39 ± 3.19   Final new_new: 7.73 ± 4.13
AUC:
  Final val: 95.46 ± 0.81   Final Test: 94.82 ± 0.77   Final old_old: 95.52 ± 0.79   Final old_new: 92.02 ± 0.73   Final new_new: 87.65 ± 1.14
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 29.53 ± 3.32   Final Test: 17.30 ± 5.05   Final old_old: 19.19 ± 5.56   Final old_new: 9.65 ± 3.14   Final new_new: 0.45 ± 1.02
Hits@20:
  Final val: 41.97 ± 4.57   Final Test: 25.20 ± 2.54   Final old_old: 27.61 ± 2.89   Final old_new: 15.57 ± 1.27   Final new_new: 1.36 ± 2.03
Hits@30:
  Final val: 50.65 ± 4.38   Final Test: 30.80 ± 2.80   Final old_old: 33.67 ± 3.15   Final old_new: 19.37 ± 1.56   Final new_new: 2.27 ± 1.61
Hits@50:
  Final val: 60.63 ± 4.67   Final Test: 40.49 ± 5.29   Final old_old: 43.76 ± 5.69   Final old_new: 27.56 ± 3.92   Final new_new: 6.36 ± 3.37
AUC:
  Final val: 95.89 ± 0.56   Final Test: 95.24 ± 0.53   Final old_old: 95.94 ± 0.53   Final old_new: 92.50 ± 0.59   Final new_new: 87.76 ± 1.25
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 38.00 ± 6.45   Final Test: 30.97 ± 2.13   Final old_old: 33.55 ± 2.25   Final old_new: 20.76 ± 1.92   Final new_new: 4.55 ± 3.21
Hits@20:
  Final val: 52.29 ± 6.72   Final Test: 39.09 ± 5.27   Final old_old: 41.98 ± 5.65   Final old_new: 27.81 ± 4.14   Final new_new: 6.82 ± 4.55
Hits@30:
  Final val: 62.70 ± 5.28   Final Test: 48.80 ± 1.74   Final old_old: 52.29 ± 1.95   Final old_new: 35.21 ± 1.39   Final new_new: 9.09 ± 5.33
Hits@50:
  Final val: 71.86 ± 4.35   Final Test: 56.93 ± 3.78   Final old_old: 60.38 ± 4.03   Final old_new: 43.53 ± 3.32   Final new_new: 16.36 ± 6.31
AUC:
  Final val: 97.36 ± 0.32   Final Test: 96.89 ± 0.25   Final old_old: 97.37 ± 0.29   Final old_new: 95.02 ± 0.16   Final new_new: 91.55 ± 0.33
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 23.34 ± 1.57   Final Test: 12.49 ± 1.33   Final old_old: 13.59 ± 1.43   Final old_new: 8.11 ± 1.06   Final new_new: 1.82 ± 1.02
Hits@20:
  Final val: 32.92 ± 2.85   Final Test: 17.02 ± 2.80   Final old_old: 18.58 ± 3.07   Final old_new: 10.73 ± 1.84   Final new_new: 3.18 ± 2.59
Hits@30:
  Final val: 41.31 ± 4.44   Final Test: 22.74 ± 3.34   Final old_old: 24.62 ± 3.64   Final old_new: 15.31 ± 2.28   Final new_new: 3.18 ± 2.59
Hits@50:
  Final val: 50.90 ± 5.85   Final Test: 31.20 ± 3.06   Final old_old: 33.67 ± 3.49   Final old_new: 21.49 ± 1.45   Final new_new: 4.55 ± 3.94
AUC:
  Final val: 94.56 ± 0.83   Final Test: 93.70 ± 0.72   Final old_old: 94.55 ± 0.73   Final old_new: 90.43 ± 0.70   Final new_new: 82.96 ± 1.77
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 23.86 ± 3.11   Final Test: 13.83 ± 4.45   Final old_old: 15.23 ± 4.88   Final old_new: 8.21 ± 2.89   Final new_new: 0.45 ± 1.02
Hits@20:
  Final val: 35.00 ± 5.33   Final Test: 21.08 ± 4.31   Final old_old: 23.05 ± 4.74   Final old_new: 13.25 ± 2.72   Final new_new: 1.36 ± 1.24
Hits@30:
  Final val: 42.74 ± 5.06   Final Test: 26.40 ± 4.43   Final old_old: 28.84 ± 4.89   Final old_new: 16.70 ± 2.73   Final new_new: 2.27 ± 1.61
Hits@50:
  Final val: 54.08 ± 5.33   Final Test: 36.25 ± 6.45   Final old_old: 39.25 ± 7.10   Final old_new: 24.46 ± 4.15   Final new_new: 4.55 ± 4.25
AUC:
  Final val: 95.12 ± 0.90   Final Test: 94.40 ± 0.91   Final old_old: 95.21 ± 0.88   Final old_new: 91.29 ± 0.99   Final new_new: 85.07 ± 2.65
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 22.91 ± 1.77   Final Test: 12.50 ± 3.08   Final old_old: 13.68 ± 3.44   Final old_new: 7.78 ± 1.65   Final new_new: 1.36 ± 1.24
Hits@20:
  Final val: 33.54 ± 1.55   Final Test: 17.89 ± 1.76   Final old_old: 19.48 ± 2.00   Final old_new: 11.54 ± 1.11   Final new_new: 2.27 ± 1.61
Hits@30:
  Final val: 41.61 ± 3.31   Final Test: 24.06 ± 3.52   Final old_old: 26.10 ± 3.75   Final old_new: 16.05 ± 2.80   Final new_new: 2.73 ± 1.90
Hits@50:
  Final val: 52.08 ± 3.77   Final Test: 32.33 ± 3.42   Final old_old: 34.92 ± 3.65   Final old_new: 22.17 ± 2.66   Final new_new: 4.55 ± 1.61
AUC:
  Final val: 94.80 ± 0.58   Final Test: 93.85 ± 0.55   Final old_old: 94.72 ± 0.54   Final old_new: 90.51 ± 0.80   Final new_new: 83.35 ± 1.96
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 27.33 ± 2.35   Final Test: 17.94 ± 8.18   Final old_old: 19.67 ± 9.08   Final old_new: 10.91 ± 4.53   Final new_new: 3.18 ± 3.45
Hits@20:
  Final val: 40.98 ± 5.09   Final Test: 25.25 ± 7.00   Final old_old: 27.51 ± 7.58   Final old_new: 16.17 ± 4.70   Final new_new: 4.09 ± 4.93
Hits@30:
  Final val: 48.77 ± 5.73   Final Test: 32.66 ± 7.54   Final old_old: 35.43 ± 8.18   Final old_new: 21.51 ± 5.07   Final new_new: 7.27 ± 3.37
Hits@50:
  Final val: 59.60 ± 4.82   Final Test: 43.49 ± 5.74   Final old_old: 46.94 ± 6.21   Final old_new: 29.85 ± 3.98   Final new_new: 8.64 ± 1.90
AUC:
  Final val: 95.93 ± 0.56   Final Test: 95.18 ± 0.64   Final old_old: 95.94 ± 0.57   Final old_new: 92.20 ± 0.90   Final new_new: 86.88 ± 1.94
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 25.96 ± 1.33   Final Test: 15.15 ± 3.77   Final old_old: 16.67 ± 4.27   Final old_new: 8.94 ± 1.86   Final new_new: 2.73 ± 1.90
Hits@20:
  Final val: 37.96 ± 4.43   Final Test: 19.58 ± 2.69   Final old_old: 21.48 ± 3.09   Final old_new: 11.89 ± 1.36   Final new_new: 3.64 ± 3.45
Hits@30:
  Final val: 45.61 ± 4.19   Final Test: 25.55 ± 3.98   Final old_old: 27.78 ± 4.51   Final old_new: 16.62 ± 2.01   Final new_new: 4.55 ± 2.27
Hits@50:
  Final val: 56.68 ± 4.01   Final Test: 37.29 ± 4.16   Final old_old: 40.28 ± 4.57   Final old_new: 25.34 ± 2.70   Final new_new: 8.64 ± 1.02
AUC:
  Final val: 95.47 ± 0.48   Final Test: 94.68 ± 0.47   Final old_old: 95.45 ± 0.41   Final old_new: 91.64 ± 0.73   Final new_new: 86.73 ± 1.34
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.91 ± 2.71   Final Test: 28.88 ± 1.87   Final old_old: 31.57 ± 2.16   Final old_new: 18.19 ± 0.88   Final new_new: 2.27 ± 1.61
Hits@20:
  Final val: 59.01 ± 3.16   Final Test: 40.60 ± 3.74   Final old_old: 43.69 ± 3.80   Final old_new: 28.64 ± 3.63   Final new_new: 4.55 ± 3.59
Hits@30:
  Final val: 66.24 ± 1.98   Final Test: 51.39 ± 2.39   Final old_old: 54.73 ± 2.55   Final old_new: 38.69 ± 1.87   Final new_new: 7.73 ± 2.59
Hits@50:
  Final val: 74.81 ± 1.34   Final Test: 59.86 ± 1.34   Final old_old: 63.35 ± 1.32   Final old_new: 46.42 ± 1.78   Final new_new: 17.27 ± 4.71
AUC:
  Final val: 97.53 ± 0.07   Final Test: 97.10 ± 0.09   Final old_old: 97.54 ± 0.10   Final old_new: 95.37 ± 0.09   Final new_new: 91.73 ± 0.63
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 26.11 ± 1.56   Final Test: 16.86 ± 5.59   Final old_old: 18.60 ± 6.19   Final old_new: 9.75 ± 3.28   Final new_new: 3.18 ± 3.45
Hits@20:
  Final val: 40.13 ± 4.38   Final Test: 22.99 ± 6.88   Final old_old: 25.13 ± 7.49   Final old_new: 14.38 ± 4.56   Final new_new: 4.09 ± 4.66
Hits@30:
  Final val: 47.15 ± 5.32   Final Test: 30.90 ± 6.09   Final old_old: 33.57 ± 6.74   Final old_new: 20.20 ± 3.53   Final new_new: 6.82 ± 3.59
Hits@50:
  Final val: 59.31 ± 5.87   Final Test: 40.25 ± 4.80   Final old_old: 43.45 ± 5.28   Final old_new: 27.46 ± 3.00   Final new_new: 10.00 ± 3.05
AUC:
  Final val: 95.78 ± 0.70   Final Test: 95.01 ± 0.71   Final old_old: 95.76 ± 0.66   Final old_new: 92.06 ± 0.96   Final new_new: 86.84 ± 1.54
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 30.39 ± 2.94   Final Test: 19.44 ± 4.75   Final old_old: 21.36 ± 5.31   Final old_new: 11.69 ± 2.62   Final new_new: 2.27 ± 2.27
Hits@20:
  Final val: 45.27 ± 2.91   Final Test: 28.12 ± 4.36   Final old_old: 30.82 ± 4.84   Final old_new: 17.28 ± 2.54   Final new_new: 4.09 ± 2.96
Hits@30:
  Final val: 54.06 ± 3.14   Final Test: 35.96 ± 2.52   Final old_old: 39.07 ± 2.67   Final old_new: 23.58 ± 1.82   Final new_new: 5.00 ± 3.37
Hits@50:
  Final val: 64.54 ± 2.96   Final Test: 46.36 ± 2.03   Final old_old: 50.00 ± 2.17   Final old_new: 31.94 ± 1.61   Final new_new: 10.00 ± 2.59
AUC:
  Final val: 96.47 ± 0.37   Final Test: 95.82 ± 0.32   Final old_old: 96.46 ± 0.32   Final old_new: 93.31 ± 0.30   Final new_new: 88.66 ± 0.96
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 27.22 ± 4.37   Final Test: 16.47 ± 3.01   Final old_old: 18.21 ± 3.36   Final old_new: 9.45 ± 1.78   Final new_new: 1.36 ± 2.03
Hits@20:
  Final val: 36.67 ± 5.82   Final Test: 22.48 ± 3.12   Final old_old: 24.45 ± 3.51   Final old_new: 14.61 ± 2.01   Final new_new: 3.64 ± 4.71
Hits@30:
  Final val: 45.63 ± 4.98   Final Test: 28.20 ± 4.94   Final old_old: 30.64 ± 5.41   Final old_new: 18.49 ± 3.24   Final new_new: 4.09 ± 1.90
Hits@50:
  Final val: 56.49 ± 5.97   Final Test: 36.45 ± 4.87   Final old_old: 39.30 ± 5.27   Final old_new: 25.06 ± 3.67   Final new_new: 8.64 ± 3.73
AUC:
  Final val: 95.31 ± 0.69   Final Test: 94.69 ± 0.66   Final old_old: 95.40 ± 0.69   Final old_new: 91.91 ± 0.52   Final new_new: 87.26 ± 1.34
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 22.46 ± 2.66   Final Test: 12.14 ± 0.35   Final old_old: 13.20 ± 0.48   Final old_new: 7.83 ± 0.84   Final new_new: 2.73 ± 1.90
Hits@20:
  Final val: 31.59 ± 2.96   Final Test: 17.16 ± 2.19   Final old_old: 18.65 ± 2.31   Final old_new: 11.26 ± 1.79   Final new_new: 2.27 ± 1.61
Hits@30:
  Final val: 39.07 ± 3.64   Final Test: 21.76 ± 1.43   Final old_old: 23.65 ± 1.51   Final old_new: 14.28 ± 1.49   Final new_new: 2.27 ± 1.61
Hits@50:
  Final val: 48.96 ± 4.45   Final Test: 30.23 ± 3.82   Final old_old: 32.65 ± 3.96   Final old_new: 20.78 ± 3.61   Final new_new: 3.18 ± 2.03
AUC:
  Final val: 94.26 ± 0.42   Final Test: 93.40 ± 0.34   Final old_old: 94.25 ± 0.30   Final old_new: 90.19 ± 0.65   Final new_new: 82.31 ± 1.77
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 28.75 ± 2.16   Final Test: 17.94 ± 2.40   Final old_old: 19.88 ± 2.72   Final old_new: 10.10 ± 1.15   Final new_new: 0.45 ± 1.02
Hits@20:
  Final val: 41.26 ± 4.53   Final Test: 24.42 ± 3.79   Final old_old: 26.80 ± 4.31   Final old_new: 14.96 ± 1.95   Final new_new: 0.91 ± 2.03
Hits@30:
  Final val: 49.70 ± 4.77   Final Test: 30.88 ± 3.15   Final old_old: 33.78 ± 3.55   Final old_new: 19.35 ± 1.73   Final new_new: 1.82 ± 1.90
Hits@50:
  Final val: 60.70 ± 5.23   Final Test: 39.44 ± 3.54   Final old_old: 42.69 ± 3.96   Final old_new: 26.55 ± 2.01   Final new_new: 6.36 ± 2.96
AUC:
  Final val: 95.81 ± 0.55   Final Test: 95.11 ± 0.44   Final old_old: 95.84 ± 0.47   Final old_new: 92.24 ± 0.38   Final new_new: 87.36 ± 0.96
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 24.24 ± 5.22   Final Test: 12.69 ± 5.35   Final old_old: 13.82 ± 5.72   Final old_new: 8.16 ± 4.02   Final new_new: 2.27 ± 0.00
Hits@20:
  Final val: 35.30 ± 5.39   Final Test: 19.39 ± 3.92   Final old_old: 20.95 ± 4.18   Final old_new: 13.25 ± 3.06   Final new_new: 2.27 ± 1.61
Hits@30:
  Final val: 43.42 ± 5.69   Final Test: 24.27 ± 2.42   Final old_old: 26.27 ± 2.65   Final old_new: 16.40 ± 1.56   Final new_new: 2.73 ± 1.02
Hits@50:
  Final val: 52.89 ± 6.24   Final Test: 33.33 ± 6.21   Final old_old: 35.81 ± 6.38   Final old_new: 23.68 ± 5.70   Final new_new: 5.00 ± 2.49
AUC:
  Final val: 94.85 ± 0.85   Final Test: 93.96 ± 0.77   Final old_old: 94.79 ± 0.77   Final old_new: 90.80 ± 0.74   Final new_new: 83.23 ± 2.76
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 28.32 ± 6.15   Final Test: 18.42 ± 4.92   Final old_old: 20.32 ± 5.51   Final old_new: 10.78 ± 2.62   Final new_new: 0.91 ± 2.03
Hits@20:
  Final val: 39.27 ± 7.47   Final Test: 22.96 ± 4.12   Final old_old: 25.12 ± 4.84   Final old_new: 14.31 ± 1.56   Final new_new: 3.18 ± 5.93
Hits@30:
  Final val: 47.83 ± 6.86   Final Test: 30.83 ± 4.69   Final old_old: 33.65 ± 5.25   Final old_new: 19.57 ± 2.81   Final new_new: 3.18 ± 3.45
Hits@50:
  Final val: 59.14 ± 7.24   Final Test: 40.40 ± 4.98   Final old_old: 43.58 ± 5.53   Final old_new: 27.91 ± 3.01   Final new_new: 5.91 ± 4.43
AUC:
  Final val: 95.64 ± 0.91   Final Test: 95.03 ± 0.94   Final old_old: 95.73 ± 0.95   Final old_new: 92.26 ± 0.92   Final new_new: 87.33 ± 1.64
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 33.16 ± 2.40   Final Test: 20.60 ± 4.72   Final old_old: 22.82 ± 5.25   Final old_new: 11.69 ± 2.78   Final new_new: 0.45 ± 1.02
Hits@20:
  Final val: 46.83 ± 1.77   Final Test: 27.80 ± 2.50   Final old_old: 30.64 ± 2.70   Final old_new: 16.42 ± 2.03   Final new_new: 1.82 ± 1.90
Hits@30:
  Final val: 55.97 ± 2.11   Final Test: 33.97 ± 3.08   Final old_old: 37.22 ± 3.33   Final old_new: 20.96 ± 2.28   Final new_new: 2.73 ± 1.90
Hits@50:
  Final val: 66.69 ± 1.57   Final Test: 47.34 ± 3.35   Final old_old: 51.15 ± 3.57   Final old_new: 32.32 ± 2.75   Final new_new: 7.27 ± 4.37
AUC:
  Final val: 96.68 ± 0.17   Final Test: 96.08 ± 0.12   Final old_old: 96.67 ± 0.16   Final old_new: 93.80 ± 0.13   Final new_new: 89.00 ± 0.82
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 28.27 ± 4.07   Final Test: 17.13 ± 5.68   Final old_old: 18.98 ± 6.40   Final old_new: 9.60 ± 2.81   Final new_new: 2.27 ± 2.27
Hits@20:
  Final val: 42.98 ± 3.76   Final Test: 23.75 ± 7.33   Final old_old: 26.12 ± 8.03   Final old_new: 14.18 ± 4.57   Final new_new: 3.18 ± 3.80
Hits@30:
  Final val: 50.30 ± 5.00   Final Test: 32.26 ± 4.24   Final old_old: 35.10 ± 4.75   Final old_new: 20.93 ± 2.14   Final new_new: 5.45 ± 3.80
Hits@50:
  Final val: 61.04 ± 4.71   Final Test: 41.47 ± 4.55   Final old_old: 44.75 ± 5.06   Final old_new: 28.31 ± 2.70   Final new_new: 10.91 ± 4.93
AUC:
  Final val: 96.04 ± 0.52   Final Test: 95.26 ± 0.45   Final old_old: 96.00 ± 0.45   Final old_new: 92.31 ± 0.53   Final new_new: 87.12 ± 0.75
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 22.71 ± 2.83   Final Test: 11.67 ± 1.96   Final old_old: 12.77 ± 2.09   Final old_new: 7.28 ± 1.82   Final new_new: 0.91 ± 1.24
Hits@20:
  Final val: 32.97 ± 2.51   Final Test: 17.21 ± 1.20   Final old_old: 18.73 ± 1.36   Final old_new: 11.16 ± 0.91   Final new_new: 2.27 ± 1.61
Hits@30:
  Final val: 39.80 ± 2.94   Final Test: 23.05 ± 3.07   Final old_old: 25.04 ± 3.43   Final old_new: 15.19 ± 2.01   Final new_new: 2.27 ± 1.61
Hits@50:
  Final val: 50.17 ± 3.89   Final Test: 29.88 ± 5.23   Final old_old: 32.30 ± 5.54   Final old_new: 20.33 ± 4.14   Final new_new: 4.09 ± 1.90
AUC:
  Final val: 94.49 ± 0.43   Final Test: 93.54 ± 0.39   Final old_old: 94.42 ± 0.38   Final old_new: 90.17 ± 0.71   Final new_new: 82.80 ± 2.37
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 27.98 ± 5.04   Final Test: 19.79 ± 5.31   Final old_old: 21.70 ± 6.06   Final old_new: 12.07 ± 2.37   Final new_new: 3.64 ± 2.59
Hits@20:
  Final val: 41.59 ± 5.08   Final Test: 24.89 ± 7.02   Final old_old: 27.29 ± 7.71   Final old_new: 15.11 ± 4.24   Final new_new: 5.91 ± 4.98
Hits@30:
  Final val: 48.94 ± 4.47   Final Test: 30.75 ± 6.82   Final old_old: 33.53 ± 7.34   Final old_new: 19.50 ± 4.95   Final new_new: 6.82 ± 3.94
Hits@50:
  Final val: 59.80 ± 4.53   Final Test: 41.62 ± 4.61   Final old_old: 44.99 ± 5.00   Final old_new: 28.04 ± 3.33   Final new_new: 11.82 ± 3.73
AUC:
  Final val: 95.95 ± 0.57   Final Test: 95.16 ± 0.65   Final old_old: 95.92 ± 0.59   Final old_new: 92.19 ± 0.87   Final new_new: 87.18 ± 1.55
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 26.01 ± 1.68   Final Test: 19.44 ± 3.64   Final old_old: 21.41 ± 3.93   Final old_new: 11.44 ± 2.68   Final new_new: 2.27 ± 1.61
Hits@20:
  Final val: 38.37 ± 4.16   Final Test: 27.50 ± 4.01   Final old_old: 30.00 ± 4.32   Final old_new: 17.53 ± 3.17   Final new_new: 3.18 ± 2.03
Hits@30:
  Final val: 46.41 ± 3.96   Final Test: 34.90 ± 3.29   Final old_old: 37.91 ± 3.71   Final old_new: 23.00 ± 1.73   Final new_new: 4.09 ± 1.90
Hits@50:
  Final val: 55.57 ± 4.29   Final Test: 43.70 ± 5.32   Final old_old: 46.99 ± 5.71   Final old_new: 30.81 ± 3.82   Final new_new: 7.27 ± 3.73
AUC:
  Final val: 93.92 ± 0.44   Final Test: 93.34 ± 0.44   Final old_old: 94.30 ± 0.39   Final old_new: 89.72 ± 0.72   Final new_new: 80.53 ± 1.87
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 24.65 ± 6.79   Final Test: 14.15 ± 5.84   Final old_old: 15.41 ± 6.27   Final old_new: 9.22 ± 4.25   Final new_new: 0.91 ± 1.24
Hits@20:
  Final val: 35.11 ± 7.20   Final Test: 17.96 ± 5.92   Final old_old: 19.53 ± 6.33   Final old_new: 11.71 ± 4.47   Final new_new: 2.27 ± 0.00
Hits@30:
  Final val: 43.15 ± 6.78   Final Test: 24.05 ± 4.69   Final old_old: 26.12 ± 4.90   Final old_new: 15.82 ± 4.01   Final new_new: 3.18 ± 1.24
Hits@50:
  Final val: 53.38 ± 6.71   Final Test: 32.98 ± 6.16   Final old_old: 35.60 ± 6.51   Final old_new: 22.70 ± 4.97   Final new_new: 4.55 ± 2.78
AUC:
  Final val: 94.79 ± 0.91   Final Test: 93.86 ± 0.87   Final old_old: 94.70 ± 0.86   Final old_new: 90.67 ± 0.84   Final new_new: 83.18 ± 2.45
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 26.17 ± 3.93   Final Test: 16.51 ± 2.39   Final old_old: 18.21 ± 2.69   Final old_new: 9.62 ± 1.34   Final new_new: 1.36 ± 2.03
Hits@20:
  Final val: 36.72 ± 6.04   Final Test: 22.13 ± 4.30   Final old_old: 24.16 ± 4.84   Final old_new: 13.95 ± 2.43   Final new_new: 4.09 ± 5.43
Hits@30:
  Final val: 44.99 ± 5.83   Final Test: 29.76 ± 5.18   Final old_old: 32.30 ± 5.63   Final old_new: 19.65 ± 3.68   Final new_new: 4.55 ± 3.59
Hits@50:
  Final val: 56.06 ± 4.78   Final Test: 35.74 ± 3.53   Final old_old: 38.59 ± 3.81   Final old_new: 24.43 ± 2.76   Final new_new: 6.82 ± 5.08
AUC:
  Final val: 95.27 ± 0.74   Final Test: 94.64 ± 0.71   Final old_old: 95.34 ± 0.75   Final old_new: 91.85 ± 0.53   Final new_new: 87.17 ± 1.21
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 27.54 ± 3.10   Final Test: 19.26 ± 6.72   Final old_old: 21.27 ± 7.47   Final old_new: 11.16 ± 3.75   Final new_new: 1.36 ± 1.24
Hits@20:
  Final val: 41.19 ± 4.95   Final Test: 25.71 ± 6.36   Final old_old: 28.15 ± 6.93   Final old_new: 15.92 ± 4.13   Final new_new: 2.73 ± 2.49
Hits@30:
  Final val: 48.53 ± 5.31   Final Test: 33.14 ± 6.27   Final old_old: 36.04 ± 6.87   Final old_new: 21.49 ± 4.06   Final new_new: 6.36 ± 1.90
Hits@50:
  Final val: 59.60 ± 4.84   Final Test: 43.25 ± 3.23   Final old_old: 46.64 ± 3.59   Final old_new: 29.82 ± 1.99   Final new_new: 9.09 ± 3.59
AUC:
  Final val: 95.93 ± 0.55   Final Test: 95.16 ± 0.62   Final old_old: 95.92 ± 0.56   Final old_new: 92.19 ± 0.82   Final new_new: 87.00 ± 1.71
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 27.65 ± 2.31   Final Test: 15.90 ± 3.17   Final old_old: 17.56 ± 3.67   Final old_new: 9.19 ± 1.41   Final new_new: 1.36 ± 2.03
Hits@20:
  Final val: 38.39 ± 3.89   Final Test: 22.06 ± 4.41   Final old_old: 24.17 ± 4.93   Final old_new: 13.58 ± 2.57   Final new_new: 3.18 ± 2.03
Hits@30:
  Final val: 46.59 ± 5.26   Final Test: 28.38 ± 2.37   Final old_old: 30.97 ± 2.78   Final old_new: 18.04 ± 1.85   Final new_new: 3.64 ± 2.03
Hits@50:
  Final val: 57.81 ± 4.93   Final Test: 37.51 ± 3.43   Final old_old: 40.48 ± 3.80   Final old_new: 25.79 ± 2.19   Final new_new: 6.36 ± 3.73
AUC:
  Final val: 95.46 ± 0.52   Final Test: 94.75 ± 0.43   Final old_old: 95.47 ± 0.43   Final old_new: 91.90 ± 0.52   Final new_new: 87.37 ± 0.96
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 25.51 ± 2.13   Final Test: 13.91 ± 2.92   Final old_old: 15.38 ± 3.30   Final old_new: 7.96 ± 1.47   Final new_new: 1.36 ± 2.03
Hits@20:
  Final val: 39.28 ± 4.42   Final Test: 22.19 ± 4.06   Final old_old: 24.25 ± 4.43   Final old_new: 13.93 ± 2.90   Final new_new: 3.18 ± 2.59
Hits@30:
  Final val: 47.60 ± 4.20   Final Test: 30.40 ± 4.51   Final old_old: 33.04 ± 4.97   Final old_new: 19.77 ± 2.96   Final new_new: 7.27 ± 2.49
Hits@50:
  Final val: 57.86 ± 3.61   Final Test: 39.21 ± 4.98   Final old_old: 42.38 ± 5.41   Final old_new: 26.42 ± 3.33   Final new_new: 10.45 ± 2.03
AUC:
  Final val: 95.76 ± 0.58   Final Test: 94.95 ± 0.54   Final old_old: 95.70 ± 0.52   Final old_new: 91.97 ± 0.69   Final new_new: 87.22 ± 0.68
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 34.98 ± 3.84   Final Test: 26.89 ± 1.54   Final old_old: 29.41 ± 1.55   Final old_new: 16.85 ± 2.00   Final new_new: 2.27 ± 1.61
Hits@20:
  Final val: 52.23 ± 5.15   Final Test: 41.13 ± 3.75   Final old_old: 44.36 ± 3.95   Final old_new: 28.41 ± 3.11   Final new_new: 6.82 ± 2.78
Hits@30:
  Final val: 60.31 ± 3.88   Final Test: 48.31 ± 4.07   Final old_old: 51.69 ± 4.15   Final old_new: 35.16 ± 3.87   Final new_new: 10.00 ± 3.45
Hits@50:
  Final val: 68.15 ± 2.66   Final Test: 55.06 ± 3.51   Final old_old: 58.59 ± 3.64   Final old_new: 41.28 ± 3.25   Final new_new: 15.45 ± 3.73
AUC:
  Final val: 95.21 ± 0.09   Final Test: 94.69 ± 0.12   Final old_old: 95.45 ± 0.09   Final old_new: 91.71 ± 0.35   Final new_new: 85.99 ± 1.31
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 23.58 ± 5.45   Final Test: 13.37 ± 3.91   Final old_old: 14.57 ± 4.21   Final old_new: 8.59 ± 2.79   Final new_new: 1.82 ± 1.02
Hits@20:
  Final val: 34.17 ± 5.65   Final Test: 18.38 ± 5.20   Final old_old: 19.96 ± 5.58   Final old_new: 12.09 ± 3.83   Final new_new: 2.27 ± 0.00
Hits@30:
  Final val: 41.85 ± 5.79   Final Test: 23.27 ± 4.00   Final old_old: 25.24 ± 4.32   Final old_new: 15.54 ± 2.86   Final new_new: 2.27 ± 0.00
Hits@50:
  Final val: 52.02 ± 5.28   Final Test: 32.00 ± 4.19   Final old_old: 34.59 ± 4.41   Final old_new: 21.79 ± 3.71   Final new_new: 5.00 ± 1.90
AUC:
  Final val: 94.77 ± 0.77   Final Test: 93.87 ± 0.71   Final old_old: 94.67 ± 0.73   Final old_new: 90.78 ± 0.58   Final new_new: 83.89 ± 2.33
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 21.25 ± 2.54   Final Test: 10.60 ± 2.47   Final old_old: 11.61 ± 2.58   Final old_new: 6.52 ± 2.13   Final new_new: 1.36 ± 1.24
Hits@20:
  Final val: 30.93 ± 3.68   Final Test: 15.59 ± 1.39   Final old_old: 17.00 ± 1.55   Final old_new: 9.95 ± 1.08   Final new_new: 1.82 ± 1.02
Hits@30:
  Final val: 38.67 ± 3.79   Final Test: 22.65 ± 3.55   Final old_old: 24.53 ± 3.78   Final old_new: 15.26 ± 2.87   Final new_new: 2.73 ± 1.90
Hits@50:
  Final val: 48.76 ± 5.41   Final Test: 29.54 ± 4.06   Final old_old: 31.85 ± 4.25   Final old_new: 20.53 ± 3.44   Final new_new: 3.64 ± 2.03
AUC:
  Final val: 94.19 ± 0.63   Final Test: 93.42 ± 0.55   Final old_old: 94.23 ± 0.50   Final old_new: 90.35 ± 0.69   Final new_new: 82.29 ± 2.69
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 30.35 ± 4.51   Final Test: 18.93 ± 3.60   Final old_old: 20.96 ± 4.16   Final old_new: 10.71 ± 1.62   Final new_new: 0.91 ± 2.03
Hits@20:
  Final val: 41.81 ± 5.59   Final Test: 23.50 ± 3.54   Final old_old: 25.86 ± 4.13   Final old_new: 14.06 ± 1.18   Final new_new: 0.91 ± 1.24
Hits@30:
  Final val: 50.01 ± 4.99   Final Test: 31.51 ± 4.00   Final old_old: 34.38 ± 4.54   Final old_new: 20.20 ± 2.28   Final new_new: 1.82 ± 1.02
Hits@50:
  Final val: 61.13 ± 5.60   Final Test: 40.68 ± 5.01   Final old_old: 43.99 ± 5.50   Final old_new: 27.63 ± 3.23   Final new_new: 5.45 ± 3.45
AUC:
  Final val: 95.82 ± 0.62   Final Test: 95.13 ± 0.52   Final old_old: 95.86 ± 0.55   Final old_new: 92.27 ± 0.46   Final new_new: 87.29 ± 1.30
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 26.54 ± 1.46   Final Test: 14.44 ± 4.23   Final old_old: 15.85 ± 4.83   Final old_new: 8.69 ± 1.79   Final new_new: 3.18 ± 3.45
Hits@20:
  Final val: 38.89 ± 4.35   Final Test: 21.02 ± 2.81   Final old_old: 23.03 ± 3.20   Final old_new: 13.00 ± 1.21   Final new_new: 2.27 ± 2.78
Hits@30:
  Final val: 47.06 ± 3.24   Final Test: 27.68 ± 4.52   Final old_old: 30.16 ± 5.13   Final old_new: 17.76 ± 2.14   Final new_new: 4.09 ± 4.07
Hits@50:
  Final val: 58.13 ± 3.58   Final Test: 37.96 ± 3.69   Final old_old: 41.19 ± 4.16   Final old_new: 25.04 ± 2.56   Final new_new: 7.73 ± 2.59
AUC:
  Final val: 95.66 ± 0.45   Final Test: 94.88 ± 0.41   Final old_old: 95.63 ± 0.39   Final old_new: 91.93 ± 0.53   Final new_new: 87.07 ± 0.86
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 27.01 ± 5.39   Final Test: 16.15 ± 3.79   Final old_old: 17.86 ± 4.39   Final old_new: 9.17 ± 1.45   Final new_new: 2.27 ± 2.27
Hits@20:
  Final val: 37.63 ± 7.21   Final Test: 20.53 ± 5.56   Final old_old: 22.46 ± 6.37   Final old_new: 12.72 ± 2.40   Final new_new: 3.64 ± 4.71
Hits@30:
  Final val: 45.84 ± 6.52   Final Test: 27.79 ± 6.13   Final old_old: 30.32 ± 6.71   Final old_new: 17.73 ± 4.04   Final new_new: 2.73 ± 2.49
Hits@50:
  Final val: 56.11 ± 6.33   Final Test: 37.44 ± 5.27   Final old_old: 40.42 ± 5.66   Final old_new: 25.49 ± 3.91   Final new_new: 10.00 ± 3.45
AUC:
  Final val: 95.32 ± 0.77   Final Test: 94.66 ± 0.83   Final old_old: 95.38 ± 0.83   Final old_new: 91.85 ± 0.85   Final new_new: 87.13 ± 1.48
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 39.84 ± 2.81   Final Test: 27.50 ± 1.58   Final old_old: 30.33 ± 1.56   Final old_new: 16.05 ± 1.88   Final new_new: 2.73 ± 1.02
Hits@20:
  Final val: 54.78 ± 3.05   Final Test: 40.34 ± 2.69   Final old_old: 43.62 ± 2.88   Final old_new: 27.20 ± 1.99   Final new_new: 9.55 ± 3.73
Hits@30:
  Final val: 62.23 ± 1.47   Final Test: 47.38 ± 1.10   Final old_old: 50.65 ± 0.99   Final old_new: 34.51 ± 1.93   Final new_new: 12.27 ± 3.45
Hits@50:
  Final val: 68.94 ± 0.63   Final Test: 56.57 ± 0.92   Final old_old: 59.76 ± 0.91   Final old_new: 44.16 ± 1.26   Final new_new: 19.55 ± 2.03
AUC:
  Final val: 94.98 ± 0.06   Final Test: 94.60 ± 0.13   Final old_old: 95.34 ± 0.07   Final old_new: 91.65 ± 0.39   Final new_new: 86.85 ± 0.71
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 21.80 ± 2.06   Final Test: 10.89 ± 2.08   Final old_old: 11.91 ± 2.32   Final old_new: 6.80 ± 1.28   Final new_new: 1.82 ± 1.02
Hits@20:
  Final val: 31.57 ± 3.84   Final Test: 16.46 ± 1.61   Final old_old: 17.90 ± 1.76   Final old_new: 10.76 ± 1.09   Final new_new: 1.36 ± 1.24
Hits@30:
  Final val: 39.19 ± 3.82   Final Test: 21.62 ± 2.67   Final old_old: 23.38 ± 2.86   Final old_new: 14.71 ± 2.24   Final new_new: 2.73 ± 1.02
Hits@50:
  Final val: 48.93 ± 4.56   Final Test: 30.81 ± 3.51   Final old_old: 33.17 ± 3.65   Final old_new: 21.66 ± 3.20   Final new_new: 3.64 ± 1.24
AUC:
  Final val: 94.27 ± 0.54   Final Test: 93.44 ± 0.42   Final old_old: 94.26 ± 0.40   Final old_new: 90.30 ± 0.53   Final new_new: 82.47 ± 1.85
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 26.69 ± 1.78   Final Test: 16.57 ± 6.58   Final old_old: 18.30 ± 7.24   Final old_new: 9.47 ± 3.88   Final new_new: 3.18 ± 3.45
Hits@20:
  Final val: 40.08 ± 3.17   Final Test: 23.93 ± 6.71   Final old_old: 26.18 ± 7.40   Final old_new: 14.74 ± 3.85   Final new_new: 5.45 ± 5.23
Hits@30:
  Final val: 46.92 ± 3.25   Final Test: 29.92 ± 5.14   Final old_old: 32.53 ± 5.79   Final old_new: 19.40 ± 2.58   Final new_new: 6.82 ± 2.78
Hits@50:
  Final val: 58.42 ± 4.64   Final Test: 42.04 ± 5.15   Final old_old: 45.34 ± 5.64   Final old_new: 28.82 ± 3.27   Final new_new: 11.36 ± 2.27
AUC:
  Final val: 95.79 ± 0.53   Final Test: 95.01 ± 0.52   Final old_old: 95.77 ± 0.48   Final old_new: 92.03 ± 0.75   Final new_new: 86.98 ± 0.80
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 21.92 ± 3.28   Final Test: 12.97 ± 2.65   Final old_old: 14.13 ± 2.82   Final old_new: 8.31 ± 1.98   Final new_new: 2.73 ± 2.49
Hits@20:
  Final val: 31.20 ± 2.97   Final Test: 17.49 ± 1.97   Final old_old: 19.07 ± 2.23   Final old_new: 11.18 ± 1.15   Final new_new: 2.27 ± 0.00
Hits@30:
  Final val: 38.91 ± 4.33   Final Test: 21.92 ± 2.32   Final old_old: 23.83 ± 2.45   Final old_new: 14.31 ± 2.10   Final new_new: 3.18 ± 2.03
Hits@50:
  Final val: 48.86 ± 5.02   Final Test: 30.01 ± 2.12   Final old_old: 32.39 ± 2.31   Final old_new: 20.71 ± 1.88   Final new_new: 4.09 ± 1.02
AUC:
  Final val: 94.24 ± 0.49   Final Test: 93.42 ± 0.41   Final old_old: 94.25 ± 0.40   Final old_new: 90.28 ± 0.53   Final new_new: 82.48 ± 2.43
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 29.68 ± 2.87   Final Test: 17.29 ± 2.97   Final old_old: 19.14 ± 3.55   Final old_new: 9.82 ± 0.76   Final new_new: 0.91 ± 2.03
Hits@20:
  Final val: 42.48 ± 4.17   Final Test: 23.58 ± 2.80   Final old_old: 25.89 ± 3.28   Final old_new: 14.41 ± 1.06   Final new_new: 0.45 ± 1.02
Hits@30:
  Final val: 49.65 ± 4.92   Final Test: 29.04 ± 2.12   Final old_old: 31.68 ± 2.50   Final old_new: 18.54 ± 1.42   Final new_new: 2.27 ± 2.27
Hits@50:
  Final val: 60.41 ± 4.13   Final Test: 40.65 ± 5.75   Final old_old: 43.86 ± 6.31   Final old_new: 27.98 ± 3.67   Final new_new: 6.36 ± 3.73
AUC:
  Final val: 95.83 ± 0.55   Final Test: 95.18 ± 0.48   Final old_old: 95.87 ± 0.51   Final old_new: 92.43 ± 0.41   Final new_new: 87.87 ± 0.58
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 22.77 ± 2.97   Final Test: 11.31 ± 2.12   Final old_old: 12.36 ± 2.31   Final old_new: 7.08 ± 1.49   Final new_new: 1.82 ± 1.02
Hits@20:
  Final val: 33.05 ± 2.99   Final Test: 17.54 ± 0.85   Final old_old: 19.10 ± 1.00   Final old_new: 11.34 ± 0.68   Final new_new: 2.73 ± 1.02
Hits@30:
  Final val: 40.03 ± 3.57   Final Test: 21.47 ± 1.86   Final old_old: 23.28 ± 1.82   Final old_new: 14.33 ± 2.42   Final new_new: 2.73 ± 1.02
Hits@50:
  Final val: 50.41 ± 3.67   Final Test: 31.44 ± 4.19   Final old_old: 33.97 ± 4.46   Final old_new: 21.54 ± 3.44   Final new_new: 3.64 ± 1.24
AUC:
  Final val: 94.48 ± 0.43   Final Test: 93.59 ± 0.31   Final old_old: 94.42 ± 0.35   Final old_new: 90.38 ± 0.42   Final new_new: 83.08 ± 1.74
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 26.42 ± 1.10   Final Test: 15.78 ± 5.51   Final old_old: 17.42 ± 6.25   Final old_new: 9.09 ± 2.56   Final new_new: 2.73 ± 1.90
Hits@20:
  Final val: 38.97 ± 3.85   Final Test: 22.88 ± 5.91   Final old_old: 25.01 ± 6.51   Final old_new: 14.31 ± 3.42   Final new_new: 4.09 ± 4.66
Hits@30:
  Final val: 46.44 ± 3.59   Final Test: 28.19 ± 6.05   Final old_old: 30.74 ± 6.72   Final old_new: 17.86 ± 3.33   Final new_new: 6.36 ± 4.37
Hits@50:
  Final val: 58.01 ± 4.66   Final Test: 39.59 ± 5.52   Final old_old: 42.79 ± 6.08   Final old_new: 26.83 ± 3.20   Final new_new: 9.09 ± 4.25
AUC:
  Final val: 95.72 ± 0.54   Final Test: 94.92 ± 0.56   Final old_old: 95.68 ± 0.50   Final old_new: 91.90 ± 0.82   Final new_new: 87.04 ± 1.30
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 28.75 ± 2.13   Final Test: 16.31 ± 3.50   Final old_old: 18.04 ± 3.89   Final old_new: 9.40 ± 2.07   Final new_new: 0.45 ± 1.02
Hits@20:
  Final val: 41.69 ± 3.92   Final Test: 24.22 ± 4.14   Final old_old: 26.52 ± 4.60   Final old_new: 15.01 ± 2.47   Final new_new: 1.82 ± 1.02
Hits@30:
  Final val: 49.45 ± 2.89   Final Test: 28.89 ± 3.70   Final old_old: 31.56 ± 4.09   Final old_new: 18.34 ± 2.36   Final new_new: 1.82 ± 1.90
Hits@50:
  Final val: 59.20 ± 4.28   Final Test: 38.90 ± 3.41   Final old_old: 42.04 ± 3.79   Final old_new: 26.57 ± 2.14   Final new_new: 5.45 ± 3.45
AUC:
  Final val: 95.66 ± 0.54   Final Test: 94.97 ± 0.46   Final old_old: 95.70 ± 0.48   Final old_new: 92.14 ± 0.42   Final new_new: 87.30 ± 0.75
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 27.26 ± 4.14   Final Test: 16.87 ± 4.80   Final old_old: 18.61 ± 5.41   Final old_new: 9.77 ± 2.42   Final new_new: 2.27 ± 3.21
Hits@20:
  Final val: 41.41 ± 3.48   Final Test: 21.21 ± 3.24   Final old_old: 23.27 ± 3.70   Final old_new: 12.90 ± 1.58   Final new_new: 2.73 ± 2.96
Hits@30:
  Final val: 48.45 ± 4.96   Final Test: 28.40 ± 5.97   Final old_old: 30.97 ± 6.71   Final old_new: 18.16 ± 2.94   Final new_new: 3.64 ± 2.59
Hits@50:
  Final val: 58.94 ± 5.59   Final Test: 39.38 ± 5.85   Final old_old: 42.63 ± 6.47   Final old_new: 26.37 ± 3.56   Final new_new: 8.64 ± 2.49
AUC:
  Final val: 95.75 ± 0.63   Final Test: 94.95 ± 0.56   Final old_old: 95.71 ± 0.55   Final old_new: 91.96 ± 0.66   Final new_new: 87.17 ± 1.11
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 23.11 ± 2.53   Final Test: 12.97 ± 2.53   Final old_old: 14.16 ± 2.92   Final old_new: 8.21 ± 1.12   Final new_new: 2.27 ± 1.61
Hits@20:
  Final val: 33.67 ± 5.54   Final Test: 18.85 ± 5.95   Final old_old: 20.55 ± 6.56   Final old_new: 11.99 ± 3.73   Final new_new: 4.09 ± 4.37
Hits@30:
  Final val: 43.47 ± 6.51   Final Test: 26.40 ± 7.07   Final old_old: 28.61 ± 7.76   Final old_new: 17.71 ± 4.44   Final new_new: 3.18 ± 2.59
Hits@50:
  Final val: 54.01 ± 5.49   Final Test: 36.36 ± 7.24   Final old_old: 39.15 ± 7.92   Final old_new: 25.39 ± 4.80   Final new_new: 6.36 ± 1.90
AUC:
  Final val: 95.17 ± 1.01   Final Test: 94.46 ± 0.93   Final old_old: 95.20 ± 0.93   Final old_new: 91.63 ± 1.02   Final new_new: 85.14 ± 1.20
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 26.91 ± 1.76   Final Test: 15.45 ± 5.25   Final old_old: 17.03 ± 5.76   Final old_new: 9.12 ± 3.37   Final new_new: 1.36 ± 2.03
Hits@20:
  Final val: 40.13 ± 4.91   Final Test: 22.14 ± 7.05   Final old_old: 24.19 ± 7.73   Final old_new: 13.93 ± 4.42   Final new_new: 3.18 ± 3.45
Hits@30:
  Final val: 46.92 ± 4.61   Final Test: 30.64 ± 7.51   Final old_old: 33.29 ± 8.20   Final old_new: 19.97 ± 4.75   Final new_new: 6.36 ± 4.37
Hits@50:
  Final val: 58.36 ± 4.13   Final Test: 40.73 ± 3.67   Final old_old: 43.95 ± 4.14   Final old_new: 27.86 ± 2.00   Final new_new: 9.55 ± 1.90
AUC:
  Final val: 95.75 ± 0.53   Final Test: 94.98 ± 0.58   Final old_old: 95.75 ± 0.52   Final old_new: 91.98 ± 0.86   Final new_new: 86.75 ± 1.54
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 26.09 ± 3.86   Final Test: 15.22 ± 3.52   Final old_old: 16.78 ± 3.91   Final old_new: 8.89 ± 2.24   Final new_new: 1.82 ± 2.49
Hits@20:
  Final val: 37.59 ± 6.44   Final Test: 21.73 ± 3.35   Final old_old: 23.75 ± 3.90   Final old_new: 13.55 ± 1.40   Final new_new: 4.55 ± 4.55
Hits@30:
  Final val: 45.71 ± 5.73   Final Test: 28.07 ± 4.11   Final old_old: 30.54 ± 4.57   Final old_new: 18.21 ± 2.48   Final new_new: 4.09 ± 2.49
Hits@50:
  Final val: 56.26 ± 5.02   Final Test: 37.32 ± 4.49   Final old_old: 40.26 ± 4.92   Final old_new: 25.69 ± 2.98   Final new_new: 7.73 ± 3.80
AUC:
  Final val: 95.29 ± 0.64   Final Test: 94.65 ± 0.62   Final old_old: 95.36 ± 0.66   Final old_new: 91.85 ± 0.46   Final new_new: 87.06 ± 1.13
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 23.13 ± 1.41   Final Test: 11.21 ± 1.77   Final old_old: 12.30 ± 1.84   Final old_new: 6.90 ± 1.68   Final new_new: 0.45 ± 1.02
Hits@20:
  Final val: 32.96 ± 3.21   Final Test: 18.97 ± 1.35   Final old_old: 20.65 ± 1.48   Final old_new: 12.29 ± 0.95   Final new_new: 2.27 ± 1.61
Hits@30:
  Final val: 39.33 ± 2.65   Final Test: 22.55 ± 2.97   Final old_old: 24.49 ± 3.33   Final old_new: 14.86 ± 1.58   Final new_new: 3.64 ± 2.03
Hits@50:
  Final val: 51.16 ± 4.02   Final Test: 30.16 ± 2.70   Final old_old: 32.70 ± 3.05   Final old_new: 20.10 ± 1.50   Final new_new: 4.09 ± 1.90
AUC:
  Final val: 94.60 ± 0.46   Final Test: 93.67 ± 0.40   Final old_old: 94.53 ± 0.43   Final old_new: 90.40 ± 0.41   Final new_new: 83.10 ± 1.59
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 27.19 ± 4.61   Final Test: 15.79 ± 4.93   Final old_old: 17.31 ± 5.41   Final old_new: 9.72 ± 3.19   Final new_new: 1.82 ± 1.90
Hits@20:
  Final val: 37.80 ± 5.68   Final Test: 22.51 ± 3.84   Final old_old: 24.53 ± 4.36   Final old_new: 14.33 ± 2.07   Final new_new: 5.45 ± 4.71
Hits@30:
  Final val: 47.52 ± 5.41   Final Test: 27.95 ± 3.37   Final old_old: 30.31 ± 3.69   Final old_new: 18.59 ± 2.18   Final new_new: 4.09 ± 1.90
Hits@50:
  Final val: 57.37 ± 5.53   Final Test: 35.92 ± 3.57   Final old_old: 38.75 ± 3.91   Final old_new: 24.66 ± 2.42   Final new_new: 8.18 ± 3.45
AUC:
  Final val: 95.40 ± 0.67   Final Test: 94.72 ± 0.63   Final old_old: 95.43 ± 0.66   Final old_new: 91.93 ± 0.50   Final new_new: 86.96 ± 1.34
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 30.86 ± 3.77   Final Test: 23.10 ± 0.71   Final old_old: 25.24 ± 0.75   Final old_new: 14.53 ± 0.79   Final new_new: 2.73 ± 2.96
Hits@20:
  Final val: 46.70 ± 3.43   Final Test: 28.24 ± 2.51   Final old_old: 30.78 ± 2.58   Final old_new: 18.01 ± 2.39   Final new_new: 5.45 ± 3.45
Hits@30:
  Final val: 54.91 ± 3.70   Final Test: 35.62 ± 1.99   Final old_old: 38.58 ± 2.07   Final old_new: 23.75 ± 1.95   Final new_new: 8.18 ± 3.05
Hits@50:
  Final val: 66.24 ± 3.75   Final Test: 48.58 ± 1.16   Final old_old: 52.11 ± 1.32   Final old_new: 34.66 ± 1.04   Final new_new: 12.27 ± 3.45
AUC:
  Final val: 96.66 ± 0.22   Final Test: 95.99 ± 0.19   Final old_old: 96.60 ± 0.20   Final old_new: 93.64 ± 0.16   Final new_new: 88.55 ± 1.19
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 27.83 ± 1.99   Final Test: 16.48 ± 1.53   Final old_old: 18.22 ± 1.80   Final old_new: 9.47 ± 1.21   Final new_new: 0.91 ± 2.03
Hits@20:
  Final val: 40.38 ± 2.96   Final Test: 23.52 ± 3.23   Final old_old: 25.83 ± 3.72   Final old_new: 14.31 ± 1.61   Final new_new: 0.91 ± 1.24
Hits@30:
  Final val: 48.31 ± 3.67   Final Test: 29.99 ± 2.95   Final old_old: 32.72 ± 3.41   Final old_new: 19.14 ± 1.53   Final new_new: 2.73 ± 2.49
Hits@50:
  Final val: 59.19 ± 3.68   Final Test: 37.78 ± 3.85   Final old_old: 41.00 ± 4.37   Final old_new: 25.06 ± 1.80   Final new_new: 5.00 ± 4.07
AUC:
  Final val: 95.70 ± 0.48   Final Test: 94.99 ± 0.38   Final old_old: 95.71 ± 0.41   Final old_new: 92.14 ± 0.33   Final new_new: 87.72 ± 1.00
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 22.96 ± 2.70   Final Test: 11.76 ± 2.47   Final old_old: 12.91 ± 2.76   Final old_new: 7.10 ± 1.39   Final new_new: 1.36 ± 1.24
Hits@20:
  Final val: 32.61 ± 2.91   Final Test: 16.84 ± 2.11   Final old_old: 18.45 ± 2.33   Final old_new: 10.45 ± 1.66   Final new_new: 1.36 ± 1.24
Hits@30:
  Final val: 40.54 ± 4.21   Final Test: 23.03 ± 3.12   Final old_old: 25.05 ± 3.53   Final old_new: 15.04 ± 1.96   Final new_new: 1.82 ± 1.02
Hits@50:
  Final val: 51.34 ± 3.33   Final Test: 29.93 ± 3.00   Final old_old: 32.44 ± 3.26   Final old_new: 20.10 ± 2.33   Final new_new: 2.73 ± 1.90
AUC:
  Final val: 94.61 ± 0.43   Final Test: 93.68 ± 0.37   Final old_old: 94.51 ± 0.40   Final old_new: 90.48 ± 0.32   Final new_new: 83.25 ± 1.14
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 26.55 ± 2.76   Final Test: 15.73 ± 5.18   Final old_old: 17.22 ± 5.76   Final old_new: 9.65 ± 3.04   Final new_new: 3.18 ± 2.59
Hits@20:
  Final val: 38.92 ± 5.55   Final Test: 20.84 ± 4.25   Final old_old: 22.71 ± 4.86   Final old_new: 13.25 ± 2.09   Final new_new: 4.55 ± 4.55
Hits@30:
  Final val: 46.31 ± 4.10   Final Test: 26.45 ± 4.39   Final old_old: 28.74 ± 4.92   Final old_new: 17.25 ± 2.46   Final new_new: 5.00 ± 2.96
Hits@50:
  Final val: 57.87 ± 4.43   Final Test: 38.77 ± 6.83   Final old_old: 41.79 ± 7.33   Final old_new: 26.73 ± 4.96   Final new_new: 9.55 ± 2.96
AUC:
  Final val: 95.46 ± 0.72   Final Test: 94.69 ± 0.69   Final old_old: 95.42 ± 0.68   Final old_new: 91.85 ± 0.76   Final new_new: 86.51 ± 1.14
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 22.82 ± 1.91   Final Test: 10.70 ± 2.01   Final old_old: 11.77 ± 2.07   Final old_new: 6.40 ± 1.88   Final new_new: 1.36 ± 1.24
Hits@20:
  Final val: 31.91 ± 1.88   Final Test: 16.52 ± 1.12   Final old_old: 17.95 ± 1.16   Final old_new: 10.83 ± 1.22   Final new_new: 2.27 ± 1.61
Hits@30:
  Final val: 38.78 ± 2.39   Final Test: 22.26 ± 1.92   Final old_old: 24.04 ± 2.15   Final old_new: 15.26 ± 1.06   Final new_new: 3.18 ± 2.59
Hits@50:
  Final val: 49.56 ± 3.31   Final Test: 29.29 ± 2.64   Final old_old: 31.63 ± 2.88   Final old_new: 20.10 ± 1.92   Final new_new: 4.09 ± 2.96
AUC:
  Final val: 94.27 ± 0.28   Final Test: 93.37 ± 0.23   Final old_old: 94.19 ± 0.24   Final old_new: 90.23 ± 0.48   Final new_new: 82.99 ± 1.08
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 21.32 ± 2.15   Final Test: 10.90 ± 1.01   Final old_old: 11.88 ± 1.09   Final old_new: 6.95 ± 0.82   Final new_new: 1.36 ± 1.24
Hits@20:
  Final val: 31.70 ± 1.55   Final Test: 16.47 ± 1.12   Final old_old: 17.95 ± 1.21   Final old_new: 10.55 ± 1.34   Final new_new: 1.82 ± 1.02
Hits@30:
  Final val: 39.53 ± 1.73   Final Test: 23.19 ± 1.75   Final old_old: 25.28 ± 2.05   Final old_new: 14.86 ± 0.94   Final new_new: 2.73 ± 1.02
Hits@50:
  Final val: 50.77 ± 3.12   Final Test: 28.98 ± 1.30   Final old_old: 31.53 ± 1.54   Final old_new: 18.87 ± 0.63   Final new_new: 3.18 ± 2.03
AUC:
  Final val: 94.68 ± 0.22   Final Test: 93.72 ± 0.22   Final old_old: 94.62 ± 0.18   Final old_new: 90.24 ± 0.51   Final new_new: 82.97 ± 1.37
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 24.95 ± 2.40   Final Test: 15.59 ± 3.47   Final old_old: 17.12 ± 4.00   Final old_new: 9.40 ± 1.66   Final new_new: 1.82 ± 2.96
Hits@20:
  Final val: 39.72 ± 3.73   Final Test: 19.86 ± 3.10   Final old_old: 21.64 ± 3.50   Final old_new: 12.75 ± 1.56   Final new_new: 3.18 ± 2.59
Hits@30:
  Final val: 46.35 ± 4.34   Final Test: 27.13 ± 3.64   Final old_old: 29.55 ± 4.12   Final old_new: 17.48 ± 1.82   Final new_new: 3.64 ± 3.45
Hits@50:
  Final val: 57.39 ± 4.18   Final Test: 36.91 ± 3.78   Final old_old: 39.85 ± 4.00   Final old_new: 25.29 ± 3.38   Final new_new: 6.36 ± 2.96
AUC:
  Final val: 95.55 ± 0.44   Final Test: 94.75 ± 0.38   Final old_old: 95.52 ± 0.34   Final old_new: 91.69 ± 0.60   Final new_new: 86.95 ± 1.31
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 31.23 ± 4.24   Final Test: 23.06 ± 1.22   Final old_old: 25.38 ± 1.33   Final old_new: 13.80 ± 0.87   Final new_new: 0.91 ± 2.03
Hits@20:
  Final val: 47.43 ± 1.89   Final Test: 29.09 ± 1.19   Final old_old: 31.72 ± 1.15   Final old_new: 18.74 ± 1.72   Final new_new: 0.45 ± 1.02
Hits@30:
  Final val: 55.69 ± 1.22   Final Test: 33.78 ± 2.12   Final old_old: 36.76 ± 2.22   Final old_new: 22.02 ± 2.10   Final new_new: 2.27 ± 1.61
Hits@50:
  Final val: 66.26 ± 1.49   Final Test: 47.57 ± 2.78   Final old_old: 51.08 ± 3.25   Final old_new: 33.83 ± 0.92   Final new_new: 8.64 ± 3.37
AUC:
  Final val: 96.64 ± 0.21   Final Test: 96.04 ± 0.21   Final old_old: 96.62 ± 0.23   Final old_new: 93.83 ± 0.21   Final new_new: 89.28 ± 0.68
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 28.32 ± 2.14   Final Test: 18.13 ± 7.10   Final old_old: 19.85 ± 7.78   Final old_new: 11.16 ± 4.42   Final new_new: 3.64 ± 2.59
Hits@20:
  Final val: 41.36 ± 3.48   Final Test: 22.52 ± 6.54   Final old_old: 24.69 ± 7.09   Final old_new: 13.75 ± 4.44   Final new_new: 3.64 ± 2.59
Hits@30:
  Final val: 49.01 ± 2.83   Final Test: 28.61 ± 5.53   Final old_old: 31.13 ± 6.03   Final old_new: 18.49 ± 3.61   Final new_new: 5.91 ± 3.80
Hits@50:
  Final val: 59.98 ± 4.39   Final Test: 42.79 ± 6.12   Final old_old: 45.98 ± 6.49   Final old_new: 29.97 ± 4.68   Final new_new: 13.64 ± 2.78
AUC:
  Final val: 95.89 ± 0.48   Final Test: 95.11 ± 0.51   Final old_old: 95.84 ± 0.49   Final old_new: 92.24 ± 0.62   Final new_new: 87.36 ± 1.33
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 30.31 ± 4.06   Final Test: 19.00 ± 1.26   Final old_old: 20.97 ± 1.59   Final old_new: 11.11 ± 0.75   Final new_new: 0.00 ± 0.00
Hits@20:
  Final val: 43.19 ± 4.42   Final Test: 25.43 ± 4.43   Final old_old: 27.94 ± 4.92   Final old_new: 15.47 ± 2.56   Final new_new: 0.91 ± 1.24
Hits@30:
  Final val: 51.63 ± 4.47   Final Test: 31.53 ± 3.82   Final old_old: 34.43 ± 4.23   Final old_new: 20.03 ± 2.14   Final new_new: 1.82 ± 1.90
Hits@50:
  Final val: 61.66 ± 4.69   Final Test: 39.80 ± 4.45   Final old_old: 43.08 ± 4.77   Final old_new: 26.85 ± 3.30   Final new_new: 5.45 ± 2.03
AUC:
  Final val: 95.94 ± 0.58   Final Test: 95.29 ± 0.54   Final old_old: 96.01 ± 0.56   Final old_new: 92.41 ± 0.53   Final new_new: 88.03 ± 0.71
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 25.80 ± 1.46   Final Test: 14.34 ± 5.05   Final old_old: 15.83 ± 5.73   Final old_new: 8.29 ± 2.31   Final new_new: 1.82 ± 1.90
Hits@20:
  Final val: 40.30 ± 5.26   Final Test: 20.38 ± 4.96   Final old_old: 22.30 ± 5.67   Final old_new: 12.64 ± 2.09   Final new_new: 3.18 ± 2.03
Hits@30:
  Final val: 47.18 ± 5.49   Final Test: 26.82 ± 5.23   Final old_old: 29.32 ± 5.95   Final old_new: 16.73 ± 2.45   Final new_new: 4.09 ± 1.90
Hits@50:
  Final val: 57.31 ± 5.56   Final Test: 37.45 ± 4.27   Final old_old: 40.61 ± 4.72   Final old_new: 24.84 ± 2.90   Final new_new: 7.73 ± 1.24
AUC:
  Final val: 95.48 ± 0.58   Final Test: 94.68 ± 0.59   Final old_old: 95.44 ± 0.54   Final old_new: 91.67 ± 0.78   Final new_new: 86.73 ± 1.90
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 26.66 ± 3.76   Final Test: 15.32 ± 1.45   Final old_old: 16.96 ± 1.67   Final old_new: 8.72 ± 1.33   Final new_new: 0.91 ± 2.03
Hits@20:
  Final val: 37.01 ± 3.75   Final Test: 21.60 ± 2.97   Final old_old: 23.56 ± 3.27   Final old_new: 13.75 ± 2.36   Final new_new: 3.18 ± 2.59
Hits@30:
  Final val: 44.79 ± 3.27   Final Test: 27.12 ± 2.62   Final old_old: 29.50 ± 2.91   Final old_new: 17.68 ± 1.77   Final new_new: 3.64 ± 2.59
Hits@50:
  Final val: 55.90 ± 2.74   Final Test: 35.56 ± 2.94   Final old_old: 38.51 ± 3.11   Final old_new: 23.88 ± 2.74   Final new_new: 5.45 ± 1.24
AUC:
  Final val: 95.33 ± 0.36   Final Test: 94.67 ± 0.41   Final old_old: 95.38 ± 0.40   Final old_new: 91.86 ± 0.47   Final new_new: 87.56 ± 1.01
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 25.96 ± 4.21   Final Test: 15.34 ± 3.99   Final old_old: 16.88 ± 4.33   Final old_new: 9.09 ± 2.75   Final new_new: 2.27 ± 2.27
Hits@20:
  Final val: 38.22 ± 5.86   Final Test: 21.70 ± 2.33   Final old_old: 23.58 ± 2.78   Final old_new: 14.18 ± 0.72   Final new_new: 3.64 ± 4.71
Hits@30:
  Final val: 46.57 ± 5.17   Final Test: 28.08 ± 3.66   Final old_old: 30.56 ± 4.14   Final old_new: 18.19 ± 1.84   Final new_new: 3.64 ± 2.59
Hits@50:
  Final val: 56.21 ± 4.89   Final Test: 35.45 ± 2.94   Final old_old: 38.34 ± 3.33   Final old_new: 23.95 ± 1.80   Final new_new: 6.82 ± 3.21
AUC:
  Final val: 95.24 ± 0.64   Final Test: 94.58 ± 0.56   Final old_old: 95.29 ± 0.59   Final old_new: 91.79 ± 0.44   Final new_new: 87.50 ± 1.25
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 21.88 ± 2.28   Final Test: 12.06 ± 1.35   Final old_old: 13.20 ± 1.41   Final old_new: 7.43 ± 1.40   Final new_new: 1.82 ± 1.02
Hits@20:
  Final val: 31.91 ± 2.90   Final Test: 17.23 ± 1.33   Final old_old: 18.80 ± 1.37   Final old_new: 10.93 ± 1.47   Final new_new: 2.27 ± 1.61
Hits@30:
  Final val: 39.32 ± 3.50   Final Test: 22.63 ± 2.06   Final old_old: 24.51 ± 2.28   Final old_new: 15.21 ± 1.48   Final new_new: 2.27 ± 1.61
Hits@50:
  Final val: 50.45 ± 4.85   Final Test: 30.44 ± 3.15   Final old_old: 32.94 ± 3.36   Final old_new: 20.60 ± 2.51   Final new_new: 4.09 ± 1.90
AUC:
  Final val: 94.40 ± 0.57   Final Test: 93.56 ± 0.44   Final old_old: 94.41 ± 0.41   Final old_new: 90.31 ± 0.54   Final new_new: 82.78 ± 2.07
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 31.73 ± 7.16   Final Test: 20.03 ± 3.64   Final old_old: 21.69 ± 3.91   Final old_new: 13.27 ± 2.42   Final new_new: 6.36 ± 3.73
Hits@20:
  Final val: 44.85 ± 5.35   Final Test: 31.17 ± 5.51   Final old_old: 33.55 ± 5.80   Final old_new: 21.51 ± 4.45   Final new_new: 10.91 ± 3.37
Hits@30:
  Final val: 53.22 ± 4.51   Final Test: 39.60 ± 4.27   Final old_old: 42.52 ± 4.28   Final old_new: 27.88 ± 4.37   Final new_new: 12.27 ± 3.45
Hits@50:
  Final val: 64.99 ± 3.54   Final Test: 50.83 ± 2.71   Final old_old: 54.26 ± 2.72   Final old_new: 37.10 ± 2.74   Final new_new: 18.18 ± 5.08
AUC:
  Final val: 96.74 ± 0.30   Final Test: 96.12 ± 0.28   Final old_old: 96.66 ± 0.28   Final old_new: 94.09 ± 0.28   Final new_new: 88.82 ± 1.24
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 22.27 ± 2.45   Final Test: 13.68 ± 3.17   Final old_old: 14.93 ± 3.45   Final old_new: 8.66 ± 2.19   Final new_new: 1.36 ± 2.03
Hits@20:
  Final val: 33.01 ± 3.62   Final Test: 17.62 ± 2.53   Final old_old: 19.20 ± 2.84   Final old_new: 11.31 ± 1.62   Final new_new: 2.27 ± 1.61
Hits@30:
  Final val: 40.91 ± 4.52   Final Test: 24.08 ± 2.17   Final old_old: 26.16 ± 2.41   Final old_new: 15.89 ± 1.37   Final new_new: 2.73 ± 1.90
Hits@50:
  Final val: 52.00 ± 4.62   Final Test: 30.83 ± 3.56   Final old_old: 33.29 ± 3.76   Final old_new: 21.26 ± 2.96   Final new_new: 3.18 ± 1.24
AUC:
  Final val: 94.77 ± 0.48   Final Test: 93.82 ± 0.35   Final old_old: 94.70 ± 0.37   Final old_new: 90.45 ± 0.38   Final new_new: 82.56 ± 1.76
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 29.15 ± 3.17   Final Test: 18.10 ± 4.70   Final old_old: 20.02 ± 5.13   Final old_new: 10.35 ± 3.23   Final new_new: 1.36 ± 2.03
Hits@20:
  Final val: 41.36 ± 3.59   Final Test: 23.93 ± 3.50   Final old_old: 26.18 ± 3.96   Final old_new: 14.89 ± 1.84   Final new_new: 3.64 ± 4.43
Hits@30:
  Final val: 48.81 ± 3.02   Final Test: 30.22 ± 3.62   Final old_old: 32.89 ± 3.97   Final old_new: 19.57 ± 2.50   Final new_new: 4.55 ± 3.21
Hits@50:
  Final val: 58.96 ± 4.11   Final Test: 39.47 ± 5.78   Final old_old: 42.54 ± 6.20   Final old_new: 27.20 ± 4.49   Final new_new: 9.55 ± 3.73
AUC:
  Final val: 95.77 ± 0.54   Final Test: 95.06 ± 0.55   Final old_old: 95.77 ± 0.54   Final old_new: 92.28 ± 0.66   Final new_new: 87.64 ± 1.15
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 26.94 ± 2.91   Final Test: 16.89 ± 6.59   Final old_old: 18.63 ± 7.35   Final old_new: 9.90 ± 3.67   Final new_new: 1.36 ± 2.03
Hits@20:
  Final val: 39.65 ± 3.91   Final Test: 22.04 ± 7.41   Final old_old: 24.10 ± 8.22   Final old_new: 13.65 ± 4.08   Final new_new: 4.55 ± 4.82
Hits@30:
  Final val: 46.96 ± 4.71   Final Test: 29.47 ± 7.32   Final old_old: 32.00 ± 8.15   Final old_new: 19.40 ± 3.98   Final new_new: 5.00 ± 1.90
Hits@50:
  Final val: 58.11 ± 4.77   Final Test: 40.88 ± 6.46   Final old_old: 43.95 ± 6.99   Final old_new: 28.59 ± 4.48   Final new_new: 11.36 ± 1.61
AUC:
  Final val: 95.68 ± 0.63   Final Test: 94.88 ± 0.66   Final old_old: 95.65 ± 0.60   Final old_new: 91.84 ± 0.90   Final new_new: 86.65 ± 1.15
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 23.17 ± 2.43   Final Test: 12.27 ± 1.37   Final old_old: 13.44 ± 1.44   Final old_new: 7.58 ± 1.48   Final new_new: 0.91 ± 1.24
Hits@20:
  Final val: 35.23 ± 1.29   Final Test: 18.24 ± 2.16   Final old_old: 19.91 ± 2.21   Final old_new: 11.59 ± 2.34   Final new_new: 1.82 ± 1.02
Hits@30:
  Final val: 43.40 ± 3.41   Final Test: 23.99 ± 2.03   Final old_old: 26.16 ± 2.28   Final old_new: 15.34 ± 1.26   Final new_new: 3.18 ± 1.24
Hits@50:
  Final val: 54.11 ± 3.50   Final Test: 32.79 ± 1.75   Final old_old: 35.45 ± 1.75   Final old_new: 22.27 ± 2.28   Final new_new: 5.45 ± 2.03
AUC:
  Final val: 95.09 ± 0.43   Final Test: 94.18 ± 0.42   Final old_old: 95.02 ± 0.45   Final old_new: 90.95 ± 0.35   Final new_new: 83.93 ± 1.67
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.33 ± 5.35   Final Test: 31.73 ± 3.38   Final old_old: 34.36 ± 3.77   Final old_new: 21.39 ± 2.08   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 57.57 ± 5.17   Final Test: 43.59 ± 3.84   Final old_old: 46.81 ± 4.36   Final old_new: 31.06 ± 2.18   Final new_new: 7.27 ± 1.90
Hits@30:
  Final val: 65.82 ± 3.69   Final Test: 50.11 ± 2.07   Final old_old: 53.67 ± 2.37   Final old_new: 36.27 ± 1.01   Final new_new: 8.64 ± 2.49
Hits@50:
  Final val: 73.71 ± 3.79   Final Test: 60.46 ± 3.00   Final old_old: 64.17 ± 3.29   Final old_new: 46.10 ± 1.94   Final new_new: 16.36 ± 6.10
AUC:
  Final val: 97.52 ± 0.21   Final Test: 97.04 ± 0.18   Final old_old: 97.55 ± 0.22   Final old_new: 95.08 ± 0.10   Final new_new: 91.30 ± 0.74
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 23.42 ± 2.09   Final Test: 13.39 ± 2.26   Final old_old: 14.75 ± 2.36   Final old_new: 7.83 ± 2.05   Final new_new: 2.27 ± 1.61
Hits@20:
  Final val: 33.26 ± 3.90   Final Test: 18.74 ± 2.25   Final old_old: 20.46 ± 2.39   Final old_new: 11.84 ± 1.88   Final new_new: 2.27 ± 1.61
Hits@30:
  Final val: 42.48 ± 5.52   Final Test: 25.07 ± 3.81   Final old_old: 27.34 ± 4.14   Final old_new: 16.07 ± 2.64   Final new_new: 2.27 ± 1.61
Hits@50:
  Final val: 53.15 ± 5.14   Final Test: 35.00 ± 3.22   Final old_old: 37.84 ± 3.42   Final old_new: 23.75 ± 2.57   Final new_new: 5.91 ± 2.59
AUC:
  Final val: 94.81 ± 0.93   Final Test: 94.11 ± 0.75   Final old_old: 94.90 ± 0.76   Final old_new: 91.09 ± 0.79   Final new_new: 84.86 ± 2.11
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 28.69 ± 4.02   Final Test: 15.79 ± 5.22   Final old_old: 17.43 ± 5.92   Final old_new: 9.12 ± 2.54   Final new_new: 2.27 ± 3.21
Hits@20:
  Final val: 41.33 ± 4.38   Final Test: 22.27 ± 5.98   Final old_old: 24.47 ± 6.62   Final old_new: 13.38 ± 3.50   Final new_new: 2.73 ± 2.49
Hits@30:
  Final val: 48.36 ± 5.00   Final Test: 28.62 ± 5.80   Final old_old: 31.22 ± 6.45   Final old_new: 18.19 ± 3.21   Final new_new: 4.55 ± 2.27
Hits@50:
  Final val: 59.71 ± 4.71   Final Test: 40.60 ± 5.98   Final old_old: 43.85 ± 6.40   Final old_new: 27.76 ± 4.53   Final new_new: 7.73 ± 2.59
AUC:
  Final val: 95.79 ± 0.65   Final Test: 95.01 ± 0.57   Final old_old: 95.76 ± 0.56   Final old_new: 92.03 ± 0.68   Final new_new: 87.12 ± 0.88
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 25.37 ± 2.43   Final Test: 11.64 ± 2.55   Final old_old: 12.74 ± 2.75   Final old_new: 7.18 ± 2.02   Final new_new: 2.27 ± 2.78
Hits@20:
  Final val: 37.47 ± 3.21   Final Test: 18.42 ± 1.25   Final old_old: 20.19 ± 1.31   Final old_new: 11.31 ± 1.10   Final new_new: 1.82 ± 2.96
Hits@30:
  Final val: 45.00 ± 3.02   Final Test: 26.09 ± 1.44   Final old_old: 28.38 ± 1.58   Final old_new: 17.00 ± 1.01   Final new_new: 3.64 ± 3.45
Hits@50:
  Final val: 54.73 ± 2.66   Final Test: 37.01 ± 0.70   Final old_old: 39.96 ± 0.52   Final old_new: 25.29 ± 1.63   Final new_new: 8.18 ± 2.03
AUC:
  Final val: 95.34 ± 0.21   Final Test: 94.54 ± 0.13   Final old_old: 95.32 ± 0.15   Final old_new: 91.43 ± 0.17   Final new_new: 86.80 ± 0.95
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 27.05 ± 2.43   Final Test: 16.16 ± 1.36   Final old_old: 17.85 ± 1.46   Final old_new: 9.37 ± 1.43   Final new_new: 1.36 ± 2.03
Hits@20:
  Final val: 38.86 ± 2.45   Final Test: 22.76 ± 4.96   Final old_old: 24.94 ± 5.60   Final old_new: 14.01 ± 2.55   Final new_new: 2.73 ± 1.90
Hits@30:
  Final val: 47.75 ± 3.63   Final Test: 28.72 ± 3.76   Final old_old: 31.22 ± 4.18   Final old_new: 18.82 ± 2.23   Final new_new: 3.18 ± 2.59
Hits@50:
  Final val: 57.34 ± 2.79   Final Test: 38.42 ± 3.14   Final old_old: 41.38 ± 3.50   Final old_new: 26.75 ± 1.86   Final new_new: 6.82 ± 1.61
AUC:
  Final val: 95.53 ± 0.43   Final Test: 94.88 ± 0.37   Final old_old: 95.58 ± 0.39   Final old_new: 92.09 ± 0.33   Final new_new: 87.70 ± 0.71
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 25.60 ± 4.44   Final Test: 14.86 ± 2.97   Final old_old: 16.37 ± 3.37   Final old_new: 8.77 ± 1.72   Final new_new: 1.36 ± 2.03
Hits@20:
  Final val: 36.74 ± 5.45   Final Test: 20.29 ± 3.61   Final old_old: 22.18 ± 4.19   Final old_new: 12.64 ± 1.58   Final new_new: 4.55 ± 5.33
Hits@30:
  Final val: 45.58 ± 4.82   Final Test: 27.33 ± 4.72   Final old_old: 29.67 ± 5.08   Final old_new: 17.96 ± 3.70   Final new_new: 4.55 ± 4.25
Hits@50:
  Final val: 55.74 ± 5.90   Final Test: 35.64 ± 4.45   Final old_old: 38.49 ± 4.80   Final old_new: 24.23 ± 3.34   Final new_new: 9.09 ± 2.78
AUC:
  Final val: 95.23 ± 0.69   Final Test: 94.57 ± 0.65   Final old_old: 95.28 ± 0.68   Final old_new: 91.78 ± 0.55   Final new_new: 87.13 ± 1.45
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 29.01 ± 2.72   Final Test: 21.21 ± 2.19   Final old_old: 23.33 ± 2.46   Final old_new: 12.75 ± 1.20   Final new_new: 0.45 ± 1.02
Hits@20:
  Final val: 44.98 ± 4.58   Final Test: 26.71 ± 2.65   Final old_old: 29.25 ± 3.02   Final old_new: 16.52 ± 1.35   Final new_new: 2.73 ± 2.96
Hits@30:
  Final val: 53.62 ± 4.57   Final Test: 34.72 ± 0.57   Final old_old: 37.81 ± 0.63   Final old_new: 22.37 ± 0.85   Final new_new: 5.91 ± 2.59
Hits@50:
  Final val: 63.82 ± 3.84   Final Test: 44.38 ± 1.80   Final old_old: 47.92 ± 1.91   Final old_new: 30.28 ± 1.39   Final new_new: 10.00 ± 2.59
AUC:
  Final val: 96.35 ± 0.48   Final Test: 95.68 ± 0.40   Final old_old: 96.34 ± 0.40   Final old_new: 93.08 ± 0.43   Final new_new: 88.27 ± 0.72
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 22.00 ± 2.84   Final Test: 12.64 ± 1.23   Final old_old: 13.83 ± 1.20   Final old_new: 7.88 ± 1.52   Final new_new: 1.36 ± 1.24
Hits@20:
  Final val: 31.70 ± 2.89   Final Test: 16.50 ± 1.29   Final old_old: 17.98 ± 1.36   Final old_new: 10.63 ± 1.31   Final new_new: 1.82 ± 1.90
Hits@30:
  Final val: 39.23 ± 3.80   Final Test: 22.34 ± 0.91   Final old_old: 24.23 ± 1.10   Final old_new: 14.91 ± 0.55   Final new_new: 2.27 ± 1.61
Hits@50:
  Final val: 49.67 ± 4.96   Final Test: 32.32 ± 4.49   Final old_old: 34.72 ± 4.68   Final old_new: 22.97 ± 3.89   Final new_new: 4.55 ± 1.61
AUC:
  Final val: 94.43 ± 0.64   Final Test: 93.61 ± 0.51   Final old_old: 94.45 ± 0.48   Final old_new: 90.45 ± 0.64   Final new_new: 82.54 ± 1.86
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.34 ± 6.96   Final Test: 28.67 ± 1.75   Final old_old: 31.23 ± 1.93   Final old_new: 18.54 ± 1.57   Final new_new: 2.27 ± 1.61
Hits@20:
  Final val: 56.96 ± 6.02   Final Test: 41.81 ± 4.99   Final old_old: 44.93 ± 5.24   Final old_new: 29.75 ± 4.17   Final new_new: 5.45 ± 2.03
Hits@30:
  Final val: 66.10 ± 3.03   Final Test: 50.84 ± 2.71   Final old_old: 54.08 ± 2.97   Final old_new: 38.49 ± 1.76   Final new_new: 9.55 ± 1.90
Hits@50:
  Final val: 73.99 ± 2.84   Final Test: 59.31 ± 0.83   Final old_old: 62.81 ± 1.04   Final old_new: 45.82 ± 1.08   Final new_new: 17.27 ± 2.59
AUC:
  Final val: 97.49 ± 0.18   Final Test: 97.03 ± 0.17   Final old_old: 97.47 ± 0.18   Final old_new: 95.35 ± 0.15   Final new_new: 91.69 ± 0.43
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 28.71 ± 4.54   Final Test: 16.48 ± 2.75   Final old_old: 18.22 ± 3.14   Final old_new: 9.45 ± 1.46   Final new_new: 1.82 ± 2.49
Hits@20:
  Final val: 41.26 ± 3.93   Final Test: 24.87 ± 4.18   Final old_old: 27.29 ± 4.71   Final old_new: 15.24 ± 2.25   Final new_new: 1.82 ± 1.90
Hits@30:
  Final val: 48.64 ± 4.32   Final Test: 30.87 ± 4.42   Final old_old: 33.69 ± 4.82   Final old_new: 19.75 ± 3.15   Final new_new: 1.36 ± 2.03
Hits@50:
  Final val: 59.39 ± 3.81   Final Test: 38.70 ± 3.13   Final old_old: 41.84 ± 3.36   Final old_new: 26.40 ± 2.55   Final new_new: 4.55 ± 2.78
AUC:
  Final val: 95.80 ± 0.52   Final Test: 95.13 ± 0.46   Final old_old: 95.86 ± 0.48   Final old_new: 92.24 ± 0.45   Final new_new: 87.70 ± 0.72
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 22.60 ± 1.99   Final Test: 12.46 ± 0.93   Final old_old: 13.57 ± 0.86   Final old_new: 8.09 ± 1.44   Final new_new: 0.91 ± 1.24
Hits@20:
  Final val: 33.87 ± 2.97   Final Test: 18.59 ± 1.74   Final old_old: 20.27 ± 1.90   Final old_new: 11.89 ± 1.49   Final new_new: 1.82 ± 1.02
Hits@30:
  Final val: 42.44 ± 4.50   Final Test: 24.38 ± 2.61   Final old_old: 26.52 ± 2.80   Final old_new: 15.92 ± 2.05   Final new_new: 2.73 ± 1.90
Hits@50:
  Final val: 53.27 ± 4.27   Final Test: 33.20 ± 5.24   Final old_old: 35.88 ± 5.48   Final old_new: 22.77 ± 4.53   Final new_new: 3.18 ± 2.59
AUC:
  Final val: 94.97 ± 0.52   Final Test: 94.02 ± 0.46   Final old_old: 94.89 ± 0.46   Final old_new: 90.70 ± 0.47   Final new_new: 83.01 ± 2.29
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 31.90 ± 2.14   Final Test: 21.07 ± 1.82   Final old_old: 23.27 ± 2.04   Final old_new: 12.24 ± 1.16   Final new_new: 0.91 ± 1.24
Hits@20:
  Final val: 45.89 ± 2.24   Final Test: 27.66 ± 1.68   Final old_old: 30.33 ± 1.80   Final old_new: 16.95 ± 1.50   Final new_new: 2.27 ± 2.78
Hits@30:
  Final val: 53.88 ± 3.48   Final Test: 33.47 ± 1.50   Final old_old: 36.64 ± 1.75   Final old_new: 20.78 ± 1.29   Final new_new: 3.18 ± 2.59
Hits@50:
  Final val: 64.20 ± 3.67   Final Test: 46.46 ± 2.34   Final old_old: 49.97 ± 2.44   Final old_new: 32.64 ± 2.16   Final new_new: 8.64 ± 5.88
AUC:
  Final val: 96.50 ± 0.30   Final Test: 95.94 ± 0.19   Final old_old: 96.53 ± 0.18   Final old_new: 93.66 ± 0.33   Final new_new: 89.23 ± 0.77
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 26.46 ± 4.66   Final Test: 16.12 ± 4.58   Final old_old: 17.77 ± 5.18   Final old_new: 9.42 ± 2.36   Final new_new: 2.27 ± 2.78
Hits@20:
  Final val: 40.08 ± 4.35   Final Test: 22.15 ± 2.67   Final old_old: 24.31 ± 3.08   Final old_new: 13.40 ± 1.44   Final new_new: 3.64 ± 3.05
Hits@30:
  Final val: 47.24 ± 4.30   Final Test: 27.48 ± 4.32   Final old_old: 29.98 ± 4.76   Final old_new: 17.43 ± 3.07   Final new_new: 5.00 ± 2.96
Hits@50:
  Final val: 58.77 ± 4.49   Final Test: 39.78 ± 3.34   Final old_old: 42.92 ± 3.68   Final old_new: 27.23 ± 2.21   Final new_new: 9.55 ± 1.02
AUC:
  Final val: 95.71 ± 0.56   Final Test: 94.92 ± 0.60   Final old_old: 95.69 ± 0.55   Final old_new: 91.91 ± 0.81   Final new_new: 86.79 ± 1.22
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 22.45 ± 2.50   Final Test: 12.14 ± 1.37   Final old_old: 13.30 ± 1.55   Final old_new: 7.41 ± 0.78   Final new_new: 2.27 ± 1.61
Hits@20:
  Final val: 31.98 ± 2.83   Final Test: 17.48 ± 2.87   Final old_old: 19.04 ± 3.24   Final old_new: 11.21 ± 1.46   Final new_new: 3.18 ± 1.24
Hits@30:
  Final val: 38.75 ± 3.71   Final Test: 23.16 ± 2.75   Final old_old: 25.05 ± 2.89   Final old_new: 15.69 ± 2.38   Final new_new: 3.18 ± 1.24
Hits@50:
  Final val: 49.47 ± 4.44   Final Test: 30.72 ± 3.29   Final old_old: 33.05 ± 3.30   Final old_new: 21.56 ± 3.83   Final new_new: 5.45 ± 2.03
AUC:
  Final val: 94.44 ± 0.42   Final Test: 93.58 ± 0.35   Final old_old: 94.40 ± 0.29   Final old_new: 90.46 ± 0.63   Final new_new: 83.34 ± 2.40
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 21.91 ± 3.24   Final Test: 12.46 ± 2.84   Final old_old: 13.63 ± 3.21   Final old_new: 7.76 ± 1.37   Final new_new: 2.27 ± 1.61
Hits@20:
  Final val: 32.44 ± 3.64   Final Test: 17.60 ± 1.55   Final old_old: 19.23 ± 1.70   Final old_new: 11.11 ± 1.31   Final new_new: 1.36 ± 1.24
Hits@30:
  Final val: 39.46 ± 2.93   Final Test: 22.20 ± 2.23   Final old_old: 24.19 ± 2.48   Final old_new: 14.33 ± 1.84   Final new_new: 1.82 ± 1.02
Hits@50:
  Final val: 50.42 ± 3.83   Final Test: 29.48 ± 3.54   Final old_old: 31.88 ± 3.82   Final old_new: 20.05 ± 2.81   Final new_new: 3.64 ± 2.03
AUC:
  Final val: 94.56 ± 0.46   Final Test: 93.68 ± 0.39   Final old_old: 94.53 ± 0.39   Final old_new: 90.44 ± 0.47   Final new_new: 83.07 ± 2.12
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 26.92 ± 3.85   Final Test: 16.48 ± 4.96   Final old_old: 18.12 ± 5.60   Final old_new: 9.82 ± 2.43   Final new_new: 2.27 ± 1.61
Hits@20:
  Final val: 39.90 ± 5.08   Final Test: 22.61 ± 4.58   Final old_old: 24.68 ± 5.00   Final old_new: 14.28 ± 2.98   Final new_new: 4.09 ± 2.96
Hits@30:
  Final val: 47.32 ± 4.95   Final Test: 29.74 ± 6.46   Final old_old: 32.38 ± 7.09   Final old_new: 19.12 ± 3.96   Final new_new: 6.36 ± 2.96
Hits@50:
  Final val: 58.66 ± 4.68   Final Test: 39.63 ± 4.33   Final old_old: 42.85 ± 4.59   Final old_new: 26.73 ± 3.66   Final new_new: 9.09 ± 2.27
AUC:
  Final val: 95.75 ± 0.60   Final Test: 94.96 ± 0.61   Final old_old: 95.72 ± 0.55   Final old_new: 91.95 ± 0.82   Final new_new: 87.05 ± 1.50
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 26.30 ± 4.26   Final Test: 15.08 ± 3.43   Final old_old: 16.59 ± 3.81   Final old_new: 8.99 ± 2.08   Final new_new: 1.82 ± 1.90
Hits@20:
  Final val: 37.31 ± 5.81   Final Test: 20.67 ± 3.01   Final old_old: 22.52 ± 3.42   Final old_new: 13.17 ± 1.76   Final new_new: 5.00 ± 4.37
Hits@30:
  Final val: 45.42 ± 5.40   Final Test: 26.92 ± 5.19   Final old_old: 29.26 ± 5.53   Final old_new: 17.53 ± 4.03   Final new_new: 5.00 ± 2.96
Hits@50:
  Final val: 56.20 ± 4.90   Final Test: 37.11 ± 3.22   Final old_old: 39.98 ± 3.49   Final old_new: 25.74 ± 2.25   Final new_new: 7.73 ± 2.59
AUC:
  Final val: 95.24 ± 0.64   Final Test: 94.58 ± 0.62   Final old_old: 95.29 ± 0.64   Final old_new: 91.78 ± 0.55   Final new_new: 87.02 ± 1.61
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 26.06 ± 0.92   Final Test: 15.27 ± 5.03   Final old_old: 16.72 ± 5.76   Final old_new: 9.45 ± 2.14   Final new_new: 2.73 ± 2.96
Hits@20:
  Final val: 39.55 ± 3.72   Final Test: 21.67 ± 4.06   Final old_old: 23.70 ± 4.63   Final old_new: 13.53 ± 1.77   Final new_new: 2.73 ± 2.96
Hits@30:
  Final val: 46.70 ± 3.81   Final Test: 27.35 ± 4.35   Final old_old: 29.78 ± 4.80   Final old_new: 17.63 ± 2.74   Final new_new: 4.09 ± 2.96
Hits@50:
  Final val: 57.50 ± 4.53   Final Test: 38.30 ± 4.26   Final old_old: 41.41 ± 4.47   Final old_new: 25.82 ± 3.52   Final new_new: 9.55 ± 4.37
AUC:
  Final val: 95.59 ± 0.58   Final Test: 94.82 ± 0.60   Final old_old: 95.56 ± 0.55   Final old_new: 91.90 ± 0.84   Final new_new: 87.02 ± 1.35
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 27.14 ± 1.47   Final Test: 19.20 ± 2.80   Final old_old: 21.25 ± 3.07   Final old_new: 10.93 ± 2.03   Final new_new: 0.91 ± 1.24
Hits@20:
  Final val: 38.59 ± 3.65   Final Test: 26.16 ± 4.84   Final old_old: 28.57 ± 5.17   Final old_new: 16.50 ± 3.82   Final new_new: 3.64 ± 2.03
Hits@30:
  Final val: 45.53 ± 3.90   Final Test: 33.57 ± 3.69   Final old_old: 36.43 ± 4.10   Final old_new: 22.27 ± 2.08   Final new_new: 4.55 ± 2.78
Hits@50:
  Final val: 55.78 ± 4.67   Final Test: 43.43 ± 3.89   Final old_old: 46.59 ± 4.24   Final old_new: 31.13 ± 2.62   Final new_new: 7.73 ± 2.59
AUC:
  Final val: 93.93 ± 0.47   Final Test: 93.37 ± 0.38   Final old_old: 94.29 ± 0.37   Final old_new: 89.90 ± 0.58   Final new_new: 80.81 ± 1.54
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 26.93 ± 3.58   Final Test: 15.82 ± 5.08   Final old_old: 17.51 ± 5.77   Final old_new: 8.92 ± 2.31   Final new_new: 2.27 ± 2.78
Hits@20:
  Final val: 41.74 ± 4.71   Final Test: 22.10 ± 5.29   Final old_old: 24.19 ± 5.95   Final old_new: 13.65 ± 2.70   Final new_new: 4.09 ± 2.96
Hits@30:
  Final val: 48.99 ± 5.49   Final Test: 30.18 ± 5.52   Final old_old: 32.81 ± 6.11   Final old_new: 19.60 ± 3.42   Final new_new: 5.91 ± 4.13
Hits@50:
  Final val: 59.84 ± 4.75   Final Test: 39.84 ± 3.98   Final old_old: 42.99 ± 4.39   Final old_new: 27.28 ± 2.58   Final new_new: 9.09 ± 2.27
AUC:
  Final val: 95.78 ± 0.61   Final Test: 94.98 ± 0.53   Final old_old: 95.74 ± 0.53   Final old_new: 91.96 ± 0.62   Final new_new: 86.82 ± 1.01
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 25.53 ± 4.48   Final Test: 14.24 ± 1.43   Final old_old: 15.67 ± 1.62   Final old_new: 8.46 ± 0.93   Final new_new: 1.36 ± 2.03
Hits@20:
  Final val: 36.52 ± 4.99   Final Test: 20.29 ± 2.36   Final old_old: 22.06 ± 2.82   Final old_new: 13.15 ± 1.05   Final new_new: 5.00 ± 5.18
Hits@30:
  Final val: 44.78 ± 3.72   Final Test: 27.65 ± 3.98   Final old_old: 29.96 ± 4.25   Final old_new: 18.41 ± 3.37   Final new_new: 5.45 ± 2.03
Hits@50:
  Final val: 55.05 ± 4.90   Final Test: 34.66 ± 3.59   Final old_old: 37.43 ± 3.80   Final old_new: 23.63 ± 3.17   Final new_new: 7.27 ± 3.73
AUC:
  Final val: 95.17 ± 0.63   Final Test: 94.50 ± 0.58   Final old_old: 95.20 ± 0.61   Final old_new: 91.74 ± 0.48   Final new_new: 87.11 ± 1.40
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 23.94 ± 4.69   Final Test: 13.72 ± 4.61   Final old_old: 14.99 ± 4.99   Final old_new: 8.61 ± 3.15   Final new_new: 1.82 ± 1.90
Hits@20:
  Final val: 34.90 ± 6.30   Final Test: 18.57 ± 4.63   Final old_old: 20.25 ± 4.99   Final old_new: 11.84 ± 3.26   Final new_new: 2.73 ± 1.02
Hits@30:
  Final val: 42.00 ± 6.45   Final Test: 23.61 ± 6.13   Final old_old: 25.52 ± 6.52   Final old_new: 16.15 ± 4.90   Final new_new: 2.73 ± 1.02
Hits@50:
  Final val: 52.55 ± 7.02   Final Test: 31.57 ± 6.57   Final old_old: 34.04 ± 6.99   Final old_new: 21.96 ± 5.04   Final new_new: 2.73 ± 1.02
AUC:
  Final val: 94.76 ± 0.88   Final Test: 93.86 ± 0.85   Final old_old: 94.69 ± 0.86   Final old_new: 90.69 ± 0.75   Final new_new: 83.31 ± 2.70
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 27.22 ± 1.89   Final Test: 15.84 ± 1.46   Final old_old: 17.47 ± 1.72   Final old_new: 9.29 ± 0.91   Final new_new: 0.91 ± 2.03
Hits@20:
  Final val: 37.45 ± 4.47   Final Test: 21.24 ± 3.68   Final old_old: 23.25 ± 4.19   Final old_new: 13.10 ± 2.07   Final new_new: 4.09 ± 5.66
Hits@30:
  Final val: 45.26 ± 5.55   Final Test: 27.40 ± 3.48   Final old_old: 29.87 ± 3.95   Final old_new: 17.61 ± 1.90   Final new_new: 2.73 ± 2.96
Hits@50:
  Final val: 55.76 ± 5.07   Final Test: 36.01 ± 2.69   Final old_old: 38.87 ± 2.96   Final old_new: 24.63 ± 1.98   Final new_new: 7.73 ± 5.23
AUC:
  Final val: 95.30 ± 0.69   Final Test: 94.62 ± 0.73   Final old_old: 95.34 ± 0.70   Final old_new: 91.76 ± 0.83   Final new_new: 87.12 ± 1.79
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 38.82 ± 5.60   Final Test: 28.46 ± 2.64   Final old_old: 31.19 ± 2.83   Final old_new: 17.51 ± 2.15   Final new_new: 3.18 ± 1.24
Hits@20:
  Final val: 52.18 ± 4.33   Final Test: 39.40 ± 1.87   Final old_old: 42.71 ± 1.86   Final old_new: 26.20 ± 2.44   Final new_new: 7.27 ± 2.49
Hits@30:
  Final val: 60.76 ± 4.16   Final Test: 47.23 ± 3.82   Final old_old: 50.71 ± 3.83   Final old_new: 33.50 ± 4.15   Final new_new: 10.45 ± 3.45
Hits@50:
  Final val: 68.82 ± 2.95   Final Test: 55.55 ± 3.88   Final old_old: 59.09 ± 4.06   Final old_new: 41.89 ± 3.60   Final new_new: 12.73 ± 5.70
AUC:
  Final val: 95.22 ± 0.06   Final Test: 94.70 ± 0.10   Final old_old: 95.44 ± 0.07   Final old_new: 91.83 ± 0.32   Final new_new: 86.31 ± 1.26
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 28.62 ± 2.07   Final Test: 17.81 ± 4.29   Final old_old: 19.76 ± 4.66   Final old_new: 9.97 ± 2.97   Final new_new: 0.45 ± 1.02
Hits@20:
  Final val: 42.51 ± 4.17   Final Test: 23.85 ± 2.80   Final old_old: 26.25 ± 3.07   Final old_new: 14.28 ± 1.84   Final new_new: 0.45 ± 1.02
Hits@30:
  Final val: 50.30 ± 4.19   Final Test: 30.41 ± 3.93   Final old_old: 33.15 ± 4.28   Final old_new: 19.62 ± 2.58   Final new_new: 1.36 ± 1.24
Hits@50:
  Final val: 60.74 ± 5.37   Final Test: 41.27 ± 4.92   Final old_old: 44.69 ± 5.23   Final old_new: 27.88 ± 3.83   Final new_new: 3.64 ± 1.24
AUC:
  Final val: 96.01 ± 0.53   Final Test: 95.36 ± 0.44   Final old_old: 96.05 ± 0.43   Final old_new: 92.68 ± 0.49   Final new_new: 87.85 ± 0.66
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 23.01 ± 3.41   Final Test: 12.37 ± 2.37   Final old_old: 13.52 ± 2.60   Final old_new: 7.73 ± 1.49   Final new_new: 2.27 ± 2.78
Hits@20:
  Final val: 32.76 ± 3.14   Final Test: 18.28 ± 3.48   Final old_old: 19.94 ± 3.81   Final old_new: 11.66 ± 2.23   Final new_new: 2.27 ± 1.61
Hits@30:
  Final val: 40.24 ± 4.31   Final Test: 23.24 ± 2.76   Final old_old: 25.22 ± 3.00   Final old_new: 15.44 ± 1.94   Final new_new: 2.27 ± 1.61
Hits@50:
  Final val: 50.77 ± 5.24   Final Test: 31.62 ± 4.10   Final old_old: 34.07 ± 4.46   Final old_new: 22.09 ± 2.90   Final new_new: 3.64 ± 1.24
AUC:
  Final val: 94.62 ± 0.58   Final Test: 93.73 ± 0.50   Final old_old: 94.58 ± 0.49   Final old_new: 90.45 ± 0.58   Final new_new: 83.27 ± 2.12
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 26.28 ± 2.36   Final Test: 17.05 ± 6.31   Final old_old: 18.66 ± 7.03   Final old_new: 10.48 ± 3.51   Final new_new: 3.64 ± 2.59
Hits@20:
  Final val: 39.63 ± 3.60   Final Test: 21.93 ± 6.43   Final old_old: 24.05 ± 7.12   Final old_new: 13.30 ± 3.64   Final new_new: 4.55 ± 4.25
Hits@30:
  Final val: 46.22 ± 3.55   Final Test: 28.30 ± 6.36   Final old_old: 30.81 ± 6.99   Final old_new: 18.21 ± 3.81   Final new_new: 5.45 ± 3.05
Hits@50:
  Final val: 57.43 ± 4.13   Final Test: 39.17 ± 4.90   Final old_old: 42.22 ± 5.38   Final old_new: 27.08 ± 2.94   Final new_new: 8.64 ± 1.90
AUC:
  Final val: 95.64 ± 0.50   Final Test: 94.84 ± 0.52   Final old_old: 95.60 ± 0.46   Final old_new: 91.85 ± 0.79   Final new_new: 86.87 ± 1.31
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 22.89 ± 2.93   Final Test: 11.59 ± 2.15   Final old_old: 12.70 ± 2.30   Final old_new: 7.15 ± 1.67   Final new_new: 0.91 ± 1.24
Hits@20:
  Final val: 32.58 ± 3.14   Final Test: 19.35 ± 3.02   Final old_old: 20.90 ± 3.30   Final old_new: 13.25 ± 2.15   Final new_new: 2.27 ± 1.61
Hits@30:
  Final val: 39.64 ± 3.35   Final Test: 23.51 ± 2.35   Final old_old: 25.44 ± 2.54   Final old_new: 15.92 ± 1.75   Final new_new: 2.73 ± 1.02
Hits@50:
  Final val: 50.26 ± 2.89   Final Test: 29.50 ± 3.33   Final old_old: 31.87 ± 3.44   Final old_new: 20.10 ± 2.91   Final new_new: 5.91 ± 3.45
AUC:
  Final val: 94.57 ± 0.36   Final Test: 93.69 ± 0.26   Final old_old: 94.51 ± 0.33   Final old_new: 90.54 ± 0.22   Final new_new: 83.36 ± 1.64
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 24.95 ± 6.03   Final Test: 13.45 ± 5.79   Final old_old: 14.56 ± 6.07   Final old_new: 9.09 ± 4.78   Final new_new: 1.82 ± 1.90
Hits@20:
  Final val: 35.18 ± 7.51   Final Test: 18.82 ± 5.96   Final old_old: 20.38 ± 6.27   Final old_new: 12.59 ± 4.93   Final new_new: 3.18 ± 1.24
Hits@30:
  Final val: 43.42 ± 6.26   Final Test: 24.93 ± 7.47   Final old_old: 26.99 ± 7.80   Final old_new: 16.75 ± 6.26   Final new_new: 4.55 ± 2.78
Hits@50:
  Final val: 53.35 ± 7.21   Final Test: 33.00 ± 8.17   Final old_old: 35.59 ± 8.53   Final old_new: 22.80 ± 6.89   Final new_new: 5.91 ± 3.45
AUC:
  Final val: 94.85 ± 0.89   Final Test: 93.99 ± 1.01   Final old_old: 94.80 ± 0.93   Final old_new: 90.88 ± 1.32   Final new_new: 83.74 ± 3.25
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 26.95 ± 1.33   Final Test: 17.24 ± 6.43   Final old_old: 19.03 ± 7.09   Final old_new: 9.97 ± 3.75   Final new_new: 2.27 ± 3.21
Hits@20:
  Final val: 39.58 ± 4.73   Final Test: 21.79 ± 6.92   Final old_old: 23.91 ± 7.64   Final old_new: 13.20 ± 3.92   Final new_new: 3.64 ± 4.71
Hits@30:
  Final val: 46.14 ± 4.81   Final Test: 28.78 ± 6.19   Final old_old: 31.29 ± 6.88   Final old_new: 18.79 ± 3.51   Final new_new: 4.09 ± 2.96
Hits@50:
  Final val: 57.81 ± 4.82   Final Test: 38.60 ± 4.91   Final old_old: 41.73 ± 5.28   Final old_new: 26.07 ± 3.40   Final new_new: 9.09 ± 2.78
AUC:
  Final val: 95.65 ± 0.58   Final Test: 94.87 ± 0.62   Final old_old: 95.65 ± 0.58   Final old_new: 91.81 ± 0.81   Final new_new: 86.71 ± 1.18
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 39.01 ± 3.94   Final Test: 27.51 ± 3.49   Final old_old: 30.21 ± 3.79   Final old_new: 16.55 ± 2.48   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 52.64 ± 5.21   Final Test: 39.86 ± 2.15   Final old_old: 42.99 ± 2.20   Final old_new: 27.36 ± 2.42   Final new_new: 10.00 ± 4.71
Hits@30:
  Final val: 61.33 ± 4.21   Final Test: 47.82 ± 0.95   Final old_old: 51.08 ± 1.01   Final old_new: 34.96 ± 1.46   Final new_new: 13.18 ± 6.10
Hits@50:
  Final val: 68.89 ± 2.74   Final Test: 55.36 ± 2.44   Final old_old: 58.60 ± 2.61   Final old_new: 42.70 ± 2.16   Final new_new: 20.00 ± 3.37
AUC:
  Final val: 94.97 ± 0.09   Final Test: 94.57 ± 0.10   Final old_old: 95.31 ± 0.05   Final old_new: 91.62 ± 0.37   Final new_new: 86.69 ± 0.33
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 25.42 ± 4.45   Final Test: 13.86 ± 3.23   Final old_old: 15.24 ± 3.55   Final old_new: 8.21 ± 2.19   Final new_new: 3.18 ± 2.03
Hits@20:
  Final val: 35.75 ± 5.18   Final Test: 21.62 ± 3.99   Final old_old: 23.47 ± 4.43   Final old_new: 14.13 ± 2.43   Final new_new: 5.91 ± 4.43
Hits@30:
  Final val: 45.51 ± 4.08   Final Test: 27.19 ± 3.88   Final old_old: 29.55 ± 4.28   Final old_new: 17.78 ± 2.39   Final new_new: 4.09 ± 2.49
Hits@50:
  Final val: 55.08 ± 4.28   Final Test: 35.37 ± 2.51   Final old_old: 38.13 ± 2.65   Final old_new: 24.41 ± 2.18   Final new_new: 8.18 ± 3.45
AUC:
  Final val: 95.16 ± 0.56   Final Test: 94.50 ± 0.49   Final old_old: 95.21 ± 0.51   Final old_new: 91.72 ± 0.42   Final new_new: 87.09 ± 0.80
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 26.26 ± 1.93   Final Test: 13.83 ± 5.04   Final old_old: 15.12 ± 5.76   Final old_new: 8.59 ± 2.13   Final new_new: 2.27 ± 3.21
Hits@20:
  Final val: 40.70 ± 3.83   Final Test: 20.87 ± 4.36   Final old_old: 22.84 ± 4.93   Final old_new: 13.00 ± 2.31   Final new_new: 2.27 ± 3.21
Hits@30:
  Final val: 49.66 ± 4.36   Final Test: 30.55 ± 4.86   Final old_old: 33.29 ± 5.40   Final old_new: 19.72 ± 2.86   Final new_new: 2.27 ± 2.78
Hits@50:
  Final val: 59.51 ± 4.19   Final Test: 38.26 ± 3.98   Final old_old: 41.32 ± 4.48   Final old_new: 26.07 ± 2.33   Final new_new: 8.18 ± 2.03
AUC:
  Final val: 95.76 ± 0.53   Final Test: 94.95 ± 0.45   Final old_old: 95.69 ± 0.44   Final old_new: 92.05 ± 0.58   Final new_new: 87.38 ± 0.82
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 25.53 ± 3.02   Final Test: 17.42 ± 6.01   Final old_old: 19.13 ± 6.63   Final old_new: 10.58 ± 3.65   Final new_new: 1.36 ± 3.05
Hits@20:
  Final val: 39.74 ± 5.30   Final Test: 21.86 ± 4.48   Final old_old: 23.82 ± 5.08   Final old_new: 13.93 ± 2.28   Final new_new: 4.55 ± 3.59
Hits@30:
  Final val: 47.79 ± 5.72   Final Test: 28.09 ± 5.33   Final old_old: 30.50 ± 5.87   Final old_new: 18.46 ± 3.45   Final new_new: 5.45 ± 2.59
Hits@50:
  Final val: 58.50 ± 6.15   Final Test: 39.81 ± 6.83   Final old_old: 42.84 ± 7.22   Final old_new: 27.71 ± 5.56   Final new_new: 10.45 ± 2.03
AUC:
  Final val: 95.64 ± 0.74   Final Test: 94.89 ± 0.72   Final old_old: 95.64 ± 0.67   Final old_new: 91.91 ± 0.98   Final new_new: 86.71 ± 1.50
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 21.18 ± 3.50   Final Test: 11.52 ± 1.65   Final old_old: 12.66 ± 1.76   Final old_new: 6.93 ± 1.44   Final new_new: 1.82 ± 1.02
Hits@20:
  Final val: 31.68 ± 2.85   Final Test: 16.43 ± 2.53   Final old_old: 17.89 ± 2.80   Final old_new: 10.63 ± 1.66   Final new_new: 1.82 ± 1.02
Hits@30:
  Final val: 39.62 ± 4.34   Final Test: 21.58 ± 0.97   Final old_old: 23.48 ± 1.01   Final old_new: 14.08 ± 1.41   Final new_new: 2.27 ± 0.00
Hits@50:
  Final val: 48.93 ± 5.63   Final Test: 31.99 ± 5.06   Final old_old: 34.43 ± 5.36   Final old_new: 22.39 ± 3.81   Final new_new: 5.91 ± 3.05
AUC:
  Final val: 94.37 ± 0.65   Final Test: 93.57 ± 0.60   Final old_old: 94.37 ± 0.56   Final old_new: 90.49 ± 0.76   Final new_new: 83.31 ± 2.24
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 28.39 ± 2.81   Final Test: 17.70 ± 2.97   Final old_old: 19.62 ± 3.51   Final old_new: 9.95 ± 0.84   Final new_new: 0.91 ± 1.24
Hits@20:
  Final val: 40.34 ± 3.80   Final Test: 22.67 ± 4.40   Final old_old: 24.96 ± 5.08   Final old_new: 13.48 ± 1.69   Final new_new: 1.82 ± 1.90
Hits@30:
  Final val: 48.99 ± 4.83   Final Test: 28.90 ± 4.66   Final old_old: 31.54 ± 5.15   Final old_new: 18.39 ± 2.86   Final new_new: 2.73 ± 2.49
Hits@50:
  Final val: 59.78 ± 4.65   Final Test: 39.04 ± 2.93   Final old_old: 42.29 ± 3.31   Final old_new: 26.22 ± 1.78   Final new_new: 5.00 ± 3.73
AUC:
  Final val: 95.65 ± 0.55   Final Test: 94.97 ± 0.49   Final old_old: 95.69 ± 0.52   Final old_new: 92.11 ± 0.40   Final new_new: 87.30 ± 0.52
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 22.42 ± 1.87   Final Test: 11.44 ± 2.25   Final old_old: 12.60 ± 2.35   Final old_new: 6.78 ± 1.97   Final new_new: 1.36 ± 1.24
Hits@20:
  Final val: 30.67 ± 3.37   Final Test: 16.52 ± 2.15   Final old_old: 17.90 ± 2.23   Final old_new: 10.88 ± 1.85   Final new_new: 5.00 ± 4.07
Hits@30:
  Final val: 37.89 ± 4.02   Final Test: 22.94 ± 4.14   Final old_old: 24.88 ± 4.38   Final old_new: 15.24 ± 3.24   Final new_new: 3.18 ± 2.03
Hits@50:
  Final val: 48.33 ± 4.91   Final Test: 29.58 ± 3.19   Final old_old: 31.89 ± 3.34   Final old_new: 20.40 ± 2.77   Final new_new: 6.82 ± 4.25
AUC:
  Final val: 94.15 ± 0.66   Final Test: 93.37 ± 0.48   Final old_old: 94.16 ± 0.50   Final old_new: 90.33 ± 0.56   Final new_new: 83.21 ± 1.04
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 23.17 ± 3.34   Final Test: 12.57 ± 2.10   Final old_old: 13.74 ± 2.25   Final old_new: 7.86 ± 1.64   Final new_new: 1.82 ± 1.90
Hits@20:
  Final val: 31.62 ± 2.91   Final Test: 16.61 ± 1.60   Final old_old: 18.11 ± 1.78   Final old_new: 10.60 ± 1.60   Final new_new: 1.82 ± 1.90
Hits@30:
  Final val: 39.35 ± 3.64   Final Test: 23.17 ± 1.28   Final old_old: 25.13 ± 1.46   Final old_new: 15.42 ± 1.50   Final new_new: 3.64 ± 1.24
Hits@50:
  Final val: 49.26 ± 3.97   Final Test: 29.06 ± 2.57   Final old_old: 31.41 ± 2.59   Final old_new: 19.85 ± 2.73   Final new_new: 3.18 ± 2.03
AUC:
  Final val: 94.36 ± 0.45   Final Test: 93.49 ± 0.36   Final old_old: 94.32 ± 0.32   Final old_new: 90.31 ± 0.66   Final new_new: 82.77 ± 1.97
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 28.77 ± 3.56   Final Test: 17.26 ± 1.92   Final old_old: 19.05 ± 2.06   Final old_new: 10.13 ± 1.48   Final new_new: 0.00 ± 0.00
Hits@20:
  Final val: 38.72 ± 4.42   Final Test: 23.30 ± 2.39   Final old_old: 25.48 ± 2.52   Final old_new: 14.63 ± 2.24   Final new_new: 1.82 ± 2.96
Hits@30:
  Final val: 46.92 ± 5.11   Final Test: 27.97 ± 3.64   Final old_old: 30.49 ± 4.01   Final old_new: 17.93 ± 2.39   Final new_new: 2.73 ± 1.02
Hits@50:
  Final val: 57.72 ± 3.70   Final Test: 36.57 ± 3.69   Final old_old: 39.58 ± 3.93   Final old_new: 24.69 ± 3.03   Final new_new: 5.45 ± 2.03
AUC:
  Final val: 95.54 ± 0.43   Final Test: 94.83 ± 0.43   Final old_old: 95.55 ± 0.44   Final old_new: 91.97 ± 0.42   Final new_new: 86.98 ± 0.92
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 27.71 ± 2.30   Final Test: 16.86 ± 7.32   Final old_old: 18.61 ± 8.31   Final old_new: 9.82 ± 3.24   Final new_new: 0.91 ± 1.24
Hits@20:
  Final val: 41.40 ± 2.84   Final Test: 23.90 ± 6.56   Final old_old: 26.17 ± 7.25   Final old_new: 14.86 ± 3.74   Final new_new: 1.36 ± 3.05
Hits@30:
  Final val: 50.34 ± 4.28   Final Test: 31.79 ± 6.22   Final old_old: 34.66 ± 6.78   Final old_new: 20.35 ± 3.96   Final new_new: 3.18 ± 3.80
Hits@50:
  Final val: 60.35 ± 3.61   Final Test: 42.60 ± 5.42   Final old_old: 46.03 ± 5.96   Final old_new: 28.89 ± 3.32   Final new_new: 9.55 ± 2.96
AUC:
  Final val: 96.02 ± 0.46   Final Test: 95.32 ± 0.48   Final old_old: 96.02 ± 0.46   Final old_new: 92.53 ± 0.62   Final new_new: 87.86 ± 0.85
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 32.52 ± 3.47   Final Test: 19.83 ± 3.97   Final old_old: 21.96 ± 4.45   Final old_new: 11.28 ± 2.11   Final new_new: 0.00 ± 0.00
Hits@20:
  Final val: 46.28 ± 0.98   Final Test: 26.06 ± 3.20   Final old_old: 28.51 ± 3.29   Final old_new: 16.35 ± 3.09   Final new_new: 0.91 ± 1.24
Hits@30:
  Final val: 53.86 ± 1.11   Final Test: 32.05 ± 1.28   Final old_old: 35.03 ± 1.36   Final old_new: 20.23 ± 1.49   Final new_new: 1.36 ± 1.24
Hits@50:
  Final val: 64.03 ± 1.19   Final Test: 44.82 ± 0.96   Final old_old: 48.31 ± 1.02   Final old_new: 31.23 ± 1.27   Final new_new: 5.45 ± 2.59
AUC:
  Final val: 96.36 ± 0.13   Final Test: 95.72 ± 0.16   Final old_old: 96.38 ± 0.14   Final old_new: 93.12 ± 0.31   Final new_new: 88.10 ± 0.87
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 28.57 ± 3.01   Final Test: 19.35 ± 4.80   Final old_old: 21.39 ± 5.39   Final old_new: 11.06 ± 2.51   Final new_new: 2.27 ± 2.78
Hits@20:
  Final val: 43.06 ± 4.86   Final Test: 25.53 ± 5.14   Final old_old: 27.89 ± 5.77   Final old_new: 16.02 ± 2.91   Final new_new: 4.55 ± 2.27
Hits@30:
  Final val: 51.45 ± 5.18   Final Test: 30.59 ± 4.42   Final old_old: 33.39 ± 4.83   Final old_new: 19.35 ± 2.90   Final new_new: 5.00 ± 1.90
Hits@50:
  Final val: 61.33 ± 5.65   Final Test: 40.11 ± 6.49   Final old_old: 43.31 ± 6.88   Final old_new: 27.30 ± 5.03   Final new_new: 10.00 ± 2.59
AUC:
  Final val: 95.99 ± 0.64   Final Test: 95.27 ± 0.63   Final old_old: 95.99 ± 0.61   Final old_new: 92.46 ± 0.75   Final new_new: 87.13 ± 1.67
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 22.82 ± 1.61   Final Test: 11.91 ± 1.93   Final old_old: 13.04 ± 2.02   Final old_new: 7.43 ± 1.75   Final new_new: 0.91 ± 1.24
Hits@20:
  Final val: 31.84 ± 2.59   Final Test: 17.94 ± 1.22   Final old_old: 19.49 ± 1.35   Final old_new: 11.79 ± 0.72   Final new_new: 2.73 ± 1.90
Hits@30:
  Final val: 38.88 ± 4.34   Final Test: 22.07 ± 2.89   Final old_old: 23.92 ± 3.04   Final old_new: 14.74 ± 2.51   Final new_new: 3.64 ± 2.03
Hits@50:
  Final val: 49.58 ± 4.97   Final Test: 28.78 ± 3.55   Final old_old: 31.09 ± 3.74   Final old_new: 19.70 ± 2.96   Final new_new: 4.09 ± 1.90
AUC:
  Final val: 94.32 ± 0.51   Final Test: 93.46 ± 0.50   Final old_old: 94.31 ± 0.44   Final old_new: 90.24 ± 0.71   Final new_new: 82.83 ± 2.30
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 26.03 ± 1.97   Final Test: 15.00 ± 2.55   Final old_old: 16.48 ± 2.98   Final old_new: 9.09 ± 1.18   Final new_new: 1.36 ± 3.05
Hits@20:
  Final val: 38.97 ± 3.66   Final Test: 21.01 ± 4.35   Final old_old: 23.00 ± 4.89   Final old_new: 13.05 ± 2.27   Final new_new: 1.82 ± 2.96
Hits@30:
  Final val: 46.37 ± 4.89   Final Test: 28.19 ± 6.18   Final old_old: 30.63 ± 6.74   Final old_new: 18.41 ± 3.95   Final new_new: 5.45 ± 2.59
Hits@50:
  Final val: 56.96 ± 5.01   Final Test: 37.68 ± 5.51   Final old_old: 40.70 ± 5.97   Final old_new: 25.69 ± 3.74   Final new_new: 8.18 ± 3.45
AUC:
  Final val: 95.52 ± 0.52   Final Test: 94.75 ± 0.53   Final old_old: 95.52 ± 0.48   Final old_new: 91.69 ± 0.76   Final new_new: 86.66 ± 1.66
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 24.77 ± 1.08   Final Test: 11.97 ± 4.65   Final old_old: 13.05 ± 5.04   Final old_new: 7.61 ± 3.20   Final new_new: 1.82 ± 1.02
Hits@20:
  Final val: 34.63 ± 4.51   Final Test: 19.03 ± 4.63   Final old_old: 20.57 ± 5.06   Final old_new: 12.90 ± 2.92   Final new_new: 4.09 ± 2.96
Hits@30:
  Final val: 43.74 ± 5.04   Final Test: 27.28 ± 4.53   Final old_old: 29.47 ± 4.86   Final old_new: 18.51 ± 3.30   Final new_new: 5.91 ± 2.59
Hits@50:
  Final val: 54.63 ± 6.84   Final Test: 34.97 ± 5.51   Final old_old: 37.67 ± 5.88   Final old_new: 24.21 ± 4.22   Final new_new: 8.64 ± 2.49
AUC:
  Final val: 95.25 ± 0.84   Final Test: 94.47 ± 0.75   Final old_old: 95.18 ± 0.73   Final old_new: 91.71 ± 0.83   Final new_new: 86.77 ± 1.25
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 26.83 ± 2.76   Final Test: 16.30 ± 2.91   Final old_old: 18.02 ± 3.35   Final old_new: 9.29 ± 1.54   Final new_new: 2.27 ± 3.21
Hits@20:
  Final val: 40.68 ± 5.01   Final Test: 22.38 ± 3.70   Final old_old: 24.53 ± 4.19   Final old_new: 13.75 ± 1.85   Final new_new: 2.73 ± 2.96
Hits@30:
  Final val: 47.70 ± 4.75   Final Test: 28.87 ± 1.84   Final old_old: 31.37 ± 2.07   Final old_new: 18.87 ± 1.09   Final new_new: 5.00 ± 3.37
Hits@50:
  Final val: 58.66 ± 4.51   Final Test: 38.84 ± 4.72   Final old_old: 41.94 ± 5.20   Final old_new: 26.45 ± 2.93   Final new_new: 9.09 ± 2.78
AUC:
  Final val: 95.64 ± 0.63   Final Test: 94.99 ± 0.57   Final old_old: 95.66 ± 0.57   Final old_new: 92.35 ± 0.64   Final new_new: 88.23 ± 0.40
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 27.69 ± 6.91   Final Test: 17.10 ± 6.48   Final old_old: 18.87 ± 7.05   Final old_new: 9.97 ± 4.36   Final new_new: 1.36 ± 2.03
Hits@20:
  Final val: 37.72 ± 7.61   Final Test: 22.85 ± 5.84   Final old_old: 24.88 ± 6.52   Final old_new: 14.66 ± 3.48   Final new_new: 5.00 ± 5.88
Hits@30:
  Final val: 45.87 ± 8.17   Final Test: 28.31 ± 5.51   Final old_old: 30.83 ± 6.04   Final old_new: 18.14 ± 3.61   Final new_new: 5.91 ± 3.45
Hits@50:
  Final val: 56.19 ± 7.14   Final Test: 36.82 ± 6.35   Final old_old: 39.77 ± 6.88   Final old_new: 25.01 ± 4.62   Final new_new: 9.09 ± 4.82
AUC:
  Final val: 95.24 ± 0.97   Final Test: 94.58 ± 1.05   Final old_old: 95.29 ± 1.04   Final old_new: 91.79 ± 1.04   Final new_new: 86.92 ± 1.92
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 27.71 ± 2.13   Final Test: 17.26 ± 4.29   Final old_old: 19.04 ± 4.79   Final old_new: 10.03 ± 2.40   Final new_new: 1.82 ± 1.90
Hits@20:
  Final val: 42.56 ± 4.27   Final Test: 24.03 ± 3.97   Final old_old: 26.27 ± 4.45   Final old_new: 15.04 ± 2.34   Final new_new: 3.18 ± 2.03
Hits@30:
  Final val: 49.59 ± 4.09   Final Test: 31.47 ± 5.02   Final old_old: 34.17 ± 5.41   Final old_new: 20.68 ± 3.65   Final new_new: 5.91 ± 1.24
Hits@50:
  Final val: 60.87 ± 4.37   Final Test: 40.13 ± 5.63   Final old_old: 43.22 ± 6.01   Final old_new: 27.91 ± 4.32   Final new_new: 7.73 ± 2.03
AUC:
  Final val: 95.97 ± 0.57   Final Test: 95.21 ± 0.53   Final old_old: 95.93 ± 0.52   Final old_new: 92.39 ± 0.64   Final new_new: 87.39 ± 1.37
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 30.34 ± 4.24   Final Test: 17.29 ± 1.82   Final old_old: 19.31 ± 1.94   Final old_new: 9.09 ± 1.53   Final new_new: 0.45 ± 1.02
Hits@20:
  Final val: 40.59 ± 2.84   Final Test: 24.02 ± 4.57   Final old_old: 26.49 ± 4.84   Final old_new: 14.11 ± 3.68   Final new_new: 0.91 ± 1.24
Hits@30:
  Final val: 49.97 ± 3.24   Final Test: 31.02 ± 3.54   Final old_old: 33.90 ± 3.69   Final old_new: 19.62 ± 3.32   Final new_new: 1.82 ± 1.90
Hits@50:
  Final val: 59.78 ± 2.81   Final Test: 39.58 ± 3.77   Final old_old: 42.82 ± 3.86   Final old_new: 26.90 ± 3.62   Final new_new: 4.55 ± 3.21
AUC:
  Final val: 95.83 ± 0.37   Final Test: 95.16 ± 0.33   Final old_old: 95.88 ± 0.31   Final old_new: 92.35 ± 0.45   Final new_new: 87.70 ± 0.85
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 24.14 ± 6.43   Final Test: 13.20 ± 4.18   Final old_old: 14.38 ± 4.40   Final old_new: 8.49 ± 3.46   Final new_new: 1.36 ± 1.24
Hits@20:
  Final val: 34.99 ± 6.79   Final Test: 18.70 ± 6.13   Final old_old: 20.47 ± 6.57   Final old_new: 11.61 ± 4.40   Final new_new: 2.27 ± 1.61
Hits@30:
  Final val: 42.99 ± 5.93   Final Test: 24.44 ± 4.82   Final old_old: 26.64 ± 5.36   Final old_new: 15.67 ± 2.79   Final new_new: 2.73 ± 1.02
Hits@50:
  Final val: 53.63 ± 6.92   Final Test: 33.69 ± 5.36   Final old_old: 36.41 ± 5.71   Final old_new: 22.80 ± 4.07   Final new_new: 7.73 ± 3.45
AUC:
  Final val: 94.96 ± 0.87   Final Test: 94.14 ± 0.81   Final old_old: 94.91 ± 0.77   Final old_new: 91.13 ± 0.93   Final new_new: 84.99 ± 1.94
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 23.31 ± 2.42   Final Test: 12.11 ± 1.38   Final old_old: 13.32 ± 1.58   Final old_new: 7.23 ± 0.84   Final new_new: 1.36 ± 1.24
Hits@20:
  Final val: 31.56 ± 3.04   Final Test: 18.26 ± 3.25   Final old_old: 19.92 ± 3.58   Final old_new: 11.69 ± 1.96   Final new_new: 1.82 ± 1.02
Hits@30:
  Final val: 40.09 ± 4.01   Final Test: 23.20 ± 3.24   Final old_old: 25.22 ± 3.49   Final old_new: 15.26 ± 2.51   Final new_new: 1.82 ± 1.02
Hits@50:
  Final val: 50.75 ± 3.15   Final Test: 30.17 ± 3.52   Final old_old: 32.47 ± 3.71   Final old_new: 21.23 ± 2.82   Final new_new: 3.18 ± 1.24
AUC:
  Final val: 94.55 ± 0.48   Final Test: 93.63 ± 0.38   Final old_old: 94.50 ± 0.42   Final old_new: 90.27 ± 0.35   Final new_new: 83.00 ± 2.12
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 33.01 ± 6.00   Final Test: 22.67 ± 4.10   Final old_old: 24.76 ± 4.31   Final old_new: 14.33 ± 3.47   Final new_new: 2.27 ± 2.78
Hits@20:
  Final val: 48.96 ± 6.49   Final Test: 30.90 ± 4.96   Final old_old: 33.48 ± 5.34   Final old_new: 20.68 ± 3.66   Final new_new: 4.55 ± 3.21
Hits@30:
  Final val: 57.28 ± 5.55   Final Test: 39.34 ± 2.62   Final old_old: 42.52 ± 2.76   Final old_new: 26.75 ± 2.34   Final new_new: 6.36 ± 3.73
Hits@50:
  Final val: 68.52 ± 4.56   Final Test: 49.96 ± 5.08   Final old_old: 53.57 ± 5.34   Final old_new: 35.79 ± 4.25   Final new_new: 10.45 ± 2.03
AUC:
  Final val: 96.87 ± 0.35   Final Test: 96.18 ± 0.29   Final old_old: 96.79 ± 0.29   Final old_new: 93.82 ± 0.29   Final new_new: 88.93 ± 0.90
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 28.42 ± 5.54   Final Test: 17.71 ± 3.75   Final old_old: 19.64 ± 4.21   Final old_new: 9.87 ± 2.00   Final new_new: 0.91 ± 2.03
Hits@20:
  Final val: 39.54 ± 7.28   Final Test: 23.43 ± 3.43   Final old_old: 25.59 ± 3.98   Final old_new: 14.81 ± 1.60   Final new_new: 2.73 ± 4.93
Hits@30:
  Final val: 47.68 ± 7.16   Final Test: 28.41 ± 5.10   Final old_old: 30.99 ± 5.56   Final old_new: 18.06 ± 3.50   Final new_new: 4.55 ± 2.27
Hits@50:
  Final val: 58.29 ± 5.64   Final Test: 38.45 ± 4.60   Final old_old: 41.46 ± 5.01   Final old_new: 26.52 ± 3.12   Final new_new: 8.18 ± 3.05
AUC:
  Final val: 95.53 ± 0.75   Final Test: 94.90 ± 0.75   Final old_old: 95.62 ± 0.74   Final old_new: 92.07 ± 0.80   Final new_new: 87.40 ± 1.20
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 22.12 ± 2.33   Final Test: 11.65 ± 1.93   Final old_old: 12.79 ± 2.06   Final old_new: 7.08 ± 1.61   Final new_new: 0.91 ± 1.24
Hits@20:
  Final val: 32.38 ± 2.44   Final Test: 18.89 ± 2.54   Final old_old: 20.60 ± 2.82   Final old_new: 12.09 ± 1.55   Final new_new: 2.27 ± 0.00
Hits@30:
  Final val: 40.51 ± 3.82   Final Test: 24.13 ± 3.59   Final old_old: 26.19 ± 3.85   Final old_new: 15.99 ± 2.72   Final new_new: 2.73 ± 1.02
Hits@50:
  Final val: 51.85 ± 3.46   Final Test: 31.06 ± 2.95   Final old_old: 33.47 ± 3.25   Final old_new: 21.56 ± 1.90   Final new_new: 5.45 ± 2.59
AUC:
  Final val: 94.67 ± 0.56   Final Test: 93.74 ± 0.41   Final old_old: 94.64 ± 0.44   Final old_new: 90.29 ± 0.27   Final new_new: 82.78 ± 2.25
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 25.87 ± 3.25   Final Test: 16.94 ± 4.83   Final old_old: 18.65 ± 5.19   Final old_new: 10.10 ± 3.53   Final new_new: 0.91 ± 2.03
Hits@20:
  Final val: 39.46 ± 2.70   Final Test: 21.61 ± 2.18   Final old_old: 23.60 ± 2.21   Final old_new: 13.65 ± 2.15   Final new_new: 2.73 ± 2.96
Hits@30:
  Final val: 48.37 ± 2.68   Final Test: 30.31 ± 4.09   Final old_old: 32.98 ± 4.39   Final old_new: 19.60 ± 2.98   Final new_new: 6.36 ± 3.37
Hits@50:
  Final val: 59.71 ± 1.99   Final Test: 40.38 ± 2.10   Final old_old: 43.52 ± 2.13   Final old_new: 27.76 ± 2.21   Final new_new: 11.82 ± 1.90
AUC:
  Final val: 95.90 ± 0.31   Final Test: 95.16 ± 0.30   Final old_old: 95.87 ± 0.28   Final old_new: 92.33 ± 0.38   Final new_new: 88.03 ± 1.27
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 28.30 ± 3.72   Final Test: 15.19 ± 6.60   Final old_old: 16.78 ± 7.31   Final old_new: 8.79 ± 3.84   Final new_new: 0.91 ± 2.03
Hits@20:
  Final val: 41.08 ± 6.62   Final Test: 22.84 ± 3.75   Final old_old: 24.97 ± 4.28   Final old_new: 14.36 ± 2.09   Final new_new: 2.27 ± 1.61
Hits@30:
  Final val: 47.77 ± 6.65   Final Test: 29.24 ± 5.54   Final old_old: 31.93 ± 6.09   Final old_new: 18.61 ± 3.42   Final new_new: 1.82 ± 1.90
Hits@50:
  Final val: 58.66 ± 6.87   Final Test: 41.55 ± 5.50   Final old_old: 44.73 ± 6.01   Final old_new: 29.04 ± 3.42   Final new_new: 8.18 ± 4.43
AUC:
  Final val: 95.56 ± 0.80   Final Test: 94.78 ± 0.73   Final old_old: 95.59 ± 0.75   Final old_new: 91.63 ± 0.66   Final new_new: 85.76 ± 1.84
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 31.45 ± 2.42   Final Test: 22.24 ± 1.42   Final old_old: 24.51 ± 1.49   Final old_new: 13.17 ± 1.42   Final new_new: 0.00 ± 0.00
Hits@20:
  Final val: 46.95 ± 1.74   Final Test: 29.34 ± 2.42   Final old_old: 32.12 ± 2.40   Final old_new: 18.31 ± 2.63   Final new_new: 1.36 ± 3.05
Hits@30:
  Final val: 55.41 ± 1.34   Final Test: 34.65 ± 2.01   Final old_old: 37.63 ± 1.93   Final old_new: 22.87 ± 2.67   Final new_new: 3.64 ± 3.45
Hits@50:
  Final val: 65.78 ± 2.11   Final Test: 49.27 ± 2.74   Final old_old: 52.88 ± 2.93   Final old_new: 35.19 ± 2.25   Final new_new: 9.09 ± 2.78
AUC:
  Final val: 96.66 ± 0.17   Final Test: 96.10 ± 0.14   Final old_old: 96.66 ± 0.15   Final old_new: 93.97 ± 0.18   Final new_new: 89.43 ± 0.87
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 27.99 ± 2.56   Final Test: 14.91 ± 3.46   Final old_old: 16.54 ± 3.90   Final old_new: 8.29 ± 1.76   Final new_new: 1.36 ± 2.03
Hits@20:
  Final val: 38.66 ± 3.67   Final Test: 20.23 ± 3.83   Final old_old: 22.15 ± 4.32   Final old_new: 12.49 ± 1.94   Final new_new: 3.18 ± 2.59
Hits@30:
  Final val: 46.50 ± 5.09   Final Test: 26.62 ± 6.22   Final old_old: 29.03 ± 6.83   Final old_new: 16.90 ± 3.94   Final new_new: 5.00 ± 1.90
Hits@50:
  Final val: 57.13 ± 3.92   Final Test: 38.43 ± 3.95   Final old_old: 41.50 ± 4.34   Final old_new: 26.20 ± 2.71   Final new_new: 8.64 ± 4.07
AUC:
  Final val: 95.51 ± 0.49   Final Test: 94.69 ± 0.48   Final old_old: 95.49 ± 0.43   Final old_new: 91.51 ± 0.75   Final new_new: 86.21 ± 1.09
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 23.69 ± 4.17   Final Test: 12.37 ± 3.48   Final old_old: 13.54 ± 3.69   Final old_new: 7.63 ± 2.72   Final new_new: 2.73 ± 1.90
Hits@20:
  Final val: 34.20 ± 5.38   Final Test: 18.60 ± 3.49   Final old_old: 20.23 ± 3.79   Final old_new: 12.07 ± 2.38   Final new_new: 3.18 ± 1.24
Hits@30:
  Final val: 42.14 ± 5.56   Final Test: 23.86 ± 4.17   Final old_old: 25.86 ± 4.56   Final old_new: 15.94 ± 2.74   Final new_new: 3.64 ± 2.03
Hits@50:
  Final val: 51.90 ± 6.38   Final Test: 31.43 ± 4.34   Final old_old: 33.88 ± 4.62   Final old_new: 21.84 ± 3.30   Final new_new: 4.55 ± 2.27
AUC:
  Final val: 94.69 ± 0.85   Final Test: 93.80 ± 0.80   Final old_old: 94.64 ± 0.76   Final old_new: 90.58 ± 0.92   Final new_new: 83.04 ± 2.30
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 26.75 ± 3.83   Final Test: 14.25 ± 2.25   Final old_old: 15.64 ± 2.48   Final old_new: 8.59 ± 1.48   Final new_new: 3.18 ± 3.05
Hits@20:
  Final val: 35.56 ± 4.80   Final Test: 21.36 ± 3.19   Final old_old: 23.15 ± 3.58   Final old_new: 14.13 ± 2.13   Final new_new: 6.36 ± 4.93
Hits@30:
  Final val: 43.38 ± 5.51   Final Test: 26.56 ± 4.36   Final old_old: 28.81 ± 4.76   Final old_new: 17.48 ± 3.01   Final new_new: 6.36 ± 4.07
Hits@50:
  Final val: 53.78 ± 5.01   Final Test: 34.92 ± 4.03   Final old_old: 37.62 ± 4.12   Final old_new: 24.06 ± 3.77   Final new_new: 10.91 ± 3.37
AUC:
  Final val: 94.91 ± 0.66   Final Test: 94.18 ± 0.71   Final old_old: 94.93 ± 0.69   Final old_new: 91.28 ± 0.78   Final new_new: 85.81 ± 2.08
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 29.30 ± 1.82   Final Test: 18.72 ± 3.41   Final old_old: 20.65 ± 3.68   Final old_new: 11.01 ± 2.54   Final new_new: 0.91 ± 1.24
Hits@20:
  Final val: 42.87 ± 4.84   Final Test: 26.36 ± 2.66   Final old_old: 28.80 ± 2.83   Final old_new: 16.60 ± 2.20   Final new_new: 2.73 ± 2.96
Hits@30:
  Final val: 50.19 ± 3.93   Final Test: 30.98 ± 2.83   Final old_old: 33.70 ± 2.98   Final old_new: 20.20 ± 2.80   Final new_new: 3.18 ± 1.24
Hits@50:
  Final val: 61.24 ± 4.56   Final Test: 40.88 ± 4.80   Final old_old: 44.12 ± 5.12   Final old_new: 28.06 ± 3.66   Final new_new: 7.27 ± 2.96
AUC:
  Final val: 96.10 ± 0.48   Final Test: 95.47 ± 0.50   Final old_old: 96.10 ± 0.45   Final old_new: 93.02 ± 0.75   Final new_new: 88.35 ± 1.11
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 27.32 ± 2.36   Final Test: 14.48 ± 4.62   Final old_old: 15.99 ± 5.10   Final old_new: 8.41 ± 2.89   Final new_new: 0.45 ± 1.02
Hits@20:
  Final val: 39.64 ± 3.80   Final Test: 21.76 ± 4.27   Final old_old: 23.71 ± 4.73   Final old_new: 13.90 ± 2.53   Final new_new: 3.64 ± 3.45
Hits@30:
  Final val: 46.58 ± 5.13   Final Test: 28.94 ± 3.61   Final old_old: 31.49 ± 3.95   Final old_new: 18.74 ± 2.42   Final new_new: 4.55 ± 1.61
Hits@50:
  Final val: 57.26 ± 4.44   Final Test: 40.15 ± 3.36   Final old_old: 43.40 ± 3.67   Final old_new: 27.18 ± 2.53   Final new_new: 9.55 ± 3.73
AUC:
  Final val: 95.56 ± 0.51   Final Test: 94.73 ± 0.46   Final old_old: 95.54 ± 0.40   Final old_new: 91.57 ± 0.73   Final new_new: 86.08 ± 1.46
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 30.33 ± 3.19   Final Test: 20.63 ± 1.69   Final old_old: 22.35 ± 1.83   Final old_new: 13.75 ± 1.41   Final new_new: 4.55 ± 2.27
Hits@20:
  Final val: 42.44 ± 4.23   Final Test: 27.35 ± 3.09   Final old_old: 29.55 ± 3.08   Final old_new: 18.64 ± 3.40   Final new_new: 5.45 ± 2.59
Hits@30:
  Final val: 51.36 ± 4.97   Final Test: 36.55 ± 4.07   Final old_old: 39.35 ± 4.46   Final old_new: 25.31 ± 2.80   Final new_new: 10.45 ± 3.45
Hits@50:
  Final val: 63.44 ± 4.69   Final Test: 48.58 ± 4.11   Final old_old: 51.91 ± 4.68   Final old_new: 35.39 ± 2.01   Final new_new: 15.00 ± 2.03
AUC:
  Final val: 96.57 ± 0.41   Final Test: 95.96 ± 0.39   Final old_old: 96.49 ± 0.47   Final old_new: 93.97 ± 0.11   Final new_new: 88.93 ± 0.77
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 30.63 ± 2.07   Final Test: 16.81 ± 2.19   Final old_old: 18.59 ± 2.32   Final old_new: 9.65 ± 2.16   Final new_new: 0.45 ± 1.02
Hits@20:
  Final val: 41.67 ± 4.45   Final Test: 25.41 ± 3.36   Final old_old: 27.72 ± 3.63   Final old_new: 16.27 ± 2.55   Final new_new: 1.82 ± 1.90
Hits@30:
  Final val: 48.81 ± 3.22   Final Test: 31.66 ± 4.19   Final old_old: 34.40 ± 4.52   Final old_new: 20.81 ± 3.12   Final new_new: 3.64 ± 4.71
Hits@50:
  Final val: 60.97 ± 5.23   Final Test: 40.18 ± 3.52   Final old_old: 43.31 ± 3.87   Final old_new: 27.71 ± 2.55   Final new_new: 9.09 ± 4.55
AUC:
  Final val: 95.83 ± 0.51   Final Test: 95.10 ± 0.46   Final old_old: 95.85 ± 0.50   Final old_new: 92.14 ± 0.41   Final new_new: 86.70 ± 1.40
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 22.87 ± 2.09   Final Test: 11.83 ± 1.79   Final old_old: 13.03 ± 1.95   Final old_new: 7.03 ± 1.27   Final new_new: 0.91 ± 1.24
Hits@20:
  Final val: 33.72 ± 2.97   Final Test: 19.21 ± 2.75   Final old_old: 20.98 ± 3.07   Final old_new: 12.17 ± 1.47   Final new_new: 2.27 ± 1.61
Hits@30:
  Final val: 41.87 ± 3.48   Final Test: 24.59 ± 3.83   Final old_old: 26.66 ± 4.19   Final old_new: 16.37 ± 2.43   Final new_new: 3.64 ± 2.03
Hits@50:
  Final val: 52.44 ± 4.05   Final Test: 32.23 ± 3.45   Final old_old: 34.71 ± 3.69   Final old_new: 22.44 ± 2.95   Final new_new: 5.91 ± 3.05
AUC:
  Final val: 94.78 ± 0.60   Final Test: 93.86 ± 0.64   Final old_old: 94.73 ± 0.64   Final old_new: 90.51 ± 0.71   Final new_new: 83.10 ± 2.61
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 23.95 ± 4.01   Final Test: 14.78 ± 3.36   Final old_old: 16.26 ± 3.49   Final old_new: 8.82 ± 2.99   Final new_new: 1.36 ± 1.24
Hits@20:
  Final val: 34.04 ± 5.68   Final Test: 21.04 ± 3.48   Final old_old: 22.84 ± 3.86   Final old_new: 13.88 ± 2.30   Final new_new: 2.73 ± 1.02
Hits@30:
  Final val: 40.56 ± 5.40   Final Test: 26.83 ± 4.22   Final old_old: 29.15 ± 4.59   Final old_new: 17.53 ± 2.82   Final new_new: 5.00 ± 2.96
Hits@50:
  Final val: 52.07 ± 5.08   Final Test: 34.88 ± 5.59   Final old_old: 37.57 ± 5.88   Final old_new: 24.26 ± 4.61   Final new_new: 6.82 ± 4.82
AUC:
  Final val: 94.89 ± 0.96   Final Test: 94.23 ± 0.86   Final old_old: 95.00 ± 0.88   Final old_new: 91.23 ± 0.83   Final new_new: 85.85 ± 1.87
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 30.78 ± 2.09   Final Test: 15.90 ± 4.93   Final old_old: 17.72 ± 5.58   Final old_new: 8.54 ± 2.49   Final new_new: 0.00 ± 0.00
Hits@20:
  Final val: 41.27 ± 3.30   Final Test: 24.57 ± 3.51   Final old_old: 26.91 ± 4.02   Final old_new: 15.19 ± 1.66   Final new_new: 2.73 ± 2.49
Hits@30:
  Final val: 48.94 ± 2.52   Final Test: 30.78 ± 3.73   Final old_old: 33.64 ± 4.04   Final old_new: 19.45 ± 2.58   Final new_new: 1.82 ± 1.90
Hits@50:
  Final val: 59.97 ± 3.73   Final Test: 39.24 ± 3.39   Final old_old: 42.28 ± 3.68   Final old_new: 27.18 ± 2.53   Final new_new: 9.09 ± 2.78
AUC:
  Final val: 95.75 ± 0.47   Final Test: 95.02 ± 0.42   Final old_old: 95.78 ± 0.46   Final old_new: 92.04 ± 0.40   Final new_new: 86.44 ± 0.97
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 28.68 ± 3.26   Final Test: 17.62 ± 2.37   Final old_old: 19.35 ± 2.60   Final old_new: 10.68 ± 1.66   Final new_new: 1.36 ± 2.03
Hits@20:
  Final val: 42.17 ± 2.99   Final Test: 24.14 ± 4.34   Final old_old: 26.32 ± 4.78   Final old_new: 15.44 ± 2.70   Final new_new: 3.18 ± 2.03
Hits@30:
  Final val: 51.79 ± 4.19   Final Test: 34.27 ± 3.23   Final old_old: 37.25 ± 3.41   Final old_new: 22.34 ± 2.72   Final new_new: 6.36 ± 1.90
Hits@50:
  Final val: 61.98 ± 3.51   Final Test: 45.50 ± 4.78   Final old_old: 48.90 ± 5.02   Final old_new: 31.96 ± 3.89   Final new_new: 12.73 ± 3.05
AUC:
  Final val: 96.21 ± 0.42   Final Test: 95.58 ± 0.40   Final old_old: 96.22 ± 0.36   Final old_new: 93.07 ± 0.61   Final new_new: 88.73 ± 1.08
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.10 ± 6.69   Final Test: 33.79 ± 3.68   Final old_old: 36.65 ± 3.75   Final old_new: 22.52 ± 3.55   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 57.13 ± 5.26   Final Test: 42.33 ± 3.62   Final old_old: 45.55 ± 4.01   Final old_new: 29.67 ± 2.27   Final new_new: 7.73 ± 2.03
Hits@30:
  Final val: 64.98 ± 4.26   Final Test: 50.43 ± 3.64   Final old_old: 53.92 ± 3.95   Final old_new: 36.78 ± 2.67   Final new_new: 11.36 ± 3.59
Hits@50:
  Final val: 74.66 ± 2.82   Final Test: 61.52 ± 4.36   Final old_old: 65.21 ± 4.48   Final old_new: 47.13 ± 3.86   Final new_new: 20.00 ± 5.18
AUC:
  Final val: 97.56 ± 0.18   Final Test: 97.06 ± 0.16   Final old_old: 97.58 ± 0.17   Final old_new: 95.03 ± 0.17   Final new_new: 91.10 ± 0.60
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 23.74 ± 5.14   Final Test: 13.92 ± 3.97   Final old_old: 15.21 ± 4.25   Final old_new: 8.82 ± 3.03   Final new_new: 0.91 ± 1.24
Hits@20:
  Final val: 34.04 ± 5.40   Final Test: 18.42 ± 4.45   Final old_old: 20.08 ± 4.87   Final old_new: 11.76 ± 2.76   Final new_new: 2.73 ± 1.02
Hits@30:
  Final val: 41.81 ± 4.54   Final Test: 23.74 ± 7.22   Final old_old: 25.80 ± 7.78   Final old_new: 15.47 ± 5.12   Final new_new: 4.55 ± 1.61
Hits@50:
  Final val: 51.99 ± 5.59   Final Test: 31.09 ± 7.12   Final old_old: 33.58 ± 7.60   Final old_new: 21.21 ± 5.31   Final new_new: 6.82 ± 2.78
AUC:
  Final val: 94.74 ± 0.80   Final Test: 93.79 ± 0.67   Final old_old: 94.70 ± 0.69   Final old_new: 90.30 ± 0.59   Final new_new: 83.02 ± 2.50
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 27.68 ± 1.58   Final Test: 15.20 ± 3.90   Final old_old: 16.79 ± 4.41   Final old_new: 8.72 ± 1.95   Final new_new: 2.27 ± 2.27
Hits@20:
  Final val: 39.28 ± 2.83   Final Test: 21.87 ± 3.81   Final old_old: 23.96 ± 4.12   Final old_new: 13.50 ± 2.79   Final new_new: 2.73 ± 2.96
Hits@30:
  Final val: 47.20 ± 3.37   Final Test: 28.32 ± 5.61   Final old_old: 30.88 ± 6.13   Final old_new: 18.06 ± 3.76   Final new_new: 4.55 ± 2.78
Hits@50:
  Final val: 58.38 ± 2.73   Final Test: 40.83 ± 2.97   Final old_old: 44.00 ± 3.08   Final old_new: 28.26 ± 3.02   Final new_new: 8.64 ± 4.07
AUC:
  Final val: 95.70 ± 0.41   Final Test: 94.90 ± 0.40   Final old_old: 95.71 ± 0.35   Final old_new: 91.68 ± 0.71   Final new_new: 86.40 ± 1.59
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 22.82 ± 3.00   Final Test: 12.47 ± 2.17   Final old_old: 13.65 ± 2.32   Final old_new: 7.68 ± 1.83   Final new_new: 1.82 ± 1.02
Hits@20:
  Final val: 32.25 ± 3.63   Final Test: 18.63 ± 1.56   Final old_old: 20.28 ± 1.66   Final old_new: 12.04 ± 1.48   Final new_new: 3.18 ± 1.24
Hits@30:
  Final val: 40.17 ± 3.88   Final Test: 23.51 ± 1.45   Final old_old: 25.49 ± 1.63   Final old_new: 15.69 ± 1.07   Final new_new: 2.73 ± 1.02
Hits@50:
  Final val: 50.30 ± 3.53   Final Test: 30.46 ± 2.93   Final old_old: 32.81 ± 3.32   Final old_new: 21.18 ± 1.55   Final new_new: 5.45 ± 1.24
AUC:
  Final val: 94.50 ± 0.44   Final Test: 93.60 ± 0.31   Final old_old: 94.49 ± 0.37   Final old_new: 90.15 ± 0.30   Final new_new: 82.44 ± 2.59
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 26.89 ± 3.48   Final Test: 13.43 ± 2.60   Final old_old: 14.84 ± 2.81   Final old_new: 7.71 ± 1.80   Final new_new: 1.82 ± 1.90
Hits@20:
  Final val: 36.51 ± 4.33   Final Test: 18.46 ± 1.13   Final old_old: 20.18 ± 1.28   Final old_new: 11.49 ± 0.70   Final new_new: 4.09 ± 2.49
Hits@30:
  Final val: 43.88 ± 3.76   Final Test: 27.24 ± 2.64   Final old_old: 29.72 ± 2.76   Final old_new: 17.28 ± 2.24   Final new_new: 3.64 ± 3.80
Hits@50:
  Final val: 54.30 ± 2.64   Final Test: 36.75 ± 3.23   Final old_old: 39.75 ± 3.50   Final old_new: 24.69 ± 2.25   Final new_new: 10.00 ± 3.80
AUC:
  Final val: 95.15 ± 0.47   Final Test: 94.38 ± 0.42   Final old_old: 95.20 ± 0.46   Final old_new: 91.16 ± 0.39   Final new_new: 85.65 ± 1.40
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 34.58 ± 3.45   Final Test: 22.90 ± 2.79   Final old_old: 25.27 ± 3.04   Final old_new: 13.40 ± 1.98   Final new_new: 0.45 ± 1.02
Hits@20:
  Final val: 47.12 ± 3.65   Final Test: 29.46 ± 2.60   Final old_old: 32.29 ± 2.74   Final old_new: 18.16 ± 2.35   Final new_new: 2.27 ± 2.27
Hits@30:
  Final val: 56.10 ± 3.66   Final Test: 36.87 ± 3.07   Final old_old: 40.20 ± 3.10   Final old_new: 23.68 ± 3.13   Final new_new: 3.18 ± 2.59
Hits@50:
  Final val: 66.49 ± 4.17   Final Test: 49.23 ± 4.28   Final old_old: 53.11 ± 4.30   Final old_new: 33.93 ± 4.33   Final new_new: 8.18 ± 4.13
AUC:
  Final val: 96.73 ± 0.30   Final Test: 96.14 ± 0.25   Final old_old: 96.70 ± 0.22   Final old_new: 93.98 ± 0.37   Final new_new: 89.36 ± 0.96
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 29.36 ± 2.90   Final Test: 15.96 ± 3.52   Final old_old: 17.80 ± 4.09   Final old_new: 8.49 ± 1.51   Final new_new: 1.36 ± 1.24
Hits@20:
  Final val: 40.47 ± 2.80   Final Test: 24.26 ± 2.72   Final old_old: 26.56 ± 3.24   Final old_new: 15.01 ± 1.16   Final new_new: 2.73 ± 2.96
Hits@30:
  Final val: 48.59 ± 3.82   Final Test: 30.28 ± 2.92   Final old_old: 33.05 ± 3.32   Final old_new: 19.24 ± 1.48   Final new_new: 3.18 ± 3.45
Hits@50:
  Final val: 59.64 ± 4.41   Final Test: 38.84 ± 4.75   Final old_old: 41.98 ± 5.11   Final old_new: 26.35 ± 3.58   Final new_new: 7.73 ± 4.13
AUC:
  Final val: 95.64 ± 0.47   Final Test: 94.91 ± 0.41   Final old_old: 95.66 ± 0.41   Final old_new: 91.99 ± 0.50   Final new_new: 86.92 ± 0.93
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 27.96 ± 2.80   Final Test: 14.52 ± 2.69   Final old_old: 16.01 ± 3.04   Final old_new: 8.46 ± 1.33   Final new_new: 1.36 ± 2.03
Hits@20:
  Final val: 39.64 ± 2.55   Final Test: 21.44 ± 3.51   Final old_old: 23.52 ± 3.95   Final old_new: 13.02 ± 1.71   Final new_new: 3.18 ± 2.59
Hits@30:
  Final val: 46.88 ± 4.25   Final Test: 27.85 ± 3.98   Final old_old: 30.29 ± 4.36   Final old_new: 18.14 ± 2.42   Final new_new: 3.64 ± 1.24
Hits@50:
  Final val: 57.28 ± 4.00   Final Test: 40.64 ± 3.85   Final old_old: 43.89 ± 4.03   Final old_new: 27.76 ± 3.30   Final new_new: 8.18 ± 6.14
AUC:
  Final val: 95.60 ± 0.44   Final Test: 94.78 ± 0.44   Final old_old: 95.60 ± 0.38   Final old_new: 91.55 ± 0.71   Final new_new: 86.05 ± 1.24
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.61 ± 4.24   Final Test: 31.15 ± 3.47   Final old_old: 33.93 ± 3.63   Final old_new: 20.13 ± 2.93   Final new_new: 2.73 ± 1.90
Hits@20:
  Final val: 58.27 ± 2.33   Final Test: 41.24 ± 2.32   Final old_old: 44.46 ± 2.55   Final old_new: 28.74 ± 1.34   Final new_new: 3.64 ± 2.03
Hits@30:
  Final val: 66.12 ± 1.47   Final Test: 50.83 ± 2.24   Final old_old: 54.14 ± 2.27   Final old_new: 38.19 ± 2.38   Final new_new: 8.64 ± 3.37
Hits@50:
  Final val: 74.36 ± 1.38   Final Test: 60.57 ± 0.44   Final old_old: 64.06 ± 0.61   Final old_new: 47.08 ± 0.99   Final new_new: 19.55 ± 4.71
AUC:
  Final val: 97.53 ± 0.08   Final Test: 97.09 ± 0.06   Final old_old: 97.53 ± 0.07   Final old_new: 95.40 ± 0.10   Final new_new: 91.93 ± 0.63
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 28.94 ± 2.60   Final Test: 15.52 ± 2.66   Final old_old: 17.27 ± 3.11   Final old_new: 8.36 ± 0.88   Final new_new: 1.82 ± 2.49
Hits@20:
  Final val: 38.09 ± 4.46   Final Test: 21.43 ± 3.80   Final old_old: 23.52 ± 4.32   Final old_new: 12.95 ± 1.85   Final new_new: 3.64 ± 4.13
Hits@30:
  Final val: 45.11 ± 4.64   Final Test: 28.87 ± 3.98   Final old_old: 31.37 ± 4.45   Final old_new: 18.82 ± 2.47   Final new_new: 5.91 ± 3.45
Hits@50:
  Final val: 56.94 ± 6.28   Final Test: 36.68 ± 4.43   Final old_old: 39.65 ± 4.93   Final old_new: 24.74 ± 2.84   Final new_new: 9.55 ± 5.43
AUC:
  Final val: 95.31 ± 0.70   Final Test: 94.56 ± 0.76   Final old_old: 95.31 ± 0.73   Final old_new: 91.60 ± 0.90   Final new_new: 86.10 ± 1.64
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 22.43 ± 2.08   Final Test: 12.05 ± 1.35   Final old_old: 13.11 ± 1.47   Final old_new: 7.81 ± 1.16   Final new_new: 1.82 ± 1.90
Hits@20:
  Final val: 30.65 ± 4.06   Final Test: 17.92 ± 1.37   Final old_old: 19.49 ± 1.43   Final old_new: 11.69 ± 1.24   Final new_new: 2.27 ± 1.61
Hits@30:
  Final val: 37.97 ± 4.80   Final Test: 22.19 ± 0.47   Final old_old: 23.99 ± 0.40   Final old_new: 15.16 ± 1.06   Final new_new: 2.73 ± 1.90
Hits@50:
  Final val: 48.05 ± 3.32   Final Test: 29.65 ± 2.03   Final old_old: 31.92 ± 2.39   Final old_new: 20.76 ± 1.29   Final new_new: 4.55 ± 1.61
AUC:
  Final val: 94.09 ± 0.49   Final Test: 93.30 ± 0.28   Final old_old: 94.14 ± 0.35   Final old_new: 90.09 ± 0.54   Final new_new: 82.41 ± 2.21
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 27.38 ± 2.51   Final Test: 19.39 ± 4.56   Final old_old: 21.26 ± 4.98   Final old_new: 11.91 ± 3.15   Final new_new: 1.36 ± 1.24
Hits@20:
  Final val: 37.24 ± 4.50   Final Test: 26.57 ± 3.40   Final old_old: 28.97 ± 3.80   Final old_new: 17.05 ± 2.35   Final new_new: 2.27 ± 1.61
Hits@30:
  Final val: 46.33 ± 5.13   Final Test: 34.58 ± 3.84   Final old_old: 37.39 ± 4.29   Final old_new: 23.58 ± 2.01   Final new_new: 3.64 ± 3.05
Hits@50:
  Final val: 55.87 ± 5.45   Final Test: 42.30 ± 5.13   Final old_old: 45.46 ± 5.60   Final old_new: 30.05 ± 3.25   Final new_new: 5.45 ± 2.59
AUC:
  Final val: 93.80 ± 0.62   Final Test: 93.29 ± 0.50   Final old_old: 94.18 ± 0.49   Final old_new: 89.89 ± 0.62   Final new_new: 81.36 ± 1.49
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 29.07 ± 5.33   Final Test: 16.05 ± 3.81   Final old_old: 17.75 ± 4.17   Final old_new: 9.17 ± 2.49   Final new_new: 1.36 ± 2.03
Hits@20:
  Final val: 40.77 ± 7.37   Final Test: 23.53 ± 3.55   Final old_old: 25.76 ± 4.05   Final old_new: 14.56 ± 1.86   Final new_new: 3.18 ± 4.98
Hits@30:
  Final val: 48.34 ± 6.73   Final Test: 30.75 ± 4.83   Final old_old: 33.43 ± 5.21   Final old_new: 20.08 ± 3.56   Final new_new: 4.09 ± 3.73
Hits@50:
  Final val: 58.50 ± 5.59   Final Test: 39.98 ± 5.03   Final old_old: 43.13 ± 5.55   Final old_new: 27.43 ± 3.30   Final new_new: 9.09 ± 4.55
AUC:
  Final val: 95.57 ± 0.83   Final Test: 94.86 ± 0.81   Final old_old: 95.61 ± 0.84   Final old_new: 91.95 ± 0.76   Final new_new: 86.31 ± 1.29
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 22.08 ± 2.42   Final Test: 12.48 ± 2.74   Final old_old: 13.61 ± 3.04   Final old_new: 8.01 ± 1.58   Final new_new: 0.91 ± 1.24
Hits@20:
  Final val: 32.38 ± 3.99   Final Test: 17.87 ± 2.05   Final old_old: 19.46 ± 2.24   Final old_new: 11.49 ± 1.49   Final new_new: 2.73 ± 2.96
Hits@30:
  Final val: 39.60 ± 4.77   Final Test: 23.49 ± 3.33   Final old_old: 25.47 ± 3.67   Final old_new: 15.69 ± 2.32   Final new_new: 3.18 ± 1.24
Hits@50:
  Final val: 50.63 ± 4.00   Final Test: 31.59 ± 2.73   Final old_old: 34.01 ± 3.03   Final old_new: 22.12 ± 1.76   Final new_new: 4.55 ± 1.61
AUC:
  Final val: 94.45 ± 0.63   Final Test: 93.58 ± 0.44   Final old_old: 94.44 ± 0.50   Final old_new: 90.26 ± 0.36   Final new_new: 82.53 ± 2.11
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 26.01 ± 3.61   Final Test: 15.24 ± 2.97   Final old_old: 16.52 ± 3.17   Final old_new: 10.25 ± 2.30   Final new_new: 0.91 ± 1.24
Hits@20:
  Final val: 38.50 ± 3.14   Final Test: 22.99 ± 4.35   Final old_old: 24.84 ± 4.67   Final old_new: 15.82 ± 3.30   Final new_new: 1.82 ± 1.90
Hits@30:
  Final val: 47.49 ± 2.85   Final Test: 30.12 ± 4.54   Final old_old: 32.52 ± 4.71   Final old_new: 20.65 ± 3.99   Final new_new: 4.55 ± 2.27
Hits@50:
  Final val: 59.33 ± 3.93   Final Test: 42.26 ± 5.30   Final old_old: 45.43 ± 5.52   Final old_new: 29.80 ± 4.60   Final new_new: 8.18 ± 2.03
AUC:
  Final val: 95.96 ± 0.44   Final Test: 95.20 ± 0.48   Final old_old: 95.94 ± 0.40   Final old_new: 92.35 ± 0.90   Final new_new: 86.90 ± 1.33
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 30.28 ± 4.47   Final Test: 14.30 ± 4.21   Final old_old: 15.81 ± 4.78   Final old_new: 8.21 ± 2.02   Final new_new: 0.91 ± 2.03
Hits@20:
  Final val: 41.44 ± 3.14   Final Test: 21.86 ± 5.22   Final old_old: 24.00 ± 5.83   Final old_new: 13.25 ± 2.90   Final new_new: 1.82 ± 1.90
Hits@30:
  Final val: 49.63 ± 4.61   Final Test: 29.14 ± 5.83   Final old_old: 31.77 ± 6.32   Final old_new: 18.64 ± 4.03   Final new_new: 3.18 ± 2.59
Hits@50:
  Final val: 60.35 ± 5.51   Final Test: 40.96 ± 4.86   Final old_old: 44.15 ± 5.38   Final old_new: 28.46 ± 3.12   Final new_new: 5.91 ± 2.59
AUC:
  Final val: 95.80 ± 0.61   Final Test: 94.93 ± 0.53   Final old_old: 95.74 ± 0.54   Final old_new: 91.78 ± 0.55   Final new_new: 86.13 ± 1.27
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 33.51 ± 3.11   Final Test: 28.60 ± 2.32   Final old_old: 31.23 ± 2.62   Final old_new: 18.16 ± 1.04   Final new_new: 2.73 ± 1.90
Hits@20:
  Final val: 50.80 ± 5.47   Final Test: 38.99 ± 2.50   Final old_old: 42.09 ± 2.86   Final old_new: 26.70 ± 1.14   Final new_new: 7.73 ± 3.05
Hits@30:
  Final val: 60.06 ± 3.21   Final Test: 46.32 ± 3.81   Final old_old: 49.70 ± 3.95   Final old_new: 33.05 ± 3.66   Final new_new: 10.00 ± 1.24
Hits@50:
  Final val: 67.83 ± 2.66   Final Test: 54.15 ± 2.54   Final old_old: 57.56 ± 2.67   Final old_new: 40.93 ± 2.40   Final new_new: 15.00 ± 2.59
AUC:
  Final val: 95.20 ± 0.06   Final Test: 94.67 ± 0.06   Final old_old: 95.43 ± 0.08   Final old_new: 91.68 ± 0.40   Final new_new: 85.78 ± 0.93
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 22.62 ± 2.95   Final Test: 11.91 ± 1.73   Final old_old: 13.08 ± 1.81   Final old_new: 7.25 ± 1.52   Final new_new: 0.91 ± 1.24
Hits@20:
  Final val: 32.48 ± 4.23   Final Test: 17.78 ± 2.03   Final old_old: 19.33 ± 2.38   Final old_new: 11.54 ± 0.94   Final new_new: 3.64 ± 3.05
Hits@30:
  Final val: 40.81 ± 5.73   Final Test: 23.07 ± 3.58   Final old_old: 24.99 ± 3.95   Final old_new: 15.44 ± 2.30   Final new_new: 4.09 ± 2.96
Hits@50:
  Final val: 51.06 ± 6.03   Final Test: 29.85 ± 4.83   Final old_old: 32.17 ± 5.13   Final old_new: 20.73 ± 4.03   Final new_new: 5.00 ± 2.96
AUC:
  Final val: 94.40 ± 0.80   Final Test: 93.51 ± 0.56   Final old_old: 94.36 ± 0.67   Final old_new: 90.29 ± 0.21   Final new_new: 82.65 ± 2.09
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 29.09 ± 1.88   Final Test: 15.57 ± 4.98   Final old_old: 17.24 ± 5.56   Final old_new: 8.87 ± 2.70   Final new_new: 0.00 ± 0.00
Hits@20:
  Final val: 39.85 ± 2.86   Final Test: 20.63 ± 5.42   Final old_old: 22.59 ± 6.02   Final old_new: 12.72 ± 3.03   Final new_new: 3.64 ± 2.03
Hits@30:
  Final val: 47.75 ± 4.12   Final Test: 29.95 ± 4.51   Final old_old: 32.54 ± 4.77   Final old_new: 19.62 ± 3.63   Final new_new: 4.55 ± 2.78
Hits@50:
  Final val: 58.15 ± 4.09   Final Test: 38.91 ± 3.36   Final old_old: 42.00 ± 3.62   Final old_new: 26.50 ± 2.48   Final new_new: 10.45 ± 2.03
AUC:
  Final val: 95.63 ± 0.54   Final Test: 94.79 ± 0.52   Final old_old: 95.60 ± 0.50   Final old_new: 91.63 ± 0.74   Final new_new: 85.99 ± 0.91
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 35.32 ± 7.22   Final Test: 24.06 ± 2.90   Final old_old: 26.42 ± 3.03   Final old_new: 14.63 ± 2.67   Final new_new: 1.36 ± 2.03
Hits@20:
  Final val: 49.62 ± 6.53   Final Test: 28.76 ± 4.26   Final old_old: 31.44 ± 4.63   Final old_new: 18.01 ± 3.20   Final new_new: 4.09 ± 4.37
Hits@30:
  Final val: 58.71 ± 6.90   Final Test: 38.83 ± 3.31   Final old_old: 42.02 ± 3.47   Final old_new: 26.17 ± 2.84   Final new_new: 6.36 ± 1.90
Hits@50:
  Final val: 69.71 ± 5.28   Final Test: 52.53 ± 4.83   Final old_old: 56.28 ± 5.12   Final old_new: 37.61 ± 3.85   Final new_new: 15.91 ± 4.55
AUC:
  Final val: 96.93 ± 0.43   Final Test: 96.28 ± 0.39   Final old_old: 96.87 ± 0.41   Final old_new: 94.04 ± 0.37   Final new_new: 88.91 ± 1.80
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 28.96 ± 2.05   Final Test: 17.87 ± 2.54   Final old_old: 19.79 ± 3.00   Final old_new: 10.15 ± 1.14   Final new_new: 0.00 ± 0.00
Hits@20:
  Final val: 41.02 ± 4.07   Final Test: 26.33 ± 3.03   Final old_old: 28.76 ± 3.32   Final old_new: 16.62 ± 2.11   Final new_new: 2.73 ± 3.73
Hits@30:
  Final val: 48.22 ± 4.42   Final Test: 32.61 ± 2.64   Final old_old: 35.52 ± 2.77   Final old_new: 21.06 ± 2.32   Final new_new: 2.73 ± 2.96
Hits@50:
  Final val: 60.09 ± 4.87   Final Test: 39.39 ± 3.27   Final old_old: 42.57 ± 3.47   Final old_new: 26.83 ± 2.77   Final new_new: 6.82 ± 3.59
AUC:
  Final val: 95.75 ± 0.49   Final Test: 95.09 ± 0.49   Final old_old: 95.83 ± 0.46   Final old_new: 92.18 ± 0.63   Final new_new: 86.93 ± 1.17
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 29.58 ± 2.30   Final Test: 15.51 ± 3.88   Final old_old: 17.13 ± 4.32   Final old_new: 8.97 ± 2.24   Final new_new: 1.36 ± 2.03
Hits@20:
  Final val: 41.60 ± 4.03   Final Test: 22.11 ± 3.48   Final old_old: 24.27 ± 3.92   Final old_new: 13.35 ± 1.95   Final new_new: 3.64 ± 2.59
Hits@30:
  Final val: 50.13 ± 5.11   Final Test: 29.68 ± 4.75   Final old_old: 32.39 ± 5.16   Final old_new: 18.79 ± 3.32   Final new_new: 4.55 ± 1.61
Hits@50:
  Final val: 60.10 ± 4.22   Final Test: 42.23 ± 5.25   Final old_old: 45.54 ± 5.47   Final old_new: 29.07 ± 4.54   Final new_new: 10.00 ± 5.23
AUC:
  Final val: 95.76 ± 0.56   Final Test: 94.95 ± 0.48   Final old_old: 95.75 ± 0.48   Final old_new: 91.81 ± 0.56   Final new_new: 86.35 ± 1.51
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 40.07 ± 2.72   Final Test: 29.02 ± 2.44   Final old_old: 31.96 ± 2.48   Final old_new: 17.08 ± 2.39   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 53.44 ± 1.79   Final Test: 40.18 ± 2.56   Final old_old: 43.37 ± 2.51   Final old_new: 27.38 ± 2.90   Final new_new: 10.45 ± 3.45
Hits@30:
  Final val: 62.88 ± 0.33   Final Test: 49.05 ± 2.46   Final old_old: 52.22 ± 2.40   Final old_new: 36.62 ± 2.86   Final new_new: 14.55 ± 5.47
Hits@50:
  Final val: 69.78 ± 0.99   Final Test: 56.80 ± 1.89   Final old_old: 59.87 ± 1.99   Final old_new: 44.89 ± 1.94   Final new_new: 20.91 ± 4.93
AUC:
  Final val: 94.99 ± 0.06   Final Test: 94.60 ± 0.10   Final old_old: 95.34 ± 0.07   Final old_new: 91.68 ± 0.34   Final new_new: 86.75 ± 0.91
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 30.49 ± 2.86   Final Test: 16.90 ± 2.96   Final old_old: 18.69 ± 3.37   Final old_new: 9.65 ± 1.53   Final new_new: 1.82 ± 1.90
Hits@20:
  Final val: 40.06 ± 4.93   Final Test: 22.57 ± 3.34   Final old_old: 24.69 ± 3.74   Final old_new: 13.98 ± 2.10   Final new_new: 4.09 ± 3.37
Hits@30:
  Final val: 48.57 ± 4.87   Final Test: 28.99 ± 4.41   Final old_old: 31.61 ± 4.92   Final old_new: 18.49 ± 2.55   Final new_new: 4.09 ± 4.07
Hits@50:
  Final val: 58.43 ± 6.00   Final Test: 38.54 ± 5.29   Final old_old: 41.56 ± 5.63   Final old_new: 26.32 ± 4.22   Final new_new: 11.82 ± 5.66
AUC:
  Final val: 95.53 ± 0.71   Final Test: 94.78 ± 0.78   Final old_old: 95.53 ± 0.73   Final old_new: 91.87 ± 1.05   Final new_new: 86.19 ± 1.06
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 23.49 ± 1.16   Final Test: 12.97 ± 2.39   Final old_old: 14.13 ± 2.54   Final old_new: 8.36 ± 2.03   Final new_new: 0.91 ± 1.24
Hits@20:
  Final val: 32.67 ± 2.86   Final Test: 17.62 ± 1.56   Final old_old: 19.11 ± 1.75   Final old_new: 11.74 ± 0.88   Final new_new: 2.27 ± 2.27
Hits@30:
  Final val: 40.14 ± 3.01   Final Test: 23.54 ± 3.71   Final old_old: 25.59 ± 4.07   Final old_new: 15.42 ± 2.42   Final new_new: 2.73 ± 1.02
Hits@50:
  Final val: 50.80 ± 4.05   Final Test: 30.82 ± 3.28   Final old_old: 33.33 ± 3.71   Final old_new: 20.91 ± 1.75   Final new_new: 5.00 ± 2.49
AUC:
  Final val: 94.49 ± 0.55   Final Test: 93.60 ± 0.51   Final old_old: 94.45 ± 0.53   Final old_new: 90.30 ± 0.39   Final new_new: 83.01 ± 2.13
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 32.44 ± 3.52   Final Test: 21.34 ± 1.37   Final old_old: 23.48 ± 1.46   Final old_new: 12.72 ± 1.25   Final new_new: 2.27 ± 3.94
Hits@20:
  Final val: 47.90 ± 4.37   Final Test: 27.75 ± 2.48   Final old_old: 30.24 ± 2.63   Final old_new: 17.81 ± 2.16   Final new_new: 4.09 ± 4.37
Hits@30:
  Final val: 56.35 ± 3.78   Final Test: 34.15 ± 1.89   Final old_old: 36.99 ± 1.91   Final old_new: 22.87 ± 2.02   Final new_new: 5.45 ± 4.98
Hits@50:
  Final val: 67.18 ± 3.81   Final Test: 49.42 ± 4.51   Final old_old: 53.05 ± 4.77   Final old_new: 35.06 ± 3.83   Final new_new: 12.27 ± 4.43
AUC:
  Final val: 96.74 ± 0.33   Final Test: 96.19 ± 0.34   Final old_old: 96.75 ± 0.32   Final old_new: 94.02 ± 0.47   Final new_new: 89.54 ± 0.88
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 30.56 ± 1.48   Final Test: 16.82 ± 4.37   Final old_old: 18.65 ± 4.96   Final old_new: 9.42 ± 2.06   Final new_new: 1.36 ± 1.24
Hits@20:
  Final val: 41.13 ± 2.80   Final Test: 25.57 ± 3.35   Final old_old: 27.97 ± 3.86   Final old_new: 16.02 ± 1.82   Final new_new: 2.73 ± 2.49
Hits@30:
  Final val: 49.46 ± 2.24   Final Test: 31.92 ± 2.72   Final old_old: 34.77 ± 3.05   Final old_new: 20.58 ± 1.68   Final new_new: 3.64 ± 2.03
Hits@50:
  Final val: 60.23 ± 2.99   Final Test: 41.40 ± 2.69   Final old_old: 44.60 ± 2.93   Final old_new: 28.74 ± 1.87   Final new_new: 8.64 ± 4.93
AUC:
  Final val: 95.76 ± 0.40   Final Test: 95.05 ± 0.38   Final old_old: 95.79 ± 0.39   Final old_new: 92.12 ± 0.46   Final new_new: 87.25 ± 0.82
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 22.36 ± 2.36   Final Test: 12.90 ± 1.66   Final old_old: 14.19 ± 1.78   Final old_new: 7.68 ± 1.56   Final new_new: 1.36 ± 1.24
Hits@20:
  Final val: 34.49 ± 3.44   Final Test: 19.27 ± 3.21   Final old_old: 21.04 ± 3.61   Final old_new: 12.24 ± 1.76   Final new_new: 1.82 ± 1.02
Hits@30:
  Final val: 42.39 ± 4.38   Final Test: 26.23 ± 3.22   Final old_old: 28.53 ± 3.62   Final old_new: 17.08 ± 1.66   Final new_new: 3.64 ± 2.59
Hits@50:
  Final val: 52.60 ± 4.49   Final Test: 33.94 ± 2.99   Final old_old: 36.60 ± 3.46   Final old_new: 23.50 ± 1.48   Final new_new: 4.55 ± 2.78
AUC:
  Final val: 94.80 ± 0.86   Final Test: 93.89 ± 0.73   Final old_old: 94.75 ± 0.77   Final old_new: 90.57 ± 0.64   Final new_new: 83.49 ± 2.03
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 27.95 ± 5.65   Final Test: 15.96 ± 4.90   Final old_old: 17.28 ± 5.24   Final old_new: 10.58 ± 3.61   Final new_new: 5.00 ± 3.73
Hits@20:
  Final val: 40.41 ± 5.34   Final Test: 25.71 ± 3.96   Final old_old: 27.83 ± 4.25   Final old_new: 17.13 ± 2.87   Final new_new: 7.27 ± 2.96
Hits@30:
  Final val: 49.44 ± 5.96   Final Test: 33.96 ± 4.70   Final old_old: 36.68 ± 4.98   Final old_new: 22.90 ± 3.79   Final new_new: 11.36 ± 2.78
Hits@50:
  Final val: 61.70 ± 4.55   Final Test: 45.04 ± 4.15   Final old_old: 48.52 ± 4.46   Final old_new: 30.83 ± 3.35   Final new_new: 17.27 ± 3.05
AUC:
  Final val: 96.29 ± 0.42   Final Test: 95.62 ± 0.38   Final old_old: 96.23 ± 0.37   Final old_new: 93.29 ± 0.50   Final new_new: 88.44 ± 0.98
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 29.99 ± 2.70   Final Test: 15.71 ± 2.64   Final old_old: 17.29 ± 2.85   Final old_new: 9.37 ± 2.13   Final new_new: 0.91 ± 2.03
Hits@20:
  Final val: 45.72 ± 2.38   Final Test: 26.81 ± 4.24   Final old_old: 29.31 ± 4.72   Final old_new: 16.83 ± 2.66   Final new_new: 3.18 ± 2.59
Hits@30:
  Final val: 52.78 ± 2.97   Final Test: 34.01 ± 4.89   Final old_old: 36.93 ± 5.37   Final old_new: 22.39 ± 3.37   Final new_new: 5.45 ± 2.03
Hits@50:
  Final val: 64.08 ± 3.50   Final Test: 43.36 ± 4.97   Final old_old: 46.67 ± 5.34   Final old_new: 30.20 ± 3.83   Final new_new: 10.00 ± 1.24
AUC:
  Final val: 96.17 ± 0.44   Final Test: 95.38 ± 0.44   Final old_old: 96.16 ± 0.43   Final old_new: 92.37 ± 0.46   Final new_new: 86.69 ± 1.65
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 28.08 ± 0.90   Final Test: 14.38 ± 1.51   Final old_old: 15.75 ± 1.66   Final old_new: 8.82 ± 1.61   Final new_new: 2.73 ± 2.96
Hits@20:
  Final val: 38.78 ± 3.07   Final Test: 20.82 ± 5.00   Final old_old: 22.72 ± 5.48   Final old_new: 13.15 ± 3.23   Final new_new: 3.64 ± 2.03
Hits@30:
  Final val: 46.40 ± 4.62   Final Test: 27.05 ± 4.40   Final old_old: 29.50 ± 4.81   Final old_new: 17.10 ± 2.75   Final new_new: 6.36 ± 1.90
Hits@50:
  Final val: 57.06 ± 3.75   Final Test: 38.75 ± 3.08   Final old_old: 41.86 ± 3.41   Final old_new: 26.25 ± 1.90   Final new_new: 10.45 ± 2.03
AUC:
  Final val: 95.48 ± 0.45   Final Test: 94.66 ± 0.40   Final old_old: 95.47 ± 0.36   Final old_new: 91.47 ± 0.59   Final new_new: 85.72 ± 1.83
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 26.96 ± 5.21   Final Test: 16.60 ± 3.26   Final old_old: 18.10 ± 3.49   Final old_new: 10.68 ± 2.42   Final new_new: 0.91 ± 1.24
Hits@20:
  Final val: 38.24 ± 8.07   Final Test: 23.20 ± 6.10   Final old_old: 25.14 ± 6.68   Final old_new: 15.44 ± 3.85   Final new_new: 4.55 ± 3.21
Hits@30:
  Final val: 47.03 ± 8.62   Final Test: 30.15 ± 6.69   Final old_old: 32.64 ± 7.11   Final old_new: 20.18 ± 5.19   Final new_new: 7.27 ± 3.37
Hits@50:
  Final val: 59.52 ± 9.06   Final Test: 41.54 ± 7.74   Final old_old: 44.56 ± 8.32   Final old_new: 29.52 ± 5.61   Final new_new: 11.82 ± 4.07
AUC:
  Final val: 95.72 ± 1.22   Final Test: 95.02 ± 1.10   Final old_old: 95.71 ± 1.11   Final old_new: 92.37 ± 1.18   Final new_new: 86.00 ± 1.72
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 23.74 ± 1.26   Final Test: 12.10 ± 1.92   Final old_old: 13.21 ± 1.99   Final old_new: 7.68 ± 1.75   Final new_new: 0.91 ± 1.24
Hits@20:
  Final val: 35.51 ± 2.48   Final Test: 19.41 ± 3.36   Final old_old: 21.10 ± 3.63   Final old_new: 12.72 ± 2.37   Final new_new: 2.27 ± 0.00
Hits@30:
  Final val: 43.45 ± 2.70   Final Test: 25.42 ± 3.32   Final old_old: 27.60 ± 3.66   Final old_new: 16.83 ± 2.04   Final new_new: 2.27 ± 0.00
Hits@50:
  Final val: 53.74 ± 2.55   Final Test: 33.85 ± 4.74   Final old_old: 36.58 ± 5.11   Final old_new: 23.00 ± 3.28   Final new_new: 6.36 ± 2.96
AUC:
  Final val: 95.11 ± 0.37   Final Test: 94.21 ± 0.39   Final old_old: 95.09 ± 0.41   Final old_new: 90.77 ± 0.40   Final new_new: 84.14 ± 1.57
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 37.08 ± 5.82   Final Test: 27.49 ± 3.73   Final old_old: 29.96 ± 4.03   Final old_new: 17.68 ± 2.64   Final new_new: 3.18 ± 2.03
Hits@20:
  Final val: 50.39 ± 6.75   Final Test: 35.13 ± 6.87   Final old_old: 38.10 ± 7.43   Final old_new: 23.15 ± 4.73   Final new_new: 8.18 ± 4.43
Hits@30:
  Final val: 61.34 ± 6.81   Final Test: 45.97 ± 5.98   Final old_old: 49.56 ± 6.33   Final old_new: 31.69 ± 4.70   Final new_new: 10.45 ± 3.80
Hits@50:
  Final val: 71.66 ± 4.96   Final Test: 54.74 ± 5.17   Final old_old: 58.51 ± 5.32   Final old_new: 39.65 ± 4.84   Final new_new: 19.09 ± 4.43
AUC:
  Final val: 97.24 ± 0.35   Final Test: 96.66 ± 0.34   Final old_old: 97.18 ± 0.35   Final old_new: 94.66 ± 0.25   Final new_new: 90.26 ± 1.05
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 27.86 ± 4.51   Final Test: 15.02 ± 3.34   Final old_old: 16.53 ± 3.80   Final old_new: 8.94 ± 1.58   Final new_new: 1.82 ± 1.90
Hits@20:
  Final val: 37.73 ± 5.77   Final Test: 21.64 ± 3.74   Final old_old: 23.61 ± 4.27   Final old_new: 13.70 ± 1.95   Final new_new: 4.09 ± 4.37
Hits@30:
  Final val: 46.58 ± 5.59   Final Test: 27.89 ± 2.31   Final old_old: 30.45 ± 2.67   Final old_new: 17.71 ± 1.38   Final new_new: 3.18 ± 3.80
Hits@50:
  Final val: 57.08 ± 4.94   Final Test: 36.51 ± 5.22   Final old_old: 39.35 ± 5.57   Final old_new: 25.21 ± 4.19   Final new_new: 8.18 ± 5.47
AUC:
  Final val: 95.29 ± 0.71   Final Test: 94.59 ± 0.72   Final old_old: 95.33 ± 0.72   Final old_new: 91.62 ± 0.77   Final new_new: 87.11 ± 1.61
All runs:
Hits@10:
  Final val: 29.07 ± 5.16   Final Test: 17.80 ± 5.56   Final old_old: 19.65 ± 6.19   Final old_new: 10.33 ± 3.11   Final new_new: 0.91 ± 2.03
Hits@20:
  Final val: 39.24 ± 6.21   Final Test: 23.73 ± 5.84   Final old_old: 25.84 ± 6.64   Final old_new: 15.21 ± 2.88   Final new_new: 4.55 ± 5.33
Hits@30:
  Final val: 46.72 ± 5.62   Final Test: 29.39 ± 5.43   Final old_old: 31.98 ± 5.99   Final old_new: 19.02 ± 3.41   Final new_new: 4.55 ± 3.21
Hits@50:
  Final val: 57.84 ± 5.22   Final Test: 39.02 ± 5.69   Final old_old: 41.98 ± 6.21   Final old_new: 27.23 ± 3.86   Final new_new: 10.45 ± 3.80
AUC:
  Final val: 95.36 ± 0.78   Final Test: 94.72 ± 0.83   Final old_old: 95.45 ± 0.83   Final old_new: 91.86 ± 0.87   Final new_new: 86.85 ± 1.45
All runs:
Hits@10:
  Final val: 34.09 ± 5.87   Final Test: 22.63 ± 4.82   Final old_old: 24.79 ± 5.25   Final old_new: 13.98 ± 3.30   Final new_new: 2.27 ± 2.78
Hits@20:
  Final val: 48.68 ± 7.13   Final Test: 31.18 ± 7.52   Final old_old: 33.96 ± 7.96   Final old_new: 20.03 ± 5.77   Final new_new: 5.45 ± 4.43
Hits@30:
  Final val: 58.43 ± 6.64   Final Test: 38.78 ± 5.83   Final old_old: 41.88 ± 6.25   Final old_new: 26.52 ± 4.25   Final new_new: 7.27 ± 4.07
Hits@50:
  Final val: 68.86 ± 4.82   Final Test: 52.20 ± 5.91   Final old_old: 56.01 ± 6.31   Final old_new: 36.98 ± 4.61   Final new_new: 15.45 ± 4.37
AUC:
  Final val: 96.88 ± 0.43   Final Test: 96.26 ± 0.41   Final old_old: 96.83 ± 0.41   Final old_new: 94.07 ± 0.39   Final new_new: 89.40 ± 1.18
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 30.84 ± 1.87   Final Test: 19.03 ± 3.29   Final old_old: 20.95 ± 3.64   Final old_new: 11.39 ± 2.04   Final new_new: 0.45 ± 1.02
Hits@20:
  Final val: 44.16 ± 2.50   Final Test: 26.01 ± 3.00   Final old_old: 28.46 ± 3.30   Final old_new: 16.30 ± 1.91   Final new_new: 1.82 ± 1.90
Hits@30:
  Final val: 51.20 ± 2.76   Final Test: 32.52 ± 2.89   Final old_old: 35.30 ± 3.16   Final old_new: 21.51 ± 2.08   Final new_new: 4.09 ± 2.96
Hits@50:
  Final val: 62.65 ± 2.97   Final Test: 42.84 ± 3.13   Final old_old: 46.21 ± 3.44   Final old_new: 29.47 ± 2.32   Final new_new: 8.64 ± 4.07
AUC:
  Final val: 96.10 ± 0.34   Final Test: 95.28 ± 0.26   Final old_old: 96.07 ± 0.27   Final old_new: 92.18 ± 0.24   Final new_new: 86.63 ± 1.32
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.12 ± 5.93   Final Test: 31.18 ± 2.31   Final old_old: 34.01 ± 2.61   Final old_new: 19.95 ± 1.34   Final new_new: 3.18 ± 2.59
Hits@20:
  Final val: 59.02 ± 4.34   Final Test: 40.37 ± 3.82   Final old_old: 43.57 ± 4.02   Final old_new: 27.81 ± 3.16   Final new_new: 5.45 ± 2.03
Hits@30:
  Final val: 65.82 ± 4.96   Final Test: 48.63 ± 5.62   Final old_old: 52.15 ± 5.90   Final old_new: 34.71 ± 4.75   Final new_new: 12.27 ± 3.45
Hits@50:
  Final val: 73.79 ± 2.72   Final Test: 59.07 ± 6.12   Final old_old: 62.91 ± 6.26   Final old_new: 43.98 ± 5.61   Final new_new: 18.64 ± 4.93
AUC:
  Final val: 97.57 ± 0.21   Final Test: 97.08 ± 0.26   Final old_old: 97.58 ± 0.25   Final old_new: 95.17 ± 0.30   Final new_new: 91.03 ± 0.31
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 30.23 ± 6.36   Final Test: 20.39 ± 4.05   Final old_old: 22.49 ± 4.48   Final old_new: 11.96 ± 2.46   Final new_new: 1.36 ± 2.03
Hits@20:
  Final val: 41.81 ± 8.07   Final Test: 26.82 ± 4.23   Final old_old: 29.30 ± 4.80   Final old_new: 16.85 ± 2.29   Final new_new: 3.64 ± 4.71
Hits@30:
  Final val: 49.38 ± 6.63   Final Test: 30.99 ± 2.38   Final old_old: 33.79 ± 2.63   Final old_new: 19.82 ± 1.65   Final new_new: 3.18 ± 3.80
Hits@50:
  Final val: 59.37 ± 4.91   Final Test: 39.79 ± 4.36   Final old_old: 42.88 ± 4.74   Final old_new: 27.68 ± 3.01   Final new_new: 6.36 ± 3.37
AUC:
  Final val: 95.71 ± 0.81   Final Test: 95.01 ± 0.81   Final old_old: 95.76 ± 0.81   Final old_new: 92.07 ± 0.87   Final new_new: 86.94 ± 1.55
All runs:
Hits@10:
  Final val: 33.79 ± 3.39   Final Test: 20.68 ± 4.82   Final old_old: 22.69 ± 5.33   Final old_new: 12.57 ± 3.02   Final new_new: 2.27 ± 3.21
Hits@20:
  Final val: 47.75 ± 5.16   Final Test: 28.35 ± 4.36   Final old_old: 30.93 ± 4.64   Final old_new: 18.09 ± 3.51   Final new_new: 2.73 ± 2.49
Hits@30:
  Final val: 57.38 ± 4.27   Final Test: 37.91 ± 6.06   Final old_old: 40.92 ± 6.33   Final old_new: 26.15 ± 5.22   Final new_new: 4.55 ± 2.78
Hits@50:
  Final val: 68.23 ± 4.35   Final Test: 50.02 ± 6.04   Final old_old: 53.62 ± 6.34   Final old_new: 35.89 ± 5.05   Final new_new: 11.36 ± 1.61
AUC:
  Final val: 96.79 ± 0.45   Final Test: 96.23 ± 0.51   Final old_old: 96.77 ± 0.49   Final old_new: 94.14 ± 0.64   Final new_new: 89.47 ± 0.57
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 28.26 ± 5.59   Final Test: 20.60 ± 2.03   Final old_old: 22.38 ± 2.40   Final old_new: 13.48 ± 0.67   Final new_new: 4.55 ± 2.27
Hits@20:
  Final val: 42.03 ± 5.92   Final Test: 27.41 ± 4.01   Final old_old: 29.82 ± 4.51   Final old_new: 17.63 ± 2.02   Final new_new: 6.82 ± 1.61
Hits@30:
  Final val: 51.09 ± 5.92   Final Test: 34.63 ± 4.03   Final old_old: 37.58 ± 4.33   Final old_new: 22.67 ± 2.93   Final new_new: 9.55 ± 1.90
Hits@50:
  Final val: 62.38 ± 4.94   Final Test: 46.25 ± 2.90   Final old_old: 49.91 ± 3.08   Final old_new: 31.41 ± 2.65   Final new_new: 14.55 ± 3.05
AUC:
  Final val: 96.49 ± 0.40   Final Test: 95.86 ± 0.37   Final old_old: 96.43 ± 0.36   Final old_new: 93.65 ± 0.42   Final new_new: 88.66 ± 1.22
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 29.03 ± 4.64   Final Test: 16.82 ± 2.95   Final old_old: 18.28 ± 3.09   Final old_new: 10.96 ± 2.36   Final new_new: 3.18 ± 2.59
Hits@20:
  Final val: 40.70 ± 6.26   Final Test: 26.86 ± 4.76   Final old_old: 29.12 ± 5.20   Final old_new: 17.76 ± 3.21   Final new_new: 6.82 ± 2.78
Hits@30:
  Final val: 51.53 ± 4.67   Final Test: 34.24 ± 5.61   Final old_old: 36.87 ± 5.90   Final old_new: 23.70 ± 4.57   Final new_new: 9.09 ± 5.33
Hits@50:
  Final val: 62.28 ± 4.44   Final Test: 44.03 ± 6.03   Final old_old: 47.16 ± 6.38   Final old_new: 31.49 ± 4.86   Final new_new: 14.55 ± 6.93
AUC:
  Final val: 96.28 ± 0.54   Final Test: 95.65 ± 0.54   Final old_old: 96.27 ± 0.52   Final old_new: 93.27 ± 0.64   Final new_new: 88.11 ± 0.97
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 37.77 ± 5.78   Final Test: 26.17 ± 4.12   Final old_old: 28.51 ± 4.52   Final old_new: 16.80 ± 2.90   Final new_new: 4.55 ± 2.78
Hits@20:
  Final val: 52.58 ± 7.66   Final Test: 37.66 ± 5.78   Final old_old: 40.68 ± 6.34   Final old_new: 25.57 ± 3.65   Final new_new: 9.09 ± 5.57
Hits@30:
  Final val: 61.83 ± 6.96   Final Test: 44.81 ± 5.41   Final old_old: 48.24 ± 5.84   Final old_new: 31.06 ± 4.03   Final new_new: 12.27 ± 3.45
Hits@50:
  Final val: 71.45 ± 6.71   Final Test: 55.93 ± 7.24   Final old_old: 59.57 ± 7.61   Final old_new: 41.51 ± 5.88   Final new_new: 19.55 ± 5.23
AUC:
  Final val: 97.32 ± 0.45   Final Test: 96.72 ± 0.37   Final old_old: 97.23 ± 0.40   Final old_new: 94.73 ± 0.26   Final new_new: 90.46 ± 0.57
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 34.70 ± 6.48   Final Test: 24.88 ± 1.30   Final old_old: 27.17 ± 1.38   Final old_new: 15.82 ± 1.51   Final new_new: 1.36 ± 2.03
Hits@20:
  Final val: 50.84 ± 1.55   Final Test: 33.49 ± 3.10   Final old_old: 36.24 ± 3.44   Final old_new: 22.57 ± 1.96   Final new_new: 6.36 ± 4.37
Hits@30:
  Final val: 60.29 ± 2.67   Final Test: 39.59 ± 2.14   Final old_old: 42.69 ± 2.27   Final old_new: 27.28 ± 1.88   Final new_new: 9.09 ± 4.82
Hits@50:
  Final val: 70.61 ± 2.35   Final Test: 54.33 ± 2.80   Final old_old: 58.04 ± 2.98   Final old_new: 39.75 ± 2.50   Final new_new: 15.00 ± 3.45
AUC:
  Final val: 97.12 ± 0.20   Final Test: 96.49 ± 0.20   Final old_old: 97.05 ± 0.20   Final old_new: 94.35 ± 0.21   Final new_new: 89.09 ± 0.73
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=65536, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0, LLP_R=0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
All runs:
Hits@10:
  Final val: 22.73 ± 3.06   Final Test: 12.11 ± 0.64   Final old_old: 13.18 ± 0.79   Final old_new: 7.76 ± 0.42   Final new_new: 2.73 ± 2.49
Hits@20:
  Final val: 32.63 ± 2.93   Final Test: 17.54 ± 1.37   Final old_old: 19.14 ± 1.57   Final old_new: 11.13 ± 0.63   Final new_new: 1.82 ± 1.02
Hits@30:
  Final val: 39.51 ± 3.41   Final Test: 24.30 ± 4.45   Final old_old: 26.31 ± 4.75   Final old_new: 16.35 ± 3.51   Final new_new: 3.64 ± 2.03
Hits@50:
  Final val: 50.93 ± 4.08   Final Test: 31.29 ± 5.09   Final old_old: 33.71 ± 5.38   Final old_new: 21.81 ± 4.12   Final new_new: 4.55 ± 1.61
AUC:
  Final val: 94.49 ± 0.52   Final Test: 93.61 ± 0.45   Final old_old: 94.45 ± 0.48   Final old_new: 90.40 ± 0.59   Final new_new: 82.76 ± 2.04
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0, LLP_R=0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
All runs:
Hits@10:
  Final val: 34.58 ± 1.51   Final Test: 23.86 ± 2.87   Final old_old: 26.12 ± 3.12   Final old_new: 14.94 ± 2.11   Final new_new: 0.91 ± 1.24
Hits@20:
  Final val: 49.88 ± 2.39   Final Test: 30.17 ± 2.37   Final old_old: 32.81 ± 2.32   Final old_new: 19.72 ± 2.57   Final new_new: 3.18 ± 3.80
Hits@30:
  Final val: 59.88 ± 2.21   Final Test: 38.83 ± 1.93   Final old_old: 41.79 ± 2.06   Final old_new: 27.23 ± 1.53   Final new_new: 6.82 ± 2.27
Hits@50:
  Final val: 67.93 ± 2.84   Final Test: 51.93 ± 1.35   Final old_old: 55.47 ± 1.43   Final old_new: 38.06 ± 1.38   Final new_new: 13.18 ± 1.02
AUC:
  Final val: 96.95 ± 0.12   Final Test: 96.40 ± 0.12   Final old_old: 96.89 ± 0.13   Final old_new: 94.51 ± 0.09   Final new_new: 90.62 ± 0.48
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 38.14 ± 3.14   Final Test: 28.74 ± 3.09   Final old_old: 31.41 ± 3.43   Final old_new: 18.09 ± 2.00   Final new_new: 2.73 ± 1.90
Hits@20:
  Final val: 53.43 ± 5.99   Final Test: 38.19 ± 4.48   Final old_old: 41.31 ± 4.86   Final old_new: 25.82 ± 3.11   Final new_new: 7.27 ± 1.90
Hits@30:
  Final val: 62.78 ± 4.08   Final Test: 47.11 ± 5.03   Final old_old: 50.49 ± 5.45   Final old_new: 33.85 ± 3.55   Final new_new: 10.91 ± 2.96
Hits@50:
  Final val: 71.68 ± 3.11   Final Test: 58.76 ± 2.54   Final old_old: 62.55 ± 2.82   Final old_new: 43.93 ± 1.73   Final new_new: 16.36 ± 4.37
AUC:
  Final val: 97.38 ± 0.22   Final Test: 96.91 ± 0.23   Final old_old: 97.38 ± 0.25   Final old_new: 95.09 ± 0.16   Final new_new: 90.90 ± 0.82
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 26.17 ± 2.56   Final Test: 15.41 ± 3.75   Final old_old: 16.96 ± 4.27   Final old_new: 9.22 ± 1.74   Final new_new: 0.91 ± 2.03
Hits@20:
  Final val: 38.01 ± 2.61   Final Test: 21.25 ± 3.48   Final old_old: 23.19 ± 3.88   Final old_new: 13.55 ± 1.92   Final new_new: 2.27 ± 2.78
Hits@30:
  Final val: 46.25 ± 3.46   Final Test: 28.38 ± 3.36   Final old_old: 30.87 ± 3.73   Final old_new: 18.51 ± 2.09   Final new_new: 3.64 ± 2.59
Hits@50:
  Final val: 57.84 ± 4.27   Final Test: 40.35 ± 3.33   Final old_old: 43.46 ± 3.47   Final old_new: 28.11 ± 3.08   Final new_new: 7.73 ± 3.80
AUC:
  Final val: 95.55 ± 0.38   Final Test: 94.78 ± 0.36   Final old_old: 95.55 ± 0.31   Final old_new: 91.78 ± 0.66   Final new_new: 86.43 ± 0.90
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0, LLP_R=0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
All runs:
Hits@10:
  Final val: 28.41 ± 2.75   Final Test: 17.80 ± 3.38   Final old_old: 19.61 ± 3.65   Final old_new: 10.43 ± 2.63   Final new_new: 2.73 ± 1.02
Hits@20:
  Final val: 38.60 ± 2.51   Final Test: 25.94 ± 1.55   Final old_old: 28.19 ± 1.59   Final old_new: 16.95 ± 1.57   Final new_new: 4.55 ± 1.61
Hits@30:
  Final val: 48.00 ± 3.71   Final Test: 34.76 ± 2.63   Final old_old: 37.70 ± 2.73   Final old_new: 23.02 ± 2.35   Final new_new: 7.27 ± 1.90
Hits@50:
  Final val: 59.39 ± 3.33   Final Test: 41.41 ± 2.95   Final old_old: 44.69 ± 3.04   Final old_new: 28.31 ± 2.66   Final new_new: 10.45 ± 4.13
AUC:
  Final val: 95.99 ± 0.35   Final Test: 95.41 ± 0.27   Final old_old: 96.02 ± 0.26   Final old_new: 93.06 ± 0.49   Final new_new: 88.65 ± 1.23
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 32.90 ± 7.90   Final Test: 24.43 ± 3.71   Final old_old: 26.45 ± 4.06   Final old_new: 16.20 ± 2.25   Final new_new: 7.73 ± 3.80
Hits@20:
  Final val: 46.11 ± 7.74   Final Test: 32.35 ± 6.30   Final old_old: 34.85 ± 6.92   Final old_new: 22.39 ± 3.95   Final new_new: 8.18 ± 4.13
Hits@30:
  Final val: 54.57 ± 8.16   Final Test: 39.67 ± 6.12   Final old_old: 42.69 ± 6.58   Final old_new: 27.48 ± 4.45   Final new_new: 13.18 ± 3.37
Hits@50:
  Final val: 67.12 ± 6.80   Final Test: 50.95 ± 5.31   Final old_old: 54.54 ± 5.71   Final old_new: 36.57 ± 3.87   Final new_new: 17.73 ± 4.07
AUC:
  Final val: 96.81 ± 0.63   Final Test: 96.23 ± 0.55   Final old_old: 96.78 ± 0.59   Final old_new: 94.14 ± 0.43   Final new_new: 89.14 ± 1.36
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 28.87 ± 3.17   Final Test: 17.71 ± 3.09   Final old_old: 19.53 ± 3.37   Final old_new: 10.28 ± 2.12   Final new_new: 2.73 ± 1.90
Hits@20:
  Final val: 39.74 ± 4.55   Final Test: 23.11 ± 4.49   Final old_old: 25.24 ± 4.89   Final old_new: 14.56 ± 3.00   Final new_new: 4.09 ± 2.96
Hits@30:
  Final val: 48.14 ± 4.42   Final Test: 27.54 ± 4.11   Final old_old: 29.96 ± 4.38   Final old_new: 17.86 ± 3.02   Final new_new: 5.00 ± 5.18
Hits@50:
  Final val: 57.91 ± 4.11   Final Test: 37.87 ± 4.87   Final old_old: 40.92 ± 5.39   Final old_new: 25.67 ± 2.74   Final new_new: 9.09 ± 5.57
AUC:
  Final val: 95.55 ± 0.53   Final Test: 94.91 ± 0.49   Final old_old: 95.61 ± 0.51   Final old_new: 92.14 ± 0.48   Final new_new: 87.62 ± 0.69
All runs:
Hits@10:
  Final val: 40.24 ± 6.59   Final Test: 26.55 ± 5.44   Final old_old: 29.21 ± 5.87   Final old_new: 15.84 ± 3.75   Final new_new: 3.18 ± 2.03
Hits@20:
  Final val: 54.36 ± 8.36   Final Test: 37.66 ± 8.82   Final old_old: 40.80 ± 9.44   Final old_new: 25.19 ± 6.44   Final new_new: 6.36 ± 4.37
Hits@30:
  Final val: 63.31 ± 7.50   Final Test: 45.50 ± 7.78   Final old_old: 49.02 ± 8.31   Final old_new: 31.59 ± 5.83   Final new_new: 8.64 ± 3.37
Hits@50:
  Final val: 72.36 ± 6.30   Final Test: 57.98 ± 6.21   Final old_old: 62.04 ± 6.60   Final old_new: 41.99 ± 4.77   Final new_new: 14.55 ± 2.03
AUC:
  Final val: 97.37 ± 0.44   Final Test: 96.79 ± 0.39   Final old_old: 97.30 ± 0.43   Final old_new: 94.79 ± 0.29   Final new_new: 90.61 ± 0.74
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.83 ± 5.24   Final Test: 36.70 ± 4.34   Final old_old: 39.74 ± 4.53   Final old_new: 24.74 ± 3.78   Final new_new: 4.09 ± 1.02
Hits@20:
  Final val: 59.90 ± 3.03   Final Test: 44.04 ± 1.97   Final old_old: 47.34 ± 1.90   Final old_new: 31.11 ± 2.65   Final new_new: 7.73 ± 3.45
Hits@30:
  Final val: 69.28 ± 1.95   Final Test: 50.23 ± 2.20   Final old_old: 53.69 ± 2.29   Final old_new: 36.75 ± 1.99   Final new_new: 10.91 ± 2.96
Hits@50:
  Final val: 76.02 ± 1.43   Final Test: 62.11 ± 2.58   Final old_old: 65.84 ± 2.63   Final old_new: 47.76 ± 2.45   Final new_new: 15.91 ± 2.27
AUC:
  Final val: 97.63 ± 0.09   Final Test: 97.17 ± 0.04   Final old_old: 97.67 ± 0.04   Final old_new: 95.19 ± 0.21   Final new_new: 91.58 ± 0.85
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.73 ± 4.70   Final Test: 30.15 ± 3.09   Final old_old: 33.12 ± 3.33   Final old_new: 18.21 ± 2.20   Final new_new: 2.73 ± 1.90
Hits@20:
  Final val: 57.39 ± 3.30   Final Test: 41.85 ± 3.43   Final old_old: 45.29 ± 3.65   Final old_new: 28.16 ± 2.59   Final new_new: 7.73 ± 3.05
Hits@30:
  Final val: 65.07 ± 2.84   Final Test: 48.61 ± 3.35   Final old_old: 52.37 ± 3.53   Final old_new: 33.70 ± 2.67   Final new_new: 10.00 ± 3.05
Hits@50:
  Final val: 74.77 ± 1.95   Final Test: 60.98 ± 1.71   Final old_old: 65.06 ± 1.83   Final old_new: 45.09 ± 1.29   Final new_new: 15.00 ± 2.59
AUC:
  Final val: 97.53 ± 0.09   Final Test: 97.13 ± 0.04   Final old_old: 97.61 ± 0.04   Final old_new: 95.30 ± 0.08   Final new_new: 91.10 ± 0.33
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 40.33 ± 4.84   Final Test: 31.38 ± 1.65   Final old_old: 34.15 ± 1.68   Final old_new: 20.43 ± 2.25   Final new_new: 3.18 ± 3.05
Hits@20:
  Final val: 58.33 ± 4.36   Final Test: 41.25 ± 3.33   Final old_old: 44.19 ± 3.73   Final old_new: 29.82 ± 2.09   Final new_new: 7.73 ± 4.71
Hits@30:
  Final val: 66.29 ± 2.18   Final Test: 49.88 ± 3.63   Final old_old: 52.85 ± 3.88   Final old_new: 38.54 ± 2.77   Final new_new: 11.36 ± 2.27
Hits@50:
  Final val: 74.79 ± 2.10   Final Test: 60.84 ± 2.60   Final old_old: 64.02 ± 2.92   Final old_new: 48.66 ± 1.62   Final new_new: 20.00 ± 4.93
AUC:
  Final val: 97.49 ± 0.19   Final Test: 97.07 ± 0.17   Final old_old: 97.49 ± 0.20   Final old_new: 95.48 ± 0.13   Final new_new: 92.13 ± 0.70
All runs:
Hits@10:
  Final val: 26.14 ± 2.71   Final Test: 17.43 ± 1.67   Final old_old: 19.12 ± 1.74   Final old_new: 10.65 ± 1.62   Final new_new: 1.82 ± 1.90
Hits@20:
  Final val: 36.35 ± 2.97   Final Test: 27.87 ± 3.65   Final old_old: 30.16 ± 4.01   Final old_new: 18.77 ± 2.32   Final new_new: 5.00 ± 2.49
Hits@30:
  Final val: 45.08 ± 2.58   Final Test: 34.87 ± 4.76   Final old_old: 37.49 ± 5.16   Final old_new: 24.61 ± 3.15   Final new_new: 5.91 ± 4.43
Hits@50:
  Final val: 56.57 ± 3.96   Final Test: 44.36 ± 4.45   Final old_old: 47.38 ± 4.71   Final old_new: 32.52 ± 3.57   Final new_new: 11.82 ± 5.66
AUC:
  Final val: 93.87 ± 0.45   Final Test: 93.48 ± 0.43   Final old_old: 94.29 ± 0.41   Final old_new: 90.43 ± 0.48   Final new_new: 82.65 ± 1.89
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 40.97 ± 8.65   Final Test: 29.48 ± 2.14   Final old_old: 32.28 ± 2.45   Final old_new: 18.24 ± 1.30   Final new_new: 4.09 ± 2.49
Hits@20:
  Final val: 56.17 ± 5.01   Final Test: 39.38 ± 4.57   Final old_old: 42.73 ± 4.90   Final old_new: 26.15 ± 3.48   Final new_new: 4.55 ± 2.27
Hits@30:
  Final val: 64.18 ± 4.16   Final Test: 50.17 ± 4.92   Final old_old: 53.83 ± 5.02   Final old_new: 35.82 ± 4.76   Final new_new: 10.45 ± 1.24
Hits@50:
  Final val: 71.78 ± 3.25   Final Test: 58.99 ± 2.92   Final old_old: 62.61 ± 2.94   Final old_new: 44.94 ± 3.17   Final new_new: 16.82 ± 4.13
AUC:
  Final val: 95.17 ± 0.06   Final Test: 94.73 ± 0.12   Final old_old: 95.51 ± 0.09   Final old_new: 91.79 ± 0.37   Final new_new: 84.59 ± 1.79
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.45 ± 3.61   Final Test: 32.47 ± 4.17   Final old_old: 35.56 ± 4.31   Final old_new: 20.08 ± 3.76   Final new_new: 4.09 ± 2.96
Hits@20:
  Final val: 57.42 ± 1.33   Final Test: 42.07 ± 2.86   Final old_old: 45.40 ± 2.98   Final old_new: 28.79 ± 2.61   Final new_new: 9.55 ± 2.96
Hits@30:
  Final val: 63.97 ± 1.44   Final Test: 50.57 ± 1.37   Final old_old: 53.73 ± 1.48   Final old_new: 38.36 ± 1.12   Final new_new: 13.18 ± 2.49
Hits@50:
  Final val: 70.99 ± 0.77   Final Test: 57.77 ± 1.85   Final old_old: 61.06 ± 1.90   Final old_new: 44.94 ± 2.15   Final new_new: 20.91 ± 3.73
AUC:
  Final val: 94.95 ± 0.08   Final Test: 94.56 ± 0.14   Final old_old: 95.32 ± 0.08   Final old_new: 91.61 ± 0.42   Final new_new: 85.80 ± 1.18
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=65536, lr=0.0003, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0, LLP_R=0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
All runs:
Hits@10:
  Final val: 12.84 ± 7.43   Final Test: 7.48 ± 5.49   Final old_old: 7.95 ± 6.04   Final old_new: 5.44 ± 3.29   Final new_new: 5.45 ± 2.03
Hits@20:
  Final val: 17.98 ± 8.98   Final Test: 10.71 ± 4.48   Final old_old: 11.21 ± 4.89   Final old_new: 8.49 ± 2.90   Final new_new: 10.00 ± 1.24
Hits@30:
  Final val: 21.58 ± 11.72   Final Test: 14.00 ± 7.46   Final old_old: 14.66 ± 8.18   Final old_new: 11.21 ± 4.60   Final new_new: 10.91 ± 1.90
Hits@50:
  Final val: 28.02 ± 15.53   Final Test: 18.74 ± 9.81   Final old_old: 19.73 ± 10.76   Final old_new: 14.56 ± 6.02   Final new_new: 13.18 ± 1.02
AUC:
  Final val: 86.86 ± 6.09   Final Test: 86.62 ± 5.86   Final old_old: 86.98 ± 6.13   Final old_new: 85.25 ± 4.85   Final new_new: 82.27 ± 2.23
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=65536, lr=0.003, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0, LLP_R=0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
All runs:
Hits@10:
  Final val: 39.85 ± 3.78   Final Test: 21.58 ± 2.79   Final old_old: 24.03 ± 2.89   Final old_new: 11.66 ± 2.61   Final new_new: 0.45 ± 1.02
Hits@20:
  Final val: 51.63 ± 3.18   Final Test: 31.97 ± 3.36   Final old_old: 35.11 ± 3.56   Final old_new: 19.37 ± 2.99   Final new_new: 2.73 ± 1.90
Hits@30:
  Final val: 58.67 ± 2.00   Final Test: 38.98 ± 2.23   Final old_old: 42.42 ± 2.57   Final old_new: 25.34 ± 1.49   Final new_new: 4.55 ± 3.21
Hits@50:
  Final val: 68.36 ± 2.12   Final Test: 48.21 ± 2.27   Final old_old: 52.02 ± 2.28   Final old_new: 33.22 ± 2.90   Final new_new: 7.73 ± 3.80
AUC:
  Final val: 96.75 ± 0.20   Final Test: 96.14 ± 0.18   Final old_old: 96.80 ± 0.21   Final old_new: 93.57 ± 0.22   Final new_new: 88.39 ± 1.33
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=65536, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0, LLP_R=0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
All runs:
Hits@10:
  Final val: 45.51 ± 3.59   Final Test: 29.95 ± 2.93   Final old_old: 32.95 ± 3.24   Final old_new: 17.91 ± 1.91   Final new_new: 1.82 ± 1.02
Hits@20:
  Final val: 58.20 ± 3.80   Final Test: 40.96 ± 4.90   Final old_old: 44.45 ± 5.34   Final old_new: 27.10 ± 3.19   Final new_new: 6.36 ± 2.49
Hits@30:
  Final val: 66.42 ± 2.65   Final Test: 46.50 ± 5.03   Final old_old: 50.17 ± 5.35   Final old_new: 32.12 ± 4.05   Final new_new: 6.36 ± 2.96
Hits@50:
  Final val: 73.92 ± 2.03   Final Test: 57.49 ± 5.92   Final old_old: 61.48 ± 6.42   Final old_new: 42.07 ± 4.09   Final new_new: 10.45 ± 1.24
AUC:
  Final val: 97.25 ± 0.09   Final Test: 96.77 ± 0.13   Final old_old: 97.34 ± 0.18   Final old_new: 94.52 ± 0.16   Final new_new: 90.30 ± 1.58
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=65536, lr=0.02, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0, LLP_R=0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
All runs:
Hits@10:
  Final val: 39.28 ± 3.18   Final Test: 27.48 ± 2.53   Final old_old: 30.25 ± 2.76   Final old_new: 16.12 ± 1.69   Final new_new: 5.91 ± 1.24
Hits@20:
  Final val: 49.94 ± 4.31   Final Test: 38.59 ± 3.70   Final old_old: 41.93 ± 3.82   Final old_new: 25.06 ± 3.28   Final new_new: 10.00 ± 3.45
Hits@30:
  Final val: 57.63 ± 3.79   Final Test: 45.13 ± 4.38   Final old_old: 48.55 ± 4.59   Final old_new: 31.51 ± 3.70   Final new_new: 11.36 ± 3.94
Hits@50:
  Final val: 67.28 ± 2.94   Final Test: 52.61 ± 2.49   Final old_old: 56.20 ± 2.71   Final old_new: 38.11 ± 2.28   Final new_new: 20.91 ± 4.07
AUC:
  Final val: 96.51 ± 0.27   Final Test: 96.14 ± 0.31   Final old_old: 96.69 ± 0.31   Final old_new: 93.97 ± 0.36   Final new_new: 90.42 ± 0.63
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=65536, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0, LLP_R=0, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
All runs:
Hits@10:
  Final val: 49.19 ± 1.89   Final Test: 36.54 ± 2.76   Final old_old: 40.07 ± 3.01   Final old_new: 22.42 ± 2.03   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 62.28 ± 0.65   Final Test: 44.94 ± 3.47   Final old_old: 48.61 ± 3.73   Final old_new: 30.48 ± 2.55   Final new_new: 6.36 ± 2.49
Hits@30:
  Final val: 68.37 ± 1.77   Final Test: 51.68 ± 2.53   Final old_old: 55.64 ± 2.69   Final old_new: 36.10 ± 2.04   Final new_new: 9.09 ± 4.25
Hits@50:
  Final val: 76.79 ± 1.56   Final Test: 60.99 ± 2.37   Final old_old: 65.21 ± 2.39   Final old_new: 44.58 ± 2.60   Final new_new: 12.73 ± 3.80
AUC:
  Final val: 97.39 ± 0.07   Final Test: 96.84 ± 0.06   Final old_old: 97.47 ± 0.11   Final old_new: 94.35 ± 0.18   Final new_new: 90.33 ± 0.82
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=65536, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0, LLP_R=0, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
All runs:
Hits@10:
  Final val: 52.83 ± 5.05   Final Test: 33.39 ± 3.65   Final old_old: 36.66 ± 4.04   Final old_new: 20.30 ± 2.32   Final new_new: 2.73 ± 1.02
Hits@20:
  Final val: 63.55 ± 4.18   Final Test: 46.09 ± 3.32   Final old_old: 49.97 ± 3.68   Final old_new: 30.68 ± 2.51   Final new_new: 7.27 ± 3.73
Hits@30:
  Final val: 69.87 ± 3.24   Final Test: 52.16 ± 3.53   Final old_old: 56.28 ± 3.79   Final old_new: 35.97 ± 3.03   Final new_new: 8.18 ± 2.59
Hits@50:
  Final val: 77.29 ± 1.35   Final Test: 60.79 ± 2.35   Final old_old: 65.05 ± 2.26   Final old_new: 44.18 ± 3.72   Final new_new: 12.27 ± 4.98
AUC:
  Final val: 97.40 ± 0.18   Final Test: 96.86 ± 0.18   Final old_old: 97.49 ± 0.19   Final old_new: 94.41 ± 0.32   Final new_new: 90.46 ± 1.72
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=65536, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=0.0, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
All runs:
Hits@10:
  Final val: 52.83 ± 5.05   Final Test: 33.39 ± 3.65   Final old_old: 36.66 ± 4.04   Final old_new: 20.30 ± 2.32   Final new_new: 2.73 ± 1.02
Hits@20:
  Final val: 63.55 ± 4.18   Final Test: 46.09 ± 3.32   Final old_old: 49.97 ± 3.68   Final old_new: 30.68 ± 2.51   Final new_new: 7.27 ± 3.73
Hits@30:
  Final val: 69.87 ± 3.24   Final Test: 52.16 ± 3.53   Final old_old: 56.28 ± 3.79   Final old_new: 35.97 ± 3.03   Final new_new: 8.18 ± 2.59
Hits@50:
  Final val: 77.29 ± 1.35   Final Test: 60.79 ± 2.35   Final old_old: 65.05 ± 2.26   Final old_new: 44.18 ± 3.72   Final new_new: 12.27 ± 4.98
AUC:
  Final val: 97.40 ± 0.18   Final Test: 96.86 ± 0.18   Final old_old: 97.49 ± 0.19   Final old_new: 94.41 ± 0.32   Final new_new: 90.46 ± 1.72
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=0.0, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=65536, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=0.0, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
All runs:
Hits@10:
  Final val: 52.83 ± 5.05   Final Test: 33.39 ± 3.65   Final old_old: 36.66 ± 4.04   Final old_new: 20.30 ± 2.32   Final new_new: 2.73 ± 1.02
Hits@20:
  Final val: 63.55 ± 4.18   Final Test: 46.09 ± 3.32   Final old_old: 49.97 ± 3.68   Final old_new: 30.68 ± 2.51   Final new_new: 7.27 ± 3.73
Hits@30:
  Final val: 69.87 ± 3.24   Final Test: 52.16 ± 3.53   Final old_old: 56.28 ± 3.79   Final old_new: 35.97 ± 3.03   Final new_new: 8.18 ± 2.59
Hits@50:
  Final val: 77.29 ± 1.35   Final Test: 60.79 ± 2.35   Final old_old: 65.05 ± 2.26   Final old_new: 44.18 ± 3.72   Final new_new: 12.27 ± 4.98
AUC:
  Final val: 97.40 ± 0.18   Final Test: 96.86 ± 0.18   Final old_old: 97.49 ± 0.19   Final old_new: 94.41 ± 0.32   Final new_new: 90.46 ± 1.72
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=0.0, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
All runs:
Hits@10:
  Final val: 44.71 ± 2.84   Final Test: 30.56 ± 6.28   Final old_old: 33.62 ± 6.75   Final old_new: 18.16 ± 4.71   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 58.87 ± 3.35   Final Test: 40.17 ± 5.61   Final old_old: 43.68 ± 5.97   Final old_new: 25.97 ± 4.52   Final new_new: 9.55 ± 4.93
Hits@30:
  Final val: 65.78 ± 2.53   Final Test: 45.22 ± 2.61   Final old_old: 49.01 ± 2.80   Final old_new: 30.08 ± 2.27   Final new_new: 9.09 ± 3.21
Hits@50:
  Final val: 73.65 ± 2.10   Final Test: 56.74 ± 3.49   Final old_old: 60.71 ± 3.65   Final old_new: 40.86 ± 3.23   Final new_new: 19.55 ± 5.23
AUC:
  Final val: 97.22 ± 0.14   Final Test: 96.72 ± 0.13   Final old_old: 97.29 ± 0.14   Final old_new: 94.47 ± 0.18   Final new_new: 90.95 ± 1.15
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=0.1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=65536, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=0.0, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
All runs:
Hits@10:
  Final val: 52.83 ± 5.05   Final Test: 33.39 ± 3.65   Final old_old: 36.66 ± 4.04   Final old_new: 20.30 ± 2.32   Final new_new: 2.73 ± 1.02
Hits@20:
  Final val: 63.55 ± 4.18   Final Test: 46.09 ± 3.32   Final old_old: 49.97 ± 3.68   Final old_new: 30.68 ± 2.51   Final new_new: 7.27 ± 3.73
Hits@30:
  Final val: 69.87 ± 3.24   Final Test: 52.16 ± 3.53   Final old_old: 56.28 ± 3.79   Final old_new: 35.97 ± 3.03   Final new_new: 8.18 ± 2.59
Hits@50:
  Final val: 77.29 ± 1.35   Final Test: 60.79 ± 2.35   Final old_old: 65.05 ± 2.26   Final old_new: 44.18 ± 3.72   Final new_new: 12.27 ± 4.98
AUC:
  Final val: 97.40 ± 0.18   Final Test: 96.86 ± 0.18   Final old_old: 97.49 ± 0.19   Final old_new: 94.41 ± 0.32   Final new_new: 90.46 ± 1.72
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=0.0, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
All runs:
Hits@10:
  Final val: 44.71 ± 2.84   Final Test: 30.56 ± 6.28   Final old_old: 33.62 ± 6.75   Final old_new: 18.16 ± 4.71   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 58.87 ± 3.35   Final Test: 40.17 ± 5.61   Final old_old: 43.68 ± 5.97   Final old_new: 25.97 ± 4.52   Final new_new: 9.55 ± 4.93
Hits@30:
  Final val: 65.78 ± 2.53   Final Test: 45.22 ± 2.61   Final old_old: 49.01 ± 2.80   Final old_new: 30.08 ± 2.27   Final new_new: 9.09 ± 3.21
Hits@50:
  Final val: 73.65 ± 2.10   Final Test: 56.74 ± 3.49   Final old_old: 60.71 ± 3.65   Final old_new: 40.86 ± 3.23   Final new_new: 19.55 ± 5.23
AUC:
  Final val: 97.22 ± 0.14   Final Test: 96.72 ± 0.13   Final old_old: 97.29 ± 0.14   Final old_new: 94.47 ± 0.18   Final new_new: 90.95 ± 1.15
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=0.1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.45 ± 1.76   Final Test: 30.58 ± 6.15   Final old_old: 33.74 ± 6.57   Final old_new: 17.81 ± 4.55   Final new_new: 3.64 ± 2.03
Hits@20:
  Final val: 61.71 ± 1.25   Final Test: 44.78 ± 3.21   Final old_old: 48.51 ± 3.43   Final old_new: 29.97 ± 2.45   Final new_new: 7.73 ± 4.98
Hits@30:
  Final val: 68.45 ± 0.60   Final Test: 49.47 ± 4.18   Final old_old: 53.45 ± 4.29   Final old_new: 33.68 ± 3.84   Final new_new: 9.55 ± 4.37
Hits@50:
  Final val: 75.78 ± 0.65   Final Test: 59.71 ± 1.41   Final old_old: 63.99 ± 1.36   Final old_new: 42.70 ± 2.05   Final new_new: 17.73 ± 2.96
AUC:
  Final val: 97.37 ± 0.10   Final Test: 96.81 ± 0.09   Final old_old: 97.44 ± 0.12   Final old_new: 94.32 ± 0.23   Final new_new: 90.59 ± 0.39
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=0.1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=65536, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=0.0, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=65536, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=0.0, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=65536, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=0.0, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
All runs:
Hits@10:
  Final val: 47.10 ± 3.89   Final Test: 30.13 ± 3.13   Final old_old: 33.18 ± 3.52   Final old_new: 17.76 ± 1.68   Final new_new: 4.09 ± 1.02
Hits@20:
  Final val: 59.00 ± 2.57   Final Test: 39.12 ± 2.99   Final old_old: 42.66 ± 3.25   Final old_new: 24.94 ± 2.13   Final new_new: 6.82 ± 0.00
Hits@30:
  Final val: 65.59 ± 2.61   Final Test: 46.03 ± 4.21   Final old_old: 49.93 ± 4.33   Final old_new: 30.43 ± 4.22   Final new_new: 8.64 ± 1.02
Hits@50:
  Final val: 73.58 ± 1.33   Final Test: 57.20 ± 1.98   Final old_old: 61.12 ± 2.12   Final old_new: 41.79 ± 2.14   Final new_new: 15.00 ± 2.59
AUC:
  Final val: 97.24 ± 0.10   Final Test: 96.71 ± 0.08   Final old_old: 97.29 ± 0.10   Final old_new: 94.42 ± 0.24   Final new_new: 90.91 ± 0.96
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.83 ± 5.05   Final Test: 33.39 ± 3.65   Final old_old: 36.66 ± 4.04   Final old_new: 20.30 ± 2.32   Final new_new: 2.73 ± 1.02
Hits@20:
  Final val: 63.55 ± 4.18   Final Test: 46.09 ± 3.32   Final old_old: 49.97 ± 3.68   Final old_new: 30.68 ± 2.51   Final new_new: 7.27 ± 3.73
Hits@30:
  Final val: 69.87 ± 3.24   Final Test: 52.16 ± 3.53   Final old_old: 56.28 ± 3.79   Final old_new: 35.97 ± 3.03   Final new_new: 8.18 ± 2.59
Hits@50:
  Final val: 77.29 ± 1.35   Final Test: 60.79 ± 2.35   Final old_old: 65.05 ± 2.26   Final old_new: 44.18 ± 3.72   Final new_new: 12.27 ± 4.98
AUC:
  Final val: 97.40 ± 0.18   Final Test: 96.86 ± 0.18   Final old_old: 97.49 ± 0.19   Final old_new: 94.41 ± 0.32   Final new_new: 90.46 ± 1.72
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=0.0, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
All runs:
Hits@10:
  Final val: 52.83 ± 5.05   Final Test: 33.39 ± 3.65   Final old_old: 36.66 ± 4.04   Final old_new: 20.30 ± 2.32   Final new_new: 2.73 ± 1.02
Hits@20:
  Final val: 63.55 ± 4.18   Final Test: 46.09 ± 3.32   Final old_old: 49.97 ± 3.68   Final old_new: 30.68 ± 2.51   Final new_new: 7.27 ± 3.73
Hits@30:
  Final val: 69.87 ± 3.24   Final Test: 52.16 ± 3.53   Final old_old: 56.28 ± 3.79   Final old_new: 35.97 ± 3.03   Final new_new: 8.18 ± 2.59
Hits@50:
  Final val: 77.29 ± 1.35   Final Test: 60.79 ± 2.35   Final old_old: 65.05 ± 2.26   Final old_new: 44.18 ± 3.72   Final new_new: 12.27 ± 4.98
AUC:
  Final val: 97.40 ± 0.18   Final Test: 96.86 ± 0.18   Final old_old: 97.49 ± 0.19   Final old_new: 94.41 ± 0.32   Final new_new: 90.46 ± 1.72
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=0.0, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
All runs:
Hits@10:
  Final val: 48.61 ± 5.61   Final Test: 32.48 ± 3.92   Final old_old: 35.78 ± 4.24   Final old_new: 19.07 ± 3.18   Final new_new: 5.91 ± 2.03
Hits@20:
  Final val: 61.03 ± 3.59   Final Test: 42.30 ± 3.08   Final old_old: 45.96 ± 3.45   Final old_new: 27.78 ± 2.39   Final new_new: 5.91 ± 2.59
Hits@30:
  Final val: 67.56 ± 2.88   Final Test: 50.98 ± 1.99   Final old_old: 55.07 ± 2.11   Final old_new: 34.76 ± 2.14   Final new_new: 10.00 ± 3.80
Hits@50:
  Final val: 75.95 ± 1.42   Final Test: 59.92 ± 1.06   Final old_old: 64.09 ± 1.03   Final old_new: 43.40 ± 1.75   Final new_new: 17.73 ± 5.88
AUC:
  Final val: 97.30 ± 0.18   Final Test: 96.82 ± 0.11   Final old_old: 97.43 ± 0.14   Final old_new: 94.40 ± 0.13   Final new_new: 90.98 ± 0.78
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.71 ± 2.84   Final Test: 30.56 ± 6.28   Final old_old: 33.62 ± 6.75   Final old_new: 18.16 ± 4.71   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 58.87 ± 3.35   Final Test: 40.17 ± 5.61   Final old_old: 43.68 ± 5.97   Final old_new: 25.97 ± 4.52   Final new_new: 9.55 ± 4.93
Hits@30:
  Final val: 65.78 ± 2.53   Final Test: 45.22 ± 2.61   Final old_old: 49.01 ± 2.80   Final old_new: 30.08 ± 2.27   Final new_new: 9.09 ± 3.21
Hits@50:
  Final val: 73.65 ± 2.10   Final Test: 56.74 ± 3.49   Final old_old: 60.71 ± 3.65   Final old_new: 40.86 ± 3.23   Final new_new: 19.55 ± 5.23
AUC:
  Final val: 97.22 ± 0.14   Final Test: 96.72 ± 0.13   Final old_old: 97.29 ± 0.14   Final old_new: 94.47 ± 0.18   Final new_new: 90.95 ± 1.15
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=0.1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.71 ± 2.84   Final Test: 30.56 ± 6.28   Final old_old: 33.62 ± 6.75   Final old_new: 18.16 ± 4.71   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 58.87 ± 3.35   Final Test: 40.17 ± 5.61   Final old_old: 43.68 ± 5.97   Final old_new: 25.97 ± 4.52   Final new_new: 9.55 ± 4.93
Hits@30:
  Final val: 65.78 ± 2.53   Final Test: 45.22 ± 2.61   Final old_old: 49.01 ± 2.80   Final old_new: 30.08 ± 2.27   Final new_new: 9.09 ± 3.21
Hits@50:
  Final val: 73.65 ± 2.10   Final Test: 56.74 ± 3.49   Final old_old: 60.71 ± 3.65   Final old_new: 40.86 ± 3.23   Final new_new: 19.55 ± 5.23
AUC:
  Final val: 97.22 ± 0.14   Final Test: 96.72 ± 0.13   Final old_old: 97.29 ± 0.14   Final old_new: 94.47 ± 0.18   Final new_new: 90.95 ± 1.15
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=0.1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.20 ± 2.50   Final Test: 31.53 ± 4.10   Final old_old: 34.65 ± 4.49   Final old_new: 18.89 ± 2.66   Final new_new: 4.55 ± 2.78
Hits@20:
  Final val: 58.75 ± 1.66   Final Test: 40.62 ± 2.76   Final old_old: 44.05 ± 3.02   Final old_new: 26.95 ± 2.00   Final new_new: 6.82 ± 3.59
Hits@30:
  Final val: 65.51 ± 1.14   Final Test: 47.67 ± 3.08   Final old_old: 51.42 ± 3.40   Final old_new: 32.85 ± 2.28   Final new_new: 9.09 ± 3.59
Hits@50:
  Final val: 74.18 ± 1.80   Final Test: 57.40 ± 2.86   Final old_old: 61.49 ± 3.11   Final old_new: 41.36 ± 2.47   Final new_new: 12.73 ± 4.43
AUC:
  Final val: 97.24 ± 0.10   Final Test: 96.76 ± 0.13   Final old_old: 97.34 ± 0.17   Final old_new: 94.49 ± 0.22   Final new_new: 90.45 ± 0.72
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=10, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.82 ± 6.22   Final Test: 28.95 ± 3.67   Final old_old: 31.92 ± 3.88   Final old_new: 16.88 ± 2.92   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 60.71 ± 6.44   Final Test: 39.81 ± 4.24   Final old_old: 43.40 ± 4.55   Final old_new: 25.37 ± 3.40   Final new_new: 7.27 ± 2.96
Hits@30:
  Final val: 66.68 ± 4.96   Final Test: 47.38 ± 7.04   Final old_old: 51.36 ± 7.48   Final old_new: 31.49 ± 5.46   Final new_new: 9.09 ± 3.21
Hits@50:
  Final val: 74.70 ± 3.81   Final Test: 59.47 ± 2.00   Final old_old: 63.66 ± 2.03   Final old_new: 42.80 ± 2.17   Final new_new: 18.18 ± 4.55
AUC:
  Final val: 97.30 ± 0.18   Final Test: 96.77 ± 0.15   Final old_old: 97.38 ± 0.18   Final old_new: 94.37 ± 0.07   Final new_new: 91.01 ± 0.46
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=0.1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.08 ± 1.79   Final Test: 30.16 ± 3.71   Final old_old: 33.18 ± 3.78   Final old_new: 17.88 ± 3.59   Final new_new: 5.00 ± 2.96
Hits@20:
  Final val: 62.60 ± 2.22   Final Test: 42.48 ± 3.21   Final old_old: 46.22 ± 3.29   Final old_new: 27.56 ± 3.04   Final new_new: 6.82 ± 1.61
Hits@30:
  Final val: 68.69 ± 2.26   Final Test: 50.56 ± 3.86   Final old_old: 54.54 ± 3.78   Final old_new: 34.79 ± 4.26   Final new_new: 10.45 ± 3.80
Hits@50:
  Final val: 76.02 ± 1.84   Final Test: 60.41 ± 3.08   Final old_old: 64.63 ± 2.86   Final old_new: 43.60 ± 4.03   Final new_new: 19.09 ± 5.93
AUC:
  Final val: 97.38 ± 0.15   Final Test: 96.89 ± 0.11   Final old_old: 97.49 ± 0.11   Final old_new: 94.53 ± 0.21   Final new_new: 90.84 ± 1.04
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=0.1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.75 ± 3.68   Final Test: 33.23 ± 4.36   Final old_old: 36.45 ± 4.68   Final old_new: 20.33 ± 3.44   Final new_new: 3.18 ± 2.03
Hits@20:
  Final val: 62.70 ± 2.97   Final Test: 43.72 ± 5.76   Final old_old: 47.40 ± 6.07   Final old_new: 29.09 ± 4.62   Final new_new: 6.82 ± 2.78
Hits@30:
  Final val: 68.79 ± 2.15   Final Test: 51.05 ± 3.15   Final old_old: 55.05 ± 3.44   Final old_new: 35.24 ± 2.05   Final new_new: 9.55 ± 2.96
Hits@50:
  Final val: 76.26 ± 0.90   Final Test: 61.39 ± 4.49   Final old_old: 65.53 ± 4.76   Final old_new: 45.16 ± 3.48   Final new_new: 16.36 ± 4.37
AUC:
  Final val: 97.33 ± 0.12   Final Test: 96.82 ± 0.16   Final old_old: 97.42 ± 0.17   Final old_new: 94.46 ± 0.27   Final new_new: 90.38 ± 0.97
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=10, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.04 ± 3.52   Final Test: 29.30 ± 5.01   Final old_old: 32.36 ± 5.46   Final old_new: 16.90 ± 3.20   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 60.18 ± 2.03   Final Test: 40.14 ± 3.32   Final old_old: 43.93 ± 3.44   Final old_new: 24.96 ± 3.09   Final new_new: 5.00 ± 1.02
Hits@30:
  Final val: 66.87 ± 1.94   Final Test: 46.79 ± 2.40   Final old_old: 50.67 ± 2.42   Final old_new: 31.39 ± 2.70   Final new_new: 8.18 ± 3.05
Hits@50:
  Final val: 74.84 ± 1.31   Final Test: 57.14 ± 2.24   Final old_old: 61.27 ± 2.19   Final old_new: 40.81 ± 2.59   Final new_new: 14.09 ± 4.93
AUC:
  Final val: 97.27 ± 0.03   Final Test: 96.74 ± 0.07   Final old_old: 97.35 ± 0.04   Final old_new: 94.36 ± 0.24   Final new_new: 90.24 ± 0.73
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.95 ± 2.89   Final Test: 32.17 ± 3.94   Final old_old: 35.28 ± 4.06   Final old_new: 19.60 ± 3.71   Final new_new: 5.00 ± 2.49
Hits@20:
  Final val: 61.28 ± 3.05   Final Test: 40.83 ± 3.85   Final old_old: 44.51 ± 4.11   Final old_new: 26.02 ± 2.87   Final new_new: 7.27 ± 2.96
Hits@30:
  Final val: 67.03 ± 2.35   Final Test: 48.56 ± 2.92   Final old_old: 52.65 ± 3.23   Final old_new: 32.19 ± 1.89   Final new_new: 10.00 ± 5.23
Hits@50:
  Final val: 74.44 ± 0.93   Final Test: 57.24 ± 2.43   Final old_old: 61.26 ± 2.56   Final old_new: 41.34 ± 2.10   Final new_new: 15.91 ± 3.94
AUC:
  Final val: 97.27 ± 0.08   Final Test: 96.78 ± 0.07   Final old_old: 97.37 ± 0.04   Final old_new: 94.45 ± 0.23   Final new_new: 90.50 ± 0.68
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.57 ± 4.82   Final Test: 30.47 ± 4.43   Final old_old: 33.63 ± 4.64   Final old_new: 17.73 ± 3.74   Final new_new: 2.27 ± 1.61
Hits@20:
  Final val: 59.15 ± 1.15   Final Test: 39.01 ± 3.23   Final old_old: 42.48 ± 3.35   Final old_new: 25.09 ± 2.96   Final new_new: 6.82 ± 0.00
Hits@30:
  Final val: 66.16 ± 0.94   Final Test: 48.70 ± 2.01   Final old_old: 52.40 ± 2.22   Final old_new: 34.08 ± 1.38   Final new_new: 10.00 ± 3.80
Hits@50:
  Final val: 74.17 ± 0.93   Final Test: 58.22 ± 3.54   Final old_old: 62.45 ± 3.61   Final old_new: 41.39 ± 3.75   Final new_new: 16.36 ± 7.08
AUC:
  Final val: 97.23 ± 0.08   Final Test: 96.72 ± 0.07   Final old_old: 97.31 ± 0.05   Final old_new: 94.40 ± 0.32   Final new_new: 90.38 ± 1.33
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.0, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.61 ± 3.72   Final Test: 31.57 ± 2.50   Final old_old: 34.81 ± 2.71   Final old_new: 18.41 ± 1.74   Final new_new: 5.00 ± 2.96
Hits@20:
  Final val: 61.49 ± 2.78   Final Test: 45.71 ± 4.75   Final old_old: 49.47 ± 4.95   Final old_new: 30.81 ± 4.25   Final new_new: 8.18 ± 4.13
Hits@30:
  Final val: 68.57 ± 2.15   Final Test: 53.55 ± 3.69   Final old_old: 57.85 ± 3.54   Final old_new: 36.52 ± 4.55   Final new_new: 10.45 ± 5.23
Hits@50:
  Final val: 76.69 ± 1.21   Final Test: 62.84 ± 3.75   Final old_old: 67.21 ± 3.48   Final old_new: 45.54 ± 5.08   Final new_new: 18.64 ± 3.73
AUC:
  Final val: 97.33 ± 0.11   Final Test: 96.88 ± 0.11   Final old_old: 97.48 ± 0.11   Final old_new: 94.47 ± 0.21   Final new_new: 90.58 ± 1.05
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 53.15 ± 2.98   Final Test: 32.51 ± 3.38   Final old_old: 35.83 ± 3.74   Final old_new: 19.07 ± 2.35   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 63.99 ± 1.77   Final Test: 44.50 ± 2.48   Final old_old: 48.39 ± 2.62   Final old_new: 29.04 ± 2.39   Final new_new: 5.91 ± 3.45
Hits@30:
  Final val: 70.08 ± 1.90   Final Test: 53.35 ± 2.14   Final old_old: 57.60 ± 2.28   Final old_new: 36.68 ± 2.17   Final new_new: 7.27 ± 3.73
Hits@50:
  Final val: 77.11 ± 0.81   Final Test: 61.93 ± 4.37   Final old_old: 66.23 ± 4.49   Final old_new: 45.04 ± 3.77   Final new_new: 15.45 ± 9.15
AUC:
  Final val: 97.43 ± 0.11   Final Test: 96.89 ± 0.09   Final old_old: 97.53 ± 0.09   Final old_new: 94.40 ± 0.22   Final new_new: 89.47 ± 1.16
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.33 ± 2.17   Final Test: 28.80 ± 6.22   Final old_old: 31.68 ± 6.78   Final old_new: 17.15 ± 4.30   Final new_new: 4.09 ± 1.02
Hits@20:
  Final val: 62.62 ± 0.59   Final Test: 43.57 ± 1.64   Final old_old: 47.44 ± 1.87   Final old_new: 28.09 ± 1.41   Final new_new: 6.82 ± 2.78
Hits@30:
  Final val: 69.88 ± 0.74   Final Test: 49.65 ± 1.99   Final old_old: 53.78 ± 2.15   Final old_new: 33.32 ± 1.69   Final new_new: 7.27 ± 2.96
Hits@50:
  Final val: 76.53 ± 1.39   Final Test: 59.81 ± 2.55   Final old_old: 64.02 ± 2.59   Final old_new: 43.32 ± 2.69   Final new_new: 13.64 ± 1.61
AUC:
  Final val: 97.40 ± 0.12   Final Test: 96.84 ± 0.10   Final old_old: 97.47 ± 0.15   Final old_new: 94.36 ± 0.23   Final new_new: 90.24 ± 0.64
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.0, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.79 ± 2.45   Final Test: 32.47 ± 3.99   Final old_old: 35.64 ± 4.20   Final old_new: 19.62 ± 3.13   Final new_new: 5.00 ± 2.96
Hits@20:
  Final val: 60.62 ± 2.37   Final Test: 39.86 ± 3.10   Final old_old: 43.50 ± 3.22   Final old_new: 25.26 ± 2.67   Final new_new: 5.45 ± 2.03
Hits@30:
  Final val: 66.94 ± 1.37   Final Test: 46.97 ± 1.48   Final old_old: 50.78 ± 1.52   Final old_new: 31.89 ± 1.62   Final new_new: 8.18 ± 1.24
Hits@50:
  Final val: 74.63 ± 1.38   Final Test: 58.20 ± 0.95   Final old_old: 62.44 ± 0.94   Final old_new: 41.51 ± 1.03   Final new_new: 13.18 ± 3.73
AUC:
  Final val: 97.31 ± 0.09   Final Test: 96.76 ± 0.15   Final old_old: 97.37 ± 0.10   Final old_new: 94.36 ± 0.43   Final new_new: 90.48 ± 1.61
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=10, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.52 ± 1.27   Final Test: 31.51 ± 2.93   Final old_old: 34.81 ± 3.08   Final old_new: 18.14 ± 2.54   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 61.00 ± 2.45   Final Test: 41.93 ± 3.63   Final old_old: 45.58 ± 3.88   Final old_new: 27.36 ± 2.71   Final new_new: 6.82 ± 3.94
Hits@30:
  Final val: 67.79 ± 2.37   Final Test: 48.83 ± 3.69   Final old_old: 52.85 ± 3.89   Final old_new: 32.87 ± 3.10   Final new_new: 8.64 ± 4.93
Hits@50:
  Final val: 75.72 ± 1.03   Final Test: 58.36 ± 3.50   Final old_old: 62.77 ± 3.65   Final old_new: 41.01 ± 2.88   Final new_new: 11.36 ± 5.33
AUC:
  Final val: 97.30 ± 0.13   Final Test: 96.79 ± 0.10   Final old_old: 97.40 ± 0.15   Final old_new: 94.41 ± 0.24   Final new_new: 90.15 ± 0.69
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=10, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.35 ± 4.24   Final Test: 30.58 ± 4.74   Final old_old: 33.55 ± 5.05   Final old_new: 18.56 ± 3.70   Final new_new: 4.55 ± 0.00
Hits@20:
  Final val: 58.77 ± 1.84   Final Test: 40.55 ± 3.20   Final old_old: 44.21 ± 3.38   Final old_new: 25.77 ± 2.74   Final new_new: 8.18 ± 1.24
Hits@30:
  Final val: 66.17 ± 2.05   Final Test: 48.06 ± 3.43   Final old_old: 51.98 ± 3.72   Final old_new: 32.49 ± 2.36   Final new_new: 8.64 ± 1.90
Hits@50:
  Final val: 73.76 ± 1.09   Final Test: 59.96 ± 4.10   Final old_old: 64.08 ± 4.10   Final old_new: 43.83 ± 4.20   Final new_new: 15.00 ± 4.43
AUC:
  Final val: 97.23 ± 0.05   Final Test: 96.78 ± 0.12   Final old_old: 97.36 ± 0.10   Final old_new: 94.49 ± 0.26   Final new_new: 90.62 ± 0.93
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.33 ± 4.80   Final Test: 32.34 ± 3.04   Final old_old: 35.56 ± 3.24   Final old_new: 19.45 ± 2.55   Final new_new: 2.73 ± 1.02
Hits@20:
  Final val: 64.98 ± 3.52   Final Test: 44.81 ± 3.45   Final old_old: 48.62 ± 3.36   Final old_new: 29.70 ± 4.12   Final new_new: 6.36 ± 2.96
Hits@30:
  Final val: 70.42 ± 1.29   Final Test: 51.04 ± 2.72   Final old_old: 55.18 ± 2.72   Final old_new: 34.58 ± 3.22   Final new_new: 10.00 ± 2.03
Hits@50:
  Final val: 77.59 ± 1.12   Final Test: 60.99 ± 1.04   Final old_old: 65.30 ± 1.29   Final old_new: 44.08 ± 1.37   Final new_new: 14.09 ± 1.90
AUC:
  Final val: 97.40 ± 0.10   Final Test: 96.85 ± 0.07   Final old_old: 97.48 ± 0.11   Final old_new: 94.31 ± 0.17   Final new_new: 91.20 ± 0.72
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=10, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.65 ± 3.98   Final Test: 31.90 ± 3.96   Final old_old: 35.05 ± 4.29   Final old_new: 19.17 ± 3.02   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 62.71 ± 2.87   Final Test: 45.39 ± 2.77   Final old_old: 49.26 ± 3.12   Final old_new: 29.90 ± 1.63   Final new_new: 8.64 ± 1.02
Hits@30:
  Final val: 68.96 ± 2.63   Final Test: 52.56 ± 3.56   Final old_old: 56.50 ± 4.00   Final old_new: 37.13 ± 1.86   Final new_new: 9.09 ± 3.59
Hits@50:
  Final val: 76.21 ± 2.02   Final Test: 59.85 ± 2.35   Final old_old: 64.04 ± 2.37   Final old_new: 43.45 ± 2.93   Final new_new: 13.18 ± 2.96
AUC:
  Final val: 97.38 ± 0.18   Final Test: 96.87 ± 0.18   Final old_old: 97.49 ± 0.20   Final old_new: 94.43 ± 0.19   Final new_new: 90.36 ± 0.71
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.0, LLP_R=10, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.73 ± 3.31   Final Test: 34.32 ± 5.49   Final old_old: 37.58 ± 5.82   Final old_new: 21.34 ± 4.43   Final new_new: 2.73 ± 1.02
Hits@20:
  Final val: 63.57 ± 1.45   Final Test: 44.41 ± 4.59   Final old_old: 48.37 ± 4.78   Final old_new: 28.56 ± 4.52   Final new_new: 7.27 ± 1.90
Hits@30:
  Final val: 69.85 ± 2.19   Final Test: 49.26 ± 2.24   Final old_old: 53.33 ± 2.63   Final old_new: 33.07 ± 1.55   Final new_new: 8.64 ± 2.96
Hits@50:
  Final val: 77.31 ± 2.15   Final Test: 62.71 ± 4.07   Final old_old: 67.00 ± 4.22   Final old_new: 45.74 ± 3.85   Final new_new: 18.64 ± 5.18
AUC:
  Final val: 97.42 ± 0.18   Final Test: 96.84 ± 0.12   Final old_old: 97.44 ± 0.15   Final old_new: 94.45 ± 0.14   Final new_new: 90.46 ± 0.66
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.80 ± 3.13   Final Test: 31.45 ± 2.13   Final old_old: 34.65 ± 2.08   Final old_new: 18.54 ± 2.57   Final new_new: 3.18 ± 2.59
Hits@20:
  Final val: 59.93 ± 2.62   Final Test: 41.08 ± 3.81   Final old_old: 44.79 ± 4.20   Final old_new: 26.25 ± 2.44   Final new_new: 6.36 ± 3.73
Hits@30:
  Final val: 66.00 ± 1.76   Final Test: 49.04 ± 4.18   Final old_old: 53.03 ± 4.51   Final old_new: 33.22 ± 2.97   Final new_new: 9.09 ± 3.59
Hits@50:
  Final val: 73.58 ± 1.22   Final Test: 56.95 ± 2.17   Final old_old: 61.32 ± 2.31   Final old_new: 39.72 ± 1.84   Final new_new: 10.91 ± 4.07
AUC:
  Final val: 97.26 ± 0.09   Final Test: 96.77 ± 0.06   Final old_old: 97.36 ± 0.05   Final old_new: 94.52 ± 0.14   Final new_new: 89.79 ± 0.42
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.0, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.71 ± 4.31   Final Test: 32.04 ± 4.65   Final old_old: 35.13 ± 4.93   Final old_new: 19.57 ± 3.86   Final new_new: 5.00 ± 1.90
Hits@20:
  Final val: 60.17 ± 4.69   Final Test: 40.75 ± 4.04   Final old_old: 44.12 ± 4.43   Final old_new: 27.33 ± 2.66   Final new_new: 7.73 ± 3.05
Hits@30:
  Final val: 66.86 ± 3.17   Final Test: 45.39 ± 5.71   Final old_old: 49.12 ± 6.17   Final old_new: 30.55 ± 4.08   Final new_new: 8.64 ± 3.73
Hits@50:
  Final val: 73.70 ± 1.26   Final Test: 59.46 ± 3.22   Final old_old: 63.52 ± 3.29   Final old_new: 43.35 ± 3.27   Final new_new: 18.18 ± 4.55
AUC:
  Final val: 97.21 ± 0.07   Final Test: 96.71 ± 0.10   Final old_old: 97.29 ± 0.10   Final old_new: 94.35 ± 0.21   Final new_new: 91.06 ± 0.46
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.59 ± 2.07   Final Test: 31.41 ± 4.94   Final old_old: 34.46 ± 5.44   Final old_new: 19.14 ± 3.29   Final new_new: 3.64 ± 2.59
Hits@20:
  Final val: 59.12 ± 2.08   Final Test: 40.88 ± 2.55   Final old_old: 44.50 ± 2.73   Final old_new: 26.32 ± 2.18   Final new_new: 8.18 ± 3.05
Hits@30:
  Final val: 66.43 ± 2.12   Final Test: 48.95 ± 2.51   Final old_old: 52.90 ± 2.80   Final old_new: 33.10 ± 1.38   Final new_new: 12.27 ± 2.03
Hits@50:
  Final val: 74.07 ± 2.05   Final Test: 58.20 ± 1.62   Final old_old: 62.25 ± 1.68   Final old_new: 42.17 ± 1.80   Final new_new: 16.82 ± 1.24
AUC:
  Final val: 97.26 ± 0.10   Final Test: 96.75 ± 0.12   Final old_old: 97.36 ± 0.15   Final old_new: 94.38 ± 0.12   Final new_new: 90.21 ± 1.16
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.0, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.90 ± 4.09   Final Test: 30.73 ± 5.02   Final old_old: 33.79 ± 5.55   Final old_new: 18.34 ± 2.88   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 64.50 ± 2.92   Final Test: 46.63 ± 4.57   Final old_old: 50.55 ± 4.75   Final old_new: 31.01 ± 4.08   Final new_new: 7.73 ± 2.03
Hits@30:
  Final val: 70.40 ± 2.12   Final Test: 51.12 ± 4.09   Final old_old: 55.32 ± 4.48   Final old_new: 34.41 ± 2.66   Final new_new: 9.55 ± 1.90
Hits@50:
  Final val: 76.99 ± 0.97   Final Test: 61.93 ± 1.40   Final old_old: 66.31 ± 1.56   Final old_new: 44.53 ± 1.30   Final new_new: 18.18 ± 4.55
AUC:
  Final val: 97.40 ± 0.14   Final Test: 96.87 ± 0.05   Final old_old: 97.51 ± 0.04   Final old_new: 94.40 ± 0.15   Final new_new: 89.52 ± 2.01
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.0, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.10 ± 2.74   Final Test: 34.34 ± 6.90   Final old_old: 37.42 ± 7.17   Final old_new: 21.99 ± 6.02   Final new_new: 5.00 ± 1.90
Hits@20:
  Final val: 64.69 ± 2.37   Final Test: 45.00 ± 3.93   Final old_old: 48.70 ± 3.90   Final old_new: 30.30 ± 4.23   Final new_new: 8.18 ± 4.98
Hits@30:
  Final val: 70.51 ± 1.65   Final Test: 50.59 ± 3.11   Final old_old: 54.63 ± 3.18   Final old_new: 34.56 ± 2.93   Final new_new: 10.00 ± 5.93
Hits@50:
  Final val: 77.19 ± 1.72   Final Test: 62.37 ± 1.80   Final old_old: 66.57 ± 1.77   Final old_new: 45.89 ± 2.34   Final new_new: 16.36 ± 5.88
AUC:
  Final val: 97.36 ± 0.08   Final Test: 96.89 ± 0.06   Final old_old: 97.51 ± 0.06   Final old_new: 94.40 ± 0.19   Final new_new: 90.53 ± 1.07
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.07 ± 1.63   Final Test: 33.20 ± 3.38   Final old_old: 36.53 ± 3.65   Final old_new: 19.72 ± 2.57   Final new_new: 5.00 ± 2.49
Hits@20:
  Final val: 63.35 ± 1.62   Final Test: 42.94 ± 3.96   Final old_old: 46.65 ± 4.23   Final old_new: 28.19 ± 3.16   Final new_new: 6.82 ± 4.25
Hits@30:
  Final val: 69.12 ± 1.18   Final Test: 50.56 ± 4.69   Final old_old: 54.56 ± 4.93   Final old_new: 34.74 ± 3.59   Final new_new: 10.00 ± 6.55
Hits@50:
  Final val: 76.73 ± 1.30   Final Test: 62.33 ± 2.37   Final old_old: 66.49 ± 2.67   Final old_new: 45.97 ± 1.73   Final new_new: 17.73 ± 5.66
AUC:
  Final val: 97.41 ± 0.08   Final Test: 96.95 ± 0.12   Final old_old: 97.54 ± 0.16   Final old_new: 94.66 ± 0.21   Final new_new: 90.26 ± 1.63
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.0, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.26 ± 1.69   Final Test: 30.72 ± 4.00   Final old_old: 33.84 ± 4.25   Final old_new: 18.11 ± 3.19   Final new_new: 3.18 ± 1.24
Hits@20:
  Final val: 59.21 ± 1.95   Final Test: 38.86 ± 3.36   Final old_old: 42.20 ± 3.49   Final old_new: 25.49 ± 3.07   Final new_new: 7.27 ± 5.43
Hits@30:
  Final val: 65.18 ± 1.07   Final Test: 46.19 ± 3.47   Final old_old: 49.84 ± 3.56   Final old_new: 31.66 ± 3.42   Final new_new: 10.00 ± 4.71
Hits@50:
  Final val: 73.17 ± 1.44   Final Test: 56.54 ± 2.09   Final old_old: 60.52 ± 2.51   Final old_new: 41.01 ± 1.09   Final new_new: 11.82 ± 2.96
AUC:
  Final val: 97.20 ± 0.13   Final Test: 96.69 ± 0.11   Final old_old: 97.29 ± 0.15   Final old_new: 94.34 ± 0.27   Final new_new: 90.00 ± 0.92
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.85 ± 1.92   Final Test: 32.31 ± 4.90   Final old_old: 35.40 ± 5.15   Final old_new: 19.77 ± 4.07   Final new_new: 6.82 ± 1.61
Hits@20:
  Final val: 59.03 ± 1.81   Final Test: 38.47 ± 5.54   Final old_old: 41.79 ± 5.88   Final old_new: 25.16 ± 4.45   Final new_new: 6.82 ± 1.61
Hits@30:
  Final val: 66.96 ± 1.75   Final Test: 45.77 ± 5.05   Final old_old: 49.50 ± 5.34   Final old_new: 30.86 ± 4.18   Final new_new: 10.45 ± 2.59
Hits@50:
  Final val: 74.34 ± 1.43   Final Test: 59.49 ± 2.48   Final old_old: 63.61 ± 2.57   Final old_new: 43.17 ± 2.27   Final new_new: 16.82 ± 3.45
AUC:
  Final val: 97.25 ± 0.06   Final Test: 96.72 ± 0.10   Final old_old: 97.31 ± 0.11   Final old_new: 94.41 ± 0.22   Final new_new: 90.29 ± 0.73
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.81 ± 1.70   Final Test: 34.81 ± 4.62   Final old_old: 38.17 ± 4.83   Final old_new: 21.18 ± 4.08   Final new_new: 6.36 ± 3.37
Hits@20:
  Final val: 59.17 ± 0.88   Final Test: 41.01 ± 3.10   Final old_old: 44.54 ± 3.18   Final old_new: 26.85 ± 3.55   Final new_new: 8.18 ± 3.05
Hits@30:
  Final val: 65.77 ± 1.55   Final Test: 47.93 ± 1.89   Final old_old: 51.77 ± 2.01   Final old_new: 32.57 ± 1.67   Final new_new: 11.36 ± 4.25
Hits@50:
  Final val: 73.95 ± 0.95   Final Test: 58.37 ± 3.18   Final old_old: 62.64 ± 3.48   Final old_new: 41.44 ± 2.85   Final new_new: 15.00 ± 2.03
AUC:
  Final val: 97.25 ± 0.10   Final Test: 96.73 ± 0.04   Final old_old: 97.33 ± 0.09   Final old_new: 94.40 ± 0.25   Final new_new: 89.78 ± 1.21
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(distill_teacher='GCN', distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.0, link_batch_size=65536, node_batch_size=65536, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=0.0001, KD_RM=0.0, KD_LM=0.0, LLP_D=0.01, LLP_R=0.001, margin=0.2, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.96 ± 0.60   Final Test: 33.28 ± 5.30   Final old_old: 36.67 ± 5.71   Final old_new: 19.65 ± 3.83   Final new_new: 2.73 ± 1.02
Hits@20:
  Final val: 62.95 ± 2.54   Final Test: 42.54 ± 5.84   Final old_old: 46.20 ± 6.04   Final old_new: 27.96 ± 5.39   Final new_new: 6.82 ± 2.27
Hits@30:
  Final val: 69.00 ± 1.37   Final Test: 51.38 ± 3.75   Final old_old: 55.45 ± 4.00   Final old_new: 35.31 ± 3.23   Final new_new: 9.09 ± 2.78
Hits@50:
  Final val: 76.99 ± 0.77   Final Test: 62.22 ± 3.24   Final old_old: 66.39 ± 3.25   Final old_new: 45.94 ± 3.56   Final new_new: 15.00 ± 3.80
AUC:
  Final val: 97.39 ± 0.08   Final Test: 96.83 ± 0.06   Final old_old: 97.41 ± 0.06   Final old_new: 94.55 ± 0.33   Final new_new: 90.69 ± 0.30
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.18 ± 3.63   Final Test: 31.42 ± 7.04   Final old_old: 34.65 ± 7.43   Final old_new: 18.34 ± 5.94   Final new_new: 4.09 ± 2.49
Hits@20:
  Final val: 62.27 ± 2.03   Final Test: 42.59 ± 1.20   Final old_old: 46.37 ± 1.07   Final old_new: 27.38 ± 2.31   Final new_new: 8.18 ± 2.03
Hits@30:
  Final val: 69.55 ± 1.02   Final Test: 51.07 ± 2.09   Final old_old: 55.25 ± 2.33   Final old_new: 34.38 ± 1.40   Final new_new: 10.45 ± 3.45
Hits@50:
  Final val: 76.14 ± 0.81   Final Test: 62.00 ± 1.08   Final old_old: 66.19 ± 1.20   Final old_new: 45.52 ± 1.37   Final new_new: 17.73 ± 4.93
AUC:
  Final val: 97.41 ± 0.09   Final Test: 96.82 ± 0.05   Final old_old: 97.45 ± 0.11   Final old_new: 94.28 ± 0.26   Final new_new: 90.46 ± 1.30
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.61 ± 2.98   Final Test: 34.47 ± 3.70   Final old_old: 37.80 ± 3.96   Final old_new: 21.13 ± 2.86   Final new_new: 3.64 ± 2.03
Hits@20:
  Final val: 63.58 ± 2.28   Final Test: 46.18 ± 4.22   Final old_old: 50.22 ± 4.66   Final old_new: 30.05 ± 2.47   Final new_new: 7.73 ± 2.59
Hits@30:
  Final val: 70.27 ± 2.22   Final Test: 53.94 ± 3.22   Final old_old: 58.22 ± 3.32   Final old_new: 36.93 ± 2.92   Final new_new: 11.36 ± 3.21
Hits@50:
  Final val: 77.38 ± 1.64   Final Test: 62.84 ± 2.69   Final old_old: 67.02 ± 2.72   Final old_new: 46.40 ± 2.86   Final new_new: 18.18 ± 3.59
AUC:
  Final val: 97.41 ± 0.12   Final Test: 96.90 ± 0.15   Final old_old: 97.52 ± 0.16   Final old_new: 94.48 ± 0.19   Final new_new: 90.37 ± 0.73
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 57.90 ± 3.02   Final Test: 30.25 ± 4.51   Final old_old: 32.90 ± 5.05   Final old_new: 19.65 ± 2.52   Final new_new: 5.00 ± 1.90
Hits@20:
  Final val: 65.59 ± 2.97   Final Test: 46.39 ± 3.68   Final old_old: 49.92 ± 3.62   Final old_new: 32.32 ± 4.06   Final new_new: 12.27 ± 5.70
Hits@30:
  Final val: 70.22 ± 2.80   Final Test: 57.33 ± 2.69   Final old_old: 61.06 ± 2.66   Final old_new: 42.57 ± 3.60   Final new_new: 19.09 ± 3.45
Hits@50:
  Final val: 76.11 ± 2.19   Final Test: 64.93 ± 1.44   Final old_old: 68.53 ± 1.66   Final old_new: 50.88 ± 0.76   Final new_new: 24.55 ± 1.90
AUC:
  Final val: 96.91 ± 0.21   Final Test: 96.25 ± 0.14   Final old_old: 96.98 ± 0.06   Final old_new: 93.52 ± 0.48   Final new_new: 85.82 ± 2.07
All runs:
Hits@10:
  Final val: 43.73 ± 1.89   Final Test: 31.20 ± 3.78   Final old_old: 34.26 ± 4.02   Final old_new: 18.89 ± 2.96   Final new_new: 3.64 ± 2.03
Hits@20:
  Final val: 57.31 ± 3.54   Final Test: 37.54 ± 2.52   Final old_old: 40.77 ± 2.77   Final old_new: 24.53 ± 1.72   Final new_new: 9.09 ± 2.27
Hits@30:
  Final val: 65.33 ± 2.35   Final Test: 45.48 ± 2.37   Final old_old: 48.97 ± 2.69   Final old_new: 31.59 ± 1.93   Final new_new: 10.91 ± 2.49
Hits@50:
  Final val: 72.87 ± 1.45   Final Test: 56.20 ± 2.98   Final old_old: 60.20 ± 3.20   Final old_new: 40.33 ± 2.69   Final new_new: 15.91 ± 2.78
AUC:
  Final val: 97.13 ± 0.07   Final Test: 96.62 ± 0.14   Final old_old: 97.18 ± 0.12   Final old_new: 94.41 ± 0.26   Final new_new: 90.76 ± 0.49
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.95 ± 2.16   Final Test: 30.53 ± 4.61   Final old_old: 33.57 ± 4.79   Final old_new: 18.29 ± 4.21   Final new_new: 4.09 ± 1.02
Hits@20:
  Final val: 58.84 ± 3.02   Final Test: 40.80 ± 1.42   Final old_old: 44.26 ± 1.70   Final old_new: 26.95 ± 1.75   Final new_new: 7.73 ± 3.80
Hits@30:
  Final val: 65.57 ± 2.97   Final Test: 47.24 ± 2.30   Final old_old: 50.97 ± 2.55   Final old_new: 32.37 ± 1.46   Final new_new: 10.91 ± 2.96
Hits@50:
  Final val: 73.72 ± 1.80   Final Test: 55.93 ± 2.29   Final old_old: 60.05 ± 2.57   Final old_new: 39.62 ± 2.25   Final new_new: 14.09 ± 1.90
AUC:
  Final val: 97.24 ± 0.16   Final Test: 96.69 ± 0.17   Final old_old: 97.29 ± 0.20   Final old_new: 94.30 ± 0.33   Final new_new: 90.25 ± 0.90
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.0, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.33 ± 1.79   Final Test: 29.53 ± 6.27   Final old_old: 32.56 ± 6.65   Final old_new: 17.10 ± 4.93   Final new_new: 5.91 ± 2.59
Hits@20:
  Final val: 58.82 ± 1.90   Final Test: 40.65 ± 2.57   Final old_old: 44.16 ± 2.71   Final old_new: 26.57 ± 2.42   Final new_new: 7.73 ± 3.80
Hits@30:
  Final val: 66.03 ± 1.22   Final Test: 49.34 ± 3.18   Final old_old: 53.36 ± 3.29   Final old_new: 33.35 ± 2.85   Final new_new: 10.00 ± 5.23
Hits@50:
  Final val: 74.03 ± 0.79   Final Test: 58.49 ± 3.78   Final old_old: 62.64 ± 3.56   Final old_new: 42.07 ± 4.92   Final new_new: 16.36 ± 6.31
AUC:
  Final val: 97.27 ± 0.09   Final Test: 96.75 ± 0.08   Final old_old: 97.35 ± 0.07   Final old_new: 94.40 ± 0.27   Final new_new: 90.15 ± 1.29
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=65536, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=65536, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=65536, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=65536, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.00 ± 5.70   Final Test: 28.79 ± 4.28   Final old_old: 31.27 ± 4.80   Final old_new: 18.82 ± 2.57   Final new_new: 5.91 ± 4.71
Hits@20:
  Final val: 54.02 ± 3.49   Final Test: 35.30 ± 2.50   Final old_old: 38.27 ± 2.83   Final old_new: 23.22 ± 1.29   Final new_new: 10.45 ± 3.80
Hits@30:
  Final val: 61.48 ± 2.90   Final Test: 46.60 ± 5.34   Final old_old: 50.09 ± 5.77   Final old_new: 32.67 ± 3.95   Final new_new: 12.73 ± 3.80
Hits@50:
  Final val: 71.91 ± 2.38   Final Test: 55.94 ± 3.27   Final old_old: 59.76 ± 3.52   Final old_new: 41.01 ± 2.90   Final new_new: 13.18 ± 6.89
AUC:
  Final val: 96.63 ± 0.30   Final Test: 95.98 ± 0.28   Final old_old: 96.62 ± 0.31   Final old_new: 93.44 ± 0.27   Final new_new: 89.88 ± 0.80
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.11 ± 4.22   Final Test: 32.29 ± 6.58   Final old_old: 34.86 ± 6.89   Final old_new: 21.94 ± 5.52   Final new_new: 9.09 ± 3.59
Hits@20:
  Final val: 54.46 ± 4.09   Final Test: 38.38 ± 3.33   Final old_old: 41.38 ± 3.45   Final old_new: 26.45 ± 3.15   Final new_new: 9.09 ± 3.94
Hits@30:
  Final val: 63.41 ± 3.23   Final Test: 46.69 ± 3.91   Final old_old: 50.04 ± 4.01   Final old_new: 33.50 ± 3.69   Final new_new: 11.82 ± 3.37
Hits@50:
  Final val: 72.49 ± 1.91   Final Test: 59.26 ± 2.78   Final old_old: 62.84 ± 2.94   Final old_new: 45.39 ± 2.76   Final new_new: 17.73 ± 5.43
AUC:
  Final val: 96.64 ± 0.21   Final Test: 96.01 ± 0.27   Final old_old: 96.65 ± 0.30   Final old_new: 93.50 ± 0.22   Final new_new: 89.20 ± 1.95
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.03 ± 6.17   Final Test: 27.08 ± 5.09   Final old_old: 29.55 ± 5.64   Final old_new: 16.95 ± 3.24   Final new_new: 8.64 ± 6.31
Hits@20:
  Final val: 54.61 ± 6.47   Final Test: 39.05 ± 4.29   Final old_old: 42.12 ± 4.76   Final old_new: 26.65 ± 2.78   Final new_new: 11.82 ± 6.10
Hits@30:
  Final val: 61.73 ± 5.11   Final Test: 43.88 ± 4.71   Final old_old: 47.29 ± 5.40   Final old_new: 30.18 ± 2.21   Final new_new: 12.73 ± 6.74
Hits@50:
  Final val: 71.36 ± 3.52   Final Test: 56.34 ± 2.80   Final old_old: 60.11 ± 3.26   Final old_new: 41.49 ± 1.15   Final new_new: 16.82 ± 6.93
AUC:
  Final val: 96.64 ± 0.26   Final Test: 96.07 ± 0.23   Final old_old: 96.75 ± 0.27   Final old_new: 93.40 ± 0.25   Final new_new: 89.29 ± 1.44
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.83 ± 3.69   Final Test: 30.88 ± 5.52   Final old_old: 33.65 ± 5.80   Final old_new: 19.52 ± 4.47   Final new_new: 9.55 ± 4.93
Hits@20:
  Final val: 55.20 ± 2.88   Final Test: 40.37 ± 4.84   Final old_old: 43.68 ± 5.09   Final old_new: 27.10 ± 3.93   Final new_new: 10.00 ± 4.71
Hits@30:
  Final val: 62.37 ± 2.61   Final Test: 45.99 ± 4.02   Final old_old: 49.48 ± 4.30   Final old_new: 32.04 ± 3.13   Final new_new: 12.27 ± 6.93
Hits@50:
  Final val: 71.00 ± 1.75   Final Test: 57.37 ± 4.12   Final old_old: 61.24 ± 4.25   Final old_new: 42.07 ± 4.20   Final new_new: 17.73 ± 6.31
AUC:
  Final val: 96.58 ± 0.19   Final Test: 95.96 ± 0.20   Final old_old: 96.64 ± 0.19   Final old_new: 93.27 ± 0.38   Final new_new: 89.38 ± 1.98
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(distill_teacher='SAGE', distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.0, link_batch_size=65536, node_batch_size=65536, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=0.0001, KD_RM=0.0, KD_LM=0.0, LLP_D=0.01, LLP_R=0.001, margin=0.2, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.48 ± 4.38   Final Test: 32.71 ± 4.10   Final old_old: 35.98 ± 4.39   Final old_new: 19.57 ± 3.06   Final new_new: 3.18 ± 1.24
Hits@20:
  Final val: 62.82 ± 2.94   Final Test: 40.98 ± 4.47   Final old_old: 44.71 ± 4.33   Final old_new: 26.02 ± 5.35   Final new_new: 5.91 ± 4.13
Hits@30:
  Final val: 69.61 ± 2.28   Final Test: 49.87 ± 3.01   Final old_old: 53.98 ± 3.24   Final old_new: 33.75 ± 2.43   Final new_new: 5.00 ± 2.49
Hits@50:
  Final val: 76.56 ± 1.36   Final Test: 60.55 ± 1.20   Final old_old: 64.90 ± 1.03   Final old_new: 43.53 ± 2.49   Final new_new: 12.73 ± 3.45
AUC:
  Final val: 97.41 ± 0.11   Final Test: 96.84 ± 0.09   Final old_old: 97.49 ± 0.11   Final old_new: 94.25 ± 0.22   Final new_new: 90.59 ± 0.34
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.75 ± 4.80   Final Test: 34.27 ± 5.72   Final old_old: 37.71 ± 6.21   Final old_new: 20.40 ± 3.96   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 61.10 ± 2.78   Final Test: 41.03 ± 2.45   Final old_old: 44.71 ± 2.67   Final old_new: 26.27 ± 2.75   Final new_new: 6.36 ± 1.90
Hits@30:
  Final val: 68.38 ± 1.81   Final Test: 50.17 ± 4.61   Final old_old: 54.24 ± 4.89   Final old_new: 33.98 ± 3.66   Final new_new: 10.45 ± 4.98
Hits@50:
  Final val: 76.49 ± 1.55   Final Test: 60.28 ± 2.25   Final old_old: 64.61 ± 2.46   Final old_new: 43.30 ± 2.16   Final new_new: 13.64 ± 4.25
AUC:
  Final val: 97.39 ± 0.20   Final Test: 96.75 ± 0.14   Final old_old: 97.39 ± 0.19   Final old_new: 94.23 ± 0.19   Final new_new: 90.54 ± 0.99
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.04 ± 3.15   Final Test: 32.48 ± 5.16   Final old_old: 35.71 ± 5.42   Final old_new: 19.45 ± 4.33   Final new_new: 4.55 ± 2.27
Hits@20:
  Final val: 61.59 ± 3.09   Final Test: 43.89 ± 3.23   Final old_old: 47.72 ± 3.49   Final old_new: 28.54 ± 2.60   Final new_new: 8.18 ± 2.59
Hits@30:
  Final val: 68.45 ± 2.43   Final Test: 51.91 ± 3.77   Final old_old: 56.10 ± 3.98   Final old_new: 35.24 ± 2.94   Final new_new: 10.00 ± 3.05
Hits@50:
  Final val: 76.17 ± 1.76   Final Test: 60.67 ± 2.19   Final old_old: 64.95 ± 2.01   Final old_new: 43.73 ± 3.20   Final new_new: 16.82 ± 4.13
AUC:
  Final val: 97.35 ± 0.09   Final Test: 96.84 ± 0.12   Final old_old: 97.47 ± 0.12   Final old_new: 94.34 ± 0.13   Final new_new: 90.60 ± 0.58
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.97 ± 5.12   Final Test: 33.17 ± 4.41   Final old_old: 36.58 ± 5.29   Final old_new: 19.35 ± 1.06   Final new_new: 4.09 ± 4.07
Hits@20:
  Final val: 60.79 ± 4.84   Final Test: 43.88 ± 6.23   Final old_old: 47.84 ± 7.04   Final old_new: 28.04 ± 3.08   Final new_new: 6.82 ± 3.59
Hits@30:
  Final val: 67.15 ± 3.34   Final Test: 54.20 ± 2.93   Final old_old: 58.85 ± 3.54   Final old_new: 35.57 ± 0.96   Final new_new: 10.91 ± 5.18
Hits@50:
  Final val: 75.00 ± 2.93   Final Test: 61.75 ± 2.70   Final old_old: 66.71 ± 3.24   Final old_new: 41.91 ± 1.04   Final new_new: 15.00 ± 4.43
AUC:
  Final val: 97.06 ± 0.44   Final Test: 95.96 ± 0.34   Final old_old: 97.45 ± 0.46   Final old_new: 90.08 ± 1.26   Final new_new: 81.07 ± 3.98
All runs:
Hits@10:
  Final val: 51.25 ± 2.94   Final Test: 30.79 ± 6.41   Final old_old: 33.82 ± 6.82   Final old_new: 18.59 ± 5.04   Final new_new: 4.09 ± 1.02
Hits@20:
  Final val: 62.93 ± 2.35   Final Test: 42.78 ± 2.76   Final old_old: 46.44 ± 2.79   Final old_new: 28.24 ± 2.96   Final new_new: 6.82 ± 1.61
Hits@30:
  Final val: 69.28 ± 1.15   Final Test: 50.26 ± 3.93   Final old_old: 54.38 ± 4.05   Final old_new: 33.98 ± 3.56   Final new_new: 7.73 ± 3.45
Hits@50:
  Final val: 77.92 ± 1.53   Final Test: 59.74 ± 1.30   Final old_old: 64.15 ± 1.38   Final old_new: 42.32 ± 1.22   Final new_new: 14.55 ± 3.45
AUC:
  Final val: 97.46 ± 0.11   Final Test: 96.83 ± 0.11   Final old_old: 97.48 ± 0.11   Final old_new: 94.27 ± 0.29   Final new_new: 90.51 ± 0.66
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=65536, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.50 ± 2.27   Final Test: 31.21 ± 5.62   Final old_old: 34.24 ± 5.82   Final old_new: 18.89 ± 4.98   Final new_new: 5.45 ± 2.59
Hits@20:
  Final val: 59.62 ± 1.91   Final Test: 40.04 ± 2.79   Final old_old: 43.46 ± 2.82   Final old_new: 26.27 ± 2.89   Final new_new: 8.64 ± 2.96
Hits@30:
  Final val: 65.82 ± 1.71   Final Test: 48.06 ± 3.12   Final old_old: 51.92 ± 3.52   Final old_new: 32.72 ± 1.81   Final new_new: 9.55 ± 2.49
Hits@50:
  Final val: 73.90 ± 1.20   Final Test: 57.82 ± 4.15   Final old_old: 61.97 ± 4.24   Final old_new: 41.34 ± 4.00   Final new_new: 16.36 ± 2.96
AUC:
  Final val: 97.27 ± 0.09   Final Test: 96.73 ± 0.08   Final old_old: 97.30 ± 0.07   Final old_new: 94.49 ± 0.21   Final new_new: 90.77 ± 1.15
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.95 ± 2.51   Final Test: 31.99 ± 2.86   Final old_old: 35.08 ± 3.13   Final old_new: 19.57 ± 1.81   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 61.37 ± 1.59   Final Test: 41.72 ± 5.28   Final old_old: 45.41 ± 5.65   Final old_new: 26.93 ± 3.80   Final new_new: 7.27 ± 3.37
Hits@30:
  Final val: 67.93 ± 1.59   Final Test: 49.74 ± 2.06   Final old_old: 53.81 ± 2.28   Final old_new: 33.48 ± 1.57   Final new_new: 10.45 ± 2.59
Hits@50:
  Final val: 75.10 ± 1.37   Final Test: 59.04 ± 1.55   Final old_old: 63.28 ± 1.38   Final old_new: 42.22 ± 2.52   Final new_new: 15.91 ± 1.61
AUC:
  Final val: 97.34 ± 0.06   Final Test: 96.81 ± 0.11   Final old_old: 97.41 ± 0.11   Final old_new: 94.47 ± 0.17   Final new_new: 90.07 ± 1.44
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.92 ± 3.96   Final Test: 30.39 ± 2.05   Final old_old: 33.51 ± 2.01   Final old_new: 17.78 ± 2.39   Final new_new: 3.64 ± 2.03
Hits@20:
  Final val: 60.97 ± 3.88   Final Test: 42.62 ± 2.47   Final old_old: 46.37 ± 2.75   Final old_new: 27.71 ± 1.84   Final new_new: 5.45 ± 2.59
Hits@30:
  Final val: 67.47 ± 3.14   Final Test: 48.49 ± 3.40   Final old_old: 52.32 ± 3.54   Final old_new: 33.30 ± 3.01   Final new_new: 10.00 ± 4.13
Hits@50:
  Final val: 74.69 ± 2.06   Final Test: 58.48 ± 2.66   Final old_old: 62.66 ± 2.63   Final old_new: 41.99 ± 3.11   Final new_new: 15.45 ± 4.07
AUC:
  Final val: 97.31 ± 0.18   Final Test: 96.80 ± 0.11   Final old_old: 97.37 ± 0.16   Final old_new: 94.55 ± 0.09   Final new_new: 90.92 ± 1.28
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=65536, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.64 ± 3.17   Final Test: 32.38 ± 4.82   Final old_old: 35.62 ± 5.24   Final old_new: 19.17 ± 3.36   Final new_new: 6.36 ± 1.90
Hits@20:
  Final val: 60.35 ± 1.01   Final Test: 42.51 ± 3.28   Final old_old: 46.19 ± 3.37   Final old_new: 27.81 ± 3.25   Final new_new: 7.73 ± 2.59
Hits@30:
  Final val: 67.13 ± 1.34   Final Test: 50.80 ± 2.00   Final old_old: 54.80 ± 2.10   Final old_new: 34.91 ± 1.87   Final new_new: 10.91 ± 3.37
Hits@50:
  Final val: 75.07 ± 1.20   Final Test: 57.87 ± 1.57   Final old_old: 62.25 ± 1.68   Final old_new: 40.38 ± 1.53   Final new_new: 15.45 ± 2.96
AUC:
  Final val: 97.30 ± 0.07   Final Test: 96.78 ± 0.12   Final old_old: 97.38 ± 0.10   Final old_new: 94.43 ± 0.22   Final new_new: 90.69 ± 0.98
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.81 ± 2.16   Final Test: 25.17 ± 4.02   Final old_old: 27.60 ± 4.37   Final old_new: 15.31 ± 2.65   Final new_new: 4.55 ± 3.21
Hits@20:
  Final val: 56.38 ± 1.76   Final Test: 39.91 ± 4.45   Final old_old: 43.18 ± 4.55   Final old_new: 26.75 ± 4.34   Final new_new: 10.00 ± 6.55
Hits@30:
  Final val: 63.79 ± 2.69   Final Test: 47.12 ± 2.18   Final old_old: 50.57 ± 2.15   Final old_new: 33.35 ± 2.74   Final new_new: 14.09 ± 4.07
Hits@50:
  Final val: 72.14 ± 1.57   Final Test: 56.19 ± 2.85   Final old_old: 60.01 ± 2.91   Final old_new: 41.26 ± 3.09   Final new_new: 14.55 ± 5.70
AUC:
  Final val: 96.64 ± 0.19   Final Test: 95.97 ± 0.22   Final old_old: 96.67 ± 0.20   Final old_new: 93.26 ± 0.40   Final new_new: 88.19 ± 2.27
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.44 ± 2.83   Final Test: 29.69 ± 3.50   Final old_old: 32.37 ± 3.95   Final old_new: 18.89 ± 1.84   Final new_new: 5.45 ± 3.45
Hits@20:
  Final val: 54.75 ± 2.02   Final Test: 41.40 ± 2.00   Final old_old: 44.63 ± 2.34   Final old_new: 28.59 ± 0.98   Final new_new: 8.64 ± 2.96
Hits@30:
  Final val: 63.44 ± 2.19   Final Test: 47.69 ± 3.58   Final old_old: 51.29 ± 4.09   Final old_new: 33.48 ± 2.05   Final new_new: 10.45 ± 5.23
Hits@50:
  Final val: 72.14 ± 1.58   Final Test: 56.56 ± 5.03   Final old_old: 60.46 ± 5.45   Final old_new: 41.21 ± 3.60   Final new_new: 15.45 ± 4.37
AUC:
  Final val: 96.65 ± 0.21   Final Test: 95.93 ± 0.20   Final old_old: 96.58 ± 0.21   Final old_new: 93.35 ± 0.25   Final new_new: 88.82 ± 2.68
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.93 ± 4.60   Final Test: 27.09 ± 6.60   Final old_old: 29.48 ± 7.22   Final old_new: 17.38 ± 4.13   Final new_new: 7.27 ± 4.66
Hits@20:
  Final val: 53.91 ± 4.50   Final Test: 34.73 ± 3.62   Final old_old: 37.52 ± 4.02   Final old_new: 23.45 ± 2.16   Final new_new: 10.45 ± 5.70
Hits@30:
  Final val: 60.87 ± 3.62   Final Test: 43.54 ± 1.99   Final old_old: 46.97 ± 1.96   Final old_new: 29.62 ± 2.39   Final new_new: 14.09 ± 3.73
Hits@50:
  Final val: 71.81 ± 1.73   Final Test: 54.92 ± 3.54   Final old_old: 58.65 ± 3.67   Final old_new: 39.92 ± 3.45   Final new_new: 20.91 ± 5.18
AUC:
  Final val: 96.64 ± 0.29   Final Test: 95.98 ± 0.21   Final old_old: 96.69 ± 0.30   Final old_new: 93.16 ± 0.33   Final new_new: 89.55 ± 1.28
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.29 ± 4.86   Final Test: 28.67 ± 7.42   Final old_old: 31.28 ± 8.04   Final old_new: 18.04 ± 5.14   Final new_new: 7.73 ± 4.71
Hits@20:
  Final val: 54.43 ± 2.96   Final Test: 38.86 ± 2.67   Final old_old: 42.01 ± 3.14   Final old_new: 26.10 ± 0.93   Final new_new: 11.82 ± 4.07
Hits@30:
  Final val: 61.92 ± 1.43   Final Test: 45.01 ± 4.32   Final old_old: 48.49 ± 4.83   Final old_new: 31.03 ± 2.15   Final new_new: 12.73 ± 3.45
Hits@50:
  Final val: 70.69 ± 1.41   Final Test: 54.21 ± 3.99   Final old_old: 58.10 ± 4.33   Final old_new: 38.82 ± 3.19   Final new_new: 15.00 ± 6.14
AUC:
  Final val: 96.58 ± 0.18   Final Test: 95.93 ± 0.15   Final old_old: 96.60 ± 0.19   Final old_new: 93.26 ± 0.23   Final new_new: 89.66 ± 1.35
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.57 ± 4.90   Final Test: 26.80 ± 6.08   Final old_old: 29.28 ± 6.76   Final old_new: 16.75 ± 3.54   Final new_new: 5.91 ± 2.59
Hits@20:
  Final val: 55.04 ± 4.61   Final Test: 37.82 ± 2.76   Final old_old: 41.04 ± 2.97   Final old_new: 24.84 ± 2.12   Final new_new: 8.64 ± 2.96
Hits@30:
  Final val: 63.39 ± 4.06   Final Test: 46.54 ± 5.75   Final old_old: 50.02 ± 5.83   Final old_new: 32.72 ± 6.01   Final new_new: 12.27 ± 1.24
Hits@50:
  Final val: 72.74 ± 2.20   Final Test: 54.92 ± 4.92   Final old_old: 58.60 ± 5.28   Final old_new: 40.38 ± 3.80   Final new_new: 17.27 ± 3.45
AUC:
  Final val: 96.67 ± 0.31   Final Test: 95.92 ± 0.25   Final old_old: 96.61 ± 0.22   Final old_new: 93.23 ± 0.54   Final new_new: 88.61 ± 1.85
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 40.93 ± 0.78   Final Test: 30.39 ± 3.58   Final old_old: 33.09 ± 3.83   Final old_new: 19.40 ± 2.68   Final new_new: 7.73 ± 5.23
Hits@20:
  Final val: 54.84 ± 2.54   Final Test: 37.74 ± 3.48   Final old_old: 40.95 ± 3.84   Final old_new: 24.79 ± 2.41   Final new_new: 9.55 ± 3.37
Hits@30:
  Final val: 62.90 ± 2.17   Final Test: 41.85 ± 1.53   Final old_old: 45.09 ± 1.94   Final old_new: 28.92 ± 1.02   Final new_new: 10.45 ± 5.23
Hits@50:
  Final val: 71.89 ± 1.80   Final Test: 55.73 ± 2.33   Final old_old: 59.69 ± 2.40   Final old_new: 40.05 ± 2.92   Final new_new: 15.45 ± 4.93
AUC:
  Final val: 96.65 ± 0.19   Final Test: 96.02 ± 0.22   Final old_old: 96.67 ± 0.25   Final old_new: 93.42 ± 0.45   Final new_new: 89.43 ± 1.30
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.76 ± 1.86   Final Test: 35.85 ± 1.28   Final old_old: 39.16 ± 1.36   Final old_new: 22.62 ± 0.97   Final new_new: 4.09 ± 2.96
Hits@20:
  Final val: 62.09 ± 2.53   Final Test: 43.18 ± 1.64   Final old_old: 46.90 ± 1.94   Final old_new: 28.39 ± 0.74   Final new_new: 6.82 ± 5.33
Hits@30:
  Final val: 68.68 ± 2.11   Final Test: 50.65 ± 1.93   Final old_old: 54.58 ± 1.97   Final old_new: 35.26 ± 2.19   Final new_new: 6.82 ± 4.25
Hits@50:
  Final val: 76.28 ± 1.74   Final Test: 61.07 ± 2.95   Final old_old: 65.31 ± 3.22   Final old_new: 44.46 ± 2.32   Final new_new: 14.55 ± 4.98
AUC:
  Final val: 97.35 ± 0.16   Final Test: 96.82 ± 0.16   Final old_old: 97.45 ± 0.19   Final old_new: 94.29 ± 0.11   Final new_new: 90.60 ± 0.23
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.10 ± 1.42   Final Test: 36.05 ± 3.30   Final old_old: 39.45 ± 3.31   Final old_new: 22.39 ± 3.70   Final new_new: 4.55 ± 2.27
Hits@20:
  Final val: 60.23 ± 2.30   Final Test: 44.43 ± 3.92   Final old_old: 48.09 ± 4.05   Final old_new: 30.00 ± 3.56   Final new_new: 6.36 ± 2.96
Hits@30:
  Final val: 67.80 ± 1.64   Final Test: 50.45 ± 3.23   Final old_old: 54.43 ± 3.35   Final old_new: 34.66 ± 2.95   Final new_new: 10.00 ± 3.05
Hits@50:
  Final val: 76.25 ± 1.30   Final Test: 60.81 ± 1.51   Final old_old: 65.00 ± 1.53   Final old_new: 44.38 ± 1.94   Final new_new: 15.00 ± 4.13
AUC:
  Final val: 97.38 ± 0.15   Final Test: 96.83 ± 0.07   Final old_old: 97.46 ± 0.08   Final old_new: 94.35 ± 0.22   Final new_new: 89.87 ± 0.93
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.55 ± 3.64   Final Test: 28.06 ± 3.14   Final old_old: 31.00 ± 3.41   Final old_new: 16.22 ± 2.39   Final new_new: 1.82 ± 1.02
Hits@20:
  Final val: 60.86 ± 2.88   Final Test: 40.25 ± 4.40   Final old_old: 43.75 ± 4.66   Final old_new: 26.22 ± 3.61   Final new_new: 7.27 ± 4.07
Hits@30:
  Final val: 68.16 ± 2.38   Final Test: 50.41 ± 3.90   Final old_old: 54.36 ± 3.95   Final old_new: 34.74 ± 3.98   Final new_new: 11.36 ± 5.08
Hits@50:
  Final val: 76.37 ± 2.22   Final Test: 60.91 ± 3.52   Final old_old: 65.17 ± 3.63   Final old_new: 44.11 ± 3.14   Final new_new: 16.36 ± 6.51
AUC:
  Final val: 97.35 ± 0.18   Final Test: 96.77 ± 0.17   Final old_old: 97.39 ± 0.21   Final old_new: 94.28 ± 0.19   Final new_new: 90.97 ± 1.06
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.01 ± 3.46   Final Test: 31.27 ± 2.95   Final old_old: 34.36 ± 3.08   Final old_new: 18.82 ± 2.61   Final new_new: 3.64 ± 2.03
Hits@20:
  Final val: 60.49 ± 1.68   Final Test: 40.70 ± 5.11   Final old_old: 44.29 ± 5.34   Final old_new: 26.42 ± 4.23   Final new_new: 5.91 ± 5.93
Hits@30:
  Final val: 66.53 ± 1.71   Final Test: 49.63 ± 3.91   Final old_old: 53.41 ± 4.08   Final old_new: 34.63 ± 3.45   Final new_new: 10.91 ± 5.18
Hits@50:
  Final val: 75.68 ± 1.36   Final Test: 60.14 ± 2.60   Final old_old: 64.25 ± 2.85   Final old_new: 43.90 ± 1.79   Final new_new: 16.82 ± 3.45
AUC:
  Final val: 97.32 ± 0.14   Final Test: 96.78 ± 0.19   Final old_old: 97.40 ± 0.21   Final old_new: 94.30 ± 0.26   Final new_new: 91.00 ± 0.84
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.06 ± 3.74   Final Test: 31.60 ± 4.52   Final old_old: 34.72 ± 4.84   Final old_new: 18.99 ± 3.34   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 62.21 ± 1.00   Final Test: 44.32 ± 2.09   Final old_old: 48.08 ± 2.19   Final old_new: 29.32 ± 1.85   Final new_new: 7.27 ± 1.90
Hits@30:
  Final val: 68.71 ± 1.26   Final Test: 53.23 ± 2.41   Final old_old: 57.42 ± 2.53   Final old_new: 36.57 ± 2.06   Final new_new: 11.36 ± 4.82
Hits@50:
  Final val: 76.40 ± 1.80   Final Test: 62.18 ± 2.28   Final old_old: 66.41 ± 2.30   Final old_new: 45.47 ± 2.50   Final new_new: 18.64 ± 5.18
AUC:
  Final val: 97.35 ± 0.16   Final Test: 96.80 ± 0.08   Final old_old: 97.44 ± 0.11   Final old_new: 94.24 ± 0.23   Final new_new: 90.09 ± 1.05
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.18 ± 2.11   Final Test: 32.37 ± 0.95   Final old_old: 35.46 ± 0.87   Final old_new: 20.00 ± 1.51   Final new_new: 3.64 ± 2.59
Hits@20:
  Final val: 62.57 ± 1.68   Final Test: 44.52 ± 4.41   Final old_old: 48.33 ± 4.76   Final old_new: 29.37 ± 3.36   Final new_new: 6.82 ± 4.25
Hits@30:
  Final val: 68.56 ± 1.34   Final Test: 51.01 ± 3.60   Final old_old: 54.92 ± 3.79   Final old_new: 35.62 ± 3.38   Final new_new: 9.55 ± 4.66
Hits@50:
  Final val: 76.40 ± 0.85   Final Test: 59.68 ± 2.49   Final old_old: 64.10 ± 2.64   Final old_new: 42.17 ± 2.04   Final new_new: 15.00 ± 5.93
AUC:
  Final val: 97.39 ± 0.15   Final Test: 96.85 ± 0.15   Final old_old: 97.49 ± 0.17   Final old_new: 94.32 ± 0.15   Final new_new: 90.59 ± 1.04
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.57 ± 3.77   Final Test: 31.14 ± 2.48   Final old_old: 33.69 ± 2.65   Final old_new: 20.86 ± 1.82   Final new_new: 8.64 ± 6.10
Hits@20:
  Final val: 55.28 ± 3.95   Final Test: 35.93 ± 4.76   Final old_old: 38.74 ± 5.12   Final old_new: 24.71 ± 3.47   Final new_new: 8.64 ± 3.73
Hits@30:
  Final val: 63.79 ± 2.58   Final Test: 46.64 ± 3.49   Final old_old: 50.02 ± 3.61   Final old_new: 33.20 ± 3.32   Final new_new: 13.64 ± 9.64
Hits@50:
  Final val: 71.56 ± 2.27   Final Test: 56.32 ± 2.67   Final old_old: 59.91 ± 2.76   Final old_new: 42.09 ± 2.71   Final new_new: 20.00 ± 7.08
AUC:
  Final val: 96.59 ± 0.21   Final Test: 95.98 ± 0.19   Final old_old: 96.65 ± 0.24   Final old_new: 93.33 ± 0.08   Final new_new: 89.49 ± 1.45
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.88 ± 2.85   Final Test: 31.38 ± 5.66   Final old_old: 34.43 ± 6.16   Final old_new: 19.07 ± 3.81   Final new_new: 4.09 ± 2.49
Hits@20:
  Final val: 59.10 ± 2.06   Final Test: 40.94 ± 1.86   Final old_old: 44.44 ± 2.02   Final old_new: 27.03 ± 1.76   Final new_new: 6.36 ± 2.96
Hits@30:
  Final val: 66.42 ± 1.23   Final Test: 48.62 ± 1.88   Final old_old: 52.54 ± 2.05   Final old_new: 33.00 ± 1.26   Final new_new: 10.45 ± 3.05
Hits@50:
  Final val: 74.04 ± 1.04   Final Test: 58.33 ± 2.29   Final old_old: 62.32 ± 2.46   Final old_new: 42.57 ± 1.93   Final new_new: 16.82 ± 4.13
AUC:
  Final val: 97.26 ± 0.09   Final Test: 96.75 ± 0.05   Final old_old: 97.33 ± 0.04   Final old_new: 94.40 ± 0.24   Final new_new: 91.46 ± 1.03
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.18 ± 1.04   Final Test: 34.10 ± 5.04   Final old_old: 37.13 ± 5.26   Final old_new: 21.81 ± 4.55   Final new_new: 7.73 ± 6.35
Hits@20:
  Final val: 58.16 ± 2.00   Final Test: 38.26 ± 5.70   Final old_old: 41.59 ± 6.06   Final old_new: 24.91 ± 4.54   Final new_new: 6.82 ± 1.61
Hits@30:
  Final val: 66.36 ± 1.44   Final Test: 48.79 ± 5.14   Final old_old: 52.66 ± 5.32   Final old_new: 33.40 ± 4.95   Final new_new: 10.91 ± 4.93
Hits@50:
  Final val: 74.29 ± 1.50   Final Test: 57.21 ± 2.80   Final old_old: 61.25 ± 2.90   Final old_new: 41.28 ± 2.77   Final new_new: 14.09 ± 2.96
AUC:
  Final val: 97.22 ± 0.09   Final Test: 96.74 ± 0.13   Final old_old: 97.32 ± 0.13   Final old_new: 94.47 ± 0.25   Final new_new: 90.31 ± 1.22
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.39 ± 1.42   Final Test: 33.00 ± 4.56   Final old_old: 36.18 ± 4.92   Final old_new: 20.18 ± 3.34   Final new_new: 5.00 ± 3.73
Hits@20:
  Final val: 59.71 ± 2.59   Final Test: 40.03 ± 6.11   Final old_old: 43.57 ± 6.39   Final old_new: 25.82 ± 5.20   Final new_new: 7.27 ± 2.96
Hits@30:
  Final val: 66.46 ± 2.22   Final Test: 46.13 ± 2.30   Final old_old: 49.98 ± 2.58   Final old_new: 30.86 ± 1.24   Final new_new: 6.82 ± 3.59
Hits@50:
  Final val: 74.01 ± 1.99   Final Test: 56.78 ± 3.03   Final old_old: 61.00 ± 3.27   Final old_new: 40.10 ± 2.63   Final new_new: 13.18 ± 1.90
AUC:
  Final val: 97.23 ± 0.06   Final Test: 96.68 ± 0.15   Final old_old: 97.27 ± 0.15   Final old_new: 94.34 ± 0.23   Final new_new: 90.70 ± 0.98
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.74 ± 2.88   Final Test: 29.75 ± 4.97   Final old_old: 32.65 ± 5.38   Final old_new: 17.96 ± 3.54   Final new_new: 5.45 ± 1.24
Hits@20:
  Final val: 59.76 ± 1.04   Final Test: 42.77 ± 3.71   Final old_old: 46.33 ± 3.82   Final old_new: 28.64 ± 3.47   Final new_new: 6.82 ± 0.00
Hits@30:
  Final val: 66.47 ± 2.06   Final Test: 49.99 ± 2.51   Final old_old: 54.01 ± 2.51   Final old_new: 34.06 ± 2.77   Final new_new: 9.55 ± 1.90
Hits@50:
  Final val: 74.61 ± 1.59   Final Test: 58.17 ± 3.79   Final old_old: 62.37 ± 3.85   Final old_new: 41.56 ± 3.95   Final new_new: 15.00 ± 2.59
AUC:
  Final val: 97.31 ± 0.12   Final Test: 96.80 ± 0.11   Final old_old: 97.40 ± 0.13   Final old_new: 94.43 ± 0.24   Final new_new: 90.83 ± 1.18
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.22 ± 4.58   Final Test: 27.43 ± 2.95   Final old_old: 29.85 ± 3.07   Final old_new: 17.66 ± 2.60   Final new_new: 6.36 ± 3.37
Hits@20:
  Final val: 57.17 ± 2.63   Final Test: 37.89 ± 4.69   Final old_old: 40.97 ± 4.78   Final old_new: 25.44 ± 4.40   Final new_new: 10.45 ± 6.14
Hits@30:
  Final val: 65.33 ± 2.54   Final Test: 47.50 ± 2.93   Final old_old: 51.02 ± 3.07   Final old_new: 33.55 ± 2.96   Final new_new: 11.36 ± 2.78
Hits@50:
  Final val: 73.67 ± 2.16   Final Test: 57.37 ± 3.83   Final old_old: 61.20 ± 4.14   Final old_new: 42.32 ± 3.28   Final new_new: 16.36 ± 5.18
AUC:
  Final val: 96.68 ± 0.29   Final Test: 96.08 ± 0.22   Final old_old: 96.72 ± 0.21   Final old_new: 93.55 ± 0.32   Final new_new: 89.54 ± 1.47
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.46 ± 3.90   Final Test: 32.33 ± 4.51   Final old_old: 35.40 ± 4.60   Final old_new: 20.08 ± 4.37   Final new_new: 3.18 ± 2.03
Hits@20:
  Final val: 62.18 ± 3.17   Final Test: 40.88 ± 3.84   Final old_old: 44.48 ± 4.30   Final old_new: 26.45 ± 2.23   Final new_new: 7.27 ± 3.73
Hits@30:
  Final val: 68.87 ± 3.02   Final Test: 51.77 ± 4.41   Final old_old: 55.65 ± 4.74   Final old_new: 36.40 ± 3.32   Final new_new: 11.82 ± 3.37
Hits@50:
  Final val: 76.46 ± 2.05   Final Test: 59.54 ± 2.77   Final old_old: 63.81 ± 2.93   Final old_new: 42.64 ± 2.49   Final new_new: 15.91 ± 2.78
AUC:
  Final val: 97.42 ± 0.19   Final Test: 96.89 ± 0.18   Final old_old: 97.51 ± 0.20   Final old_new: 94.41 ± 0.20   Final new_new: 90.75 ± 0.65
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=65536, lr=0.005, epochs=20000, eval_steps=5, runs=10, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=0.1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.03 ± 6.57   Final Test: 29.17 ± 4.91   Final old_old: 31.89 ± 5.45   Final old_new: 18.06 ± 3.54   Final new_new: 7.27 ± 4.93
Hits@20:
  Final val: 56.05 ± 4.71   Final Test: 36.24 ± 3.09   Final old_old: 39.17 ± 3.50   Final old_new: 24.41 ± 2.32   Final new_new: 10.00 ± 6.55
Hits@30:
  Final val: 63.51 ± 3.38   Final Test: 43.44 ± 3.55   Final old_old: 46.85 ± 3.98   Final old_new: 29.85 ± 2.12   Final new_new: 10.45 ± 6.14
Hits@50:
  Final val: 72.01 ± 2.47   Final Test: 57.42 ± 3.10   Final old_old: 60.98 ± 3.39   Final old_new: 43.55 ± 2.51   Final new_new: 16.82 ± 6.74
AUC:
  Final val: 96.60 ± 0.30   Final Test: 95.97 ± 0.29   Final old_old: 96.62 ± 0.30   Final old_new: 93.38 ± 0.37   Final new_new: 89.52 ± 0.42
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=65536, lr=0.005, epochs=20000, eval_steps=5, runs=10, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=0.1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.93 ± 6.88   Final Test: 28.83 ± 6.57   Final old_old: 31.42 ± 7.24   Final old_new: 18.39 ± 3.97   Final new_new: 5.91 ± 2.59
Hits@20:
  Final val: 54.74 ± 4.25   Final Test: 38.34 ± 6.37   Final old_old: 41.64 ± 6.88   Final old_new: 25.06 ± 4.63   Final new_new: 8.64 ± 4.66
Hits@30:
  Final val: 62.99 ± 2.75   Final Test: 44.72 ± 3.34   Final old_old: 48.29 ± 3.62   Final old_new: 30.45 ± 2.44   Final new_new: 10.45 ± 3.45
Hits@50:
  Final val: 71.59 ± 2.61   Final Test: 55.61 ± 2.23   Final old_old: 59.49 ± 2.53   Final old_new: 40.40 ± 1.47   Final new_new: 13.64 ± 3.59
AUC:
  Final val: 96.66 ± 0.31   Final Test: 96.06 ± 0.24   Final old_old: 96.70 ± 0.30   Final old_new: 93.50 ± 0.32   Final new_new: 90.16 ± 1.00
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=65536, lr=0.005, epochs=20000, eval_steps=5, runs=10, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=0.1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.80 ± 6.37   Final Test: 27.80 ± 2.91   Final old_old: 30.29 ± 3.20   Final old_new: 17.56 ± 2.10   Final new_new: 9.09 ± 7.01
Hits@20:
  Final val: 56.51 ± 4.06   Final Test: 36.86 ± 2.75   Final old_old: 40.01 ± 2.89   Final old_new: 24.13 ± 2.68   Final new_new: 10.00 ± 6.55
Hits@30:
  Final val: 63.67 ± 3.22   Final Test: 45.07 ± 4.56   Final old_old: 48.58 ± 4.80   Final old_new: 31.13 ± 4.04   Final new_new: 10.45 ± 6.14
Hits@50:
  Final val: 71.70 ± 1.99   Final Test: 56.10 ± 4.29   Final old_old: 59.83 ± 4.71   Final old_new: 41.51 ± 2.93   Final new_new: 14.09 ± 4.93
AUC:
  Final val: 96.65 ± 0.24   Final Test: 95.96 ± 0.33   Final old_old: 96.59 ± 0.32   Final old_new: 93.45 ± 0.44   Final new_new: 89.59 ± 2.01
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=65536, lr=0.005, epochs=20000, eval_steps=5, runs=10, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=0.1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=65536, lr=0.005, epochs=20000, eval_steps=5, runs=10, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=0.1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.18 ± 3.38   Final Test: 30.22 ± 4.59   Final old_old: 33.07 ± 4.93   Final old_new: 18.69 ± 3.51   Final new_new: 5.45 ± 1.24
Hits@20:
  Final val: 53.49 ± 3.23   Final Test: 34.65 ± 2.17   Final old_old: 37.72 ± 2.37   Final old_new: 22.29 ± 1.63   Final new_new: 7.27 ± 2.96
Hits@30:
  Final val: 62.23 ± 3.57   Final Test: 43.74 ± 5.10   Final old_old: 47.12 ± 5.09   Final old_new: 30.23 ± 5.29   Final new_new: 11.82 ± 4.37
Hits@50:
  Final val: 72.13 ± 2.72   Final Test: 53.58 ± 4.73   Final old_old: 57.37 ± 4.66   Final old_new: 38.59 ± 5.21   Final new_new: 14.55 ± 3.05
AUC:
  Final val: 96.70 ± 0.33   Final Test: 95.96 ± 0.32   Final old_old: 96.64 ± 0.32   Final old_new: 93.21 ± 0.43   Final new_new: 89.75 ± 1.28
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.05 ± 5.77   Final Test: 29.89 ± 8.59   Final old_old: 32.49 ± 9.22   Final old_new: 19.37 ± 6.28   Final new_new: 7.73 ± 6.14
Hits@20:
  Final val: 56.09 ± 3.71   Final Test: 41.06 ± 5.36   Final old_old: 44.31 ± 5.97   Final old_new: 28.16 ± 3.11   Final new_new: 8.18 ± 4.43
Hits@30:
  Final val: 64.03 ± 3.10   Final Test: 47.05 ± 5.12   Final old_old: 50.51 ± 5.42   Final old_new: 33.50 ± 4.34   Final new_new: 9.55 ± 6.31
Hits@50:
  Final val: 72.01 ± 2.00   Final Test: 57.38 ± 2.61   Final old_old: 61.15 ± 2.79   Final old_new: 42.75 ± 2.46   Final new_new: 13.64 ± 2.78
AUC:
  Final val: 96.66 ± 0.18   Final Test: 96.02 ± 0.21   Final old_old: 96.70 ± 0.21   Final old_new: 93.32 ± 0.31   Final new_new: 89.63 ± 1.37
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.52 ± 3.68   Final Test: 33.28 ± 7.07   Final old_old: 36.49 ± 7.69   Final old_new: 20.38 ± 4.78   Final new_new: 4.09 ± 3.37
Hits@20:
  Final val: 63.15 ± 2.37   Final Test: 44.44 ± 3.86   Final old_old: 48.20 ± 4.30   Final old_new: 29.45 ± 2.64   Final new_new: 7.73 ± 4.13
Hits@30:
  Final val: 70.12 ± 0.87   Final Test: 50.66 ± 2.63   Final old_old: 54.53 ± 2.81   Final old_new: 35.26 ± 2.26   Final new_new: 12.27 ± 3.45
Hits@50:
  Final val: 76.36 ± 0.98   Final Test: 62.14 ± 1.94   Final old_old: 66.32 ± 2.10   Final old_new: 45.77 ± 2.24   Final new_new: 15.91 ± 6.63
AUC:
  Final val: 97.41 ± 0.13   Final Test: 96.87 ± 0.14   Final old_old: 97.47 ± 0.19   Final old_new: 94.47 ± 0.07   Final new_new: 91.17 ± 1.06
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.19 ± 3.73   Final Test: 33.06 ± 2.92   Final old_old: 36.34 ± 3.08   Final old_new: 19.87 ± 2.39   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 61.91 ± 3.37   Final Test: 39.57 ± 6.84   Final old_old: 43.17 ± 7.27   Final old_new: 25.04 ± 5.27   Final new_new: 7.73 ± 1.24
Hits@30:
  Final val: 68.80 ± 3.13   Final Test: 50.50 ± 4.64   Final old_old: 54.47 ± 4.83   Final old_new: 34.76 ± 4.03   Final new_new: 10.45 ± 3.45
Hits@50:
  Final val: 76.28 ± 2.51   Final Test: 60.84 ± 2.97   Final old_old: 65.00 ± 3.10   Final old_new: 44.43 ± 2.59   Final new_new: 16.36 ± 3.73
AUC:
  Final val: 97.32 ± 0.20   Final Test: 96.84 ± 0.16   Final old_old: 97.45 ± 0.18   Final old_new: 94.43 ± 0.19   Final new_new: 90.84 ± 0.98
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.07 ± 4.50   Final Test: 35.31 ± 3.16   Final old_old: 38.64 ± 3.58   Final old_new: 22.02 ± 1.55   Final new_new: 3.64 ± 2.03
Hits@20:
  Final val: 62.17 ± 4.12   Final Test: 41.91 ± 2.40   Final old_old: 45.41 ± 2.28   Final old_new: 28.09 ± 3.22   Final new_new: 5.91 ± 2.59
Hits@30:
  Final val: 68.98 ± 2.76   Final Test: 51.16 ± 3.22   Final old_old: 55.18 ± 3.25   Final old_new: 35.26 ± 3.62   Final new_new: 9.55 ± 3.73
Hits@50:
  Final val: 77.20 ± 1.34   Final Test: 59.53 ± 2.14   Final old_old: 63.86 ± 2.27   Final old_new: 42.47 ± 2.08   Final new_new: 13.64 ± 4.82
AUC:
  Final val: 97.40 ± 0.18   Final Test: 96.86 ± 0.19   Final old_old: 97.49 ± 0.23   Final old_new: 94.35 ± 0.22   Final new_new: 90.95 ± 0.95
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.53 ± 1.49   Final Test: 34.82 ± 4.65   Final old_old: 38.13 ± 4.94   Final old_new: 21.54 ± 3.64   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 63.51 ± 2.00   Final Test: 45.00 ± 3.74   Final old_old: 49.00 ± 3.94   Final old_new: 29.09 ± 3.19   Final new_new: 5.00 ± 2.49
Hits@30:
  Final val: 70.49 ± 1.37   Final Test: 52.51 ± 2.82   Final old_old: 56.64 ± 2.90   Final old_new: 36.30 ± 2.89   Final new_new: 8.18 ± 3.45
Hits@50:
  Final val: 77.69 ± 1.52   Final Test: 64.02 ± 1.40   Final old_old: 68.34 ± 1.63   Final old_new: 47.08 ± 0.81   Final new_new: 16.36 ± 7.61
AUC:
  Final val: 97.43 ± 0.15   Final Test: 96.86 ± 0.14   Final old_old: 97.48 ± 0.16   Final old_new: 94.41 ± 0.22   Final new_new: 90.27 ± 0.88
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.98 ± 3.37   Final Test: 32.08 ± 4.41   Final old_old: 35.12 ± 4.51   Final old_new: 19.75 ± 4.04   Final new_new: 6.36 ± 4.07
Hits@20:
  Final val: 61.38 ± 1.47   Final Test: 43.10 ± 2.47   Final old_old: 46.76 ± 2.74   Final old_new: 28.41 ± 1.63   Final new_new: 9.09 ± 2.78
Hits@30:
  Final val: 68.91 ± 1.39   Final Test: 53.05 ± 2.80   Final old_old: 57.14 ± 2.98   Final old_new: 36.73 ± 2.37   Final new_new: 13.64 ± 3.59
Hits@50:
  Final val: 76.51 ± 1.46   Final Test: 63.11 ± 2.26   Final old_old: 67.40 ± 2.25   Final old_new: 46.10 ± 3.02   Final new_new: 20.00 ± 3.37
AUC:
  Final val: 97.43 ± 0.23   Final Test: 96.85 ± 0.19   Final old_old: 97.54 ± 0.20   Final old_new: 94.10 ± 0.24   Final new_new: 90.44 ± 0.54
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.85 ± 0.99   Final Test: 30.75 ± 3.63   Final old_old: 33.70 ± 3.72   Final old_new: 18.79 ± 3.47   Final new_new: 5.45 ± 3.45
Hits@20:
  Final val: 58.94 ± 1.34   Final Test: 39.81 ± 5.16   Final old_old: 43.26 ± 5.42   Final old_new: 25.94 ± 4.26   Final new_new: 8.18 ± 3.45
Hits@30:
  Final val: 67.45 ± 1.19   Final Test: 47.70 ± 2.09   Final old_old: 51.57 ± 2.11   Final old_new: 32.27 ± 2.42   Final new_new: 10.45 ± 4.71
Hits@50:
  Final val: 75.65 ± 1.24   Final Test: 57.06 ± 2.35   Final old_old: 61.33 ± 2.53   Final old_new: 40.10 ± 2.43   Final new_new: 14.55 ± 5.93
AUC:
  Final val: 97.29 ± 0.08   Final Test: 96.80 ± 0.16   Final old_old: 97.40 ± 0.16   Final old_new: 94.37 ± 0.29   Final new_new: 91.02 ± 1.36
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.01 ± 4.43   Final Test: 28.89 ± 2.95   Final old_old: 31.51 ± 3.08   Final old_new: 18.26 ± 2.66   Final new_new: 7.27 ± 1.90
Hits@20:
  Final val: 58.43 ± 3.77   Final Test: 40.37 ± 3.44   Final old_old: 43.55 ± 3.55   Final old_new: 27.66 ± 3.38   Final new_new: 10.45 ± 3.05
Hits@30:
  Final val: 66.58 ± 2.20   Final Test: 47.42 ± 3.18   Final old_old: 50.90 ± 3.27   Final old_new: 33.55 ± 3.05   Final new_new: 13.18 ± 2.49
Hits@50:
  Final val: 74.75 ± 1.44   Final Test: 56.37 ± 2.55   Final old_old: 60.09 ± 2.23   Final old_new: 41.54 ± 4.25   Final new_new: 20.00 ± 4.37
AUC:
  Final val: 96.79 ± 0.21   Final Test: 96.23 ± 0.29   Final old_old: 96.83 ± 0.26   Final old_new: 93.80 ± 0.43   Final new_new: 91.77 ± 0.95
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.46 ± 1.44   Final Test: 32.39 ± 3.12   Final old_old: 35.62 ± 3.47   Final old_new: 19.29 ± 1.97   Final new_new: 5.00 ± 1.90
Hits@20:
  Final val: 58.93 ± 1.26   Final Test: 43.59 ± 3.45   Final old_old: 47.24 ± 3.60   Final old_new: 28.92 ± 3.26   Final new_new: 10.45 ± 4.43
Hits@30:
  Final val: 66.01 ± 1.46   Final Test: 48.48 ± 4.76   Final old_old: 52.49 ± 5.04   Final old_new: 32.34 ± 3.90   Final new_new: 12.27 ± 1.24
Hits@50:
  Final val: 74.22 ± 1.75   Final Test: 58.84 ± 2.56   Final old_old: 63.08 ± 2.81   Final old_new: 42.12 ± 2.00   Final new_new: 14.55 ± 3.80
AUC:
  Final val: 97.30 ± 0.08   Final Test: 96.79 ± 0.10   Final old_old: 97.34 ± 0.06   Final old_new: 94.63 ± 0.30   Final new_new: 91.10 ± 1.11
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=65536, lr=0.005, epochs=20000, eval_steps=5, runs=10, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=0.1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=65536, lr=0.005, epochs=20000, eval_steps=5, runs=10, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=0.1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.67 ± 2.69   Final Test: 32.45 ± 3.69   Final old_old: 35.79 ± 4.04   Final old_new: 18.89 ± 2.66   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 61.02 ± 1.12   Final Test: 44.96 ± 0.76   Final old_old: 48.69 ± 0.81   Final old_new: 30.08 ± 1.29   Final new_new: 8.64 ± 2.96
Hits@30:
  Final val: 67.81 ± 1.49   Final Test: 49.85 ± 3.66   Final old_old: 53.70 ± 3.96   Final old_new: 34.63 ± 2.88   Final new_new: 10.00 ± 4.13
Hits@50:
  Final val: 75.50 ± 1.31   Final Test: 58.34 ± 2.07   Final old_old: 62.54 ± 2.26   Final old_new: 41.69 ± 1.64   Final new_new: 16.36 ± 4.07
AUC:
  Final val: 97.37 ± 0.10   Final Test: 96.78 ± 0.09   Final old_old: 97.37 ± 0.09   Final old_new: 94.46 ± 0.25   Final new_new: 90.71 ± 1.15
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=65536, lr=0.005, epochs=20000, eval_steps=5, runs=10, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=0.1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.50 ± 6.38   Final Test: 31.74 ± 3.33   Final old_old: 34.92 ± 3.49   Final old_new: 18.79 ± 3.22   Final new_new: 5.91 ± 2.03
Hits@20:
  Final val: 56.14 ± 4.79   Final Test: 40.58 ± 1.76   Final old_old: 44.18 ± 2.27   Final old_new: 26.12 ± 1.09   Final new_new: 7.27 ± 2.96
Hits@30:
  Final val: 65.34 ± 3.94   Final Test: 50.65 ± 2.98   Final old_old: 54.96 ± 3.44   Final old_new: 33.22 ± 1.47   Final new_new: 12.73 ± 3.05
Hits@50:
  Final val: 72.94 ± 3.25   Final Test: 59.77 ± 2.44   Final old_old: 64.45 ± 2.88   Final old_new: 41.08 ± 0.94   Final new_new: 15.00 ± 2.59
AUC:
  Final val: 97.20 ± 0.29   Final Test: 96.33 ± 0.20   Final old_old: 97.32 ± 0.28   Final old_new: 92.35 ± 0.52   Final new_new: 87.55 ± 1.88
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=65536, lr=0.005, epochs=20000, eval_steps=5, runs=10, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=0.1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=65536, lr=0.005, epochs=20000, eval_steps=5, runs=10, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=0.1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.31 ± 1.15   Final Test: 32.76 ± 3.07   Final old_old: 35.92 ± 3.35   Final old_new: 20.05 ± 1.94   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 59.51 ± 1.68   Final Test: 42.13 ± 7.43   Final old_old: 45.55 ± 8.03   Final old_new: 28.39 ± 5.35   Final new_new: 10.45 ± 3.45
Hits@30:
  Final val: 66.81 ± 1.30   Final Test: 50.57 ± 2.88   Final old_old: 54.50 ± 3.00   Final old_new: 34.89 ± 2.80   Final new_new: 12.27 ± 1.24
Hits@50:
  Final val: 74.49 ± 1.87   Final Test: 57.14 ± 2.10   Final old_old: 61.32 ± 2.40   Final old_new: 40.60 ± 1.18   Final new_new: 14.55 ± 4.43
AUC:
  Final val: 97.25 ± 0.07   Final Test: 96.72 ± 0.17   Final old_old: 97.31 ± 0.16   Final old_new: 94.38 ± 0.27   Final new_new: 90.26 ± 0.92
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=65536, lr=0.005, epochs=20000, eval_steps=5, runs=10, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=0.1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.68 ± 3.98   Final Test: 30.26 ± 4.24   Final old_old: 33.24 ± 4.55   Final old_new: 18.19 ± 3.11   Final new_new: 5.00 ± 2.49
Hits@20:
  Final val: 60.35 ± 2.65   Final Test: 43.10 ± 2.86   Final old_old: 46.70 ± 3.01   Final old_new: 28.61 ± 2.42   Final new_new: 10.45 ± 3.45
Hits@30:
  Final val: 67.59 ± 1.90   Final Test: 47.15 ± 2.88   Final old_old: 50.94 ± 3.29   Final old_new: 31.96 ± 1.61   Final new_new: 11.36 ± 3.21
Hits@50:
  Final val: 75.15 ± 1.67   Final Test: 57.99 ± 1.82   Final old_old: 62.34 ± 1.96   Final old_new: 40.60 ± 1.77   Final new_new: 15.91 ± 5.33
AUC:
  Final val: 97.32 ± 0.12   Final Test: 96.74 ± 0.07   Final old_old: 97.35 ± 0.06   Final old_new: 94.36 ± 0.21   Final new_new: 90.27 ± 1.50
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=65536, lr=0.005, epochs=20000, eval_steps=5, runs=10, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=0.1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.93 ± 4.59   Final Test: 32.89 ± 4.66   Final old_old: 36.11 ± 5.04   Final old_new: 19.90 ± 3.25   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 65.66 ± 2.45   Final Test: 43.29 ± 1.89   Final old_old: 47.02 ± 1.99   Final old_new: 28.44 ± 1.95   Final new_new: 6.82 ± 2.78
Hits@30:
  Final val: 72.24 ± 1.48   Final Test: 51.48 ± 3.38   Final old_old: 55.64 ± 3.40   Final old_new: 35.09 ± 3.52   Final new_new: 8.18 ± 3.80
Hits@50:
  Final val: 78.77 ± 1.04   Final Test: 62.37 ± 1.46   Final old_old: 66.80 ± 1.49   Final old_new: 45.01 ± 2.09   Final new_new: 14.09 ± 2.96
AUC:
  Final val: 97.62 ± 0.08   Final Test: 97.11 ± 0.06   Final old_old: 97.66 ± 0.09   Final old_new: 94.92 ± 0.10   Final new_new: 91.87 ± 1.02
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.66 ± 6.71   Final Test: 28.41 ± 3.68   Final old_old: 31.00 ± 4.23   Final old_new: 17.91 ± 1.81   Final new_new: 5.91 ± 2.59
Hits@20:
  Final val: 56.46 ± 4.30   Final Test: 38.02 ± 3.21   Final old_old: 41.05 ± 3.67   Final old_new: 25.92 ± 1.68   Final new_new: 9.55 ± 1.90
Hits@30:
  Final val: 63.67 ± 2.91   Final Test: 45.76 ± 4.09   Final old_old: 49.16 ± 4.46   Final old_new: 32.34 ± 3.06   Final new_new: 10.00 ± 3.05
Hits@50:
  Final val: 71.66 ± 2.36   Final Test: 57.69 ± 2.50   Final old_old: 61.54 ± 2.57   Final old_new: 42.59 ± 2.85   Final new_new: 16.36 ± 2.96
AUC:
  Final val: 96.67 ± 0.24   Final Test: 96.04 ± 0.17   Final old_old: 96.69 ± 0.21   Final old_new: 93.44 ± 0.13   Final new_new: 90.07 ± 0.48
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.12 ± 2.61   Final Test: 36.84 ± 2.08   Final old_old: 40.47 ± 2.27   Final old_new: 22.17 ± 1.56   Final new_new: 5.45 ± 3.80
Hits@20:
  Final val: 62.95 ± 2.28   Final Test: 45.21 ± 4.33   Final old_old: 49.46 ± 4.65   Final old_new: 28.19 ± 3.28   Final new_new: 5.91 ± 4.98
Hits@30:
  Final val: 69.65 ± 1.37   Final Test: 53.64 ± 3.12   Final old_old: 58.35 ± 3.49   Final old_new: 34.84 ± 1.81   Final new_new: 8.64 ± 5.18
Hits@50:
  Final val: 78.32 ± 1.12   Final Test: 62.92 ± 1.38   Final old_old: 67.94 ± 1.39   Final old_new: 42.90 ± 1.52   Final new_new: 14.09 ± 8.10
AUC:
  Final val: 97.85 ± 0.15   Final Test: 96.64 ± 0.15   Final old_old: 97.92 ± 0.10   Final old_new: 91.51 ± 0.48   Final new_new: 85.22 ± 2.80
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.00 ± 4.68   Final Test: 31.15 ± 3.95   Final old_old: 33.94 ± 4.38   Final old_new: 19.87 ± 2.69   Final new_new: 6.82 ± 4.55
Hits@20:
  Final val: 54.12 ± 3.51   Final Test: 37.02 ± 2.55   Final old_old: 40.02 ± 2.91   Final old_new: 24.91 ± 1.29   Final new_new: 10.45 ± 5.70
Hits@30:
  Final val: 62.05 ± 3.28   Final Test: 45.36 ± 3.82   Final old_old: 48.56 ± 3.87   Final old_new: 32.67 ± 3.60   Final new_new: 13.18 ± 7.08
Hits@50:
  Final val: 71.27 ± 3.07   Final Test: 55.24 ± 5.79   Final old_old: 58.78 ± 6.14   Final old_new: 41.26 ± 4.53   Final new_new: 18.64 ± 5.88
AUC:
  Final val: 96.70 ± 0.28   Final Test: 95.98 ± 0.23   Final old_old: 96.62 ± 0.19   Final old_new: 93.42 ± 0.42   Final new_new: 89.80 ± 1.51
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=65536, lr=0.005, epochs=20000, eval_steps=5, runs=10, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=0.1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.25 ± 2.44   Final Test: 25.01 ± 3.30   Final old_old: 27.22 ± 3.54   Final old_new: 16.05 ± 2.40   Final new_new: 5.91 ± 3.05
Hits@20:
  Final val: 55.34 ± 1.32   Final Test: 36.98 ± 2.79   Final old_old: 40.03 ± 3.11   Final old_new: 24.79 ± 1.77   Final new_new: 7.73 ± 4.71
Hits@30:
  Final val: 63.92 ± 1.68   Final Test: 45.43 ± 3.83   Final old_old: 48.90 ± 3.95   Final old_new: 31.64 ± 3.27   Final new_new: 10.45 ± 4.98
Hits@50:
  Final val: 72.26 ± 2.53   Final Test: 55.50 ± 3.48   Final old_old: 59.18 ± 3.61   Final old_new: 41.08 ± 3.14   Final new_new: 15.00 ± 5.23
AUC:
  Final val: 96.67 ± 0.27   Final Test: 95.92 ± 0.35   Final old_old: 96.62 ± 0.28   Final old_new: 93.18 ± 0.67   Final new_new: 88.40 ± 1.45
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=65536, lr=0.005, epochs=20000, eval_steps=5, runs=10, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=0.1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.13 ± 4.16   Final Test: 30.86 ± 2.88   Final old_old: 33.39 ± 3.35   Final old_new: 20.68 ± 1.33   Final new_new: 8.18 ± 4.13
Hits@20:
  Final val: 56.44 ± 3.30   Final Test: 36.95 ± 6.11   Final old_old: 40.13 ± 6.70   Final old_new: 24.11 ± 4.28   Final new_new: 9.09 ± 1.61
Hits@30:
  Final val: 63.40 ± 2.90   Final Test: 43.58 ± 4.84   Final old_old: 46.89 ± 5.06   Final old_new: 30.28 ± 4.37   Final new_new: 13.64 ± 5.08
Hits@50:
  Final val: 72.61 ± 1.15   Final Test: 57.35 ± 2.72   Final old_old: 61.14 ± 2.88   Final old_new: 42.37 ± 2.95   Final new_new: 18.18 ± 6.63
AUC:
  Final val: 96.74 ± 0.29   Final Test: 96.10 ± 0.11   Final old_old: 96.77 ± 0.15   Final old_new: 93.49 ± 0.12   Final new_new: 88.67 ± 1.53
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=65536, lr=0.005, epochs=20000, eval_steps=5, runs=10, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=0.1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=65536, lr=0.005, epochs=20000, eval_steps=5, runs=10, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=0.1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=65536, lr=0.005, epochs=20000, eval_steps=5, runs=10, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=0.1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.10 ± 5.02   Final Test: 29.12 ± 5.75   Final old_old: 31.79 ± 6.27   Final old_new: 18.29 ± 3.99   Final new_new: 6.82 ± 4.25
Hits@20:
  Final val: 54.01 ± 3.75   Final Test: 38.82 ± 3.69   Final old_old: 42.08 ± 4.05   Final old_new: 25.72 ± 2.69   Final new_new: 9.55 ± 4.93
Hits@30:
  Final val: 62.02 ± 2.84   Final Test: 44.93 ± 4.95   Final old_old: 48.46 ± 5.44   Final old_new: 30.86 ± 3.32   Final new_new: 10.91 ± 6.31
Hits@50:
  Final val: 71.46 ± 1.96   Final Test: 56.57 ± 4.90   Final old_old: 60.30 ± 5.14   Final old_new: 41.81 ± 4.31   Final new_new: 17.73 ± 6.70
AUC:
  Final val: 96.63 ± 0.33   Final Test: 95.99 ± 0.39   Final old_old: 96.64 ± 0.39   Final old_new: 93.35 ± 0.45   Final new_new: 90.18 ± 1.00
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.05 ± 4.99   Final Test: 36.35 ± 3.78   Final old_old: 39.84 ± 4.12   Final old_new: 22.34 ± 2.76   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 62.96 ± 3.33   Final Test: 44.18 ± 2.20   Final old_old: 47.94 ± 2.40   Final old_new: 29.22 ± 1.47   Final new_new: 6.36 ± 2.96
Hits@30:
  Final val: 69.68 ± 2.80   Final Test: 52.52 ± 2.70   Final old_old: 56.74 ± 2.59   Final old_new: 35.87 ± 3.15   Final new_new: 8.64 ± 4.93
Hits@50:
  Final val: 77.13 ± 2.02   Final Test: 60.97 ± 0.87   Final old_old: 65.39 ± 1.18   Final old_new: 43.48 ± 0.65   Final new_new: 15.00 ± 4.13
AUC:
  Final val: 97.39 ± 0.17   Final Test: 96.84 ± 0.15   Final old_old: 97.45 ± 0.17   Final old_new: 94.37 ± 0.12   Final new_new: 91.01 ± 0.66
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.69 ± 5.32   Final Test: 31.31 ± 7.22   Final old_old: 33.69 ± 7.62   Final old_new: 21.84 ± 5.83   Final new_new: 7.73 ± 3.80
Hits@20:
  Final val: 61.08 ± 3.92   Final Test: 44.96 ± 4.25   Final old_old: 48.40 ± 4.54   Final old_new: 31.21 ± 3.22   Final new_new: 11.82 ± 1.90
Hits@30:
  Final val: 67.57 ± 2.04   Final Test: 51.40 ± 2.03   Final old_old: 55.06 ± 2.17   Final old_new: 36.90 ± 1.72   Final new_new: 13.64 ± 2.78
Hits@50:
  Final val: 75.16 ± 1.23   Final Test: 58.70 ± 4.52   Final old_old: 62.42 ± 4.69   Final old_new: 43.95 ± 4.32   Final new_new: 20.91 ± 7.08
AUC:
  Final val: 97.03 ± 0.16   Final Test: 96.46 ± 0.17   Final old_old: 97.06 ± 0.15   Final old_new: 94.08 ± 0.26   Final new_new: 89.94 ± 1.28
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.59 ± 5.96   Final Test: 31.39 ± 4.13   Final old_old: 34.37 ± 4.49   Final old_new: 19.42 ± 2.89   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 64.79 ± 2.29   Final Test: 43.08 ± 3.06   Final old_old: 47.01 ± 3.31   Final old_new: 27.28 ± 2.35   Final new_new: 7.27 ± 3.37
Hits@30:
  Final val: 71.30 ± 1.15   Final Test: 52.20 ± 4.72   Final old_old: 56.43 ± 4.99   Final old_new: 35.37 ± 4.18   Final new_new: 10.00 ± 4.98
Hits@50:
  Final val: 78.12 ± 1.26   Final Test: 60.88 ± 3.58   Final old_old: 65.07 ± 3.73   Final old_new: 44.41 ± 3.73   Final new_new: 15.91 ± 5.79
AUC:
  Final val: 97.49 ± 0.14   Final Test: 96.88 ± 0.17   Final old_old: 97.48 ± 0.17   Final old_new: 94.54 ± 0.40   Final new_new: 90.52 ± 1.16
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.97 ± 2.44   Final Test: 33.29 ± 3.80   Final old_old: 36.60 ± 4.22   Final old_new: 19.90 ± 2.16   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 64.44 ± 2.18   Final Test: 44.68 ± 3.96   Final old_old: 48.61 ± 4.24   Final old_new: 28.97 ± 3.06   Final new_new: 7.27 ± 3.73
Hits@30:
  Final val: 71.21 ± 2.34   Final Test: 54.32 ± 2.91   Final old_old: 58.71 ± 3.17   Final old_new: 36.95 ± 2.12   Final new_new: 9.55 ± 2.96
Hits@50:
  Final val: 78.00 ± 1.74   Final Test: 62.67 ± 1.51   Final old_old: 67.10 ± 1.56   Final old_new: 45.24 ± 2.10   Final new_new: 15.45 ± 5.43
AUC:
  Final val: 97.50 ± 0.12   Final Test: 96.92 ± 0.13   Final old_old: 97.57 ± 0.12   Final old_new: 94.38 ± 0.38   Final new_new: 90.28 ± 0.65
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.74 ± 4.05   Final Test: 35.27 ± 7.43   Final old_old: 38.64 ± 7.94   Final old_new: 21.76 ± 5.53   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 63.12 ± 2.75   Final Test: 46.25 ± 4.46   Final old_old: 50.11 ± 4.91   Final old_new: 30.91 ± 2.98   Final new_new: 7.27 ± 3.37
Hits@30:
  Final val: 70.16 ± 2.53   Final Test: 54.55 ± 4.23   Final old_old: 58.59 ± 4.49   Final old_new: 38.74 ± 3.34   Final new_new: 10.45 ± 2.59
Hits@50:
  Final val: 77.43 ± 2.02   Final Test: 62.34 ± 2.90   Final old_old: 66.69 ± 3.11   Final old_new: 45.34 ± 2.38   Final new_new: 14.09 ± 2.96
AUC:
  Final val: 97.42 ± 0.22   Final Test: 96.93 ± 0.11   Final old_old: 97.55 ± 0.17   Final old_new: 94.43 ± 0.13   Final new_new: 90.90 ± 1.63
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.61 ± 2.83   Final Test: 32.74 ± 1.98   Final old_old: 36.12 ± 2.36   Final old_new: 19.07 ± 1.01   Final new_new: 3.18 ± 1.24
Hits@20:
  Final val: 62.54 ± 2.35   Final Test: 45.74 ± 1.17   Final old_old: 49.75 ± 1.10   Final old_new: 29.85 ± 1.63   Final new_new: 5.00 ± 2.96
Hits@30:
  Final val: 68.82 ± 1.13   Final Test: 53.63 ± 3.51   Final old_old: 57.88 ± 3.85   Final old_new: 36.80 ± 2.72   Final new_new: 10.00 ± 1.24
Hits@50:
  Final val: 76.52 ± 1.40   Final Test: 62.08 ± 1.96   Final old_old: 66.42 ± 1.87   Final old_new: 45.11 ± 2.63   Final new_new: 13.64 ± 3.59
AUC:
  Final val: 97.35 ± 0.18   Final Test: 96.86 ± 0.09   Final old_old: 97.52 ± 0.13   Final old_new: 94.26 ± 0.16   Final new_new: 89.94 ± 0.80
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.42 ± 2.69   Final Test: 34.16 ± 2.96   Final old_old: 37.36 ± 3.30   Final old_new: 21.18 ± 1.73   Final new_new: 7.27 ± 3.73
Hits@20:
  Final val: 59.62 ± 2.26   Final Test: 41.67 ± 4.26   Final old_old: 45.30 ± 4.64   Final old_new: 27.08 ± 2.94   Final new_new: 8.64 ± 4.07
Hits@30:
  Final val: 66.97 ± 1.60   Final Test: 49.14 ± 5.21   Final old_old: 52.90 ± 5.31   Final old_new: 34.13 ± 4.95   Final new_new: 12.73 ± 5.23
Hits@50:
  Final val: 73.66 ± 0.90   Final Test: 58.06 ± 1.71   Final old_old: 62.33 ± 1.70   Final old_new: 41.06 ± 1.80   Final new_new: 15.91 ± 6.22
AUC:
  Final val: 97.24 ± 0.11   Final Test: 96.76 ± 0.09   Final old_old: 97.34 ± 0.09   Final old_new: 94.51 ± 0.23   Final new_new: 90.50 ± 0.93
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.81 ± 2.63   Final Test: 34.05 ± 2.92   Final old_old: 37.16 ± 3.05   Final old_new: 21.61 ± 2.58   Final new_new: 4.09 ± 3.37
Hits@20:
  Final val: 64.77 ± 2.75   Final Test: 46.21 ± 2.57   Final old_old: 49.66 ± 2.71   Final old_new: 32.62 ± 2.23   Final new_new: 9.55 ± 3.73
Hits@30:
  Final val: 70.54 ± 2.29   Final Test: 51.62 ± 2.65   Final old_old: 55.44 ± 2.82   Final old_new: 36.73 ± 2.37   Final new_new: 8.18 ± 2.59
Hits@50:
  Final val: 77.88 ± 1.11   Final Test: 62.91 ± 2.67   Final old_old: 66.79 ± 2.65   Final old_new: 47.81 ± 3.11   Final new_new: 19.09 ± 4.71
AUC:
  Final val: 97.63 ± 0.09   Final Test: 97.20 ± 0.08   Final old_old: 97.67 ± 0.11   Final old_new: 95.33 ± 0.15   Final new_new: 92.38 ± 0.91
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.24 ± 1.49   Final Test: 33.36 ± 5.53   Final old_old: 36.59 ± 5.96   Final old_new: 20.28 ± 4.00   Final new_new: 5.00 ± 1.90
Hits@20:
  Final val: 58.94 ± 2.54   Final Test: 39.73 ± 2.04   Final old_old: 43.18 ± 2.23   Final old_new: 25.84 ± 1.63   Final new_new: 8.64 ± 2.96
Hits@30:
  Final val: 65.42 ± 0.92   Final Test: 46.94 ± 3.48   Final old_old: 50.89 ± 3.67   Final old_new: 31.08 ± 3.08   Final new_new: 10.45 ± 2.03
Hits@50:
  Final val: 73.85 ± 0.98   Final Test: 56.58 ± 2.29   Final old_old: 60.73 ± 2.47   Final old_new: 40.08 ± 2.09   Final new_new: 15.45 ± 3.37
AUC:
  Final val: 97.26 ± 0.09   Final Test: 96.76 ± 0.10   Final old_old: 97.33 ± 0.07   Final old_new: 94.48 ± 0.37   Final new_new: 90.54 ± 0.89
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.24 ± 3.13   Final Test: 32.54 ± 3.82   Final old_old: 35.69 ± 4.08   Final old_new: 19.80 ± 2.94   Final new_new: 5.45 ± 2.59
Hits@20:
  Final val: 58.51 ± 2.58   Final Test: 42.90 ± 4.09   Final old_old: 46.56 ± 4.39   Final old_new: 28.19 ± 3.47   Final new_new: 9.55 ± 3.37
Hits@30:
  Final val: 66.05 ± 2.11   Final Test: 49.02 ± 2.10   Final old_old: 52.85 ± 1.99   Final old_new: 33.78 ± 2.88   Final new_new: 10.91 ± 2.49
Hits@50:
  Final val: 74.41 ± 0.55   Final Test: 58.67 ± 0.97   Final old_old: 62.84 ± 1.17   Final old_new: 42.09 ± 0.91   Final new_new: 16.82 ± 2.59
AUC:
  Final val: 97.25 ± 0.10   Final Test: 96.80 ± 0.04   Final old_old: 97.40 ± 0.04   Final old_new: 94.41 ± 0.23   Final new_new: 90.32 ± 1.05
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.54 ± 2.47   Final Test: 30.54 ± 4.12   Final old_old: 33.78 ± 4.33   Final old_new: 17.41 ± 3.55   Final new_new: 3.18 ± 1.24
Hits@20:
  Final val: 58.57 ± 2.73   Final Test: 40.05 ± 3.34   Final old_old: 43.57 ± 3.50   Final old_new: 25.99 ± 2.88   Final new_new: 5.91 ± 3.80
Hits@30:
  Final val: 65.99 ± 2.04   Final Test: 49.55 ± 3.62   Final old_old: 53.48 ± 3.77   Final old_new: 33.95 ± 3.59   Final new_new: 10.00 ± 4.43
Hits@50:
  Final val: 74.81 ± 1.35   Final Test: 57.53 ± 2.47   Final old_old: 61.64 ± 2.37   Final old_new: 41.34 ± 3.28   Final new_new: 14.09 ± 7.77
AUC:
  Final val: 97.28 ± 0.07   Final Test: 96.77 ± 0.07   Final old_old: 97.32 ± 0.09   Final old_new: 94.58 ± 0.22   Final new_new: 91.07 ± 0.85
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.32 ± 4.34   Final Test: 29.56 ± 4.90   Final old_old: 32.16 ± 5.37   Final old_new: 18.94 ± 3.16   Final new_new: 8.64 ± 4.37
Hits@20:
  Final val: 55.89 ± 2.96   Final Test: 39.35 ± 4.27   Final old_old: 42.45 ± 4.58   Final old_new: 26.88 ± 3.50   Final new_new: 11.36 ± 5.33
Hits@30:
  Final val: 62.94 ± 2.86   Final Test: 46.74 ± 4.42   Final old_old: 50.00 ± 4.83   Final old_new: 33.83 ± 3.43   Final new_new: 13.64 ± 6.43
Hits@50:
  Final val: 72.39 ± 2.20   Final Test: 57.48 ± 3.22   Final old_old: 61.26 ± 3.69   Final old_new: 42.34 ± 1.56   Final new_new: 22.27 ± 8.10
AUC:
  Final val: 96.70 ± 0.32   Final Test: 96.03 ± 0.21   Final old_old: 96.71 ± 0.31   Final old_new: 93.28 ± 0.24   Final new_new: 90.11 ± 0.97
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.91 ± 8.58   Final Test: 25.96 ± 5.03   Final old_old: 28.30 ± 5.21   Final old_new: 16.50 ± 4.85   Final new_new: 5.91 ± 6.14
Hits@20:
  Final val: 54.98 ± 5.66   Final Test: 35.32 ± 2.88   Final old_old: 38.19 ± 3.10   Final old_new: 23.75 ± 2.33   Final new_new: 9.55 ± 5.66
Hits@30:
  Final val: 62.35 ± 3.93   Final Test: 43.80 ± 1.77   Final old_old: 47.24 ± 1.93   Final old_new: 29.97 ± 1.87   Final new_new: 11.82 ± 5.18
Hits@50:
  Final val: 70.64 ± 2.66   Final Test: 53.83 ± 2.96   Final old_old: 57.61 ± 2.92   Final old_new: 38.69 ± 3.90   Final new_new: 18.18 ± 5.79
AUC:
  Final val: 96.67 ± 0.22   Final Test: 95.95 ± 0.23   Final old_old: 96.62 ± 0.26   Final old_new: 93.31 ± 0.18   Final new_new: 89.37 ± 2.04
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.57 ± 2.76   Final Test: 33.24 ± 5.10   Final old_old: 36.09 ± 5.52   Final old_new: 21.79 ± 3.60   Final new_new: 7.73 ± 4.43
Hits@20:
  Final val: 56.57 ± 3.90   Final Test: 41.43 ± 5.00   Final old_old: 44.74 ± 5.43   Final old_new: 28.06 ± 3.28   Final new_new: 12.27 ± 4.43
Hits@30:
  Final val: 63.00 ± 3.73   Final Test: 48.20 ± 4.40   Final old_old: 51.71 ± 5.03   Final old_new: 34.23 ± 2.17   Final new_new: 13.64 ± 4.55
Hits@50:
  Final val: 71.91 ± 2.56   Final Test: 58.02 ± 2.32   Final old_old: 61.89 ± 2.66   Final old_new: 42.64 ± 1.26   Final new_new: 19.55 ± 5.93
AUC:
  Final val: 96.74 ± 0.43   Final Test: 96.11 ± 0.34   Final old_old: 96.81 ± 0.35   Final old_new: 93.28 ± 0.48   Final new_new: 89.76 ± 1.00
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.74 ± 2.97   Final Test: 28.97 ± 5.04   Final old_old: 31.46 ± 5.55   Final old_new: 18.84 ± 3.39   Final new_new: 8.64 ± 2.96
Hits@20:
  Final val: 54.06 ± 2.80   Final Test: 35.55 ± 3.40   Final old_old: 38.49 ± 3.69   Final old_new: 23.65 ± 2.84   Final new_new: 10.45 ± 3.05
Hits@30:
  Final val: 62.71 ± 2.44   Final Test: 42.38 ± 5.82   Final old_old: 45.57 ± 5.90   Final old_new: 29.62 ± 5.76   Final new_new: 11.82 ± 4.93
Hits@50:
  Final val: 71.11 ± 1.67   Final Test: 55.23 ± 2.99   Final old_old: 58.97 ± 3.26   Final old_new: 40.45 ± 2.35   Final new_new: 16.82 ± 6.14
AUC:
  Final val: 96.64 ± 0.24   Final Test: 95.92 ± 0.23   Final old_old: 96.55 ± 0.24   Final old_new: 93.37 ± 0.37   Final new_new: 90.08 ± 1.02
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.03 ± 8.25   Final Test: 26.63 ± 8.20   Final old_old: 28.91 ± 8.94   Final old_new: 17.43 ± 5.42   Final new_new: 5.91 ± 4.13
Hits@20:
  Final val: 56.03 ± 5.24   Final Test: 32.97 ± 6.01   Final old_old: 35.73 ± 6.36   Final old_new: 21.84 ± 4.83   Final new_new: 8.18 ± 1.24
Hits@30:
  Final val: 63.23 ± 4.33   Final Test: 43.66 ± 3.95   Final old_old: 47.12 ± 4.16   Final old_new: 29.82 ± 3.24   Final new_new: 10.91 ± 3.37
Hits@50:
  Final val: 71.68 ± 2.68   Final Test: 55.39 ± 3.74   Final old_old: 58.98 ± 4.06   Final old_new: 41.26 ± 2.52   Final new_new: 16.82 ± 5.23
AUC:
  Final val: 96.63 ± 0.20   Final Test: 95.96 ± 0.21   Final old_old: 96.58 ± 0.20   Final old_new: 93.46 ± 0.31   Final new_new: 89.59 ± 1.71
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.57 ± 1.81   Final Test: 28.63 ± 6.62   Final old_old: 31.29 ± 6.99   Final old_new: 17.76 ± 5.11   Final new_new: 6.82 ± 4.82
Hits@20:
  Final val: 55.43 ± 1.69   Final Test: 37.20 ± 3.17   Final old_old: 40.20 ± 3.31   Final old_new: 25.04 ± 2.87   Final new_new: 11.36 ± 3.59
Hits@30:
  Final val: 63.15 ± 1.58   Final Test: 47.33 ± 1.15   Final old_old: 50.88 ± 1.25   Final old_new: 33.20 ± 0.70   Final new_new: 12.27 ± 3.80
Hits@50:
  Final val: 72.41 ± 1.91   Final Test: 57.51 ± 2.52   Final old_old: 61.32 ± 2.72   Final old_new: 42.52 ± 1.85   Final new_new: 16.36 ± 2.49
AUC:
  Final val: 96.71 ± 0.12   Final Test: 96.06 ± 0.18   Final old_old: 96.72 ± 0.17   Final old_new: 93.45 ± 0.31   Final new_new: 89.08 ± 0.71
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.04 ± 2.52   Final Test: 33.27 ± 2.58   Final old_old: 36.60 ± 2.81   Final old_new: 19.90 ± 1.97   Final new_new: 2.27 ± 1.61
Hits@20:
  Final val: 61.81 ± 1.37   Final Test: 44.24 ± 3.67   Final old_old: 48.01 ± 3.98   Final old_new: 29.24 ± 2.57   Final new_new: 6.82 ± 2.78
Hits@30:
  Final val: 68.18 ± 0.76   Final Test: 50.77 ± 5.07   Final old_old: 54.57 ± 5.41   Final old_new: 35.87 ± 4.01   Final new_new: 8.64 ± 2.96
Hits@50:
  Final val: 75.56 ± 1.34   Final Test: 60.35 ± 2.58   Final old_old: 64.43 ± 2.65   Final old_new: 44.43 ± 3.15   Final new_new: 15.00 ± 3.45
AUC:
  Final val: 97.43 ± 0.19   Final Test: 96.85 ± 0.15   Final old_old: 97.46 ± 0.20   Final old_new: 94.44 ± 0.16   Final new_new: 90.43 ± 1.08
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.79 ± 2.72   Final Test: 32.99 ± 4.31   Final old_old: 36.19 ± 4.58   Final old_new: 20.03 ± 3.34   Final new_new: 5.00 ± 1.90
Hits@20:
  Final val: 62.18 ± 1.33   Final Test: 42.47 ± 2.65   Final old_old: 46.23 ± 2.83   Final old_new: 27.33 ± 2.01   Final new_new: 8.64 ± 2.96
Hits@30:
  Final val: 69.78 ± 1.52   Final Test: 51.21 ± 4.56   Final old_old: 55.25 ± 4.61   Final old_new: 35.11 ± 4.57   Final new_new: 11.82 ± 4.07
Hits@50:
  Final val: 77.55 ± 1.34   Final Test: 60.95 ± 2.30   Final old_old: 65.16 ± 2.17   Final old_new: 44.36 ± 2.96   Final new_new: 16.36 ± 4.37
AUC:
  Final val: 97.40 ± 0.15   Final Test: 96.82 ± 0.22   Final old_old: 97.45 ± 0.25   Final old_new: 94.37 ± 0.21   Final new_new: 89.88 ± 0.81
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.71 ± 5.15   Final Test: 33.96 ± 4.13   Final old_old: 37.19 ± 4.44   Final old_new: 20.96 ± 3.13   Final new_new: 5.00 ± 1.02
Hits@20:
  Final val: 64.65 ± 2.29   Final Test: 41.73 ± 3.55   Final old_old: 45.44 ± 3.75   Final old_new: 26.88 ± 3.13   Final new_new: 6.82 ± 1.61
Hits@30:
  Final val: 69.96 ± 1.66   Final Test: 50.20 ± 1.04   Final old_old: 54.31 ± 1.11   Final old_new: 33.85 ± 1.29   Final new_new: 10.00 ± 3.45
Hits@50:
  Final val: 77.33 ± 1.25   Final Test: 60.83 ± 1.96   Final old_old: 65.21 ± 2.12   Final old_new: 43.43 ± 1.97   Final new_new: 16.82 ± 4.71
AUC:
  Final val: 97.39 ± 0.15   Final Test: 96.90 ± 0.12   Final old_old: 97.50 ± 0.14   Final old_new: 94.56 ± 0.24   Final new_new: 90.50 ± 1.22
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.26 ± 3.25   Final Test: 31.82 ± 5.28   Final old_old: 34.97 ± 5.73   Final old_new: 19.12 ± 3.66   Final new_new: 3.64 ± 2.03
Hits@20:
  Final val: 62.83 ± 2.07   Final Test: 42.19 ± 4.05   Final old_old: 45.83 ± 4.05   Final old_new: 27.66 ± 4.59   Final new_new: 6.82 ± 2.27
Hits@30:
  Final val: 69.16 ± 1.28   Final Test: 51.75 ± 2.70   Final old_old: 55.81 ± 2.87   Final old_new: 35.84 ± 2.09   Final new_new: 7.73 ± 3.80
Hits@50:
  Final val: 76.33 ± 2.01   Final Test: 60.19 ± 2.89   Final old_old: 64.56 ± 2.97   Final old_new: 43.00 ± 2.56   Final new_new: 13.64 ± 4.55
AUC:
  Final val: 97.43 ± 0.19   Final Test: 96.86 ± 0.17   Final old_old: 97.50 ± 0.18   Final old_new: 94.39 ± 0.24   Final new_new: 89.74 ± 0.81
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.87 ± 5.34   Final Test: 30.99 ± 7.50   Final old_old: 34.21 ± 8.11   Final old_new: 17.96 ± 5.22   Final new_new: 3.18 ± 2.03
Hits@20:
  Final val: 63.37 ± 1.64   Final Test: 43.48 ± 2.60   Final old_old: 47.25 ± 2.79   Final old_new: 28.49 ± 1.96   Final new_new: 5.91 ± 3.45
Hits@30:
  Final val: 69.50 ± 1.35   Final Test: 51.18 ± 3.78   Final old_old: 55.28 ± 4.08   Final old_new: 34.91 ± 2.87   Final new_new: 9.55 ± 5.43
Hits@50:
  Final val: 76.61 ± 2.00   Final Test: 62.47 ± 3.77   Final old_old: 66.73 ± 4.07   Final old_new: 45.72 ± 2.77   Final new_new: 16.82 ± 2.59
AUC:
  Final val: 97.40 ± 0.19   Final Test: 96.91 ± 0.15   Final old_old: 97.52 ± 0.17   Final old_new: 94.48 ± 0.17   Final new_new: 90.71 ± 1.23
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 39.52 ± 3.58   Final Test: 27.99 ± 7.23   Final old_old: 30.42 ± 7.75   Final old_new: 18.21 ± 5.28   Final new_new: 6.36 ± 3.37
Hits@20:
  Final val: 51.59 ± 2.23   Final Test: 36.35 ± 5.16   Final old_old: 39.36 ± 5.29   Final old_new: 24.18 ± 4.98   Final new_new: 10.45 ± 6.93
Hits@30:
  Final val: 59.44 ± 1.93   Final Test: 46.38 ± 3.72   Final old_old: 49.81 ± 3.93   Final old_new: 32.77 ± 3.26   Final new_new: 12.27 ± 5.70
Hits@50:
  Final val: 69.64 ± 1.87   Final Test: 55.81 ± 2.81   Final old_old: 59.42 ± 2.80   Final old_new: 41.61 ± 3.16   Final new_new: 17.27 ± 5.23
AUC:
  Final val: 96.61 ± 0.24   Final Test: 96.04 ± 0.23   Final old_old: 96.73 ± 0.26   Final old_new: 93.32 ± 0.30   Final new_new: 89.57 ± 1.16
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.49 ± 3.16   Final Test: 34.06 ± 5.50   Final old_old: 37.38 ± 5.81   Final old_new: 20.68 ± 4.38   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 63.97 ± 1.51   Final Test: 44.87 ± 3.74   Final old_old: 48.69 ± 3.89   Final old_new: 29.70 ± 3.43   Final new_new: 6.82 ± 2.78
Hits@30:
  Final val: 70.66 ± 1.48   Final Test: 53.41 ± 3.51   Final old_old: 57.70 ± 3.68   Final old_new: 36.50 ± 3.02   Final new_new: 8.64 ± 1.90
Hits@50:
  Final val: 77.46 ± 1.56   Final Test: 62.09 ± 2.63   Final old_old: 66.65 ± 2.91   Final old_new: 44.08 ± 1.99   Final new_new: 14.09 ± 4.07
AUC:
  Final val: 97.47 ± 0.19   Final Test: 96.91 ± 0.15   Final old_old: 97.52 ± 0.18   Final old_new: 94.54 ± 0.24   Final new_new: 90.00 ± 1.05
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.01 ± 5.36   Final Test: 29.47 ± 6.91   Final old_old: 32.35 ± 7.37   Final old_new: 17.81 ± 5.10   Final new_new: 4.55 ± 2.78
Hits@20:
  Final val: 55.77 ± 5.11   Final Test: 40.02 ± 6.91   Final old_old: 43.42 ± 7.42   Final old_new: 26.35 ± 5.22   Final new_new: 9.09 ± 2.78
Hits@30:
  Final val: 63.74 ± 3.28   Final Test: 47.04 ± 3.66   Final old_old: 50.82 ± 3.96   Final old_new: 32.02 ± 3.07   Final new_new: 9.55 ± 2.49
Hits@50:
  Final val: 72.28 ± 2.20   Final Test: 57.33 ± 3.06   Final old_old: 61.39 ± 3.41   Final old_new: 41.34 ± 1.92   Final new_new: 14.09 ± 3.73
AUC:
  Final val: 97.15 ± 0.15   Final Test: 96.68 ± 0.11   Final old_old: 97.24 ± 0.13   Final old_new: 94.48 ± 0.34   Final new_new: 90.74 ± 0.75
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.37 ± 3.75   Final Test: 30.03 ± 3.47   Final old_old: 33.10 ± 3.94   Final old_new: 17.56 ± 1.46   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 59.90 ± 2.67   Final Test: 39.91 ± 1.53   Final old_old: 43.44 ± 1.61   Final old_new: 25.74 ± 1.54   Final new_new: 7.73 ± 2.59
Hits@30:
  Final val: 66.73 ± 0.63   Final Test: 48.77 ± 1.49   Final old_old: 52.74 ± 1.62   Final old_new: 32.85 ± 1.18   Final new_new: 12.27 ± 4.71
Hits@50:
  Final val: 74.24 ± 1.24   Final Test: 58.66 ± 2.25   Final old_old: 63.06 ± 2.28   Final old_new: 41.13 ± 2.85   Final new_new: 16.36 ± 2.96
AUC:
  Final val: 97.27 ± 0.10   Final Test: 96.75 ± 0.07   Final old_old: 97.36 ± 0.10   Final old_new: 94.36 ± 0.17   Final new_new: 90.18 ± 1.79
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.90 ± 2.97   Final Test: 29.18 ± 2.86   Final old_old: 32.11 ± 3.14   Final old_new: 17.23 ± 2.10   Final new_new: 5.91 ± 1.24
Hits@20:
  Final val: 58.64 ± 3.18   Final Test: 37.30 ± 2.72   Final old_old: 40.76 ± 2.98   Final old_new: 23.32 ± 1.79   Final new_new: 6.82 ± 1.61
Hits@30:
  Final val: 66.14 ± 2.84   Final Test: 46.59 ± 2.17   Final old_old: 50.41 ± 2.34   Final old_new: 31.36 ± 1.59   Final new_new: 9.55 ± 1.90
Hits@50:
  Final val: 73.38 ± 2.31   Final Test: 56.88 ± 4.96   Final old_old: 60.95 ± 4.91   Final old_new: 40.86 ± 5.42   Final new_new: 13.64 ± 4.55
AUC:
  Final val: 97.21 ± 0.15   Final Test: 96.74 ± 0.13   Final old_old: 97.30 ± 0.15   Final old_new: 94.55 ± 0.11   Final new_new: 90.63 ± 0.60
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.50 ± 2.98   Final Test: 28.40 ± 5.24   Final old_old: 30.91 ± 5.58   Final old_new: 18.31 ± 4.01   Final new_new: 5.45 ± 3.45
Hits@20:
  Final val: 56.58 ± 4.53   Final Test: 37.16 ± 5.45   Final old_old: 40.12 ± 5.70   Final old_new: 25.31 ± 4.84   Final new_new: 8.64 ± 3.37
Hits@30:
  Final val: 63.89 ± 2.37   Final Test: 44.22 ± 3.38   Final old_old: 47.56 ± 3.48   Final old_new: 31.01 ± 3.00   Final new_new: 10.00 ± 6.35
Hits@50:
  Final val: 73.24 ± 2.01   Final Test: 58.88 ± 3.15   Final old_old: 62.65 ± 3.15   Final old_new: 44.03 ± 3.17   Final new_new: 18.64 ± 9.83
AUC:
  Final val: 96.66 ± 0.19   Final Test: 96.00 ± 0.19   Final old_old: 96.66 ± 0.19   Final old_new: 93.42 ± 0.27   Final new_new: 89.24 ± 1.40
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.01 ± 1.60   Final Test: 33.26 ± 5.33   Final old_old: 36.52 ± 5.86   Final old_new: 20.20 ± 3.34   Final new_new: 3.18 ± 1.24
Hits@20:
  Final val: 61.77 ± 1.39   Final Test: 43.77 ± 2.98   Final old_old: 47.58 ± 3.02   Final old_new: 28.51 ± 3.02   Final new_new: 8.18 ± 2.03
Hits@30:
  Final val: 68.95 ± 1.14   Final Test: 50.68 ± 2.56   Final old_old: 54.86 ± 2.69   Final old_new: 34.06 ± 2.49   Final new_new: 9.09 ± 1.61
Hits@50:
  Final val: 76.99 ± 0.63   Final Test: 62.60 ± 1.64   Final old_old: 67.14 ± 1.70   Final old_new: 44.66 ± 2.00   Final new_new: 15.45 ± 3.37
AUC:
  Final val: 97.43 ± 0.15   Final Test: 96.87 ± 0.14   Final old_old: 97.59 ± 0.18   Final old_new: 93.98 ± 0.18   Final new_new: 89.96 ± 1.52
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.58 ± 3.16   Final Test: 29.88 ± 3.76   Final old_old: 32.96 ± 4.12   Final old_new: 17.38 ± 2.37   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 61.02 ± 2.79   Final Test: 42.04 ± 3.79   Final old_old: 45.85 ± 4.12   Final old_new: 26.78 ± 2.78   Final new_new: 6.36 ± 2.96
Hits@30:
  Final val: 66.81 ± 2.23   Final Test: 46.77 ± 5.02   Final old_old: 50.68 ± 5.36   Final old_new: 31.21 ± 3.85   Final new_new: 8.64 ± 3.73
Hits@50:
  Final val: 75.02 ± 1.42   Final Test: 57.73 ± 1.69   Final old_old: 62.08 ± 1.68   Final old_new: 40.43 ± 2.19   Final new_new: 15.00 ± 4.98
AUC:
  Final val: 97.29 ± 0.09   Final Test: 96.76 ± 0.07   Final old_old: 97.36 ± 0.06   Final old_new: 94.39 ± 0.13   Final new_new: 90.44 ± 0.72
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.36 ± 5.36   Final Test: 28.49 ± 6.08   Final old_old: 31.10 ± 6.50   Final old_new: 17.91 ± 4.45   Final new_new: 5.91 ± 4.43
Hits@20:
  Final val: 53.79 ± 4.28   Final Test: 36.74 ± 10.73   Final old_old: 39.65 ± 11.31   Final old_new: 25.06 ± 8.42   Final new_new: 9.55 ± 6.10
Hits@30:
  Final val: 61.38 ± 3.70   Final Test: 46.01 ± 4.01   Final old_old: 49.54 ± 4.16   Final old_new: 31.99 ± 4.02   Final new_new: 10.91 ± 2.96
Hits@50:
  Final val: 71.01 ± 2.10   Final Test: 55.70 ± 5.27   Final old_old: 59.40 ± 5.42   Final old_new: 41.28 ± 4.93   Final new_new: 13.64 ± 3.59
AUC:
  Final val: 96.62 ± 0.20   Final Test: 95.98 ± 0.21   Final old_old: 96.62 ± 0.23   Final old_new: 93.40 ± 0.24   Final new_new: 89.87 ± 0.88
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.45 ± 3.44   Final Test: 27.97 ± 5.30   Final old_old: 30.56 ± 5.66   Final old_new: 17.43 ± 4.22   Final new_new: 6.36 ± 1.90
Hits@20:
  Final val: 55.76 ± 4.48   Final Test: 38.70 ± 3.53   Final old_old: 41.68 ± 3.96   Final old_new: 26.78 ± 2.29   Final new_new: 10.00 ± 6.14
Hits@30:
  Final val: 63.62 ± 3.57   Final Test: 44.58 ± 1.33   Final old_old: 48.03 ± 1.35   Final old_new: 30.78 ± 1.65   Final new_new: 11.82 ± 4.93
Hits@50:
  Final val: 72.89 ± 2.57   Final Test: 57.12 ± 1.14   Final old_old: 60.76 ± 1.55   Final old_new: 43.00 ± 1.96   Final new_new: 15.45 ± 4.07
AUC:
  Final val: 96.68 ± 0.27   Final Test: 95.97 ± 0.29   Final old_old: 96.64 ± 0.29   Final old_new: 93.27 ± 0.29   Final new_new: 89.31 ± 0.47
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.94 ± 6.74   Final Test: 28.59 ± 6.19   Final old_old: 31.17 ± 6.56   Final old_new: 18.19 ± 4.85   Final new_new: 5.91 ± 3.45
Hits@20:
  Final val: 54.86 ± 4.30   Final Test: 37.60 ± 6.36   Final old_old: 40.54 ± 6.58   Final old_new: 25.82 ± 5.59   Final new_new: 10.00 ± 5.23
Hits@30:
  Final val: 62.75 ± 3.19   Final Test: 43.62 ± 4.48   Final old_old: 47.01 ± 4.71   Final old_new: 30.00 ± 3.70   Final new_new: 12.27 ± 7.64
Hits@50:
  Final val: 72.25 ± 2.38   Final Test: 56.88 ± 1.88   Final old_old: 60.62 ± 1.81   Final old_new: 41.99 ± 2.48   Final new_new: 20.00 ± 7.43
AUC:
  Final val: 96.65 ± 0.34   Final Test: 95.89 ± 0.20   Final old_old: 96.55 ± 0.26   Final old_new: 93.26 ± 0.26   Final new_new: 89.72 ± 2.10
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.09 ± 3.58   Final Test: 33.32 ± 6.57   Final old_old: 36.68 ± 7.21   Final old_new: 19.77 ± 4.01   Final new_new: 3.64 ± 2.03
Hits@20:
  Final val: 63.97 ± 2.12   Final Test: 45.60 ± 7.43   Final old_old: 49.52 ± 8.02   Final old_new: 29.90 ± 5.30   Final new_new: 8.18 ± 2.59
Hits@30:
  Final val: 70.58 ± 1.72   Final Test: 53.48 ± 3.80   Final old_old: 57.70 ± 3.85   Final old_new: 36.75 ± 4.16   Final new_new: 10.91 ± 1.02
Hits@50:
  Final val: 78.18 ± 1.44   Final Test: 61.79 ± 3.79   Final old_old: 66.10 ± 3.77   Final old_new: 44.86 ± 4.22   Final new_new: 15.91 ± 4.25
AUC:
  Final val: 97.50 ± 0.17   Final Test: 96.96 ± 0.17   Final old_old: 97.59 ± 0.20   Final old_new: 94.46 ± 0.19   Final new_new: 90.22 ± 0.94
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.02 ± 2.97   Final Test: 31.98 ± 7.04   Final old_old: 34.72 ± 7.54   Final old_new: 20.78 ± 5.12   Final new_new: 10.45 ± 3.45
Hits@20:
  Final val: 56.74 ± 2.90   Final Test: 40.00 ± 3.76   Final old_old: 43.04 ± 4.00   Final old_new: 27.78 ± 2.88   Final new_new: 11.82 ± 3.73
Hits@30:
  Final val: 64.27 ± 2.28   Final Test: 46.40 ± 2.71   Final old_old: 49.63 ± 2.87   Final old_new: 33.65 ± 2.19   Final new_new: 13.18 ± 5.18
Hits@50:
  Final val: 73.01 ± 1.27   Final Test: 57.14 ± 3.26   Final old_old: 60.80 ± 3.29   Final old_new: 42.72 ± 3.18   Final new_new: 19.09 ± 4.71
AUC:
  Final val: 96.71 ± 0.29   Final Test: 96.08 ± 0.20   Final old_old: 96.73 ± 0.21   Final old_new: 93.46 ± 0.37   Final new_new: 89.83 ± 1.57
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.19 ± 2.97   Final Test: 37.91 ± 5.21   Final old_old: 41.67 ± 5.46   Final old_new: 22.59 ± 4.65   Final new_new: 6.82 ± 2.78
Hits@20:
  Final val: 61.49 ± 2.63   Final Test: 45.26 ± 5.21   Final old_old: 49.42 ± 5.48   Final old_new: 28.49 ± 4.51   Final new_new: 8.64 ± 4.07
Hits@30:
  Final val: 67.58 ± 1.94   Final Test: 53.25 ± 5.21   Final old_old: 57.78 ± 5.60   Final old_new: 35.16 ± 4.31   Final new_new: 9.55 ± 2.49
Hits@50:
  Final val: 75.86 ± 1.61   Final Test: 62.93 ± 3.91   Final old_old: 67.84 ± 4.13   Final old_new: 43.43 ± 3.24   Final new_new: 14.09 ± 5.18
AUC:
  Final val: 97.48 ± 0.10   Final Test: 96.35 ± 0.21   Final old_old: 97.62 ± 0.24   Final old_new: 91.22 ± 0.49   Final new_new: 85.59 ± 1.69
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.30 ± 2.95   Final Test: 31.42 ± 2.61   Final old_old: 34.41 ± 2.59   Final old_new: 19.42 ± 2.93   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 62.96 ± 1.96   Final Test: 41.65 ± 2.64   Final old_old: 45.52 ± 2.87   Final old_new: 26.12 ± 1.78   Final new_new: 5.91 ± 3.05
Hits@30:
  Final val: 69.60 ± 1.03   Final Test: 49.97 ± 5.04   Final old_old: 53.93 ± 4.93   Final old_new: 34.21 ± 5.62   Final new_new: 10.45 ± 5.47
Hits@50:
  Final val: 77.56 ± 1.01   Final Test: 61.60 ± 2.87   Final old_old: 65.92 ± 2.75   Final old_new: 44.58 ± 3.84   Final new_new: 16.36 ± 5.18
AUC:
  Final val: 97.41 ± 0.18   Final Test: 96.83 ± 0.12   Final old_old: 97.48 ± 0.14   Final old_new: 94.24 ± 0.20   Final new_new: 90.45 ± 0.60
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.74 ± 3.90   Final Test: 33.08 ± 4.93   Final old_old: 36.51 ± 5.28   Final old_new: 19.14 ± 3.75   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 63.40 ± 2.20   Final Test: 43.79 ± 1.42   Final old_old: 47.58 ± 1.42   Final old_new: 28.56 ± 1.96   Final new_new: 8.64 ± 1.90
Hits@30:
  Final val: 69.50 ± 2.24   Final Test: 50.86 ± 2.51   Final old_old: 54.93 ± 2.43   Final old_new: 34.66 ± 3.00   Final new_new: 10.91 ± 2.96
Hits@50:
  Final val: 76.42 ± 1.82   Final Test: 58.56 ± 2.67   Final old_old: 62.84 ± 2.67   Final old_new: 41.64 ± 2.71   Final new_new: 15.00 ± 4.71
AUC:
  Final val: 97.41 ± 0.14   Final Test: 96.83 ± 0.13   Final old_old: 97.46 ± 0.15   Final old_new: 94.34 ± 0.35   Final new_new: 89.90 ± 1.10
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.00 ± 4.44   Final Test: 30.79 ± 5.43   Final old_old: 34.00 ± 5.81   Final old_new: 17.83 ± 4.19   Final new_new: 2.27 ± 0.00
Hits@20:
  Final val: 63.63 ± 4.83   Final Test: 43.06 ± 2.56   Final old_old: 46.87 ± 2.62   Final old_new: 27.98 ± 2.54   Final new_new: 4.09 ± 1.90
Hits@30:
  Final val: 70.15 ± 2.33   Final Test: 52.14 ± 2.63   Final old_old: 56.37 ± 2.70   Final old_new: 35.47 ± 2.58   Final new_new: 8.18 ± 2.03
Hits@50:
  Final val: 77.09 ± 2.06   Final Test: 60.93 ± 0.68   Final old_old: 65.24 ± 0.88   Final old_new: 43.98 ± 1.52   Final new_new: 14.55 ± 2.03
AUC:
  Final val: 97.44 ± 0.15   Final Test: 96.90 ± 0.11   Final old_old: 97.49 ± 0.15   Final old_new: 94.58 ± 0.13   Final new_new: 90.83 ± 1.44
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.11 ± 7.10   Final Test: 33.34 ± 7.22   Final old_old: 36.22 ± 7.61   Final old_new: 21.96 ± 5.94   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 59.97 ± 3.65   Final Test: 42.17 ± 7.79   Final old_old: 45.44 ± 8.12   Final old_new: 29.24 ± 6.61   Final new_new: 7.73 ± 3.45
Hits@30:
  Final val: 67.41 ± 3.17   Final Test: 47.42 ± 7.72   Final old_old: 50.86 ± 7.95   Final old_new: 33.95 ± 6.95   Final new_new: 9.55 ± 3.73
Hits@50:
  Final val: 75.09 ± 2.83   Final Test: 57.92 ± 2.62   Final old_old: 61.70 ± 2.78   Final old_new: 43.22 ± 2.31   Final new_new: 15.00 ± 2.59
AUC:
  Final val: 96.82 ± 0.19   Final Test: 96.19 ± 0.27   Final old_old: 96.81 ± 0.23   Final old_new: 93.67 ± 0.45   Final new_new: 90.40 ± 0.96
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.20 ± 3.70   Final Test: 35.87 ± 1.83   Final old_old: 39.62 ± 1.93   Final old_new: 20.78 ± 1.54   Final new_new: 2.73 ± 1.90
Hits@20:
  Final val: 62.22 ± 2.83   Final Test: 46.34 ± 3.82   Final old_old: 50.72 ± 3.93   Final old_new: 28.74 ± 3.33   Final new_new: 6.82 ± 5.08
Hits@30:
  Final val: 69.73 ± 1.18   Final Test: 55.23 ± 1.87   Final old_old: 59.99 ± 1.77   Final old_new: 36.22 ± 2.24   Final new_new: 9.55 ± 4.93
Hits@50:
  Final val: 77.78 ± 0.68   Final Test: 64.28 ± 4.06   Final old_old: 69.22 ± 4.11   Final old_new: 44.56 ± 3.92   Final new_new: 16.82 ± 5.23
AUC:
  Final val: 97.81 ± 0.10   Final Test: 96.64 ± 0.12   Final old_old: 97.96 ± 0.06   Final old_new: 91.29 ± 0.56   Final new_new: 85.66 ± 1.82
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.62 ± 3.39   Final Test: 31.64 ± 5.44   Final old_old: 34.85 ± 5.77   Final old_new: 18.64 ± 4.32   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 64.03 ± 1.76   Final Test: 42.25 ± 1.88   Final old_old: 46.01 ± 2.00   Final old_new: 27.15 ± 1.77   Final new_new: 8.18 ± 2.59
Hits@30:
  Final val: 70.32 ± 2.80   Final Test: 51.34 ± 2.73   Final old_old: 55.46 ± 2.81   Final old_new: 34.94 ± 2.77   Final new_new: 10.91 ± 3.73
Hits@50:
  Final val: 77.36 ± 2.01   Final Test: 60.55 ± 2.54   Final old_old: 64.85 ± 2.86   Final old_new: 43.58 ± 1.77   Final new_new: 15.00 ± 5.70
AUC:
  Final val: 97.45 ± 0.15   Final Test: 96.84 ± 0.17   Final old_old: 97.47 ± 0.17   Final old_new: 94.32 ± 0.27   Final new_new: 90.39 ± 0.46
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.81 ± 1.36   Final Test: 32.22 ± 5.20   Final old_old: 35.34 ± 5.68   Final old_new: 19.70 ± 3.36   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 59.86 ± 2.27   Final Test: 41.41 ± 3.28   Final old_old: 44.93 ± 3.44   Final old_new: 27.36 ± 3.12   Final new_new: 7.73 ± 2.03
Hits@30:
  Final val: 66.25 ± 2.86   Final Test: 51.21 ± 3.48   Final old_old: 55.23 ± 3.60   Final old_new: 35.21 ± 3.92   Final new_new: 11.36 ± 3.59
Hits@50:
  Final val: 74.27 ± 1.27   Final Test: 59.60 ± 1.36   Final old_old: 63.90 ± 1.08   Final old_new: 42.59 ± 2.82   Final new_new: 15.45 ± 6.10
AUC:
  Final val: 97.28 ± 0.10   Final Test: 96.80 ± 0.03   Final old_old: 97.41 ± 0.09   Final old_new: 94.41 ± 0.28   Final new_new: 90.31 ± 0.89
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.81 ± 3.09   Final Test: 31.52 ± 3.50   Final old_old: 34.63 ± 3.49   Final old_new: 18.94 ± 3.82   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 59.70 ± 2.20   Final Test: 43.63 ± 5.04   Final old_old: 47.36 ± 5.34   Final old_new: 28.69 ± 3.95   Final new_new: 9.09 ± 1.61
Hits@30:
  Final val: 66.20 ± 2.05   Final Test: 48.13 ± 3.24   Final old_old: 52.10 ± 3.41   Final old_new: 32.29 ± 2.61   Final new_new: 9.55 ± 1.02
Hits@50:
  Final val: 74.76 ± 0.43   Final Test: 59.54 ± 1.70   Final old_old: 63.83 ± 1.79   Final old_new: 42.49 ± 1.35   Final new_new: 16.36 ± 1.90
AUC:
  Final val: 97.27 ± 0.12   Final Test: 96.76 ± 0.10   Final old_old: 97.35 ± 0.08   Final old_new: 94.44 ± 0.18   Final new_new: 90.55 ± 0.81
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.76 ± 2.07   Final Test: 33.13 ± 3.06   Final old_old: 36.26 ± 3.14   Final old_new: 20.53 ± 2.97   Final new_new: 5.00 ± 2.96
Hits@20:
  Final val: 61.60 ± 2.28   Final Test: 42.88 ± 1.51   Final old_old: 46.41 ± 1.60   Final old_new: 28.94 ± 1.92   Final new_new: 5.91 ± 3.45
Hits@30:
  Final val: 68.70 ± 2.62   Final Test: 49.61 ± 4.18   Final old_old: 53.42 ± 4.16   Final old_new: 34.74 ± 4.37   Final new_new: 7.27 ± 4.66
Hits@50:
  Final val: 75.73 ± 1.24   Final Test: 60.86 ± 2.76   Final old_old: 64.93 ± 2.75   Final old_new: 45.09 ± 3.01   Final new_new: 12.73 ± 3.80
AUC:
  Final val: 97.33 ± 0.09   Final Test: 96.87 ± 0.07   Final old_old: 97.46 ± 0.05   Final old_new: 94.58 ± 0.18   Final new_new: 90.03 ± 0.80
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.27 ± 2.96   Final Test: 34.05 ± 3.54   Final old_old: 37.39 ± 3.77   Final old_new: 20.45 ± 2.92   Final new_new: 5.91 ± 2.59
Hits@20:
  Final val: 65.92 ± 2.57   Final Test: 41.59 ± 1.35   Final old_old: 45.32 ± 1.33   Final old_new: 26.68 ± 1.64   Final new_new: 6.36 ± 1.90
Hits@30:
  Final val: 72.11 ± 2.26   Final Test: 51.31 ± 2.19   Final old_old: 55.25 ± 2.32   Final old_new: 35.84 ± 1.77   Final new_new: 8.64 ± 1.90
Hits@50:
  Final val: 79.11 ± 1.31   Final Test: 62.35 ± 1.21   Final old_old: 66.64 ± 1.35   Final old_new: 45.67 ± 1.23   Final new_new: 13.18 ± 4.37
AUC:
  Final val: 97.63 ± 0.10   Final Test: 97.09 ± 0.12   Final old_old: 97.64 ± 0.14   Final old_new: 94.97 ± 0.12   Final new_new: 90.99 ± 1.68
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.20 ± 5.37   Final Test: 29.73 ± 6.09   Final old_old: 32.42 ± 6.57   Final old_new: 18.66 ± 4.49   Final new_new: 10.00 ± 2.59
Hits@20:
  Final val: 55.72 ± 4.15   Final Test: 37.20 ± 4.32   Final old_old: 40.24 ± 4.66   Final old_new: 24.94 ± 3.54   Final new_new: 10.00 ± 2.59
Hits@30:
  Final val: 62.52 ± 2.86   Final Test: 44.27 ± 2.59   Final old_old: 47.64 ± 2.72   Final old_new: 30.81 ± 2.70   Final new_new: 11.82 ± 4.37
Hits@50:
  Final val: 72.33 ± 2.44   Final Test: 57.22 ± 2.47   Final old_old: 60.90 ± 2.61   Final old_new: 42.64 ± 2.32   Final new_new: 20.45 ± 8.65
AUC:
  Final val: 96.64 ± 0.31   Final Test: 96.03 ± 0.25   Final old_old: 96.69 ± 0.29   Final old_new: 93.36 ± 0.27   Final new_new: 90.29 ± 1.22
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.52 ± 5.55   Final Test: 29.25 ± 4.32   Final old_old: 31.91 ± 4.73   Final old_new: 18.44 ± 2.93   Final new_new: 7.27 ± 4.07
Hits@20:
  Final val: 54.99 ± 4.91   Final Test: 38.09 ± 4.75   Final old_old: 41.35 ± 5.03   Final old_new: 24.91 ± 4.02   Final new_new: 9.55 ± 4.37
Hits@30:
  Final val: 62.94 ± 4.26   Final Test: 46.62 ± 2.84   Final old_old: 50.05 ± 2.71   Final old_new: 32.95 ± 3.47   Final new_new: 12.73 ± 7.29
Hits@50:
  Final val: 71.00 ± 3.55   Final Test: 56.51 ± 3.58   Final old_old: 60.16 ± 3.74   Final old_new: 42.17 ± 3.31   Final new_new: 17.27 ± 6.14
AUC:
  Final val: 96.59 ± 0.29   Final Test: 95.91 ± 0.18   Final old_old: 96.58 ± 0.23   Final old_new: 93.18 ± 0.24   Final new_new: 90.65 ± 1.04
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.66 ± 3.47   Final Test: 30.56 ± 2.91   Final old_old: 33.12 ± 3.11   Final old_new: 20.25 ± 2.16   Final new_new: 7.73 ± 2.59
Hits@20:
  Final val: 56.85 ± 2.95   Final Test: 39.79 ± 2.85   Final old_old: 42.73 ± 3.20   Final old_new: 28.11 ± 1.56   Final new_new: 10.45 ± 3.80
Hits@30:
  Final val: 63.49 ± 3.18   Final Test: 47.93 ± 3.49   Final old_old: 51.49 ± 3.75   Final old_new: 33.68 ± 2.90   Final new_new: 14.09 ± 2.96
Hits@50:
  Final val: 71.10 ± 2.50   Final Test: 56.34 ± 1.71   Final old_old: 60.11 ± 2.03   Final old_new: 41.41 ± 1.53   Final new_new: 18.18 ± 5.08
AUC:
  Final val: 96.54 ± 0.29   Final Test: 95.83 ± 0.30   Final old_old: 96.59 ± 0.31   Final old_new: 92.77 ± 0.46   Final new_new: 88.80 ± 0.92
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.36 ± 3.07   Final Test: 32.14 ± 3.07   Final old_old: 35.23 ± 3.27   Final old_new: 19.72 ± 2.82   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 58.74 ± 3.90   Final Test: 41.95 ± 4.60   Final old_old: 45.51 ± 4.95   Final old_new: 27.81 ± 3.23   Final new_new: 7.27 ± 3.37
Hits@30:
  Final val: 66.65 ± 2.75   Final Test: 48.72 ± 3.79   Final old_old: 52.54 ± 4.15   Final old_new: 33.60 ± 2.52   Final new_new: 9.55 ± 1.90
Hits@50:
  Final val: 74.82 ± 0.95   Final Test: 57.58 ± 0.86   Final old_old: 61.85 ± 1.04   Final old_new: 40.76 ± 1.09   Final new_new: 12.27 ± 2.59
AUC:
  Final val: 97.27 ± 0.08   Final Test: 96.76 ± 0.13   Final old_old: 97.35 ± 0.11   Final old_new: 94.43 ± 0.24   Final new_new: 90.29 ± 0.97
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.77 ± 7.45   Final Test: 22.05 ± 6.30   Final old_old: 24.21 ± 6.93   Final old_new: 13.25 ± 3.88   Final new_new: 4.55 ± 2.78
Hits@20:
  Final val: 54.82 ± 6.17   Final Test: 35.21 ± 4.83   Final old_old: 38.30 ± 5.23   Final old_new: 22.67 ± 3.61   Final new_new: 9.55 ± 4.93
Hits@30:
  Final val: 61.54 ± 4.24   Final Test: 43.01 ± 2.24   Final old_old: 46.31 ± 2.72   Final old_new: 29.82 ± 1.03   Final new_new: 11.36 ± 6.63
Hits@50:
  Final val: 70.83 ± 3.75   Final Test: 56.19 ± 5.31   Final old_old: 60.14 ± 5.53   Final old_new: 40.58 ± 4.75   Final new_new: 15.91 ± 6.22
AUC:
  Final val: 96.62 ± 0.20   Final Test: 96.09 ± 0.29   Final old_old: 96.77 ± 0.23   Final old_new: 93.36 ± 0.64   Final new_new: 89.81 ± 1.45
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.10 ± 4.92   Final Test: 34.32 ± 7.36   Final old_old: 37.09 ± 7.61   Final old_new: 23.17 ± 6.52   Final new_new: 9.09 ± 3.59
Hits@20:
  Final val: 61.67 ± 4.12   Final Test: 44.07 ± 6.90   Final old_old: 47.16 ± 7.50   Final old_new: 31.89 ± 4.78   Final new_new: 11.36 ± 3.59
Hits@30:
  Final val: 67.61 ± 3.79   Final Test: 51.31 ± 1.31   Final old_old: 54.75 ± 1.43   Final old_new: 37.68 ± 1.17   Final new_new: 15.91 ± 4.25
Hits@50:
  Final val: 75.04 ± 2.14   Final Test: 59.52 ± 4.04   Final old_old: 63.22 ± 4.07   Final old_new: 44.94 ± 4.14   Final new_new: 20.00 ± 2.96
AUC:
  Final val: 97.07 ± 0.10   Final Test: 96.50 ± 0.14   Final old_old: 97.09 ± 0.14   Final old_new: 94.14 ± 0.34   Final new_new: 90.60 ± 1.20
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.35 ± 4.25   Final Test: 30.76 ± 4.45   Final old_old: 33.86 ± 4.76   Final old_new: 18.24 ± 3.31   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 62.48 ± 2.88   Final Test: 41.74 ± 4.24   Final old_old: 45.38 ± 4.39   Final old_new: 27.23 ± 3.72   Final new_new: 5.91 ± 3.05
Hits@30:
  Final val: 69.17 ± 1.72   Final Test: 50.41 ± 4.33   Final old_old: 54.34 ± 4.53   Final old_new: 34.94 ± 3.73   Final new_new: 8.18 ± 1.24
Hits@50:
  Final val: 76.50 ± 1.13   Final Test: 61.11 ± 4.71   Final old_old: 65.52 ± 4.78   Final old_new: 43.78 ± 4.70   Final new_new: 13.64 ± 4.25
AUC:
  Final val: 97.39 ± 0.11   Final Test: 96.84 ± 0.11   Final old_old: 97.46 ± 0.13   Final old_new: 94.42 ± 0.20   Final new_new: 90.29 ± 1.33
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.90 ± 3.48   Final Test: 26.08 ± 2.85   Final old_old: 28.66 ± 2.97   Final old_new: 15.57 ± 2.82   Final new_new: 5.45 ± 2.59
Hits@20:
  Final val: 55.69 ± 2.58   Final Test: 35.42 ± 5.90   Final old_old: 38.36 ± 6.18   Final old_new: 23.60 ± 4.93   Final new_new: 8.64 ± 2.96
Hits@30:
  Final val: 62.42 ± 2.22   Final Test: 46.54 ± 2.02   Final old_old: 50.05 ± 2.21   Final old_new: 32.44 ± 1.75   Final new_new: 15.00 ± 3.45
Hits@50:
  Final val: 71.09 ± 1.55   Final Test: 57.62 ± 3.62   Final old_old: 61.32 ± 3.83   Final old_new: 42.95 ± 3.16   Final new_new: 20.45 ± 4.25
AUC:
  Final val: 96.63 ± 0.25   Final Test: 95.96 ± 0.23   Final old_old: 96.64 ± 0.28   Final old_new: 93.26 ± 0.47   Final new_new: 89.52 ± 1.61
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.89 ± 2.21   Final Test: 30.80 ± 5.29   Final old_old: 33.88 ± 5.74   Final old_new: 18.26 ± 3.55   Final new_new: 5.00 ± 2.96
Hits@20:
  Final val: 64.27 ± 1.54   Final Test: 44.94 ± 3.30   Final old_old: 48.70 ± 3.53   Final old_new: 30.00 ± 2.34   Final new_new: 7.27 ± 5.18
Hits@30:
  Final val: 69.62 ± 1.27   Final Test: 53.51 ± 2.59   Final old_old: 57.63 ± 2.73   Final old_new: 37.20 ± 2.47   Final new_new: 11.36 ± 2.27
Hits@50:
  Final val: 77.65 ± 1.39   Final Test: 60.65 ± 2.03   Final old_old: 64.97 ± 2.35   Final old_new: 43.68 ± 1.19   Final new_new: 14.55 ± 2.59
AUC:
  Final val: 97.37 ± 0.19   Final Test: 96.92 ± 0.12   Final old_old: 97.56 ± 0.14   Final old_new: 94.37 ± 0.24   Final new_new: 90.33 ± 0.64
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.35 ± 2.38   Final Test: 34.19 ± 4.46   Final old_old: 37.72 ± 5.00   Final old_new: 19.87 ± 2.69   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 61.75 ± 2.13   Final Test: 44.00 ± 4.58   Final old_old: 48.14 ± 4.86   Final old_new: 27.43 ± 3.94   Final new_new: 5.45 ± 3.45
Hits@30:
  Final val: 68.42 ± 1.45   Final Test: 51.67 ± 1.81   Final old_old: 56.14 ± 2.01   Final old_new: 33.88 ± 1.69   Final new_new: 8.18 ± 4.13
Hits@50:
  Final val: 76.22 ± 1.54   Final Test: 59.90 ± 1.02   Final old_old: 64.49 ± 1.24   Final old_new: 41.74 ± 0.90   Final new_new: 12.27 ± 3.45
AUC:
  Final val: 97.33 ± 0.23   Final Test: 96.61 ± 0.22   Final old_old: 97.43 ± 0.22   Final old_new: 93.35 ± 0.44   Final new_new: 88.76 ± 0.87
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.70 ± 5.78   Final Test: 31.38 ± 2.04   Final old_old: 34.66 ± 2.25   Final old_new: 18.06 ± 1.98   Final new_new: 4.09 ± 1.02
Hits@20:
  Final val: 62.89 ± 4.92   Final Test: 41.66 ± 2.97   Final old_old: 45.48 ± 3.15   Final old_new: 26.37 ± 2.51   Final new_new: 5.45 ± 2.03
Hits@30:
  Final val: 69.22 ± 3.74   Final Test: 51.56 ± 3.95   Final old_old: 55.90 ± 4.13   Final old_new: 34.23 ± 3.41   Final new_new: 10.00 ± 4.13
Hits@50:
  Final val: 76.35 ± 2.14   Final Test: 61.08 ± 3.54   Final old_old: 65.49 ± 3.73   Final old_new: 43.63 ± 2.67   Final new_new: 16.36 ± 6.89
AUC:
  Final val: 97.38 ± 0.13   Final Test: 96.91 ± 0.09   Final old_old: 97.51 ± 0.13   Final old_new: 94.49 ± 0.33   Final new_new: 91.48 ± 0.84
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 54.11 ± 3.87   Final Test: 36.67 ± 7.16   Final old_old: 40.08 ± 7.53   Final old_new: 23.10 ± 6.04   Final new_new: 3.64 ± 3.05
Hits@20:
  Final val: 66.72 ± 3.45   Final Test: 45.46 ± 5.41   Final old_old: 49.22 ± 5.84   Final old_new: 30.55 ± 3.85   Final new_new: 7.27 ± 2.96
Hits@30:
  Final val: 72.07 ± 2.18   Final Test: 54.57 ± 3.54   Final old_old: 58.44 ± 3.89   Final old_new: 39.37 ± 2.31   Final new_new: 12.73 ± 5.23
Hits@50:
  Final val: 78.55 ± 1.67   Final Test: 64.76 ± 2.16   Final old_old: 68.72 ± 2.26   Final old_new: 49.40 ± 2.56   Final new_new: 18.64 ± 3.37
AUC:
  Final val: 97.67 ± 0.16   Final Test: 97.31 ± 0.10   Final old_old: 97.81 ± 0.10   Final old_new: 95.34 ± 0.10   Final new_new: 92.19 ± 0.97
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.53 ± 4.29   Final Test: 31.42 ± 3.65   Final old_old: 34.60 ± 3.81   Final old_new: 18.54 ± 3.12   Final new_new: 3.64 ± 2.03
Hits@20:
  Final val: 58.74 ± 1.46   Final Test: 39.45 ± 2.28   Final old_old: 42.94 ± 2.44   Final old_new: 25.54 ± 1.83   Final new_new: 5.91 ± 2.03
Hits@30:
  Final val: 65.85 ± 1.19   Final Test: 47.38 ± 2.17   Final old_old: 51.12 ± 2.33   Final old_new: 32.54 ± 1.73   Final new_new: 9.55 ± 1.90
Hits@50:
  Final val: 74.19 ± 0.66   Final Test: 59.32 ± 2.01   Final old_old: 63.40 ± 2.16   Final old_new: 43.27 ± 1.96   Final new_new: 16.36 ± 2.96
AUC:
  Final val: 97.24 ± 0.08   Final Test: 96.69 ± 0.06   Final old_old: 97.30 ± 0.10   Final old_new: 94.29 ± 0.25   Final new_new: 90.65 ± 0.95
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 40.25 ± 2.85   Final Test: 28.60 ± 3.81   Final old_old: 31.33 ± 4.13   Final old_new: 17.46 ± 2.70   Final new_new: 6.36 ± 3.73
Hits@20:
  Final val: 51.61 ± 3.98   Final Test: 37.57 ± 6.73   Final old_old: 40.67 ± 7.26   Final old_new: 25.01 ± 4.90   Final new_new: 10.91 ± 6.10
Hits@30:
  Final val: 60.33 ± 3.57   Final Test: 42.32 ± 5.26   Final old_old: 45.57 ± 5.69   Final old_new: 29.14 ± 3.76   Final new_new: 14.55 ± 8.29
Hits@50:
  Final val: 70.56 ± 2.71   Final Test: 54.78 ± 4.56   Final old_old: 58.36 ± 4.68   Final old_new: 40.43 ± 4.61   Final new_new: 21.82 ± 7.29
AUC:
  Final val: 96.57 ± 0.37   Final Test: 95.97 ± 0.24   Final old_old: 96.61 ± 0.28   Final old_new: 93.37 ± 0.40   Final new_new: 90.33 ± 1.93
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.01 ± 4.05   Final Test: 31.56 ± 4.72   Final old_old: 34.59 ± 5.10   Final old_new: 19.37 ± 3.20   Final new_new: 4.09 ± 3.37
Hits@20:
  Final val: 63.26 ± 1.41   Final Test: 43.00 ± 4.88   Final old_old: 46.68 ± 5.24   Final old_new: 28.24 ± 3.92   Final new_new: 8.64 ± 1.90
Hits@30:
  Final val: 69.99 ± 1.68   Final Test: 51.42 ± 3.79   Final old_old: 55.42 ± 4.09   Final old_new: 35.52 ± 3.00   Final new_new: 12.27 ± 2.59
Hits@50:
  Final val: 77.05 ± 1.44   Final Test: 60.46 ± 3.21   Final old_old: 64.64 ± 3.45   Final old_new: 43.95 ± 2.73   Final new_new: 17.27 ± 4.71
AUC:
  Final val: 97.33 ± 0.17   Final Test: 96.80 ± 0.17   Final old_old: 97.43 ± 0.17   Final old_new: 94.30 ± 0.15   Final new_new: 90.62 ± 0.74
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.37 ± 2.71   Final Test: 31.81 ± 3.43   Final old_old: 35.09 ± 3.73   Final old_new: 18.44 ± 2.32   Final new_new: 5.45 ± 2.03
Hits@20:
  Final val: 60.17 ± 1.85   Final Test: 41.31 ± 6.24   Final old_old: 45.03 ± 6.66   Final old_new: 26.35 ± 4.65   Final new_new: 7.27 ± 2.96
Hits@30:
  Final val: 66.72 ± 0.98   Final Test: 47.81 ± 2.64   Final old_old: 51.56 ± 2.69   Final old_new: 32.92 ± 2.44   Final new_new: 10.45 ± 4.98
Hits@50:
  Final val: 74.70 ± 0.92   Final Test: 58.01 ± 2.94   Final old_old: 62.27 ± 3.01   Final old_new: 41.11 ± 3.39   Final new_new: 15.45 ± 4.66
AUC:
  Final val: 97.32 ± 0.12   Final Test: 96.79 ± 0.07   Final old_old: 97.37 ± 0.07   Final old_new: 94.46 ± 0.26   Final new_new: 90.80 ± 1.32
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.74 ± 3.42   Final Test: 28.73 ± 5.18   Final old_old: 31.81 ± 5.46   Final old_new: 16.22 ± 4.17   Final new_new: 3.18 ± 1.24
Hits@20:
  Final val: 60.90 ± 1.82   Final Test: 41.93 ± 1.86   Final old_old: 45.77 ± 1.76   Final old_new: 26.68 ± 2.53   Final new_new: 3.64 ± 3.45
Hits@30:
  Final val: 67.72 ± 1.23   Final Test: 48.86 ± 1.94   Final old_old: 52.93 ± 1.85   Final old_new: 32.77 ± 2.40   Final new_new: 7.27 ± 4.37
Hits@50:
  Final val: 74.89 ± 1.39   Final Test: 57.97 ± 1.96   Final old_old: 62.15 ± 2.22   Final old_new: 41.66 ± 1.07   Final new_new: 11.36 ± 3.59
AUC:
  Final val: 97.33 ± 0.15   Final Test: 96.89 ± 0.15   Final old_old: 97.42 ± 0.11   Final old_new: 94.79 ± 0.39   Final new_new: 90.92 ± 0.97
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.30 ± 5.67   Final Test: 26.90 ± 5.11   Final old_old: 29.41 ± 5.47   Final old_new: 16.57 ± 3.68   Final new_new: 8.18 ± 4.71
Hits@20:
  Final val: 55.13 ± 6.98   Final Test: 37.75 ± 4.14   Final old_old: 40.62 ± 4.55   Final old_new: 26.30 ± 2.54   Final new_new: 10.45 ± 4.71
Hits@30:
  Final val: 62.24 ± 6.49   Final Test: 48.04 ± 3.25   Final old_old: 51.49 ± 3.40   Final old_new: 34.26 ± 2.77   Final new_new: 15.00 ± 4.13
Hits@50:
  Final val: 71.83 ± 4.07   Final Test: 56.91 ± 3.40   Final old_old: 60.52 ± 3.69   Final old_new: 42.64 ± 2.21   Final new_new: 19.55 ± 3.80
AUC:
  Final val: 96.75 ± 0.17   Final Test: 96.16 ± 0.28   Final old_old: 96.80 ± 0.25   Final old_new: 93.59 ± 0.46   Final new_new: 90.23 ± 1.20
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.39 ± 5.71   Final Test: 26.44 ± 5.95   Final old_old: 28.81 ± 6.53   Final old_new: 16.78 ± 3.77   Final new_new: 6.82 ± 3.21
Hits@20:
  Final val: 53.83 ± 2.83   Final Test: 39.62 ± 2.97   Final old_old: 42.71 ± 3.24   Final old_new: 27.28 ± 2.44   Final new_new: 10.45 ± 4.43
Hits@30:
  Final val: 61.90 ± 1.24   Final Test: 46.21 ± 3.62   Final old_old: 49.58 ± 3.91   Final old_new: 32.64 ± 3.09   Final new_new: 15.91 ± 5.79
Hits@50:
  Final val: 71.17 ± 1.39   Final Test: 57.54 ± 1.72   Final old_old: 61.30 ± 2.00   Final old_new: 42.67 ± 1.41   Final new_new: 18.18 ± 6.22
AUC:
  Final val: 96.64 ± 0.30   Final Test: 95.97 ± 0.32   Final old_old: 96.63 ± 0.35   Final old_new: 93.32 ± 0.23   Final new_new: 90.08 ± 1.38
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.03 ± 2.10   Final Test: 31.93 ± 5.14   Final old_old: 35.16 ± 5.34   Final old_new: 18.79 ± 4.37   Final new_new: 5.00 ± 4.93
Hits@20:
  Final val: 58.15 ± 1.64   Final Test: 38.33 ± 3.45   Final old_old: 41.80 ± 3.55   Final old_new: 24.33 ± 3.26   Final new_new: 7.27 ± 1.90
Hits@30:
  Final val: 65.73 ± 1.27   Final Test: 46.11 ± 2.50   Final old_old: 49.83 ± 2.73   Final old_new: 31.26 ± 1.98   Final new_new: 10.45 ± 4.43
Hits@50:
  Final val: 73.52 ± 0.87   Final Test: 58.00 ± 2.46   Final old_old: 61.99 ± 2.55   Final old_new: 42.27 ± 2.17   Final new_new: 16.82 ± 5.23
AUC:
  Final val: 97.23 ± 0.10   Final Test: 96.69 ± 0.09   Final old_old: 97.25 ± 0.10   Final old_new: 94.49 ± 0.09   Final new_new: 90.63 ± 0.49
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.51 ± 5.10   Final Test: 30.77 ± 6.34   Final old_old: 33.52 ± 6.64   Final old_new: 19.62 ± 5.28   Final new_new: 7.73 ± 3.80
Hits@20:
  Final val: 54.14 ± 3.08   Final Test: 35.25 ± 5.76   Final old_old: 38.32 ± 6.06   Final old_new: 22.75 ± 4.57   Final new_new: 10.00 ± 7.64
Hits@30:
  Final val: 61.93 ± 3.02   Final Test: 46.99 ± 3.16   Final old_old: 50.43 ± 3.13   Final old_new: 33.10 ± 3.66   Final new_new: 16.36 ± 11.85
Hits@50:
  Final val: 71.71 ± 3.84   Final Test: 55.81 ± 3.28   Final old_old: 59.49 ± 3.33   Final old_new: 41.16 ± 3.47   Final new_new: 20.00 ± 10.83
AUC:
  Final val: 96.61 ± 0.29   Final Test: 95.97 ± 0.20   Final old_old: 96.66 ± 0.18   Final old_new: 93.18 ± 0.48   Final new_new: 89.50 ± 1.33
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.90 ± 3.77   Final Test: 32.83 ± 7.50   Final old_old: 36.25 ± 8.08   Final old_new: 19.04 ± 5.38   Final new_new: 2.73 ± 1.90
Hits@20:
  Final val: 58.75 ± 3.81   Final Test: 40.57 ± 4.19   Final old_old: 44.41 ± 4.37   Final old_new: 25.19 ± 3.89   Final new_new: 4.55 ± 1.61
Hits@30:
  Final val: 66.39 ± 2.47   Final Test: 52.81 ± 2.08   Final old_old: 57.19 ± 2.23   Final old_new: 35.42 ± 2.50   Final new_new: 9.09 ± 2.27
Hits@50:
  Final val: 75.37 ± 1.63   Final Test: 60.59 ± 3.69   Final old_old: 65.10 ± 3.88   Final old_new: 42.64 ± 3.03   Final new_new: 15.45 ± 1.90
AUC:
  Final val: 97.30 ± 0.16   Final Test: 96.65 ± 0.17   Final old_old: 97.43 ± 0.14   Final old_new: 93.56 ± 0.41   Final new_new: 88.96 ± 1.36
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.82 ± 3.68   Final Test: 26.08 ± 3.54   Final old_old: 28.43 ± 3.85   Final old_new: 16.60 ± 2.57   Final new_new: 4.55 ± 4.25
Hits@20:
  Final val: 54.12 ± 3.78   Final Test: 35.69 ± 5.84   Final old_old: 38.59 ± 6.40   Final old_new: 24.28 ± 4.10   Final new_new: 5.45 ± 2.59
Hits@30:
  Final val: 61.60 ± 3.93   Final Test: 46.11 ± 2.59   Final old_old: 49.47 ± 3.08   Final old_new: 32.64 ± 1.36   Final new_new: 14.55 ± 6.74
Hits@50:
  Final val: 71.84 ± 1.88   Final Test: 55.21 ± 3.19   Final old_old: 58.83 ± 3.46   Final old_new: 40.81 ± 2.66   Final new_new: 19.55 ± 6.14
AUC:
  Final val: 96.73 ± 0.19   Final Test: 96.24 ± 0.22   Final old_old: 96.86 ± 0.22   Final old_new: 93.75 ± 0.36   Final new_new: 89.75 ± 1.64
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 53.67 ± 3.13   Final Test: 33.55 ± 5.95   Final old_old: 36.97 ± 6.41   Final old_new: 19.72 ± 4.08   Final new_new: 4.09 ± 3.37
Hits@20:
  Final val: 65.09 ± 4.10   Final Test: 46.80 ± 6.82   Final old_old: 50.74 ± 7.22   Final old_new: 31.11 ± 5.38   Final new_new: 7.73 ± 3.80
Hits@30:
  Final val: 71.89 ± 3.30   Final Test: 53.72 ± 4.81   Final old_old: 58.08 ± 5.12   Final old_new: 36.42 ± 3.86   Final new_new: 10.00 ± 3.45
Hits@50:
  Final val: 79.03 ± 2.66   Final Test: 63.77 ± 2.69   Final old_old: 68.18 ± 2.80   Final old_new: 46.55 ± 2.81   Final new_new: 14.55 ± 4.13
AUC:
  Final val: 97.56 ± 0.16   Final Test: 97.11 ± 0.19   Final old_old: 97.66 ± 0.20   Final old_new: 94.92 ± 0.25   Final new_new: 91.05 ± 1.51
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.12 ± 5.19   Final Test: 28.93 ± 4.45   Final old_old: 31.69 ± 4.57   Final old_new: 17.71 ± 4.16   Final new_new: 5.45 ± 2.03
Hits@20:
  Final val: 54.61 ± 4.54   Final Test: 37.83 ± 2.56   Final old_old: 40.95 ± 2.54   Final old_new: 25.04 ± 3.01   Final new_new: 13.64 ± 7.01
Hits@30:
  Final val: 62.67 ± 2.64   Final Test: 47.61 ± 2.10   Final old_old: 51.11 ± 2.17   Final old_new: 33.65 ± 1.71   Final new_new: 14.09 ± 9.43
Hits@50:
  Final val: 72.01 ± 1.26   Final Test: 55.63 ± 3.52   Final old_old: 59.33 ± 3.37   Final old_new: 40.93 ± 4.14   Final new_new: 18.64 ± 10.59
AUC:
  Final val: 96.60 ± 0.33   Final Test: 96.01 ± 0.21   Final old_old: 96.70 ± 0.22   Final old_new: 93.23 ± 0.44   Final new_new: 89.99 ± 0.97
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.34 ± 1.35   Final Test: 30.96 ± 4.16   Final old_old: 33.94 ± 4.42   Final old_new: 18.94 ± 3.18   Final new_new: 5.00 ± 1.02
Hits@20:
  Final val: 61.44 ± 1.36   Final Test: 40.54 ± 2.08   Final old_old: 44.03 ± 2.32   Final old_new: 26.50 ± 1.64   Final new_new: 8.64 ± 4.07
Hits@30:
  Final val: 68.82 ± 1.84   Final Test: 48.74 ± 3.17   Final old_old: 52.67 ± 3.44   Final old_new: 33.05 ± 2.34   Final new_new: 11.36 ± 1.61
Hits@50:
  Final val: 76.44 ± 1.32   Final Test: 59.98 ± 3.94   Final old_old: 64.35 ± 4.23   Final old_new: 42.59 ± 3.10   Final new_new: 16.36 ± 2.96
AUC:
  Final val: 97.31 ± 0.16   Final Test: 96.81 ± 0.17   Final old_old: 97.44 ± 0.20   Final old_new: 94.35 ± 0.25   Final new_new: 90.02 ± 0.98
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.25 ± 4.02   Final Test: 33.29 ± 3.55   Final old_old: 36.59 ± 3.78   Final old_new: 20.00 ± 3.38   Final new_new: 3.64 ± 3.45
Hits@20:
  Final val: 64.29 ± 2.29   Final Test: 44.35 ± 5.52   Final old_old: 48.26 ± 5.94   Final old_new: 28.74 ± 4.09   Final new_new: 6.82 ± 2.27
Hits@30:
  Final val: 71.20 ± 1.77   Final Test: 51.71 ± 5.30   Final old_old: 56.01 ± 5.49   Final old_new: 34.66 ± 4.88   Final new_new: 8.18 ± 3.05
Hits@50:
  Final val: 77.83 ± 1.22   Final Test: 62.69 ± 1.76   Final old_old: 67.23 ± 1.76   Final old_new: 44.79 ± 1.71   Final new_new: 15.00 ± 6.14
AUC:
  Final val: 97.47 ± 0.18   Final Test: 96.93 ± 0.16   Final old_old: 97.54 ± 0.16   Final old_new: 94.47 ± 0.28   Final new_new: 91.30 ± 1.28
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.06 ± 7.21   Final Test: 27.23 ± 5.39   Final old_old: 29.90 ± 5.77   Final old_new: 16.35 ± 4.10   Final new_new: 5.91 ± 1.24
Hits@20:
  Final val: 53.91 ± 4.70   Final Test: 37.40 ± 6.85   Final old_old: 40.67 ± 7.01   Final old_new: 24.03 ± 6.19   Final new_new: 11.36 ± 7.01
Hits@30:
  Final val: 61.36 ± 2.79   Final Test: 47.11 ± 5.68   Final old_old: 50.49 ± 5.63   Final old_new: 33.70 ± 5.93   Final new_new: 13.64 ± 7.71
Hits@50:
  Final val: 70.54 ± 2.46   Final Test: 55.36 ± 4.51   Final old_old: 59.28 ± 4.80   Final old_new: 39.85 ± 3.44   Final new_new: 15.45 ± 5.18
AUC:
  Final val: 96.55 ± 0.30   Final Test: 95.96 ± 0.24   Final old_old: 96.63 ± 0.21   Final old_new: 93.25 ± 0.47   Final new_new: 89.85 ± 0.90
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 53.13 ± 4.62   Final Test: 32.71 ± 7.41   Final old_old: 36.05 ± 7.87   Final old_new: 19.14 ± 5.79   Final new_new: 4.55 ± 3.59
Hits@20:
  Final val: 63.74 ± 2.49   Final Test: 44.26 ± 2.93   Final old_old: 48.11 ± 2.93   Final old_new: 28.79 ± 3.40   Final new_new: 9.09 ± 2.78
Hits@30:
  Final val: 69.78 ± 1.84   Final Test: 55.26 ± 1.41   Final old_old: 59.49 ± 1.20   Final old_new: 38.54 ± 2.58   Final new_new: 11.36 ± 1.61
Hits@50:
  Final val: 77.05 ± 1.18   Final Test: 59.81 ± 3.47   Final old_old: 64.13 ± 3.47   Final old_new: 42.72 ± 3.84   Final new_new: 15.00 ± 1.24
AUC:
  Final val: 97.48 ± 0.08   Final Test: 96.93 ± 0.14   Final old_old: 97.51 ± 0.13   Final old_new: 94.64 ± 0.23   Final new_new: 90.50 ± 0.99
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.15 ± 5.67   Final Test: 27.68 ± 4.96   Final old_old: 30.29 ± 5.24   Final old_new: 17.13 ± 4.09   Final new_new: 5.00 ± 2.49
Hits@20:
  Final val: 59.43 ± 3.97   Final Test: 39.10 ± 3.58   Final old_old: 42.30 ± 3.71   Final old_new: 26.22 ± 3.11   Final new_new: 10.00 ± 2.59
Hits@30:
  Final val: 65.92 ± 2.11   Final Test: 46.26 ± 3.12   Final old_old: 49.90 ± 3.27   Final old_new: 31.51 ± 2.64   Final new_new: 15.00 ± 4.71
Hits@50:
  Final val: 74.27 ± 1.87   Final Test: 57.31 ± 2.85   Final old_old: 61.21 ± 2.90   Final old_new: 41.84 ± 3.36   Final new_new: 18.64 ± 3.73
AUC:
  Final val: 96.83 ± 0.25   Final Test: 96.23 ± 0.22   Final old_old: 96.83 ± 0.28   Final old_new: 93.79 ± 0.20   Final new_new: 91.44 ± 0.65
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.58 ± 2.80   Final Test: 33.66 ± 9.30   Final old_old: 36.85 ± 9.97   Final old_new: 20.88 ± 6.70   Final new_new: 3.64 ± 2.59
Hits@20:
  Final val: 62.55 ± 2.86   Final Test: 41.31 ± 3.03   Final old_old: 44.93 ± 3.30   Final old_new: 26.93 ± 2.26   Final new_new: 5.45 ± 4.43
Hits@30:
  Final val: 68.77 ± 1.63   Final Test: 49.54 ± 3.68   Final old_old: 53.43 ± 4.01   Final old_new: 34.21 ± 2.86   Final new_new: 8.64 ± 5.43
Hits@50:
  Final val: 76.49 ± 1.80   Final Test: 59.83 ± 3.11   Final old_old: 64.20 ± 3.62   Final old_new: 42.64 ± 2.05   Final new_new: 13.18 ± 6.31
AUC:
  Final val: 97.36 ± 0.17   Final Test: 96.83 ± 0.19   Final old_old: 97.46 ± 0.22   Final old_new: 94.34 ± 0.28   Final new_new: 90.22 ± 0.86
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.90 ± 5.79   Final Test: 31.95 ± 3.73   Final old_old: 35.21 ± 3.86   Final old_new: 18.66 ± 3.20   Final new_new: 5.91 ± 4.43
Hits@20:
  Final val: 59.95 ± 5.56   Final Test: 39.87 ± 3.61   Final old_old: 43.41 ± 3.84   Final old_new: 25.64 ± 2.90   Final new_new: 7.73 ± 4.13
Hits@30:
  Final val: 66.60 ± 3.85   Final Test: 47.89 ± 3.32   Final old_old: 51.90 ± 3.28   Final old_new: 31.79 ± 3.80   Final new_new: 10.91 ± 2.96
Hits@50:
  Final val: 74.35 ± 2.06   Final Test: 56.24 ± 1.43   Final old_old: 60.54 ± 1.38   Final old_new: 39.19 ± 2.48   Final new_new: 12.27 ± 6.74
AUC:
  Final val: 97.30 ± 0.15   Final Test: 96.73 ± 0.15   Final old_old: 97.30 ± 0.19   Final old_new: 94.46 ± 0.23   Final new_new: 90.70 ± 0.68
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.37 ± 2.54   Final Test: 31.66 ± 4.95   Final old_old: 34.81 ± 5.20   Final old_new: 18.99 ± 4.32   Final new_new: 3.18 ± 1.24
Hits@20:
  Final val: 60.16 ± 1.66   Final Test: 41.90 ± 3.78   Final old_old: 45.46 ± 4.29   Final old_new: 27.73 ± 2.52   Final new_new: 6.82 ± 2.78
Hits@30:
  Final val: 66.94 ± 1.35   Final Test: 48.79 ± 1.98   Final old_old: 52.85 ± 1.98   Final old_new: 32.54 ± 2.93   Final new_new: 10.91 ± 4.66
Hits@50:
  Final val: 74.13 ± 1.32   Final Test: 58.97 ± 2.52   Final old_old: 63.12 ± 2.77   Final old_new: 42.49 ± 1.96   Final new_new: 17.27 ± 5.93
AUC:
  Final val: 97.29 ± 0.17   Final Test: 96.83 ± 0.09   Final old_old: 97.44 ± 0.13   Final old_new: 94.37 ± 0.16   Final new_new: 91.23 ± 0.97
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.96 ± 1.24   Final Test: 31.30 ± 5.95   Final old_old: 34.44 ± 6.49   Final old_new: 18.59 ± 4.11   Final new_new: 3.64 ± 2.03
Hits@20:
  Final val: 61.96 ± 1.23   Final Test: 42.78 ± 5.19   Final old_old: 46.55 ± 5.57   Final old_new: 27.76 ± 3.96   Final new_new: 6.36 ± 2.96
Hits@30:
  Final val: 67.69 ± 0.72   Final Test: 50.47 ± 3.73   Final old_old: 54.66 ± 3.81   Final old_new: 33.80 ± 3.47   Final new_new: 9.09 ± 7.01
Hits@50:
  Final val: 75.83 ± 1.48   Final Test: 62.50 ± 1.72   Final old_old: 67.15 ± 1.84   Final old_new: 44.03 ± 2.01   Final new_new: 15.91 ± 8.04
AUC:
  Final val: 97.41 ± 0.19   Final Test: 96.72 ± 0.19   Final old_old: 97.52 ± 0.21   Final old_new: 93.47 ± 0.24   Final new_new: 89.20 ± 0.99
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.17 ± 2.16   Final Test: 34.36 ± 6.31   Final old_old: 37.68 ± 6.82   Final old_new: 20.96 ± 4.58   Final new_new: 4.55 ± 0.00
Hits@20:
  Final val: 60.61 ± 2.01   Final Test: 40.53 ± 4.59   Final old_old: 44.16 ± 4.78   Final old_new: 26.02 ± 3.84   Final new_new: 5.45 ± 3.05
Hits@30:
  Final val: 66.84 ± 2.03   Final Test: 47.51 ± 4.87   Final old_old: 51.49 ± 5.11   Final old_new: 31.69 ± 3.96   Final new_new: 8.18 ± 2.59
Hits@50:
  Final val: 74.34 ± 1.74   Final Test: 57.30 ± 2.85   Final old_old: 61.48 ± 3.02   Final old_new: 40.76 ± 2.43   Final new_new: 15.00 ± 2.03
AUC:
  Final val: 97.37 ± 0.08   Final Test: 96.92 ± 0.10   Final old_old: 97.47 ± 0.09   Final old_new: 94.78 ± 0.29   Final new_new: 90.82 ± 1.64
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.56 ± 3.30   Final Test: 30.61 ± 5.26   Final old_old: 33.58 ± 5.59   Final old_new: 18.69 ± 4.08   Final new_new: 3.18 ± 1.24
Hits@20:
  Final val: 59.44 ± 3.51   Final Test: 39.98 ± 5.41   Final old_old: 43.50 ± 5.77   Final old_new: 25.89 ± 4.05   Final new_new: 6.36 ± 2.96
Hits@30:
  Final val: 65.56 ± 2.28   Final Test: 47.06 ± 3.35   Final old_old: 50.88 ± 3.61   Final old_new: 31.94 ± 2.59   Final new_new: 8.18 ± 2.03
Hits@50:
  Final val: 73.18 ± 1.42   Final Test: 56.56 ± 3.08   Final old_old: 60.75 ± 3.21   Final old_new: 40.00 ± 2.85   Final new_new: 13.18 ± 3.73
AUC:
  Final val: 97.20 ± 0.11   Final Test: 96.70 ± 0.11   Final old_old: 97.29 ± 0.13   Final old_new: 94.37 ± 0.20   Final new_new: 90.41 ± 1.29
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 54.16 ± 3.98   Final Test: 34.52 ± 4.81   Final old_old: 38.04 ± 5.14   Final old_new: 20.35 ± 3.79   Final new_new: 3.18 ± 2.59
Hits@20:
  Final val: 66.10 ± 3.30   Final Test: 47.62 ± 3.95   Final old_old: 51.69 ± 4.49   Final old_new: 31.41 ± 1.81   Final new_new: 7.73 ± 2.59
Hits@30:
  Final val: 72.34 ± 2.87   Final Test: 56.48 ± 5.06   Final old_old: 60.91 ± 5.53   Final old_new: 38.82 ± 3.36   Final new_new: 13.64 ± 1.61
Hits@50:
  Final val: 78.70 ± 1.43   Final Test: 63.85 ± 3.19   Final old_old: 68.39 ± 3.51   Final old_new: 45.87 ± 2.33   Final new_new: 17.73 ± 4.07
AUC:
  Final val: 97.56 ± 0.13   Final Test: 97.07 ± 0.14   Final old_old: 97.64 ± 0.21   Final old_new: 94.82 ± 0.12   Final new_new: 91.42 ± 1.67
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.08 ± 3.46   Final Test: 28.55 ± 3.33   Final old_old: 31.11 ± 3.66   Final old_new: 18.24 ± 2.21   Final new_new: 5.91 ± 6.14
Hits@20:
  Final val: 54.34 ± 3.59   Final Test: 37.87 ± 3.37   Final old_old: 40.97 ± 3.71   Final old_new: 25.39 ± 2.09   Final new_new: 10.45 ± 2.59
Hits@30:
  Final val: 62.85 ± 2.49   Final Test: 45.13 ± 5.69   Final old_old: 48.58 ± 5.94   Final old_new: 31.26 ± 4.96   Final new_new: 13.64 ± 3.59
Hits@50:
  Final val: 71.40 ± 1.45   Final Test: 54.35 ± 2.55   Final old_old: 57.93 ± 2.76   Final old_new: 40.23 ± 1.84   Final new_new: 17.27 ± 4.43
AUC:
  Final val: 96.68 ± 0.31   Final Test: 95.99 ± 0.26   Final old_old: 96.61 ± 0.27   Final old_new: 93.52 ± 0.25   Final new_new: 89.83 ± 1.12
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.22 ± 1.93   Final Test: 26.52 ± 3.46   Final old_old: 29.22 ± 3.61   Final old_new: 15.44 ± 2.92   Final new_new: 5.45 ± 4.13
Hits@20:
  Final val: 56.95 ± 0.19   Final Test: 42.13 ± 2.82   Final old_old: 45.68 ± 2.72   Final old_new: 27.81 ± 3.19   Final new_new: 10.45 ± 7.64
Hits@30:
  Final val: 63.37 ± 1.01   Final Test: 47.35 ± 4.04   Final old_old: 50.98 ± 4.12   Final old_new: 32.92 ± 3.75   Final new_new: 11.82 ± 7.43
Hits@50:
  Final val: 72.70 ± 1.02   Final Test: 55.33 ± 1.69   Final old_old: 59.30 ± 1.64   Final old_new: 39.55 ± 2.31   Final new_new: 16.36 ± 7.61
AUC:
  Final val: 96.67 ± 0.27   Final Test: 96.02 ± 0.28   Final old_old: 96.70 ± 0.31   Final old_new: 93.28 ± 0.38   Final new_new: 89.71 ± 1.86
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.18 ± 4.50   Final Test: 33.12 ± 5.74   Final old_old: 36.32 ± 6.00   Final old_new: 20.13 ± 4.94   Final new_new: 6.36 ± 1.90
Hits@20:
  Final val: 57.86 ± 3.48   Final Test: 44.35 ± 2.85   Final old_old: 48.19 ± 3.19   Final old_new: 28.89 ± 2.30   Final new_new: 10.00 ± 3.45
Hits@30:
  Final val: 65.65 ± 3.62   Final Test: 52.35 ± 5.20   Final old_old: 56.58 ± 5.65   Final old_new: 35.42 ± 3.91   Final new_new: 12.73 ± 1.24
Hits@50:
  Final val: 74.54 ± 1.95   Final Test: 61.12 ± 3.17   Final old_old: 65.68 ± 3.56   Final old_new: 43.10 ± 1.92   Final new_new: 14.09 ± 1.02
AUC:
  Final val: 97.19 ± 0.23   Final Test: 96.33 ± 0.17   Final old_old: 97.36 ± 0.23   Final old_new: 92.14 ± 0.88   Final new_new: 87.96 ± 2.35
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.96 ± 2.83   Final Test: 29.88 ± 5.11   Final old_old: 32.82 ± 5.27   Final old_new: 17.91 ± 4.66   Final new_new: 6.36 ± 1.90
Hits@20:
  Final val: 60.49 ± 3.25   Final Test: 39.93 ± 9.10   Final old_old: 43.21 ± 9.63   Final old_new: 26.75 ± 7.07   Final new_new: 10.00 ± 3.45
Hits@30:
  Final val: 68.00 ± 2.04   Final Test: 46.80 ± 6.02   Final old_old: 50.56 ± 6.34   Final old_new: 31.64 ± 4.73   Final new_new: 13.18 ± 4.93
Hits@50:
  Final val: 75.16 ± 1.46   Final Test: 58.35 ± 2.92   Final old_old: 62.34 ± 2.84   Final old_new: 42.42 ± 3.11   Final new_new: 20.00 ± 7.61
AUC:
  Final val: 96.85 ± 0.13   Final Test: 96.24 ± 0.19   Final old_old: 96.90 ± 0.19   Final old_new: 93.65 ± 0.26   Final new_new: 89.33 ± 1.51
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.88 ± 3.21   Final Test: 30.16 ± 3.49   Final old_old: 32.94 ± 3.97   Final old_new: 18.97 ± 2.26   Final new_new: 5.91 ± 4.71
Hits@20:
  Final val: 55.35 ± 2.33   Final Test: 38.61 ± 5.59   Final old_old: 41.91 ± 6.12   Final old_new: 25.54 ± 3.56   Final new_new: 5.00 ± 2.96
Hits@30:
  Final val: 63.48 ± 2.78   Final Test: 48.76 ± 2.97   Final old_old: 52.41 ± 2.91   Final old_new: 34.28 ± 3.73   Final new_new: 11.82 ± 7.94
Hits@50:
  Final val: 71.73 ± 1.72   Final Test: 56.46 ± 1.72   Final old_old: 60.40 ± 2.02   Final old_new: 40.86 ± 1.54   Final new_new: 16.36 ± 6.31
AUC:
  Final val: 96.64 ± 0.34   Final Test: 95.98 ± 0.19   Final old_old: 96.68 ± 0.25   Final old_new: 93.17 ± 0.34   Final new_new: 89.68 ± 1.84
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.93 ± 4.60   Final Test: 31.74 ± 4.11   Final old_old: 34.90 ± 4.38   Final old_new: 19.04 ± 3.18   Final new_new: 3.18 ± 2.03
Hits@20:
  Final val: 62.55 ± 1.46   Final Test: 40.49 ± 3.26   Final old_old: 44.17 ± 3.61   Final old_new: 25.82 ± 2.11   Final new_new: 4.55 ± 2.78
Hits@30:
  Final val: 70.17 ± 1.49   Final Test: 53.25 ± 4.16   Final old_old: 57.29 ± 4.54   Final old_new: 37.28 ± 3.08   Final new_new: 11.36 ± 7.01
Hits@50:
  Final val: 77.67 ± 1.66   Final Test: 62.27 ± 1.21   Final old_old: 66.66 ± 1.29   Final old_new: 45.09 ± 2.69   Final new_new: 14.09 ± 5.88
AUC:
  Final val: 97.45 ± 0.18   Final Test: 96.85 ± 0.17   Final old_old: 97.48 ± 0.20   Final old_new: 94.31 ± 0.24   Final new_new: 91.03 ± 1.01
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.77 ± 2.79   Final Test: 33.07 ± 6.20   Final old_old: 36.28 ± 6.70   Final old_new: 20.15 ± 4.30   Final new_new: 4.09 ± 1.02
Hits@20:
  Final val: 61.89 ± 2.64   Final Test: 40.07 ± 2.92   Final old_old: 43.62 ± 3.29   Final old_new: 25.84 ± 1.75   Final new_new: 6.82 ± 4.25
Hits@30:
  Final val: 68.94 ± 1.96   Final Test: 49.15 ± 3.90   Final old_old: 53.21 ± 4.24   Final old_new: 33.02 ± 2.79   Final new_new: 8.64 ± 3.73
Hits@50:
  Final val: 76.46 ± 1.76   Final Test: 59.59 ± 2.07   Final old_old: 63.77 ± 2.48   Final old_new: 43.07 ± 0.50   Final new_new: 16.36 ± 3.37
AUC:
  Final val: 97.34 ± 0.14   Final Test: 96.81 ± 0.19   Final old_old: 97.41 ± 0.20   Final old_new: 94.43 ± 0.29   Final new_new: 90.18 ± 1.03
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.15 ± 5.09   Final Test: 33.09 ± 5.11   Final old_old: 36.06 ± 5.61   Final old_new: 21.16 ± 3.25   Final new_new: 6.36 ± 1.02
Hits@20:
  Final val: 61.18 ± 3.59   Final Test: 42.61 ± 6.35   Final old_old: 45.95 ± 6.80   Final old_new: 29.14 ± 4.67   Final new_new: 12.73 ± 2.59
Hits@30:
  Final val: 67.50 ± 3.21   Final Test: 50.22 ± 3.13   Final old_old: 53.99 ± 3.33   Final old_new: 35.21 ± 2.65   Final new_new: 13.64 ± 3.21
Hits@50:
  Final val: 75.16 ± 2.11   Final Test: 58.87 ± 2.87   Final old_old: 62.83 ± 3.20   Final old_new: 43.10 ± 1.70   Final new_new: 20.45 ± 2.78
AUC:
  Final val: 96.86 ± 0.23   Final Test: 96.29 ± 0.23   Final old_old: 96.95 ± 0.16   Final old_new: 93.62 ± 0.60   Final new_new: 90.13 ± 2.22
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.19 ± 2.14   Final Test: 39.37 ± 3.39   Final old_old: 43.44 ± 3.64   Final old_new: 22.97 ± 2.53   Final new_new: 2.73 ± 1.90
Hits@20:
  Final val: 62.01 ± 1.99   Final Test: 49.50 ± 3.76   Final old_old: 54.18 ± 3.70   Final old_new: 30.63 ± 4.19   Final new_new: 7.73 ± 5.23
Hits@30:
  Final val: 70.05 ± 1.56   Final Test: 54.50 ± 3.53   Final old_old: 59.27 ± 3.84   Final old_new: 35.44 ± 2.97   Final new_new: 9.09 ± 4.55
Hits@50:
  Final val: 77.91 ± 0.99   Final Test: 65.08 ± 1.05   Final old_old: 70.19 ± 1.00   Final old_new: 44.71 ± 2.03   Final new_new: 15.00 ± 7.29
AUC:
  Final val: 97.81 ± 0.09   Final Test: 96.44 ± 0.22   Final old_old: 97.95 ± 0.06   Final old_new: 90.29 ± 1.23   Final new_new: 84.06 ± 2.48
All runs:
Hits@10:
  Final val: 50.19 ± 5.10   Final Test: 35.60 ± 4.75   Final old_old: 38.86 ± 5.13   Final old_new: 22.47 ± 3.28   Final new_new: 6.36 ± 3.37
Hits@20:
  Final val: 63.33 ± 1.84   Final Test: 43.85 ± 2.16   Final old_old: 47.56 ± 2.15   Final old_new: 28.99 ± 2.75   Final new_new: 8.64 ± 4.93
Hits@30:
  Final val: 70.24 ± 2.54   Final Test: 52.35 ± 2.85   Final old_old: 56.30 ± 3.14   Final old_new: 36.50 ± 1.93   Final new_new: 15.45 ± 3.73
Hits@50:
  Final val: 78.34 ± 1.55   Final Test: 63.08 ± 1.90   Final old_old: 67.35 ± 1.83   Final old_new: 46.10 ± 2.43   Final new_new: 20.91 ± 4.66
AUC:
  Final val: 97.57 ± 0.05   Final Test: 97.10 ± 0.09   Final old_old: 97.64 ± 0.08   Final old_new: 95.00 ± 0.31   Final new_new: 91.54 ± 1.64
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.61 ± 3.67   Final Test: 34.56 ± 3.16   Final old_old: 37.88 ± 3.49   Final old_new: 21.16 ± 1.86   Final new_new: 4.55 ± 4.25
Hits@20:
  Final val: 61.57 ± 3.86   Final Test: 45.85 ± 5.40   Final old_old: 49.58 ± 5.77   Final old_new: 31.03 ± 3.99   Final new_new: 8.64 ± 4.37
Hits@30:
  Final val: 69.04 ± 2.70   Final Test: 51.80 ± 3.63   Final old_old: 55.93 ± 4.01   Final old_new: 35.42 ± 2.50   Final new_new: 10.91 ± 3.73
Hits@50:
  Final val: 76.43 ± 1.80   Final Test: 60.86 ± 2.63   Final old_old: 65.16 ± 3.12   Final old_new: 43.88 ± 1.01   Final new_new: 16.36 ± 6.31
AUC:
  Final val: 97.36 ± 0.12   Final Test: 96.87 ± 0.16   Final old_old: 97.49 ± 0.16   Final old_new: 94.43 ± 0.23   Final new_new: 90.46 ± 1.12
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.40 ± 3.03   Final Test: 28.61 ± 3.21   Final old_old: 31.47 ± 3.48   Final old_new: 17.05 ± 2.29   Final new_new: 3.18 ± 1.24
Hits@20:
  Final val: 59.09 ± 1.92   Final Test: 39.86 ± 2.50   Final old_old: 43.39 ± 2.58   Final old_new: 25.77 ± 2.40   Final new_new: 5.45 ± 3.05
Hits@30:
  Final val: 65.49 ± 1.99   Final Test: 45.36 ± 5.34   Final old_old: 49.21 ± 5.64   Final old_new: 30.05 ± 4.42   Final new_new: 7.27 ± 2.96
Hits@50:
  Final val: 73.70 ± 1.64   Final Test: 56.27 ± 3.95   Final old_old: 60.77 ± 4.03   Final old_new: 38.36 ± 3.92   Final new_new: 11.82 ± 4.37
AUC:
  Final val: 97.24 ± 0.07   Final Test: 96.68 ± 0.06   Final old_old: 97.28 ± 0.09   Final old_new: 94.33 ± 0.17   Final new_new: 90.13 ± 0.83
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.48 ± 2.76   Final Test: 31.68 ± 6.09   Final old_old: 34.95 ± 6.53   Final old_new: 18.29 ± 4.33   Final new_new: 6.82 ± 3.59
Hits@20:
  Final val: 60.09 ± 0.89   Final Test: 39.92 ± 4.42   Final old_old: 43.49 ± 4.52   Final old_new: 25.49 ± 4.03   Final new_new: 8.64 ± 3.73
Hits@30:
  Final val: 66.49 ± 0.60   Final Test: 49.19 ± 2.33   Final old_old: 53.08 ± 2.25   Final old_new: 33.70 ± 2.65   Final new_new: 11.36 ± 4.25
Hits@50:
  Final val: 73.99 ± 0.46   Final Test: 57.85 ± 2.28   Final old_old: 62.04 ± 2.34   Final old_new: 41.23 ± 2.35   Final new_new: 15.00 ± 1.24
AUC:
  Final val: 97.30 ± 0.11   Final Test: 96.77 ± 0.08   Final old_old: 97.36 ± 0.07   Final old_new: 94.42 ± 0.16   Final new_new: 90.43 ± 0.57
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.08 ± 7.11   Final Test: 33.84 ± 8.75   Final old_old: 37.05 ± 9.50   Final old_new: 20.96 ± 5.80   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 62.22 ± 4.22   Final Test: 43.22 ± 6.56   Final old_old: 46.85 ± 7.18   Final old_new: 28.79 ± 4.16   Final new_new: 7.73 ± 2.03
Hits@30:
  Final val: 70.29 ± 3.34   Final Test: 51.80 ± 5.88   Final old_old: 55.96 ± 6.45   Final old_new: 35.31 ± 3.57   Final new_new: 9.09 ± 2.27
Hits@50:
  Final val: 77.83 ± 2.09   Final Test: 62.22 ± 4.19   Final old_old: 66.54 ± 4.61   Final old_new: 45.29 ± 2.55   Final new_new: 15.00 ± 4.98
AUC:
  Final val: 97.55 ± 0.15   Final Test: 97.07 ± 0.18   Final old_old: 97.59 ± 0.18   Final old_new: 95.01 ± 0.32   Final new_new: 91.95 ± 0.28
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.93 ± 4.28   Final Test: 33.91 ± 3.39   Final old_old: 37.15 ± 3.48   Final old_new: 20.91 ± 3.32   Final new_new: 4.09 ± 1.02
Hits@20:
  Final val: 60.65 ± 2.65   Final Test: 42.04 ± 4.25   Final old_old: 45.71 ± 4.39   Final old_new: 27.46 ± 3.97   Final new_new: 5.00 ± 1.02
Hits@30:
  Final val: 67.70 ± 1.48   Final Test: 51.45 ± 3.92   Final old_old: 55.42 ± 4.21   Final old_new: 35.84 ± 2.92   Final new_new: 8.18 ± 5.47
Hits@50:
  Final val: 75.31 ± 1.87   Final Test: 59.65 ± 2.17   Final old_old: 63.75 ± 2.29   Final old_new: 43.70 ± 1.82   Final new_new: 12.73 ± 4.43
AUC:
  Final val: 97.47 ± 0.07   Final Test: 97.07 ± 0.16   Final old_old: 97.55 ± 0.18   Final old_new: 95.19 ± 0.20   Final new_new: 91.91 ± 0.34
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.00 ± 2.44   Final Test: 31.14 ± 6.98   Final old_old: 34.32 ± 7.46   Final old_new: 18.24 ± 5.07   Final new_new: 4.55 ± 2.78
Hits@20:
  Final val: 59.86 ± 1.35   Final Test: 40.87 ± 2.93   Final old_old: 44.39 ± 3.06   Final old_new: 26.88 ± 2.70   Final new_new: 5.45 ± 2.03
Hits@30:
  Final val: 65.96 ± 1.34   Final Test: 48.64 ± 2.56   Final old_old: 52.50 ± 2.62   Final old_new: 33.48 ± 2.53   Final new_new: 7.73 ± 4.13
Hits@50:
  Final val: 73.44 ± 1.08   Final Test: 59.11 ± 2.53   Final old_old: 63.13 ± 2.42   Final old_new: 43.35 ± 3.20   Final new_new: 14.55 ± 5.47
AUC:
  Final val: 97.27 ± 0.04   Final Test: 96.73 ± 0.05   Final old_old: 97.33 ± 0.06   Final old_new: 94.32 ± 0.13   Final new_new: 90.98 ± 0.97
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.21 ± 5.77   Final Test: 29.13 ± 3.38   Final old_old: 31.75 ± 3.80   Final old_new: 18.56 ± 2.03   Final new_new: 5.91 ± 4.43
Hits@20:
  Final val: 55.46 ± 4.54   Final Test: 36.65 ± 2.34   Final old_old: 39.73 ± 2.51   Final old_new: 24.23 ± 1.92   Final new_new: 8.64 ± 4.07
Hits@30:
  Final val: 63.41 ± 2.73   Final Test: 44.50 ± 3.47   Final old_old: 47.86 ± 3.55   Final old_new: 31.11 ± 3.40   Final new_new: 11.36 ± 4.25
Hits@50:
  Final val: 72.19 ± 2.00   Final Test: 53.73 ± 4.97   Final old_old: 57.41 ± 5.26   Final old_new: 39.19 ± 4.26   Final new_new: 15.45 ± 4.37
AUC:
  Final val: 96.58 ± 0.29   Final Test: 95.99 ± 0.23   Final old_old: 96.64 ± 0.23   Final old_new: 93.39 ± 0.50   Final new_new: 89.99 ± 1.43
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.72 ± 5.00   Final Test: 34.52 ± 5.90   Final old_old: 37.36 ± 6.40   Final old_new: 23.07 ± 3.98   Final new_new: 9.09 ± 4.25
Hits@20:
  Final val: 59.65 ± 3.64   Final Test: 44.98 ± 6.19   Final old_old: 48.32 ± 6.53   Final old_new: 31.71 ± 4.74   Final new_new: 12.27 ± 7.29
Hits@30:
  Final val: 67.52 ± 3.42   Final Test: 54.15 ± 2.41   Final old_old: 57.75 ± 2.71   Final old_new: 39.85 ± 1.42   Final new_new: 18.18 ± 8.19
Hits@50:
  Final val: 75.73 ± 1.45   Final Test: 61.10 ± 2.34   Final old_old: 64.96 ± 2.55   Final old_new: 45.84 ± 1.59   Final new_new: 20.91 ± 5.43
AUC:
  Final val: 96.96 ± 0.19   Final Test: 96.45 ± 0.14   Final old_old: 97.14 ± 0.12   Final old_new: 93.78 ± 0.35   Final new_new: 88.15 ± 1.79
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.95 ± 6.43   Final Test: 26.14 ± 4.75   Final old_old: 28.73 ± 4.90   Final old_new: 15.57 ± 4.36   Final new_new: 5.45 ± 1.24
Hits@20:
  Final val: 57.19 ± 4.75   Final Test: 36.72 ± 3.49   Final old_old: 39.84 ± 3.73   Final old_new: 24.16 ± 2.70   Final new_new: 9.09 ± 1.61
Hits@30:
  Final val: 63.26 ± 3.60   Final Test: 45.62 ± 4.18   Final old_old: 48.88 ± 4.55   Final old_new: 32.62 ± 3.04   Final new_new: 14.09 ± 5.18
Hits@50:
  Final val: 72.25 ± 2.46   Final Test: 57.54 ± 2.57   Final old_old: 61.34 ± 2.75   Final old_new: 42.67 ± 2.20   Final new_new: 15.45 ± 4.07
AUC:
  Final val: 96.72 ± 0.25   Final Test: 95.95 ± 0.27   Final old_old: 96.65 ± 0.29   Final old_new: 93.19 ± 0.27   Final new_new: 88.87 ± 1.38
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.56 ± 6.11   Final Test: 28.31 ± 2.85   Final old_old: 30.94 ± 2.66   Final old_new: 17.61 ± 3.75   Final new_new: 6.36 ± 4.93
Hits@20:
  Final val: 58.22 ± 3.72   Final Test: 40.40 ± 3.83   Final old_old: 43.76 ± 3.94   Final old_new: 26.90 ± 3.68   Final new_new: 9.09 ± 4.25
Hits@30:
  Final val: 64.51 ± 2.08   Final Test: 48.52 ± 3.71   Final old_old: 52.11 ± 3.89   Final old_new: 34.31 ± 3.53   Final new_new: 11.82 ± 6.10
Hits@50:
  Final val: 71.82 ± 1.01   Final Test: 57.59 ± 3.18   Final old_old: 61.13 ± 3.29   Final old_new: 43.83 ± 2.98   Final new_new: 16.36 ± 4.93
AUC:
  Final val: 95.56 ± 0.13   Final Test: 94.92 ± 0.18   Final old_old: 95.63 ± 0.14   Final old_new: 92.30 ± 0.31   Final new_new: 84.36 ± 2.88
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.06 ± 5.89   Final Test: 26.40 ± 5.72   Final old_old: 28.86 ± 6.12   Final old_new: 16.42 ± 4.29   Final new_new: 5.91 ± 3.80
Hits@20:
  Final val: 55.89 ± 5.42   Final Test: 36.98 ± 2.96   Final old_old: 40.02 ± 3.29   Final old_new: 24.79 ± 2.60   Final new_new: 9.09 ± 4.25
Hits@30:
  Final val: 63.83 ± 3.75   Final Test: 44.35 ± 4.11   Final old_old: 47.84 ± 4.43   Final old_new: 30.45 ± 3.13   Final new_new: 10.00 ± 4.13
Hits@50:
  Final val: 72.19 ± 2.79   Final Test: 56.65 ± 1.91   Final old_old: 60.31 ± 2.36   Final old_new: 42.24 ± 1.06   Final new_new: 17.73 ± 6.10
AUC:
  Final val: 96.57 ± 0.37   Final Test: 95.98 ± 0.15   Final old_old: 96.63 ± 0.19   Final old_new: 93.34 ± 0.11   Final new_new: 90.00 ± 1.85
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=65536, lr=0.005, epochs=20000, eval_steps=5, runs=10, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=0.1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, rw_step=5, ns_rate=5, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.30 ± 2.71   Final Test: 32.49 ± 5.74   Final old_old: 35.56 ± 6.00   Final old_new: 20.15 ± 4.80   Final new_new: 4.55 ± 2.27
Hits@20:
  Final val: 59.83 ± 2.13   Final Test: 40.88 ± 5.26   Final old_old: 44.33 ± 5.67   Final old_new: 27.15 ± 3.77   Final new_new: 7.27 ± 1.90
Hits@30:
  Final val: 67.31 ± 1.38   Final Test: 50.67 ± 3.28   Final old_old: 54.65 ± 3.53   Final old_new: 34.99 ± 2.74   Final new_new: 8.64 ± 3.37
Hits@50:
  Final val: 75.50 ± 0.61   Final Test: 59.40 ± 2.52   Final old_old: 63.52 ± 2.88   Final old_new: 43.17 ± 1.76   Final new_new: 15.00 ± 5.47
AUC:
  Final val: 97.30 ± 0.15   Final Test: 96.74 ± 0.19   Final old_old: 97.33 ± 0.16   Final old_new: 94.43 ± 0.42   Final new_new: 90.68 ± 0.83
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 54.08 ± 3.73   Final Test: 36.47 ± 2.13   Final old_old: 39.86 ± 2.27   Final old_new: 22.95 ± 1.66   Final new_new: 3.64 ± 3.80
Hits@20:
  Final val: 66.61 ± 2.48   Final Test: 49.18 ± 3.32   Final old_old: 53.00 ± 3.63   Final old_new: 34.16 ± 2.22   Final new_new: 8.18 ± 4.71
Hits@30:
  Final val: 72.64 ± 2.23   Final Test: 54.41 ± 4.55   Final old_old: 58.30 ± 4.90   Final old_new: 39.19 ± 3.17   Final new_new: 11.36 ± 5.57
Hits@50:
  Final val: 79.45 ± 1.51   Final Test: 65.18 ± 3.12   Final old_old: 69.12 ± 3.07   Final old_new: 49.85 ± 3.43   Final new_new: 20.00 ± 4.07
AUC:
  Final val: 97.64 ± 0.13   Final Test: 97.24 ± 0.08   Final old_old: 97.74 ± 0.11   Final old_new: 95.31 ± 0.09   Final new_new: 91.61 ± 0.53
All runs:
Hits@10:
  Final val: 51.89 ± 2.29   Final Test: 33.04 ± 4.19   Final old_old: 36.38 ± 4.50   Final old_new: 19.60 ± 3.17   Final new_new: 3.18 ± 2.59
Hits@20:
  Final val: 63.92 ± 1.92   Final Test: 47.29 ± 3.42   Final old_old: 51.24 ± 3.66   Final old_new: 31.66 ± 2.72   Final new_new: 6.36 ± 1.90
Hits@30:
  Final val: 70.74 ± 0.57   Final Test: 53.11 ± 2.47   Final old_old: 57.22 ± 2.57   Final old_new: 37.03 ± 2.43   Final new_new: 7.73 ± 1.24
Hits@50:
  Final val: 77.49 ± 1.09   Final Test: 62.62 ± 2.79   Final old_old: 67.14 ± 3.00   Final old_new: 45.01 ± 2.49   Final new_new: 10.91 ± 2.96
AUC:
  Final val: 97.44 ± 0.18   Final Test: 96.86 ± 0.17   Final old_old: 97.50 ± 0.20   Final old_new: 94.31 ± 0.21   Final new_new: 90.61 ± 0.87
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.71 ± 3.87   Final Test: 33.62 ± 3.90   Final old_old: 37.13 ± 4.05   Final old_new: 19.40 ± 3.52   Final new_new: 3.64 ± 2.59
Hits@20:
  Final val: 61.27 ± 0.81   Final Test: 40.88 ± 3.53   Final old_old: 44.61 ± 3.84   Final old_new: 25.87 ± 2.35   Final new_new: 7.73 ± 3.05
Hits@30:
  Final val: 66.71 ± 0.98   Final Test: 46.47 ± 2.65   Final old_old: 50.42 ± 2.75   Final old_new: 30.65 ± 2.32   Final new_new: 9.09 ± 2.27
Hits@50:
  Final val: 73.46 ± 0.93   Final Test: 58.07 ± 1.15   Final old_old: 61.96 ± 1.20   Final old_new: 42.70 ± 1.17   Final new_new: 18.18 ± 2.27
AUC:
  Final val: 95.56 ± 0.31   Final Test: 95.17 ± 0.25   Final old_old: 95.84 ± 0.27   Final old_new: 92.56 ± 0.31   Final new_new: 87.40 ± 1.65
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.58 ± 1.46   Final Test: 33.97 ± 2.66   Final old_old: 37.33 ± 2.72   Final old_new: 20.48 ± 2.54   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 59.65 ± 2.13   Final Test: 39.74 ± 2.51   Final old_old: 43.17 ± 2.64   Final old_new: 26.10 ± 2.23   Final new_new: 5.91 ± 2.59
Hits@30:
  Final val: 66.56 ± 1.58   Final Test: 47.84 ± 2.90   Final old_old: 51.59 ± 2.67   Final old_new: 33.07 ± 4.15   Final new_new: 8.18 ± 2.03
Hits@50:
  Final val: 74.16 ± 1.32   Final Test: 56.46 ± 2.50   Final old_old: 60.46 ± 2.83   Final old_new: 40.88 ± 1.33   Final new_new: 10.91 ± 4.37
AUC:
  Final val: 97.26 ± 0.09   Final Test: 96.71 ± 0.06   Final old_old: 97.30 ± 0.06   Final old_new: 94.37 ± 0.22   Final new_new: 90.61 ± 0.64
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.68 ± 3.68   Final Test: 29.59 ± 4.84   Final old_old: 32.53 ± 5.05   Final old_new: 17.71 ± 4.20   Final new_new: 3.64 ± 3.45
Hits@20:
  Final val: 57.98 ± 2.23   Final Test: 39.21 ± 4.27   Final old_old: 42.60 ± 4.72   Final old_new: 25.62 ± 2.57   Final new_new: 8.18 ± 2.03
Hits@30:
  Final val: 65.64 ± 2.39   Final Test: 46.71 ± 2.62   Final old_old: 50.48 ± 2.80   Final old_new: 31.71 ± 2.30   Final new_new: 10.00 ± 3.45
Hits@50:
  Final val: 73.74 ± 1.51   Final Test: 58.02 ± 2.25   Final old_old: 62.12 ± 2.29   Final old_new: 41.84 ± 2.68   Final new_new: 15.45 ± 1.90
AUC:
  Final val: 97.27 ± 0.12   Final Test: 96.72 ± 0.18   Final old_old: 97.31 ± 0.21   Final old_new: 94.40 ± 0.18   Final new_new: 90.77 ± 0.85
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 53.33 ± 2.88   Final Test: 33.54 ± 8.67   Final old_old: 37.03 ± 9.38   Final old_new: 19.37 ± 5.75   Final new_new: 4.09 ± 4.37
Hits@20:
  Final val: 65.48 ± 1.98   Final Test: 41.83 ± 4.46   Final old_old: 45.80 ± 4.93   Final old_new: 25.99 ± 2.74   Final new_new: 3.64 ± 3.05
Hits@30:
  Final val: 71.15 ± 1.24   Final Test: 52.42 ± 4.11   Final old_old: 56.69 ± 4.57   Final old_new: 35.64 ± 2.32   Final new_new: 6.82 ± 1.61
Hits@50:
  Final val: 77.41 ± 1.49   Final Test: 63.16 ± 2.45   Final old_old: 67.68 ± 2.74   Final old_new: 45.31 ± 1.61   Final new_new: 16.36 ± 1.02
AUC:
  Final val: 97.43 ± 0.15   Final Test: 96.92 ± 0.14   Final old_old: 97.57 ± 0.14   Final old_new: 94.33 ± 0.16   Final new_new: 89.74 ± 1.12
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.08 ± 2.22   Final Test: 30.41 ± 5.10   Final old_old: 33.78 ± 5.27   Final old_new: 16.57 ± 4.68   Final new_new: 5.00 ± 1.90
Hits@20:
  Final val: 60.81 ± 1.14   Final Test: 40.42 ± 4.66   Final old_old: 43.95 ± 4.84   Final old_new: 26.22 ± 4.21   Final new_new: 8.64 ± 2.49
Hits@30:
  Final val: 65.94 ± 0.77   Final Test: 49.72 ± 2.28   Final old_old: 53.32 ± 2.17   Final old_new: 35.31 ± 3.00   Final new_new: 15.45 ± 4.93
Hits@50:
  Final val: 72.33 ± 0.38   Final Test: 58.14 ± 1.29   Final old_old: 61.63 ± 1.06   Final old_new: 44.38 ± 2.56   Final new_new: 20.91 ± 4.93
AUC:
  Final val: 95.39 ± 0.26   Final Test: 94.95 ± 0.15   Final old_old: 95.69 ± 0.17   Final old_new: 92.13 ± 0.36   Final new_new: 84.93 ± 1.91
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.98 ± 6.27   Final Test: 29.44 ± 7.24   Final old_old: 32.16 ± 7.76   Final old_new: 18.41 ± 5.37   Final new_new: 5.91 ± 1.24
Hits@20:
  Final val: 58.04 ± 4.07   Final Test: 40.62 ± 5.50   Final old_old: 43.80 ± 5.80   Final old_new: 28.06 ± 4.42   Final new_new: 7.73 ± 2.03
Hits@30:
  Final val: 65.00 ± 2.55   Final Test: 46.05 ± 3.71   Final old_old: 49.64 ± 3.87   Final old_new: 31.74 ± 3.11   Final new_new: 10.91 ± 3.37
Hits@50:
  Final val: 72.92 ± 1.83   Final Test: 56.85 ± 2.67   Final old_old: 60.59 ± 2.63   Final old_new: 42.17 ± 3.07   Final new_new: 16.82 ± 2.03
AUC:
  Final val: 96.67 ± 0.27   Final Test: 95.96 ± 0.15   Final old_old: 96.62 ± 0.21   Final old_new: 93.37 ± 0.26   Final new_new: 89.56 ± 0.88
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.24 ± 4.48   Final Test: 32.19 ± 2.52   Final old_old: 35.39 ± 2.70   Final old_new: 19.19 ± 1.93   Final new_new: 5.00 ± 2.49
Hits@20:
  Final val: 60.95 ± 2.57   Final Test: 39.12 ± 2.20   Final old_old: 42.78 ± 2.39   Final old_new: 24.38 ± 1.79   Final new_new: 5.91 ± 2.59
Hits@30:
  Final val: 67.27 ± 1.92   Final Test: 47.63 ± 1.47   Final old_old: 51.53 ± 1.58   Final old_new: 32.14 ± 1.21   Final new_new: 8.64 ± 2.49
Hits@50:
  Final val: 74.79 ± 1.01   Final Test: 59.01 ± 1.49   Final old_old: 63.11 ± 1.53   Final old_new: 42.95 ± 1.85   Final new_new: 14.09 ± 5.18
AUC:
  Final val: 97.30 ± 0.09   Final Test: 96.77 ± 0.09   Final old_old: 97.36 ± 0.08   Final old_new: 94.40 ± 0.20   Final new_new: 90.60 ± 0.58
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.53 ± 3.33   Final Test: 28.35 ± 2.71   Final old_old: 30.92 ± 3.05   Final old_new: 17.98 ± 2.02   Final new_new: 5.45 ± 4.71
Hits@20:
  Final val: 56.02 ± 2.10   Final Test: 39.10 ± 3.65   Final old_old: 42.28 ± 3.80   Final old_new: 26.35 ± 3.35   Final new_new: 9.09 ± 6.63
Hits@30:
  Final val: 61.72 ± 1.91   Final Test: 44.11 ± 4.29   Final old_old: 47.51 ± 4.50   Final old_new: 30.48 ± 3.69   Final new_new: 12.27 ± 7.97
Hits@50:
  Final val: 68.62 ± 1.48   Final Test: 55.75 ± 2.37   Final old_old: 59.36 ± 2.32   Final old_new: 41.61 ± 3.16   Final new_new: 15.91 ± 4.82
AUC:
  Final val: 94.98 ± 0.27   Final Test: 94.56 ± 0.43   Final old_old: 95.17 ± 0.45   Final old_new: 92.28 ± 0.37   Final new_new: 86.08 ± 2.03
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.25 ± 3.37   Final Test: 28.77 ± 4.86   Final old_old: 31.47 ± 5.20   Final old_new: 17.88 ± 3.54   Final new_new: 5.00 ± 2.96
Hits@20:
  Final val: 58.58 ± 2.04   Final Test: 40.70 ± 4.50   Final old_old: 44.08 ± 4.88   Final old_new: 27.23 ± 3.92   Final new_new: 7.27 ± 2.49
Hits@30:
  Final val: 66.78 ± 1.75   Final Test: 48.67 ± 5.62   Final old_old: 52.32 ± 5.77   Final old_new: 34.18 ± 5.51   Final new_new: 11.82 ± 4.93
Hits@50:
  Final val: 74.59 ± 1.40   Final Test: 57.96 ± 3.13   Final old_old: 61.98 ± 3.46   Final old_new: 42.12 ± 2.97   Final new_new: 16.36 ± 1.90
AUC:
  Final val: 96.75 ± 0.24   Final Test: 96.00 ± 0.19   Final old_old: 96.70 ± 0.23   Final old_new: 93.19 ± 0.29   Final new_new: 89.64 ± 0.74
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.97 ± 3.16   Final Test: 31.58 ± 3.03   Final old_old: 34.68 ± 3.42   Final old_new: 18.97 ± 1.48   Final new_new: 5.91 ± 3.05
Hits@20:
  Final val: 58.52 ± 2.19   Final Test: 41.18 ± 5.65   Final old_old: 44.75 ± 5.96   Final old_new: 26.88 ± 4.47   Final new_new: 8.18 ± 2.59
Hits@30:
  Final val: 64.78 ± 0.39   Final Test: 48.29 ± 3.48   Final old_old: 51.96 ± 3.62   Final old_new: 33.70 ± 2.94   Final new_new: 11.36 ± 4.25
Hits@50:
  Final val: 71.68 ± 1.18   Final Test: 58.57 ± 4.35   Final old_old: 62.19 ± 4.21   Final old_new: 44.46 ± 5.19   Final new_new: 17.73 ± 4.37
AUC:
  Final val: 94.80 ± 0.23   Final Test: 94.40 ± 0.20   Final old_old: 95.03 ± 0.24   Final old_new: 92.01 ± 0.42   Final new_new: 86.28 ± 1.19
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.97 ± 3.80   Final Test: 31.99 ± 3.53   Final old_old: 35.12 ± 3.83   Final old_new: 19.22 ± 2.95   Final new_new: 6.36 ± 2.96
Hits@20:
  Final val: 60.51 ± 1.26   Final Test: 39.71 ± 5.02   Final old_old: 43.01 ± 5.31   Final old_new: 26.57 ± 3.98   Final new_new: 7.73 ± 4.13
Hits@30:
  Final val: 66.71 ± 0.97   Final Test: 51.20 ± 1.29   Final old_old: 55.17 ± 1.34   Final old_new: 35.44 ± 1.78   Final new_new: 10.91 ± 4.07
Hits@50:
  Final val: 74.94 ± 2.23   Final Test: 60.91 ± 2.65   Final old_old: 65.19 ± 2.56   Final old_new: 44.06 ± 3.34   Final new_new: 15.91 ± 3.59
AUC:
  Final val: 97.31 ± 0.10   Final Test: 96.78 ± 0.15   Final old_old: 97.42 ± 0.20   Final old_new: 94.26 ± 0.16   Final new_new: 90.65 ± 1.50
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.51 ± 2.50   Final Test: 33.67 ± 4.91   Final old_old: 36.92 ± 5.14   Final old_new: 20.55 ± 4.03   Final new_new: 4.55 ± 3.21
Hits@20:
  Final val: 63.87 ± 1.94   Final Test: 41.87 ± 3.91   Final old_old: 45.62 ± 4.26   Final old_new: 26.93 ± 2.46   Final new_new: 5.45 ± 1.24
Hits@30:
  Final val: 69.22 ± 1.69   Final Test: 51.31 ± 4.49   Final old_old: 55.35 ± 4.95   Final old_new: 35.31 ± 2.64   Final new_new: 9.55 ± 3.73
Hits@50:
  Final val: 76.60 ± 2.12   Final Test: 62.92 ± 3.28   Final old_old: 67.12 ± 3.54   Final old_new: 46.37 ± 2.42   Final new_new: 18.18 ± 8.04
AUC:
  Final val: 97.39 ± 0.15   Final Test: 96.90 ± 0.15   Final old_old: 97.51 ± 0.17   Final old_new: 94.51 ± 0.17   Final new_new: 90.31 ± 1.22
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.76 ± 2.08   Final Test: 24.90 ± 3.21   Final old_old: 27.35 ± 3.64   Final old_new: 14.86 ± 1.86   Final new_new: 5.45 ± 2.59
Hits@20:
  Final val: 58.51 ± 1.49   Final Test: 36.02 ± 5.24   Final old_old: 39.14 ± 5.55   Final old_new: 23.43 ± 4.38   Final new_new: 8.64 ± 4.37
Hits@30:
  Final val: 65.27 ± 0.51   Final Test: 45.53 ± 3.04   Final old_old: 48.98 ± 3.09   Final old_new: 31.86 ± 3.29   Final new_new: 10.45 ± 4.98
Hits@50:
  Final val: 73.65 ± 1.29   Final Test: 55.59 ± 4.59   Final old_old: 59.35 ± 4.56   Final old_new: 40.78 ± 5.22   Final new_new: 15.91 ± 7.01
AUC:
  Final val: 96.69 ± 0.36   Final Test: 96.00 ± 0.14   Final old_old: 96.60 ± 0.18   Final old_new: 93.64 ± 0.25   Final new_new: 89.74 ± 0.76
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.68 ± 2.82   Final Test: 28.21 ± 5.31   Final old_old: 31.39 ± 5.50   Final old_new: 15.21 ± 4.86   Final new_new: 2.73 ± 2.96
Hits@20:
  Final val: 58.86 ± 2.23   Final Test: 38.12 ± 2.08   Final old_old: 41.49 ± 2.23   Final old_new: 24.69 ± 1.95   Final new_new: 5.91 ± 4.13
Hits@30:
  Final val: 64.03 ± 1.11   Final Test: 47.49 ± 2.86   Final old_old: 50.90 ± 2.94   Final old_new: 34.03 ± 2.80   Final new_new: 12.27 ± 5.23
Hits@50:
  Final val: 70.61 ± 0.31   Final Test: 57.27 ± 2.11   Final old_old: 60.65 ± 2.18   Final old_new: 44.03 ± 2.09   Final new_new: 20.45 ± 3.94
AUC:
  Final val: 94.68 ± 0.11   Final Test: 94.34 ± 0.24   Final old_old: 95.03 ± 0.20   Final old_new: 91.60 ± 0.44   Final new_new: 87.02 ± 1.37
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.13 ± 3.78   Final Test: 31.82 ± 2.87   Final old_old: 34.73 ± 3.08   Final old_new: 20.18 ± 2.45   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 57.27 ± 3.76   Final Test: 37.77 ± 4.47   Final old_old: 41.16 ± 4.84   Final old_new: 24.16 ± 3.14   Final new_new: 6.36 ± 3.37
Hits@30:
  Final val: 65.05 ± 2.51   Final Test: 45.55 ± 2.46   Final old_old: 49.28 ± 2.73   Final old_new: 30.71 ± 1.68   Final new_new: 8.64 ± 2.96
Hits@50:
  Final val: 73.60 ± 2.17   Final Test: 56.72 ± 2.00   Final old_old: 60.87 ± 1.83   Final old_new: 40.35 ± 3.02   Final new_new: 13.64 ± 4.82
AUC:
  Final val: 97.27 ± 0.19   Final Test: 96.74 ± 0.08   Final old_old: 97.29 ± 0.14   Final old_new: 94.60 ± 0.20   Final new_new: 90.85 ± 1.04
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.37 ± 2.37   Final Test: 32.88 ± 3.33   Final old_old: 36.15 ± 3.71   Final old_new: 19.72 ± 1.91   Final new_new: 3.18 ± 2.03
Hits@20:
  Final val: 64.69 ± 1.22   Final Test: 44.20 ± 3.22   Final old_old: 48.00 ± 3.50   Final old_new: 29.02 ± 2.55   Final new_new: 7.73 ± 4.71
Hits@30:
  Final val: 70.79 ± 3.04   Final Test: 50.63 ± 1.70   Final old_old: 54.79 ± 1.62   Final old_new: 34.06 ± 2.68   Final new_new: 10.00 ± 2.59
Hits@50:
  Final val: 77.20 ± 1.98   Final Test: 60.56 ± 3.29   Final old_old: 64.85 ± 3.31   Final old_new: 43.73 ± 3.43   Final new_new: 13.64 ± 3.59
AUC:
  Final val: 97.42 ± 0.17   Final Test: 96.89 ± 0.15   Final old_old: 97.50 ± 0.18   Final old_new: 94.46 ± 0.18   Final new_new: 90.70 ± 0.52
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.28 ± 4.02   Final Test: 37.16 ± 2.51   Final old_old: 40.23 ± 2.65   Final old_new: 25.04 ± 2.09   Final new_new: 5.45 ± 3.80
Hits@20:
  Final val: 60.67 ± 1.68   Final Test: 44.22 ± 3.33   Final old_old: 47.82 ± 3.49   Final old_new: 29.92 ± 2.62   Final new_new: 9.09 ± 5.79
Hits@30:
  Final val: 67.32 ± 0.94   Final Test: 51.59 ± 3.86   Final old_old: 55.36 ± 3.80   Final old_new: 36.70 ± 4.53   Final new_new: 11.82 ± 2.49
Hits@50:
  Final val: 75.07 ± 1.45   Final Test: 58.16 ± 2.87   Final old_old: 62.01 ± 2.83   Final old_new: 43.02 ± 3.50   Final new_new: 16.82 ± 3.05
AUC:
  Final val: 96.78 ± 0.18   Final Test: 96.18 ± 0.32   Final old_old: 96.80 ± 0.29   Final old_new: 93.69 ± 0.41   Final new_new: 90.33 ± 1.12
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.52 ± 1.97   Final Test: 31.65 ± 3.91   Final old_old: 35.01 ± 4.14   Final old_new: 18.06 ± 3.25   Final new_new: 3.18 ± 2.03
Hits@20:
  Final val: 60.87 ± 1.63   Final Test: 41.25 ± 2.33   Final old_old: 45.05 ± 2.53   Final old_new: 26.07 ± 2.17   Final new_new: 5.00 ± 1.90
Hits@30:
  Final val: 67.21 ± 0.69   Final Test: 50.17 ± 2.05   Final old_old: 54.27 ± 2.29   Final old_new: 33.88 ± 1.49   Final new_new: 9.55 ± 2.96
Hits@50:
  Final val: 74.88 ± 1.17   Final Test: 59.14 ± 3.92   Final old_old: 63.63 ± 3.95   Final old_new: 41.36 ± 4.22   Final new_new: 13.64 ± 3.59
AUC:
  Final val: 97.29 ± 0.15   Final Test: 96.81 ± 0.14   Final old_old: 97.40 ± 0.21   Final old_new: 94.46 ± 0.28   Final new_new: 90.54 ± 1.04
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.62 ± 2.87   Final Test: 31.10 ± 5.21   Final old_old: 33.72 ± 5.57   Final old_new: 20.50 ± 3.97   Final new_new: 8.64 ± 6.51
Hits@20:
  Final val: 57.65 ± 2.74   Final Test: 37.31 ± 5.66   Final old_old: 40.19 ± 6.06   Final old_new: 25.67 ± 4.13   Final new_new: 12.27 ± 6.55
Hits@30:
  Final val: 64.17 ± 2.97   Final Test: 46.67 ± 0.69   Final old_old: 50.02 ± 0.46   Final old_new: 33.32 ± 1.87   Final new_new: 14.09 ± 5.88
Hits@50:
  Final val: 72.02 ± 2.21   Final Test: 56.74 ± 0.55   Final old_old: 60.44 ± 0.59   Final old_new: 42.07 ± 1.50   Final new_new: 19.09 ± 3.45
AUC:
  Final val: 96.55 ± 0.25   Final Test: 95.92 ± 0.24   Final old_old: 96.61 ± 0.25   Final old_new: 93.13 ± 0.32   Final new_new: 89.66 ± 1.81
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.76 ± 6.47   Final Test: 28.00 ± 6.49   Final old_old: 30.34 ± 6.85   Final old_new: 18.56 ± 5.53   Final new_new: 6.82 ± 4.25
Hits@20:
  Final val: 53.67 ± 5.02   Final Test: 35.47 ± 3.99   Final old_old: 38.33 ± 4.48   Final old_new: 24.06 ± 2.31   Final new_new: 8.18 ± 3.05
Hits@30:
  Final val: 61.81 ± 4.62   Final Test: 43.71 ± 1.45   Final old_old: 46.75 ± 1.90   Final old_new: 31.71 ± 0.89   Final new_new: 11.82 ± 4.66
Hits@50:
  Final val: 71.02 ± 4.21   Final Test: 54.71 ± 3.67   Final old_old: 58.22 ± 3.92   Final old_new: 40.88 ± 3.38   Final new_new: 16.82 ± 3.05
AUC:
  Final val: 96.61 ± 0.32   Final Test: 95.94 ± 0.30   Final old_old: 96.62 ± 0.29   Final old_new: 93.23 ± 0.64   Final new_new: 89.79 ± 1.86
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.86 ± 3.49   Final Test: 28.89 ± 4.06   Final old_old: 31.81 ± 4.30   Final old_new: 17.00 ± 3.33   Final new_new: 4.55 ± 2.27
Hits@20:
  Final val: 58.69 ± 1.27   Final Test: 39.06 ± 1.28   Final old_old: 42.44 ± 1.35   Final old_new: 25.59 ± 1.68   Final new_new: 5.91 ± 1.24
Hits@30:
  Final val: 65.63 ± 0.29   Final Test: 46.34 ± 2.47   Final old_old: 50.04 ± 2.61   Final old_new: 31.71 ± 2.17   Final new_new: 7.73 ± 3.80
Hits@50:
  Final val: 74.17 ± 1.44   Final Test: 58.68 ± 4.75   Final old_old: 62.64 ± 4.80   Final old_new: 43.15 ± 4.97   Final new_new: 15.91 ± 6.01
AUC:
  Final val: 97.21 ± 0.13   Final Test: 96.71 ± 0.09   Final old_old: 97.30 ± 0.14   Final old_new: 94.36 ± 0.16   Final new_new: 90.39 ± 0.64
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.88 ± 1.41   Final Test: 33.25 ± 4.77   Final old_old: 36.29 ± 5.01   Final old_new: 21.13 ± 4.02   Final new_new: 3.64 ± 2.59
Hits@20:
  Final val: 63.70 ± 1.16   Final Test: 42.23 ± 4.74   Final old_old: 45.56 ± 4.86   Final old_new: 29.14 ± 4.66   Final new_new: 6.36 ± 2.96
Hits@30:
  Final val: 70.60 ± 0.71   Final Test: 52.35 ± 1.97   Final old_old: 56.11 ± 2.14   Final old_new: 37.66 ± 1.45   Final new_new: 10.45 ± 2.59
Hits@50:
  Final val: 77.51 ± 0.46   Final Test: 60.32 ± 1.81   Final old_old: 64.15 ± 1.83   Final old_new: 45.49 ± 1.97   Final new_new: 14.55 ± 2.59
AUC:
  Final val: 97.27 ± 0.09   Final Test: 96.82 ± 0.13   Final old_old: 97.32 ± 0.11   Final old_new: 94.83 ± 0.20   Final new_new: 91.40 ± 0.83
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.42 ± 2.39   Final Test: 34.68 ± 4.40   Final old_old: 38.11 ± 4.77   Final old_new: 20.86 ± 2.97   Final new_new: 4.55 ± 3.59
Hits@20:
  Final val: 62.45 ± 2.12   Final Test: 43.31 ± 2.92   Final old_old: 46.98 ± 3.04   Final old_new: 28.61 ± 2.74   Final new_new: 9.09 ± 5.33
Hits@30:
  Final val: 68.59 ± 1.35   Final Test: 51.27 ± 4.54   Final old_old: 55.10 ± 4.63   Final old_new: 36.15 ± 4.33   Final new_new: 11.36 ± 4.82
Hits@50:
  Final val: 75.97 ± 1.14   Final Test: 61.85 ± 1.94   Final old_old: 65.99 ± 2.00   Final old_new: 45.52 ± 2.41   Final new_new: 18.18 ± 5.33
AUC:
  Final val: 97.31 ± 0.13   Final Test: 96.86 ± 0.09   Final old_old: 97.46 ± 0.09   Final old_new: 94.47 ± 0.23   Final new_new: 91.41 ± 0.32
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.63 ± 4.59   Final Test: 31.37 ± 3.52   Final old_old: 34.54 ± 3.75   Final old_new: 18.59 ± 2.83   Final new_new: 3.18 ± 2.59
Hits@20:
  Final val: 60.88 ± 3.24   Final Test: 41.95 ± 5.21   Final old_old: 45.54 ± 5.59   Final old_new: 27.71 ± 3.93   Final new_new: 5.91 ± 2.03
Hits@30:
  Final val: 67.95 ± 2.49   Final Test: 50.51 ± 3.08   Final old_old: 54.68 ± 3.19   Final old_new: 33.98 ± 2.80   Final new_new: 8.18 ± 4.13
Hits@50:
  Final val: 75.12 ± 1.79   Final Test: 62.29 ± 2.13   Final old_old: 66.65 ± 2.24   Final old_new: 45.04 ± 2.19   Final new_new: 18.18 ± 5.33
AUC:
  Final val: 97.32 ± 0.09   Final Test: 96.79 ± 0.11   Final old_old: 97.42 ± 0.11   Final old_new: 94.29 ± 0.21   Final new_new: 91.21 ± 1.16
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.75 ± 3.96   Final Test: 33.13 ± 3.33   Final old_old: 36.25 ± 3.55   Final old_new: 20.68 ± 2.69   Final new_new: 3.18 ± 1.24
Hits@20:
  Final val: 63.56 ± 1.98   Final Test: 42.73 ± 1.59   Final old_old: 46.16 ± 1.57   Final old_new: 29.24 ± 1.85   Final new_new: 5.45 ± 2.59
Hits@30:
  Final val: 69.33 ± 1.48   Final Test: 50.90 ± 3.01   Final old_old: 54.52 ± 3.19   Final old_new: 36.85 ± 2.38   Final new_new: 9.09 ± 1.61
Hits@50:
  Final val: 76.85 ± 0.96   Final Test: 60.75 ± 1.85   Final old_old: 64.64 ± 1.82   Final old_new: 45.59 ± 2.08   Final new_new: 16.82 ± 4.13
AUC:
  Final val: 97.43 ± 0.06   Final Test: 97.00 ± 0.11   Final old_old: 97.49 ± 0.10   Final old_new: 95.07 ± 0.16   Final new_new: 91.26 ± 0.30
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.35 ± 7.93   Final Test: 28.68 ± 3.69   Final old_old: 31.13 ± 3.86   Final old_new: 18.92 ± 3.31   Final new_new: 5.45 ± 4.71
Hits@20:
  Final val: 57.96 ± 6.48   Final Test: 37.77 ± 4.48   Final old_old: 40.65 ± 5.01   Final old_new: 26.40 ± 2.97   Final new_new: 8.18 ± 3.05
Hits@30:
  Final val: 65.27 ± 3.72   Final Test: 45.33 ± 3.39   Final old_old: 48.54 ± 3.70   Final old_new: 32.72 ± 2.47   Final new_new: 10.45 ± 3.45
Hits@50:
  Final val: 73.57 ± 2.46   Final Test: 54.67 ± 3.20   Final old_old: 58.30 ± 3.23   Final old_new: 40.28 ± 3.61   Final new_new: 18.18 ± 5.33
AUC:
  Final val: 96.69 ± 0.27   Final Test: 96.06 ± 0.26   Final old_old: 96.66 ± 0.31   Final old_new: 93.61 ± 0.30   Final new_new: 90.87 ± 0.70
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.15 ± 4.01   Final Test: 34.42 ± 4.09   Final old_old: 37.68 ± 4.55   Final old_new: 21.34 ± 2.43   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 59.36 ± 1.98   Final Test: 39.97 ± 4.11   Final old_old: 43.52 ± 4.33   Final old_new: 25.79 ± 3.72   Final new_new: 6.36 ± 2.96
Hits@30:
  Final val: 66.12 ± 1.56   Final Test: 48.23 ± 3.08   Final old_old: 52.13 ± 3.46   Final old_new: 32.62 ± 1.41   Final new_new: 10.91 ± 2.96
Hits@50:
  Final val: 73.94 ± 1.02   Final Test: 57.84 ± 3.33   Final old_old: 62.05 ± 3.51   Final old_new: 41.13 ± 2.51   Final new_new: 15.45 ± 6.51
AUC:
  Final val: 97.27 ± 0.09   Final Test: 96.82 ± 0.08   Final old_old: 97.41 ± 0.10   Final old_new: 94.51 ± 0.15   Final new_new: 90.36 ± 1.28
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.83 ± 1.90   Final Test: 32.02 ± 3.76   Final old_old: 35.02 ± 3.89   Final old_new: 19.95 ± 3.55   Final new_new: 5.00 ± 2.49
Hits@20:
  Final val: 60.41 ± 1.84   Final Test: 40.99 ± 3.56   Final old_old: 44.54 ± 3.67   Final old_new: 26.80 ± 3.34   Final new_new: 7.73 ± 3.05
Hits@30:
  Final val: 66.67 ± 1.28   Final Test: 46.90 ± 4.30   Final old_old: 50.64 ± 4.57   Final old_new: 32.07 ± 3.36   Final new_new: 9.09 ± 2.27
Hits@50:
  Final val: 74.17 ± 0.71   Final Test: 57.00 ± 2.19   Final old_old: 61.07 ± 2.32   Final old_new: 40.98 ± 1.84   Final new_new: 13.18 ± 2.96
AUC:
  Final val: 97.28 ± 0.10   Final Test: 96.75 ± 0.11   Final old_old: 97.34 ± 0.13   Final old_new: 94.46 ± 0.16   Final new_new: 90.53 ± 1.04
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.63 ± 4.00   Final Test: 31.20 ± 5.61   Final old_old: 34.38 ± 6.20   Final old_new: 18.26 ± 3.50   Final new_new: 5.00 ± 2.49
Hits@20:
  Final val: 63.16 ± 3.09   Final Test: 44.69 ± 6.96   Final old_old: 48.59 ± 7.52   Final old_new: 29.02 ± 5.00   Final new_new: 8.18 ± 2.03
Hits@30:
  Final val: 69.31 ± 2.57   Final Test: 53.11 ± 3.92   Final old_old: 57.46 ± 4.35   Final old_new: 35.87 ± 2.78   Final new_new: 8.64 ± 1.02
Hits@50:
  Final val: 76.26 ± 3.46   Final Test: 62.07 ± 2.32   Final old_old: 66.46 ± 2.28   Final old_new: 44.84 ± 3.06   Final new_new: 14.55 ± 3.80
AUC:
  Final val: 97.41 ± 0.22   Final Test: 96.86 ± 0.21   Final old_old: 97.49 ± 0.25   Final old_new: 94.40 ± 0.24   Final new_new: 90.21 ± 1.53
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.79 ± 1.38   Final Test: 27.79 ± 2.73   Final old_old: 30.40 ± 2.96   Final old_new: 17.25 ± 2.14   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 56.37 ± 2.46   Final Test: 40.55 ± 2.43   Final old_old: 43.67 ± 2.78   Final old_new: 27.96 ± 1.82   Final new_new: 12.27 ± 3.45
Hits@30:
  Final val: 63.67 ± 2.63   Final Test: 46.33 ± 2.16   Final old_old: 49.83 ± 2.32   Final old_new: 32.49 ± 1.88   Final new_new: 10.45 ± 3.80
Hits@50:
  Final val: 73.48 ± 1.29   Final Test: 55.39 ± 2.38   Final old_old: 59.08 ± 2.44   Final old_new: 40.91 ± 2.62   Final new_new: 15.00 ± 4.71
AUC:
  Final val: 96.69 ± 0.14   Final Test: 95.97 ± 0.13   Final old_old: 96.64 ± 0.10   Final old_new: 93.36 ± 0.37   Final new_new: 88.62 ± 2.14
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.17 ± 4.73   Final Test: 28.06 ± 5.17   Final old_old: 30.63 ± 5.57   Final old_new: 17.68 ± 3.98   Final new_new: 5.45 ± 3.80
Hits@20:
  Final val: 54.40 ± 3.35   Final Test: 36.44 ± 4.00   Final old_old: 39.58 ± 4.32   Final old_new: 23.90 ± 3.04   Final new_new: 5.91 ± 2.03
Hits@30:
  Final val: 62.17 ± 2.46   Final Test: 44.40 ± 2.92   Final old_old: 47.72 ± 3.12   Final old_new: 31.26 ± 2.47   Final new_new: 10.91 ± 4.93
Hits@50:
  Final val: 71.00 ± 1.28   Final Test: 55.55 ± 1.90   Final old_old: 59.46 ± 2.00   Final old_new: 40.23 ± 1.71   Final new_new: 12.73 ± 5.70
AUC:
  Final val: 96.59 ± 0.22   Final Test: 95.97 ± 0.26   Final old_old: 96.62 ± 0.27   Final old_new: 93.39 ± 0.45   Final new_new: 89.41 ± 1.53
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.08 ± 4.77   Final Test: 26.44 ± 5.12   Final old_old: 28.88 ± 5.45   Final old_new: 16.60 ± 3.91   Final new_new: 4.55 ± 3.21
Hits@20:
  Final val: 57.21 ± 3.24   Final Test: 41.60 ± 2.20   Final old_old: 44.74 ± 2.44   Final old_new: 29.24 ± 1.61   Final new_new: 7.73 ± 2.59
Hits@30:
  Final val: 64.84 ± 1.75   Final Test: 47.10 ± 2.12   Final old_old: 50.59 ± 2.18   Final old_new: 33.45 ± 2.46   Final new_new: 8.64 ± 2.49
Hits@50:
  Final val: 73.31 ± 1.13   Final Test: 56.50 ± 3.90   Final old_old: 60.19 ± 4.28   Final old_new: 42.04 ± 2.60   Final new_new: 15.91 ± 1.61
AUC:
  Final val: 96.63 ± 0.19   Final Test: 95.94 ± 0.42   Final old_old: 96.62 ± 0.34   Final old_new: 93.29 ± 0.72   Final new_new: 88.27 ± 2.02
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.00 ± 1.93   Final Test: 31.02 ± 2.80   Final old_old: 34.07 ± 2.89   Final old_new: 18.66 ± 2.69   Final new_new: 5.00 ± 1.90
Hits@20:
  Final val: 57.15 ± 0.60   Final Test: 38.84 ± 2.86   Final old_old: 42.23 ± 2.97   Final old_new: 25.24 ± 2.57   Final new_new: 6.82 ± 3.21
Hits@30:
  Final val: 65.18 ± 0.87   Final Test: 45.74 ± 3.88   Final old_old: 49.47 ± 3.90   Final old_new: 30.86 ± 4.08   Final new_new: 10.00 ± 2.03
Hits@50:
  Final val: 73.88 ± 1.16   Final Test: 57.98 ± 2.28   Final old_old: 61.95 ± 2.38   Final old_new: 42.34 ± 2.17   Final new_new: 15.91 ± 3.21
AUC:
  Final val: 97.27 ± 0.11   Final Test: 96.74 ± 0.14   Final old_old: 97.30 ± 0.19   Final old_new: 94.52 ± 0.10   Final new_new: 91.11 ± 0.38
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.81 ± 2.10   Final Test: 34.30 ± 3.82   Final old_old: 37.66 ± 4.13   Final old_new: 20.73 ± 2.63   Final new_new: 4.55 ± 2.78
Hits@20:
  Final val: 61.84 ± 3.65   Final Test: 42.20 ± 6.08   Final old_old: 45.95 ± 6.45   Final old_new: 27.13 ± 4.77   Final new_new: 8.18 ± 3.05
Hits@30:
  Final val: 68.76 ± 3.38   Final Test: 50.23 ± 6.67   Final old_old: 54.32 ± 6.86   Final old_new: 33.88 ± 6.07   Final new_new: 10.91 ± 5.43
Hits@50:
  Final val: 76.69 ± 1.59   Final Test: 60.93 ± 3.12   Final old_old: 65.02 ± 3.33   Final old_new: 44.86 ± 2.27   Final new_new: 16.82 ± 3.80
AUC:
  Final val: 97.36 ± 0.06   Final Test: 96.89 ± 0.09   Final old_old: 97.48 ± 0.11   Final old_new: 94.48 ± 0.14   Final new_new: 91.37 ± 0.76
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.83 ± 2.44   Final Test: 35.13 ± 6.00   Final old_old: 38.34 ± 6.35   Final old_new: 22.32 ± 4.85   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 62.68 ± 1.46   Final Test: 42.90 ± 4.36   Final old_old: 46.61 ± 4.69   Final old_new: 28.14 ± 3.29   Final new_new: 5.91 ± 2.03
Hits@30:
  Final val: 69.44 ± 1.21   Final Test: 50.78 ± 2.71   Final old_old: 54.89 ± 2.98   Final old_new: 34.51 ± 2.21   Final new_new: 8.64 ± 3.37
Hits@50:
  Final val: 76.24 ± 1.13   Final Test: 61.90 ± 1.99   Final old_old: 66.19 ± 2.40   Final old_new: 45.11 ± 0.51   Final new_new: 15.00 ± 4.13
AUC:
  Final val: 97.35 ± 0.09   Final Test: 96.85 ± 0.17   Final old_old: 97.48 ± 0.18   Final old_new: 94.33 ± 0.27   Final new_new: 90.29 ± 0.84
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.46 ± 2.45   Final Test: 34.04 ± 5.62   Final old_old: 37.38 ± 5.94   Final old_new: 20.53 ± 4.56   Final new_new: 4.55 ± 2.27
Hits@20:
  Final val: 62.95 ± 1.03   Final Test: 45.80 ± 4.55   Final old_old: 49.59 ± 4.59   Final old_new: 30.73 ± 4.66   Final new_new: 7.73 ± 3.05
Hits@30:
  Final val: 69.58 ± 0.81   Final Test: 50.86 ± 2.47   Final old_old: 55.05 ± 2.62   Final old_new: 34.26 ± 2.23   Final new_new: 8.18 ± 2.59
Hits@50:
  Final val: 76.72 ± 0.89   Final Test: 61.16 ± 3.47   Final old_old: 65.38 ± 3.59   Final old_new: 44.74 ± 3.33   Final new_new: 13.18 ± 2.49
AUC:
  Final val: 97.37 ± 0.09   Final Test: 96.86 ± 0.19   Final old_old: 97.48 ± 0.18   Final old_new: 94.42 ± 0.28   Final new_new: 90.40 ± 0.80
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.78 ± 6.28   Final Test: 29.19 ± 4.64   Final old_old: 31.69 ± 4.93   Final old_new: 19.07 ± 3.36   Final new_new: 7.73 ± 5.23
Hits@20:
  Final val: 52.72 ± 4.45   Final Test: 39.36 ± 3.58   Final old_old: 42.45 ± 3.76   Final old_new: 26.93 ± 3.05   Final new_new: 11.36 ± 4.55
Hits@30:
  Final val: 61.61 ± 2.96   Final Test: 45.61 ± 4.00   Final old_old: 49.06 ± 4.38   Final old_new: 31.79 ± 2.44   Final new_new: 13.18 ± 6.51
Hits@50:
  Final val: 72.15 ± 2.44   Final Test: 56.47 ± 3.65   Final old_old: 60.04 ± 3.78   Final old_new: 42.52 ± 3.33   Final new_new: 16.82 ± 5.93
AUC:
  Final val: 96.69 ± 0.24   Final Test: 96.08 ± 0.36   Final old_old: 96.71 ± 0.37   Final old_new: 93.60 ± 0.42   Final new_new: 89.64 ± 1.67
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.04 ± 2.06   Final Test: 33.32 ± 4.20   Final old_old: 36.62 ± 4.39   Final old_new: 20.05 ± 3.61   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 61.01 ± 1.86   Final Test: 41.11 ± 4.08   Final old_old: 44.92 ± 4.38   Final old_new: 25.89 ± 3.00   Final new_new: 5.00 ± 2.96
Hits@30:
  Final val: 67.93 ± 0.51   Final Test: 48.39 ± 3.07   Final old_old: 52.48 ± 2.98   Final old_new: 32.09 ± 3.65   Final new_new: 8.64 ± 2.96
Hits@50:
  Final val: 75.01 ± 0.60   Final Test: 58.64 ± 1.70   Final old_old: 63.06 ± 1.87   Final old_new: 41.21 ± 1.39   Final new_new: 12.73 ± 2.59
AUC:
  Final val: 97.33 ± 0.08   Final Test: 96.86 ± 0.15   Final old_old: 97.46 ± 0.11   Final old_new: 94.50 ± 0.43   Final new_new: 90.65 ± 0.72
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.49 ± 2.14   Final Test: 31.56 ± 2.21   Final old_old: 34.63 ± 2.50   Final old_new: 19.12 ± 1.18   Final new_new: 5.45 ± 1.24
Hits@20:
  Final val: 59.82 ± 2.77   Final Test: 39.09 ± 4.51   Final old_old: 42.64 ± 4.85   Final old_new: 24.76 ± 3.27   Final new_new: 7.73 ± 2.03
Hits@30:
  Final val: 65.92 ± 2.00   Final Test: 47.42 ± 3.47   Final old_old: 51.19 ± 3.72   Final old_new: 32.42 ± 2.62   Final new_new: 9.55 ± 2.49
Hits@50:
  Final val: 74.02 ± 0.72   Final Test: 59.33 ± 1.45   Final old_old: 63.40 ± 1.24   Final old_new: 43.30 ± 2.90   Final new_new: 16.36 ± 3.73
AUC:
  Final val: 97.24 ± 0.07   Final Test: 96.78 ± 0.11   Final old_old: 97.36 ± 0.10   Final old_new: 94.52 ± 0.37   Final new_new: 90.43 ± 1.24
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.30 ± 4.00   Final Test: 35.73 ± 3.28   Final old_old: 39.12 ± 3.54   Final old_new: 22.12 ± 2.47   Final new_new: 5.00 ± 1.90
Hits@20:
  Final val: 59.19 ± 1.80   Final Test: 40.19 ± 3.00   Final old_old: 43.56 ± 3.13   Final old_new: 26.80 ± 2.91   Final new_new: 6.82 ± 4.25
Hits@30:
  Final val: 66.28 ± 1.33   Final Test: 48.67 ± 3.37   Final old_old: 52.56 ± 3.84   Final old_new: 33.20 ± 2.19   Final new_new: 10.00 ± 4.71
Hits@50:
  Final val: 74.04 ± 1.57   Final Test: 58.28 ± 4.87   Final old_old: 62.44 ± 5.10   Final old_new: 41.76 ± 4.31   Final new_new: 16.36 ± 3.73
AUC:
  Final val: 97.26 ± 0.16   Final Test: 96.77 ± 0.09   Final old_old: 97.37 ± 0.12   Final old_new: 94.41 ± 0.23   Final new_new: 89.96 ± 1.77
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.50 ± 3.03   Final Test: 30.92 ± 4.77   Final old_old: 34.02 ± 5.10   Final old_new: 18.34 ± 3.48   Final new_new: 4.55 ± 2.27
Hits@20:
  Final val: 63.83 ± 1.91   Final Test: 43.22 ± 2.52   Final old_old: 47.06 ± 2.62   Final old_new: 27.71 ± 2.46   Final new_new: 9.09 ± 2.78
Hits@30:
  Final val: 69.92 ± 1.35   Final Test: 51.50 ± 1.07   Final old_old: 55.62 ± 1.34   Final old_new: 35.16 ± 0.58   Final new_new: 9.55 ± 1.90
Hits@50:
  Final val: 76.92 ± 1.04   Final Test: 60.88 ± 2.13   Final old_old: 65.19 ± 2.44   Final old_new: 44.03 ± 1.19   Final new_new: 13.18 ± 3.73
AUC:
  Final val: 97.34 ± 0.09   Final Test: 96.82 ± 0.10   Final old_old: 97.45 ± 0.12   Final old_new: 94.33 ± 0.16   Final new_new: 90.56 ± 0.91
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.59 ± 2.00   Final Test: 27.53 ± 6.32   Final old_old: 30.00 ± 6.84   Final old_new: 17.71 ± 4.32   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 55.52 ± 0.87   Final Test: 39.17 ± 2.54   Final old_old: 42.16 ± 2.71   Final old_new: 27.41 ± 2.03   Final new_new: 7.27 ± 3.37
Hits@30:
  Final val: 63.23 ± 1.59   Final Test: 46.74 ± 2.04   Final old_old: 50.19 ± 2.10   Final old_new: 33.22 ± 2.02   Final new_new: 9.55 ± 3.37
Hits@50:
  Final val: 72.39 ± 0.82   Final Test: 56.31 ± 0.97   Final old_old: 60.13 ± 0.95   Final old_new: 41.39 ± 1.59   Final new_new: 13.64 ± 3.21
AUC:
  Final val: 96.62 ± 0.19   Final Test: 95.96 ± 0.31   Final old_old: 96.63 ± 0.23   Final old_new: 93.31 ± 0.69   Final new_new: 89.22 ± 0.85
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.20 ± 4.11   Final Test: 28.72 ± 5.38   Final old_old: 31.28 ± 5.72   Final old_new: 18.44 ± 3.98   Final new_new: 5.00 ± 3.73
Hits@20:
  Final val: 55.43 ± 4.21   Final Test: 34.84 ± 3.13   Final old_old: 37.84 ± 3.32   Final old_new: 22.85 ± 2.60   Final new_new: 5.91 ± 2.59
Hits@30:
  Final val: 64.77 ± 3.15   Final Test: 44.44 ± 3.13   Final old_old: 47.84 ± 3.43   Final old_new: 31.01 ± 1.82   Final new_new: 9.09 ± 4.55
Hits@50:
  Final val: 73.15 ± 0.95   Final Test: 54.91 ± 2.53   Final old_old: 58.75 ± 2.63   Final old_new: 39.67 ± 2.55   Final new_new: 16.36 ± 4.37
AUC:
  Final val: 96.61 ± 0.21   Final Test: 95.97 ± 0.28   Final old_old: 96.65 ± 0.21   Final old_new: 93.31 ± 0.61   Final new_new: 88.72 ± 2.23
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.21 ± 3.65   Final Test: 33.32 ± 2.77   Final old_old: 36.64 ± 2.96   Final old_new: 19.85 ± 2.45   Final new_new: 5.00 ± 1.90
Hits@20:
  Final val: 59.89 ± 1.06   Final Test: 41.55 ± 3.04   Final old_old: 45.29 ± 3.28   Final old_new: 26.52 ± 2.65   Final new_new: 7.73 ± 3.80
Hits@30:
  Final val: 66.79 ± 1.37   Final Test: 48.12 ± 2.27   Final old_old: 52.15 ± 2.27   Final old_new: 31.99 ± 2.73   Final new_new: 9.55 ± 4.07
Hits@50:
  Final val: 74.21 ± 0.89   Final Test: 56.20 ± 0.25   Final old_old: 60.24 ± 0.32   Final old_new: 40.33 ± 2.08   Final new_new: 13.18 ± 2.96
AUC:
  Final val: 97.26 ± 0.11   Final Test: 96.75 ± 0.15   Final old_old: 97.34 ± 0.12   Final old_new: 94.43 ± 0.38   Final new_new: 90.17 ± 0.95
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.09 ± 1.70   Final Test: 29.49 ± 5.79   Final old_old: 32.07 ± 6.27   Final old_new: 19.17 ± 3.91   Final new_new: 5.00 ± 2.96
Hits@20:
  Final val: 58.03 ± 1.65   Final Test: 37.74 ± 2.63   Final old_old: 40.78 ± 2.97   Final old_new: 25.69 ± 1.52   Final new_new: 6.82 ± 5.79
Hits@30:
  Final val: 64.29 ± 1.33   Final Test: 47.15 ± 2.42   Final old_old: 50.49 ± 2.50   Final old_new: 33.95 ± 2.10   Final new_new: 12.73 ± 5.70
Hits@50:
  Final val: 72.80 ± 0.90   Final Test: 57.55 ± 4.69   Final old_old: 61.34 ± 4.98   Final old_new: 42.57 ± 3.66   Final new_new: 19.09 ± 7.29
AUC:
  Final val: 96.70 ± 0.23   Final Test: 96.00 ± 0.26   Final old_old: 96.70 ± 0.24   Final old_new: 93.30 ± 0.42   Final new_new: 88.37 ± 1.16
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.54 ± 3.95   Final Test: 33.98 ± 3.54   Final old_old: 37.27 ± 3.80   Final old_new: 20.76 ± 2.71   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 64.01 ± 2.20   Final Test: 43.62 ± 3.25   Final old_old: 47.49 ± 3.46   Final old_new: 28.16 ± 2.79   Final new_new: 6.36 ± 1.90
Hits@30:
  Final val: 69.82 ± 1.28   Final Test: 51.90 ± 2.26   Final old_old: 56.02 ± 2.14   Final old_new: 35.54 ± 2.82   Final new_new: 10.45 ± 4.71
Hits@50:
  Final val: 76.51 ± 1.29   Final Test: 61.16 ± 1.75   Final old_old: 65.46 ± 1.71   Final old_new: 44.21 ± 2.23   Final new_new: 15.45 ± 1.90
AUC:
  Final val: 97.41 ± 0.16   Final Test: 96.86 ± 0.13   Final old_old: 97.50 ± 0.14   Final old_new: 94.39 ± 0.25   Final new_new: 89.52 ± 0.64
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.54 ± 2.59   Final Test: 31.43 ± 5.93   Final old_old: 34.57 ± 6.33   Final old_new: 18.61 ± 4.34   Final new_new: 5.91 ± 3.80
Hits@20:
  Final val: 62.80 ± 2.36   Final Test: 44.38 ± 3.10   Final old_old: 48.25 ± 3.55   Final old_new: 28.69 ± 1.59   Final new_new: 10.91 ± 4.37
Hits@30:
  Final val: 69.57 ± 1.96   Final Test: 50.98 ± 5.09   Final old_old: 55.26 ± 5.31   Final old_new: 33.80 ± 4.44   Final new_new: 10.91 ± 2.49
Hits@50:
  Final val: 75.75 ± 1.06   Final Test: 62.18 ± 3.67   Final old_old: 66.48 ± 3.74   Final old_new: 44.91 ± 3.47   Final new_new: 21.82 ± 5.70
AUC:
  Final val: 97.41 ± 0.12   Final Test: 96.83 ± 0.16   Final old_old: 97.45 ± 0.19   Final old_new: 94.37 ± 0.18   Final new_new: 90.59 ± 1.52
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.60 ± 6.51   Final Test: 29.26 ± 7.45   Final old_old: 31.87 ± 7.86   Final old_new: 18.77 ± 6.07   Final new_new: 5.45 ± 1.24
Hits@20:
  Final val: 54.77 ± 6.36   Final Test: 39.38 ± 5.99   Final old_old: 42.64 ± 6.43   Final old_new: 26.32 ± 4.62   Final new_new: 8.64 ± 4.66
Hits@30:
  Final val: 63.10 ± 5.07   Final Test: 45.22 ± 4.00   Final old_old: 48.74 ± 4.35   Final old_new: 31.26 ± 2.87   Final new_new: 10.00 ± 5.23
Hits@50:
  Final val: 72.35 ± 2.96   Final Test: 56.73 ± 4.46   Final old_old: 60.47 ± 4.57   Final old_new: 42.02 ± 4.39   Final new_new: 17.27 ± 4.13
AUC:
  Final val: 96.62 ± 0.21   Final Test: 96.01 ± 0.20   Final old_old: 96.66 ± 0.19   Final old_new: 93.46 ± 0.36   Final new_new: 88.96 ± 0.90
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.16 ± 1.21   Final Test: 31.37 ± 8.22   Final old_old: 34.50 ± 8.80   Final old_new: 18.72 ± 6.06   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 65.01 ± 0.87   Final Test: 43.79 ± 4.61   Final old_old: 47.61 ± 4.80   Final old_new: 28.46 ± 4.00   Final new_new: 8.18 ± 2.03
Hits@30:
  Final val: 70.35 ± 1.12   Final Test: 50.97 ± 2.48   Final old_old: 55.10 ± 2.71   Final old_new: 34.56 ± 1.63   Final new_new: 9.55 ± 2.96
Hits@50:
  Final val: 77.15 ± 1.26   Final Test: 61.07 ± 3.61   Final old_old: 65.40 ± 3.70   Final old_new: 43.90 ± 3.73   Final new_new: 16.82 ± 3.05
AUC:
  Final val: 97.42 ± 0.07   Final Test: 96.85 ± 0.08   Final old_old: 97.47 ± 0.11   Final old_new: 94.38 ± 0.11   Final new_new: 90.34 ± 1.27
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.50 ± 2.93   Final Test: 31.43 ± 3.38   Final old_old: 34.61 ± 3.64   Final old_new: 18.56 ± 2.52   Final new_new: 4.09 ± 2.49
Hits@20:
  Final val: 59.21 ± 1.36   Final Test: 41.16 ± 3.81   Final old_old: 44.63 ± 3.91   Final old_new: 27.41 ± 3.84   Final new_new: 6.36 ± 3.73
Hits@30:
  Final val: 66.92 ± 1.00   Final Test: 49.39 ± 2.58   Final old_old: 53.22 ± 2.68   Final old_new: 34.18 ± 2.57   Final new_new: 10.91 ± 2.49
Hits@50:
  Final val: 74.42 ± 0.71   Final Test: 59.80 ± 1.61   Final old_old: 63.81 ± 1.79   Final old_new: 44.03 ± 1.25   Final new_new: 16.36 ± 4.37
AUC:
  Final val: 97.27 ± 0.14   Final Test: 96.81 ± 0.13   Final old_old: 97.37 ± 0.13   Final old_new: 94.61 ± 0.27   Final new_new: 90.45 ± 1.47
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.94 ± 1.25   Final Test: 30.47 ± 2.95   Final old_old: 33.58 ± 3.17   Final old_new: 17.91 ± 2.65   Final new_new: 3.18 ± 1.24
Hits@20:
  Final val: 58.80 ± 3.02   Final Test: 39.38 ± 2.67   Final old_old: 42.86 ± 2.80   Final old_new: 25.34 ± 2.47   Final new_new: 8.64 ± 2.96
Hits@30:
  Final val: 65.97 ± 1.76   Final Test: 46.85 ± 4.98   Final old_old: 50.64 ± 5.09   Final old_new: 31.74 ± 4.97   Final new_new: 10.00 ± 3.45
Hits@50:
  Final val: 73.64 ± 1.80   Final Test: 58.04 ± 0.85   Final old_old: 62.19 ± 0.78   Final old_new: 41.54 ± 2.10   Final new_new: 16.82 ± 3.80
AUC:
  Final val: 97.26 ± 0.16   Final Test: 96.71 ± 0.12   Final old_old: 97.29 ± 0.13   Final old_new: 94.47 ± 0.32   Final new_new: 90.50 ± 1.10
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.29 ± 2.96   Final Test: 33.04 ± 1.54   Final old_old: 36.31 ± 1.43   Final old_new: 19.87 ± 2.26   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 62.04 ± 3.02   Final Test: 42.50 ± 3.37   Final old_old: 46.24 ± 3.59   Final old_new: 27.56 ± 2.74   Final new_new: 6.36 ± 1.90
Hits@30:
  Final val: 68.43 ± 2.67   Final Test: 50.42 ± 0.86   Final old_old: 54.56 ± 0.61   Final old_new: 33.98 ± 2.07   Final new_new: 8.64 ± 2.96
Hits@50:
  Final val: 76.11 ± 1.67   Final Test: 60.28 ± 1.62   Final old_old: 64.65 ± 1.59   Final old_new: 43.10 ± 2.79   Final new_new: 14.09 ± 1.90
AUC:
  Final val: 97.33 ± 0.07   Final Test: 96.84 ± 0.11   Final old_old: 97.47 ± 0.13   Final old_new: 94.35 ± 0.23   Final new_new: 90.06 ± 1.08
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.16 ± 2.62   Final Test: 32.27 ± 3.68   Final old_old: 35.37 ± 3.89   Final old_new: 19.70 ± 2.95   Final new_new: 5.91 ± 4.43
Hits@20:
  Final val: 60.48 ± 1.48   Final Test: 40.10 ± 1.99   Final old_old: 43.82 ± 2.26   Final old_new: 25.14 ± 1.16   Final new_new: 6.36 ± 3.37
Hits@30:
  Final val: 66.82 ± 1.83   Final Test: 46.50 ± 2.97   Final old_old: 50.43 ± 3.26   Final old_new: 30.88 ± 2.10   Final new_new: 7.73 ± 2.03
Hits@50:
  Final val: 74.77 ± 1.22   Final Test: 59.13 ± 2.46   Final old_old: 63.43 ± 2.44   Final old_new: 42.14 ± 2.88   Final new_new: 14.55 ± 3.80
AUC:
  Final val: 97.28 ± 0.11   Final Test: 96.75 ± 0.03   Final old_old: 97.34 ± 0.06   Final old_new: 94.46 ± 0.18   Final new_new: 90.21 ± 1.51
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.83 ± 2.00   Final Test: 29.66 ± 4.11   Final old_old: 32.39 ± 4.47   Final old_new: 18.77 ± 2.79   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 55.48 ± 2.49   Final Test: 37.62 ± 2.42   Final old_old: 40.70 ± 2.58   Final old_new: 25.34 ± 2.15   Final new_new: 7.73 ± 2.59
Hits@30:
  Final val: 63.38 ± 3.54   Final Test: 44.31 ± 2.66   Final old_old: 47.65 ± 2.57   Final old_new: 31.21 ± 3.13   Final new_new: 7.73 ± 4.13
Hits@50:
  Final val: 71.83 ± 1.46   Final Test: 55.90 ± 3.30   Final old_old: 59.60 ± 3.45   Final old_new: 41.31 ± 2.97   Final new_new: 16.82 ± 6.74
AUC:
  Final val: 96.64 ± 0.21   Final Test: 96.04 ± 0.35   Final old_old: 96.68 ± 0.32   Final old_new: 93.47 ± 0.48   Final new_new: 89.54 ± 1.26
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.56 ± 4.71   Final Test: 29.29 ± 7.38   Final old_old: 31.80 ± 7.96   Final old_new: 19.35 ± 5.04   Final new_new: 3.64 ± 2.59
Hits@20:
  Final val: 58.09 ± 5.56   Final Test: 41.09 ± 3.36   Final old_old: 44.29 ± 3.39   Final old_new: 28.56 ± 3.47   Final new_new: 5.91 ± 2.59
Hits@30:
  Final val: 64.94 ± 5.15   Final Test: 46.68 ± 1.42   Final old_old: 50.29 ± 1.58   Final old_new: 32.44 ± 1.20   Final new_new: 8.18 ± 2.59
Hits@50:
  Final val: 72.82 ± 2.64   Final Test: 56.59 ± 2.54   Final old_old: 60.40 ± 2.58   Final old_new: 41.71 ± 2.52   Final new_new: 14.09 ± 3.73
AUC:
  Final val: 96.72 ± 0.17   Final Test: 96.06 ± 0.16   Final old_old: 96.69 ± 0.15   Final old_new: 93.56 ± 0.25   Final new_new: 89.37 ± 1.10
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.94 ± 2.72   Final Test: 32.09 ± 3.49   Final old_old: 35.32 ± 3.51   Final old_new: 19.02 ± 4.00   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 60.79 ± 0.80   Final Test: 40.91 ± 2.47   Final old_old: 44.60 ± 2.55   Final old_new: 26.22 ± 2.51   Final new_new: 5.00 ± 2.96
Hits@30:
  Final val: 67.56 ± 0.65   Final Test: 48.73 ± 3.38   Final old_old: 52.89 ± 3.41   Final old_new: 32.24 ± 3.81   Final new_new: 6.82 ± 1.61
Hits@50:
  Final val: 75.02 ± 0.63   Final Test: 57.59 ± 1.74   Final old_old: 62.00 ± 1.96   Final old_new: 40.13 ± 2.24   Final new_new: 11.82 ± 4.37
AUC:
  Final val: 97.30 ± 0.13   Final Test: 96.79 ± 0.09   Final old_old: 97.40 ± 0.12   Final old_new: 94.39 ± 0.32   Final new_new: 90.25 ± 0.85
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.83 ± 5.51   Final Test: 29.50 ± 5.21   Final old_old: 32.16 ± 5.46   Final old_new: 18.77 ± 4.49   Final new_new: 5.91 ± 2.59
Hits@20:
  Final val: 55.18 ± 3.59   Final Test: 39.79 ± 5.07   Final old_old: 43.11 ± 5.12   Final old_new: 26.55 ± 5.07   Final new_new: 8.18 ± 3.45
Hits@30:
  Final val: 63.74 ± 2.03   Final Test: 45.99 ± 3.07   Final old_old: 49.37 ± 2.90   Final old_new: 32.57 ± 4.19   Final new_new: 12.27 ± 5.23
Hits@50:
  Final val: 71.75 ± 2.73   Final Test: 55.11 ± 3.52   Final old_old: 58.67 ± 3.71   Final old_new: 41.21 ± 3.52   Final new_new: 15.00 ± 4.98
AUC:
  Final val: 96.65 ± 0.15   Final Test: 96.03 ± 0.24   Final old_old: 96.73 ± 0.21   Final old_new: 93.32 ± 0.40   Final new_new: 87.63 ± 1.44
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.08 ± 3.02   Final Test: 32.97 ± 2.93   Final old_old: 36.26 ± 3.39   Final old_new: 19.60 ± 1.55   Final new_new: 5.00 ± 1.90
Hits@20:
  Final val: 62.74 ± 2.38   Final Test: 43.87 ± 3.68   Final old_old: 47.68 ± 4.00   Final old_new: 28.59 ± 2.46   Final new_new: 9.09 ± 2.27
Hits@30:
  Final val: 69.10 ± 1.63   Final Test: 51.70 ± 3.31   Final old_old: 55.57 ± 3.63   Final old_new: 36.42 ± 2.13   Final new_new: 10.91 ± 2.49
Hits@50:
  Final val: 75.93 ± 1.28   Final Test: 61.16 ± 2.66   Final old_old: 65.47 ± 2.92   Final old_new: 44.16 ± 2.12   Final new_new: 16.82 ± 2.03
AUC:
  Final val: 97.36 ± 0.11   Final Test: 96.81 ± 0.12   Final old_old: 97.41 ± 0.12   Final old_new: 94.43 ± 0.26   Final new_new: 90.11 ± 1.21
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.18 ± 4.92   Final Test: 31.95 ± 2.29   Final old_old: 35.12 ± 2.49   Final old_new: 19.17 ± 1.67   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 62.87 ± 2.18   Final Test: 43.21 ± 2.60   Final old_old: 46.96 ± 3.01   Final old_new: 28.21 ± 1.41   Final new_new: 7.73 ± 4.13
Hits@30:
  Final val: 68.91 ± 1.92   Final Test: 50.84 ± 0.79   Final old_old: 55.06 ± 0.54   Final old_new: 34.01 ± 2.17   Final new_new: 10.00 ± 3.05
Hits@50:
  Final val: 75.92 ± 1.57   Final Test: 61.39 ± 1.52   Final old_old: 65.57 ± 1.73   Final old_new: 44.84 ± 1.04   Final new_new: 19.09 ± 5.23
AUC:
  Final val: 97.32 ± 0.13   Final Test: 96.78 ± 0.06   Final old_old: 97.39 ± 0.07   Final old_new: 94.36 ± 0.17   Final new_new: 90.71 ± 0.57
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.48 ± 4.75   Final Test: 30.77 ± 5.67   Final old_old: 33.33 ± 6.00   Final old_new: 20.58 ± 4.70   Final new_new: 6.36 ± 5.18
Hits@20:
  Final val: 54.24 ± 4.94   Final Test: 37.56 ± 5.11   Final old_old: 40.61 ± 5.53   Final old_new: 25.42 ± 3.69   Final new_new: 7.73 ± 5.93
Hits@30:
  Final val: 62.73 ± 3.44   Final Test: 44.84 ± 1.93   Final old_old: 48.37 ± 1.93   Final old_new: 30.76 ± 2.01   Final new_new: 10.91 ± 6.51
Hits@50:
  Final val: 71.59 ± 2.16   Final Test: 56.05 ± 3.00   Final old_old: 59.68 ± 2.90   Final old_new: 41.91 ± 3.50   Final new_new: 14.09 ± 7.77
AUC:
  Final val: 96.65 ± 0.19   Final Test: 96.11 ± 0.29   Final old_old: 96.76 ± 0.25   Final old_new: 93.56 ± 0.49   Final new_new: 89.17 ± 0.99
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.64 ± 3.41   Final Test: 32.94 ± 6.52   Final old_old: 36.01 ± 6.89   Final old_new: 20.58 ± 5.19   Final new_new: 5.45 ± 2.59
Hits@20:
  Final val: 61.48 ± 5.26   Final Test: 41.85 ± 3.68   Final old_old: 45.44 ± 3.78   Final old_new: 27.46 ± 3.51   Final new_new: 7.73 ± 1.24
Hits@30:
  Final val: 68.28 ± 5.08   Final Test: 48.86 ± 3.71   Final old_old: 52.68 ± 3.89   Final old_new: 33.68 ± 3.11   Final new_new: 10.91 ± 4.07
Hits@50:
  Final val: 75.21 ± 2.91   Final Test: 58.65 ± 3.47   Final old_old: 62.80 ± 3.53   Final old_new: 42.19 ± 3.23   Final new_new: 16.82 ± 5.23
AUC:
  Final val: 97.31 ± 0.12   Final Test: 96.72 ± 0.19   Final old_old: 97.33 ± 0.18   Final old_new: 94.34 ± 0.25   Final new_new: 90.00 ± 0.95
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.88 ± 3.23   Final Test: 32.12 ± 2.01   Final old_old: 35.07 ± 2.04   Final old_new: 20.20 ± 1.79   Final new_new: 6.82 ± 5.57
Hits@20:
  Final val: 60.29 ± 1.26   Final Test: 41.97 ± 4.33   Final old_old: 45.44 ± 4.55   Final old_new: 28.04 ± 3.71   Final new_new: 10.00 ± 2.59
Hits@30:
  Final val: 66.97 ± 1.72   Final Test: 49.59 ± 2.60   Final old_old: 53.68 ± 2.80   Final old_new: 33.27 ± 2.47   Final new_new: 10.45 ± 4.13
Hits@50:
  Final val: 74.18 ± 1.56   Final Test: 58.27 ± 3.41   Final old_old: 62.50 ± 3.42   Final old_new: 41.49 ± 3.79   Final new_new: 15.91 ± 3.94
AUC:
  Final val: 97.27 ± 0.14   Final Test: 96.78 ± 0.10   Final old_old: 97.38 ± 0.16   Final old_new: 94.43 ± 0.19   Final new_new: 89.97 ± 1.05
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.57 ± 2.59   Final Test: 33.06 ± 5.60   Final old_old: 36.34 ± 5.95   Final old_new: 19.77 ± 4.29   Final new_new: 5.45 ± 2.03
Hits@20:
  Final val: 62.05 ± 3.37   Final Test: 42.40 ± 3.01   Final old_old: 46.05 ± 3.20   Final old_new: 27.83 ± 2.90   Final new_new: 7.27 ± 2.96
Hits@30:
  Final val: 68.11 ± 2.38   Final Test: 50.16 ± 2.31   Final old_old: 54.04 ± 2.37   Final old_new: 34.91 ± 2.24   Final new_new: 9.09 ± 4.55
Hits@50:
  Final val: 75.26 ± 1.45   Final Test: 60.61 ± 2.02   Final old_old: 64.76 ± 1.96   Final old_new: 44.28 ± 2.43   Final new_new: 16.82 ± 4.43
AUC:
  Final val: 97.33 ± 0.09   Final Test: 96.83 ± 0.14   Final old_old: 97.45 ± 0.13   Final old_new: 94.36 ± 0.22   Final new_new: 90.54 ± 0.94
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.73 ± 1.90   Final Test: 31.52 ± 4.47   Final old_old: 34.65 ± 4.80   Final old_new: 18.84 ± 3.37   Final new_new: 5.00 ± 4.07
Hits@20:
  Final val: 59.66 ± 1.90   Final Test: 41.52 ± 3.22   Final old_old: 45.34 ± 3.62   Final old_new: 26.22 ± 1.82   Final new_new: 5.91 ± 2.03
Hits@30:
  Final val: 67.13 ± 1.10   Final Test: 48.75 ± 0.99   Final old_old: 52.83 ± 1.09   Final old_new: 32.52 ± 1.72   Final new_new: 8.18 ± 3.45
Hits@50:
  Final val: 74.66 ± 1.11   Final Test: 57.72 ± 2.37   Final old_old: 62.19 ± 2.61   Final old_new: 40.03 ± 2.48   Final new_new: 11.82 ± 1.90
AUC:
  Final val: 97.30 ± 0.11   Final Test: 96.79 ± 0.13   Final old_old: 97.39 ± 0.18   Final old_new: 94.42 ± 0.26   Final new_new: 90.52 ± 1.86
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.62 ± 3.02   Final Test: 33.37 ± 3.63   Final old_old: 36.64 ± 4.05   Final old_new: 20.08 ± 2.16   Final new_new: 6.36 ± 2.96
Hits@20:
  Final val: 60.51 ± 0.84   Final Test: 40.36 ± 3.02   Final old_old: 43.89 ± 3.36   Final old_new: 26.17 ± 1.88   Final new_new: 8.18 ± 2.03
Hits@30:
  Final val: 66.95 ± 0.72   Final Test: 48.62 ± 3.64   Final old_old: 52.56 ± 3.96   Final old_new: 32.87 ± 2.47   Final new_new: 10.91 ± 2.49
Hits@50:
  Final val: 74.72 ± 0.98   Final Test: 58.28 ± 1.10   Final old_old: 62.56 ± 1.00   Final old_new: 41.41 ± 1.84   Final new_new: 13.18 ± 1.90
AUC:
  Final val: 97.29 ± 0.07   Final Test: 96.73 ± 0.11   Final old_old: 97.34 ± 0.09   Final old_new: 94.31 ± 0.32   Final new_new: 90.37 ± 1.10
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.87 ± 4.07   Final Test: 28.83 ± 6.02   Final old_old: 31.44 ± 6.49   Final old_new: 18.36 ± 4.46   Final new_new: 4.55 ± 3.21
Hits@20:
  Final val: 55.16 ± 4.43   Final Test: 39.59 ± 7.21   Final old_old: 42.95 ± 7.72   Final old_new: 26.10 ± 5.39   Final new_new: 8.64 ± 5.18
Hits@30:
  Final val: 63.00 ± 2.97   Final Test: 45.38 ± 2.68   Final old_old: 48.93 ± 2.83   Final old_new: 31.26 ± 2.34   Final new_new: 10.00 ± 3.45
Hits@50:
  Final val: 72.40 ± 1.73   Final Test: 54.87 ± 2.52   Final old_old: 58.36 ± 2.69   Final old_new: 41.16 ± 2.37   Final new_new: 17.27 ± 6.35
AUC:
  Final val: 96.68 ± 0.22   Final Test: 96.07 ± 0.25   Final old_old: 96.74 ± 0.30   Final old_new: 93.44 ± 0.33   Final new_new: 89.09 ± 2.05
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.38 ± 7.06   Final Test: 28.13 ± 4.48   Final old_old: 30.75 ± 4.88   Final old_new: 17.56 ± 3.70   Final new_new: 5.00 ± 2.49
Hits@20:
  Final val: 52.89 ± 4.52   Final Test: 37.45 ± 7.16   Final old_old: 40.48 ± 7.68   Final old_new: 25.24 ± 5.14   Final new_new: 9.55 ± 2.96
Hits@30:
  Final val: 62.30 ± 3.57   Final Test: 44.08 ± 5.04   Final old_old: 47.45 ± 5.39   Final old_new: 30.76 ± 3.83   Final new_new: 10.00 ± 6.74
Hits@50:
  Final val: 71.37 ± 1.79   Final Test: 57.50 ± 3.02   Final old_old: 61.27 ± 2.99   Final old_new: 42.67 ± 3.40   Final new_new: 16.82 ± 2.59
AUC:
  Final val: 96.61 ± 0.15   Final Test: 95.98 ± 0.12   Final old_old: 96.64 ± 0.08   Final old_new: 93.39 ± 0.49   Final new_new: 88.95 ± 1.25
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.37 ± 3.03   Final Test: 31.51 ± 3.09   Final old_old: 34.76 ± 3.31   Final old_new: 18.31 ± 2.35   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 60.01 ± 1.17   Final Test: 40.91 ± 3.40   Final old_old: 44.58 ± 3.40   Final old_new: 26.22 ± 3.64   Final new_new: 6.36 ± 1.90
Hits@30:
  Final val: 66.92 ± 0.55   Final Test: 49.42 ± 2.17   Final old_old: 53.40 ± 2.12   Final old_new: 33.58 ± 2.88   Final new_new: 10.00 ± 2.59
Hits@50:
  Final val: 73.97 ± 0.53   Final Test: 59.82 ± 1.43   Final old_old: 64.12 ± 1.26   Final old_new: 42.97 ± 2.79   Final new_new: 12.73 ± 2.59
AUC:
  Final val: 97.27 ± 0.09   Final Test: 96.75 ± 0.16   Final old_old: 97.35 ± 0.14   Final old_new: 94.40 ± 0.40   Final new_new: 90.07 ± 0.61
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.71 ± 4.68   Final Test: 28.54 ± 4.96   Final old_old: 31.08 ± 5.37   Final old_new: 18.19 ± 3.48   Final new_new: 7.27 ± 1.90
Hits@20:
  Final val: 56.63 ± 2.89   Final Test: 36.99 ± 4.27   Final old_old: 40.04 ± 4.57   Final old_new: 24.76 ± 3.17   Final new_new: 8.64 ± 2.96
Hits@30:
  Final val: 64.18 ± 2.82   Final Test: 45.89 ± 2.99   Final old_old: 49.33 ± 3.06   Final old_new: 32.19 ± 2.77   Final new_new: 12.73 ± 4.13
Hits@50:
  Final val: 72.47 ± 1.42   Final Test: 55.66 ± 2.52   Final old_old: 59.20 ± 2.61   Final old_new: 41.64 ± 2.48   Final new_new: 19.55 ± 2.59
AUC:
  Final val: 96.63 ± 0.18   Final Test: 95.99 ± 0.21   Final old_old: 96.64 ± 0.18   Final old_new: 93.42 ± 0.45   Final new_new: 89.23 ± 0.48
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.15 ± 3.54   Final Test: 31.46 ± 2.91   Final old_old: 34.67 ± 3.06   Final old_new: 18.46 ± 2.61   Final new_new: 3.64 ± 2.03
Hits@20:
  Final val: 62.50 ± 2.11   Final Test: 43.81 ± 1.98   Final old_old: 47.74 ± 1.91   Final old_new: 28.11 ± 2.63   Final new_new: 6.36 ± 1.90
Hits@30:
  Final val: 69.57 ± 2.10   Final Test: 51.41 ± 2.53   Final old_old: 55.51 ± 2.79   Final old_new: 35.31 ± 2.26   Final new_new: 6.82 ± 1.61
Hits@50:
  Final val: 77.13 ± 1.14   Final Test: 62.40 ± 1.79   Final old_old: 66.74 ± 2.09   Final old_new: 45.49 ± 1.15   Final new_new: 13.18 ± 2.96
AUC:
  Final val: 97.39 ± 0.11   Final Test: 96.89 ± 0.10   Final old_old: 97.53 ± 0.12   Final old_new: 94.34 ± 0.26   Final new_new: 90.89 ± 0.68
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.72 ± 6.14   Final Test: 37.66 ± 6.48   Final old_old: 41.30 ± 7.08   Final old_new: 22.90 ± 4.05   Final new_new: 6.82 ± 4.55
Hits@20:
  Final val: 63.40 ± 2.21   Final Test: 46.98 ± 4.70   Final old_old: 50.93 ± 4.98   Final old_new: 31.16 ± 3.61   Final new_new: 10.00 ± 4.71
Hits@30:
  Final val: 70.42 ± 1.43   Final Test: 53.07 ± 4.61   Final old_old: 57.24 ± 5.06   Final old_new: 36.42 ± 3.12   Final new_new: 12.27 ± 3.45
Hits@50:
  Final val: 76.88 ± 1.59   Final Test: 63.84 ± 2.82   Final old_old: 68.00 ± 2.98   Final old_new: 47.41 ± 2.92   Final new_new: 20.45 ± 2.78
AUC:
  Final val: 97.39 ± 0.16   Final Test: 96.91 ± 0.11   Final old_old: 97.51 ± 0.13   Final old_new: 94.55 ± 0.13   Final new_new: 90.30 ± 1.42
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.47 ± 4.85   Final Test: 30.44 ± 2.97   Final old_old: 33.13 ± 3.00   Final old_new: 19.50 ± 2.95   Final new_new: 8.18 ± 2.59
Hits@20:
  Final val: 57.72 ± 3.08   Final Test: 39.51 ± 0.78   Final old_old: 42.68 ± 0.95   Final old_new: 26.85 ± 1.07   Final new_new: 9.55 ± 2.96
Hits@30:
  Final val: 65.61 ± 1.45   Final Test: 46.79 ± 2.76   Final old_old: 50.35 ± 2.72   Final old_new: 32.67 ± 3.37   Final new_new: 10.45 ± 2.59
Hits@50:
  Final val: 73.76 ± 0.67   Final Test: 54.72 ± 1.35   Final old_old: 58.50 ± 1.56   Final old_new: 39.87 ± 1.39   Final new_new: 14.55 ± 4.13
AUC:
  Final val: 96.64 ± 0.14   Final Test: 96.05 ± 0.22   Final old_old: 96.72 ± 0.21   Final old_new: 93.43 ± 0.47   Final new_new: 89.03 ± 1.13
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.20 ± 5.03   Final Test: 29.96 ± 4.74   Final old_old: 33.00 ± 5.20   Final old_new: 17.63 ± 2.88   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 62.87 ± 2.35   Final Test: 39.58 ± 4.38   Final old_old: 43.25 ± 4.74   Final old_new: 24.81 ± 3.09   Final new_new: 6.82 ± 2.27
Hits@30:
  Final val: 68.74 ± 1.06   Final Test: 47.78 ± 2.87   Final old_old: 51.71 ± 3.15   Final old_new: 32.12 ± 2.09   Final new_new: 9.09 ± 3.21
Hits@50:
  Final val: 75.80 ± 1.45   Final Test: 61.66 ± 2.00   Final old_old: 65.89 ± 2.23   Final old_new: 44.99 ± 1.29   Final new_new: 16.82 ± 2.59
AUC:
  Final val: 97.36 ± 0.15   Final Test: 96.80 ± 0.10   Final old_old: 97.42 ± 0.11   Final old_new: 94.34 ± 0.25   Final new_new: 90.47 ± 0.91
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.84 ± 4.20   Final Test: 30.37 ± 2.64   Final old_old: 33.54 ± 2.95   Final old_new: 17.56 ± 1.67   Final new_new: 2.73 ± 1.02
Hits@20:
  Final val: 60.83 ± 1.82   Final Test: 40.65 ± 4.01   Final old_old: 44.31 ± 4.26   Final old_new: 25.94 ± 3.16   Final new_new: 6.82 ± 2.27
Hits@30:
  Final val: 67.64 ± 1.44   Final Test: 47.76 ± 3.41   Final old_old: 51.85 ± 3.66   Final old_new: 31.54 ± 2.69   Final new_new: 6.82 ± 4.25
Hits@50:
  Final val: 75.17 ± 1.20   Final Test: 59.38 ± 1.28   Final old_old: 63.73 ± 1.34   Final old_new: 42.37 ± 1.73   Final new_new: 11.36 ± 4.25
AUC:
  Final val: 97.33 ± 0.06   Final Test: 96.81 ± 0.08   Final old_old: 97.42 ± 0.07   Final old_new: 94.39 ± 0.22   Final new_new: 90.26 ± 1.35
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.50 ± 3.89   Final Test: 32.80 ± 3.70   Final old_old: 36.03 ± 3.86   Final old_new: 19.77 ± 3.35   Final new_new: 4.55 ± 3.21
Hits@20:
  Final val: 61.28 ± 1.44   Final Test: 40.50 ± 5.27   Final old_old: 43.87 ± 5.77   Final old_new: 27.05 ± 3.45   Final new_new: 7.27 ± 4.07
Hits@30:
  Final val: 67.05 ± 1.10   Final Test: 50.92 ± 2.59   Final old_old: 54.85 ± 2.93   Final old_new: 35.49 ± 1.78   Final new_new: 9.09 ± 1.61
Hits@50:
  Final val: 74.59 ± 0.74   Final Test: 59.44 ± 1.81   Final old_old: 63.79 ± 1.92   Final old_new: 42.29 ± 1.68   Final new_new: 13.18 ± 1.90
AUC:
  Final val: 97.28 ± 0.08   Final Test: 96.79 ± 0.08   Final old_old: 97.38 ± 0.13   Final old_new: 94.48 ± 0.26   Final new_new: 90.36 ± 0.59
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.64 ± 4.12   Final Test: 29.44 ± 7.01   Final old_old: 32.48 ± 7.33   Final old_new: 17.10 ± 5.88   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 63.15 ± 1.67   Final Test: 41.33 ± 4.05   Final old_old: 45.05 ± 4.15   Final old_new: 26.37 ± 3.73   Final new_new: 6.82 ± 2.78
Hits@30:
  Final val: 68.92 ± 1.95   Final Test: 49.08 ± 2.35   Final old_old: 52.92 ± 2.34   Final old_new: 33.85 ± 2.47   Final new_new: 10.45 ± 5.47
Hits@50:
  Final val: 76.49 ± 1.31   Final Test: 60.31 ± 1.35   Final old_old: 64.47 ± 1.42   Final old_new: 43.90 ± 1.45   Final new_new: 16.36 ± 2.49
AUC:
  Final val: 97.40 ± 0.14   Final Test: 96.82 ± 0.11   Final old_old: 97.44 ± 0.12   Final old_new: 94.36 ± 0.17   Final new_new: 90.58 ± 0.81
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.62 ± 5.74   Final Test: 32.14 ± 5.71   Final old_old: 35.37 ± 5.90   Final old_new: 19.04 ± 5.35   Final new_new: 5.00 ± 1.90
Hits@20:
  Final val: 61.25 ± 3.36   Final Test: 40.72 ± 3.26   Final old_old: 44.39 ± 3.62   Final old_new: 26.02 ± 1.98   Final new_new: 6.36 ± 1.02
Hits@30:
  Final val: 66.72 ± 2.44   Final Test: 50.46 ± 1.53   Final old_old: 54.34 ± 1.95   Final old_new: 35.01 ± 1.31   Final new_new: 12.27 ± 2.59
Hits@50:
  Final val: 74.20 ± 1.76   Final Test: 59.29 ± 3.27   Final old_old: 63.68 ± 3.41   Final old_new: 41.89 ± 2.75   Final new_new: 15.00 ± 8.59
AUC:
  Final val: 97.24 ± 0.16   Final Test: 96.72 ± 0.11   Final old_old: 97.33 ± 0.13   Final old_new: 94.33 ± 0.27   Final new_new: 90.85 ± 1.52
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.47 ± 0.84   Final Test: 26.43 ± 7.25   Final old_old: 28.96 ± 7.82   Final old_new: 16.17 ± 5.12   Final new_new: 5.45 ± 3.80
Hits@20:
  Final val: 53.66 ± 2.26   Final Test: 35.92 ± 5.28   Final old_old: 39.00 ± 5.57   Final old_new: 23.53 ± 4.38   Final new_new: 7.73 ± 3.45
Hits@30:
  Final val: 61.80 ± 2.32   Final Test: 44.61 ± 4.55   Final old_old: 47.97 ± 4.83   Final old_new: 31.34 ± 3.72   Final new_new: 10.00 ± 6.93
Hits@50:
  Final val: 71.79 ± 1.90   Final Test: 53.90 ± 4.10   Final old_old: 57.72 ± 4.15   Final old_new: 38.74 ± 4.08   Final new_new: 15.00 ± 6.55
AUC:
  Final val: 96.66 ± 0.12   Final Test: 95.99 ± 0.23   Final old_old: 96.65 ± 0.20   Final old_new: 93.36 ± 0.41   Final new_new: 89.58 ± 1.45
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 40.59 ± 3.21   Final Test: 24.50 ± 2.36   Final old_old: 26.62 ± 2.66   Final old_new: 15.94 ± 1.61   Final new_new: 6.36 ± 4.37
Hits@20:
  Final val: 53.94 ± 3.28   Final Test: 36.59 ± 4.08   Final old_old: 39.37 ± 4.18   Final old_new: 25.42 ± 4.26   Final new_new: 11.36 ± 3.59
Hits@30:
  Final val: 61.86 ± 3.80   Final Test: 42.93 ± 2.92   Final old_old: 46.14 ± 2.98   Final old_new: 30.20 ± 2.93   Final new_new: 10.45 ± 3.45
Hits@50:
  Final val: 70.96 ± 2.44   Final Test: 56.34 ± 2.95   Final old_old: 59.86 ± 2.94   Final old_new: 42.42 ± 3.37   Final new_new: 20.45 ± 5.79
AUC:
  Final val: 96.63 ± 0.18   Final Test: 95.97 ± 0.19   Final old_old: 96.62 ± 0.17   Final old_new: 93.38 ± 0.39   Final new_new: 89.86 ± 1.81
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.24 ± 3.94   Final Test: 29.74 ± 2.90   Final old_old: 32.65 ± 3.16   Final old_new: 18.01 ± 1.96   Final new_new: 3.64 ± 2.03
Hits@20:
  Final val: 58.81 ± 2.45   Final Test: 39.91 ± 2.55   Final old_old: 43.42 ± 2.67   Final old_new: 25.87 ± 2.38   Final new_new: 6.36 ± 2.96
Hits@30:
  Final val: 66.17 ± 1.08   Final Test: 47.02 ± 2.52   Final old_old: 50.91 ± 2.60   Final old_new: 31.61 ± 2.31   Final new_new: 7.27 ± 2.96
Hits@50:
  Final val: 74.57 ± 0.51   Final Test: 59.48 ± 4.30   Final old_old: 63.82 ± 4.21   Final old_new: 42.42 ± 4.93   Final new_new: 12.27 ± 5.93
AUC:
  Final val: 97.21 ± 0.09   Final Test: 96.73 ± 0.11   Final old_old: 97.33 ± 0.08   Final old_new: 94.37 ± 0.30   Final new_new: 90.43 ± 0.32
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.03 ± 2.06   Final Test: 29.76 ± 6.95   Final old_old: 32.51 ± 7.58   Final old_new: 18.64 ± 4.43   Final new_new: 5.91 ± 3.45
Hits@20:
  Final val: 55.27 ± 1.93   Final Test: 38.74 ± 4.01   Final old_old: 42.01 ± 4.35   Final old_new: 25.69 ± 2.79   Final new_new: 7.27 ± 3.73
Hits@30:
  Final val: 64.22 ± 1.69   Final Test: 46.96 ± 2.51   Final old_old: 50.53 ± 2.56   Final old_new: 32.90 ± 3.03   Final new_new: 9.55 ± 3.73
Hits@50:
  Final val: 72.99 ± 1.30   Final Test: 57.69 ± 4.34   Final old_old: 61.76 ± 4.45   Final old_new: 41.74 ± 4.29   Final new_new: 13.18 ± 3.73
AUC:
  Final val: 96.68 ± 0.19   Final Test: 95.99 ± 0.27   Final old_old: 96.62 ± 0.25   Final old_new: 93.51 ± 0.44   Final new_new: 89.89 ± 1.36
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.73 ± 3.31   Final Test: 34.32 ± 5.49   Final old_old: 37.58 ± 5.82   Final old_new: 21.34 ± 4.43   Final new_new: 2.73 ± 1.02
Hits@20:
  Final val: 63.57 ± 1.45   Final Test: 44.41 ± 4.59   Final old_old: 48.37 ± 4.78   Final old_new: 28.56 ± 4.52   Final new_new: 7.27 ± 1.90
Hits@30:
  Final val: 69.85 ± 2.19   Final Test: 49.26 ± 2.24   Final old_old: 53.33 ± 2.63   Final old_new: 33.07 ± 1.55   Final new_new: 8.64 ± 2.96
Hits@50:
  Final val: 77.31 ± 2.15   Final Test: 62.71 ± 4.07   Final old_old: 67.00 ± 4.22   Final old_new: 45.74 ± 3.85   Final new_new: 18.64 ± 5.18
AUC:
  Final val: 97.42 ± 0.18   Final Test: 96.84 ± 0.12   Final old_old: 97.44 ± 0.15   Final old_new: 94.45 ± 0.14   Final new_new: 90.46 ± 0.66
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.53 ± 4.78   Final Test: 31.85 ± 5.48   Final old_old: 35.14 ± 5.80   Final old_new: 18.51 ± 4.33   Final new_new: 3.64 ± 2.59
Hits@20:
  Final val: 62.62 ± 5.67   Final Test: 42.04 ± 7.50   Final old_old: 45.73 ± 7.94   Final old_new: 27.33 ± 5.93   Final new_new: 6.36 ± 2.96
Hits@30:
  Final val: 68.65 ± 4.31   Final Test: 49.33 ± 6.74   Final old_old: 53.54 ± 7.17   Final old_new: 32.64 ± 5.21   Final new_new: 7.27 ± 2.96
Hits@50:
  Final val: 76.67 ± 2.52   Final Test: 60.70 ± 4.79   Final old_old: 64.80 ± 5.04   Final old_new: 44.74 ± 4.22   Final new_new: 13.18 ± 2.49
AUC:
  Final val: 97.38 ± 0.18   Final Test: 96.88 ± 0.24   Final old_old: 97.48 ± 0.27   Final old_new: 94.50 ± 0.24   Final new_new: 90.72 ± 0.69
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.35 ± 4.45   Final Test: 29.24 ± 1.59   Final old_old: 31.97 ± 1.69   Final old_new: 18.16 ± 1.41   Final new_new: 5.91 ± 3.45
Hits@20:
  Final val: 56.84 ± 4.05   Final Test: 39.63 ± 2.37   Final old_old: 42.93 ± 2.43   Final old_new: 26.42 ± 2.18   Final new_new: 8.64 ± 4.07
Hits@30:
  Final val: 64.17 ± 2.49   Final Test: 46.52 ± 2.73   Final old_old: 50.02 ± 2.59   Final old_new: 32.67 ± 3.58   Final new_new: 10.91 ± 3.37
Hits@50:
  Final val: 72.36 ± 2.51   Final Test: 56.02 ± 0.96   Final old_old: 59.66 ± 0.94   Final old_new: 41.71 ± 1.41   Final new_new: 17.27 ± 5.93
AUC:
  Final val: 96.68 ± 0.14   Final Test: 96.03 ± 0.21   Final old_old: 96.70 ± 0.15   Final old_new: 93.37 ± 0.51   Final new_new: 88.91 ± 1.52
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.61 ± 2.98   Final Test: 34.47 ± 3.70   Final old_old: 37.80 ± 3.96   Final old_new: 21.13 ± 2.86   Final new_new: 3.64 ± 2.03
Hits@20:
  Final val: 63.58 ± 2.28   Final Test: 46.18 ± 4.22   Final old_old: 50.22 ± 4.66   Final old_new: 30.05 ± 2.47   Final new_new: 7.73 ± 2.59
Hits@30:
  Final val: 70.27 ± 2.22   Final Test: 53.94 ± 3.22   Final old_old: 58.22 ± 3.32   Final old_new: 36.93 ± 2.92   Final new_new: 11.36 ± 3.21
Hits@50:
  Final val: 77.38 ± 1.64   Final Test: 62.84 ± 2.69   Final old_old: 67.02 ± 2.72   Final old_new: 46.40 ± 2.86   Final new_new: 18.18 ± 3.59
AUC:
  Final val: 97.41 ± 0.12   Final Test: 96.90 ± 0.15   Final old_old: 97.52 ± 0.16   Final old_new: 94.48 ± 0.19   Final new_new: 90.37 ± 0.73
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.71 ± 4.31   Final Test: 32.04 ± 4.65   Final old_old: 35.13 ± 4.93   Final old_new: 19.57 ± 3.86   Final new_new: 5.00 ± 1.90
Hits@20:
  Final val: 60.17 ± 4.69   Final Test: 40.75 ± 4.04   Final old_old: 44.12 ± 4.43   Final old_new: 27.33 ± 2.66   Final new_new: 7.73 ± 3.05
Hits@30:
  Final val: 66.86 ± 3.17   Final Test: 45.39 ± 5.71   Final old_old: 49.12 ± 6.17   Final old_new: 30.55 ± 4.08   Final new_new: 8.64 ± 3.73
Hits@50:
  Final val: 73.70 ± 1.26   Final Test: 59.46 ± 3.22   Final old_old: 63.52 ± 3.29   Final old_new: 43.35 ± 3.27   Final new_new: 18.18 ± 4.55
AUC:
  Final val: 97.21 ± 0.07   Final Test: 96.71 ± 0.10   Final old_old: 97.29 ± 0.10   Final old_new: 94.35 ± 0.21   Final new_new: 91.06 ± 0.46
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.71 ± 2.10   Final Test: 31.18 ± 2.29   Final old_old: 34.40 ± 2.41   Final old_new: 18.24 ± 2.04   Final new_new: 2.27 ± 1.61
Hits@20:
  Final val: 60.84 ± 2.70   Final Test: 41.95 ± 3.01   Final old_old: 45.63 ± 3.29   Final old_new: 27.36 ± 1.98   Final new_new: 4.55 ± 2.78
Hits@30:
  Final val: 67.42 ± 2.71   Final Test: 49.73 ± 1.78   Final old_old: 53.82 ± 1.89   Final old_new: 33.55 ± 1.48   Final new_new: 7.27 ± 3.37
Hits@50:
  Final val: 74.62 ± 1.49   Final Test: 59.18 ± 2.10   Final old_old: 63.54 ± 2.38   Final old_new: 42.27 ± 1.09   Final new_new: 8.64 ± 2.96
AUC:
  Final val: 97.33 ± 0.07   Final Test: 96.87 ± 0.07   Final old_old: 97.47 ± 0.09   Final old_new: 94.49 ± 0.23   Final new_new: 90.48 ± 0.68
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.18 ± 3.63   Final Test: 31.42 ± 7.04   Final old_old: 34.65 ± 7.43   Final old_new: 18.34 ± 5.94   Final new_new: 4.09 ± 2.49
Hits@20:
  Final val: 62.27 ± 2.03   Final Test: 42.59 ± 1.20   Final old_old: 46.37 ± 1.07   Final old_new: 27.38 ± 2.31   Final new_new: 8.18 ± 2.03
Hits@30:
  Final val: 69.55 ± 1.02   Final Test: 51.07 ± 2.09   Final old_old: 55.25 ± 2.33   Final old_new: 34.38 ± 1.40   Final new_new: 10.45 ± 3.45
Hits@50:
  Final val: 76.14 ± 0.81   Final Test: 62.00 ± 1.08   Final old_old: 66.19 ± 1.20   Final old_new: 45.52 ± 1.37   Final new_new: 17.73 ± 4.93
AUC:
  Final val: 97.41 ± 0.09   Final Test: 96.82 ± 0.05   Final old_old: 97.45 ± 0.11   Final old_new: 94.28 ± 0.26   Final new_new: 90.46 ± 1.30
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 40.61 ± 4.38   Final Test: 26.71 ± 5.08   Final old_old: 29.04 ± 5.62   Final old_new: 17.43 ± 3.35   Final new_new: 4.09 ± 2.96
Hits@20:
  Final val: 52.81 ± 3.51   Final Test: 34.80 ± 1.83   Final old_old: 37.63 ± 2.15   Final old_new: 23.40 ± 1.35   Final new_new: 9.09 ± 2.78
Hits@30:
  Final val: 61.25 ± 2.85   Final Test: 44.58 ± 2.92   Final old_old: 48.00 ± 2.93   Final old_new: 31.11 ± 3.04   Final new_new: 8.64 ± 4.93
Hits@50:
  Final val: 70.52 ± 2.59   Final Test: 54.65 ± 1.53   Final old_old: 58.07 ± 1.60   Final old_new: 41.28 ± 1.93   Final new_new: 16.36 ± 5.88
AUC:
  Final val: 96.61 ± 0.14   Final Test: 95.92 ± 0.24   Final old_old: 96.62 ± 0.23   Final old_new: 93.16 ± 0.40   Final new_new: 88.70 ± 0.64
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.33 ± 1.79   Final Test: 29.53 ± 6.27   Final old_old: 32.56 ± 6.65   Final old_new: 17.10 ± 4.93   Final new_new: 5.91 ± 2.59
Hits@20:
  Final val: 58.82 ± 1.90   Final Test: 40.65 ± 2.57   Final old_old: 44.16 ± 2.71   Final old_new: 26.57 ± 2.42   Final new_new: 7.73 ± 3.80
Hits@30:
  Final val: 66.03 ± 1.22   Final Test: 49.34 ± 3.18   Final old_old: 53.36 ± 3.29   Final old_new: 33.35 ± 2.85   Final new_new: 10.00 ± 5.23
Hits@50:
  Final val: 74.03 ± 0.79   Final Test: 58.49 ± 3.78   Final old_old: 62.64 ± 3.56   Final old_new: 42.07 ± 4.92   Final new_new: 16.36 ± 6.31
AUC:
  Final val: 97.27 ± 0.09   Final Test: 96.75 ± 0.08   Final old_old: 97.35 ± 0.07   Final old_new: 94.40 ± 0.27   Final new_new: 90.15 ± 1.29
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.73 ± 1.89   Final Test: 31.20 ± 3.78   Final old_old: 34.26 ± 4.02   Final old_new: 18.89 ± 2.96   Final new_new: 3.64 ± 2.03
Hits@20:
  Final val: 57.31 ± 3.54   Final Test: 37.54 ± 2.52   Final old_old: 40.77 ± 2.77   Final old_new: 24.53 ± 1.72   Final new_new: 9.09 ± 2.27
Hits@30:
  Final val: 65.33 ± 2.35   Final Test: 45.48 ± 2.37   Final old_old: 48.97 ± 2.69   Final old_new: 31.59 ± 1.93   Final new_new: 10.91 ± 2.49
Hits@50:
  Final val: 72.87 ± 1.45   Final Test: 56.20 ± 2.98   Final old_old: 60.20 ± 3.20   Final old_new: 40.33 ± 2.69   Final new_new: 15.91 ± 2.78
AUC:
  Final val: 97.13 ± 0.07   Final Test: 96.62 ± 0.14   Final old_old: 97.18 ± 0.12   Final old_new: 94.41 ± 0.26   Final new_new: 90.76 ± 0.49
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.89 ± 3.85   Final Test: 29.42 ± 6.37   Final old_old: 31.97 ± 6.57   Final old_new: 19.07 ± 5.64   Final new_new: 7.27 ± 5.18
Hits@20:
  Final val: 57.36 ± 2.80   Final Test: 40.57 ± 5.38   Final old_old: 43.96 ± 5.53   Final old_new: 26.98 ± 4.89   Final new_new: 8.64 ± 4.66
Hits@30:
  Final val: 64.86 ± 1.85   Final Test: 45.66 ± 3.30   Final old_old: 49.21 ± 3.52   Final old_new: 31.51 ± 2.54   Final new_new: 11.36 ± 4.25
Hits@50:
  Final val: 73.79 ± 1.69   Final Test: 56.95 ± 2.73   Final old_old: 60.84 ± 2.72   Final old_new: 41.46 ± 2.73   Final new_new: 19.09 ± 6.14
AUC:
  Final val: 96.64 ± 0.13   Final Test: 96.07 ± 0.16   Final old_old: 96.72 ± 0.17   Final old_new: 93.46 ± 0.15   Final new_new: 89.87 ± 1.61
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.41 ± 4.80   Final Test: 30.20 ± 5.36   Final old_old: 33.15 ± 5.77   Final old_new: 18.29 ± 3.77   Final new_new: 4.09 ± 2.96
Hits@20:
  Final val: 61.09 ± 5.03   Final Test: 40.74 ± 3.49   Final old_old: 44.12 ± 3.76   Final old_new: 27.36 ± 2.63   Final new_new: 6.82 ± 3.94
Hits@30:
  Final val: 66.69 ± 4.36   Final Test: 47.80 ± 2.06   Final old_old: 51.67 ± 2.32   Final old_new: 32.42 ± 1.52   Final new_new: 9.09 ± 6.01
Hits@50:
  Final val: 74.96 ± 3.38   Final Test: 60.11 ± 1.34   Final old_old: 64.25 ± 1.65   Final old_new: 43.78 ± 0.80   Final new_new: 16.36 ± 3.73
AUC:
  Final val: 97.31 ± 0.14   Final Test: 96.78 ± 0.12   Final old_old: 97.38 ± 0.13   Final old_new: 94.40 ± 0.22   Final new_new: 90.64 ± 0.56
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.73 ± 7.14   Final Test: 30.98 ± 5.90   Final old_old: 33.78 ± 6.57   Final old_new: 19.77 ± 3.54   Final new_new: 4.09 ± 2.49
Hits@20:
  Final val: 54.71 ± 5.47   Final Test: 37.92 ± 3.82   Final old_old: 41.14 ± 4.05   Final old_new: 25.06 ± 3.52   Final new_new: 7.27 ± 5.66
Hits@30:
  Final val: 62.49 ± 4.79   Final Test: 47.05 ± 3.23   Final old_old: 50.65 ± 3.83   Final old_new: 32.85 ± 1.22   Final new_new: 9.09 ± 5.57
Hits@50:
  Final val: 71.57 ± 2.79   Final Test: 58.15 ± 5.99   Final old_old: 61.88 ± 6.42   Final old_new: 43.60 ± 4.84   Final new_new: 15.91 ± 8.04
AUC:
  Final val: 96.58 ± 0.21   Final Test: 96.00 ± 0.29   Final old_old: 96.68 ± 0.30   Final old_new: 93.30 ± 0.51   Final new_new: 88.81 ± 1.93
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.23 ± 1.81   Final Test: 27.27 ± 9.26   Final old_old: 29.71 ± 9.84   Final old_new: 17.36 ± 7.04   Final new_new: 7.27 ± 5.18
Hits@20:
  Final val: 56.86 ± 3.10   Final Test: 39.17 ± 3.01   Final old_old: 42.52 ± 3.13   Final old_new: 25.67 ± 3.05   Final new_new: 10.00 ± 3.05
Hits@30:
  Final val: 65.60 ± 2.08   Final Test: 44.44 ± 1.99   Final old_old: 47.83 ± 2.02   Final old_new: 30.98 ± 2.22   Final new_new: 10.45 ± 2.59
Hits@50:
  Final val: 73.80 ± 1.45   Final Test: 58.21 ± 2.75   Final old_old: 61.90 ± 2.88   Final old_new: 43.78 ± 2.70   Final new_new: 17.27 ± 5.70
AUC:
  Final val: 96.65 ± 0.25   Final Test: 95.96 ± 0.33   Final old_old: 96.65 ± 0.29   Final old_new: 93.17 ± 0.53   Final new_new: 89.16 ± 1.69
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.30 ± 2.07   Final Test: 29.48 ± 2.99   Final old_old: 32.56 ± 3.03   Final old_new: 16.98 ± 3.38   Final new_new: 3.64 ± 2.03
Hits@20:
  Final val: 59.83 ± 1.55   Final Test: 40.63 ± 2.62   Final old_old: 44.04 ± 2.73   Final old_new: 27.10 ± 2.38   Final new_new: 6.82 ± 2.78
Hits@30:
  Final val: 66.61 ± 0.77   Final Test: 47.35 ± 5.78   Final old_old: 51.07 ± 6.01   Final old_new: 32.64 ± 4.92   Final new_new: 9.55 ± 4.93
Hits@50:
  Final val: 74.08 ± 0.70   Final Test: 58.06 ± 2.86   Final old_old: 62.14 ± 3.06   Final old_new: 41.96 ± 2.20   Final new_new: 15.00 ± 4.98
AUC:
  Final val: 97.26 ± 0.11   Final Test: 96.71 ± 0.17   Final old_old: 97.31 ± 0.14   Final old_new: 94.37 ± 0.45   Final new_new: 89.81 ± 1.11
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 54.10 ± 3.62   Final Test: 35.11 ± 6.84   Final old_old: 38.54 ± 7.21   Final old_new: 21.23 ± 5.53   Final new_new: 5.45 ± 3.05
Hits@20:
  Final val: 64.81 ± 2.35   Final Test: 43.22 ± 3.07   Final old_old: 46.87 ± 3.40   Final old_new: 28.66 ± 1.60   Final new_new: 7.27 ± 3.73
Hits@30:
  Final val: 70.85 ± 1.73   Final Test: 50.12 ± 4.06   Final old_old: 54.05 ± 4.20   Final old_new: 34.56 ± 3.50   Final new_new: 10.45 ± 4.43
Hits@50:
  Final val: 77.48 ± 1.53   Final Test: 62.11 ± 1.37   Final old_old: 66.32 ± 1.69   Final old_new: 45.47 ± 0.93   Final new_new: 18.18 ± 2.78
AUC:
  Final val: 97.51 ± 0.06   Final Test: 96.97 ± 0.15   Final old_old: 97.56 ± 0.19   Final old_new: 94.62 ± 0.17   Final new_new: 90.36 ± 1.44
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.80 ± 2.75   Final Test: 35.19 ± 3.43   Final old_old: 38.42 ± 3.42   Final old_new: 22.24 ± 3.66   Final new_new: 5.45 ± 4.43
Hits@20:
  Final val: 64.40 ± 2.36   Final Test: 46.88 ± 2.58   Final old_old: 50.89 ± 2.76   Final old_new: 30.86 ± 2.07   Final new_new: 9.09 ± 1.61
Hits@30:
  Final val: 70.59 ± 1.86   Final Test: 55.13 ± 1.94   Final old_old: 59.22 ± 1.98   Final old_new: 39.07 ± 2.00   Final new_new: 11.36 ± 2.27
Hits@50:
  Final val: 77.34 ± 1.81   Final Test: 62.87 ± 1.74   Final old_old: 67.11 ± 1.82   Final old_new: 46.27 ± 1.59   Final new_new: 16.36 ± 4.66
AUC:
  Final val: 97.44 ± 0.12   Final Test: 96.89 ± 0.11   Final old_old: 97.53 ± 0.09   Final old_new: 94.40 ± 0.20   Final new_new: 90.26 ± 0.86
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.43 ± 5.84   Final Test: 32.14 ± 7.03   Final old_old: 35.34 ± 7.58   Final old_new: 19.19 ± 5.00   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 61.35 ± 4.89   Final Test: 43.90 ± 4.61   Final old_old: 47.50 ± 4.91   Final old_new: 29.60 ± 3.66   Final new_new: 7.73 ± 2.03
Hits@30:
  Final val: 67.41 ± 3.74   Final Test: 51.03 ± 2.81   Final old_old: 54.85 ± 3.15   Final old_new: 36.05 ± 1.52   Final new_new: 9.55 ± 4.37
Hits@50:
  Final val: 75.14 ± 2.35   Final Test: 60.85 ± 2.45   Final old_old: 64.97 ± 2.57   Final old_new: 44.71 ± 2.50   Final new_new: 15.45 ± 5.88
AUC:
  Final val: 97.34 ± 0.22   Final Test: 96.82 ± 0.16   Final old_old: 97.45 ± 0.23   Final old_new: 94.30 ± 0.28   Final new_new: 90.34 ± 0.73
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.59 ± 2.77   Final Test: 32.12 ± 3.02   Final old_old: 35.37 ± 3.07   Final old_new: 19.04 ± 3.14   Final new_new: 2.73 ± 1.02
Hits@20:
  Final val: 59.19 ± 1.98   Final Test: 38.86 ± 1.14   Final old_old: 42.20 ± 1.08   Final old_new: 25.62 ± 2.17   Final new_new: 5.45 ± 1.24
Hits@30:
  Final val: 65.64 ± 1.50   Final Test: 46.88 ± 1.93   Final old_old: 50.64 ± 2.10   Final old_new: 32.14 ± 1.98   Final new_new: 6.36 ± 1.90
Hits@50:
  Final val: 73.39 ± 0.78   Final Test: 57.68 ± 2.63   Final old_old: 61.77 ± 2.45   Final old_new: 41.49 ± 3.42   Final new_new: 15.45 ± 5.88
AUC:
  Final val: 97.27 ± 0.09   Final Test: 96.75 ± 0.04   Final old_old: 97.31 ± 0.07   Final old_new: 94.54 ± 0.11   Final new_new: 91.19 ± 0.37
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.96 ± 6.19   Final Test: 27.93 ± 4.19   Final old_old: 30.51 ± 4.48   Final old_new: 17.48 ± 3.14   Final new_new: 5.45 ± 4.13
Hits@20:
  Final val: 56.29 ± 4.12   Final Test: 37.41 ± 5.58   Final old_old: 40.53 ± 6.07   Final old_new: 24.91 ± 3.76   Final new_new: 8.18 ± 5.23
Hits@30:
  Final val: 63.92 ± 2.71   Final Test: 43.98 ± 3.71   Final old_old: 47.46 ± 3.97   Final old_new: 30.00 ± 2.91   Final new_new: 11.36 ± 5.57
Hits@50:
  Final val: 72.46 ± 1.87   Final Test: 57.63 ± 2.37   Final old_old: 61.49 ± 2.41   Final old_new: 42.37 ± 2.68   Final new_new: 18.18 ± 5.79
AUC:
  Final val: 96.66 ± 0.18   Final Test: 95.92 ± 0.28   Final old_old: 96.57 ± 0.26   Final old_new: 93.34 ± 0.56   Final new_new: 89.46 ± 2.86
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.35 ± 1.12   Final Test: 31.46 ± 3.96   Final old_old: 34.56 ± 4.32   Final old_new: 18.87 ± 2.73   Final new_new: 5.45 ± 3.45
Hits@20:
  Final val: 59.55 ± 2.81   Final Test: 42.44 ± 4.22   Final old_old: 46.06 ± 4.37   Final old_new: 28.04 ± 4.11   Final new_new: 7.27 ± 3.73
Hits@30:
  Final val: 66.40 ± 1.66   Final Test: 47.15 ± 1.78   Final old_old: 50.93 ± 1.89   Final old_new: 32.12 ± 1.81   Final new_new: 10.45 ± 4.71
Hits@50:
  Final val: 73.83 ± 0.94   Final Test: 58.80 ± 3.12   Final old_old: 62.95 ± 3.26   Final old_new: 42.39 ± 3.02   Final new_new: 15.91 ± 4.82
AUC:
  Final val: 97.23 ± 0.13   Final Test: 96.73 ± 0.18   Final old_old: 97.33 ± 0.16   Final old_new: 94.39 ± 0.38   Final new_new: 90.24 ± 0.77
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.73 ± 3.19   Final Test: 30.02 ± 8.71   Final old_old: 32.94 ± 9.29   Final old_new: 18.19 ± 6.49   Final new_new: 5.45 ± 4.43
Hits@20:
  Final val: 59.74 ± 3.61   Final Test: 40.07 ± 3.29   Final old_old: 43.48 ± 3.59   Final old_new: 26.42 ± 2.20   Final new_new: 8.18 ± 3.05
Hits@30:
  Final val: 66.52 ± 1.50   Final Test: 48.29 ± 5.62   Final old_old: 52.02 ± 5.78   Final old_new: 33.53 ± 5.02   Final new_new: 10.00 ± 4.13
Hits@50:
  Final val: 73.22 ± 1.89   Final Test: 58.94 ± 2.87   Final old_old: 63.14 ± 2.91   Final old_new: 42.47 ± 2.91   Final new_new: 13.18 ± 4.37
AUC:
  Final val: 97.23 ± 0.10   Final Test: 96.73 ± 0.18   Final old_old: 97.32 ± 0.17   Final old_new: 94.41 ± 0.26   Final new_new: 90.15 ± 1.32
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.74 ± 3.99   Final Test: 31.50 ± 5.47   Final old_old: 34.44 ± 5.86   Final old_new: 19.70 ± 4.03   Final new_new: 4.09 ± 2.96
Hits@20:
  Final val: 60.47 ± 3.57   Final Test: 41.85 ± 2.48   Final old_old: 45.43 ± 2.66   Final old_new: 27.58 ± 1.94   Final new_new: 6.82 ± 1.61
Hits@30:
  Final val: 66.78 ± 2.63   Final Test: 53.52 ± 2.25   Final old_old: 57.33 ± 2.08   Final old_new: 38.44 ± 3.09   Final new_new: 14.55 ± 3.80
Hits@50:
  Final val: 75.52 ± 1.29   Final Test: 61.54 ± 1.19   Final old_old: 65.57 ± 1.04   Final old_new: 45.82 ± 2.23   Final new_new: 16.82 ± 5.23
AUC:
  Final val: 97.32 ± 0.17   Final Test: 96.79 ± 0.11   Final old_old: 97.39 ± 0.12   Final old_new: 94.42 ± 0.11   Final new_new: 90.46 ± 0.57
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.08 ± 7.50   Final Test: 27.61 ± 10.18   Final old_old: 30.05 ± 10.79   Final old_new: 17.68 ± 8.09   Final new_new: 7.73 ± 3.45
Hits@20:
  Final val: 56.06 ± 7.03   Final Test: 37.45 ± 10.28   Final old_old: 40.45 ± 10.82   Final old_new: 25.37 ± 8.29   Final new_new: 10.45 ± 4.71
Hits@30:
  Final val: 62.96 ± 5.30   Final Test: 45.99 ± 7.21   Final old_old: 49.69 ± 7.62   Final old_new: 31.08 ± 5.96   Final new_new: 12.73 ± 4.13
Hits@50:
  Final val: 71.65 ± 3.79   Final Test: 54.28 ± 5.61   Final old_old: 58.09 ± 5.71   Final old_new: 38.89 ± 5.60   Final new_new: 20.91 ± 4.07
AUC:
  Final val: 96.61 ± 0.22   Final Test: 96.07 ± 0.05   Final old_old: 96.72 ± 0.18   Final old_new: 93.46 ± 0.51   Final new_new: 89.51 ± 2.29
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.47 ± 3.92   Final Test: 25.97 ± 4.05   Final old_old: 28.37 ± 4.17   Final old_new: 16.20 ± 3.70   Final new_new: 6.36 ± 4.07
Hits@20:
  Final val: 55.40 ± 3.75   Final Test: 38.91 ± 4.39   Final old_old: 42.05 ± 4.69   Final old_new: 26.32 ± 3.86   Final new_new: 9.55 ± 2.96
Hits@30:
  Final val: 63.18 ± 3.62   Final Test: 44.83 ± 1.28   Final old_old: 48.34 ± 1.24   Final old_new: 30.78 ± 3.12   Final new_new: 11.82 ± 4.66
Hits@50:
  Final val: 71.67 ± 2.36   Final Test: 57.68 ± 2.00   Final old_old: 61.40 ± 2.14   Final old_new: 43.10 ± 2.54   Final new_new: 16.82 ± 6.14
AUC:
  Final val: 96.60 ± 0.20   Final Test: 96.03 ± 0.24   Final old_old: 96.70 ± 0.22   Final old_new: 93.36 ± 0.43   Final new_new: 89.41 ± 1.20
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.18 ± 6.09   Final Test: 20.49 ± 7.90   Final old_old: 22.37 ± 8.50   Final old_new: 12.82 ± 5.61   Final new_new: 5.00 ± 5.18
Hits@20:
  Final val: 55.80 ± 3.79   Final Test: 40.19 ± 5.92   Final old_old: 43.40 ± 6.33   Final old_new: 27.30 ± 4.58   Final new_new: 10.91 ± 1.02
Hits@30:
  Final val: 63.61 ± 3.78   Final Test: 46.85 ± 2.64   Final old_old: 50.31 ± 2.77   Final old_new: 33.17 ± 2.65   Final new_new: 11.36 ± 4.82
Hits@50:
  Final val: 71.60 ± 1.99   Final Test: 57.13 ± 3.12   Final old_old: 60.82 ± 3.13   Final old_new: 42.59 ± 3.51   Final new_new: 18.18 ± 10.29
AUC:
  Final val: 96.65 ± 0.18   Final Test: 96.00 ± 0.26   Final old_old: 96.61 ± 0.20   Final old_new: 93.53 ± 0.64   Final new_new: 90.38 ± 0.99
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.94 ± 2.46   Final Test: 30.71 ± 5.11   Final old_old: 33.79 ± 5.41   Final old_new: 18.26 ± 4.12   Final new_new: 4.09 ± 3.37
Hits@20:
  Final val: 60.00 ± 1.94   Final Test: 40.21 ± 4.56   Final old_old: 43.87 ± 4.92   Final old_new: 25.47 ± 3.61   Final new_new: 7.27 ± 2.96
Hits@30:
  Final val: 66.49 ± 1.27   Final Test: 49.16 ± 3.60   Final old_old: 53.12 ± 3.85   Final old_new: 33.38 ± 2.70   Final new_new: 10.91 ± 2.49
Hits@50:
  Final val: 74.36 ± 1.65   Final Test: 57.55 ± 3.09   Final old_old: 61.73 ± 3.17   Final old_new: 41.08 ± 3.05   Final new_new: 13.64 ± 2.27
AUC:
  Final val: 97.26 ± 0.12   Final Test: 96.75 ± 0.15   Final old_old: 97.36 ± 0.19   Final old_new: 94.30 ± 0.18   Final new_new: 90.47 ± 0.86
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.02 ± 2.30   Final Test: 32.90 ± 3.08   Final old_old: 36.11 ± 3.29   Final old_new: 19.90 ± 2.35   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 62.62 ± 2.49   Final Test: 43.14 ± 5.05   Final old_old: 46.75 ± 5.27   Final old_new: 28.84 ± 4.37   Final new_new: 5.91 ± 2.59
Hits@30:
  Final val: 69.55 ± 1.02   Final Test: 49.81 ± 2.98   Final old_old: 53.98 ± 3.02   Final old_new: 33.35 ± 3.22   Final new_new: 6.36 ± 1.90
Hits@50:
  Final val: 76.91 ± 1.17   Final Test: 58.83 ± 0.83   Final old_old: 63.34 ± 0.74   Final old_new: 41.03 ± 1.35   Final new_new: 11.36 ± 2.78
AUC:
  Final val: 97.39 ± 0.05   Final Test: 96.84 ± 0.10   Final old_old: 97.45 ± 0.11   Final old_new: 94.40 ± 0.13   Final new_new: 91.06 ± 0.60
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.46 ± 2.45   Final Test: 34.76 ± 7.34   Final old_old: 38.15 ± 7.92   Final old_new: 21.13 ± 5.15   Final new_new: 4.09 ± 2.96
Hits@20:
  Final val: 62.79 ± 2.57   Final Test: 43.90 ± 4.11   Final old_old: 47.77 ± 4.36   Final old_new: 28.51 ± 3.51   Final new_new: 5.45 ± 1.24
Hits@30:
  Final val: 68.92 ± 2.50   Final Test: 53.44 ± 3.28   Final old_old: 57.63 ± 3.53   Final old_new: 36.98 ± 2.42   Final new_new: 8.18 ± 3.05
Hits@50:
  Final val: 76.60 ± 1.56   Final Test: 60.86 ± 2.77   Final old_old: 65.14 ± 3.09   Final old_new: 44.13 ± 1.79   Final new_new: 13.18 ± 2.49
AUC:
  Final val: 97.40 ± 0.11   Final Test: 96.88 ± 0.10   Final old_old: 97.49 ± 0.13   Final old_new: 94.48 ± 0.28   Final new_new: 89.95 ± 0.82
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.24 ± 4.22   Final Test: 25.55 ± 4.27   Final old_old: 27.79 ± 4.49   Final old_new: 16.60 ± 3.61   Final new_new: 3.64 ± 3.45
Hits@20:
  Final val: 54.49 ± 1.88   Final Test: 35.63 ± 5.61   Final old_old: 38.49 ± 5.84   Final old_new: 24.26 ± 4.99   Final new_new: 7.27 ± 2.96
Hits@30:
  Final val: 62.92 ± 1.25   Final Test: 44.04 ± 4.38   Final old_old: 47.41 ± 4.53   Final old_new: 30.73 ± 3.93   Final new_new: 8.64 ± 4.07
Hits@50:
  Final val: 72.01 ± 1.31   Final Test: 54.39 ± 2.77   Final old_old: 58.12 ± 2.66   Final old_new: 39.62 ± 3.74   Final new_new: 15.91 ± 4.55
AUC:
  Final val: 96.58 ± 0.15   Final Test: 95.90 ± 0.25   Final old_old: 96.56 ± 0.23   Final old_new: 93.22 ± 0.47   Final new_new: 89.93 ± 1.33
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.10 ± 3.65   Final Test: 28.52 ± 3.38   Final old_old: 31.51 ± 3.57   Final old_new: 16.35 ± 2.91   Final new_new: 4.55 ± 2.27
Hits@20:
  Final val: 60.23 ± 2.42   Final Test: 39.43 ± 1.28   Final old_old: 42.93 ± 1.54   Final old_new: 25.39 ± 1.52   Final new_new: 6.36 ± 4.07
Hits@30:
  Final val: 67.29 ± 0.95   Final Test: 47.59 ± 4.37   Final old_old: 51.39 ± 4.68   Final old_new: 32.54 ± 3.37   Final new_new: 8.64 ± 1.90
Hits@50:
  Final val: 73.95 ± 1.05   Final Test: 57.42 ± 2.58   Final old_old: 61.34 ± 2.65   Final old_new: 42.12 ± 3.18   Final new_new: 13.18 ± 4.37
AUC:
  Final val: 97.29 ± 0.09   Final Test: 96.79 ± 0.17   Final old_old: 97.33 ± 0.17   Final old_new: 94.67 ± 0.32   Final new_new: 90.86 ± 1.45
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.38 ± 4.30   Final Test: 33.93 ± 7.54   Final old_old: 37.30 ± 8.02   Final old_new: 20.30 ± 5.72   Final new_new: 4.55 ± 2.27
Hits@20:
  Final val: 62.33 ± 1.66   Final Test: 47.66 ± 4.11   Final old_old: 51.59 ± 4.35   Final old_new: 32.07 ± 3.28   Final new_new: 7.27 ± 1.02
Hits@30:
  Final val: 70.13 ± 0.94   Final Test: 53.20 ± 3.37   Final old_old: 57.41 ± 3.55   Final old_new: 36.60 ± 2.95   Final new_new: 9.09 ± 2.27
Hits@50:
  Final val: 77.02 ± 0.25   Final Test: 64.05 ± 1.99   Final old_old: 68.44 ± 2.11   Final old_new: 46.83 ± 2.07   Final new_new: 16.36 ± 6.31
AUC:
  Final val: 97.40 ± 0.11   Final Test: 96.89 ± 0.09   Final old_old: 97.50 ± 0.07   Final old_new: 94.50 ± 0.26   Final new_new: 90.54 ± 0.63
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.04 ± 1.31   Final Test: 29.88 ± 2.10   Final old_old: 32.93 ± 2.44   Final old_new: 17.56 ± 0.70   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 60.62 ± 2.84   Final Test: 38.68 ± 3.82   Final old_old: 42.00 ± 4.04   Final old_new: 25.47 ± 2.96   Final new_new: 5.91 ± 4.13
Hits@30:
  Final val: 66.18 ± 1.93   Final Test: 47.16 ± 1.98   Final old_old: 50.92 ± 2.20   Final old_new: 32.24 ± 2.23   Final new_new: 9.55 ± 4.07
Hits@50:
  Final val: 73.65 ± 0.87   Final Test: 58.20 ± 2.56   Final old_old: 62.30 ± 2.83   Final old_new: 42.14 ± 1.80   Final new_new: 13.18 ± 1.90
AUC:
  Final val: 97.25 ± 0.11   Final Test: 96.75 ± 0.14   Final old_old: 97.34 ± 0.16   Final old_new: 94.37 ± 0.26   Final new_new: 90.74 ± 0.71
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 40.94 ± 5.89   Final Test: 26.88 ± 4.65   Final old_old: 29.27 ± 4.83   Final old_new: 17.15 ± 4.19   Final new_new: 7.27 ± 5.88
Hits@20:
  Final val: 54.11 ± 4.97   Final Test: 33.76 ± 5.14   Final old_old: 36.50 ± 5.44   Final old_new: 22.85 ± 4.18   Final new_new: 6.82 ± 5.79
Hits@30:
  Final val: 62.21 ± 3.76   Final Test: 46.58 ± 4.47   Final old_old: 49.89 ± 4.67   Final old_new: 33.40 ± 3.75   Final new_new: 14.09 ± 4.37
Hits@50:
  Final val: 70.93 ± 3.64   Final Test: 53.60 ± 5.42   Final old_old: 57.12 ± 5.59   Final old_new: 39.65 ± 4.80   Final new_new: 17.27 ± 8.29
AUC:
  Final val: 96.68 ± 0.28   Final Test: 96.16 ± 0.14   Final old_old: 96.75 ± 0.12   Final old_new: 93.81 ± 0.32   Final new_new: 90.24 ± 2.19
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.10 ± 2.74   Final Test: 34.34 ± 6.90   Final old_old: 37.42 ± 7.17   Final old_new: 21.99 ± 6.02   Final new_new: 5.00 ± 1.90
Hits@20:
  Final val: 64.69 ± 2.37   Final Test: 45.00 ± 3.93   Final old_old: 48.70 ± 3.90   Final old_new: 30.30 ± 4.23   Final new_new: 8.18 ± 4.98
Hits@30:
  Final val: 70.51 ± 1.65   Final Test: 50.59 ± 3.11   Final old_old: 54.63 ± 3.18   Final old_new: 34.56 ± 2.93   Final new_new: 10.00 ± 5.93
Hits@50:
  Final val: 77.19 ± 1.72   Final Test: 62.37 ± 1.80   Final old_old: 66.57 ± 1.77   Final old_new: 45.89 ± 2.34   Final new_new: 16.36 ± 5.88
AUC:
  Final val: 97.36 ± 0.08   Final Test: 96.89 ± 0.06   Final old_old: 97.51 ± 0.06   Final old_new: 94.40 ± 0.19   Final new_new: 90.53 ± 1.07
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.87 ± 1.70   Final Test: 33.16 ± 3.37   Final old_old: 36.54 ± 3.71   Final old_new: 19.42 ± 2.36   Final new_new: 5.00 ± 1.90
Hits@20:
  Final val: 60.20 ± 1.11   Final Test: 42.61 ± 1.37   Final old_old: 46.28 ± 1.63   Final old_new: 27.98 ± 0.89   Final new_new: 6.82 ± 2.78
Hits@30:
  Final val: 66.27 ± 0.77   Final Test: 49.02 ± 1.87   Final old_old: 53.07 ± 2.18   Final old_new: 32.92 ± 2.06   Final new_new: 9.09 ± 4.55
Hits@50:
  Final val: 74.61 ± 1.21   Final Test: 58.52 ± 1.89   Final old_old: 62.92 ± 2.08   Final old_new: 41.11 ± 1.63   Final new_new: 13.18 ± 2.49
AUC:
  Final val: 97.29 ± 0.09   Final Test: 96.72 ± 0.11   Final old_old: 97.34 ± 0.13   Final old_new: 94.28 ± 0.35   Final new_new: 90.14 ± 1.37
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.06 ± 4.35   Final Test: 29.75 ± 3.54   Final old_old: 32.46 ± 3.91   Final old_new: 18.84 ± 2.23   Final new_new: 5.45 ± 4.71
Hits@20:
  Final val: 58.29 ± 2.94   Final Test: 37.71 ± 4.66   Final old_old: 40.88 ± 4.88   Final old_new: 25.09 ± 4.00   Final new_new: 6.36 ± 4.93
Hits@30:
  Final val: 64.71 ± 2.14   Final Test: 44.89 ± 3.72   Final old_old: 48.29 ± 4.14   Final old_new: 31.36 ± 2.43   Final new_new: 11.82 ± 7.08
Hits@50:
  Final val: 72.58 ± 1.99   Final Test: 57.10 ± 3.82   Final old_old: 60.97 ± 3.88   Final old_new: 41.76 ± 3.86   Final new_new: 17.27 ± 9.03
AUC:
  Final val: 96.67 ± 0.19   Final Test: 96.10 ± 0.27   Final old_old: 96.77 ± 0.22   Final old_new: 93.43 ± 0.51   Final new_new: 89.79 ± 2.06
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.26 ± 1.69   Final Test: 30.72 ± 4.00   Final old_old: 33.84 ± 4.25   Final old_new: 18.11 ± 3.19   Final new_new: 3.18 ± 1.24
Hits@20:
  Final val: 59.21 ± 1.95   Final Test: 38.86 ± 3.36   Final old_old: 42.20 ± 3.49   Final old_new: 25.49 ± 3.07   Final new_new: 7.27 ± 5.43
Hits@30:
  Final val: 65.18 ± 1.07   Final Test: 46.19 ± 3.47   Final old_old: 49.84 ± 3.56   Final old_new: 31.66 ± 3.42   Final new_new: 10.00 ± 4.71
Hits@50:
  Final val: 73.17 ± 1.44   Final Test: 56.54 ± 2.09   Final old_old: 60.52 ± 2.51   Final old_new: 41.01 ± 1.09   Final new_new: 11.82 ± 2.96
AUC:
  Final val: 97.20 ± 0.13   Final Test: 96.69 ± 0.11   Final old_old: 97.29 ± 0.15   Final old_new: 94.34 ± 0.27   Final new_new: 90.00 ± 0.92
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 53.19 ± 3.81   Final Test: 32.02 ± 7.62   Final old_old: 35.29 ± 8.12   Final old_new: 18.79 ± 5.77   Final new_new: 4.09 ± 2.96
Hits@20:
  Final val: 65.08 ± 2.07   Final Test: 42.71 ± 2.58   Final old_old: 46.53 ± 2.82   Final old_new: 27.36 ± 1.89   Final new_new: 7.73 ± 3.05
Hits@30:
  Final val: 70.46 ± 1.30   Final Test: 49.85 ± 3.96   Final old_old: 54.00 ± 4.30   Final old_new: 33.30 ± 2.73   Final new_new: 10.00 ± 5.23
Hits@50:
  Final val: 77.49 ± 0.79   Final Test: 60.72 ± 3.87   Final old_old: 65.03 ± 4.03   Final old_new: 43.83 ± 3.60   Final new_new: 13.64 ± 5.57
AUC:
  Final val: 97.52 ± 0.07   Final Test: 96.99 ± 0.14   Final old_old: 97.56 ± 0.15   Final old_new: 94.71 ± 0.33   Final new_new: 91.33 ± 1.44
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.50 ± 4.99   Final Test: 27.16 ± 3.19   Final old_old: 29.70 ± 3.37   Final old_new: 16.80 ± 2.64   Final new_new: 6.82 ± 4.25
Hits@20:
  Final val: 56.55 ± 3.56   Final Test: 38.00 ± 3.94   Final old_old: 41.05 ± 4.19   Final old_new: 25.92 ± 3.15   Final new_new: 7.27 ± 4.93
Hits@30:
  Final val: 65.17 ± 1.75   Final Test: 47.50 ± 2.96   Final old_old: 51.08 ± 3.22   Final old_new: 33.48 ± 2.44   Final new_new: 8.18 ± 5.70
Hits@50:
  Final val: 73.28 ± 1.19   Final Test: 55.14 ± 2.63   Final old_old: 58.99 ± 2.92   Final old_new: 40.10 ± 2.32   Final new_new: 12.27 ± 4.13
AUC:
  Final val: 96.67 ± 0.20   Final Test: 96.04 ± 0.24   Final old_old: 96.71 ± 0.17   Final old_new: 93.37 ± 0.65   Final new_new: 89.67 ± 1.77
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.91 ± 4.69   Final Test: 30.23 ± 5.61   Final old_old: 33.15 ± 6.09   Final old_new: 18.46 ± 3.84   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 62.09 ± 3.62   Final Test: 43.51 ± 3.46   Final old_old: 47.02 ± 3.72   Final old_new: 29.60 ± 2.52   Final new_new: 8.18 ± 4.13
Hits@30:
  Final val: 68.88 ± 2.94   Final Test: 50.19 ± 2.18   Final old_old: 54.13 ± 2.33   Final old_new: 34.53 ± 1.90   Final new_new: 10.45 ± 5.23
Hits@50:
  Final val: 75.78 ± 2.45   Final Test: 60.27 ± 0.84   Final old_old: 64.40 ± 0.68   Final old_new: 43.93 ± 1.73   Final new_new: 17.73 ± 4.93
AUC:
  Final val: 97.34 ± 0.13   Final Test: 96.81 ± 0.14   Final old_old: 97.43 ± 0.12   Final old_new: 94.37 ± 0.28   Final new_new: 90.10 ± 0.71
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.45 ± 2.67   Final Test: 32.20 ± 3.04   Final old_old: 35.36 ± 3.21   Final old_new: 19.37 ± 2.56   Final new_new: 5.00 ± 1.90
Hits@20:
  Final val: 60.02 ± 2.93   Final Test: 42.38 ± 1.83   Final old_old: 46.05 ± 2.12   Final old_new: 27.73 ± 1.23   Final new_new: 6.82 ± 3.59
Hits@30:
  Final val: 66.96 ± 1.88   Final Test: 48.00 ± 3.60   Final old_old: 51.72 ± 3.63   Final old_new: 33.32 ± 3.75   Final new_new: 9.09 ± 4.55
Hits@50:
  Final val: 74.90 ± 1.31   Final Test: 59.79 ± 3.09   Final old_old: 63.80 ± 3.00   Final old_new: 44.23 ± 3.60   Final new_new: 12.73 ± 5.47
AUC:
  Final val: 97.43 ± 0.09   Final Test: 96.90 ± 0.10   Final old_old: 97.44 ± 0.14   Final old_new: 94.85 ± 0.22   Final new_new: 90.70 ± 0.57
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.18 ± 5.94   Final Test: 29.69 ± 5.72   Final old_old: 32.37 ± 6.13   Final old_new: 18.87 ± 4.17   Final new_new: 5.91 ± 4.71
Hits@20:
  Final val: 58.60 ± 4.00   Final Test: 38.83 ± 6.74   Final old_old: 42.04 ± 7.12   Final old_new: 26.05 ± 5.44   Final new_new: 6.82 ± 3.59
Hits@30:
  Final val: 64.94 ± 2.39   Final Test: 47.51 ± 3.61   Final old_old: 51.17 ± 3.84   Final old_new: 33.02 ± 2.98   Final new_new: 10.45 ± 2.59
Hits@50:
  Final val: 72.80 ± 0.80   Final Test: 55.39 ± 4.45   Final old_old: 59.05 ± 4.85   Final old_new: 41.26 ± 2.97   Final new_new: 10.91 ± 2.96
AUC:
  Final val: 96.64 ± 0.18   Final Test: 95.91 ± 0.20   Final old_old: 96.60 ± 0.23   Final old_new: 93.16 ± 0.29   Final new_new: 88.98 ± 1.51
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.00 ± 3.45   Final Test: 35.33 ± 6.08   Final old_old: 38.75 ± 6.33   Final old_new: 21.44 ± 5.49   Final new_new: 6.36 ± 2.49
Hits@20:
  Final val: 63.31 ± 2.11   Final Test: 45.48 ± 3.78   Final old_old: 49.28 ± 3.85   Final old_new: 30.18 ± 3.85   Final new_new: 10.45 ± 2.59
Hits@30:
  Final val: 69.12 ± 2.18   Final Test: 52.10 ± 2.70   Final old_old: 56.10 ± 2.83   Final old_new: 36.12 ± 2.44   Final new_new: 13.64 ± 2.27
Hits@50:
  Final val: 76.11 ± 0.77   Final Test: 63.26 ± 1.83   Final old_old: 67.58 ± 1.82   Final old_new: 46.20 ± 1.83   Final new_new: 18.64 ± 5.66
AUC:
  Final val: 97.38 ± 0.04   Final Test: 96.90 ± 0.11   Final old_old: 97.51 ± 0.11   Final old_new: 94.47 ± 0.20   Final new_new: 90.70 ± 1.52
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.90 ± 3.01   Final Test: 32.62 ± 3.53   Final old_old: 35.83 ± 3.67   Final old_new: 19.55 ± 3.53   Final new_new: 6.36 ± 2.96
Hits@20:
  Final val: 61.22 ± 1.86   Final Test: 40.42 ± 3.42   Final old_old: 43.93 ± 3.50   Final old_new: 26.37 ± 3.35   Final new_new: 7.27 ± 2.96
Hits@30:
  Final val: 67.42 ± 1.56   Final Test: 46.75 ± 2.87   Final old_old: 50.62 ± 3.09   Final old_new: 31.16 ± 2.27   Final new_new: 12.27 ± 3.45
Hits@50:
  Final val: 75.18 ± 1.07   Final Test: 58.22 ± 3.12   Final old_old: 62.57 ± 3.32   Final old_new: 40.98 ± 3.05   Final new_new: 14.55 ± 5.70
AUC:
  Final val: 97.31 ± 0.09   Final Test: 96.74 ± 0.08   Final old_old: 97.37 ± 0.06   Final old_new: 94.19 ± 0.45   Final new_new: 90.88 ± 1.42
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.11 ± 6.88   Final Test: 29.67 ± 3.01   Final old_old: 32.22 ± 3.26   Final old_new: 19.29 ± 2.79   Final new_new: 8.64 ± 1.90
Hits@20:
  Final val: 60.17 ± 6.20   Final Test: 40.08 ± 6.75   Final old_old: 43.11 ± 6.82   Final old_new: 27.91 ± 6.64   Final new_new: 12.73 ± 4.98
Hits@30:
  Final val: 67.97 ± 2.84   Final Test: 48.46 ± 4.53   Final old_old: 51.97 ± 4.63   Final old_new: 34.43 ± 4.37   Final new_new: 15.00 ± 4.98
Hits@50:
  Final val: 75.26 ± 2.15   Final Test: 59.88 ± 2.08   Final old_old: 63.77 ± 1.85   Final old_new: 44.43 ± 3.22   Final new_new: 20.45 ± 4.25
AUC:
  Final val: 96.86 ± 0.15   Final Test: 96.27 ± 0.20   Final old_old: 96.88 ± 0.19   Final old_new: 93.84 ± 0.35   Final new_new: 90.07 ± 0.58
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 53.65 ± 3.31   Final Test: 36.57 ± 2.37   Final old_old: 40.18 ± 2.49   Final old_new: 22.02 ± 1.89   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 64.88 ± 2.22   Final Test: 46.47 ± 5.90   Final old_old: 50.38 ± 6.28   Final old_new: 30.86 ± 4.52   Final new_new: 9.55 ± 4.07
Hits@30:
  Final val: 70.86 ± 1.90   Final Test: 53.35 ± 5.09   Final old_old: 57.55 ± 5.39   Final old_new: 36.70 ± 4.00   Final new_new: 11.36 ± 4.55
Hits@50:
  Final val: 77.24 ± 1.66   Final Test: 61.80 ± 2.57   Final old_old: 66.00 ± 2.55   Final old_new: 45.26 ± 2.93   Final new_new: 16.82 ± 2.59
AUC:
  Final val: 97.41 ± 0.08   Final Test: 96.86 ± 0.13   Final old_old: 97.49 ± 0.14   Final old_new: 94.36 ± 0.19   Final new_new: 90.48 ± 1.56
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.27 ± 3.16   Final Test: 30.71 ± 3.29   Final old_old: 33.82 ± 3.59   Final old_new: 18.06 ± 2.26   Final new_new: 4.55 ± 2.27
Hits@20:
  Final val: 60.19 ± 3.04   Final Test: 41.45 ± 4.16   Final old_old: 45.03 ± 4.43   Final old_new: 27.20 ± 3.21   Final new_new: 6.36 ± 1.02
Hits@30:
  Final val: 66.92 ± 2.18   Final Test: 49.54 ± 5.47   Final old_old: 53.36 ± 5.66   Final old_new: 34.41 ± 4.77   Final new_new: 10.45 ± 4.71
Hits@50:
  Final val: 73.97 ± 1.23   Final Test: 57.15 ± 1.94   Final old_old: 61.21 ± 2.03   Final old_new: 41.21 ± 1.77   Final new_new: 13.64 ± 3.59
AUC:
  Final val: 97.24 ± 0.16   Final Test: 96.71 ± 0.14   Final old_old: 97.31 ± 0.19   Final old_new: 94.35 ± 0.32   Final new_new: 90.30 ± 0.80
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.42 ± 5.89   Final Test: 32.40 ± 6.85   Final old_old: 35.28 ± 7.45   Final old_new: 20.71 ± 4.66   Final new_new: 8.18 ± 2.59
Hits@20:
  Final val: 57.71 ± 3.36   Final Test: 40.24 ± 2.89   Final old_old: 43.60 ± 3.38   Final old_new: 26.83 ± 1.27   Final new_new: 8.64 ± 2.96
Hits@30:
  Final val: 64.31 ± 3.23   Final Test: 47.42 ± 4.35   Final old_old: 51.06 ± 4.46   Final old_new: 32.85 ± 4.30   Final new_new: 12.73 ± 5.23
Hits@50:
  Final val: 72.67 ± 1.44   Final Test: 56.01 ± 3.26   Final old_old: 59.92 ± 3.43   Final old_new: 40.48 ± 3.13   Final new_new: 17.73 ± 4.37
AUC:
  Final val: 96.68 ± 0.16   Final Test: 96.08 ± 0.21   Final old_old: 96.76 ± 0.16   Final old_new: 93.43 ± 0.42   Final new_new: 88.85 ± 0.85
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.62 ± 3.43   Final Test: 30.66 ± 3.79   Final old_old: 33.75 ± 3.97   Final old_new: 18.14 ± 3.28   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 58.75 ± 4.69   Final Test: 39.19 ± 2.87   Final old_old: 42.47 ± 2.92   Final old_new: 25.99 ± 2.91   Final new_new: 9.55 ± 6.31
Hits@30:
  Final val: 64.75 ± 3.78   Final Test: 45.70 ± 1.67   Final old_old: 49.39 ± 1.91   Final old_new: 31.06 ± 1.56   Final new_new: 8.64 ± 4.07
Hits@50:
  Final val: 72.44 ± 2.59   Final Test: 56.70 ± 2.50   Final old_old: 60.63 ± 2.59   Final old_new: 41.03 ± 3.40   Final new_new: 17.73 ± 4.37
AUC:
  Final val: 97.17 ± 0.17   Final Test: 96.64 ± 0.19   Final old_old: 97.23 ± 0.23   Final old_new: 94.37 ± 0.32   Final new_new: 90.03 ± 1.15
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.39 ± 6.11   Final Test: 33.71 ± 6.58   Final old_old: 36.72 ± 7.17   Final old_new: 21.61 ± 4.25   Final new_new: 6.36 ± 3.73
Hits@20:
  Final val: 62.30 ± 4.42   Final Test: 42.12 ± 6.67   Final old_old: 45.49 ± 7.10   Final old_new: 28.69 ± 5.30   Final new_new: 9.09 ± 4.82
Hits@30:
  Final val: 69.51 ± 2.66   Final Test: 52.82 ± 6.51   Final old_old: 56.64 ± 6.77   Final old_new: 37.71 ± 5.78   Final new_new: 13.64 ± 4.55
Hits@50:
  Final val: 76.65 ± 1.81   Final Test: 62.12 ± 2.94   Final old_old: 66.16 ± 3.09   Final old_new: 46.30 ± 2.76   Final new_new: 18.18 ± 3.59
AUC:
  Final val: 97.51 ± 0.11   Final Test: 97.08 ± 0.14   Final old_old: 97.58 ± 0.17   Final old_new: 95.12 ± 0.19   Final new_new: 91.54 ± 0.79
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.28 ± 2.87   Final Test: 25.74 ± 3.49   Final old_old: 28.03 ± 3.64   Final old_new: 16.42 ± 3.38   Final new_new: 6.36 ± 4.07
Hits@20:
  Final val: 54.78 ± 2.93   Final Test: 35.89 ± 1.52   Final old_old: 38.79 ± 1.34   Final old_new: 24.26 ± 2.65   Final new_new: 8.64 ± 4.07
Hits@30:
  Final val: 62.25 ± 1.21   Final Test: 45.17 ± 1.39   Final old_old: 48.54 ± 1.60   Final old_new: 31.84 ± 1.99   Final new_new: 10.91 ± 2.96
Hits@50:
  Final val: 71.92 ± 1.23   Final Test: 55.02 ± 2.55   Final old_old: 58.78 ± 2.46   Final old_new: 40.13 ± 3.48   Final new_new: 16.82 ± 3.45
AUC:
  Final val: 96.62 ± 0.14   Final Test: 95.99 ± 0.25   Final old_old: 96.62 ± 0.22   Final old_new: 93.47 ± 0.44   Final new_new: 89.95 ± 1.70
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.02 ± 1.96   Final Test: 33.05 ± 1.82   Final old_old: 36.11 ± 2.01   Final old_new: 20.73 ± 1.03   Final new_new: 5.45 ± 1.24
Hits@20:
  Final val: 59.72 ± 3.34   Final Test: 41.71 ± 1.69   Final old_old: 45.02 ± 1.97   Final old_new: 28.49 ± 0.73   Final new_new: 9.55 ± 3.73
Hits@30:
  Final val: 67.24 ± 2.49   Final Test: 48.08 ± 2.20   Final old_old: 51.69 ± 2.24   Final old_new: 33.73 ± 2.48   Final new_new: 12.73 ± 6.14
Hits@50:
  Final val: 75.23 ± 1.80   Final Test: 59.04 ± 2.51   Final old_old: 62.87 ± 2.88   Final old_new: 44.08 ± 1.37   Final new_new: 15.91 ± 5.57
AUC:
  Final val: 97.44 ± 0.11   Final Test: 97.00 ± 0.03   Final old_old: 97.48 ± 0.08   Final old_new: 95.10 ± 0.18   Final new_new: 91.98 ± 0.33
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.94 ± 3.33   Final Test: 36.16 ± 3.68   Final old_old: 39.59 ± 4.05   Final old_new: 22.29 ± 2.30   Final new_new: 5.91 ± 2.59
Hits@20:
  Final val: 62.25 ± 1.62   Final Test: 44.12 ± 2.98   Final old_old: 47.83 ± 3.14   Final old_new: 29.35 ± 2.51   Final new_new: 7.27 ± 1.90
Hits@30:
  Final val: 69.37 ± 1.48   Final Test: 50.37 ± 2.02   Final old_old: 54.46 ± 2.04   Final old_new: 34.13 ± 2.43   Final new_new: 9.55 ± 4.07
Hits@50:
  Final val: 76.95 ± 1.49   Final Test: 61.86 ± 1.12   Final old_old: 66.07 ± 1.13   Final old_new: 45.44 ± 1.25   Final new_new: 14.09 ± 1.90
AUC:
  Final val: 97.32 ± 0.16   Final Test: 96.84 ± 0.16   Final old_old: 97.43 ± 0.16   Final old_new: 94.48 ± 0.25   Final new_new: 90.68 ± 0.92
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.43 ± 3.95   Final Test: 26.70 ± 7.46   Final old_old: 29.15 ± 8.07   Final old_new: 16.75 ± 5.15   Final new_new: 5.91 ± 2.03
Hits@20:
  Final val: 55.94 ± 3.28   Final Test: 39.68 ± 3.27   Final old_old: 43.00 ± 3.29   Final old_new: 26.37 ± 3.59   Final new_new: 9.09 ± 1.61
Hits@30:
  Final val: 64.33 ± 3.25   Final Test: 45.81 ± 3.59   Final old_old: 49.45 ± 3.83   Final old_new: 31.36 ± 3.02   Final new_new: 9.55 ± 1.90
Hits@50:
  Final val: 72.49 ± 2.03   Final Test: 56.80 ± 1.75   Final old_old: 60.76 ± 1.68   Final old_new: 41.26 ± 2.30   Final new_new: 14.09 ± 3.73
AUC:
  Final val: 96.73 ± 0.24   Final Test: 96.04 ± 0.20   Final old_old: 96.73 ± 0.17   Final old_new: 93.29 ± 0.37   Final new_new: 89.18 ± 1.13
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.64 ± 4.20   Final Test: 32.34 ± 5.98   Final old_old: 35.71 ± 6.51   Final old_new: 18.61 ± 4.04   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 63.67 ± 1.81   Final Test: 45.08 ± 2.35   Final old_old: 48.85 ± 2.58   Final old_new: 30.13 ± 1.66   Final new_new: 7.27 ± 2.49
Hits@30:
  Final val: 69.63 ± 1.68   Final Test: 51.25 ± 4.02   Final old_old: 55.26 ± 4.43   Final old_new: 35.49 ± 2.57   Final new_new: 8.18 ± 2.03
Hits@50:
  Final val: 76.26 ± 1.80   Final Test: 60.42 ± 2.13   Final old_old: 64.45 ± 2.22   Final old_new: 44.71 ± 1.81   Final new_new: 15.00 ± 2.03
AUC:
  Final val: 97.40 ± 0.11   Final Test: 96.87 ± 0.10   Final old_old: 97.48 ± 0.14   Final old_new: 94.46 ± 0.18   Final new_new: 90.54 ± 1.23
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.62 ± 4.09   Final Test: 32.68 ± 2.66   Final old_old: 35.78 ± 2.90   Final old_new: 20.03 ± 2.14   Final new_new: 7.73 ± 2.59
Hits@20:
  Final val: 57.78 ± 1.97   Final Test: 40.19 ± 5.28   Final old_old: 43.71 ± 5.80   Final old_new: 25.89 ± 3.29   Final new_new: 10.45 ± 4.13
Hits@30:
  Final val: 64.76 ± 1.11   Final Test: 45.80 ± 3.02   Final old_old: 49.44 ± 3.29   Final old_new: 31.13 ± 1.99   Final new_new: 12.73 ± 5.70
Hits@50:
  Final val: 71.70 ± 1.58   Final Test: 57.51 ± 3.13   Final old_old: 61.42 ± 3.36   Final old_new: 42.04 ± 2.23   Final new_new: 17.73 ± 2.96
AUC:
  Final val: 95.54 ± 0.23   Final Test: 95.10 ± 0.22   Final old_old: 95.72 ± 0.17   Final old_new: 92.75 ± 0.57   Final new_new: 86.51 ± 1.25
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.73 ± 3.20   Final Test: 30.87 ± 4.30   Final old_old: 33.76 ± 4.51   Final old_new: 19.35 ± 3.70   Final new_new: 3.18 ± 1.24
Hits@20:
  Final val: 59.54 ± 1.87   Final Test: 40.11 ± 4.29   Final old_old: 43.50 ± 4.51   Final old_new: 26.55 ± 3.63   Final new_new: 7.73 ± 2.59
Hits@30:
  Final val: 65.92 ± 0.71   Final Test: 47.03 ± 2.11   Final old_old: 50.79 ± 2.06   Final old_new: 32.09 ± 2.70   Final new_new: 9.55 ± 1.02
Hits@50:
  Final val: 73.15 ± 0.84   Final Test: 57.69 ± 2.30   Final old_old: 61.65 ± 2.12   Final old_new: 41.96 ± 3.61   Final new_new: 18.18 ± 6.22
AUC:
  Final val: 97.23 ± 0.09   Final Test: 96.72 ± 0.09   Final old_old: 97.30 ± 0.07   Final old_new: 94.42 ± 0.29   Final new_new: 90.82 ± 0.80
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.18 ± 5.02   Final Test: 36.07 ± 1.36   Final old_old: 39.45 ± 1.47   Final old_new: 22.57 ± 1.79   Final new_new: 3.64 ± 2.59
Hits@20:
  Final val: 63.59 ± 3.14   Final Test: 45.57 ± 3.36   Final old_old: 49.34 ± 3.53   Final old_new: 30.68 ± 3.19   Final new_new: 6.82 ± 2.27
Hits@30:
  Final val: 70.02 ± 2.91   Final Test: 50.68 ± 2.84   Final old_old: 54.52 ± 3.11   Final old_new: 35.69 ± 2.26   Final new_new: 8.18 ± 3.45
Hits@50:
  Final val: 76.53 ± 2.36   Final Test: 61.58 ± 3.46   Final old_old: 65.66 ± 3.84   Final old_new: 45.59 ± 2.08   Final new_new: 17.27 ± 4.71
AUC:
  Final val: 97.33 ± 0.15   Final Test: 96.85 ± 0.13   Final old_old: 97.45 ± 0.16   Final old_new: 94.47 ± 0.26   Final new_new: 90.76 ± 0.87
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.72 ± 3.50   Final Test: 32.81 ± 4.06   Final old_old: 35.95 ± 4.32   Final old_new: 20.23 ± 3.33   Final new_new: 4.09 ± 2.49
Hits@20:
  Final val: 60.13 ± 2.17   Final Test: 44.44 ± 3.27   Final old_old: 48.14 ± 3.46   Final old_new: 29.72 ± 2.77   Final new_new: 7.73 ± 3.80
Hits@30:
  Final val: 66.64 ± 1.81   Final Test: 48.75 ± 3.53   Final old_old: 52.59 ± 3.65   Final old_new: 33.63 ± 3.27   Final new_new: 8.64 ± 3.73
Hits@50:
  Final val: 73.89 ± 1.54   Final Test: 58.87 ± 2.60   Final old_old: 62.97 ± 2.55   Final old_new: 42.85 ± 3.40   Final new_new: 12.73 ± 6.14
AUC:
  Final val: 97.25 ± 0.14   Final Test: 96.76 ± 0.13   Final old_old: 97.35 ± 0.19   Final old_new: 94.43 ± 0.33   Final new_new: 90.54 ± 0.53
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.28 ± 2.35   Final Test: 31.99 ± 4.80   Final old_old: 35.11 ± 5.07   Final old_new: 19.42 ± 4.20   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 62.34 ± 0.95   Final Test: 42.72 ± 3.51   Final old_old: 46.31 ± 3.51   Final old_new: 28.34 ± 3.73   Final new_new: 9.55 ± 2.96
Hits@30:
  Final val: 67.48 ± 0.18   Final Test: 49.15 ± 3.21   Final old_old: 52.92 ± 3.29   Final old_new: 34.18 ± 3.18   Final new_new: 11.36 ± 2.27
Hits@50:
  Final val: 73.25 ± 0.65   Final Test: 60.61 ± 3.04   Final old_old: 64.36 ± 2.86   Final old_new: 46.02 ± 3.79   Final new_new: 17.73 ± 5.88
AUC:
  Final val: 95.55 ± 0.27   Final Test: 95.06 ± 0.20   Final old_old: 95.78 ± 0.18   Final old_new: 92.30 ± 0.34   Final new_new: 86.13 ± 1.62
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.06 ± 3.95   Final Test: 31.45 ± 3.76   Final old_old: 34.14 ± 4.07   Final old_new: 20.73 ± 2.56   Final new_new: 5.00 ± 1.90
Hits@20:
  Final val: 58.66 ± 3.84   Final Test: 40.69 ± 6.62   Final old_old: 43.70 ± 7.09   Final old_new: 28.74 ± 4.80   Final new_new: 10.45 ± 4.71
Hits@30:
  Final val: 65.51 ± 3.20   Final Test: 47.36 ± 3.17   Final old_old: 50.80 ± 3.19   Final old_new: 33.73 ± 3.45   Final new_new: 11.82 ± 4.93
Hits@50:
  Final val: 73.84 ± 2.56   Final Test: 58.40 ± 2.60   Final old_old: 62.09 ± 2.45   Final old_new: 43.93 ± 3.38   Final new_new: 17.73 ± 4.66
AUC:
  Final val: 96.71 ± 0.23   Final Test: 96.01 ± 0.28   Final old_old: 96.64 ± 0.19   Final old_new: 93.55 ± 0.77   Final new_new: 88.61 ± 2.29
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.22 ± 4.14   Final Test: 31.74 ± 1.72   Final old_old: 34.86 ± 2.01   Final old_new: 19.14 ± 1.09   Final new_new: 4.55 ± 2.78
Hits@20:
  Final val: 60.06 ± 2.10   Final Test: 43.00 ± 1.13   Final old_old: 46.60 ± 1.17   Final old_new: 28.64 ± 1.79   Final new_new: 8.18 ± 2.59
Hits@30:
  Final val: 67.98 ± 1.68   Final Test: 50.05 ± 2.44   Final old_old: 53.98 ± 2.46   Final old_new: 34.51 ± 2.50   Final new_new: 9.09 ± 3.21
Hits@50:
  Final val: 75.42 ± 1.21   Final Test: 60.60 ± 1.46   Final old_old: 64.83 ± 1.51   Final old_new: 43.88 ± 2.00   Final new_new: 16.82 ± 3.45
AUC:
  Final val: 97.30 ± 0.09   Final Test: 96.75 ± 0.06   Final old_old: 97.35 ± 0.05   Final old_new: 94.40 ± 0.36   Final new_new: 90.44 ± 1.71
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.88 ± 2.97   Final Test: 29.78 ± 2.42   Final old_old: 32.49 ± 2.50   Final old_new: 18.84 ± 2.37   Final new_new: 5.45 ± 2.59
Hits@20:
  Final val: 55.94 ± 1.91   Final Test: 40.16 ± 1.74   Final old_old: 43.32 ± 1.94   Final old_new: 27.53 ± 2.11   Final new_new: 10.45 ± 3.45
Hits@30:
  Final val: 63.76 ± 2.04   Final Test: 44.64 ± 1.36   Final old_old: 48.21 ± 1.54   Final old_new: 30.35 ± 1.50   Final new_new: 11.36 ± 2.27
Hits@50:
  Final val: 72.98 ± 0.93   Final Test: 55.44 ± 2.35   Final old_old: 59.23 ± 2.27   Final old_new: 40.50 ± 3.13   Final new_new: 15.45 ± 4.37
AUC:
  Final val: 96.75 ± 0.23   Final Test: 95.96 ± 0.23   Final old_old: 96.66 ± 0.23   Final old_new: 93.19 ± 0.34   Final new_new: 89.01 ± 1.13
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.40 ± 2.41   Final Test: 28.96 ± 2.83   Final old_old: 32.06 ± 2.95   Final old_new: 16.22 ± 2.72   Final new_new: 5.00 ± 1.90
Hits@20:
  Final val: 59.58 ± 1.08   Final Test: 41.37 ± 1.84   Final old_old: 44.87 ± 1.71   Final old_new: 27.25 ± 2.63   Final new_new: 10.45 ± 4.98
Hits@30:
  Final val: 66.32 ± 0.59   Final Test: 48.87 ± 4.09   Final old_old: 52.52 ± 4.07   Final old_new: 34.26 ± 4.24   Final new_new: 14.09 ± 6.51
Hits@50:
  Final val: 72.59 ± 0.91   Final Test: 59.74 ± 0.89   Final old_old: 63.34 ± 1.01   Final old_new: 45.52 ± 0.57   Final new_new: 22.73 ± 4.55
AUC:
  Final val: 95.52 ± 0.18   Final Test: 95.12 ± 0.12   Final old_old: 95.85 ± 0.13   Final old_new: 92.25 ± 0.39   Final new_new: 86.79 ± 2.21
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.01 ± 3.80   Final Test: 32.30 ± 4.86   Final old_old: 35.30 ± 5.02   Final old_new: 20.25 ± 4.75   Final new_new: 4.09 ± 3.73
Hits@20:
  Final val: 58.24 ± 3.50   Final Test: 41.53 ± 2.76   Final old_old: 45.05 ± 3.07   Final old_new: 27.51 ± 1.89   Final new_new: 6.82 ± 2.27
Hits@30:
  Final val: 66.34 ± 2.83   Final Test: 52.31 ± 3.08   Final old_old: 55.98 ± 3.60   Final old_new: 37.88 ± 1.41   Final new_new: 12.73 ± 6.93
Hits@50:
  Final val: 74.94 ± 1.91   Final Test: 59.66 ± 1.38   Final old_old: 63.69 ± 1.51   Final old_new: 43.80 ± 1.33   Final new_new: 16.82 ± 7.29
AUC:
  Final val: 97.27 ± 0.15   Final Test: 96.75 ± 0.12   Final old_old: 97.37 ± 0.12   Final old_new: 94.29 ± 0.14   Final new_new: 90.68 ± 0.77
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.78 ± 5.21   Final Test: 29.03 ± 3.85   Final old_old: 31.59 ± 4.01   Final old_new: 18.61 ± 3.40   Final new_new: 8.18 ± 3.45
Hits@20:
  Final val: 56.17 ± 4.89   Final Test: 38.75 ± 5.08   Final old_old: 41.73 ± 5.28   Final old_new: 26.70 ± 4.55   Final new_new: 12.73 ± 4.98
Hits@30:
  Final val: 64.93 ± 4.70   Final Test: 46.93 ± 3.64   Final old_old: 50.36 ± 3.81   Final old_new: 33.32 ± 3.12   Final new_new: 12.27 ± 6.35
Hits@50:
  Final val: 73.75 ± 2.74   Final Test: 55.34 ± 4.97   Final old_old: 59.16 ± 5.31   Final old_new: 40.10 ± 4.03   Final new_new: 18.18 ± 4.25
AUC:
  Final val: 96.70 ± 0.22   Final Test: 95.98 ± 0.15   Final old_old: 96.63 ± 0.19   Final old_new: 93.43 ± 0.10   Final new_new: 88.84 ± 1.30
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.53 ± 5.38   Final Test: 26.04 ± 5.98   Final old_old: 28.52 ± 6.64   Final old_new: 15.92 ± 3.38   Final new_new: 6.36 ± 4.93
Hits@20:
  Final val: 53.75 ± 3.03   Final Test: 39.61 ± 5.58   Final old_old: 42.87 ± 5.91   Final old_new: 26.47 ± 4.46   Final new_new: 10.45 ± 2.59
Hits@30:
  Final val: 60.72 ± 2.33   Final Test: 48.68 ± 3.29   Final old_old: 52.03 ± 3.38   Final old_new: 35.29 ± 2.96   Final new_new: 15.91 ± 4.55
Hits@50:
  Final val: 69.37 ± 1.30   Final Test: 56.31 ± 3.11   Final old_old: 59.81 ± 2.94   Final old_new: 42.42 ± 4.29   Final new_new: 20.91 ± 3.73
AUC:
  Final val: 94.78 ± 0.31   Final Test: 94.33 ± 0.42   Final old_old: 95.08 ± 0.37   Final old_new: 91.46 ± 1.02   Final new_new: 84.88 ± 2.29
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.34 ± 2.90   Final Test: 31.33 ± 3.36   Final old_old: 34.26 ± 3.48   Final old_new: 19.60 ± 3.07   Final new_new: 4.55 ± 3.59
Hits@20:
  Final val: 64.76 ± 2.50   Final Test: 43.28 ± 5.39   Final old_old: 47.04 ± 5.80   Final old_new: 28.24 ± 3.76   Final new_new: 7.73 ± 3.45
Hits@30:
  Final val: 70.57 ± 1.51   Final Test: 52.55 ± 3.79   Final old_old: 56.61 ± 4.26   Final old_new: 36.42 ± 2.27   Final new_new: 11.82 ± 1.02
Hits@50:
  Final val: 78.44 ± 0.71   Final Test: 63.61 ± 3.43   Final old_old: 67.87 ± 3.83   Final old_new: 46.78 ± 2.02   Final new_new: 20.00 ± 3.73
AUC:
  Final val: 97.45 ± 0.15   Final Test: 96.92 ± 0.06   Final old_old: 97.52 ± 0.11   Final old_new: 94.56 ± 0.23   Final new_new: 90.71 ± 1.45
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.23 ± 4.61   Final Test: 30.37 ± 4.80   Final old_old: 33.48 ± 5.11   Final old_new: 17.76 ± 3.73   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 59.32 ± 1.32   Final Test: 40.41 ± 1.78   Final old_old: 44.00 ± 1.76   Final old_new: 25.94 ± 2.01   Final new_new: 8.18 ± 1.24
Hits@30:
  Final val: 64.33 ± 0.74   Final Test: 48.76 ± 2.98   Final old_old: 52.44 ± 3.02   Final old_new: 34.21 ± 2.86   Final new_new: 10.91 ± 2.49
Hits@50:
  Final val: 70.90 ± 0.69   Final Test: 59.55 ± 1.56   Final old_old: 63.13 ± 1.54   Final old_new: 45.62 ± 2.03   Final new_new: 18.64 ± 1.90
AUC:
  Final val: 94.88 ± 0.28   Final Test: 94.60 ± 0.21   Final old_old: 95.26 ± 0.25   Final old_new: 92.04 ± 0.18   Final new_new: 87.10 ± 1.24
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.14 ± 2.93   Final Test: 35.92 ± 4.44   Final old_old: 39.24 ± 4.49   Final old_new: 22.52 ± 4.43   Final new_new: 6.36 ± 5.43
Hits@20:
  Final val: 59.22 ± 1.75   Final Test: 41.11 ± 2.70   Final old_old: 44.66 ± 2.86   Final old_new: 26.95 ± 2.17   Final new_new: 6.82 ± 5.79
Hits@30:
  Final val: 66.40 ± 1.92   Final Test: 47.50 ± 2.62   Final old_old: 51.43 ± 2.57   Final old_new: 31.94 ± 2.79   Final new_new: 7.27 ± 5.18
Hits@50:
  Final val: 73.84 ± 1.31   Final Test: 59.21 ± 1.98   Final old_old: 63.50 ± 2.25   Final old_new: 42.27 ± 1.27   Final new_new: 14.09 ± 5.18
AUC:
  Final val: 97.23 ± 0.06   Final Test: 96.79 ± 0.03   Final old_old: 97.41 ± 0.04   Final old_new: 94.34 ± 0.19   Final new_new: 90.35 ± 0.44
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.00 ± 1.62   Final Test: 30.74 ± 2.73   Final old_old: 33.95 ± 2.88   Final old_new: 17.63 ± 2.23   Final new_new: 5.00 ± 2.96
Hits@20:
  Final val: 59.17 ± 2.19   Final Test: 40.73 ± 2.87   Final old_old: 44.45 ± 3.02   Final old_new: 25.72 ± 2.67   Final new_new: 7.73 ± 4.98
Hits@30:
  Final val: 65.95 ± 1.82   Final Test: 47.15 ± 4.07   Final old_old: 51.09 ± 4.45   Final old_new: 31.31 ± 2.68   Final new_new: 10.91 ± 4.66
Hits@50:
  Final val: 74.08 ± 1.26   Final Test: 57.56 ± 1.96   Final old_old: 61.71 ± 2.17   Final old_new: 40.98 ± 1.98   Final new_new: 17.73 ± 3.37
AUC:
  Final val: 97.19 ± 0.10   Final Test: 96.75 ± 0.09   Final old_old: 97.33 ± 0.12   Final old_new: 94.44 ± 0.14   Final new_new: 90.51 ± 1.17
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.96 ± 0.60   Final Test: 33.28 ± 5.30   Final old_old: 36.67 ± 5.71   Final old_new: 19.65 ± 3.83   Final new_new: 2.73 ± 1.02
Hits@20:
  Final val: 62.95 ± 2.54   Final Test: 42.54 ± 5.84   Final old_old: 46.20 ± 6.04   Final old_new: 27.96 ± 5.39   Final new_new: 6.82 ± 2.27
Hits@30:
  Final val: 69.00 ± 1.37   Final Test: 51.38 ± 3.75   Final old_old: 55.45 ± 4.00   Final old_new: 35.31 ± 3.23   Final new_new: 9.09 ± 2.78
Hits@50:
  Final val: 76.99 ± 0.77   Final Test: 62.22 ± 3.24   Final old_old: 66.39 ± 3.25   Final old_new: 45.94 ± 3.56   Final new_new: 15.00 ± 3.80
AUC:
  Final val: 97.39 ± 0.08   Final Test: 96.83 ± 0.06   Final old_old: 97.41 ± 0.06   Final old_new: 94.55 ± 0.33   Final new_new: 90.69 ± 0.30
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.46 ± 2.08   Final Test: 29.22 ± 3.06   Final old_old: 32.57 ± 3.18   Final old_new: 15.44 ± 2.82   Final new_new: 3.64 ± 2.03
Hits@20:
  Final val: 58.91 ± 1.70   Final Test: 43.46 ± 1.40   Final old_old: 47.07 ± 1.40   Final old_new: 29.04 ± 1.44   Final new_new: 8.64 ± 1.02
Hits@30:
  Final val: 64.43 ± 1.11   Final Test: 49.99 ± 2.29   Final old_old: 53.63 ± 2.31   Final old_new: 35.52 ± 2.65   Final new_new: 14.55 ± 3.80
Hits@50:
  Final val: 70.27 ± 0.93   Final Test: 57.35 ± 1.48   Final old_old: 60.94 ± 1.81   Final old_new: 43.32 ± 0.43   Final new_new: 18.18 ± 1.61
AUC:
  Final val: 94.78 ± 0.19   Final Test: 94.33 ± 0.22   Final old_old: 95.11 ± 0.17   Final old_new: 91.34 ± 0.52   Final new_new: 84.17 ± 1.71
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 40.88 ± 4.70   Final Test: 27.46 ± 3.07   Final old_old: 30.13 ± 3.36   Final old_new: 16.60 ± 2.37   Final new_new: 5.45 ± 2.59
Hits@20:
  Final val: 52.89 ± 3.45   Final Test: 32.74 ± 2.02   Final old_old: 35.59 ± 2.15   Final old_new: 21.11 ± 2.36   Final new_new: 9.55 ± 4.37
Hits@30:
  Final val: 61.11 ± 2.50   Final Test: 43.59 ± 4.57   Final old_old: 46.94 ± 4.74   Final old_new: 30.13 ± 4.16   Final new_new: 13.18 ± 4.07
Hits@50:
  Final val: 71.33 ± 2.04   Final Test: 54.39 ± 4.31   Final old_old: 58.26 ± 4.66   Final old_new: 38.97 ± 2.87   Final new_new: 17.27 ± 6.14
AUC:
  Final val: 96.57 ± 0.23   Final Test: 96.02 ± 0.16   Final old_old: 96.65 ± 0.24   Final old_new: 93.47 ± 0.28   Final new_new: 90.36 ± 2.51
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 39.15 ± 4.68   Final Test: 22.85 ± 4.76   Final old_old: 24.95 ± 5.12   Final old_new: 14.36 ± 3.55   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 52.94 ± 2.79   Final Test: 34.08 ± 4.15   Final old_old: 36.85 ± 4.40   Final old_new: 23.00 ± 3.45   Final new_new: 8.18 ± 3.45
Hits@30:
  Final val: 61.36 ± 3.10   Final Test: 40.88 ± 2.73   Final old_old: 44.17 ± 2.93   Final old_new: 27.88 ± 2.14   Final new_new: 6.82 ± 3.21
Hits@50:
  Final val: 70.88 ± 2.04   Final Test: 53.87 ± 3.62   Final old_old: 57.61 ± 3.78   Final old_new: 39.19 ± 3.20   Final new_new: 13.18 ± 2.49
AUC:
  Final val: 96.60 ± 0.21   Final Test: 96.07 ± 0.12   Final old_old: 96.71 ± 0.12   Final old_new: 93.55 ± 0.24   Final new_new: 89.44 ± 1.96
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.95 ± 2.16   Final Test: 30.53 ± 4.61   Final old_old: 33.57 ± 4.79   Final old_new: 18.29 ± 4.21   Final new_new: 4.09 ± 1.02
Hits@20:
  Final val: 58.84 ± 3.02   Final Test: 40.80 ± 1.42   Final old_old: 44.26 ± 1.70   Final old_new: 26.95 ± 1.75   Final new_new: 7.73 ± 3.80
Hits@30:
  Final val: 65.57 ± 2.97   Final Test: 47.24 ± 2.30   Final old_old: 50.97 ± 2.55   Final old_new: 32.37 ± 1.46   Final new_new: 10.91 ± 2.96
Hits@50:
  Final val: 73.72 ± 1.80   Final Test: 55.93 ± 2.29   Final old_old: 60.05 ± 2.57   Final old_new: 39.62 ± 2.25   Final new_new: 14.09 ± 1.90
AUC:
  Final val: 97.24 ± 0.16   Final Test: 96.69 ± 0.17   Final old_old: 97.29 ± 0.20   Final old_new: 94.30 ± 0.33   Final new_new: 90.25 ± 0.90
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.92 ± 5.99   Final Test: 24.44 ± 7.05   Final old_old: 26.69 ± 7.44   Final old_new: 15.39 ± 5.64   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 59.82 ± 3.60   Final Test: 40.64 ± 4.36   Final old_old: 43.85 ± 4.59   Final old_new: 27.91 ± 3.91   Final new_new: 8.18 ± 3.05
Hits@30:
  Final val: 66.31 ± 2.10   Final Test: 48.16 ± 4.43   Final old_old: 51.72 ± 4.51   Final old_new: 34.06 ± 4.37   Final new_new: 11.36 ± 4.25
Hits@50:
  Final val: 74.34 ± 0.92   Final Test: 58.25 ± 3.77   Final old_old: 62.16 ± 3.91   Final old_new: 42.85 ± 3.26   Final new_new: 16.82 ± 6.74
AUC:
  Final val: 96.77 ± 0.13   Final Test: 96.18 ± 0.22   Final old_old: 96.80 ± 0.18   Final old_new: 93.63 ± 0.40   Final new_new: 91.42 ± 0.99
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.38 ± 3.85   Final Test: 35.15 ± 7.81   Final old_old: 38.34 ± 8.33   Final old_new: 22.44 ± 5.91   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 63.88 ± 3.34   Final Test: 44.40 ± 2.48   Final old_old: 48.13 ± 2.56   Final old_new: 29.55 ± 2.22   Final new_new: 7.73 ± 2.59
Hits@30:
  Final val: 70.10 ± 1.74   Final Test: 51.40 ± 1.28   Final old_old: 55.30 ± 1.44   Final old_new: 35.92 ± 1.38   Final new_new: 12.73 ± 4.13
Hits@50:
  Final val: 76.79 ± 1.28   Final Test: 60.45 ± 2.48   Final old_old: 64.61 ± 2.46   Final old_new: 44.08 ± 2.78   Final new_new: 15.91 ± 4.55
AUC:
  Final val: 97.38 ± 0.10   Final Test: 96.85 ± 0.14   Final old_old: 97.45 ± 0.11   Final old_new: 94.49 ± 0.36   Final new_new: 90.34 ± 0.76
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.19 ± 1.43   Final Test: 31.80 ± 3.15   Final old_old: 34.83 ± 3.14   Final old_new: 19.60 ± 3.40   Final new_new: 4.09 ± 2.96
Hits@20:
  Final val: 62.91 ± 2.61   Final Test: 43.02 ± 3.51   Final old_old: 46.76 ± 3.63   Final old_new: 28.11 ± 3.32   Final new_new: 6.82 ± 3.21
Hits@30:
  Final val: 69.11 ± 1.51   Final Test: 50.80 ± 4.31   Final old_old: 54.75 ± 4.43   Final old_new: 35.24 ± 3.99   Final new_new: 8.64 ± 5.88
Hits@50:
  Final val: 77.06 ± 1.57   Final Test: 62.56 ± 1.65   Final old_old: 66.76 ± 1.69   Final old_new: 46.12 ± 1.92   Final new_new: 16.82 ± 4.43
AUC:
  Final val: 97.40 ± 0.08   Final Test: 96.84 ± 0.10   Final old_old: 97.48 ± 0.16   Final old_new: 94.26 ± 0.19   Final new_new: 90.66 ± 1.00
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.24 ± 4.03   Final Test: 30.10 ± 5.96   Final old_old: 32.70 ± 6.48   Final old_new: 19.50 ± 3.86   Final new_new: 8.64 ± 2.49
Hits@20:
  Final val: 57.15 ± 4.31   Final Test: 38.21 ± 2.86   Final old_old: 41.39 ± 3.07   Final old_new: 25.34 ± 2.59   Final new_new: 10.45 ± 2.03
Hits@30:
  Final val: 65.24 ± 2.81   Final Test: 45.94 ± 2.57   Final old_old: 49.49 ± 2.78   Final old_new: 31.71 ± 2.40   Final new_new: 12.73 ± 3.80
Hits@50:
  Final val: 73.05 ± 1.49   Final Test: 55.22 ± 3.29   Final old_old: 59.01 ± 3.40   Final old_new: 40.03 ± 3.31   Final new_new: 19.55 ± 6.93
AUC:
  Final val: 96.65 ± 0.13   Final Test: 95.96 ± 0.19   Final old_old: 96.61 ± 0.24   Final old_new: 93.33 ± 0.33   Final new_new: 90.08 ± 1.43
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.80 ± 2.00   Final Test: 31.34 ± 5.07   Final old_old: 34.26 ± 5.31   Final old_new: 19.72 ± 4.31   Final new_new: 3.18 ± 3.05
Hits@20:
  Final val: 64.70 ± 3.05   Final Test: 43.55 ± 3.21   Final old_old: 47.03 ± 3.31   Final old_new: 29.77 ± 2.84   Final new_new: 8.18 ± 3.80
Hits@30:
  Final val: 70.77 ± 2.17   Final Test: 51.78 ± 2.21   Final old_old: 55.59 ± 2.08   Final old_new: 36.83 ± 2.79   Final new_new: 10.45 ± 3.05
Hits@50:
  Final val: 77.43 ± 1.63   Final Test: 61.10 ± 2.17   Final old_old: 64.90 ± 2.22   Final old_new: 46.37 ± 2.28   Final new_new: 16.36 ± 4.93
AUC:
  Final val: 97.29 ± 0.07   Final Test: 96.84 ± 0.12   Final old_old: 97.31 ± 0.09   Final old_new: 95.02 ± 0.28   Final new_new: 91.37 ± 0.68
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.81 ± 2.35   Final Test: 31.79 ± 1.26   Final old_old: 34.88 ± 1.30   Final old_new: 19.32 ± 1.27   Final new_new: 4.09 ± 2.96
Hits@20:
  Final val: 59.62 ± 1.09   Final Test: 40.64 ± 1.79   Final old_old: 44.01 ± 1.82   Final old_new: 27.28 ± 1.80   Final new_new: 6.36 ± 3.37
Hits@30:
  Final val: 66.09 ± 2.03   Final Test: 47.60 ± 3.02   Final old_old: 51.37 ± 3.10   Final old_new: 32.75 ± 2.98   Final new_new: 7.27 ± 3.73
Hits@50:
  Final val: 74.21 ± 1.34   Final Test: 60.05 ± 1.91   Final old_old: 64.19 ± 2.15   Final old_new: 43.83 ± 1.32   Final new_new: 15.45 ± 5.66
AUC:
  Final val: 97.22 ± 0.08   Final Test: 96.80 ± 0.07   Final old_old: 97.39 ± 0.08   Final old_new: 94.50 ± 0.20   Final new_new: 90.34 ± 0.93
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.11 ± 2.20   Final Test: 33.59 ± 3.40   Final old_old: 36.73 ± 3.46   Final old_new: 21.01 ± 3.36   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 60.74 ± 2.97   Final Test: 41.68 ± 1.95   Final old_old: 45.32 ± 2.06   Final old_new: 27.18 ± 1.71   Final new_new: 6.82 ± 2.27
Hits@30:
  Final val: 67.38 ± 1.90   Final Test: 48.17 ± 2.38   Final old_old: 52.24 ± 2.52   Final old_new: 31.96 ± 1.98   Final new_new: 7.73 ± 3.05
Hits@50:
  Final val: 74.86 ± 1.27   Final Test: 56.28 ± 3.67   Final old_old: 60.43 ± 3.90   Final old_new: 39.87 ± 3.05   Final new_new: 12.73 ± 3.05
AUC:
  Final val: 97.30 ± 0.11   Final Test: 96.84 ± 0.10   Final old_old: 97.42 ± 0.11   Final old_new: 94.55 ± 0.17   Final new_new: 91.03 ± 0.51
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.78 ± 5.72   Final Test: 31.90 ± 6.49   Final old_old: 35.02 ± 6.82   Final old_new: 19.27 ± 5.26   Final new_new: 5.00 ± 2.49
Hits@20:
  Final val: 61.57 ± 1.31   Final Test: 43.75 ± 6.12   Final old_old: 47.27 ± 6.45   Final old_new: 29.80 ± 5.00   Final new_new: 8.18 ± 2.59
Hits@30:
  Final val: 68.23 ± 1.41   Final Test: 51.93 ± 4.35   Final old_old: 55.80 ± 4.42   Final old_new: 36.62 ± 4.41   Final new_new: 11.36 ± 4.25
Hits@50:
  Final val: 76.00 ± 1.87   Final Test: 61.62 ± 3.38   Final old_old: 65.62 ± 3.47   Final old_new: 46.02 ± 3.21   Final new_new: 16.36 ± 4.07
AUC:
  Final val: 97.31 ± 0.16   Final Test: 96.80 ± 0.20   Final old_old: 97.38 ± 0.22   Final old_new: 94.53 ± 0.20   Final new_new: 90.18 ± 0.97
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.03 ± 2.51   Final Test: 32.89 ± 3.78   Final old_old: 36.01 ± 4.02   Final old_new: 20.40 ± 2.92   Final new_new: 3.64 ± 2.03
Hits@20:
  Final val: 62.56 ± 2.77   Final Test: 40.28 ± 2.23   Final old_old: 43.54 ± 2.19   Final old_new: 27.33 ± 2.57   Final new_new: 8.18 ± 2.03
Hits@30:
  Final val: 68.69 ± 1.72   Final Test: 48.30 ± 1.39   Final old_old: 51.92 ± 1.46   Final old_new: 34.06 ± 1.14   Final new_new: 10.00 ± 2.59
Hits@50:
  Final val: 75.96 ± 0.59   Final Test: 59.54 ± 2.46   Final old_old: 63.21 ± 2.63   Final old_new: 45.39 ± 1.91   Final new_new: 15.00 ± 4.98
AUC:
  Final val: 97.37 ± 0.09   Final Test: 96.98 ± 0.09   Final old_old: 97.45 ± 0.10   Final old_new: 95.18 ± 0.10   Final new_new: 91.62 ± 0.25
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.35 ± 3.47   Final Test: 29.00 ± 5.73   Final old_old: 31.77 ± 6.04   Final old_new: 17.68 ± 4.59   Final new_new: 7.27 ± 3.73
Hits@20:
  Final val: 58.08 ± 3.91   Final Test: 38.31 ± 2.61   Final old_old: 41.43 ± 2.71   Final old_new: 25.72 ± 2.47   Final new_new: 10.45 ± 2.03
Hits@30:
  Final val: 64.90 ± 2.94   Final Test: 44.84 ± 3.19   Final old_old: 48.28 ± 3.37   Final old_new: 30.98 ± 2.59   Final new_new: 14.55 ± 2.03
Hits@50:
  Final val: 73.08 ± 1.95   Final Test: 56.49 ± 5.41   Final old_old: 60.32 ± 5.40   Final old_new: 41.34 ± 5.66   Final new_new: 17.73 ± 3.73
AUC:
  Final val: 96.65 ± 0.22   Final Test: 95.96 ± 0.17   Final old_old: 96.62 ± 0.19   Final old_new: 93.31 ± 0.31   Final new_new: 89.46 ± 1.74
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.86 ± 2.14   Final Test: 30.81 ± 4.15   Final old_old: 34.04 ± 4.27   Final old_new: 17.66 ± 3.78   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 59.88 ± 2.96   Final Test: 40.34 ± 2.86   Final old_old: 44.02 ± 3.02   Final old_new: 25.67 ± 2.45   Final new_new: 4.55 ± 1.61
Hits@30:
  Final val: 66.19 ± 2.09   Final Test: 47.47 ± 4.08   Final old_old: 51.38 ± 4.47   Final old_new: 32.07 ± 2.75   Final new_new: 6.36 ± 1.02
Hits@50:
  Final val: 73.77 ± 0.77   Final Test: 60.36 ± 2.26   Final old_old: 64.71 ± 2.33   Final old_new: 43.20 ± 2.26   Final new_new: 14.09 ± 2.96
AUC:
  Final val: 97.21 ± 0.10   Final Test: 96.75 ± 0.15   Final old_old: 97.35 ± 0.16   Final old_new: 94.41 ± 0.30   Final new_new: 90.42 ± 1.00
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.66 ± 6.80   Final Test: 28.86 ± 5.93   Final old_old: 31.52 ± 6.54   Final old_new: 18.01 ± 3.56   Final new_new: 7.73 ± 3.05
Hits@20:
  Final val: 57.36 ± 4.81   Final Test: 36.07 ± 6.26   Final old_old: 39.11 ± 7.06   Final old_new: 23.85 ± 3.18   Final new_new: 8.18 ± 3.05
Hits@30:
  Final val: 63.82 ± 3.42   Final Test: 44.23 ± 5.65   Final old_old: 47.59 ± 6.37   Final old_new: 30.76 ± 3.08   Final new_new: 12.73 ± 2.59
Hits@50:
  Final val: 72.80 ± 1.80   Final Test: 58.89 ± 1.88   Final old_old: 62.83 ± 2.14   Final old_new: 43.25 ± 2.04   Final new_new: 19.55 ± 6.14
AUC:
  Final val: 96.58 ± 0.26   Final Test: 95.98 ± 0.27   Final old_old: 96.66 ± 0.27   Final old_new: 93.28 ± 0.42   Final new_new: 89.34 ± 1.92
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.56 ± 3.46   Final Test: 35.42 ± 5.36   Final old_old: 38.52 ± 5.94   Final old_new: 22.82 ± 3.01   Final new_new: 9.55 ± 3.73
Hits@20:
  Final val: 55.21 ± 2.16   Final Test: 39.63 ± 3.64   Final old_old: 42.88 ± 3.96   Final old_new: 26.40 ± 2.56   Final new_new: 12.73 ± 2.03
Hits@30:
  Final val: 64.04 ± 2.14   Final Test: 49.01 ± 2.53   Final old_old: 52.62 ± 2.77   Final old_new: 34.66 ± 1.96   Final new_new: 13.18 ± 5.66
Hits@50:
  Final val: 73.84 ± 2.09   Final Test: 58.50 ± 1.67   Final old_old: 62.20 ± 1.95   Final old_new: 43.60 ± 0.97   Final new_new: 25.00 ± 3.21
AUC:
  Final val: 96.78 ± 0.13   Final Test: 96.22 ± 0.17   Final old_old: 96.85 ± 0.22   Final old_new: 93.71 ± 0.29   Final new_new: 90.20 ± 1.19
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.97 ± 2.34   Final Test: 33.83 ± 2.63   Final old_old: 37.11 ± 2.90   Final old_new: 20.65 ± 1.95   Final new_new: 3.64 ± 2.03
Hits@20:
  Final val: 63.18 ± 1.31   Final Test: 41.14 ± 3.47   Final old_old: 44.78 ± 3.69   Final old_new: 26.62 ± 3.07   Final new_new: 5.91 ± 1.24
Hits@30:
  Final val: 69.94 ± 1.24   Final Test: 49.26 ± 2.38   Final old_old: 53.30 ± 2.50   Final old_new: 33.12 ± 1.97   Final new_new: 10.45 ± 4.71
Hits@50:
  Final val: 76.78 ± 1.35   Final Test: 60.89 ± 1.83   Final old_old: 65.00 ± 2.13   Final old_new: 44.79 ± 1.07   Final new_new: 15.45 ± 4.37
AUC:
  Final val: 97.33 ± 0.14   Final Test: 96.80 ± 0.13   Final old_old: 97.39 ± 0.15   Final old_new: 94.47 ± 0.16   Final new_new: 90.56 ± 1.26
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.64 ± 6.15   Final Test: 24.53 ± 3.77   Final old_old: 26.73 ± 4.10   Final old_new: 15.57 ± 2.98   Final new_new: 6.82 ± 5.57
Hits@20:
  Final val: 53.86 ± 5.54   Final Test: 34.45 ± 3.83   Final old_old: 37.38 ± 4.12   Final old_new: 22.59 ± 2.84   Final new_new: 9.55 ± 4.37
Hits@30:
  Final val: 61.44 ± 4.04   Final Test: 42.88 ± 2.26   Final old_old: 46.08 ± 2.36   Final old_new: 30.23 ± 1.83   Final new_new: 10.00 ± 6.55
Hits@50:
  Final val: 70.67 ± 2.34   Final Test: 54.52 ± 3.19   Final old_old: 58.11 ± 3.24   Final old_new: 40.38 ± 3.00   Final new_new: 15.91 ± 6.63
AUC:
  Final val: 96.61 ± 0.28   Final Test: 95.98 ± 0.17   Final old_old: 96.64 ± 0.23   Final old_new: 93.36 ± 0.15   Final new_new: 90.00 ± 1.35
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.55 ± 3.38   Final Test: 32.71 ± 1.91   Final old_old: 35.91 ± 2.09   Final old_new: 19.82 ± 1.43   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 61.94 ± 3.06   Final Test: 42.55 ± 3.54   Final old_old: 46.23 ± 3.88   Final old_new: 27.88 ± 2.22   Final new_new: 6.82 ± 4.55
Hits@30:
  Final val: 68.40 ± 2.39   Final Test: 48.86 ± 5.84   Final old_old: 52.84 ± 6.23   Final old_new: 33.12 ± 4.41   Final new_new: 7.73 ± 4.43
Hits@50:
  Final val: 75.78 ± 2.66   Final Test: 61.56 ± 3.82   Final old_old: 65.66 ± 4.01   Final old_new: 45.69 ± 3.13   Final new_new: 13.64 ± 5.33
AUC:
  Final val: 97.27 ± 0.13   Final Test: 96.79 ± 0.14   Final old_old: 97.41 ± 0.14   Final old_new: 94.30 ± 0.23   Final new_new: 90.65 ± 0.29
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.62 ± 2.63   Final Test: 33.71 ± 2.87   Final old_old: 36.97 ± 3.14   Final old_new: 20.65 ± 2.02   Final new_new: 3.64 ± 3.45
Hits@20:
  Final val: 65.53 ± 2.55   Final Test: 45.45 ± 3.35   Final old_old: 49.31 ± 3.44   Final old_new: 30.05 ± 3.25   Final new_new: 8.64 ± 2.96
Hits@30:
  Final val: 71.72 ± 2.00   Final Test: 52.96 ± 2.06   Final old_old: 57.19 ± 2.19   Final old_new: 36.12 ± 1.44   Final new_new: 11.36 ± 4.25
Hits@50:
  Final val: 78.05 ± 1.15   Final Test: 64.70 ± 1.59   Final old_old: 69.08 ± 1.57   Final old_new: 47.48 ± 2.08   Final new_new: 18.18 ± 5.57
AUC:
  Final val: 97.56 ± 0.11   Final Test: 97.03 ± 0.12   Final old_old: 97.59 ± 0.13   Final old_new: 94.81 ± 0.13   Final new_new: 91.66 ± 0.48
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.41 ± 3.14   Final Test: 35.03 ± 3.76   Final old_old: 38.35 ± 4.09   Final old_new: 21.61 ± 2.61   Final new_new: 5.45 ± 3.45
Hits@20:
  Final val: 59.63 ± 1.48   Final Test: 42.09 ± 3.65   Final old_old: 45.80 ± 3.97   Final old_new: 27.10 ± 2.30   Final new_new: 10.00 ± 6.14
Hits@30:
  Final val: 66.19 ± 0.94   Final Test: 48.15 ± 5.06   Final old_old: 52.07 ± 5.49   Final old_new: 32.47 ± 3.10   Final new_new: 10.91 ± 5.88
Hits@50:
  Final val: 73.51 ± 0.44   Final Test: 56.43 ± 1.72   Final old_old: 60.49 ± 1.97   Final old_new: 40.35 ± 0.80   Final new_new: 15.45 ± 7.08
AUC:
  Final val: 97.21 ± 0.09   Final Test: 96.69 ± 0.07   Final old_old: 97.27 ± 0.14   Final old_new: 94.36 ± 0.20   Final new_new: 90.52 ± 0.96
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.44 ± 3.43   Final Test: 35.17 ± 2.43   Final old_old: 38.44 ± 2.62   Final old_new: 22.07 ± 2.16   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 61.22 ± 2.62   Final Test: 41.24 ± 2.30   Final old_old: 44.86 ± 2.59   Final old_new: 26.78 ± 1.30   Final new_new: 6.82 ± 1.61
Hits@30:
  Final val: 68.81 ± 0.86   Final Test: 49.31 ± 2.30   Final old_old: 53.21 ± 2.27   Final old_new: 33.90 ± 2.72   Final new_new: 9.09 ± 2.78
Hits@50:
  Final val: 76.45 ± 1.33   Final Test: 59.79 ± 1.85   Final old_old: 64.01 ± 1.86   Final old_new: 43.12 ± 2.34   Final new_new: 15.91 ± 5.57
AUC:
  Final val: 97.34 ± 0.18   Final Test: 96.83 ± 0.12   Final old_old: 97.46 ± 0.14   Final old_new: 94.30 ± 0.26   Final new_new: 90.82 ± 0.82
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.23 ± 0.74   Final Test: 29.54 ± 3.14   Final old_old: 32.51 ± 3.12   Final old_new: 17.58 ± 3.51   Final new_new: 3.18 ± 1.24
Hits@20:
  Final val: 58.42 ± 2.13   Final Test: 40.05 ± 4.34   Final old_old: 43.43 ± 4.21   Final old_new: 26.50 ± 5.12   Final new_new: 7.73 ± 3.05
Hits@30:
  Final val: 65.86 ± 1.26   Final Test: 48.63 ± 2.12   Final old_old: 52.39 ± 2.52   Final old_new: 33.55 ± 1.07   Final new_new: 13.18 ± 2.49
Hits@50:
  Final val: 73.65 ± 1.28   Final Test: 57.91 ± 2.04   Final old_old: 62.02 ± 2.27   Final old_new: 41.66 ± 1.58   Final new_new: 15.45 ± 4.37
AUC:
  Final val: 97.24 ± 0.10   Final Test: 96.73 ± 0.07   Final old_old: 97.29 ± 0.09   Final old_new: 94.51 ± 0.31   Final new_new: 91.02 ± 0.62
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.64 ± 5.03   Final Test: 30.77 ± 5.57   Final old_old: 33.83 ± 5.84   Final old_new: 18.34 ± 4.85   Final new_new: 5.45 ± 2.03
Hits@20:
  Final val: 61.08 ± 4.87   Final Test: 42.65 ± 5.20   Final old_old: 46.51 ± 5.63   Final old_new: 27.15 ± 3.52   Final new_new: 6.36 ± 2.96
Hits@30:
  Final val: 67.89 ± 2.94   Final Test: 49.79 ± 4.50   Final old_old: 54.01 ± 4.73   Final old_new: 33.02 ± 3.75   Final new_new: 7.73 ± 3.05
Hits@50:
  Final val: 75.18 ± 1.82   Final Test: 59.95 ± 2.46   Final old_old: 64.27 ± 2.58   Final old_new: 43.05 ± 2.54   Final new_new: 11.82 ± 2.96
AUC:
  Final val: 97.40 ± 0.14   Final Test: 96.96 ± 0.18   Final old_old: 97.51 ± 0.18   Final old_new: 94.80 ± 0.21   Final new_new: 91.18 ± 0.67
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.19 ± 4.07   Final Test: 31.50 ± 5.77   Final old_old: 34.09 ± 6.09   Final old_new: 21.03 ± 4.84   Final new_new: 8.18 ± 4.13
Hits@20:
  Final val: 53.88 ± 3.63   Final Test: 39.84 ± 5.56   Final old_old: 42.87 ± 5.84   Final old_new: 27.81 ± 4.89   Final new_new: 10.00 ± 5.23
Hits@30:
  Final val: 61.13 ± 2.79   Final Test: 46.16 ± 6.50   Final old_old: 49.71 ± 6.83   Final old_new: 31.96 ± 5.51   Final new_new: 12.27 ± 5.47
Hits@50:
  Final val: 70.90 ± 2.12   Final Test: 53.30 ± 5.36   Final old_old: 56.72 ± 5.48   Final old_new: 39.67 ± 5.11   Final new_new: 19.55 ± 3.45
AUC:
  Final val: 96.65 ± 0.24   Final Test: 95.91 ± 0.30   Final old_old: 96.54 ± 0.32   Final old_new: 93.39 ± 0.27   Final new_new: 89.65 ± 1.55
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.43 ± 2.19   Final Test: 32.64 ± 2.65   Final old_old: 35.84 ± 2.80   Final old_new: 19.80 ± 2.31   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 60.41 ± 2.38   Final Test: 39.43 ± 3.35   Final old_old: 43.06 ± 3.69   Final old_new: 24.91 ± 2.11   Final new_new: 4.55 ± 1.61
Hits@30:
  Final val: 67.25 ± 1.92   Final Test: 48.30 ± 4.36   Final old_old: 52.23 ± 4.70   Final old_new: 32.75 ± 3.08   Final new_new: 7.27 ± 1.90
Hits@50:
  Final val: 74.70 ± 1.51   Final Test: 58.03 ± 1.79   Final old_old: 62.53 ± 1.78   Final old_new: 40.23 ± 2.37   Final new_new: 11.82 ± 2.96
AUC:
  Final val: 97.28 ± 0.07   Final Test: 96.74 ± 0.08   Final old_old: 97.37 ± 0.09   Final old_new: 94.33 ± 0.20   Final new_new: 89.56 ± 0.52
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.09 ± 5.02   Final Test: 30.28 ± 1.66   Final old_old: 32.89 ± 1.66   Final old_new: 19.62 ± 2.35   Final new_new: 9.09 ± 6.22
Hits@20:
  Final val: 54.77 ± 4.78   Final Test: 39.02 ± 5.45   Final old_old: 42.11 ± 5.95   Final old_new: 26.60 ± 3.92   Final new_new: 10.45 ± 6.14
Hits@30:
  Final val: 62.80 ± 3.29   Final Test: 45.56 ± 2.95   Final old_old: 49.02 ± 3.00   Final old_new: 31.64 ± 2.98   Final new_new: 14.09 ± 7.43
Hits@50:
  Final val: 71.36 ± 2.16   Final Test: 55.96 ± 5.01   Final old_old: 59.54 ± 5.37   Final old_new: 41.76 ± 4.22   Final new_new: 20.45 ± 6.63
AUC:
  Final val: 96.65 ± 0.35   Final Test: 95.95 ± 0.34   Final old_old: 96.61 ± 0.35   Final old_new: 93.29 ± 0.43   Final new_new: 90.06 ± 1.37
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.67 ± 5.44   Final Test: 29.04 ± 7.01   Final old_old: 31.75 ± 7.40   Final old_new: 18.01 ± 5.68   Final new_new: 7.27 ± 2.49
Hits@20:
  Final val: 59.26 ± 2.92   Final Test: 39.29 ± 4.38   Final old_old: 42.74 ± 4.62   Final old_new: 25.39 ± 3.63   Final new_new: 8.18 ± 4.43
Hits@30:
  Final val: 66.41 ± 2.27   Final Test: 46.12 ± 2.27   Final old_old: 49.83 ± 2.27   Final old_new: 31.21 ± 2.61   Final new_new: 11.82 ± 3.37
Hits@50:
  Final val: 74.67 ± 1.01   Final Test: 58.64 ± 3.79   Final old_old: 62.58 ± 3.88   Final old_new: 43.00 ± 3.63   Final new_new: 19.09 ± 4.71
AUC:
  Final val: 96.80 ± 0.16   Final Test: 96.13 ± 0.16   Final old_old: 96.79 ± 0.15   Final old_new: 93.49 ± 0.24   Final new_new: 89.67 ± 1.63
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.89 ± 2.14   Final Test: 31.55 ± 3.56   Final old_old: 34.70 ± 3.76   Final old_new: 18.82 ± 2.88   Final new_new: 5.00 ± 1.90
Hits@20:
  Final val: 61.25 ± 1.22   Final Test: 43.29 ± 4.71   Final old_old: 46.95 ± 5.02   Final old_new: 28.77 ± 3.79   Final new_new: 6.82 ± 2.27
Hits@30:
  Final val: 68.91 ± 0.93   Final Test: 49.92 ± 5.14   Final old_old: 53.79 ± 5.35   Final old_new: 34.71 ± 4.66   Final new_new: 7.73 ± 2.03
Hits@50:
  Final val: 76.83 ± 0.95   Final Test: 62.73 ± 2.71   Final old_old: 66.88 ± 2.72   Final old_new: 46.45 ± 3.55   Final new_new: 17.27 ± 5.23
AUC:
  Final val: 97.35 ± 0.08   Final Test: 96.77 ± 0.10   Final old_old: 97.43 ± 0.07   Final old_new: 94.18 ± 0.31   Final new_new: 89.91 ± 0.86
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.24 ± 3.56   Final Test: 29.00 ± 3.76   Final old_old: 31.61 ± 3.92   Final old_new: 18.31 ± 3.09   Final new_new: 9.55 ± 4.37
Hits@20:
  Final val: 53.90 ± 2.38   Final Test: 34.75 ± 5.10   Final old_old: 37.54 ± 5.38   Final old_new: 23.38 ± 4.10   Final new_new: 11.82 ± 4.37
Hits@30:
  Final val: 62.02 ± 1.93   Final Test: 42.29 ± 4.51   Final old_old: 45.53 ± 4.65   Final old_new: 29.12 ± 4.06   Final new_new: 15.45 ± 4.66
Hits@50:
  Final val: 72.19 ± 1.73   Final Test: 55.41 ± 2.16   Final old_old: 58.99 ± 2.30   Final old_new: 41.23 ± 1.93   Final new_new: 19.55 ± 8.29
AUC:
  Final val: 96.64 ± 0.29   Final Test: 95.87 ± 0.25   Final old_old: 96.55 ± 0.25   Final old_new: 93.12 ± 0.40   Final new_new: 90.08 ± 1.35
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.69 ± 4.82   Final Test: 34.08 ± 2.79   Final old_old: 37.36 ± 3.05   Final old_new: 20.93 ± 1.79   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 63.51 ± 2.18   Final Test: 43.35 ± 1.70   Final old_old: 47.03 ± 1.66   Final old_new: 28.82 ± 2.04   Final new_new: 5.00 ± 3.37
Hits@30:
  Final val: 69.81 ± 1.55   Final Test: 50.82 ± 1.45   Final old_old: 54.91 ± 1.55   Final old_new: 34.63 ± 1.92   Final new_new: 9.55 ± 3.37
Hits@50:
  Final val: 76.88 ± 1.61   Final Test: 60.90 ± 1.77   Final old_old: 65.18 ± 2.09   Final old_new: 44.06 ± 0.79   Final new_new: 15.45 ± 5.18
AUC:
  Final val: 97.36 ± 0.21   Final Test: 96.71 ± 0.16   Final old_old: 97.40 ± 0.23   Final old_new: 93.98 ± 0.25   Final new_new: 89.28 ± 1.96
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.68 ± 1.68   Final Test: 33.49 ± 3.62   Final old_old: 36.85 ± 3.65   Final old_new: 19.85 ± 3.68   Final new_new: 5.00 ± 3.37
Hits@20:
  Final val: 66.40 ± 2.41   Final Test: 44.10 ± 5.16   Final old_old: 47.78 ± 5.40   Final old_new: 29.52 ± 4.51   Final new_new: 6.82 ± 3.59
Hits@30:
  Final val: 72.44 ± 1.55   Final Test: 51.92 ± 3.96   Final old_old: 56.06 ± 4.21   Final old_new: 35.57 ± 3.21   Final new_new: 8.64 ± 3.37
Hits@50:
  Final val: 78.36 ± 0.99   Final Test: 63.37 ± 2.18   Final old_old: 67.71 ± 2.23   Final old_new: 46.42 ± 2.53   Final new_new: 15.00 ± 5.70
AUC:
  Final val: 97.58 ± 0.11   Final Test: 97.06 ± 0.09   Final old_old: 97.62 ± 0.10   Final old_new: 94.85 ± 0.18   Final new_new: 91.11 ± 0.84
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.07 ± 2.41   Final Test: 30.13 ± 4.56   Final old_old: 33.13 ± 4.80   Final old_new: 17.91 ± 3.67   Final new_new: 5.91 ± 2.03
Hits@20:
  Final val: 59.21 ± 1.41   Final Test: 39.53 ± 4.22   Final old_old: 43.10 ± 4.53   Final old_new: 25.21 ± 3.26   Final new_new: 6.36 ± 2.96
Hits@30:
  Final val: 66.53 ± 2.08   Final Test: 48.35 ± 2.51   Final old_old: 52.26 ± 2.78   Final old_new: 32.75 ± 2.15   Final new_new: 10.91 ± 4.37
Hits@50:
  Final val: 74.43 ± 1.77   Final Test: 58.98 ± 1.95   Final old_old: 63.38 ± 1.94   Final old_new: 41.49 ± 2.65   Final new_new: 15.00 ± 2.03
AUC:
  Final val: 97.26 ± 0.14   Final Test: 96.76 ± 0.04   Final old_old: 97.36 ± 0.10   Final old_new: 94.40 ± 0.21   Final new_new: 90.68 ± 1.33
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.38 ± 2.24   Final Test: 33.81 ± 3.74   Final old_old: 37.02 ± 4.00   Final old_new: 20.83 ± 3.11   Final new_new: 5.91 ± 3.45
Hits@20:
  Final val: 62.18 ± 1.18   Final Test: 45.29 ± 4.12   Final old_old: 49.15 ± 4.31   Final old_new: 29.97 ± 3.66   Final new_new: 6.36 ± 2.49
Hits@30:
  Final val: 68.48 ± 1.16   Final Test: 51.65 ± 2.30   Final old_old: 55.90 ± 2.24   Final old_new: 34.86 ± 2.82   Final new_new: 7.73 ± 4.13
Hits@50:
  Final val: 76.95 ± 1.21   Final Test: 62.65 ± 2.71   Final old_old: 67.10 ± 2.95   Final old_new: 45.16 ± 2.11   Final new_new: 15.45 ± 6.10
AUC:
  Final val: 97.37 ± 0.18   Final Test: 96.82 ± 0.14   Final old_old: 97.47 ± 0.13   Final old_new: 94.25 ± 0.37   Final new_new: 90.03 ± 0.69
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.73 ± 1.46   Final Test: 30.39 ± 2.64   Final old_old: 33.41 ± 2.71   Final old_new: 18.24 ± 2.37   Final new_new: 3.64 ± 3.05
Hits@20:
  Final val: 58.20 ± 1.67   Final Test: 40.94 ± 2.50   Final old_old: 44.45 ± 2.75   Final old_new: 26.98 ± 2.19   Final new_new: 6.36 ± 1.90
Hits@30:
  Final val: 65.12 ± 1.53   Final Test: 48.71 ± 3.16   Final old_old: 52.66 ± 3.48   Final old_new: 32.97 ± 2.83   Final new_new: 9.55 ± 4.07
Hits@50:
  Final val: 73.29 ± 1.22   Final Test: 58.95 ± 3.33   Final old_old: 63.22 ± 3.69   Final old_new: 42.17 ± 2.46   Final new_new: 13.64 ± 3.94
AUC:
  Final val: 97.26 ± 0.11   Final Test: 96.73 ± 0.15   Final old_old: 97.33 ± 0.17   Final old_new: 94.37 ± 0.21   Final new_new: 90.19 ± 0.56
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.76 ± 4.39   Final Test: 28.44 ± 7.32   Final old_old: 30.84 ± 7.87   Final old_new: 18.69 ± 5.12   Final new_new: 7.73 ± 5.23
Hits@20:
  Final val: 53.86 ± 4.04   Final Test: 37.21 ± 8.18   Final old_old: 40.11 ± 8.47   Final old_new: 25.57 ± 7.15   Final new_new: 10.45 ± 6.55
Hits@30:
  Final val: 61.66 ± 3.37   Final Test: 45.06 ± 3.17   Final old_old: 48.42 ± 3.48   Final old_new: 31.66 ± 2.92   Final new_new: 12.73 ± 5.70
Hits@50:
  Final val: 70.95 ± 1.07   Final Test: 54.11 ± 3.32   Final old_old: 57.75 ± 3.28   Final old_new: 39.72 ± 3.71   Final new_new: 17.27 ± 5.70
AUC:
  Final val: 96.61 ± 0.19   Final Test: 95.98 ± 0.26   Final old_old: 96.63 ± 0.25   Final old_new: 93.38 ± 0.32   Final new_new: 89.69 ± 0.78
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.04 ± 3.01   Final Test: 33.30 ± 5.70   Final old_old: 36.54 ± 6.16   Final old_new: 20.33 ± 3.96   Final new_new: 2.73 ± 1.02
Hits@20:
  Final val: 61.02 ± 1.78   Final Test: 41.60 ± 4.56   Final old_old: 45.34 ± 4.77   Final old_new: 26.70 ± 3.83   Final new_new: 5.45 ± 1.24
Hits@30:
  Final val: 67.83 ± 1.49   Final Test: 50.66 ± 3.25   Final old_old: 54.71 ± 3.35   Final old_new: 34.63 ± 3.17   Final new_new: 9.09 ± 2.27
Hits@50:
  Final val: 75.55 ± 1.06   Final Test: 60.52 ± 1.24   Final old_old: 64.63 ± 1.13   Final old_new: 44.46 ± 2.46   Final new_new: 15.00 ± 4.71
AUC:
  Final val: 97.46 ± 0.08   Final Test: 97.01 ± 0.08   Final old_old: 97.55 ± 0.10   Final old_new: 94.87 ± 0.16   Final new_new: 91.27 ± 0.74
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.82 ± 2.47   Final Test: 33.17 ± 4.03   Final old_old: 36.33 ± 4.34   Final old_new: 20.45 ± 2.90   Final new_new: 4.09 ± 1.02
Hits@20:
  Final val: 59.20 ± 3.12   Final Test: 39.66 ± 3.90   Final old_old: 43.04 ± 4.13   Final old_new: 26.10 ± 3.04   Final new_new: 8.18 ± 3.05
Hits@30:
  Final val: 65.42 ± 2.60   Final Test: 48.14 ± 5.11   Final old_old: 51.96 ± 5.42   Final old_new: 32.92 ± 3.93   Final new_new: 10.45 ± 5.23
Hits@50:
  Final val: 73.72 ± 1.15   Final Test: 57.43 ± 2.10   Final old_old: 61.63 ± 2.40   Final old_new: 40.73 ± 1.09   Final new_new: 15.91 ± 3.59
AUC:
  Final val: 97.23 ± 0.06   Final Test: 96.69 ± 0.14   Final old_old: 97.29 ± 0.14   Final old_new: 94.36 ± 0.29   Final new_new: 90.02 ± 0.93
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.28 ± 4.39   Final Test: 29.57 ± 7.34   Final old_old: 32.02 ± 7.79   Final old_new: 19.87 ± 5.69   Final new_new: 5.00 ± 4.07
Hits@20:
  Final val: 53.28 ± 3.32   Final Test: 36.24 ± 2.83   Final old_old: 39.20 ± 3.00   Final old_new: 24.38 ± 2.74   Final new_new: 8.64 ± 4.93
Hits@30:
  Final val: 62.44 ± 2.18   Final Test: 44.73 ± 3.14   Final old_old: 48.14 ± 3.54   Final old_new: 31.21 ± 2.15   Final new_new: 10.91 ± 3.37
Hits@50:
  Final val: 71.57 ± 2.79   Final Test: 55.47 ± 3.91   Final old_old: 59.09 ± 4.10   Final old_new: 41.23 ± 3.28   Final new_new: 17.27 ± 4.13
AUC:
  Final val: 96.61 ± 0.29   Final Test: 95.97 ± 0.31   Final old_old: 96.61 ± 0.33   Final old_new: 93.35 ± 0.29   Final new_new: 90.33 ± 0.85
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.26 ± 3.22   Final Test: 33.00 ± 2.25   Final old_old: 36.34 ± 2.21   Final old_new: 19.50 ± 3.19   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 63.51 ± 2.69   Final Test: 43.76 ± 4.72   Final old_old: 47.58 ± 4.87   Final old_new: 28.49 ± 4.34   Final new_new: 7.73 ± 3.80
Hits@30:
  Final val: 69.71 ± 2.21   Final Test: 51.77 ± 3.55   Final old_old: 55.96 ± 3.72   Final old_new: 35.24 ± 3.44   Final new_new: 7.73 ± 3.45
Hits@50:
  Final val: 77.25 ± 1.87   Final Test: 59.49 ± 1.50   Final old_old: 63.77 ± 1.73   Final old_new: 42.87 ± 1.29   Final new_new: 10.45 ± 3.45
AUC:
  Final val: 97.37 ± 0.18   Final Test: 96.75 ± 0.11   Final old_old: 97.40 ± 0.17   Final old_new: 94.18 ± 0.23   Final new_new: 89.55 ± 1.19
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.21 ± 2.24   Final Test: 30.89 ± 3.10   Final old_old: 33.60 ± 3.49   Final old_new: 19.87 ± 1.82   Final new_new: 8.18 ± 1.24
Hits@20:
  Final val: 54.73 ± 1.82   Final Test: 40.32 ± 2.96   Final old_old: 43.38 ± 3.07   Final old_new: 27.88 ± 2.77   Final new_new: 14.09 ± 4.07
Hits@30:
  Final val: 63.40 ± 1.40   Final Test: 43.21 ± 5.73   Final old_old: 46.59 ± 6.10   Final old_new: 29.50 ± 4.46   Final new_new: 15.00 ± 2.59
Hits@50:
  Final val: 71.04 ± 1.42   Final Test: 56.27 ± 2.18   Final old_old: 60.18 ± 2.39   Final old_new: 40.48 ± 2.61   Final new_new: 22.27 ± 3.73
AUC:
  Final val: 96.61 ± 0.31   Final Test: 95.94 ± 0.34   Final old_old: 96.60 ± 0.38   Final old_new: 93.26 ± 0.47   Final new_new: 90.29 ± 0.39
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.87 ± 5.04   Final Test: 28.94 ± 7.78   Final old_old: 31.37 ± 8.63   Final old_new: 19.07 ± 4.43   Final new_new: 8.18 ± 3.45
Hits@20:
  Final val: 58.39 ± 3.68   Final Test: 40.23 ± 3.40   Final old_old: 43.59 ± 3.47   Final old_new: 26.55 ± 3.60   Final new_new: 12.27 ± 5.47
Hits@30:
  Final val: 66.20 ± 2.55   Final Test: 48.79 ± 2.13   Final old_old: 52.37 ± 2.13   Final old_new: 34.38 ± 2.27   Final new_new: 15.91 ± 5.79
Hits@50:
  Final val: 74.43 ± 1.33   Final Test: 57.56 ± 1.66   Final old_old: 61.45 ± 1.73   Final old_new: 42.12 ± 1.49   Final new_new: 18.64 ± 6.89
AUC:
  Final val: 96.76 ± 0.22   Final Test: 96.10 ± 0.19   Final old_old: 96.77 ± 0.17   Final old_new: 93.52 ± 0.33   Final new_new: 88.73 ± 1.24
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.17 ± 4.74   Final Test: 32.92 ± 6.48   Final old_old: 36.21 ± 7.02   Final old_new: 19.57 ± 4.43   Final new_new: 5.45 ± 1.24
Hits@20:
  Final val: 61.58 ± 3.03   Final Test: 43.67 ± 5.62   Final old_old: 47.35 ± 5.97   Final old_new: 28.84 ± 4.41   Final new_new: 10.91 ± 4.07
Hits@30:
  Final val: 68.81 ± 1.68   Final Test: 52.00 ± 6.19   Final old_old: 56.09 ± 6.48   Final old_new: 35.77 ± 5.21   Final new_new: 10.91 ± 3.73
Hits@50:
  Final val: 76.52 ± 2.36   Final Test: 59.77 ± 3.77   Final old_old: 63.96 ± 3.92   Final old_new: 43.27 ± 3.40   Final new_new: 15.45 ± 2.49
AUC:
  Final val: 97.36 ± 0.16   Final Test: 96.82 ± 0.18   Final old_old: 97.48 ± 0.20   Final old_new: 94.22 ± 0.29   Final new_new: 90.43 ± 0.79
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.93 ± 1.37   Final Test: 34.45 ± 4.16   Final old_old: 37.72 ± 4.41   Final old_new: 21.31 ± 3.21   Final new_new: 5.00 ± 2.49
Hits@20:
  Final val: 59.15 ± 1.16   Final Test: 40.83 ± 4.39   Final old_old: 44.50 ± 4.52   Final old_new: 26.07 ± 4.02   Final new_new: 7.27 ± 2.96
Hits@30:
  Final val: 66.83 ± 1.81   Final Test: 47.77 ± 4.32   Final old_old: 51.68 ± 4.28   Final old_new: 32.27 ± 4.45   Final new_new: 8.18 ± 5.47
Hits@50:
  Final val: 74.33 ± 0.53   Final Test: 57.44 ± 1.34   Final old_old: 61.88 ± 1.35   Final old_new: 39.77 ± 1.34   Final new_new: 13.64 ± 3.59
AUC:
  Final val: 97.29 ± 0.06   Final Test: 96.75 ± 0.06   Final old_old: 97.37 ± 0.05   Final old_new: 94.31 ± 0.23   Final new_new: 90.56 ± 0.81
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.16 ± 2.27   Final Test: 29.88 ± 4.73   Final old_old: 32.84 ± 4.95   Final old_new: 17.93 ± 4.02   Final new_new: 3.64 ± 3.45
Hits@20:
  Final val: 65.21 ± 1.55   Final Test: 42.39 ± 6.58   Final old_old: 46.02 ± 6.82   Final old_new: 27.93 ± 5.66   Final new_new: 6.82 ± 5.33
Hits@30:
  Final val: 71.50 ± 1.43   Final Test: 50.90 ± 4.45   Final old_old: 54.92 ± 4.52   Final old_new: 34.96 ± 4.34   Final new_new: 10.45 ± 5.47
Hits@50:
  Final val: 78.00 ± 0.78   Final Test: 62.66 ± 2.07   Final old_old: 66.98 ± 2.04   Final old_new: 45.69 ± 2.24   Final new_new: 15.91 ± 5.33
AUC:
  Final val: 97.53 ± 0.10   Final Test: 97.00 ± 0.13   Final old_old: 97.56 ± 0.13   Final old_new: 94.82 ± 0.30   Final new_new: 91.22 ± 0.70
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.79 ± 1.99   Final Test: 34.21 ± 2.35   Final old_old: 37.61 ± 2.44   Final old_new: 20.38 ± 2.20   Final new_new: 6.36 ± 3.73
Hits@20:
  Final val: 62.96 ± 2.25   Final Test: 46.33 ± 3.65   Final old_old: 50.18 ± 3.78   Final old_new: 30.98 ± 3.55   Final new_new: 9.09 ± 2.78
Hits@30:
  Final val: 69.69 ± 0.92   Final Test: 52.43 ± 3.29   Final old_old: 56.53 ± 3.50   Final old_new: 36.22 ± 2.58   Final new_new: 10.45 ± 4.13
Hits@50:
  Final val: 77.47 ± 0.91   Final Test: 61.28 ± 2.05   Final old_old: 65.43 ± 2.31   Final old_new: 45.06 ± 1.96   Final new_new: 15.00 ± 3.05
AUC:
  Final val: 97.36 ± 0.14   Final Test: 96.84 ± 0.10   Final old_old: 97.48 ± 0.13   Final old_new: 94.29 ± 0.17   Final new_new: 90.32 ± 0.77
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.44 ± 1.99   Final Test: 32.09 ± 3.41   Final old_old: 35.13 ± 3.60   Final old_new: 19.75 ± 3.07   Final new_new: 5.91 ± 1.24
Hits@20:
  Final val: 60.83 ± 2.75   Final Test: 40.70 ± 1.10   Final old_old: 44.07 ± 1.19   Final old_new: 27.23 ± 0.97   Final new_new: 7.73 ± 2.59
Hits@30:
  Final val: 67.77 ± 1.17   Final Test: 48.38 ± 1.33   Final old_old: 52.37 ± 1.51   Final old_new: 32.42 ± 0.92   Final new_new: 10.91 ± 2.96
Hits@50:
  Final val: 75.10 ± 0.85   Final Test: 58.55 ± 1.29   Final old_old: 62.92 ± 1.53   Final old_new: 41.23 ± 1.13   Final new_new: 13.64 ± 6.43
AUC:
  Final val: 97.34 ± 0.08   Final Test: 96.78 ± 0.06   Final old_old: 97.40 ± 0.04   Final old_new: 94.33 ± 0.25   Final new_new: 90.43 ± 0.73
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.08 ± 4.16   Final Test: 30.56 ± 1.48   Final old_old: 33.69 ± 1.57   Final old_new: 17.88 ± 1.34   Final new_new: 3.64 ± 2.03
Hits@20:
  Final val: 60.75 ± 2.10   Final Test: 43.40 ± 2.17   Final old_old: 46.97 ± 2.29   Final old_new: 29.22 ± 2.11   Final new_new: 8.18 ± 3.05
Hits@30:
  Final val: 67.29 ± 2.08   Final Test: 50.64 ± 2.29   Final old_old: 54.68 ± 2.42   Final old_new: 34.69 ± 1.82   Final new_new: 8.64 ± 2.49
Hits@50:
  Final val: 75.18 ± 1.51   Final Test: 61.45 ± 2.25   Final old_old: 65.64 ± 2.41   Final old_new: 45.06 ± 2.24   Final new_new: 15.45 ± 2.49
AUC:
  Final val: 97.40 ± 0.07   Final Test: 96.96 ± 0.06   Final old_old: 97.49 ± 0.08   Final old_new: 94.89 ± 0.18   Final new_new: 91.23 ± 0.94
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.18 ± 5.65   Final Test: 28.57 ± 6.72   Final old_old: 31.29 ± 7.22   Final old_new: 17.56 ± 4.77   Final new_new: 5.45 ± 4.71
Hits@20:
  Final val: 55.99 ± 3.51   Final Test: 38.42 ± 1.31   Final old_old: 41.53 ± 1.48   Final old_new: 25.87 ± 1.65   Final new_new: 11.36 ± 7.71
Hits@30:
  Final val: 62.62 ± 3.17   Final Test: 44.81 ± 3.71   Final old_old: 48.27 ± 4.06   Final old_new: 31.01 ± 2.79   Final new_new: 11.82 ± 7.08
Hits@50:
  Final val: 71.16 ± 2.44   Final Test: 54.52 ± 4.13   Final old_old: 58.06 ± 4.29   Final old_new: 40.55 ± 3.76   Final new_new: 17.27 ± 4.13
AUC:
  Final val: 96.62 ± 0.33   Final Test: 95.97 ± 0.26   Final old_old: 96.64 ± 0.29   Final old_new: 93.28 ± 0.26   Final new_new: 90.00 ± 1.46
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.85 ± 2.98   Final Test: 28.11 ± 4.38   Final old_old: 31.07 ± 4.71   Final old_new: 16.15 ± 3.24   Final new_new: 2.73 ± 1.02
Hits@20:
  Final val: 59.25 ± 3.58   Final Test: 41.44 ± 4.12   Final old_old: 45.04 ± 4.68   Final old_new: 27.00 ± 2.16   Final new_new: 8.18 ± 2.03
Hits@30:
  Final val: 65.87 ± 2.92   Final Test: 47.45 ± 3.57   Final old_old: 51.24 ± 3.79   Final old_new: 32.52 ± 3.25   Final new_new: 7.27 ± 3.37
Hits@50:
  Final val: 74.31 ± 1.20   Final Test: 58.26 ± 1.61   Final old_old: 62.55 ± 1.76   Final old_new: 41.34 ± 2.01   Final new_new: 12.73 ± 3.80
AUC:
  Final val: 97.23 ± 0.12   Final Test: 96.72 ± 0.13   Final old_old: 97.33 ± 0.15   Final old_new: 94.34 ± 0.36   Final new_new: 89.99 ± 1.37
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.14 ± 6.46   Final Test: 28.28 ± 9.06   Final old_old: 30.87 ± 9.79   Final old_new: 17.73 ± 6.20   Final new_new: 6.82 ± 2.78
Hits@20:
  Final val: 55.02 ± 3.97   Final Test: 37.19 ± 5.54   Final old_old: 40.26 ± 6.32   Final old_new: 24.89 ± 2.58   Final new_new: 8.64 ± 2.96
Hits@30:
  Final val: 62.83 ± 3.73   Final Test: 44.26 ± 5.80   Final old_old: 47.55 ± 6.24   Final old_new: 31.16 ± 4.24   Final new_new: 12.27 ± 2.03
Hits@50:
  Final val: 71.96 ± 2.11   Final Test: 55.86 ± 3.27   Final old_old: 59.64 ± 3.55   Final old_new: 40.78 ± 2.39   Final new_new: 18.64 ± 4.07
AUC:
  Final val: 96.57 ± 0.22   Final Test: 95.85 ± 0.17   Final old_old: 96.53 ± 0.21   Final old_new: 93.13 ± 0.31   Final new_new: 89.29 ± 0.99
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.98 ± 6.26   Final Test: 31.42 ± 5.18   Final old_old: 34.07 ± 5.59   Final old_new: 20.78 ± 3.85   Final new_new: 7.73 ± 2.03
Hits@20:
  Final val: 56.58 ± 5.88   Final Test: 40.72 ± 6.82   Final old_old: 43.78 ± 7.39   Final old_new: 28.41 ± 4.97   Final new_new: 12.73 ± 4.43
Hits@30:
  Final val: 64.42 ± 3.30   Final Test: 47.02 ± 6.74   Final old_old: 50.55 ± 6.95   Final old_new: 32.95 ± 6.27   Final new_new: 12.73 ± 1.24
Hits@50:
  Final val: 72.76 ± 3.10   Final Test: 58.66 ± 5.62   Final old_old: 62.40 ± 5.79   Final old_new: 43.93 ± 5.44   Final new_new: 19.09 ± 3.80
AUC:
  Final val: 96.68 ± 0.20   Final Test: 96.17 ± 0.28   Final old_old: 96.81 ± 0.22   Final old_new: 93.70 ± 0.54   Final new_new: 89.16 ± 1.96
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.67 ± 5.45   Final Test: 28.50 ± 5.74   Final old_old: 31.04 ± 6.42   Final old_new: 18.19 ± 3.08   Final new_new: 7.27 ± 1.90
Hits@20:
  Final val: 54.57 ± 3.95   Final Test: 35.89 ± 4.81   Final old_old: 38.88 ± 5.44   Final old_new: 23.85 ± 2.43   Final new_new: 8.64 ± 1.90
Hits@30:
  Final val: 61.18 ± 2.62   Final Test: 45.09 ± 3.92   Final old_old: 48.55 ± 4.40   Final old_new: 31.16 ± 2.32   Final new_new: 14.09 ± 4.37
Hits@50:
  Final val: 70.89 ± 2.79   Final Test: 56.81 ± 4.08   Final old_old: 60.51 ± 4.31   Final old_new: 42.07 ± 3.60   Final new_new: 21.36 ± 4.43
AUC:
  Final val: 96.59 ± 0.29   Final Test: 95.94 ± 0.27   Final old_old: 96.59 ± 0.32   Final old_new: 93.31 ± 0.12   Final new_new: 90.22 ± 1.79
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.94 ± 3.06   Final Test: 35.30 ± 1.81   Final old_old: 38.76 ± 1.92   Final old_new: 21.46 ± 1.74   Final new_new: 2.27 ± 2.27
Hits@20:
  Final val: 63.10 ± 2.86   Final Test: 46.07 ± 4.64   Final old_old: 50.15 ± 4.77   Final old_new: 29.75 ± 4.61   Final new_new: 6.82 ± 2.78
Hits@30:
  Final val: 69.90 ± 2.57   Final Test: 53.40 ± 3.93   Final old_old: 57.63 ± 3.89   Final old_new: 36.65 ± 4.34   Final new_new: 9.55 ± 4.37
Hits@50:
  Final val: 77.26 ± 1.94   Final Test: 60.40 ± 3.49   Final old_old: 64.58 ± 3.66   Final old_new: 44.13 ± 3.30   Final new_new: 12.73 ± 4.71
AUC:
  Final val: 97.38 ± 0.17   Final Test: 96.77 ± 0.20   Final old_old: 97.45 ± 0.19   Final old_new: 94.10 ± 0.38   Final new_new: 89.69 ± 0.72
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.63 ± 4.60   Final Test: 31.89 ± 5.54   Final old_old: 34.98 ± 5.94   Final old_new: 19.47 ± 3.92   Final new_new: 4.09 ± 2.96
Hits@20:
  Final val: 62.58 ± 1.82   Final Test: 43.60 ± 3.36   Final old_old: 47.21 ± 3.27   Final old_new: 29.19 ± 3.90   Final new_new: 8.18 ± 2.59
Hits@30:
  Final val: 69.20 ± 1.26   Final Test: 50.10 ± 2.91   Final old_old: 54.15 ± 3.00   Final old_new: 34.11 ± 2.91   Final new_new: 7.73 ± 3.05
Hits@50:
  Final val: 76.66 ± 1.75   Final Test: 59.65 ± 1.38   Final old_old: 63.71 ± 1.62   Final old_new: 43.78 ± 1.04   Final new_new: 14.09 ± 1.90
AUC:
  Final val: 97.38 ± 0.15   Final Test: 96.75 ± 0.10   Final old_old: 97.40 ± 0.13   Final old_new: 94.16 ± 0.28   Final new_new: 90.41 ± 1.14
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.52 ± 1.28   Final Test: 30.42 ± 1.95   Final old_old: 33.35 ± 2.18   Final old_new: 18.41 ± 1.90   Final new_new: 7.73 ± 3.45
Hits@20:
  Final val: 59.01 ± 0.99   Final Test: 40.60 ± 2.62   Final old_old: 44.08 ± 2.62   Final old_new: 26.55 ± 2.64   Final new_new: 10.00 ± 5.70
Hits@30:
  Final val: 66.06 ± 1.48   Final Test: 45.81 ± 3.47   Final old_old: 49.66 ± 3.63   Final old_new: 30.30 ± 2.87   Final new_new: 11.82 ± 4.07
Hits@50:
  Final val: 73.81 ± 1.84   Final Test: 55.89 ± 2.59   Final old_old: 59.97 ± 2.58   Final old_new: 39.52 ± 2.82   Final new_new: 18.18 ± 4.25
AUC:
  Final val: 97.25 ± 0.10   Final Test: 96.71 ± 0.08   Final old_old: 97.31 ± 0.10   Final old_new: 94.32 ± 0.18   Final new_new: 90.63 ± 0.52
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 55.22 ± 2.96   Final Test: 36.13 ± 4.30   Final old_old: 39.55 ± 4.67   Final old_new: 22.39 ± 2.96   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 67.38 ± 3.97   Final Test: 46.80 ± 2.53   Final old_old: 50.80 ± 2.74   Final old_new: 30.78 ± 1.76   Final new_new: 9.55 ± 1.90
Hits@30:
  Final val: 72.91 ± 2.76   Final Test: 55.32 ± 3.12   Final old_old: 59.67 ± 3.13   Final old_new: 37.98 ± 3.27   Final new_new: 13.18 ± 2.49
Hits@50:
  Final val: 78.90 ± 1.60   Final Test: 65.83 ± 1.95   Final old_old: 70.19 ± 1.89   Final old_new: 48.66 ± 2.26   Final new_new: 19.55 ± 8.59
AUC:
  Final val: 97.66 ± 0.14   Final Test: 97.15 ± 0.13   Final old_old: 97.70 ± 0.13   Final old_new: 94.97 ± 0.25   Final new_new: 90.96 ± 0.93
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.04 ± 6.14   Final Test: 34.91 ± 1.54   Final old_old: 38.17 ± 1.78   Final old_new: 21.89 ± 0.61   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 63.87 ± 3.04   Final Test: 45.35 ± 1.69   Final old_old: 48.97 ± 1.82   Final old_new: 31.13 ± 1.13   Final new_new: 6.36 ± 1.90
Hits@30:
  Final val: 69.61 ± 2.68   Final Test: 54.39 ± 2.43   Final old_old: 58.45 ± 2.63   Final old_new: 38.61 ± 1.69   Final new_new: 8.18 ± 2.03
Hits@50:
  Final val: 76.51 ± 1.95   Final Test: 63.76 ± 2.25   Final old_old: 68.33 ± 2.39   Final old_new: 45.84 ± 1.75   Final new_new: 14.09 ± 2.49
AUC:
  Final val: 97.39 ± 0.11   Final Test: 96.84 ± 0.12   Final old_old: 97.49 ± 0.17   Final old_new: 94.31 ± 0.17   Final new_new: 90.13 ± 0.86
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.69 ± 4.68   Final Test: 31.66 ± 3.05   Final old_old: 34.84 ± 3.25   Final old_new: 18.82 ± 2.33   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 58.62 ± 4.54   Final Test: 38.41 ± 4.39   Final old_old: 41.76 ± 4.62   Final old_new: 24.89 ± 3.63   Final new_new: 8.18 ± 4.71
Hits@30:
  Final val: 65.53 ± 2.83   Final Test: 45.83 ± 4.08   Final old_old: 49.56 ± 4.38   Final old_new: 30.93 ± 3.18   Final new_new: 10.00 ± 5.23
Hits@50:
  Final val: 73.69 ± 1.22   Final Test: 56.17 ± 2.41   Final old_old: 60.25 ± 2.50   Final old_new: 40.08 ± 2.08   Final new_new: 13.18 ± 5.88
AUC:
  Final val: 97.22 ± 0.11   Final Test: 96.69 ± 0.16   Final old_old: 97.29 ± 0.14   Final old_new: 94.32 ± 0.29   Final new_new: 90.41 ± 0.81
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.17 ± 5.04   Final Test: 24.75 ± 7.17   Final old_old: 27.01 ± 7.82   Final old_new: 15.52 ± 4.74   Final new_new: 6.82 ± 2.27
Hits@20:
  Final val: 53.62 ± 2.92   Final Test: 35.44 ± 5.80   Final old_old: 38.36 ± 6.16   Final old_new: 23.58 ± 4.54   Final new_new: 10.45 ± 4.13
Hits@30:
  Final val: 61.20 ± 1.51   Final Test: 46.54 ± 3.53   Final old_old: 49.86 ± 3.81   Final old_new: 33.10 ± 2.66   Final new_new: 17.73 ± 5.43
Hits@50:
  Final val: 70.77 ± 1.49   Final Test: 54.08 ± 4.31   Final old_old: 57.75 ± 4.62   Final old_new: 39.42 ± 3.44   Final new_new: 18.64 ± 4.07
AUC:
  Final val: 96.61 ± 0.23   Final Test: 95.90 ± 0.23   Final old_old: 96.59 ± 0.22   Final old_new: 93.13 ± 0.30   Final new_new: 90.01 ± 1.69
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.54 ± 3.23   Final Test: 33.07 ± 2.40   Final old_old: 36.36 ± 2.45   Final old_new: 19.75 ± 2.43   Final new_new: 5.00 ± 1.02
Hits@20:
  Final val: 58.92 ± 1.48   Final Test: 41.72 ± 3.71   Final old_old: 45.12 ± 3.73   Final old_new: 28.21 ± 3.70   Final new_new: 8.64 ± 2.96
Hits@30:
  Final val: 66.73 ± 1.42   Final Test: 49.24 ± 3.43   Final old_old: 53.24 ± 3.64   Final old_new: 33.35 ± 2.97   Final new_new: 9.09 ± 4.25
Hits@50:
  Final val: 73.75 ± 0.97   Final Test: 57.84 ± 0.61   Final old_old: 61.94 ± 0.68   Final old_new: 41.59 ± 1.22   Final new_new: 15.91 ± 2.27
AUC:
  Final val: 97.25 ± 0.11   Final Test: 96.72 ± 0.07   Final old_old: 97.33 ± 0.05   Final old_new: 94.35 ± 0.36   Final new_new: 90.56 ± 1.14
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.34 ± 2.07   Final Test: 31.83 ± 6.06   Final old_old: 35.22 ± 6.46   Final old_new: 18.04 ± 4.58   Final new_new: 4.09 ± 2.96
Hits@20:
  Final val: 62.03 ± 2.17   Final Test: 40.79 ± 2.66   Final old_old: 44.63 ± 2.72   Final old_new: 25.24 ± 2.58   Final new_new: 7.27 ± 1.90
Hits@30:
  Final val: 68.55 ± 2.15   Final Test: 50.06 ± 2.95   Final old_old: 54.00 ± 2.82   Final old_new: 34.33 ± 3.69   Final new_new: 11.82 ± 3.73
Hits@50:
  Final val: 75.49 ± 1.68   Final Test: 60.52 ± 1.40   Final old_old: 64.74 ± 1.35   Final old_new: 43.90 ± 2.26   Final new_new: 16.36 ± 3.73
AUC:
  Final val: 97.48 ± 0.08   Final Test: 97.02 ± 0.08   Final old_old: 97.54 ± 0.07   Final old_new: 94.95 ± 0.15   Final new_new: 91.34 ± 1.06
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.85 ± 7.00   Final Test: 25.00 ± 7.77   Final old_old: 27.22 ± 8.41   Final old_new: 15.99 ± 5.44   Final new_new: 5.91 ± 4.43
Hits@20:
  Final val: 54.34 ± 5.68   Final Test: 37.15 ± 4.42   Final old_old: 39.98 ± 4.80   Final old_new: 25.77 ± 3.11   Final new_new: 11.36 ± 6.43
Hits@30:
  Final val: 61.67 ± 5.11   Final Test: 44.75 ± 4.29   Final old_old: 48.00 ± 4.51   Final old_new: 31.81 ± 3.91   Final new_new: 12.27 ± 5.23
Hits@50:
  Final val: 70.74 ± 3.73   Final Test: 54.56 ± 3.66   Final old_old: 58.25 ± 4.05   Final old_new: 39.85 ± 2.67   Final new_new: 18.18 ± 8.19
AUC:
  Final val: 96.64 ± 0.30   Final Test: 95.87 ± 0.38   Final old_old: 96.54 ± 0.40   Final old_new: 93.15 ± 0.49   Final new_new: 90.28 ± 0.94
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.03 ± 2.67   Final Test: 28.98 ± 5.60   Final old_old: 31.45 ± 5.77   Final old_new: 19.02 ± 5.29   Final new_new: 7.27 ± 3.37
Hits@20:
  Final val: 54.03 ± 3.81   Final Test: 36.98 ± 6.03   Final old_old: 40.01 ± 6.29   Final old_new: 24.91 ± 5.38   Final new_new: 8.18 ± 3.45
Hits@30:
  Final val: 62.20 ± 4.95   Final Test: 43.36 ± 5.36   Final old_old: 46.53 ± 5.64   Final old_new: 30.76 ± 4.44   Final new_new: 11.36 ± 3.59
Hits@50:
  Final val: 71.01 ± 3.48   Final Test: 54.61 ± 5.82   Final old_old: 58.08 ± 6.19   Final old_new: 41.03 ± 4.59   Final new_new: 16.36 ± 1.90
AUC:
  Final val: 96.66 ± 0.25   Final Test: 95.91 ± 0.19   Final old_old: 96.56 ± 0.22   Final old_new: 93.32 ± 0.24   Final new_new: 89.88 ± 2.16
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.63 ± 4.74   Final Test: 31.18 ± 4.37   Final old_old: 33.96 ± 4.68   Final old_new: 19.85 ± 3.45   Final new_new: 8.18 ± 4.43
Hits@20:
  Final val: 58.65 ± 1.91   Final Test: 38.67 ± 6.08   Final old_old: 41.89 ± 6.18   Final old_new: 25.69 ± 5.81   Final new_new: 10.45 ± 3.80
Hits@30:
  Final val: 65.14 ± 1.63   Final Test: 48.74 ± 4.44   Final old_old: 52.47 ± 4.38   Final old_new: 33.80 ± 4.82   Final new_new: 13.64 ± 5.08
Hits@50:
  Final val: 73.95 ± 1.13   Final Test: 57.72 ± 1.55   Final old_old: 61.68 ± 1.65   Final old_new: 42.09 ± 1.26   Final new_new: 16.36 ± 4.37
AUC:
  Final val: 96.79 ± 0.20   Final Test: 96.24 ± 0.28   Final old_old: 96.89 ± 0.20   Final old_new: 93.73 ± 0.58   Final new_new: 89.05 ± 1.78
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.58 ± 2.40   Final Test: 32.54 ± 1.62   Final old_old: 35.86 ± 1.82   Final old_new: 18.99 ± 1.08   Final new_new: 5.45 ± 1.24
Hits@20:
  Final val: 63.41 ± 2.42   Final Test: 43.51 ± 3.60   Final old_old: 47.23 ± 3.75   Final old_new: 28.66 ± 3.18   Final new_new: 7.27 ± 2.49
Hits@30:
  Final val: 69.76 ± 1.08   Final Test: 53.88 ± 1.92   Final old_old: 57.97 ± 2.00   Final old_new: 37.58 ± 1.71   Final new_new: 13.64 ± 2.27
Hits@50:
  Final val: 77.01 ± 1.05   Final Test: 63.39 ± 1.58   Final old_old: 67.64 ± 1.54   Final old_new: 46.60 ± 2.10   Final new_new: 18.64 ± 2.96
AUC:
  Final val: 97.37 ± 0.12   Final Test: 96.78 ± 0.07   Final old_old: 97.43 ± 0.11   Final old_new: 94.23 ± 0.15   Final new_new: 89.28 ± 0.68
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.58 ± 4.75   Final Test: 32.21 ± 5.83   Final old_old: 35.26 ± 5.99   Final old_new: 19.97 ± 5.44   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 62.75 ± 3.45   Final Test: 45.31 ± 2.26   Final old_old: 49.13 ± 2.37   Final old_new: 30.15 ± 2.16   Final new_new: 7.27 ± 2.49
Hits@30:
  Final val: 69.08 ± 1.63   Final Test: 52.06 ± 1.36   Final old_old: 56.30 ± 1.47   Final old_new: 35.24 ± 2.15   Final new_new: 8.64 ± 1.02
Hits@50:
  Final val: 77.06 ± 1.68   Final Test: 61.29 ± 2.67   Final old_old: 65.65 ± 2.97   Final old_new: 44.16 ± 1.74   Final new_new: 14.09 ± 3.37
AUC:
  Final val: 97.41 ± 0.17   Final Test: 96.85 ± 0.18   Final old_old: 97.50 ± 0.19   Final old_new: 94.26 ± 0.34   Final new_new: 90.72 ± 1.47
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.17 ± 3.17   Final Test: 30.67 ± 4.33   Final old_old: 33.59 ± 4.72   Final old_new: 18.94 ± 3.45   Final new_new: 4.55 ± 2.27
Hits@20:
  Final val: 58.51 ± 2.38   Final Test: 39.19 ± 1.78   Final old_old: 42.53 ± 1.86   Final old_new: 25.97 ± 1.63   Final new_new: 5.45 ± 3.45
Hits@30:
  Final val: 65.68 ± 1.51   Final Test: 45.71 ± 3.31   Final old_old: 49.44 ± 3.40   Final old_new: 30.91 ± 2.92   Final new_new: 7.73 ± 4.98
Hits@50:
  Final val: 73.61 ± 1.21   Final Test: 58.12 ± 1.36   Final old_old: 62.22 ± 1.58   Final old_new: 41.96 ± 0.54   Final new_new: 15.45 ± 1.90
AUC:
  Final val: 97.22 ± 0.09   Final Test: 96.69 ± 0.10   Final old_old: 97.30 ± 0.08   Final old_new: 94.33 ± 0.31   Final new_new: 89.68 ± 0.62
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.77 ± 5.16   Final Test: 34.37 ± 4.79   Final old_old: 37.67 ± 5.14   Final old_new: 21.16 ± 3.76   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 64.36 ± 1.78   Final Test: 44.96 ± 5.65   Final old_old: 48.74 ± 5.98   Final old_new: 29.87 ± 4.53   Final new_new: 8.18 ± 1.24
Hits@30:
  Final val: 71.16 ± 2.64   Final Test: 52.21 ± 4.56   Final old_old: 56.38 ± 4.89   Final old_new: 35.67 ± 3.95   Final new_new: 10.45 ± 3.80
Hits@50:
  Final val: 77.75 ± 1.85   Final Test: 62.89 ± 3.03   Final old_old: 67.36 ± 3.46   Final old_new: 45.14 ± 2.23   Final new_new: 18.18 ± 4.25
AUC:
  Final val: 97.43 ± 0.18   Final Test: 96.89 ± 0.19   Final old_old: 97.51 ± 0.24   Final old_new: 94.46 ± 0.17   Final new_new: 90.35 ± 0.66
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.97 ± 2.30   Final Test: 37.88 ± 6.22   Final old_old: 41.48 ± 6.56   Final old_new: 23.38 ± 5.01   Final new_new: 5.45 ± 2.59
Hits@20:
  Final val: 66.43 ± 1.16   Final Test: 48.07 ± 5.82   Final old_old: 52.05 ± 6.14   Final old_new: 32.24 ± 4.66   Final new_new: 8.18 ± 4.13
Hits@30:
  Final val: 71.77 ± 1.91   Final Test: 53.57 ± 4.88   Final old_old: 57.79 ± 5.21   Final old_new: 36.93 ± 3.84   Final new_new: 9.55 ± 3.37
Hits@50:
  Final val: 79.03 ± 0.97   Final Test: 63.87 ± 2.96   Final old_old: 68.17 ± 3.10   Final old_new: 47.00 ± 2.90   Final new_new: 17.27 ± 3.45
AUC:
  Final val: 97.65 ± 0.12   Final Test: 97.12 ± 0.19   Final old_old: 97.66 ± 0.20   Final old_new: 95.03 ± 0.22   Final new_new: 90.67 ± 1.43
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.37 ± 3.06   Final Test: 34.88 ± 3.51   Final old_old: 38.18 ± 3.77   Final old_new: 21.54 ± 2.59   Final new_new: 5.91 ± 3.45
Hits@20:
  Final val: 59.32 ± 1.41   Final Test: 43.71 ± 3.44   Final old_old: 47.56 ± 3.62   Final old_new: 28.19 ± 3.05   Final new_new: 9.09 ± 4.55
Hits@30:
  Final val: 66.51 ± 1.57   Final Test: 50.65 ± 3.85   Final old_old: 54.80 ± 3.99   Final old_new: 34.03 ± 3.48   Final new_new: 12.27 ± 2.59
Hits@50:
  Final val: 74.30 ± 2.10   Final Test: 59.65 ± 2.19   Final old_old: 63.90 ± 2.54   Final old_new: 42.77 ± 0.93   Final new_new: 17.27 ± 3.45
AUC:
  Final val: 97.24 ± 0.15   Final Test: 96.73 ± 0.16   Final old_old: 97.35 ± 0.16   Final old_new: 94.28 ± 0.22   Final new_new: 90.04 ± 1.11
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.89 ± 4.89   Final Test: 33.92 ± 5.08   Final old_old: 36.99 ± 5.57   Final old_new: 21.61 ± 3.22   Final new_new: 5.45 ± 1.24
Hits@20:
  Final val: 57.75 ± 4.11   Final Test: 41.53 ± 4.79   Final old_old: 44.89 ± 5.33   Final old_new: 28.09 ± 2.71   Final new_new: 10.00 ± 3.05
Hits@30:
  Final val: 65.21 ± 2.69   Final Test: 46.58 ± 2.79   Final old_old: 50.47 ± 3.00   Final old_new: 31.06 ± 2.39   Final new_new: 9.09 ± 1.61
Hits@50:
  Final val: 73.26 ± 1.28   Final Test: 56.70 ± 1.09   Final old_old: 60.85 ± 0.98   Final old_new: 40.33 ± 2.02   Final new_new: 13.64 ± 4.55
AUC:
  Final val: 97.20 ± 0.10   Final Test: 96.67 ± 0.08   Final old_old: 97.26 ± 0.12   Final old_new: 94.31 ± 0.34   Final new_new: 90.51 ± 1.13
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.70 ± 4.39   Final Test: 28.04 ± 8.00   Final old_old: 30.53 ± 8.72   Final old_new: 17.93 ± 5.25   Final new_new: 7.27 ± 2.96
Hits@20:
  Final val: 56.90 ± 3.98   Final Test: 40.48 ± 3.12   Final old_old: 43.78 ± 3.41   Final old_new: 27.28 ± 2.81   Final new_new: 9.55 ± 3.73
Hits@30:
  Final val: 64.11 ± 4.03   Final Test: 45.78 ± 3.11   Final old_old: 49.46 ± 3.51   Final old_new: 31.08 ± 1.71   Final new_new: 10.45 ± 4.13
Hits@50:
  Final val: 72.47 ± 2.35   Final Test: 58.43 ± 1.82   Final old_old: 62.22 ± 2.19   Final old_new: 43.63 ± 0.72   Final new_new: 15.91 ± 4.82
AUC:
  Final val: 96.69 ± 0.27   Final Test: 96.02 ± 0.29   Final old_old: 96.70 ± 0.28   Final old_new: 93.31 ± 0.38   Final new_new: 89.46 ± 1.70
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.31 ± 2.66   Final Test: 31.89 ± 3.59   Final old_old: 35.01 ± 4.04   Final old_new: 19.37 ± 1.95   Final new_new: 3.64 ± 2.03
Hits@20:
  Final val: 63.61 ± 1.50   Final Test: 43.08 ± 2.94   Final old_old: 46.74 ± 3.15   Final old_new: 28.61 ± 2.55   Final new_new: 5.91 ± 1.24
Hits@30:
  Final val: 68.92 ± 1.52   Final Test: 51.14 ± 0.78   Final old_old: 55.16 ± 1.06   Final old_new: 35.39 ± 1.23   Final new_new: 7.73 ± 2.59
Hits@50:
  Final val: 76.45 ± 1.27   Final Test: 59.64 ± 3.24   Final old_old: 63.84 ± 3.55   Final old_new: 43.22 ± 2.20   Final new_new: 13.64 ± 2.78
AUC:
  Final val: 97.49 ± 0.08   Final Test: 97.01 ± 0.09   Final old_old: 97.55 ± 0.11   Final old_new: 94.91 ± 0.19   Final new_new: 91.11 ± 1.05
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.63 ± 5.37   Final Test: 27.72 ± 7.47   Final old_old: 30.24 ± 8.19   Final old_new: 17.48 ± 4.48   Final new_new: 6.82 ± 5.79
Hits@20:
  Final val: 54.43 ± 4.55   Final Test: 37.69 ± 4.20   Final old_old: 40.76 ± 4.42   Final old_new: 25.26 ± 3.36   Final new_new: 11.36 ± 7.36
Hits@30:
  Final val: 62.14 ± 1.95   Final Test: 44.52 ± 3.53   Final old_old: 47.77 ± 3.70   Final old_new: 31.28 ± 3.08   Final new_new: 17.73 ± 4.07
Hits@50:
  Final val: 72.24 ± 2.45   Final Test: 53.61 ± 3.38   Final old_old: 57.21 ± 3.43   Final old_new: 39.19 ± 3.77   Final new_new: 20.00 ± 3.73
AUC:
  Final val: 96.65 ± 0.30   Final Test: 96.02 ± 0.30   Final old_old: 96.68 ± 0.33   Final old_new: 93.42 ± 0.21   Final new_new: 89.23 ± 1.50
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.14 ± 2.89   Final Test: 30.74 ± 4.39   Final old_old: 33.23 ± 4.58   Final old_new: 20.60 ± 3.85   Final new_new: 10.45 ± 5.70
Hits@20:
  Final val: 57.23 ± 3.22   Final Test: 36.93 ± 3.70   Final old_old: 39.89 ± 4.00   Final old_new: 25.01 ± 2.85   Final new_new: 10.45 ± 3.45
Hits@30:
  Final val: 64.43 ± 2.84   Final Test: 45.16 ± 3.42   Final old_old: 48.47 ± 3.47   Final old_new: 31.99 ± 3.83   Final new_new: 12.73 ± 3.45
Hits@50:
  Final val: 72.95 ± 1.44   Final Test: 56.57 ± 3.34   Final old_old: 60.23 ± 3.44   Final old_new: 42.12 ± 3.29   Final new_new: 19.09 ± 2.59
AUC:
  Final val: 96.65 ± 0.25   Final Test: 95.95 ± 0.18   Final old_old: 96.62 ± 0.24   Final old_new: 93.28 ± 0.21   Final new_new: 89.10 ± 1.16
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.70 ± 4.11   Final Test: 35.20 ± 6.27   Final old_old: 38.41 ± 6.59   Final old_new: 22.24 ± 5.24   Final new_new: 7.27 ± 5.18
Hits@20:
  Final val: 63.76 ± 3.11   Final Test: 41.45 ± 4.21   Final old_old: 45.13 ± 4.56   Final old_new: 26.62 ± 2.95   Final new_new: 8.64 ± 4.07
Hits@30:
  Final val: 70.18 ± 2.01   Final Test: 49.27 ± 2.99   Final old_old: 53.18 ± 3.10   Final old_new: 33.68 ± 2.76   Final new_new: 11.36 ± 5.33
Hits@50:
  Final val: 76.52 ± 1.71   Final Test: 59.11 ± 2.13   Final old_old: 63.45 ± 2.35   Final old_new: 41.79 ± 1.75   Final new_new: 16.82 ± 4.13
AUC:
  Final val: 97.41 ± 0.22   Final Test: 96.81 ± 0.19   Final old_old: 97.44 ± 0.24   Final old_new: 94.33 ± 0.32   Final new_new: 90.11 ± 1.56
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.18 ± 5.31   Final Test: 29.69 ± 8.41   Final old_old: 32.36 ± 9.26   Final old_new: 18.79 ± 5.15   Final new_new: 8.18 ± 2.59
Hits@20:
  Final val: 57.98 ± 4.39   Final Test: 38.39 ± 7.69   Final old_old: 41.53 ± 8.25   Final old_new: 25.69 ± 5.65   Final new_new: 11.82 ± 4.37
Hits@30:
  Final val: 65.98 ± 3.07   Final Test: 47.78 ± 4.37   Final old_old: 51.09 ± 4.98   Final old_new: 34.63 ± 2.64   Final new_new: 15.00 ± 6.55
Hits@50:
  Final val: 74.37 ± 1.05   Final Test: 57.82 ± 1.83   Final old_old: 61.66 ± 2.18   Final old_new: 42.54 ± 1.67   Final new_new: 19.55 ± 6.14
AUC:
  Final val: 96.81 ± 0.20   Final Test: 96.10 ± 0.13   Final old_old: 96.73 ± 0.11   Final old_new: 93.57 ± 0.18   Final new_new: 90.25 ± 1.99
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.63 ± 4.36   Final Test: 33.58 ± 2.76   Final old_old: 36.88 ± 2.89   Final old_new: 20.30 ± 2.38   Final new_new: 3.64 ± 2.59
Hits@20:
  Final val: 62.04 ± 2.47   Final Test: 43.01 ± 3.55   Final old_old: 46.59 ± 3.88   Final old_new: 28.74 ± 2.81   Final new_new: 8.18 ± 2.59
Hits@30:
  Final val: 68.91 ± 1.82   Final Test: 49.88 ± 4.57   Final old_old: 53.99 ± 4.85   Final old_new: 33.55 ± 3.79   Final new_new: 8.18 ± 2.59
Hits@50:
  Final val: 76.67 ± 2.51   Final Test: 60.04 ± 2.85   Final old_old: 64.38 ± 3.17   Final old_new: 43.02 ± 2.18   Final new_new: 12.73 ± 3.05
AUC:
  Final val: 97.37 ± 0.16   Final Test: 96.80 ± 0.09   Final old_old: 97.44 ± 0.13   Final old_new: 94.31 ± 0.27   Final new_new: 89.08 ± 1.59
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.81 ± 2.99   Final Test: 33.09 ± 5.21   Final old_old: 36.20 ± 5.65   Final old_new: 20.53 ± 3.59   Final new_new: 5.00 ± 1.90
Hits@20:
  Final val: 59.30 ± 1.97   Final Test: 40.96 ± 2.72   Final old_old: 44.48 ± 2.78   Final old_new: 26.95 ± 2.63   Final new_new: 6.36 ± 1.90
Hits@30:
  Final val: 66.74 ± 1.03   Final Test: 47.83 ± 4.02   Final old_old: 51.75 ± 4.23   Final old_new: 32.27 ± 3.27   Final new_new: 8.64 ± 1.90
Hits@50:
  Final val: 73.99 ± 1.09   Final Test: 57.66 ± 3.35   Final old_old: 61.86 ± 3.53   Final old_new: 41.06 ± 2.65   Final new_new: 14.55 ± 5.70
AUC:
  Final val: 97.25 ± 0.07   Final Test: 96.72 ± 0.10   Final old_old: 97.33 ± 0.06   Final old_new: 94.32 ± 0.25   Final new_new: 90.03 ± 1.20
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.47 ± 3.73   Final Test: 34.92 ± 6.42   Final old_old: 38.25 ± 6.87   Final old_new: 21.61 ± 4.82   Final new_new: 3.64 ± 3.05
Hits@20:
  Final val: 64.90 ± 2.29   Final Test: 44.82 ± 4.06   Final old_old: 48.64 ± 4.34   Final old_new: 29.55 ± 3.23   Final new_new: 8.18 ± 2.03
Hits@30:
  Final val: 70.18 ± 2.60   Final Test: 51.22 ± 2.69   Final old_old: 55.42 ± 2.88   Final old_new: 34.61 ± 2.23   Final new_new: 8.18 ± 3.05
Hits@50:
  Final val: 76.99 ± 2.02   Final Test: 62.27 ± 0.52   Final old_old: 66.64 ± 0.72   Final old_new: 45.16 ± 1.85   Final new_new: 14.55 ± 4.13
AUC:
  Final val: 97.40 ± 0.15   Final Test: 96.81 ± 0.13   Final old_old: 97.47 ± 0.16   Final old_new: 94.21 ± 0.36   Final new_new: 89.88 ± 0.60
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 53.60 ± 3.42   Final Test: 33.56 ± 3.51   Final old_old: 36.82 ± 3.83   Final old_new: 20.43 ± 2.46   Final new_new: 4.55 ± 2.27
Hits@20:
  Final val: 65.81 ± 3.17   Final Test: 43.38 ± 2.36   Final old_old: 47.10 ± 2.71   Final old_new: 28.49 ± 1.33   Final new_new: 8.18 ± 3.45
Hits@30:
  Final val: 71.83 ± 2.00   Final Test: 49.56 ± 1.79   Final old_old: 53.47 ± 1.49   Final old_new: 34.11 ± 3.27   Final new_new: 10.00 ± 5.23
Hits@50:
  Final val: 78.87 ± 1.06   Final Test: 59.69 ± 2.77   Final old_old: 63.92 ± 3.14   Final old_new: 42.97 ± 2.11   Final new_new: 15.91 ± 3.94
AUC:
  Final val: 97.63 ± 0.12   Final Test: 97.02 ± 0.14   Final old_old: 97.58 ± 0.16   Final old_new: 94.77 ± 0.14   Final new_new: 91.36 ± 0.47
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.46 ± 5.54   Final Test: 27.47 ± 3.20   Final old_old: 29.97 ± 3.44   Final old_new: 17.25 ± 2.38   Final new_new: 7.27 ± 3.73
Hits@20:
  Final val: 53.55 ± 3.35   Final Test: 37.69 ± 2.90   Final old_old: 40.66 ± 3.02   Final old_new: 25.67 ± 2.77   Final new_new: 12.73 ± 5.23
Hits@30:
  Final val: 62.11 ± 1.56   Final Test: 43.87 ± 2.09   Final old_old: 47.26 ± 2.15   Final old_new: 30.23 ± 1.98   Final new_new: 12.73 ± 4.43
Hits@50:
  Final val: 71.89 ± 1.50   Final Test: 54.54 ± 3.92   Final old_old: 58.04 ± 4.31   Final old_new: 40.50 ± 2.90   Final new_new: 21.82 ± 6.74
AUC:
  Final val: 96.63 ± 0.33   Final Test: 96.00 ± 0.24   Final old_old: 96.64 ± 0.25   Final old_new: 93.43 ± 0.29   Final new_new: 89.88 ± 1.71
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=65536, lr=0.005, epochs=20000, eval_steps=5, runs=10, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=0.1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=65536, lr=0.005, epochs=20000, eval_steps=5, runs=10, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=0.1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.65 ± 1.65   Final Test: 33.37 ± 3.35   Final old_old: 36.60 ± 3.79   Final old_new: 20.28 ± 1.87   Final new_new: 6.36 ± 1.90
Hits@20:
  Final val: 58.53 ± 3.64   Final Test: 41.11 ± 5.06   Final old_old: 44.73 ± 5.32   Final old_new: 26.73 ± 4.34   Final new_new: 5.45 ± 2.59
Hits@30:
  Final val: 65.87 ± 2.42   Final Test: 46.98 ± 2.93   Final old_old: 50.93 ± 3.25   Final old_new: 31.23 ± 2.17   Final new_new: 8.18 ± 4.43
Hits@50:
  Final val: 73.68 ± 1.53   Final Test: 56.62 ± 1.49   Final old_old: 60.77 ± 1.80   Final old_new: 40.18 ± 0.96   Final new_new: 14.55 ± 5.23
AUC:
  Final val: 97.23 ± 0.09   Final Test: 96.68 ± 0.11   Final old_old: 97.29 ± 0.11   Final old_new: 94.22 ± 0.26   Final new_new: 90.60 ± 0.38
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.29 ± 3.52   Final Test: 34.33 ± 5.37   Final old_old: 37.51 ± 5.78   Final old_new: 21.46 ± 4.03   Final new_new: 6.36 ± 3.37
Hits@20:
  Final val: 60.45 ± 2.45   Final Test: 43.36 ± 6.68   Final old_old: 47.07 ± 7.37   Final old_new: 28.46 ± 4.05   Final new_new: 8.64 ± 1.90
Hits@30:
  Final val: 66.33 ± 1.61   Final Test: 47.38 ± 3.23   Final old_old: 51.27 ± 3.56   Final old_new: 31.94 ± 2.28   Final new_new: 8.64 ± 1.90
Hits@50:
  Final val: 74.59 ± 0.96   Final Test: 57.03 ± 2.02   Final old_old: 61.46 ± 2.31   Final old_new: 39.47 ± 1.57   Final new_new: 11.82 ± 4.37
AUC:
  Final val: 97.26 ± 0.07   Final Test: 96.72 ± 0.09   Final old_old: 97.32 ± 0.10   Final old_new: 94.36 ± 0.12   Final new_new: 90.18 ± 0.65
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.67 ± 2.15   Final Test: 35.55 ± 5.16   Final old_old: 38.86 ± 5.42   Final old_new: 22.12 ± 4.32   Final new_new: 6.82 ± 3.59
Hits@20:
  Final val: 63.40 ± 2.10   Final Test: 41.50 ± 3.08   Final old_old: 45.08 ± 3.48   Final old_new: 27.23 ± 1.87   Final new_new: 6.82 ± 3.59
Hits@30:
  Final val: 69.00 ± 1.83   Final Test: 49.92 ± 2.94   Final old_old: 53.82 ± 3.12   Final old_new: 34.53 ± 2.24   Final new_new: 9.09 ± 4.25
Hits@50:
  Final val: 76.03 ± 0.93   Final Test: 59.55 ± 2.00   Final old_old: 63.56 ± 2.20   Final old_new: 43.93 ± 1.45   Final new_new: 13.64 ± 2.27
AUC:
  Final val: 97.53 ± 0.06   Final Test: 97.05 ± 0.06   Final old_old: 97.59 ± 0.08   Final old_new: 94.94 ± 0.23   Final new_new: 91.37 ± 1.26
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=65536, lr=0.005, epochs=20000, eval_steps=5, runs=10, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=0.1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.33 ± 2.14   Final Test: 32.55 ± 3.03   Final old_old: 35.74 ± 3.17   Final old_new: 19.65 ± 2.55   Final new_new: 5.45 ± 3.45
Hits@20:
  Final val: 61.35 ± 1.09   Final Test: 41.95 ± 3.30   Final old_old: 45.53 ± 3.53   Final old_new: 27.63 ± 2.58   Final new_new: 7.73 ± 3.45
Hits@30:
  Final val: 67.91 ± 2.07   Final Test: 50.62 ± 2.68   Final old_old: 54.49 ± 2.96   Final old_new: 35.21 ± 1.87   Final new_new: 11.82 ± 6.89
Hits@50:
  Final val: 76.06 ± 1.28   Final Test: 59.50 ± 2.23   Final old_old: 63.76 ± 2.58   Final old_new: 42.64 ± 1.49   Final new_new: 15.45 ± 6.70
AUC:
  Final val: 97.31 ± 0.12   Final Test: 96.74 ± 0.08   Final old_old: 97.36 ± 0.09   Final old_new: 94.30 ± 0.24   Final new_new: 90.12 ± 1.45
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.09 ± 5.17   Final Test: 28.52 ± 5.00   Final old_old: 31.09 ± 5.51   Final old_new: 17.93 ± 3.09   Final new_new: 9.55 ± 3.37
Hits@20:
  Final val: 55.21 ± 3.22   Final Test: 38.12 ± 2.48   Final old_old: 41.25 ± 2.56   Final old_new: 25.26 ± 2.41   Final new_new: 14.09 ± 4.93
Hits@30:
  Final val: 62.01 ± 2.51   Final Test: 46.41 ± 3.05   Final old_old: 49.87 ± 3.19   Final old_new: 32.44 ± 2.89   Final new_new: 15.91 ± 6.43
Hits@50:
  Final val: 72.00 ± 1.30   Final Test: 56.27 ± 3.50   Final old_old: 60.13 ± 3.85   Final old_new: 40.73 ± 2.33   Final new_new: 21.36 ± 2.59
AUC:
  Final val: 96.60 ± 0.27   Final Test: 95.95 ± 0.21   Final old_old: 96.60 ± 0.24   Final old_new: 93.33 ± 0.43   Final new_new: 90.21 ± 1.08
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.30 ± 6.14   Final Test: 28.35 ± 4.52   Final old_old: 30.95 ± 4.87   Final old_new: 17.83 ± 3.32   Final new_new: 5.45 ± 2.59
Hits@20:
  Final val: 56.98 ± 3.76   Final Test: 36.38 ± 5.68   Final old_old: 39.39 ± 6.20   Final old_new: 24.38 ± 4.00   Final new_new: 6.82 ± 4.55
Hits@30:
  Final val: 63.71 ± 2.79   Final Test: 43.69 ± 4.26   Final old_old: 47.06 ± 4.70   Final old_new: 30.15 ± 2.82   Final new_new: 12.73 ± 6.14
Hits@50:
  Final val: 71.95 ± 2.54   Final Test: 55.16 ± 3.76   Final old_old: 58.93 ± 4.08   Final old_new: 40.18 ± 2.85   Final new_new: 17.27 ± 4.71
AUC:
  Final val: 96.65 ± 0.28   Final Test: 95.97 ± 0.33   Final old_old: 96.63 ± 0.33   Final old_new: 93.36 ± 0.36   Final new_new: 89.78 ± 1.36
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.67 ± 5.36   Final Test: 26.50 ± 8.61   Final old_old: 28.84 ± 9.16   Final old_new: 16.85 ± 6.47   Final new_new: 9.55 ± 5.18
Hits@20:
  Final val: 56.35 ± 5.46   Final Test: 38.81 ± 4.96   Final old_old: 41.87 ± 5.43   Final old_new: 26.52 ± 3.49   Final new_new: 10.45 ± 3.05
Hits@30:
  Final val: 63.99 ± 5.05   Final Test: 47.26 ± 4.24   Final old_old: 50.67 ± 4.52   Final old_new: 33.70 ± 3.64   Final new_new: 14.09 ± 7.08
Hits@50:
  Final val: 72.80 ± 2.91   Final Test: 55.42 ± 3.60   Final old_old: 59.23 ± 3.74   Final old_new: 40.08 ± 3.23   Final new_new: 20.91 ± 3.37
AUC:
  Final val: 96.75 ± 0.19   Final Test: 96.12 ± 0.16   Final old_old: 96.81 ± 0.10   Final old_new: 93.28 ± 0.42   Final new_new: 90.21 ± 1.09
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.51 ± 2.86   Final Test: 30.18 ± 2.77   Final old_old: 33.32 ± 3.02   Final old_new: 17.46 ± 1.79   Final new_new: 3.18 ± 1.24
Hits@20:
  Final val: 60.61 ± 1.49   Final Test: 41.39 ± 2.82   Final old_old: 45.01 ± 2.97   Final old_new: 26.90 ± 2.43   Final new_new: 7.27 ± 2.96
Hits@30:
  Final val: 66.48 ± 1.80   Final Test: 48.59 ± 3.84   Final old_old: 52.61 ± 3.99   Final old_new: 32.70 ± 3.68   Final new_new: 7.73 ± 3.45
Hits@50:
  Final val: 73.98 ± 1.33   Final Test: 59.74 ± 1.11   Final old_old: 63.99 ± 1.09   Final old_new: 43.10 ± 1.22   Final new_new: 13.64 ± 3.59
AUC:
  Final val: 97.28 ± 0.11   Final Test: 96.77 ± 0.08   Final old_old: 97.38 ± 0.07   Final old_new: 94.43 ± 0.24   Final new_new: 89.86 ± 0.90
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.00 ± 5.06   Final Test: 31.55 ± 1.77   Final old_old: 34.67 ± 1.95   Final old_new: 18.99 ± 1.71   Final new_new: 2.73 ± 1.02
Hits@20:
  Final val: 62.31 ± 4.47   Final Test: 43.25 ± 5.87   Final old_old: 46.95 ± 6.43   Final old_new: 28.54 ± 3.74   Final new_new: 6.36 ± 4.66
Hits@30:
  Final val: 69.49 ± 3.46   Final Test: 49.29 ± 3.26   Final old_old: 53.33 ± 3.49   Final old_new: 33.22 ± 2.65   Final new_new: 9.55 ± 5.88
Hits@50:
  Final val: 76.70 ± 2.25   Final Test: 60.28 ± 2.95   Final old_old: 64.42 ± 3.23   Final old_new: 43.88 ± 2.04   Final new_new: 18.64 ± 7.43
AUC:
  Final val: 97.33 ± 0.18   Final Test: 96.80 ± 0.16   Final old_old: 97.46 ± 0.23   Final old_new: 94.17 ± 0.32   Final new_new: 90.24 ± 1.56
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.38 ± 2.81   Final Test: 33.93 ± 3.37   Final old_old: 37.26 ± 3.72   Final old_new: 20.53 ± 2.30   Final new_new: 3.64 ± 2.59
Hits@20:
  Final val: 60.81 ± 1.51   Final Test: 44.44 ± 2.96   Final old_old: 48.08 ± 3.42   Final old_new: 29.97 ± 1.81   Final new_new: 8.64 ± 1.90
Hits@30:
  Final val: 67.95 ± 1.41   Final Test: 49.91 ± 2.45   Final old_old: 53.89 ± 2.88   Final old_new: 34.08 ± 1.38   Final new_new: 10.91 ± 2.96
Hits@50:
  Final val: 76.01 ± 1.22   Final Test: 59.86 ± 2.94   Final old_old: 64.19 ± 3.00   Final old_new: 42.82 ± 3.06   Final new_new: 13.18 ± 2.96
AUC:
  Final val: 97.31 ± 0.10   Final Test: 96.73 ± 0.06   Final old_old: 97.39 ± 0.08   Final old_new: 94.15 ± 0.06   Final new_new: 89.95 ± 1.12
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.77 ± 3.34   Final Test: 35.21 ± 4.06   Final old_old: 38.60 ± 4.34   Final old_new: 21.66 ± 3.06   Final new_new: 2.73 ± 2.49
Hits@20:
  Final val: 65.34 ± 3.66   Final Test: 48.00 ± 3.56   Final old_old: 52.04 ± 3.97   Final old_new: 31.99 ± 2.10   Final new_new: 7.27 ± 2.96
Hits@30:
  Final val: 71.06 ± 2.43   Final Test: 56.05 ± 3.37   Final old_old: 60.24 ± 3.59   Final old_new: 39.60 ± 2.83   Final new_new: 10.91 ± 2.49
Hits@50:
  Final val: 78.52 ± 1.04   Final Test: 62.17 ± 3.52   Final old_old: 66.59 ± 3.71   Final old_new: 44.91 ± 3.05   Final new_new: 13.18 ± 4.93
AUC:
  Final val: 97.55 ± 0.07   Final Test: 97.05 ± 0.12   Final old_old: 97.64 ± 0.13   Final old_new: 94.75 ± 0.17   Final new_new: 90.85 ± 1.17
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 40.78 ± 5.44   Final Test: 29.14 ± 4.82   Final old_old: 31.45 ± 5.15   Final old_new: 19.85 ± 3.89   Final new_new: 7.73 ± 1.24
Hits@20:
  Final val: 54.02 ± 5.15   Final Test: 36.59 ± 1.58   Final old_old: 39.72 ± 1.54   Final old_new: 24.03 ± 2.01   Final new_new: 7.73 ± 2.03
Hits@30:
  Final val: 63.03 ± 3.90   Final Test: 46.54 ± 4.34   Final old_old: 49.97 ± 4.60   Final old_new: 32.80 ± 3.38   Final new_new: 14.09 ± 7.94
Hits@50:
  Final val: 72.23 ± 2.44   Final Test: 55.17 ± 3.65   Final old_old: 58.91 ± 3.89   Final old_new: 40.23 ± 2.77   Final new_new: 19.55 ± 5.23
AUC:
  Final val: 96.64 ± 0.28   Final Test: 95.98 ± 0.23   Final old_old: 96.64 ± 0.22   Final old_new: 93.35 ± 0.28   Final new_new: 90.21 ± 1.97
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.63 ± 2.24   Final Test: 32.19 ± 3.97   Final old_old: 35.42 ± 4.22   Final old_new: 18.99 ± 3.10   Final new_new: 6.36 ± 1.90
Hits@20:
  Final val: 59.19 ± 2.19   Final Test: 40.18 ± 1.96   Final old_old: 43.76 ± 2.10   Final old_new: 25.87 ± 1.52   Final new_new: 5.91 ± 1.24
Hits@30:
  Final val: 66.65 ± 2.05   Final Test: 48.76 ± 5.54   Final old_old: 52.80 ± 5.94   Final old_new: 32.64 ± 3.98   Final new_new: 10.00 ± 2.03
Hits@50:
  Final val: 74.00 ± 1.55   Final Test: 57.26 ± 2.98   Final old_old: 61.53 ± 3.01   Final old_new: 40.28 ± 2.83   Final new_new: 15.91 ± 5.33
AUC:
  Final val: 97.23 ± 0.10   Final Test: 96.77 ± 0.06   Final old_old: 97.37 ± 0.06   Final old_new: 94.45 ± 0.15   Final new_new: 90.52 ± 0.92
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.81 ± 2.06   Final Test: 33.40 ± 4.15   Final old_old: 36.54 ± 4.61   Final old_new: 20.78 ± 2.32   Final new_new: 5.00 ± 2.96
Hits@20:
  Final val: 59.03 ± 2.74   Final Test: 39.50 ± 4.69   Final old_old: 42.93 ± 5.11   Final old_new: 25.82 ± 3.26   Final new_new: 6.82 ± 2.78
Hits@30:
  Final val: 65.70 ± 2.25   Final Test: 47.35 ± 2.64   Final old_old: 51.26 ± 2.74   Final old_new: 31.81 ± 2.62   Final new_new: 8.64 ± 2.96
Hits@50:
  Final val: 73.08 ± 1.61   Final Test: 57.53 ± 1.72   Final old_old: 61.71 ± 1.93   Final old_new: 40.96 ± 1.70   Final new_new: 15.00 ± 2.03
AUC:
  Final val: 97.21 ± 0.07   Final Test: 96.73 ± 0.07   Final old_old: 97.30 ± 0.12   Final old_new: 94.46 ± 0.17   Final new_new: 90.68 ± 0.67
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.47 ± 1.06   Final Test: 30.37 ± 1.62   Final old_old: 33.39 ± 1.82   Final old_new: 18.16 ± 1.14   Final new_new: 4.09 ± 2.49
Hits@20:
  Final val: 60.38 ± 0.94   Final Test: 45.06 ± 2.27   Final old_old: 48.68 ± 2.33   Final old_new: 30.86 ± 2.32   Final new_new: 6.36 ± 2.49
Hits@30:
  Final val: 67.91 ± 1.73   Final Test: 50.27 ± 3.41   Final old_old: 54.07 ± 3.34   Final old_new: 35.31 ± 3.70   Final new_new: 10.00 ± 5.93
Hits@50:
  Final val: 75.17 ± 1.77   Final Test: 58.28 ± 3.06   Final old_old: 62.20 ± 3.22   Final old_new: 43.10 ± 2.53   Final new_new: 12.27 ± 4.43
AUC:
  Final val: 97.43 ± 0.09   Final Test: 96.95 ± 0.12   Final old_old: 97.48 ± 0.09   Final old_new: 94.88 ± 0.34   Final new_new: 91.32 ± 0.89
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.12 ± 5.87   Final Test: 28.44 ± 2.43   Final old_old: 31.31 ± 2.81   Final old_new: 16.88 ± 0.93   Final new_new: 2.73 ± 1.02
Hits@20:
  Final val: 61.61 ± 2.54   Final Test: 42.50 ± 3.23   Final old_old: 46.08 ± 3.56   Final old_new: 28.34 ± 2.18   Final new_new: 5.45 ± 3.05
Hits@30:
  Final val: 68.87 ± 1.95   Final Test: 53.28 ± 2.81   Final old_old: 57.37 ± 3.19   Final old_new: 37.23 ± 1.72   Final new_new: 8.64 ± 2.96
Hits@50:
  Final val: 76.62 ± 2.19   Final Test: 61.22 ± 2.70   Final old_old: 65.44 ± 3.10   Final old_new: 44.84 ± 1.94   Final new_new: 12.27 ± 1.24
AUC:
  Final val: 97.34 ± 0.22   Final Test: 96.80 ± 0.23   Final old_old: 97.44 ± 0.27   Final old_new: 94.32 ± 0.18   Final new_new: 90.11 ± 0.94
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 40.61 ± 7.05   Final Test: 28.22 ± 6.87   Final old_old: 30.94 ± 7.33   Final old_new: 17.13 ± 5.13   Final new_new: 6.36 ± 2.49
Hits@20:
  Final val: 53.19 ± 5.56   Final Test: 35.92 ± 7.27   Final old_old: 38.86 ± 7.64   Final old_new: 23.95 ± 5.95   Final new_new: 11.82 ± 4.93
Hits@30:
  Final val: 61.72 ± 3.74   Final Test: 45.71 ± 5.05   Final old_old: 49.17 ± 5.01   Final old_new: 31.89 ± 5.42   Final new_new: 12.27 ± 6.74
Hits@50:
  Final val: 70.80 ± 2.85   Final Test: 54.11 ± 2.61   Final old_old: 57.85 ± 2.72   Final old_new: 39.29 ± 2.57   Final new_new: 16.36 ± 4.37
AUC:
  Final val: 96.50 ± 0.12   Final Test: 96.00 ± 0.15   Final old_old: 96.67 ± 0.18   Final old_new: 93.35 ± 0.28   Final new_new: 89.60 ± 1.67
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.98 ± 6.77   Final Test: 30.34 ± 4.95   Final old_old: 32.98 ± 5.17   Final old_new: 19.70 ± 4.21   Final new_new: 7.27 ± 3.73
Hits@20:
  Final val: 58.55 ± 5.51   Final Test: 40.66 ± 2.01   Final old_old: 44.05 ± 2.14   Final old_new: 27.05 ± 1.86   Final new_new: 9.55 ± 2.49
Hits@30:
  Final val: 66.09 ± 3.98   Final Test: 45.50 ± 3.50   Final old_old: 49.08 ± 3.85   Final old_new: 31.18 ± 2.04   Final new_new: 10.91 ± 4.93
Hits@50:
  Final val: 74.50 ± 2.39   Final Test: 57.10 ± 1.91   Final old_old: 60.95 ± 2.17   Final old_new: 41.81 ± 1.75   Final new_new: 19.09 ± 5.93
AUC:
  Final val: 96.84 ± 0.21   Final Test: 96.18 ± 0.17   Final old_old: 96.84 ± 0.19   Final old_new: 93.58 ± 0.17   Final new_new: 89.31 ± 1.69
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.25 ± 2.99   Final Test: 33.16 ± 2.78   Final old_old: 36.45 ± 3.00   Final old_new: 19.90 ± 2.00   Final new_new: 3.64 ± 3.05
Hits@20:
  Final val: 59.20 ± 2.27   Final Test: 41.58 ± 2.05   Final old_old: 45.33 ± 2.23   Final old_new: 26.52 ± 1.43   Final new_new: 6.82 ± 1.61
Hits@30:
  Final val: 65.78 ± 1.30   Final Test: 49.76 ± 3.39   Final old_old: 53.70 ± 3.59   Final old_new: 34.16 ± 2.63   Final new_new: 9.55 ± 3.73
Hits@50:
  Final val: 73.44 ± 1.24   Final Test: 59.45 ± 1.82   Final old_old: 63.76 ± 1.84   Final old_new: 42.52 ± 1.94   Final new_new: 13.64 ± 1.61
AUC:
  Final val: 97.24 ± 0.09   Final Test: 96.77 ± 0.02   Final old_old: 97.37 ± 0.04   Final old_new: 94.42 ± 0.13   Final new_new: 90.42 ± 0.92
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.10 ± 4.71   Final Test: 31.78 ± 2.94   Final old_old: 34.78 ± 3.33   Final old_new: 19.40 ± 2.37   Final new_new: 9.55 ± 6.31
Hits@20:
  Final val: 55.96 ± 3.02   Final Test: 38.50 ± 3.77   Final old_old: 41.72 ± 3.71   Final old_new: 25.49 ± 4.25   Final new_new: 10.45 ± 7.97
Hits@30:
  Final val: 63.22 ± 1.33   Final Test: 46.69 ± 1.57   Final old_old: 50.36 ± 1.60   Final old_new: 31.86 ± 2.43   Final new_new: 14.55 ± 5.23
Hits@50:
  Final val: 71.55 ± 1.37   Final Test: 58.22 ± 3.13   Final old_old: 61.98 ± 3.10   Final old_new: 43.15 ± 4.50   Final new_new: 22.73 ± 8.65
AUC:
  Final val: 96.65 ± 0.21   Final Test: 95.95 ± 0.13   Final old_old: 96.66 ± 0.19   Final old_new: 93.15 ± 0.46   Final new_new: 88.84 ± 2.52
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.80 ± 4.93   Final Test: 35.10 ± 6.11   Final old_old: 38.44 ± 6.58   Final old_new: 21.66 ± 4.28   Final new_new: 4.55 ± 2.27
Hits@20:
  Final val: 62.87 ± 1.84   Final Test: 44.64 ± 3.94   Final old_old: 48.47 ± 4.24   Final old_new: 29.40 ± 2.96   Final new_new: 7.27 ± 1.90
Hits@30:
  Final val: 68.95 ± 2.47   Final Test: 52.18 ± 2.81   Final old_old: 56.32 ± 3.16   Final old_new: 35.79 ± 1.40   Final new_new: 10.45 ± 2.03
Hits@50:
  Final val: 75.95 ± 1.19   Final Test: 61.36 ± 2.41   Final old_old: 65.63 ± 2.81   Final old_new: 44.41 ± 1.19   Final new_new: 18.18 ± 4.25
AUC:
  Final val: 97.31 ± 0.22   Final Test: 96.80 ± 0.18   Final old_old: 97.46 ± 0.23   Final old_new: 94.14 ± 0.13   Final new_new: 90.39 ± 1.46
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.97 ± 5.45   Final Test: 26.68 ± 8.32   Final old_old: 29.07 ± 8.98   Final old_new: 16.98 ± 5.75   Final new_new: 7.27 ± 5.43
Hits@20:
  Final val: 54.89 ± 4.20   Final Test: 35.40 ± 5.01   Final old_old: 38.39 ± 5.42   Final old_new: 23.43 ± 3.28   Final new_new: 7.27 ± 4.93
Hits@30:
  Final val: 62.03 ± 2.88   Final Test: 43.23 ± 1.98   Final old_old: 46.57 ± 2.06   Final old_new: 29.82 ± 2.02   Final new_new: 12.27 ± 5.47
Hits@50:
  Final val: 70.70 ± 2.91   Final Test: 52.42 ± 2.80   Final old_old: 56.14 ± 2.93   Final old_new: 37.41 ± 2.46   Final new_new: 19.09 ± 3.80
AUC:
  Final val: 96.57 ± 0.28   Final Test: 95.87 ± 0.21   Final old_old: 96.57 ± 0.23   Final old_new: 93.09 ± 0.51   Final new_new: 89.03 ± 0.88
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.42 ± 4.53   Final Test: 34.98 ± 5.89   Final old_old: 38.44 ± 6.33   Final old_new: 20.98 ± 4.27   Final new_new: 5.00 ± 3.73
Hits@20:
  Final val: 63.85 ± 2.01   Final Test: 45.44 ± 6.73   Final old_old: 49.39 ± 7.06   Final old_new: 29.67 ± 5.61   Final new_new: 7.73 ± 4.71
Hits@30:
  Final val: 71.52 ± 1.15   Final Test: 53.14 ± 5.48   Final old_old: 57.33 ± 5.67   Final old_new: 36.47 ± 4.77   Final new_new: 11.82 ± 6.70
Hits@50:
  Final val: 78.00 ± 0.93   Final Test: 63.82 ± 3.00   Final old_old: 68.04 ± 3.15   Final old_new: 47.25 ± 2.63   Final new_new: 18.64 ± 5.88
AUC:
  Final val: 97.49 ± 0.10   Final Test: 97.01 ± 0.11   Final old_old: 97.59 ± 0.13   Final old_new: 94.72 ± 0.24   Final new_new: 90.94 ± 1.15
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.70 ± 4.52   Final Test: 32.71 ± 2.85   Final old_old: 35.83 ± 3.21   Final old_new: 20.25 ± 1.67   Final new_new: 2.73 ± 1.02
Hits@20:
  Final val: 62.81 ± 4.74   Final Test: 44.62 ± 3.88   Final old_old: 48.50 ± 4.03   Final old_new: 29.07 ± 3.58   Final new_new: 8.18 ± 3.45
Hits@30:
  Final val: 68.81 ± 3.23   Final Test: 51.17 ± 5.07   Final old_old: 55.20 ± 5.32   Final old_new: 35.24 ± 4.48   Final new_new: 9.55 ± 3.37
Hits@50:
  Final val: 76.16 ± 2.33   Final Test: 61.46 ± 3.73   Final old_old: 65.67 ± 4.00   Final old_new: 44.94 ± 2.86   Final new_new: 15.45 ± 4.07
AUC:
  Final val: 97.34 ± 0.10   Final Test: 96.83 ± 0.12   Final old_old: 97.47 ± 0.17   Final old_new: 94.31 ± 0.13   Final new_new: 90.07 ± 1.03
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.27 ± 3.50   Final Test: 29.05 ± 6.05   Final old_old: 31.96 ± 6.52   Final old_new: 17.18 ± 4.26   Final new_new: 5.00 ± 4.07
Hits@20:
  Final val: 58.71 ± 4.13   Final Test: 39.58 ± 5.29   Final old_old: 43.23 ± 5.84   Final old_new: 24.84 ± 3.52   Final new_new: 7.27 ± 2.96
Hits@30:
  Final val: 65.66 ± 3.18   Final Test: 47.04 ± 5.40   Final old_old: 51.00 ± 5.83   Final old_new: 31.26 ± 3.86   Final new_new: 8.18 ± 2.59
Hits@50:
  Final val: 73.43 ± 2.27   Final Test: 55.21 ± 3.36   Final old_old: 59.40 ± 3.66   Final old_new: 38.69 ± 2.44   Final new_new: 10.91 ± 3.37
AUC:
  Final val: 97.23 ± 0.11   Final Test: 96.69 ± 0.14   Final old_old: 97.29 ± 0.16   Final old_new: 94.34 ± 0.24   Final new_new: 90.45 ± 0.82
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.83 ± 2.20   Final Test: 33.06 ± 3.55   Final old_old: 36.34 ± 3.92   Final old_new: 19.85 ± 2.20   Final new_new: 3.18 ± 2.03
Hits@20:
  Final val: 63.51 ± 1.94   Final Test: 43.46 ± 2.45   Final old_old: 47.18 ± 2.82   Final old_new: 28.79 ± 1.09   Final new_new: 4.55 ± 0.00
Hits@30:
  Final val: 69.58 ± 1.50   Final Test: 50.72 ± 3.30   Final old_old: 54.88 ± 3.33   Final old_new: 34.26 ± 3.63   Final new_new: 8.18 ± 2.59
Hits@50:
  Final val: 76.94 ± 1.28   Final Test: 61.03 ± 1.55   Final old_old: 65.37 ± 1.68   Final old_new: 44.01 ± 1.63   Final new_new: 14.55 ± 5.23
AUC:
  Final val: 97.35 ± 0.17   Final Test: 96.81 ± 0.14   Final old_old: 97.46 ± 0.15   Final old_new: 94.23 ± 0.22   Final new_new: 90.33 ± 1.12
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.21 ± 1.75   Final Test: 34.48 ± 4.88   Final old_old: 37.83 ± 4.96   Final old_new: 20.98 ± 4.80   Final new_new: 4.55 ± 2.78
Hits@20:
  Final val: 60.86 ± 2.18   Final Test: 42.04 ± 1.70   Final old_old: 45.67 ± 1.80   Final old_new: 27.51 ± 1.54   Final new_new: 8.18 ± 2.59
Hits@30:
  Final val: 68.44 ± 2.02   Final Test: 50.22 ± 3.15   Final old_old: 53.89 ± 2.96   Final old_new: 35.79 ± 4.14   Final new_new: 11.36 ± 2.78
Hits@50:
  Final val: 75.55 ± 2.38   Final Test: 58.84 ± 2.19   Final old_old: 62.82 ± 2.02   Final old_new: 43.30 ± 3.00   Final new_new: 14.09 ± 4.93
AUC:
  Final val: 97.44 ± 0.13   Final Test: 96.96 ± 0.11   Final old_old: 97.49 ± 0.10   Final old_new: 94.88 ± 0.23   Final new_new: 91.31 ± 0.80
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.48 ± 3.66   Final Test: 33.13 ± 3.09   Final old_old: 36.48 ± 3.34   Final old_new: 19.62 ± 2.28   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 60.28 ± 2.10   Final Test: 41.80 ± 3.04   Final old_old: 45.45 ± 3.12   Final old_new: 27.20 ± 3.00   Final new_new: 6.82 ± 1.61
Hits@30:
  Final val: 66.51 ± 1.00   Final Test: 46.82 ± 2.85   Final old_old: 50.78 ± 2.97   Final old_new: 31.03 ± 2.72   Final new_new: 8.64 ± 5.18
Hits@50:
  Final val: 73.97 ± 1.14   Final Test: 56.23 ± 2.77   Final old_old: 60.51 ± 2.85   Final old_new: 39.29 ± 3.25   Final new_new: 12.27 ± 5.23
AUC:
  Final val: 97.26 ± 0.11   Final Test: 96.69 ± 0.08   Final old_old: 97.28 ± 0.07   Final old_new: 94.37 ± 0.17   Final new_new: 90.44 ± 1.11
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.36 ± 4.85   Final Test: 27.87 ± 5.85   Final old_old: 30.39 ± 6.20   Final old_new: 17.68 ± 4.79   Final new_new: 5.91 ± 3.45
Hits@20:
  Final val: 55.01 ± 3.68   Final Test: 38.58 ± 5.67   Final old_old: 41.60 ± 6.13   Final old_new: 26.45 ± 4.33   Final new_new: 10.91 ± 4.37
Hits@30:
  Final val: 62.48 ± 2.52   Final Test: 44.75 ± 2.50   Final old_old: 48.13 ± 2.75   Final old_new: 31.21 ± 2.07   Final new_new: 12.73 ± 6.14
Hits@50:
  Final val: 71.70 ± 1.65   Final Test: 54.98 ± 2.52   Final old_old: 58.75 ± 2.59   Final old_new: 39.95 ± 3.36   Final new_new: 17.73 ± 6.89
AUC:
  Final val: 96.63 ± 0.30   Final Test: 96.00 ± 0.18   Final old_old: 96.67 ± 0.20   Final old_new: 93.31 ± 0.30   Final new_new: 89.87 ± 2.82
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.13 ± 5.75   Final Test: 29.80 ± 5.94   Final old_old: 32.22 ± 6.41   Final old_new: 19.97 ± 4.41   Final new_new: 9.55 ± 2.49
Hits@20:
  Final val: 58.03 ± 6.69   Final Test: 39.30 ± 6.58   Final old_old: 42.37 ± 7.18   Final old_new: 27.03 ± 4.40   Final new_new: 10.45 ± 2.59
Hits@30:
  Final val: 65.98 ± 5.28   Final Test: 46.22 ± 1.73   Final old_old: 49.76 ± 2.03   Final old_new: 31.96 ± 1.68   Final new_new: 14.09 ± 4.37
Hits@50:
  Final val: 74.02 ± 3.27   Final Test: 56.92 ± 1.38   Final old_old: 60.77 ± 1.25   Final old_new: 41.74 ± 2.77   Final new_new: 16.82 ± 5.93
AUC:
  Final val: 96.85 ± 0.17   Final Test: 96.22 ± 0.27   Final old_old: 96.81 ± 0.27   Final old_new: 93.90 ± 0.45   Final new_new: 89.93 ± 0.40
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.71 ± 1.43   Final Test: 32.92 ± 3.88   Final old_old: 36.29 ± 4.28   Final old_new: 19.27 ± 2.34   Final new_new: 3.64 ± 2.59
Hits@20:
  Final val: 60.57 ± 1.74   Final Test: 40.73 ± 2.32   Final old_old: 44.44 ± 2.59   Final old_new: 25.87 ± 1.24   Final new_new: 5.91 ± 2.03
Hits@30:
  Final val: 67.56 ± 1.58   Final Test: 49.56 ± 2.14   Final old_old: 53.58 ± 2.29   Final old_new: 33.63 ± 1.89   Final new_new: 8.64 ± 1.90
Hits@50:
  Final val: 74.42 ± 1.24   Final Test: 58.60 ± 1.30   Final old_old: 62.84 ± 1.43   Final old_new: 41.89 ± 1.17   Final new_new: 14.09 ± 5.43
AUC:
  Final val: 97.31 ± 0.16   Final Test: 96.78 ± 0.05   Final old_old: 97.43 ± 0.07   Final old_new: 94.20 ± 0.29   Final new_new: 89.93 ± 0.53
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.97 ± 5.14   Final Test: 28.47 ± 4.47   Final old_old: 30.90 ± 4.63   Final old_new: 18.69 ± 4.25   Final new_new: 6.36 ± 8.10
Hits@20:
  Final val: 56.95 ± 3.96   Final Test: 38.29 ± 5.28   Final old_old: 41.14 ± 5.70   Final old_new: 26.93 ± 3.93   Final new_new: 10.91 ± 4.93
Hits@30:
  Final val: 64.27 ± 3.48   Final Test: 46.04 ± 2.98   Final old_old: 49.47 ± 3.27   Final old_new: 32.14 ± 2.15   Final new_new: 16.82 ± 10.37
Hits@50:
  Final val: 72.46 ± 2.95   Final Test: 57.29 ± 2.52   Final old_old: 61.00 ± 2.82   Final old_new: 42.54 ± 1.75   Final new_new: 20.91 ± 9.29
AUC:
  Final val: 96.58 ± 0.31   Final Test: 96.00 ± 0.20   Final old_old: 96.67 ± 0.24   Final old_new: 93.33 ± 0.24   Final new_new: 89.79 ± 1.96
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.79 ± 4.27   Final Test: 32.48 ± 2.09   Final old_old: 35.62 ± 2.36   Final old_new: 19.95 ± 1.45   Final new_new: 2.73 ± 1.90
Hits@20:
  Final val: 62.24 ± 1.58   Final Test: 44.41 ± 3.27   Final old_old: 48.23 ± 3.24   Final old_new: 29.24 ± 3.81   Final new_new: 5.45 ± 1.24
Hits@30:
  Final val: 69.19 ± 0.21   Final Test: 52.78 ± 1.31   Final old_old: 56.98 ± 1.36   Final old_new: 36.20 ± 1.36   Final new_new: 8.64 ± 1.90
Hits@50:
  Final val: 76.51 ± 1.31   Final Test: 62.00 ± 2.63   Final old_old: 66.28 ± 2.59   Final old_new: 45.37 ± 3.20   Final new_new: 13.18 ± 1.02
AUC:
  Final val: 97.35 ± 0.14   Final Test: 96.79 ± 0.12   Final old_old: 97.47 ± 0.16   Final old_new: 94.12 ± 0.32   Final new_new: 89.25 ± 1.60
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.96 ± 5.33   Final Test: 35.31 ± 2.11   Final old_old: 38.73 ± 2.34   Final old_new: 21.41 ± 1.52   Final new_new: 7.27 ± 3.37
Hits@20:
  Final val: 65.96 ± 4.54   Final Test: 47.62 ± 3.04   Final old_old: 51.65 ± 3.25   Final old_new: 31.41 ± 2.67   Final new_new: 11.36 ± 3.59
Hits@30:
  Final val: 71.78 ± 3.82   Final Test: 55.15 ± 6.00   Final old_old: 59.27 ± 6.26   Final old_new: 38.79 ± 4.96   Final new_new: 14.09 ± 6.70
Hits@50:
  Final val: 78.93 ± 1.34   Final Test: 64.11 ± 2.53   Final old_old: 68.38 ± 2.50   Final old_new: 47.30 ± 2.63   Final new_new: 18.18 ± 6.22
AUC:
  Final val: 97.60 ± 0.10   Final Test: 97.12 ± 0.10   Final old_old: 97.67 ± 0.07   Final old_new: 94.92 ± 0.19   Final new_new: 91.83 ± 0.60
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.63 ± 3.01   Final Test: 33.68 ± 5.72   Final old_old: 36.92 ± 6.22   Final old_new: 20.71 ± 3.81   Final new_new: 3.18 ± 1.24
Hits@20:
  Final val: 59.99 ± 2.38   Final Test: 41.23 ± 4.69   Final old_old: 44.77 ± 5.10   Final old_new: 27.15 ± 3.19   Final new_new: 6.36 ± 1.90
Hits@30:
  Final val: 67.22 ± 1.91   Final Test: 48.82 ± 6.36   Final old_old: 52.80 ± 6.87   Final old_new: 33.02 ± 4.64   Final new_new: 9.09 ± 1.61
Hits@50:
  Final val: 75.85 ± 1.77   Final Test: 58.64 ± 2.90   Final old_old: 62.90 ± 3.27   Final old_new: 41.89 ± 1.49   Final new_new: 12.73 ± 4.13
AUC:
  Final val: 97.27 ± 0.15   Final Test: 96.75 ± 0.21   Final old_old: 97.41 ± 0.25   Final old_new: 94.17 ± 0.22   Final new_new: 89.71 ± 1.89
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.93 ± 2.70   Final Test: 33.57 ± 6.82   Final old_old: 36.26 ± 7.31   Final old_new: 22.85 ± 4.95   Final new_new: 6.82 ± 2.78
Hits@20:
  Final val: 57.66 ± 2.74   Final Test: 40.14 ± 5.90   Final old_old: 43.35 ± 6.38   Final old_new: 27.28 ± 4.21   Final new_new: 10.45 ± 2.03
Hits@30:
  Final val: 65.37 ± 1.46   Final Test: 47.15 ± 5.10   Final old_old: 50.58 ± 5.43   Final old_new: 33.58 ± 4.22   Final new_new: 11.82 ± 4.07
Hits@50:
  Final val: 72.55 ± 1.65   Final Test: 58.17 ± 2.47   Final old_old: 61.81 ± 2.83   Final old_new: 43.80 ± 1.69   Final new_new: 20.00 ± 6.10
AUC:
  Final val: 96.64 ± 0.30   Final Test: 96.08 ± 0.28   Final old_old: 96.70 ± 0.33   Final old_new: 93.61 ± 0.20   Final new_new: 90.73 ± 0.66
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.55 ± 3.71   Final Test: 31.03 ± 4.42   Final old_old: 34.07 ± 4.75   Final old_new: 18.72 ± 3.24   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 59.37 ± 2.59   Final Test: 41.97 ± 2.67   Final old_old: 45.67 ± 2.73   Final old_new: 27.15 ± 2.60   Final new_new: 7.27 ± 1.90
Hits@30:
  Final val: 67.03 ± 1.46   Final Test: 49.49 ± 2.97   Final old_old: 53.57 ± 3.13   Final old_new: 33.32 ± 2.71   Final new_new: 8.18 ± 4.71
Hits@50:
  Final val: 74.27 ± 1.31   Final Test: 57.99 ± 2.45   Final old_old: 62.18 ± 2.40   Final old_new: 41.39 ± 3.06   Final new_new: 15.00 ± 4.43
AUC:
  Final val: 97.22 ± 0.10   Final Test: 96.73 ± 0.07   Final old_old: 97.34 ± 0.10   Final old_new: 94.36 ± 0.29   Final new_new: 89.89 ± 0.82
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.59 ± 3.33   Final Test: 31.89 ± 3.62   Final old_old: 34.97 ± 3.77   Final old_new: 19.50 ± 3.36   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 61.45 ± 2.63   Final Test: 41.30 ± 3.27   Final old_old: 44.66 ± 3.65   Final old_new: 28.01 ± 1.76   Final new_new: 6.82 ± 1.61
Hits@30:
  Final val: 68.64 ± 1.58   Final Test: 47.16 ± 3.20   Final old_old: 50.80 ± 3.26   Final old_new: 32.82 ± 3.30   Final new_new: 9.09 ± 2.78
Hits@50:
  Final val: 75.36 ± 1.84   Final Test: 59.41 ± 1.67   Final old_old: 63.04 ± 1.40   Final old_new: 45.14 ± 2.84   Final new_new: 20.00 ± 6.31
AUC:
  Final val: 97.47 ± 0.09   Final Test: 97.08 ± 0.10   Final old_old: 97.53 ± 0.11   Final old_new: 95.27 ± 0.07   Final new_new: 92.29 ± 0.47
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.27 ± 3.43   Final Test: 33.16 ± 2.38   Final old_old: 36.45 ± 2.56   Final old_new: 19.85 ± 2.14   Final new_new: 4.55 ± 2.78
Hits@20:
  Final val: 60.61 ± 1.25   Final Test: 39.37 ± 2.39   Final old_old: 43.08 ± 2.42   Final old_new: 24.36 ± 2.88   Final new_new: 6.82 ± 2.78
Hits@30:
  Final val: 67.39 ± 0.74   Final Test: 46.80 ± 3.16   Final old_old: 50.84 ± 3.53   Final old_new: 30.68 ± 2.16   Final new_new: 7.73 ± 4.13
Hits@50:
  Final val: 74.66 ± 0.97   Final Test: 56.76 ± 3.71   Final old_old: 61.10 ± 4.02   Final old_new: 39.45 ± 2.63   Final new_new: 14.55 ± 2.59
AUC:
  Final val: 97.27 ± 0.12   Final Test: 96.73 ± 0.04   Final old_old: 97.33 ± 0.04   Final old_new: 94.36 ± 0.08   Final new_new: 90.45 ± 0.75
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.69 ± 6.29   Final Test: 28.92 ± 4.61   Final old_old: 31.53 ± 5.20   Final old_new: 18.21 ± 2.38   Final new_new: 8.64 ± 3.73
Hits@20:
  Final val: 54.24 ± 5.05   Final Test: 37.08 ± 6.43   Final old_old: 39.94 ± 7.02   Final old_new: 25.69 ± 4.21   Final new_new: 9.09 ± 3.59
Hits@30:
  Final val: 61.88 ± 3.36   Final Test: 46.01 ± 3.52   Final old_old: 49.27 ± 3.83   Final old_new: 33.10 ± 2.75   Final new_new: 13.18 ± 4.07
Hits@50:
  Final val: 71.38 ± 2.33   Final Test: 53.84 ± 5.65   Final old_old: 57.37 ± 6.03   Final old_new: 39.95 ± 4.39   Final new_new: 16.36 ± 1.90
AUC:
  Final val: 96.62 ± 0.34   Final Test: 95.89 ± 0.21   Final old_old: 96.56 ± 0.23   Final old_new: 93.25 ± 0.15   Final new_new: 88.58 ± 1.45
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.70 ± 3.91   Final Test: 31.05 ± 5.76   Final old_old: 34.28 ± 6.09   Final old_new: 18.06 ± 4.65   Final new_new: 2.27 ± 1.61
Hits@20:
  Final val: 62.80 ± 2.99   Final Test: 43.57 ± 2.28   Final old_old: 47.52 ± 2.45   Final old_new: 27.73 ± 1.79   Final new_new: 6.36 ± 3.37
Hits@30:
  Final val: 70.13 ± 2.50   Final Test: 50.72 ± 3.43   Final old_old: 55.06 ± 3.54   Final old_new: 33.43 ± 2.92   Final new_new: 9.09 ± 6.22
Hits@50:
  Final val: 76.69 ± 2.03   Final Test: 59.97 ± 2.55   Final old_old: 64.56 ± 2.82   Final old_new: 41.76 ± 1.44   Final new_new: 14.09 ± 5.18
AUC:
  Final val: 97.35 ± 0.15   Final Test: 96.78 ± 0.18   Final old_old: 97.44 ± 0.21   Final old_new: 94.26 ± 0.25   Final new_new: 88.78 ± 1.91
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.00 ± 2.51   Final Test: 27.65 ± 4.75   Final old_old: 30.38 ± 5.22   Final old_new: 16.55 ± 2.87   Final new_new: 5.00 ± 3.73
Hits@20:
  Final val: 61.01 ± 1.85   Final Test: 40.36 ± 3.96   Final old_old: 43.81 ± 4.42   Final old_new: 26.52 ± 2.24   Final new_new: 8.64 ± 5.88
Hits@30:
  Final val: 65.96 ± 1.80   Final Test: 47.50 ± 2.51   Final old_old: 51.32 ± 2.48   Final old_new: 32.09 ± 3.24   Final new_new: 13.64 ± 5.33
Hits@50:
  Final val: 72.32 ± 0.84   Final Test: 57.07 ± 2.10   Final old_old: 60.90 ± 1.94   Final old_new: 41.94 ± 3.06   Final new_new: 16.82 ± 4.71
AUC:
  Final val: 95.48 ± 0.28   Final Test: 94.96 ± 0.19   Final old_old: 95.60 ± 0.20   Final old_new: 92.59 ± 0.20   Final new_new: 86.11 ± 2.86
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.78 ± 2.65   Final Test: 35.28 ± 5.19   Final old_old: 38.58 ± 5.82   Final old_new: 22.12 ± 2.57   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 62.51 ± 1.87   Final Test: 45.96 ± 5.78   Final old_old: 49.81 ± 6.15   Final old_new: 30.68 ± 4.28   Final new_new: 7.27 ± 4.07
Hits@30:
  Final val: 69.08 ± 1.17   Final Test: 51.32 ± 4.99   Final old_old: 55.39 ± 5.30   Final old_new: 35.29 ± 3.78   Final new_new: 9.09 ± 3.59
Hits@50:
  Final val: 76.16 ± 2.02   Final Test: 61.27 ± 1.19   Final old_old: 65.54 ± 1.47   Final old_new: 44.69 ± 1.16   Final new_new: 12.27 ± 2.59
AUC:
  Final val: 97.36 ± 0.16   Final Test: 96.76 ± 0.24   Final old_old: 97.43 ± 0.24   Final old_new: 94.14 ± 0.35   Final new_new: 90.12 ± 0.23
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.61 ± 3.73   Final Test: 25.06 ± 6.11   Final old_old: 27.48 ± 6.58   Final old_new: 15.11 ± 4.42   Final new_new: 6.82 ± 2.78
Hits@20:
  Final val: 55.52 ± 3.26   Final Test: 39.37 ± 2.68   Final old_old: 42.33 ± 2.62   Final old_new: 27.25 ± 3.08   Final new_new: 16.36 ± 7.94
Hits@30:
  Final val: 63.15 ± 3.49   Final Test: 43.96 ± 5.47   Final old_old: 47.33 ± 5.77   Final old_new: 30.35 ± 4.66   Final new_new: 15.00 ± 5.70
Hits@50:
  Final val: 72.14 ± 2.22   Final Test: 59.10 ± 3.98   Final old_old: 62.88 ± 4.04   Final old_new: 43.88 ± 3.96   Final new_new: 25.00 ± 2.78
AUC:
  Final val: 96.62 ± 0.21   Final Test: 96.00 ± 0.29   Final old_old: 96.63 ± 0.33   Final old_new: 93.43 ± 0.37   Final new_new: 91.14 ± 1.45
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.70 ± 1.38   Final Test: 35.09 ± 2.77   Final old_old: 38.33 ± 3.20   Final old_new: 22.09 ± 1.22   Final new_new: 5.45 ± 2.03
Hits@20:
  Final val: 59.70 ± 1.15   Final Test: 44.07 ± 1.28   Final old_old: 47.88 ± 1.40   Final old_new: 28.74 ± 1.38   Final new_new: 9.09 ± 2.78
Hits@30:
  Final val: 66.87 ± 1.23   Final Test: 51.07 ± 2.89   Final old_old: 55.16 ± 2.92   Final old_new: 34.76 ± 3.05   Final new_new: 11.36 ± 3.21
Hits@50:
  Final val: 74.67 ± 0.42   Final Test: 60.23 ± 1.19   Final old_old: 64.39 ± 1.37   Final old_new: 43.65 ± 0.72   Final new_new: 19.09 ± 6.55
AUC:
  Final val: 97.29 ± 0.13   Final Test: 96.81 ± 0.06   Final old_old: 97.44 ± 0.05   Final old_new: 94.30 ± 0.25   Final new_new: 90.49 ± 0.73
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.73 ± 2.84   Final Test: 30.62 ± 1.66   Final old_old: 33.84 ± 1.81   Final old_new: 17.46 ± 1.64   Final new_new: 4.55 ± 2.27
Hits@20:
  Final val: 61.70 ± 1.98   Final Test: 40.26 ± 2.60   Final old_old: 43.96 ± 2.76   Final old_new: 25.31 ± 2.36   Final new_new: 7.73 ± 2.03
Hits@30:
  Final val: 67.11 ± 0.77   Final Test: 46.96 ± 2.63   Final old_old: 50.74 ± 2.64   Final old_new: 31.86 ± 2.93   Final new_new: 10.45 ± 3.45
Hits@50:
  Final val: 73.42 ± 0.84   Final Test: 58.57 ± 2.18   Final old_old: 62.37 ± 2.32   Final old_new: 43.70 ± 1.82   Final new_new: 16.82 ± 7.81
AUC:
  Final val: 95.65 ± 0.30   Final Test: 95.23 ± 0.34   Final old_old: 95.92 ± 0.25   Final old_new: 92.59 ± 0.72   Final new_new: 86.60 ± 2.21
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.91 ± 4.36   Final Test: 32.30 ± 4.40   Final old_old: 35.41 ± 4.73   Final old_new: 19.77 ± 3.21   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 60.90 ± 1.21   Final Test: 42.25 ± 3.76   Final old_old: 45.76 ± 4.00   Final old_new: 28.31 ± 3.06   Final new_new: 7.27 ± 3.37
Hits@30:
  Final val: 68.09 ± 2.15   Final Test: 49.22 ± 3.08   Final old_old: 53.06 ± 3.34   Final old_new: 34.06 ± 2.37   Final new_new: 9.09 ± 4.25
Hits@50:
  Final val: 76.06 ± 2.40   Final Test: 61.19 ± 3.24   Final old_old: 65.44 ± 3.39   Final old_new: 44.53 ± 3.12   Final new_new: 14.55 ± 6.14
AUC:
  Final val: 97.28 ± 0.20   Final Test: 96.85 ± 0.11   Final old_old: 97.48 ± 0.16   Final old_new: 94.35 ± 0.14   Final new_new: 90.09 ± 1.24
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.96 ± 5.52   Final Test: 28.03 ± 4.60   Final old_old: 30.63 ± 5.01   Final old_new: 17.53 ± 3.33   Final new_new: 5.45 ± 3.45
Hits@20:
  Final val: 54.88 ± 5.21   Final Test: 37.50 ± 4.75   Final old_old: 40.65 ± 5.11   Final old_new: 24.86 ± 3.91   Final new_new: 8.18 ± 5.47
Hits@30:
  Final val: 61.89 ± 4.95   Final Test: 44.30 ± 3.90   Final old_old: 47.55 ± 4.31   Final old_new: 31.31 ± 2.82   Final new_new: 13.18 ± 4.93
Hits@50:
  Final val: 71.45 ± 3.56   Final Test: 53.29 ± 6.77   Final old_old: 57.02 ± 7.16   Final old_new: 38.34 ± 5.52   Final new_new: 18.64 ± 6.89
AUC:
  Final val: 96.60 ± 0.35   Final Test: 95.93 ± 0.25   Final old_old: 96.62 ± 0.27   Final old_new: 93.21 ± 0.24   Final new_new: 89.04 ± 0.99
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.20 ± 2.69   Final Test: 32.91 ± 4.91   Final old_old: 36.30 ± 5.17   Final old_new: 19.14 ± 4.01   Final new_new: 4.09 ± 2.49
Hits@20:
  Final val: 58.23 ± 3.17   Final Test: 40.94 ± 2.85   Final old_old: 44.44 ± 2.98   Final old_new: 27.05 ± 2.53   Final new_new: 5.91 ± 3.05
Hits@30:
  Final val: 65.37 ± 1.82   Final Test: 47.71 ± 5.40   Final old_old: 51.58 ± 5.59   Final old_new: 32.37 ± 4.79   Final new_new: 8.64 ± 2.49
Hits@50:
  Final val: 73.95 ± 0.78   Final Test: 58.15 ± 2.67   Final old_old: 62.35 ± 2.87   Final old_new: 41.71 ± 2.27   Final new_new: 12.27 ± 3.80
AUC:
  Final val: 97.27 ± 0.09   Final Test: 96.76 ± 0.12   Final old_old: 97.38 ± 0.15   Final old_new: 94.33 ± 0.23   Final new_new: 89.94 ± 0.89
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.93 ± 1.86   Final Test: 30.99 ± 4.29   Final old_old: 34.44 ± 4.37   Final old_new: 16.78 ± 3.99   Final new_new: 5.45 ± 6.55
Hits@20:
  Final val: 60.80 ± 2.08   Final Test: 44.23 ± 2.78   Final old_old: 47.90 ± 2.84   Final old_new: 29.45 ± 2.80   Final new_new: 11.36 ± 6.22
Hits@30:
  Final val: 66.62 ± 1.00   Final Test: 49.29 ± 3.88   Final old_old: 52.93 ± 3.85   Final old_new: 34.79 ± 4.06   Final new_new: 14.09 ± 5.43
Hits@50:
  Final val: 73.19 ± 0.20   Final Test: 58.39 ± 1.31   Final old_old: 61.96 ± 1.41   Final old_new: 44.43 ± 1.18   Final new_new: 18.18 ± 3.59
AUC:
  Final val: 95.34 ± 0.23   Final Test: 94.89 ± 0.26   Final old_old: 95.60 ± 0.22   Final old_new: 92.10 ± 0.59   Final new_new: 86.93 ± 1.56
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.12 ± 3.28   Final Test: 32.66 ± 2.12   Final old_old: 35.89 ± 2.07   Final old_new: 19.62 ± 2.52   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 59.71 ± 2.03   Final Test: 41.13 ± 3.10   Final old_old: 44.71 ± 3.18   Final old_new: 26.80 ± 2.96   Final new_new: 7.27 ± 2.49
Hits@30:
  Final val: 66.81 ± 1.06   Final Test: 46.77 ± 2.11   Final old_old: 50.72 ± 2.23   Final old_new: 31.06 ± 1.73   Final new_new: 8.64 ± 1.90
Hits@50:
  Final val: 74.31 ± 0.43   Final Test: 57.79 ± 2.88   Final old_old: 62.11 ± 3.10   Final old_new: 40.60 ± 2.08   Final new_new: 15.00 ± 3.05
AUC:
  Final val: 97.24 ± 0.10   Final Test: 96.72 ± 0.07   Final old_old: 97.34 ± 0.08   Final old_new: 94.26 ± 0.11   Final new_new: 90.36 ± 0.49
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.37 ± 2.81   Final Test: 28.81 ± 3.39   Final old_old: 31.28 ± 3.60   Final old_new: 18.72 ± 2.92   Final new_new: 9.55 ± 3.37
Hits@20:
  Final val: 56.70 ± 2.18   Final Test: 37.59 ± 3.34   Final old_old: 40.63 ± 3.55   Final old_new: 25.34 ± 2.88   Final new_new: 10.00 ± 3.80
Hits@30:
  Final val: 63.61 ± 2.46   Final Test: 44.07 ± 3.78   Final old_old: 47.25 ± 3.97   Final old_new: 31.51 ± 3.18   Final new_new: 11.36 ± 4.55
Hits@50:
  Final val: 71.86 ± 2.36   Final Test: 56.87 ± 3.81   Final old_old: 60.47 ± 4.18   Final old_new: 42.70 ± 2.50   Final new_new: 18.18 ± 5.57
AUC:
  Final val: 96.64 ± 0.30   Final Test: 95.98 ± 0.21   Final old_old: 96.61 ± 0.27   Final old_new: 93.44 ± 0.12   Final new_new: 90.15 ± 1.43
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.25 ± 2.38   Final Test: 33.22 ± 3.80   Final old_old: 36.40 ± 4.13   Final old_new: 20.45 ± 2.85   Final new_new: 3.64 ± 2.03
Hits@20:
  Final val: 64.46 ± 3.78   Final Test: 43.05 ± 5.69   Final old_old: 46.68 ± 6.26   Final old_new: 28.54 ± 3.87   Final new_new: 8.18 ± 2.59
Hits@30:
  Final val: 70.66 ± 2.86   Final Test: 52.72 ± 2.18   Final old_old: 56.96 ± 2.31   Final old_new: 35.92 ± 2.67   Final new_new: 9.55 ± 2.49
Hits@50:
  Final val: 77.46 ± 1.43   Final Test: 61.51 ± 2.22   Final old_old: 65.89 ± 2.35   Final old_new: 44.21 ± 2.46   Final new_new: 15.91 ± 2.78
AUC:
  Final val: 97.39 ± 0.10   Final Test: 96.79 ± 0.13   Final old_old: 97.44 ± 0.12   Final old_new: 94.20 ± 0.33   Final new_new: 90.60 ± 1.25
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.54 ± 3.26   Final Test: 26.90 ± 1.85   Final old_old: 29.41 ± 2.22   Final old_new: 16.73 ± 0.74   Final new_new: 5.45 ± 1.24
Hits@20:
  Final val: 57.32 ± 3.13   Final Test: 39.35 ± 2.79   Final old_old: 42.59 ± 3.28   Final old_new: 26.32 ± 1.36   Final new_new: 9.55 ± 3.37
Hits@30:
  Final val: 62.83 ± 2.31   Final Test: 46.11 ± 2.34   Final old_old: 49.57 ± 2.90   Final old_new: 32.14 ± 0.89   Final new_new: 15.00 ± 4.13
Hits@50:
  Final val: 69.83 ± 1.68   Final Test: 57.88 ± 2.54   Final old_old: 61.37 ± 2.60   Final old_new: 44.13 ± 2.84   Final new_new: 21.36 ± 4.13
AUC:
  Final val: 95.03 ± 0.27   Final Test: 94.65 ± 0.40   Final old_old: 95.39 ± 0.28   Final old_new: 91.83 ± 0.99   Final new_new: 85.43 ± 2.52
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.86 ± 5.79   Final Test: 28.54 ± 6.10   Final old_old: 30.95 ± 6.67   Final old_new: 18.79 ± 4.00   Final new_new: 7.27 ± 3.73
Hits@20:
  Final val: 53.97 ± 6.27   Final Test: 37.51 ± 3.92   Final old_old: 40.21 ± 4.45   Final old_new: 26.70 ± 2.12   Final new_new: 12.27 ± 3.05
Hits@30:
  Final val: 60.70 ± 5.36   Final Test: 43.23 ± 6.04   Final old_old: 46.28 ± 6.64   Final old_new: 31.03 ± 3.81   Final new_new: 14.55 ± 5.23
Hits@50:
  Final val: 69.83 ± 3.77   Final Test: 52.44 ± 2.72   Final old_old: 55.88 ± 3.14   Final old_new: 38.92 ± 1.60   Final new_new: 16.36 ± 7.43
AUC:
  Final val: 96.53 ± 0.42   Final Test: 95.95 ± 0.28   Final old_old: 96.58 ± 0.34   Final old_new: 93.45 ± 0.32   Final new_new: 89.84 ± 1.29
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.91 ± 1.72   Final Test: 32.02 ± 6.38   Final old_old: 35.31 ± 6.91   Final old_new: 18.66 ± 4.31   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 59.89 ± 0.87   Final Test: 40.84 ± 3.62   Final old_old: 44.31 ± 3.58   Final old_new: 26.73 ± 3.88   Final new_new: 11.82 ± 5.66
Hits@30:
  Final val: 65.14 ± 0.69   Final Test: 49.14 ± 2.52   Final old_old: 52.72 ± 2.49   Final old_new: 35.04 ± 2.87   Final new_new: 11.36 ± 5.57
Hits@50:
  Final val: 71.66 ± 1.01   Final Test: 57.71 ± 1.45   Final old_old: 61.24 ± 1.39   Final old_new: 43.98 ± 2.15   Final new_new: 16.82 ± 7.29
AUC:
  Final val: 94.90 ± 0.41   Final Test: 94.55 ± 0.26   Final old_old: 95.17 ± 0.27   Final old_new: 92.16 ± 0.31   Final new_new: 86.40 ± 1.75
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.81 ± 3.02   Final Test: 33.20 ± 5.76   Final old_old: 36.42 ± 6.08   Final old_new: 20.28 ± 4.66   Final new_new: 4.09 ± 2.96
Hits@20:
  Final val: 62.85 ± 0.75   Final Test: 43.66 ± 2.89   Final old_old: 47.48 ± 3.14   Final old_new: 28.31 ± 1.96   Final new_new: 8.18 ± 4.43
Hits@30:
  Final val: 69.71 ± 1.41   Final Test: 49.23 ± 1.73   Final old_old: 53.21 ± 1.64   Final old_new: 33.45 ± 2.58   Final new_new: 8.18 ± 2.59
Hits@50:
  Final val: 76.85 ± 1.72   Final Test: 60.65 ± 1.22   Final old_old: 64.88 ± 1.41   Final old_new: 44.13 ± 1.09   Final new_new: 13.18 ± 3.73
AUC:
  Final val: 97.37 ± 0.19   Final Test: 96.83 ± 0.18   Final old_old: 97.48 ± 0.18   Final old_new: 94.30 ± 0.24   Final new_new: 89.95 ± 1.08
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.51 ± 3.20   Final Test: 32.22 ± 2.57   Final old_old: 35.54 ± 2.66   Final old_new: 18.74 ± 2.63   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 59.07 ± 2.50   Final Test: 39.16 ± 1.54   Final old_old: 42.88 ± 1.60   Final old_new: 24.16 ± 2.24   Final new_new: 6.82 ± 2.27
Hits@30:
  Final val: 66.30 ± 1.98   Final Test: 47.06 ± 1.65   Final old_old: 51.21 ± 1.70   Final old_new: 30.38 ± 2.44   Final new_new: 9.09 ± 5.08
Hits@50:
  Final val: 74.49 ± 0.86   Final Test: 58.21 ± 1.75   Final old_old: 62.33 ± 1.86   Final old_new: 42.07 ± 2.02   Final new_new: 13.18 ± 1.90
AUC:
  Final val: 97.26 ± 0.11   Final Test: 96.75 ± 0.04   Final old_old: 97.35 ± 0.09   Final old_new: 94.31 ± 0.32   Final new_new: 91.07 ± 0.49
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.98 ± 3.35   Final Test: 28.03 ± 3.29   Final old_old: 31.04 ± 3.44   Final old_new: 15.64 ± 2.95   Final new_new: 5.45 ± 2.03
Hits@20:
  Final val: 59.12 ± 1.65   Final Test: 40.51 ± 3.15   Final old_old: 44.01 ± 3.32   Final old_new: 26.47 ± 2.79   Final new_new: 7.73 ± 3.05
Hits@30:
  Final val: 64.48 ± 0.95   Final Test: 47.24 ± 1.86   Final old_old: 50.66 ± 1.79   Final old_new: 33.75 ± 2.45   Final new_new: 10.91 ± 2.49
Hits@50:
  Final val: 70.76 ± 0.58   Final Test: 57.88 ± 1.16   Final old_old: 61.46 ± 1.29   Final old_new: 43.93 ± 1.12   Final new_new: 17.27 ± 3.45
AUC:
  Final val: 94.71 ± 0.21   Final Test: 94.43 ± 0.35   Final old_old: 95.10 ± 0.30   Final old_new: 91.87 ± 0.74   Final new_new: 85.80 ± 1.79
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.06 ± 2.14   Final Test: 33.36 ± 6.41   Final old_old: 36.70 ± 6.96   Final old_new: 19.85 ± 4.40   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 64.71 ± 1.69   Final Test: 45.25 ± 4.62   Final old_old: 49.18 ± 5.08   Final old_new: 29.52 ± 2.98   Final new_new: 8.18 ± 3.05
Hits@30:
  Final val: 70.68 ± 2.28   Final Test: 54.11 ± 2.79   Final old_old: 58.58 ± 3.06   Final old_new: 36.40 ± 2.09   Final new_new: 9.09 ± 2.78
Hits@50:
  Final val: 76.28 ± 2.49   Final Test: 64.18 ± 1.91   Final old_old: 68.68 ± 2.16   Final old_new: 46.55 ± 1.24   Final new_new: 15.45 ± 3.73
AUC:
  Final val: 97.40 ± 0.17   Final Test: 96.91 ± 0.17   Final old_old: 97.58 ± 0.16   Final old_new: 94.22 ± 0.32   Final new_new: 90.61 ± 1.03
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.10 ± 5.43   Final Test: 24.08 ± 3.28   Final old_old: 26.25 ± 3.51   Final old_new: 15.21 ± 2.39   Final new_new: 6.36 ± 4.07
Hits@20:
  Final val: 54.19 ± 2.61   Final Test: 34.20 ± 2.08   Final old_old: 36.98 ± 2.26   Final old_new: 23.10 ± 1.71   Final new_new: 8.18 ± 4.13
Hits@30:
  Final val: 62.39 ± 1.34   Final Test: 44.96 ± 2.17   Final old_old: 48.39 ± 2.68   Final old_new: 31.49 ± 1.28   Final new_new: 8.18 ± 4.13
Hits@50:
  Final val: 71.59 ± 0.54   Final Test: 53.91 ± 3.22   Final old_old: 57.54 ± 3.50   Final old_new: 39.77 ± 3.13   Final new_new: 12.73 ± 7.11
AUC:
  Final val: 96.58 ± 0.26   Final Test: 95.88 ± 0.32   Final old_old: 96.53 ± 0.35   Final old_new: 93.31 ± 0.36   Final new_new: 89.82 ± 1.34
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.45 ± 1.86   Final Test: 27.69 ± 3.14   Final old_old: 30.75 ± 3.30   Final old_new: 15.24 ± 3.37   Final new_new: 2.73 ± 1.02
Hits@20:
  Final val: 61.25 ± 1.85   Final Test: 41.08 ± 2.91   Final old_old: 44.83 ± 2.83   Final old_new: 26.10 ± 3.55   Final new_new: 5.91 ± 3.05
Hits@30:
  Final val: 66.74 ± 1.34   Final Test: 47.80 ± 3.41   Final old_old: 51.72 ± 3.52   Final old_new: 32.22 ± 3.27   Final new_new: 8.64 ± 5.18
Hits@50:
  Final val: 74.18 ± 1.60   Final Test: 57.38 ± 1.82   Final old_old: 61.65 ± 1.80   Final old_new: 40.45 ± 2.32   Final new_new: 14.55 ± 6.14
AUC:
  Final val: 97.29 ± 0.09   Final Test: 96.75 ± 0.10   Final old_old: 97.37 ± 0.09   Final old_new: 94.31 ± 0.28   Final new_new: 90.67 ± 0.88
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.94 ± 3.54   Final Test: 28.53 ± 9.67   Final old_old: 30.87 ± 10.44   Final old_new: 19.09 ± 6.65   Final new_new: 7.73 ± 4.13
Hits@20:
  Final val: 60.57 ± 3.41   Final Test: 41.06 ± 4.77   Final old_old: 44.06 ± 5.04   Final old_new: 29.17 ± 3.79   Final new_new: 10.91 ± 5.43
Hits@30:
  Final val: 67.48 ± 2.21   Final Test: 50.37 ± 2.92   Final old_old: 53.93 ± 2.99   Final old_new: 36.25 ± 3.00   Final new_new: 14.55 ± 7.29
Hits@50:
  Final val: 75.30 ± 1.07   Final Test: 58.82 ± 4.24   Final old_old: 62.58 ± 4.16   Final old_new: 43.98 ± 4.61   Final new_new: 19.55 ± 8.74
AUC:
  Final val: 96.77 ± 0.14   Final Test: 96.31 ± 0.12   Final old_old: 96.91 ± 0.07   Final old_new: 93.91 ± 0.48   Final new_new: 89.88 ± 0.74
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.52 ± 3.04   Final Test: 32.35 ± 4.02   Final old_old: 35.49 ± 4.50   Final old_new: 19.67 ± 2.14   Final new_new: 5.00 ± 2.96
Hits@20:
  Final val: 56.42 ± 3.18   Final Test: 40.04 ± 5.36   Final old_old: 43.56 ± 5.75   Final old_new: 25.82 ± 4.45   Final new_new: 8.64 ± 4.07
Hits@30:
  Final val: 64.06 ± 1.91   Final Test: 46.50 ± 2.13   Final old_old: 50.26 ± 2.24   Final old_new: 31.46 ± 1.79   Final new_new: 10.91 ± 6.31
Hits@50:
  Final val: 73.19 ± 2.12   Final Test: 56.88 ± 2.45   Final old_old: 60.94 ± 2.59   Final old_new: 40.76 ± 2.36   Final new_new: 15.91 ± 5.08
AUC:
  Final val: 97.14 ± 0.13   Final Test: 96.62 ± 0.13   Final old_old: 97.20 ± 0.17   Final old_new: 94.32 ± 0.15   Final new_new: 90.95 ± 0.40
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.15 ± 2.26   Final Test: 33.15 ± 4.82   Final old_old: 36.25 ± 5.08   Final old_new: 20.65 ± 3.94   Final new_new: 5.45 ± 3.05
Hits@20:
  Final val: 61.61 ± 2.23   Final Test: 44.63 ± 5.22   Final old_old: 48.41 ± 5.44   Final old_new: 29.55 ± 4.48   Final new_new: 7.73 ± 5.23
Hits@30:
  Final val: 68.20 ± 1.89   Final Test: 48.29 ± 3.68   Final old_old: 52.26 ± 3.85   Final old_new: 32.47 ± 3.39   Final new_new: 9.09 ± 2.27
Hits@50:
  Final val: 75.90 ± 1.65   Final Test: 60.57 ± 1.22   Final old_old: 64.94 ± 1.47   Final old_new: 43.35 ± 0.46   Final new_new: 15.00 ± 3.45
AUC:
  Final val: 97.34 ± 0.12   Final Test: 96.78 ± 0.15   Final old_old: 97.44 ± 0.14   Final old_new: 94.15 ± 0.24   Final new_new: 90.61 ± 0.99
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.67 ± 5.20   Final Test: 30.30 ± 4.71   Final old_old: 32.93 ± 5.12   Final old_new: 19.72 ± 3.72   Final new_new: 6.82 ± 4.55
Hits@20:
  Final val: 52.16 ± 5.57   Final Test: 33.97 ± 4.62   Final old_old: 36.84 ± 5.17   Final old_new: 22.47 ± 2.61   Final new_new: 7.27 ± 3.73
Hits@30:
  Final val: 60.63 ± 4.67   Final Test: 44.02 ± 4.76   Final old_old: 47.20 ± 5.22   Final old_new: 31.44 ± 4.02   Final new_new: 10.91 ± 4.37
Hits@50:
  Final val: 70.93 ± 3.55   Final Test: 55.62 ± 3.96   Final old_old: 59.22 ± 4.09   Final old_new: 41.49 ± 3.91   Final new_new: 17.27 ± 6.55
AUC:
  Final val: 96.56 ± 0.28   Final Test: 95.85 ± 0.20   Final old_old: 96.50 ± 0.23   Final old_new: 93.26 ± 0.32   Final new_new: 89.24 ± 2.44
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.52 ± 3.25   Final Test: 29.11 ± 6.08   Final old_old: 31.83 ± 6.60   Final old_new: 18.31 ± 4.16   Final new_new: 1.82 ± 1.90
Hits@20:
  Final val: 64.45 ± 2.54   Final Test: 42.60 ± 2.62   Final old_old: 45.91 ± 2.55   Final old_new: 29.57 ± 3.20   Final new_new: 7.27 ± 3.73
Hits@30:
  Final val: 70.68 ± 2.02   Final Test: 53.68 ± 2.79   Final old_old: 57.29 ± 2.67   Final old_new: 39.67 ± 3.46   Final new_new: 11.82 ± 4.07
Hits@50:
  Final val: 77.25 ± 1.07   Final Test: 63.18 ± 1.07   Final old_old: 66.85 ± 1.31   Final old_new: 49.09 ± 1.04   Final new_new: 17.73 ± 1.90
AUC:
  Final val: 97.25 ± 0.06   Final Test: 96.83 ± 0.15   Final old_old: 97.30 ± 0.14   Final old_new: 94.98 ± 0.32   Final new_new: 91.55 ± 1.13
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.95 ± 3.04   Final Test: 33.83 ± 5.13   Final old_old: 37.08 ± 5.22   Final old_new: 20.76 ± 4.99   Final new_new: 5.00 ± 2.96
Hits@20:
  Final val: 58.28 ± 2.75   Final Test: 40.77 ± 2.91   Final old_old: 44.37 ± 3.15   Final old_new: 26.42 ± 2.45   Final new_new: 5.45 ± 1.24
Hits@30:
  Final val: 66.10 ± 2.78   Final Test: 45.52 ± 2.39   Final old_old: 49.40 ± 2.58   Final old_new: 30.18 ± 1.76   Final new_new: 5.91 ± 2.03
Hits@50:
  Final val: 74.41 ± 1.38   Final Test: 57.54 ± 2.32   Final old_old: 61.75 ± 2.46   Final old_new: 40.78 ± 2.15   Final new_new: 15.45 ± 2.49
AUC:
  Final val: 97.25 ± 0.15   Final Test: 96.75 ± 0.07   Final old_old: 97.33 ± 0.15   Final old_new: 94.47 ± 0.24   Final new_new: 90.26 ± 1.05
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.11 ± 5.62   Final Test: 29.48 ± 5.68   Final old_old: 32.11 ± 6.01   Final old_new: 18.89 ± 4.52   Final new_new: 5.91 ± 4.43
Hits@20:
  Final val: 59.08 ± 4.47   Final Test: 38.65 ± 3.43   Final old_old: 41.72 ± 3.49   Final old_new: 26.40 ± 3.35   Final new_new: 8.64 ± 4.37
Hits@30:
  Final val: 66.25 ± 2.53   Final Test: 47.29 ± 3.76   Final old_old: 50.83 ± 3.61   Final old_new: 33.20 ± 4.59   Final new_new: 12.27 ± 5.23
Hits@50:
  Final val: 73.58 ± 1.17   Final Test: 57.35 ± 1.48   Final old_old: 61.00 ± 1.66   Final old_new: 43.05 ± 1.67   Final new_new: 18.18 ± 8.19
AUC:
  Final val: 96.66 ± 0.25   Final Test: 95.93 ± 0.25   Final old_old: 96.63 ± 0.27   Final old_new: 93.20 ± 0.28   Final new_new: 88.53 ± 1.84
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.98 ± 2.78   Final Test: 36.49 ± 6.85   Final old_old: 40.04 ± 7.41   Final old_new: 22.12 ± 4.72   Final new_new: 5.45 ± 2.59
Hits@20:
  Final val: 61.80 ± 0.75   Final Test: 46.33 ± 1.62   Final old_old: 50.27 ± 1.51   Final old_new: 30.65 ± 2.47   Final new_new: 7.27 ± 1.90
Hits@30:
  Final val: 69.15 ± 1.41   Final Test: 52.64 ± 2.91   Final old_old: 56.82 ± 3.16   Final old_new: 36.07 ± 2.47   Final new_new: 10.45 ± 2.03
Hits@50:
  Final val: 76.61 ± 1.98   Final Test: 61.32 ± 3.44   Final old_old: 65.61 ± 3.88   Final old_new: 44.58 ± 2.00   Final new_new: 13.64 ± 3.59
AUC:
  Final val: 97.34 ± 0.14   Final Test: 96.81 ± 0.18   Final old_old: 97.46 ± 0.22   Final old_new: 94.28 ± 0.03   Final new_new: 89.31 ± 1.50
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.75 ± 3.13   Final Test: 33.07 ± 4.01   Final old_old: 36.06 ± 4.19   Final old_new: 21.18 ± 3.41   Final new_new: 3.18 ± 3.80
Hits@20:
  Final val: 63.53 ± 0.83   Final Test: 43.75 ± 3.02   Final old_old: 47.21 ± 2.96   Final old_new: 30.10 ± 3.53   Final new_new: 7.73 ± 3.80
Hits@30:
  Final val: 69.46 ± 0.66   Final Test: 52.52 ± 1.02   Final old_old: 56.36 ± 1.10   Final old_new: 37.66 ± 1.38   Final new_new: 6.82 ± 3.59
Hits@50:
  Final val: 76.89 ± 1.14   Final Test: 61.61 ± 1.42   Final old_old: 65.46 ± 1.60   Final old_new: 46.75 ± 1.04   Final new_new: 15.00 ± 2.03
AUC:
  Final val: 97.45 ± 0.07   Final Test: 97.02 ± 0.08   Final old_old: 97.51 ± 0.09   Final old_new: 95.14 ± 0.16   Final new_new: 91.41 ± 0.69
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.47 ± 3.03   Final Test: 26.51 ± 6.29   Final old_old: 28.97 ± 6.81   Final old_new: 16.50 ± 4.36   Final new_new: 6.82 ± 5.33
Hits@20:
  Final val: 55.87 ± 3.84   Final Test: 36.71 ± 1.62   Final old_old: 39.63 ± 1.74   Final old_new: 24.94 ± 1.86   Final new_new: 10.00 ± 5.47
Hits@30:
  Final val: 63.20 ± 4.05   Final Test: 43.14 ± 4.21   Final old_old: 46.42 ± 4.45   Final old_new: 29.95 ± 3.61   Final new_new: 12.73 ± 4.43
Hits@50:
  Final val: 71.99 ± 3.26   Final Test: 54.58 ± 3.60   Final old_old: 58.31 ± 3.60   Final old_new: 39.82 ± 4.06   Final new_new: 15.91 ± 5.79
AUC:
  Final val: 96.66 ± 0.28   Final Test: 95.91 ± 0.26   Final old_old: 96.57 ± 0.25   Final old_new: 93.26 ± 0.41   Final new_new: 90.07 ± 0.93
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.36 ± 3.32   Final Test: 31.01 ± 3.89   Final old_old: 34.22 ± 4.08   Final old_new: 17.98 ± 3.31   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 60.38 ± 2.17   Final Test: 41.35 ± 3.44   Final old_old: 45.01 ± 3.66   Final old_new: 26.68 ± 3.04   Final new_new: 7.27 ± 2.96
Hits@30:
  Final val: 66.35 ± 2.28   Final Test: 47.70 ± 2.86   Final old_old: 51.72 ± 2.90   Final old_new: 31.74 ± 2.92   Final new_new: 7.73 ± 2.59
Hits@50:
  Final val: 73.94 ± 1.09   Final Test: 58.10 ± 2.09   Final old_old: 62.23 ± 2.21   Final old_new: 41.71 ± 1.66   Final new_new: 16.36 ± 7.08
AUC:
  Final val: 97.29 ± 0.10   Final Test: 96.77 ± 0.19   Final old_old: 97.38 ± 0.18   Final old_new: 94.31 ± 0.38   Final new_new: 90.99 ± 0.34
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.44 ± 1.66   Final Test: 30.07 ± 4.80   Final old_old: 33.14 ± 5.01   Final old_new: 17.66 ± 4.21   Final new_new: 2.73 ± 1.90
Hits@20:
  Final val: 63.88 ± 1.48   Final Test: 44.87 ± 4.78   Final old_old: 48.92 ± 5.20   Final old_new: 28.79 ± 4.04   Final new_new: 4.55 ± 1.61
Hits@30:
  Final val: 69.89 ± 2.03   Final Test: 53.19 ± 3.84   Final old_old: 57.37 ± 4.34   Final old_new: 36.83 ± 2.07   Final new_new: 7.27 ± 1.02
Hits@50:
  Final val: 76.74 ± 1.68   Final Test: 62.36 ± 3.87   Final old_old: 66.69 ± 4.03   Final old_new: 45.57 ± 3.77   Final new_new: 11.82 ± 4.93
AUC:
  Final val: 97.40 ± 0.18   Final Test: 96.81 ± 0.18   Final old_old: 97.46 ± 0.22   Final old_new: 94.19 ± 0.20   Final new_new: 90.90 ± 1.19
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.31 ± 5.82   Final Test: 27.73 ± 10.34   Final old_old: 30.12 ± 11.21   Final old_new: 18.26 ± 7.02   Final new_new: 3.64 ± 2.59
Hits@20:
  Final val: 58.31 ± 4.23   Final Test: 41.92 ± 2.15   Final old_old: 45.13 ± 2.30   Final old_new: 29.17 ± 1.87   Final new_new: 9.55 ± 2.96
Hits@30:
  Final val: 64.62 ± 3.23   Final Test: 48.71 ± 3.69   Final old_old: 52.13 ± 4.06   Final old_new: 35.34 ± 2.36   Final new_new: 10.45 ± 4.13
Hits@50:
  Final val: 73.61 ± 1.97   Final Test: 57.97 ± 4.74   Final old_old: 61.97 ± 4.93   Final old_new: 42.27 ± 4.27   Final new_new: 14.55 ± 5.47
AUC:
  Final val: 96.83 ± 0.20   Final Test: 96.11 ± 0.20   Final old_old: 96.73 ± 0.15   Final old_new: 93.63 ± 0.48   Final new_new: 89.49 ± 3.42
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.39 ± 4.67   Final Test: 33.43 ± 3.60   Final old_old: 36.58 ± 4.12   Final old_new: 20.83 ± 1.71   Final new_new: 3.64 ± 3.05
Hits@20:
  Final val: 62.87 ± 3.62   Final Test: 43.97 ± 3.68   Final old_old: 47.67 ± 4.11   Final old_new: 29.27 ± 2.70   Final new_new: 6.82 ± 4.25
Hits@30:
  Final val: 68.77 ± 2.61   Final Test: 50.14 ± 5.47   Final old_old: 54.03 ± 5.86   Final old_new: 34.79 ± 4.29   Final new_new: 9.09 ± 4.55
Hits@50:
  Final val: 76.30 ± 1.66   Final Test: 59.53 ± 2.47   Final old_old: 63.75 ± 2.71   Final old_new: 43.00 ± 1.84   Final new_new: 13.64 ± 6.01
AUC:
  Final val: 97.38 ± 0.19   Final Test: 96.83 ± 0.16   Final old_old: 97.47 ± 0.18   Final old_new: 94.30 ± 0.27   Final new_new: 90.33 ± 0.83
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.31 ± 4.25   Final Test: 28.48 ± 5.53   Final old_old: 30.99 ± 6.03   Final old_new: 18.44 ± 3.70   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 54.91 ± 4.10   Final Test: 37.83 ± 4.79   Final old_old: 40.91 ± 5.05   Final old_new: 25.42 ± 4.39   Final new_new: 10.45 ± 4.71
Hits@30:
  Final val: 61.30 ± 3.32   Final Test: 45.59 ± 4.62   Final old_old: 48.95 ± 4.63   Final old_new: 32.32 ± 4.99   Final new_new: 10.45 ± 4.71
Hits@50:
  Final val: 70.95 ± 2.31   Final Test: 54.95 ± 3.48   Final old_old: 58.74 ± 3.48   Final old_new: 40.05 ± 3.95   Final new_new: 14.09 ± 3.73
AUC:
  Final val: 96.63 ± 0.26   Final Test: 95.91 ± 0.27   Final old_old: 96.57 ± 0.29   Final old_new: 93.25 ± 0.51   Final new_new: 89.42 ± 2.48
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.72 ± 2.33   Final Test: 28.14 ± 5.52   Final old_old: 31.12 ± 6.01   Final old_new: 15.97 ± 3.75   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 61.30 ± 1.87   Final Test: 39.94 ± 4.43   Final old_old: 43.53 ± 4.73   Final old_new: 25.52 ± 3.44   Final new_new: 6.36 ± 1.90
Hits@30:
  Final val: 67.10 ± 1.29   Final Test: 48.09 ± 2.07   Final old_old: 52.12 ± 1.91   Final old_new: 32.04 ± 2.95   Final new_new: 8.64 ± 1.90
Hits@50:
  Final val: 74.31 ± 0.47   Final Test: 58.70 ± 2.21   Final old_old: 62.89 ± 2.14   Final old_new: 42.14 ± 2.65   Final new_new: 14.55 ± 2.59
AUC:
  Final val: 97.28 ± 0.11   Final Test: 96.69 ± 0.10   Final old_old: 97.32 ± 0.11   Final old_new: 94.22 ± 0.22   Final new_new: 89.76 ± 0.76
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.99 ± 3.19   Final Test: 32.36 ± 2.83   Final old_old: 35.42 ± 3.03   Final old_new: 20.08 ± 2.47   Final new_new: 4.09 ± 1.02
Hits@20:
  Final val: 64.26 ± 2.95   Final Test: 41.72 ± 1.83   Final old_old: 45.22 ± 1.95   Final old_new: 27.81 ± 1.60   Final new_new: 7.27 ± 2.49
Hits@30:
  Final val: 70.22 ± 1.93   Final Test: 53.07 ± 2.80   Final old_old: 57.17 ± 2.74   Final old_new: 37.03 ± 3.27   Final new_new: 7.73 ± 1.24
Hits@50:
  Final val: 76.78 ± 1.29   Final Test: 61.71 ± 2.34   Final old_old: 66.02 ± 2.11   Final old_new: 44.74 ± 3.34   Final new_new: 15.91 ± 4.82
AUC:
  Final val: 97.52 ± 0.10   Final Test: 97.03 ± 0.09   Final old_old: 97.60 ± 0.05   Final old_new: 94.78 ± 0.33   Final new_new: 91.04 ± 0.85
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.60 ± 3.23   Final Test: 34.77 ± 3.55   Final old_old: 38.12 ± 3.90   Final old_new: 21.26 ± 2.46   Final new_new: 5.00 ± 1.90
Hits@20:
  Final val: 59.90 ± 2.13   Final Test: 43.54 ± 2.20   Final old_old: 47.36 ± 2.55   Final old_new: 28.16 ± 1.05   Final new_new: 8.64 ± 1.90
Hits@30:
  Final val: 66.21 ± 1.89   Final Test: 47.29 ± 3.11   Final old_old: 51.17 ± 3.22   Final old_new: 31.94 ± 2.68   Final new_new: 7.73 ± 3.05
Hits@50:
  Final val: 74.52 ± 1.12   Final Test: 59.20 ± 3.68   Final old_old: 63.52 ± 3.62   Final old_new: 42.02 ± 4.14   Final new_new: 15.91 ± 6.22
AUC:
  Final val: 97.28 ± 0.16   Final Test: 96.76 ± 0.07   Final old_old: 97.39 ± 0.06   Final old_new: 94.27 ± 0.15   Final new_new: 90.21 ± 0.90
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.70 ± 2.37   Final Test: 36.46 ± 4.36   Final old_old: 39.73 ± 4.71   Final old_new: 23.38 ± 3.05   Final new_new: 5.45 ± 1.24
Hits@20:
  Final val: 62.22 ± 1.21   Final Test: 44.25 ± 1.00   Final old_old: 47.97 ± 1.13   Final old_new: 29.40 ± 1.23   Final new_new: 8.64 ± 1.90
Hits@30:
  Final val: 68.50 ± 0.37   Final Test: 52.26 ± 1.11   Final old_old: 56.24 ± 1.32   Final old_new: 36.57 ± 1.35   Final new_new: 10.91 ± 4.07
Hits@50:
  Final val: 76.68 ± 0.74   Final Test: 61.40 ± 1.02   Final old_old: 65.59 ± 0.91   Final old_new: 44.94 ± 1.90   Final new_new: 15.91 ± 6.63
AUC:
  Final val: 97.36 ± 0.08   Final Test: 96.83 ± 0.12   Final old_old: 97.44 ± 0.14   Final old_new: 94.41 ± 0.22   Final new_new: 90.24 ± 0.89
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.27 ± 2.91   Final Test: 30.97 ± 5.16   Final old_old: 34.03 ± 5.47   Final old_new: 18.64 ± 3.97   Final new_new: 3.64 ± 2.03
Hits@20:
  Final val: 60.06 ± 1.89   Final Test: 41.53 ± 3.19   Final old_old: 45.11 ± 3.39   Final old_new: 27.33 ± 2.36   Final new_new: 5.00 ± 2.96
Hits@30:
  Final val: 65.89 ± 2.02   Final Test: 48.75 ± 1.24   Final old_old: 52.46 ± 1.21   Final old_new: 34.08 ± 1.63   Final new_new: 10.45 ± 1.24
Hits@50:
  Final val: 74.52 ± 1.43   Final Test: 58.21 ± 1.48   Final old_old: 62.30 ± 1.30   Final old_new: 42.17 ± 2.28   Final new_new: 13.18 ± 3.37
AUC:
  Final val: 97.39 ± 0.10   Final Test: 96.88 ± 0.10   Final old_old: 97.41 ± 0.11   Final old_new: 94.82 ± 0.15   Final new_new: 91.07 ± 0.37
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.06 ± 3.43   Final Test: 27.80 ± 5.13   Final old_old: 30.20 ± 5.52   Final old_new: 18.14 ± 3.74   Final new_new: 5.91 ± 3.80
Hits@20:
  Final val: 55.46 ± 3.07   Final Test: 37.98 ± 3.85   Final old_old: 41.11 ± 4.14   Final old_new: 25.44 ± 3.14   Final new_new: 8.18 ± 4.13
Hits@30:
  Final val: 63.38 ± 2.54   Final Test: 47.26 ± 2.22   Final old_old: 50.78 ± 2.54   Final old_new: 33.27 ± 2.50   Final new_new: 12.27 ± 4.13
Hits@50:
  Final val: 72.33 ± 1.84   Final Test: 55.02 ± 2.75   Final old_old: 58.64 ± 3.14   Final old_new: 40.81 ± 2.01   Final new_new: 16.36 ± 2.49
AUC:
  Final val: 96.69 ± 0.27   Final Test: 95.97 ± 0.17   Final old_old: 96.63 ± 0.21   Final old_new: 93.35 ± 0.12   Final new_new: 89.71 ± 1.73
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 40.82 ± 4.14   Final Test: 25.33 ± 5.39   Final old_old: 27.65 ± 5.71   Final old_new: 15.97 ± 4.31   Final new_new: 5.00 ± 2.49
Hits@20:
  Final val: 54.98 ± 3.73   Final Test: 37.18 ± 2.75   Final old_old: 40.08 ± 2.86   Final old_new: 25.52 ± 2.56   Final new_new: 10.91 ± 1.90
Hits@30:
  Final val: 62.14 ± 2.93   Final Test: 45.18 ± 2.83   Final old_old: 48.31 ± 2.97   Final old_new: 32.80 ± 3.09   Final new_new: 12.73 ± 7.64
Hits@50:
  Final val: 70.85 ± 1.53   Final Test: 54.36 ± 1.69   Final old_old: 57.77 ± 1.79   Final old_new: 40.93 ± 2.07   Final new_new: 18.18 ± 5.57
AUC:
  Final val: 96.71 ± 0.16   Final Test: 96.03 ± 0.24   Final old_old: 96.69 ± 0.22   Final old_new: 93.49 ± 0.32   Final new_new: 88.52 ± 2.72
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.67 ± 5.57   Final Test: 31.06 ± 3.71   Final old_old: 34.14 ± 3.96   Final old_new: 18.54 ± 2.96   Final new_new: 5.00 ± 2.49
Hits@20:
  Final val: 59.12 ± 3.36   Final Test: 40.45 ± 2.62   Final old_old: 43.76 ± 2.73   Final old_new: 27.20 ± 2.26   Final new_new: 9.55 ± 3.37
Hits@30:
  Final val: 66.04 ± 2.77   Final Test: 47.22 ± 5.18   Final old_old: 50.98 ± 5.48   Final old_new: 32.29 ± 4.31   Final new_new: 9.09 ± 1.61
Hits@50:
  Final val: 74.21 ± 1.84   Final Test: 57.65 ± 4.93   Final old_old: 61.90 ± 5.17   Final old_new: 40.86 ± 4.19   Final new_new: 13.64 ± 2.27
AUC:
  Final val: 97.21 ± 0.14   Final Test: 96.67 ± 0.20   Final old_old: 97.28 ± 0.24   Final old_new: 94.23 ± 0.16   Final new_new: 90.40 ± 1.11
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.66 ± 5.72   Final Test: 30.70 ± 6.00   Final old_old: 33.44 ± 6.53   Final old_new: 19.52 ± 4.14   Final new_new: 8.64 ± 4.37
Hits@20:
  Final val: 61.06 ± 3.42   Final Test: 43.19 ± 3.48   Final old_old: 46.64 ± 3.84   Final old_new: 29.29 ± 2.44   Final new_new: 12.27 ± 2.59
Hits@30:
  Final val: 67.05 ± 3.08   Final Test: 48.62 ± 2.53   Final old_old: 52.31 ± 2.43   Final old_new: 33.83 ± 3.27   Final new_new: 14.09 ± 2.49
Hits@50:
  Final val: 74.79 ± 1.30   Final Test: 58.17 ± 2.22   Final old_old: 61.93 ± 2.27   Final old_new: 43.22 ± 2.44   Final new_new: 21.36 ± 4.43
AUC:
  Final val: 96.79 ± 0.18   Final Test: 96.08 ± 0.22   Final old_old: 96.76 ± 0.17   Final old_new: 93.44 ± 0.63   Final new_new: 88.57 ± 1.31
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.39 ± 3.20   Final Test: 33.13 ± 2.35   Final old_old: 36.76 ± 2.53   Final old_new: 18.36 ± 1.77   Final new_new: 3.18 ± 2.59
Hits@20:
  Final val: 63.71 ± 1.52   Final Test: 43.71 ± 1.73   Final old_old: 47.65 ± 1.89   Final old_new: 28.01 ± 1.36   Final new_new: 5.45 ± 3.80
Hits@30:
  Final val: 69.88 ± 1.64   Final Test: 51.02 ± 3.32   Final old_old: 55.42 ± 3.64   Final old_new: 33.48 ± 2.56   Final new_new: 7.73 ± 4.13
Hits@50:
  Final val: 76.13 ± 1.31   Final Test: 61.32 ± 1.89   Final old_old: 65.79 ± 2.06   Final old_new: 43.73 ± 2.34   Final new_new: 14.09 ± 3.37
AUC:
  Final val: 97.34 ± 0.14   Final Test: 96.80 ± 0.21   Final old_old: 97.45 ± 0.19   Final old_new: 94.23 ± 0.32   Final new_new: 89.62 ± 0.84
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.58 ± 0.84   Final Test: 33.40 ± 2.55   Final old_old: 36.60 ± 2.91   Final old_new: 20.50 ± 1.35   Final new_new: 5.00 ± 1.90
Hits@20:
  Final val: 62.57 ± 2.27   Final Test: 43.02 ± 3.79   Final old_old: 46.57 ± 3.98   Final old_new: 28.87 ± 3.32   Final new_new: 8.18 ± 1.24
Hits@30:
  Final val: 68.47 ± 1.54   Final Test: 50.09 ± 4.33   Final old_old: 54.07 ± 4.46   Final old_new: 34.26 ± 4.10   Final new_new: 10.45 ± 1.24
Hits@50:
  Final val: 76.19 ± 0.83   Final Test: 60.73 ± 2.98   Final old_old: 64.90 ± 2.85   Final old_new: 44.31 ± 3.76   Final new_new: 16.36 ± 2.49
AUC:
  Final val: 97.33 ± 0.09   Final Test: 96.79 ± 0.18   Final old_old: 97.42 ± 0.16   Final old_new: 94.30 ± 0.32   Final new_new: 90.13 ± 0.77
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.44 ± 4.47   Final Test: 27.93 ± 7.50   Final old_old: 30.35 ± 8.00   Final old_new: 18.19 ± 5.63   Final new_new: 5.91 ± 2.03
Hits@20:
  Final val: 54.26 ± 3.03   Final Test: 39.07 ± 2.90   Final old_old: 42.01 ± 2.80   Final old_new: 27.25 ± 3.66   Final new_new: 11.82 ± 3.37
Hits@30:
  Final val: 60.46 ± 2.16   Final Test: 45.43 ± 3.46   Final old_old: 48.80 ± 3.24   Final old_new: 31.99 ± 4.58   Final new_new: 13.18 ± 4.37
Hits@50:
  Final val: 69.64 ± 2.00   Final Test: 55.25 ± 3.52   Final old_old: 58.90 ± 3.69   Final old_new: 40.76 ± 3.26   Final new_new: 18.64 ± 6.10
AUC:
  Final val: 96.58 ± 0.18   Final Test: 95.96 ± 0.25   Final old_old: 96.60 ± 0.25   Final old_new: 93.36 ± 0.51   Final new_new: 89.95 ± 1.15
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.27 ± 2.27   Final Test: 28.88 ± 3.97   Final old_old: 31.92 ± 4.23   Final old_new: 16.45 ± 2.87   Final new_new: 5.00 ± 2.96
Hits@20:
  Final val: 57.23 ± 2.66   Final Test: 41.72 ± 4.96   Final old_old: 45.48 ± 5.31   Final old_new: 26.62 ± 3.62   Final new_new: 6.82 ± 3.59
Hits@30:
  Final val: 64.22 ± 1.64   Final Test: 47.95 ± 4.57   Final old_old: 51.90 ± 4.80   Final old_new: 32.29 ± 3.81   Final new_new: 8.18 ± 3.80
Hits@50:
  Final val: 72.73 ± 1.63   Final Test: 55.64 ± 3.61   Final old_old: 59.77 ± 3.83   Final old_new: 39.24 ± 2.93   Final new_new: 14.55 ± 4.71
AUC:
  Final val: 97.18 ± 0.13   Final Test: 96.71 ± 0.16   Final old_old: 97.32 ± 0.20   Final old_new: 94.29 ± 0.12   Final new_new: 90.40 ± 0.72
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.60 ± 3.24   Final Test: 36.06 ± 4.34   Final old_old: 39.35 ± 4.39   Final old_new: 22.87 ± 4.27   Final new_new: 4.55 ± 3.94
Hits@20:
  Final val: 65.34 ± 2.57   Final Test: 47.88 ± 4.91   Final old_old: 51.69 ± 5.07   Final old_new: 32.75 ± 4.40   Final new_new: 9.55 ± 2.96
Hits@30:
  Final val: 71.41 ± 2.05   Final Test: 54.48 ± 3.44   Final old_old: 58.59 ± 3.56   Final old_new: 38.31 ± 3.29   Final new_new: 10.91 ± 4.93
Hits@50:
  Final val: 77.86 ± 1.85   Final Test: 62.75 ± 2.67   Final old_old: 67.16 ± 2.67   Final old_new: 45.39 ± 2.90   Final new_new: 15.91 ± 4.25
AUC:
  Final val: 97.55 ± 0.14   Final Test: 97.08 ± 0.15   Final old_old: 97.65 ± 0.16   Final old_new: 94.81 ± 0.12   Final new_new: 91.07 ± 0.46
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.25 ± 3.54   Final Test: 31.85 ± 4.11   Final old_old: 35.13 ± 4.42   Final old_new: 18.51 ± 2.99   Final new_new: 4.55 ± 2.27
Hits@20:
  Final val: 59.97 ± 2.53   Final Test: 38.66 ± 3.11   Final old_old: 42.39 ± 3.36   Final old_new: 23.60 ± 2.25   Final new_new: 5.91 ± 1.24
Hits@30:
  Final val: 67.08 ± 2.39   Final Test: 47.96 ± 3.26   Final old_old: 51.87 ± 3.41   Final old_new: 32.44 ± 2.76   Final new_new: 8.64 ± 3.73
Hits@50:
  Final val: 74.22 ± 1.87   Final Test: 57.57 ± 1.18   Final old_old: 61.74 ± 1.11   Final old_new: 41.16 ± 1.90   Final new_new: 12.73 ± 2.03
AUC:
  Final val: 97.27 ± 0.14   Final Test: 96.66 ± 0.09   Final old_old: 97.29 ± 0.08   Final old_new: 94.26 ± 0.34   Final new_new: 89.27 ± 1.59
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.51 ± 2.72   Final Test: 34.95 ± 3.11   Final old_old: 38.18 ± 3.27   Final old_new: 21.99 ± 2.83   Final new_new: 5.00 ± 2.96
Hits@20:
  Final val: 61.43 ± 2.64   Final Test: 42.83 ± 1.42   Final old_old: 46.62 ± 1.52   Final old_new: 27.78 ± 1.57   Final new_new: 5.45 ± 2.59
Hits@30:
  Final val: 68.36 ± 1.39   Final Test: 50.53 ± 0.69   Final old_old: 54.43 ± 0.88   Final old_new: 35.14 ± 0.62   Final new_new: 9.09 ± 2.27
Hits@50:
  Final val: 75.98 ± 1.36   Final Test: 60.63 ± 1.27   Final old_old: 64.84 ± 1.30   Final old_new: 44.18 ± 1.68   Final new_new: 13.18 ± 3.73
AUC:
  Final val: 97.32 ± 0.10   Final Test: 96.86 ± 0.05   Final old_old: 97.50 ± 0.11   Final old_new: 94.37 ± 0.17   Final new_new: 89.92 ± 1.17
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.49 ± 3.72   Final Test: 28.07 ± 2.86   Final old_old: 30.56 ± 3.04   Final old_new: 17.88 ± 2.57   Final new_new: 7.73 ± 5.23
Hits@20:
  Final val: 53.92 ± 3.45   Final Test: 40.79 ± 3.79   Final old_old: 43.96 ± 3.95   Final old_new: 28.09 ± 3.83   Final new_new: 11.36 ± 7.54
Hits@30:
  Final val: 62.44 ± 2.48   Final Test: 46.69 ± 3.59   Final old_old: 50.14 ± 3.66   Final old_new: 32.97 ± 3.90   Final new_new: 12.73 ± 6.35
Hits@50:
  Final val: 70.94 ± 1.39   Final Test: 56.87 ± 2.92   Final old_old: 60.43 ± 2.93   Final old_new: 42.80 ± 3.51   Final new_new: 20.00 ± 7.08
AUC:
  Final val: 96.65 ± 0.19   Final Test: 96.04 ± 0.27   Final old_old: 96.68 ± 0.27   Final old_new: 93.54 ± 0.38   Final new_new: 89.27 ± 2.06
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.71 ± 4.04   Final Test: 31.94 ± 4.87   Final old_old: 35.17 ± 5.49   Final old_new: 18.87 ± 2.31   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 63.08 ± 2.45   Final Test: 40.72 ± 3.14   Final old_old: 44.41 ± 3.48   Final old_new: 25.97 ± 1.86   Final new_new: 5.91 ± 2.03
Hits@30:
  Final val: 69.41 ± 1.45   Final Test: 49.06 ± 4.70   Final old_old: 53.08 ± 5.27   Final old_new: 33.20 ± 2.40   Final new_new: 7.27 ± 2.96
Hits@50:
  Final val: 76.28 ± 1.38   Final Test: 60.00 ± 3.66   Final old_old: 64.39 ± 3.86   Final old_new: 42.82 ± 2.99   Final new_new: 11.36 ± 4.55
AUC:
  Final val: 97.48 ± 0.07   Final Test: 97.01 ± 0.10   Final old_old: 97.57 ± 0.11   Final old_new: 94.84 ± 0.21   Final new_new: 90.95 ± 0.85
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 40.98 ± 4.22   Final Test: 25.51 ± 8.09   Final old_old: 27.78 ± 8.64   Final old_new: 16.30 ± 5.99   Final new_new: 6.36 ± 4.07
Hits@20:
  Final val: 53.96 ± 2.88   Final Test: 36.96 ± 5.47   Final old_old: 39.97 ± 5.86   Final old_new: 24.86 ± 4.12   Final new_new: 9.55 ± 2.96
Hits@30:
  Final val: 62.23 ± 2.26   Final Test: 44.08 ± 2.31   Final old_old: 47.51 ± 2.25   Final old_new: 30.45 ± 2.76   Final new_new: 10.00 ± 4.71
Hits@50:
  Final val: 72.03 ± 1.98   Final Test: 56.33 ± 2.76   Final old_old: 60.07 ± 2.81   Final old_new: 41.71 ± 2.92   Final new_new: 15.45 ± 5.18
AUC:
  Final val: 96.63 ± 0.23   Final Test: 95.88 ± 0.18   Final old_old: 96.54 ± 0.22   Final old_new: 93.25 ± 0.39   Final new_new: 89.23 ± 2.62
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.17 ± 1.40   Final Test: 30.77 ± 1.36   Final old_old: 33.97 ± 1.46   Final old_new: 17.66 ± 1.12   Final new_new: 5.91 ± 3.45
Hits@20:
  Final val: 59.58 ± 1.62   Final Test: 38.55 ± 2.36   Final old_old: 41.99 ± 2.46   Final old_new: 24.76 ± 2.09   Final new_new: 5.91 ± 2.03
Hits@30:
  Final val: 66.44 ± 1.43   Final Test: 46.34 ± 2.60   Final old_old: 50.26 ± 2.73   Final old_new: 30.73 ± 2.20   Final new_new: 8.18 ± 3.45
Hits@50:
  Final val: 74.34 ± 1.25   Final Test: 57.75 ± 2.86   Final old_old: 62.02 ± 2.83   Final old_new: 40.91 ± 3.41   Final new_new: 13.64 ± 2.27
AUC:
  Final val: 97.27 ± 0.07   Final Test: 96.72 ± 0.08   Final old_old: 97.33 ± 0.07   Final old_new: 94.29 ± 0.22   Final new_new: 90.50 ± 0.58
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.61 ± 2.95   Final Test: 31.65 ± 6.18   Final old_old: 34.61 ± 6.71   Final old_new: 19.70 ± 4.25   Final new_new: 5.45 ± 1.24
Hits@20:
  Final val: 62.45 ± 1.75   Final Test: 41.22 ± 4.64   Final old_old: 44.93 ± 4.92   Final old_new: 26.40 ± 3.65   Final new_new: 6.36 ± 1.90
Hits@30:
  Final val: 68.54 ± 1.55   Final Test: 48.77 ± 3.23   Final old_old: 52.83 ± 3.45   Final old_new: 32.57 ± 2.53   Final new_new: 9.09 ± 3.59
Hits@50:
  Final val: 75.96 ± 1.17   Final Test: 59.85 ± 0.78   Final old_old: 64.15 ± 0.99   Final old_new: 42.80 ± 0.36   Final new_new: 15.91 ± 2.27
AUC:
  Final val: 97.33 ± 0.10   Final Test: 96.74 ± 0.13   Final old_old: 97.38 ± 0.13   Final old_new: 94.19 ± 0.22   Final new_new: 90.44 ± 0.72
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.91 ± 4.54   Final Test: 32.13 ± 2.69   Final old_old: 35.34 ± 2.77   Final old_new: 19.19 ± 2.52   Final new_new: 3.18 ± 2.59
Hits@20:
  Final val: 59.51 ± 4.48   Final Test: 41.21 ± 5.57   Final old_old: 44.81 ± 5.94   Final old_new: 26.83 ± 4.55   Final new_new: 6.82 ± 2.27
Hits@30:
  Final val: 66.57 ± 3.71   Final Test: 51.39 ± 3.24   Final old_old: 55.51 ± 3.26   Final old_new: 34.94 ± 3.59   Final new_new: 11.82 ± 5.66
Hits@50:
  Final val: 74.58 ± 2.54   Final Test: 60.11 ± 2.03   Final old_old: 64.26 ± 2.13   Final old_new: 43.88 ± 2.07   Final new_new: 14.09 ± 6.31
AUC:
  Final val: 97.30 ± 0.12   Final Test: 96.76 ± 0.10   Final old_old: 97.36 ± 0.10   Final old_new: 94.32 ± 0.22   Final new_new: 91.01 ± 1.03
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.84 ± 3.22   Final Test: 33.07 ± 6.73   Final old_old: 35.86 ± 7.13   Final old_new: 21.84 ± 5.45   Final new_new: 8.18 ± 5.23
Hits@20:
  Final val: 58.13 ± 2.17   Final Test: 39.18 ± 3.52   Final old_old: 42.33 ± 3.67   Final old_new: 26.62 ± 3.50   Final new_new: 9.09 ± 4.25
Hits@30:
  Final val: 65.05 ± 1.84   Final Test: 46.56 ± 4.87   Final old_old: 49.77 ± 5.12   Final old_new: 33.90 ± 4.30   Final new_new: 12.73 ± 5.70
Hits@50:
  Final val: 73.25 ± 1.29   Final Test: 55.36 ± 2.43   Final old_old: 59.15 ± 2.69   Final old_new: 40.28 ± 2.45   Final new_new: 18.18 ± 5.79
AUC:
  Final val: 96.69 ± 0.20   Final Test: 96.11 ± 0.18   Final old_old: 96.78 ± 0.19   Final old_new: 93.46 ± 0.35   Final new_new: 88.94 ± 2.10
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.01 ± 2.85   Final Test: 28.24 ± 6.18   Final old_old: 30.79 ± 6.63   Final old_new: 17.88 ± 4.67   Final new_new: 6.82 ± 3.21
Hits@20:
  Final val: 54.87 ± 2.49   Final Test: 36.67 ± 3.58   Final old_old: 39.68 ± 3.65   Final old_new: 24.46 ± 3.68   Final new_new: 11.36 ± 3.59
Hits@30:
  Final val: 62.32 ± 2.51   Final Test: 46.15 ± 2.41   Final old_old: 49.63 ± 2.46   Final old_new: 32.14 ± 2.66   Final new_new: 15.00 ± 6.14
Hits@50:
  Final val: 70.84 ± 1.91   Final Test: 54.99 ± 4.39   Final old_old: 58.68 ± 4.78   Final old_new: 40.33 ± 3.45   Final new_new: 19.09 ± 6.93
AUC:
  Final val: 96.55 ± 0.21   Final Test: 95.95 ± 0.28   Final old_old: 96.65 ± 0.28   Final old_new: 93.13 ± 0.47   Final new_new: 89.42 ± 1.85
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.90 ± 4.37   Final Test: 30.93 ± 3.61   Final old_old: 33.97 ± 3.94   Final old_new: 18.59 ± 2.99   Final new_new: 5.00 ± 1.90
Hits@20:
  Final val: 58.26 ± 3.06   Final Test: 40.70 ± 2.66   Final old_old: 44.27 ± 2.93   Final old_new: 26.40 ± 2.43   Final new_new: 7.27 ± 2.96
Hits@30:
  Final val: 64.76 ± 3.18   Final Test: 48.77 ± 4.20   Final old_old: 52.69 ± 4.35   Final old_new: 33.25 ± 3.92   Final new_new: 8.64 ± 3.73
Hits@50:
  Final val: 73.81 ± 1.96   Final Test: 57.44 ± 3.64   Final old_old: 61.73 ± 3.70   Final old_new: 40.53 ± 3.92   Final new_new: 12.27 ± 2.59
AUC:
  Final val: 97.18 ± 0.13   Final Test: 96.64 ± 0.23   Final old_old: 97.25 ± 0.23   Final old_new: 94.23 ± 0.34   Final new_new: 90.62 ± 1.52
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.35 ± 1.54   Final Test: 31.86 ± 3.92   Final old_old: 35.02 ± 4.10   Final old_new: 19.07 ± 3.27   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 59.20 ± 2.19   Final Test: 41.67 ± 3.58   Final old_old: 45.26 ± 3.80   Final old_new: 27.33 ± 2.85   Final new_new: 7.73 ± 3.80
Hits@30:
  Final val: 65.71 ± 2.64   Final Test: 48.14 ± 2.49   Final old_old: 52.07 ± 2.79   Final old_new: 32.57 ± 1.56   Final new_new: 7.73 ± 3.45
Hits@50:
  Final val: 73.89 ± 1.01   Final Test: 57.75 ± 1.20   Final old_old: 61.95 ± 1.07   Final old_new: 41.13 ± 2.19   Final new_new: 14.55 ± 3.45
AUC:
  Final val: 97.23 ± 0.06   Final Test: 96.68 ± 0.08   Final old_old: 97.26 ± 0.08   Final old_new: 94.39 ± 0.24   Final new_new: 90.62 ± 0.63
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.99 ± 1.27   Final Test: 32.61 ± 5.30   Final old_old: 35.90 ± 5.72   Final old_new: 19.29 ± 3.61   Final new_new: 4.09 ± 3.37
Hits@20:
  Final val: 64.73 ± 1.81   Final Test: 44.85 ± 6.33   Final old_old: 48.79 ± 6.72   Final old_new: 29.02 ± 4.96   Final new_new: 8.64 ± 4.07
Hits@30:
  Final val: 70.87 ± 1.30   Final Test: 53.19 ± 2.57   Final old_old: 57.53 ± 2.79   Final old_new: 35.92 ± 2.04   Final new_new: 10.45 ± 5.23
Hits@50:
  Final val: 77.57 ± 1.25   Final Test: 62.71 ± 3.90   Final old_old: 66.98 ± 4.04   Final old_new: 45.97 ± 3.87   Final new_new: 16.36 ± 1.02
AUC:
  Final val: 97.54 ± 0.09   Final Test: 97.07 ± 0.13   Final old_old: 97.64 ± 0.13   Final old_new: 94.85 ± 0.29   Final new_new: 91.17 ± 0.79
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.41 ± 3.26   Final Test: 33.67 ± 3.12   Final old_old: 36.94 ± 3.43   Final old_new: 20.48 ± 1.90   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 62.19 ± 1.29   Final Test: 44.66 ± 2.57   Final old_old: 48.51 ± 2.64   Final old_new: 29.32 ± 2.50   Final new_new: 7.27 ± 3.73
Hits@30:
  Final val: 69.15 ± 1.29   Final Test: 51.01 ± 3.42   Final old_old: 55.12 ± 3.63   Final old_new: 34.71 ± 3.03   Final new_new: 9.55 ± 2.49
Hits@50:
  Final val: 76.77 ± 1.15   Final Test: 61.84 ± 1.03   Final old_old: 66.25 ± 0.90   Final old_new: 44.61 ± 2.21   Final new_new: 12.73 ± 3.05
AUC:
  Final val: 97.35 ± 0.08   Final Test: 96.81 ± 0.05   Final old_old: 97.44 ± 0.07   Final old_new: 94.30 ± 0.11   Final new_new: 90.03 ± 0.85
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 40.17 ± 1.82   Final Test: 28.38 ± 6.24   Final old_old: 30.93 ± 6.77   Final old_new: 18.11 ± 4.52   Final new_new: 5.45 ± 3.45
Hits@20:
  Final val: 53.60 ± 2.25   Final Test: 38.56 ± 4.17   Final old_old: 41.65 ± 4.54   Final old_new: 26.07 ± 2.79   Final new_new: 10.91 ± 5.88
Hits@30:
  Final val: 61.61 ± 1.30   Final Test: 43.47 ± 4.53   Final old_old: 46.75 ± 4.81   Final old_new: 30.45 ± 3.73   Final new_new: 10.45 ± 5.93
Hits@50:
  Final val: 71.53 ± 1.69   Final Test: 58.45 ± 3.28   Final old_old: 62.22 ± 3.48   Final old_new: 43.63 ± 2.92   Final new_new: 18.18 ± 6.22
AUC:
  Final val: 96.66 ± 0.21   Final Test: 95.92 ± 0.32   Final old_old: 96.56 ± 0.28   Final old_new: 93.41 ± 0.53   Final new_new: 88.93 ± 1.81
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.44 ± 3.58   Final Test: 24.18 ± 4.98   Final old_old: 26.45 ± 5.11   Final old_new: 14.99 ± 4.61   Final new_new: 4.55 ± 2.78
Hits@20:
  Final val: 53.67 ± 4.04   Final Test: 33.70 ± 2.98   Final old_old: 36.54 ± 3.06   Final old_new: 22.37 ± 3.07   Final new_new: 6.36 ± 3.73
Hits@30:
  Final val: 61.73 ± 3.06   Final Test: 43.97 ± 2.97   Final old_old: 47.36 ± 2.98   Final old_new: 30.48 ± 3.25   Final new_new: 10.45 ± 3.45
Hits@50:
  Final val: 70.91 ± 1.50   Final Test: 55.53 ± 1.75   Final old_old: 59.25 ± 1.88   Final old_new: 41.13 ± 1.45   Final new_new: 11.82 ± 3.37
AUC:
  Final val: 96.63 ± 0.16   Final Test: 95.81 ± 0.25   Final old_old: 96.52 ± 0.19   Final old_new: 93.06 ± 0.62   Final new_new: 87.53 ± 4.01
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.68 ± 3.21   Final Test: 34.06 ± 1.80   Final old_old: 37.23 ± 1.88   Final old_new: 21.36 ± 2.02   Final new_new: 4.55 ± 2.27
Hits@20:
  Final val: 61.20 ± 1.75   Final Test: 41.23 ± 2.99   Final old_old: 44.66 ± 3.37   Final old_new: 27.66 ± 1.88   Final new_new: 5.91 ± 3.80
Hits@30:
  Final val: 67.47 ± 1.63   Final Test: 50.16 ± 2.84   Final old_old: 54.21 ± 2.87   Final old_new: 34.11 ± 3.07   Final new_new: 9.55 ± 5.18
Hits@50:
  Final val: 75.34 ± 1.08   Final Test: 58.74 ± 2.45   Final old_old: 62.91 ± 2.72   Final old_new: 42.49 ± 1.47   Final new_new: 10.91 ± 8.10
AUC:
  Final val: 97.41 ± 0.09   Final Test: 96.92 ± 0.09   Final old_old: 97.46 ± 0.10   Final old_new: 94.80 ± 0.25   Final new_new: 91.33 ± 0.91
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.95 ± 2.08   Final Test: 32.65 ± 2.48   Final old_old: 35.86 ± 2.72   Final old_new: 19.67 ± 1.59   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 58.37 ± 3.42   Final Test: 38.37 ± 3.68   Final old_old: 41.89 ± 3.99   Final old_new: 24.23 ± 2.59   Final new_new: 5.45 ± 2.03
Hits@30:
  Final val: 65.68 ± 2.13   Final Test: 47.05 ± 2.43   Final old_old: 50.90 ± 2.45   Final old_new: 31.76 ± 2.55   Final new_new: 8.18 ± 3.45
Hits@50:
  Final val: 73.33 ± 2.02   Final Test: 56.36 ± 4.18   Final old_old: 60.60 ± 4.25   Final old_new: 39.60 ± 4.06   Final new_new: 13.18 ± 3.73
AUC:
  Final val: 97.17 ± 0.13   Final Test: 96.66 ± 0.10   Final old_old: 97.25 ± 0.11   Final old_new: 94.34 ± 0.18   Final new_new: 90.57 ± 0.59
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.61 ± 1.86   Final Test: 31.84 ± 1.96   Final old_old: 35.03 ± 2.11   Final old_new: 18.92 ± 1.68   Final new_new: 4.55 ± 2.27
Hits@20:
  Final val: 62.91 ± 2.17   Final Test: 41.83 ± 1.25   Final old_old: 45.63 ± 1.35   Final old_new: 26.60 ± 1.27   Final new_new: 6.36 ± 1.02
Hits@30:
  Final val: 69.98 ± 1.24   Final Test: 50.69 ± 1.09   Final old_old: 54.85 ± 1.07   Final old_new: 34.26 ± 1.38   Final new_new: 7.73 ± 2.59
Hits@50:
  Final val: 76.69 ± 1.37   Final Test: 63.04 ± 0.84   Final old_old: 67.43 ± 0.95   Final old_new: 45.89 ± 0.57   Final new_new: 14.09 ± 2.49
AUC:
  Final val: 97.37 ± 0.15   Final Test: 96.83 ± 0.13   Final old_old: 97.46 ± 0.17   Final old_new: 94.40 ± 0.08   Final new_new: 90.10 ± 1.29
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.37 ± 7.37   Final Test: 34.76 ± 6.80   Final old_old: 38.01 ± 7.27   Final old_new: 21.61 ± 4.98   Final new_new: 7.27 ± 3.73
Hits@20:
  Final val: 61.66 ± 4.31   Final Test: 42.28 ± 8.30   Final old_old: 46.03 ± 8.86   Final old_new: 27.13 ± 6.17   Final new_new: 9.55 ± 2.96
Hits@30:
  Final val: 68.87 ± 3.12   Final Test: 49.34 ± 6.77   Final old_old: 53.43 ± 7.22   Final old_new: 32.95 ± 5.29   Final new_new: 11.36 ± 4.25
Hits@50:
  Final val: 75.85 ± 2.55   Final Test: 58.75 ± 3.89   Final old_old: 63.01 ± 3.93   Final old_new: 41.91 ± 4.01   Final new_new: 15.00 ± 3.45
AUC:
  Final val: 97.32 ± 0.18   Final Test: 96.79 ± 0.21   Final old_old: 97.37 ± 0.24   Final old_new: 94.49 ± 0.30   Final new_new: 91.08 ± 1.28
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.79 ± 6.58   Final Test: 24.16 ± 5.92   Final old_old: 26.36 ± 6.44   Final old_new: 15.16 ± 4.15   Final new_new: 7.27 ± 2.96
Hits@20:
  Final val: 55.03 ± 7.21   Final Test: 37.45 ± 7.64   Final old_old: 40.26 ± 8.07   Final old_new: 26.07 ± 6.02   Final new_new: 13.18 ± 3.37
Hits@30:
  Final val: 63.24 ± 5.92   Final Test: 47.35 ± 6.05   Final old_old: 50.77 ± 6.27   Final old_new: 33.68 ± 5.32   Final new_new: 14.55 ± 4.71
Hits@50:
  Final val: 72.09 ± 4.24   Final Test: 56.24 ± 5.92   Final old_old: 60.00 ± 6.28   Final old_new: 41.21 ± 4.61   Final new_new: 20.45 ± 2.78
AUC:
  Final val: 96.70 ± 0.21   Final Test: 96.15 ± 0.29   Final old_old: 96.78 ± 0.30   Final old_new: 93.69 ± 0.36   Final new_new: 89.28 ± 1.91
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.31 ± 4.49   Final Test: 25.15 ± 6.50   Final old_old: 27.47 ± 7.06   Final old_new: 15.72 ± 4.37   Final new_new: 6.36 ± 4.93
Hits@20:
  Final val: 56.08 ± 2.08   Final Test: 37.08 ± 2.54   Final old_old: 40.23 ± 2.78   Final old_new: 24.48 ± 1.68   Final new_new: 7.27 ± 3.37
Hits@30:
  Final val: 62.83 ± 1.03   Final Test: 43.20 ± 2.63   Final old_old: 46.62 ± 2.63   Final old_new: 29.47 ± 2.79   Final new_new: 10.91 ± 5.43
Hits@50:
  Final val: 71.48 ± 0.87   Final Test: 54.81 ± 3.68   Final old_old: 58.41 ± 3.72   Final old_new: 40.65 ± 4.00   Final new_new: 16.36 ± 7.43
AUC:
  Final val: 96.58 ± 0.19   Final Test: 95.92 ± 0.36   Final old_old: 96.61 ± 0.31   Final old_new: 93.19 ± 0.64   Final new_new: 89.34 ± 2.01
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.71 ± 4.61   Final Test: 31.90 ± 4.46   Final old_old: 35.03 ± 4.76   Final old_new: 19.19 ± 3.38   Final new_new: 5.00 ± 1.90
Hits@20:
  Final val: 58.73 ± 3.50   Final Test: 38.54 ± 5.56   Final old_old: 41.78 ± 5.95   Final old_new: 25.59 ± 4.55   Final new_new: 7.27 ± 2.96
Hits@30:
  Final val: 66.24 ± 1.61   Final Test: 45.93 ± 3.86   Final old_old: 49.79 ± 4.26   Final old_new: 30.55 ± 3.00   Final new_new: 7.73 ± 1.24
Hits@50:
  Final val: 73.46 ± 1.90   Final Test: 57.54 ± 1.88   Final old_old: 61.54 ± 2.12   Final old_new: 41.74 ± 1.32   Final new_new: 15.91 ± 4.55
AUC:
  Final val: 97.21 ± 0.22   Final Test: 96.65 ± 0.21   Final old_old: 97.24 ± 0.23   Final old_new: 94.38 ± 0.33   Final new_new: 89.64 ± 1.09
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.60 ± 3.86   Final Test: 30.09 ± 3.31   Final old_old: 33.15 ± 3.48   Final old_new: 17.66 ± 3.31   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 61.38 ± 3.01   Final Test: 39.76 ± 2.77   Final old_old: 43.35 ± 2.71   Final old_new: 25.42 ± 3.33   Final new_new: 5.91 ± 2.03
Hits@30:
  Final val: 67.47 ± 1.78   Final Test: 47.12 ± 4.00   Final old_old: 51.00 ± 4.21   Final old_new: 31.66 ± 3.32   Final new_new: 9.55 ± 2.49
Hits@50:
  Final val: 75.19 ± 1.15   Final Test: 58.18 ± 4.75   Final old_old: 62.40 ± 4.84   Final old_new: 41.51 ± 4.72   Final new_new: 14.09 ± 4.07
AUC:
  Final val: 97.24 ± 0.11   Final Test: 96.76 ± 0.16   Final old_old: 97.38 ± 0.14   Final old_new: 94.32 ± 0.30   Final new_new: 90.20 ± 1.08
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.66 ± 2.98   Final Test: 30.47 ± 3.83   Final old_old: 33.58 ± 4.10   Final old_new: 17.88 ± 3.03   Final new_new: 3.64 ± 2.03
Hits@20:
  Final val: 65.79 ± 2.37   Final Test: 43.84 ± 3.32   Final old_old: 47.59 ± 3.58   Final old_new: 28.94 ± 2.51   Final new_new: 5.91 ± 2.03
Hits@30:
  Final val: 72.20 ± 1.08   Final Test: 52.57 ± 1.76   Final old_old: 56.54 ± 2.19   Final old_new: 37.03 ± 0.96   Final new_new: 8.18 ± 2.59
Hits@50:
  Final val: 79.04 ± 1.26   Final Test: 63.67 ± 2.66   Final old_old: 67.66 ± 2.89   Final old_new: 48.34 ± 2.73   Final new_new: 14.09 ± 5.66
AUC:
  Final val: 97.59 ± 0.13   Final Test: 97.04 ± 0.10   Final old_old: 97.58 ± 0.12   Final old_new: 94.92 ± 0.09   Final new_new: 91.16 ± 0.91
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.80 ± 4.87   Final Test: 30.71 ± 3.82   Final old_old: 33.76 ± 4.21   Final old_new: 18.39 ± 2.57   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 62.20 ± 2.92   Final Test: 39.64 ± 3.77   Final old_old: 43.03 ± 4.09   Final old_new: 26.15 ± 2.69   Final new_new: 6.82 ± 4.82
Hits@30:
  Final val: 69.19 ± 2.13   Final Test: 50.22 ± 2.69   Final old_old: 54.26 ± 2.94   Final old_new: 34.13 ± 2.72   Final new_new: 10.91 ± 2.96
Hits@50:
  Final val: 76.50 ± 2.08   Final Test: 59.27 ± 2.43   Final old_old: 63.44 ± 2.70   Final old_new: 42.85 ± 1.99   Final new_new: 15.45 ± 5.18
AUC:
  Final val: 97.37 ± 0.15   Final Test: 96.80 ± 0.10   Final old_old: 97.42 ± 0.14   Final old_new: 94.34 ± 0.20   Final new_new: 90.01 ± 1.07
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.38 ± 4.21   Final Test: 30.12 ± 7.50   Final old_old: 32.70 ± 8.02   Final old_new: 19.85 ± 5.72   Final new_new: 5.00 ± 2.96
Hits@20:
  Final val: 53.54 ± 4.72   Final Test: 37.42 ± 7.81   Final old_old: 40.35 ± 8.22   Final old_new: 25.84 ± 6.42   Final new_new: 7.73 ± 1.24
Hits@30:
  Final val: 60.88 ± 2.69   Final Test: 46.45 ± 4.31   Final old_old: 49.97 ± 4.42   Final old_new: 32.49 ± 4.07   Final new_new: 11.36 ± 3.21
Hits@50:
  Final val: 70.46 ± 1.17   Final Test: 55.90 ± 4.73   Final old_old: 59.72 ± 4.87   Final old_new: 40.81 ± 4.57   Final new_new: 16.36 ± 4.37
AUC:
  Final val: 96.58 ± 0.29   Final Test: 95.98 ± 0.25   Final old_old: 96.63 ± 0.30   Final old_new: 93.43 ± 0.14   Final new_new: 89.30 ± 1.71
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.23 ± 5.28   Final Test: 26.43 ± 3.65   Final old_old: 28.63 ± 3.75   Final old_new: 17.73 ± 3.59   Final new_new: 4.09 ± 3.73
Hits@20:
  Final val: 57.85 ± 5.71   Final Test: 38.08 ± 5.43   Final old_old: 41.10 ± 5.84   Final old_new: 26.07 ± 4.05   Final new_new: 8.18 ± 5.23
Hits@30:
  Final val: 64.00 ± 5.20   Final Test: 46.90 ± 4.72   Final old_old: 50.33 ± 4.88   Final old_new: 33.30 ± 4.33   Final new_new: 11.82 ± 2.96
Hits@50:
  Final val: 72.68 ± 3.00   Final Test: 55.78 ± 3.04   Final old_old: 59.54 ± 3.21   Final old_new: 40.93 ± 2.66   Final new_new: 17.27 ± 5.23
AUC:
  Final val: 96.61 ± 0.25   Final Test: 96.01 ± 0.31   Final old_old: 96.64 ± 0.30   Final old_new: 93.59 ± 0.53   Final new_new: 88.83 ± 3.42
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.24 ± 3.04   Final Test: 33.03 ± 2.99   Final old_old: 36.24 ± 3.18   Final old_new: 20.08 ± 2.31   Final new_new: 4.09 ± 2.96
Hits@20:
  Final val: 63.08 ± 1.13   Final Test: 42.31 ± 5.23   Final old_old: 45.87 ± 5.30   Final old_new: 28.21 ± 5.17   Final new_new: 6.36 ± 3.37
Hits@30:
  Final val: 69.00 ± 1.54   Final Test: 47.87 ± 4.32   Final old_old: 51.88 ± 4.16   Final old_new: 31.91 ± 5.24   Final new_new: 8.18 ± 4.43
Hits@50:
  Final val: 76.90 ± 1.05   Final Test: 59.75 ± 2.53   Final old_old: 63.97 ± 2.60   Final old_new: 43.17 ± 2.78   Final new_new: 14.55 ± 3.05
AUC:
  Final val: 97.49 ± 0.09   Final Test: 96.99 ± 0.08   Final old_old: 97.51 ± 0.10   Final old_new: 94.97 ± 0.15   Final new_new: 91.16 ± 1.11
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.71 ± 2.67   Final Test: 33.95 ± 3.11   Final old_old: 37.32 ± 3.19   Final old_new: 20.38 ± 3.01   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 58.94 ± 2.79   Final Test: 40.25 ± 5.70   Final old_old: 43.92 ± 6.12   Final old_new: 25.57 ± 4.19   Final new_new: 5.00 ± 2.49
Hits@30:
  Final val: 66.08 ± 2.22   Final Test: 46.26 ± 4.06   Final old_old: 50.24 ± 4.22   Final old_new: 30.43 ± 3.66   Final new_new: 6.82 ± 3.21
Hits@50:
  Final val: 74.09 ± 0.96   Final Test: 57.88 ± 1.98   Final old_old: 62.10 ± 1.96   Final old_new: 41.34 ± 2.38   Final new_new: 11.82 ± 4.07
AUC:
  Final val: 97.24 ± 0.11   Final Test: 96.73 ± 0.10   Final old_old: 97.32 ± 0.10   Final old_new: 94.39 ± 0.40   Final new_new: 90.67 ± 1.21
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.93 ± 2.44   Final Test: 31.13 ± 5.13   Final old_old: 34.17 ± 5.56   Final old_new: 18.79 ± 3.50   Final new_new: 5.00 ± 2.96
Hits@20:
  Final val: 61.30 ± 0.53   Final Test: 40.41 ± 3.13   Final old_old: 43.93 ± 3.37   Final old_new: 26.37 ± 2.61   Final new_new: 6.36 ± 1.02
Hits@30:
  Final val: 67.75 ± 1.14   Final Test: 49.71 ± 1.48   Final old_old: 53.69 ± 1.73   Final old_new: 33.93 ± 1.40   Final new_new: 9.55 ± 2.96
Hits@50:
  Final val: 75.50 ± 0.87   Final Test: 59.73 ± 2.40   Final old_old: 63.94 ± 2.40   Final old_new: 43.22 ± 2.97   Final new_new: 14.09 ± 3.73
AUC:
  Final val: 97.30 ± 0.09   Final Test: 96.78 ± 0.13   Final old_old: 97.41 ± 0.14   Final old_new: 94.30 ± 0.35   Final new_new: 90.54 ± 0.58
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.001, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.61 ± 3.68   Final Test: 32.27 ± 6.34   Final old_old: 35.49 ± 6.73   Final old_new: 19.14 ± 4.96   Final new_new: 6.36 ± 4.07
Hits@20:
  Final val: 63.19 ± 1.72   Final Test: 43.07 ± 2.52   Final old_old: 46.91 ± 2.69   Final old_new: 27.58 ± 2.32   Final new_new: 9.09 ± 2.27
Hits@30:
  Final val: 69.58 ± 2.38   Final Test: 50.60 ± 2.42   Final old_old: 54.80 ± 2.51   Final old_new: 33.78 ± 2.46   Final new_new: 11.36 ± 2.78
Hits@50:
  Final val: 76.58 ± 1.59   Final Test: 60.56 ± 2.82   Final old_old: 64.83 ± 3.06   Final old_new: 43.68 ± 1.97   Final new_new: 16.36 ± 4.93
AUC:
  Final val: 97.38 ± 0.09   Final Test: 96.75 ± 0.10   Final old_old: 97.41 ± 0.13   Final old_new: 94.17 ± 0.30   Final new_new: 89.86 ± 0.79
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.48 ± 5.12   Final Test: 27.87 ± 6.10   Final old_old: 30.40 ± 6.37   Final old_new: 17.48 ± 5.33   Final new_new: 8.18 ± 3.80
Hits@20:
  Final val: 54.82 ± 3.63   Final Test: 37.31 ± 4.38   Final old_old: 40.23 ± 4.81   Final old_new: 25.54 ± 3.49   Final new_new: 10.91 ± 5.66
Hits@30:
  Final val: 62.25 ± 2.53   Final Test: 45.30 ± 6.02   Final old_old: 48.63 ± 6.30   Final old_new: 32.02 ± 5.16   Final new_new: 12.73 ± 3.45
Hits@50:
  Final val: 70.99 ± 1.81   Final Test: 55.06 ± 4.21   Final old_old: 58.68 ± 4.54   Final old_new: 40.86 ± 3.11   Final new_new: 16.36 ± 1.90
AUC:
  Final val: 96.59 ± 0.19   Final Test: 95.92 ± 0.20   Final old_old: 96.61 ± 0.21   Final old_new: 93.21 ± 0.56   Final new_new: 88.94 ± 0.95
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.15 ± 3.31   Final Test: 26.33 ± 4.46   Final old_old: 28.58 ± 4.76   Final old_new: 17.23 ± 3.38   Final new_new: 6.82 ± 2.27
Hits@20:
  Final val: 60.52 ± 2.05   Final Test: 40.73 ± 3.19   Final old_old: 44.04 ± 3.28   Final old_new: 27.46 ± 3.09   Final new_new: 10.00 ± 2.59
Hits@30:
  Final val: 67.32 ± 0.62   Final Test: 48.54 ± 5.41   Final old_old: 52.15 ± 5.74   Final old_new: 34.03 ± 4.35   Final new_new: 15.91 ± 3.59
Hits@50:
  Final val: 75.32 ± 1.33   Final Test: 58.11 ± 2.69   Final old_old: 61.92 ± 2.75   Final old_new: 43.00 ± 2.95   Final new_new: 20.00 ± 7.08
AUC:
  Final val: 96.88 ± 0.14   Final Test: 96.25 ± 0.26   Final old_old: 96.88 ± 0.23   Final old_new: 93.74 ± 0.42   Final new_new: 90.12 ± 0.91
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.35 ± 2.78   Final Test: 31.42 ± 3.98   Final old_old: 34.46 ± 4.41   Final old_new: 19.17 ± 2.46   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 59.81 ± 3.19   Final Test: 39.91 ± 4.16   Final old_old: 43.42 ± 4.40   Final old_new: 25.84 ± 3.42   Final new_new: 6.82 ± 1.61
Hits@30:
  Final val: 66.01 ± 2.20   Final Test: 48.04 ± 4.46   Final old_old: 52.12 ± 4.85   Final old_new: 31.69 ± 2.99   Final new_new: 10.00 ± 3.80
Hits@50:
  Final val: 73.47 ± 1.42   Final Test: 58.57 ± 2.44   Final old_old: 62.76 ± 2.59   Final old_new: 41.99 ± 2.16   Final new_new: 15.91 ± 4.55
AUC:
  Final val: 97.20 ± 0.09   Final Test: 96.73 ± 0.21   Final old_old: 97.33 ± 0.23   Final old_new: 94.34 ± 0.25   Final new_new: 91.24 ± 0.71
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.21 ± 2.50   Final Test: 30.21 ± 6.17   Final old_old: 33.12 ± 6.65   Final old_new: 18.44 ± 4.47   Final new_new: 5.45 ± 2.59
Hits@20:
  Final val: 60.25 ± 2.68   Final Test: 36.60 ± 2.12   Final old_old: 39.82 ± 2.36   Final old_new: 23.70 ± 1.39   Final new_new: 6.36 ± 2.49
Hits@30:
  Final val: 66.30 ± 2.21   Final Test: 45.39 ± 3.70   Final old_old: 49.07 ± 3.72   Final old_new: 30.81 ± 4.07   Final new_new: 8.18 ± 4.13
Hits@50:
  Final val: 73.49 ± 1.80   Final Test: 57.74 ± 1.68   Final old_old: 61.93 ± 1.84   Final old_new: 41.13 ± 1.58   Final new_new: 15.45 ± 2.96
AUC:
  Final val: 97.20 ± 0.09   Final Test: 96.61 ± 0.16   Final old_old: 97.24 ± 0.16   Final old_new: 94.13 ± 0.28   Final new_new: 90.24 ± 0.93
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.13 ± 1.72   Final Test: 31.83 ± 2.27   Final old_old: 35.01 ± 2.39   Final old_new: 18.97 ± 1.94   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 62.78 ± 1.20   Final Test: 42.63 ± 2.47   Final old_old: 46.36 ± 2.56   Final old_new: 27.61 ± 2.31   Final new_new: 9.09 ± 2.78
Hits@30:
  Final val: 68.66 ± 1.12   Final Test: 50.81 ± 4.05   Final old_old: 54.90 ± 4.10   Final old_new: 34.58 ± 3.95   Final new_new: 10.00 ± 3.45
Hits@50:
  Final val: 75.41 ± 1.63   Final Test: 59.06 ± 3.50   Final old_old: 63.19 ± 3.57   Final old_new: 42.77 ± 3.48   Final new_new: 15.45 ± 4.93
AUC:
  Final val: 97.35 ± 0.06   Final Test: 96.76 ± 0.06   Final old_old: 97.40 ± 0.08   Final old_new: 94.27 ± 0.15   Final new_new: 89.89 ± 0.94
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.99 ± 3.80   Final Test: 37.41 ± 5.05   Final old_old: 40.85 ± 5.35   Final old_new: 23.73 ± 4.00   Final new_new: 3.18 ± 1.24
Hits@20:
  Final val: 64.22 ± 4.89   Final Test: 46.05 ± 2.94   Final old_old: 49.87 ± 3.13   Final old_new: 31.08 ± 2.46   Final new_new: 4.55 ± 1.61
Hits@30:
  Final val: 70.58 ± 3.04   Final Test: 53.03 ± 4.72   Final old_old: 56.98 ± 4.98   Final old_new: 37.61 ± 3.69   Final new_new: 8.18 ± 5.70
Hits@50:
  Final val: 77.72 ± 2.33   Final Test: 65.10 ± 3.15   Final old_old: 69.05 ± 3.08   Final old_new: 49.67 ± 3.24   Final new_new: 20.45 ± 8.65
AUC:
  Final val: 97.56 ± 0.11   Final Test: 97.11 ± 0.12   Final old_old: 97.66 ± 0.13   Final old_new: 94.94 ± 0.21   Final new_new: 91.21 ± 0.81
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.82 ± 2.49   Final Test: 32.50 ± 6.42   Final old_old: 35.23 ± 6.90   Final old_new: 21.59 ± 4.41   Final new_new: 6.36 ± 4.37
Hits@20:
  Final val: 55.40 ± 1.72   Final Test: 41.01 ± 0.83   Final old_old: 44.24 ± 1.00   Final old_new: 28.21 ± 1.40   Final new_new: 8.64 ± 3.37
Hits@30:
  Final val: 63.44 ± 1.84   Final Test: 48.55 ± 4.10   Final old_old: 52.07 ± 4.38   Final old_new: 34.61 ± 3.24   Final new_new: 12.73 ± 2.03
Hits@50:
  Final val: 71.76 ± 1.46   Final Test: 57.40 ± 3.01   Final old_old: 61.26 ± 3.00   Final old_new: 42.24 ± 3.91   Final new_new: 15.91 ± 6.63
AUC:
  Final val: 96.52 ± 0.14   Final Test: 95.98 ± 0.25   Final old_old: 96.61 ± 0.23   Final old_new: 93.47 ± 0.52   Final new_new: 89.58 ± 1.84
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.96 ± 3.77   Final Test: 28.35 ± 4.03   Final old_old: 30.90 ± 4.28   Final old_new: 17.98 ± 3.46   Final new_new: 6.36 ± 3.73
Hits@20:
  Final val: 56.59 ± 3.10   Final Test: 38.85 ± 3.44   Final old_old: 41.91 ± 3.51   Final old_new: 26.60 ± 3.73   Final new_new: 10.00 ± 6.93
Hits@30:
  Final val: 63.63 ± 3.25   Final Test: 46.14 ± 3.23   Final old_old: 49.61 ± 3.10   Final old_new: 32.42 ± 4.64   Final new_new: 10.45 ± 5.93
Hits@50:
  Final val: 71.89 ± 1.91   Final Test: 55.46 ± 3.02   Final old_old: 59.13 ± 3.26   Final old_new: 41.13 ± 2.29   Final new_new: 15.00 ± 4.71
AUC:
  Final val: 96.68 ± 0.23   Final Test: 96.02 ± 0.30   Final old_old: 96.70 ± 0.32   Final old_new: 93.35 ± 0.32   Final new_new: 88.84 ± 2.04
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.96 ± 3.69   Final Test: 32.53 ± 2.62   Final old_old: 35.74 ± 2.75   Final old_new: 19.52 ± 2.07   Final new_new: 4.55 ± 3.21
Hits@20:
  Final val: 59.46 ± 1.89   Final Test: 39.43 ± 5.20   Final old_old: 43.02 ± 5.45   Final old_new: 24.94 ± 4.35   Final new_new: 7.73 ± 5.47
Hits@30:
  Final val: 66.55 ± 1.29   Final Test: 47.65 ± 2.65   Final old_old: 51.49 ± 2.73   Final old_new: 32.39 ± 2.34   Final new_new: 9.55 ± 3.37
Hits@50:
  Final val: 74.59 ± 1.21   Final Test: 57.40 ± 1.44   Final old_old: 61.70 ± 1.57   Final old_new: 40.38 ± 0.83   Final new_new: 13.64 ± 2.78
AUC:
  Final val: 97.22 ± 0.12   Final Test: 96.68 ± 0.08   Final old_old: 97.29 ± 0.09   Final old_new: 94.26 ± 0.28   Final new_new: 90.58 ± 0.45
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.71 ± 2.45   Final Test: 33.59 ± 2.63   Final old_old: 36.82 ± 2.97   Final old_new: 20.58 ± 1.37   Final new_new: 5.00 ± 1.90
Hits@20:
  Final val: 60.22 ± 2.15   Final Test: 40.89 ± 2.20   Final old_old: 44.36 ± 2.21   Final old_new: 26.93 ± 2.52   Final new_new: 9.55 ± 2.49
Hits@30:
  Final val: 67.11 ± 1.78   Final Test: 49.07 ± 3.97   Final old_old: 53.08 ± 4.06   Final old_new: 33.12 ± 3.85   Final new_new: 10.00 ± 2.03
Hits@50:
  Final val: 75.75 ± 1.27   Final Test: 59.68 ± 2.27   Final old_old: 63.81 ± 2.22   Final old_new: 43.32 ± 2.84   Final new_new: 17.73 ± 3.73
AUC:
  Final val: 97.41 ± 0.09   Final Test: 96.95 ± 0.11   Final old_old: 97.49 ± 0.12   Final old_new: 94.84 ± 0.20   Final new_new: 91.16 ± 0.53
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.26 ± 3.04   Final Test: 30.20 ± 6.54   Final old_old: 33.33 ± 6.93   Final old_new: 17.46 ± 5.08   Final new_new: 4.55 ± 3.21
Hits@20:
  Final val: 61.66 ± 1.70   Final Test: 37.62 ± 5.96   Final old_old: 40.98 ± 6.29   Final old_new: 24.18 ± 4.79   Final new_new: 5.91 ± 2.03
Hits@30:
  Final val: 69.49 ± 1.65   Final Test: 50.42 ± 2.83   Final old_old: 54.33 ± 2.94   Final old_new: 35.01 ± 2.69   Final new_new: 9.09 ± 1.61
Hits@50:
  Final val: 76.35 ± 1.14   Final Test: 62.97 ± 1.71   Final old_old: 67.12 ± 1.90   Final old_new: 46.57 ± 1.56   Final new_new: 19.55 ± 5.47
AUC:
  Final val: 97.34 ± 0.10   Final Test: 96.81 ± 0.15   Final old_old: 97.43 ± 0.14   Final old_new: 94.39 ± 0.42   Final new_new: 90.18 ± 1.14
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.63 ± 2.29   Final Test: 31.29 ± 3.21   Final old_old: 34.47 ± 3.46   Final old_new: 18.41 ± 2.59   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 60.86 ± 2.60   Final Test: 42.58 ± 3.28   Final old_old: 46.29 ± 3.50   Final old_new: 27.76 ± 2.84   Final new_new: 6.82 ± 2.78
Hits@30:
  Final val: 68.64 ± 2.14   Final Test: 49.60 ± 3.77   Final old_old: 53.58 ± 3.95   Final old_new: 33.90 ± 3.56   Final new_new: 7.73 ± 2.03
Hits@50:
  Final val: 76.06 ± 1.23   Final Test: 60.26 ± 2.11   Final old_old: 64.49 ± 2.13   Final old_new: 43.68 ± 2.39   Final new_new: 14.09 ± 3.37
AUC:
  Final val: 97.33 ± 0.07   Final Test: 96.75 ± 0.07   Final old_old: 97.40 ± 0.08   Final old_new: 94.22 ± 0.17   Final new_new: 89.63 ± 1.17
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.92 ± 3.61   Final Test: 31.66 ± 3.69   Final old_old: 34.41 ± 3.86   Final old_new: 20.48 ± 3.48   Final new_new: 9.09 ± 3.59
Hits@20:
  Final val: 58.13 ± 3.87   Final Test: 38.57 ± 3.55   Final old_old: 41.75 ± 3.82   Final old_new: 25.74 ± 2.61   Final new_new: 10.00 ± 2.59
Hits@30:
  Final val: 64.17 ± 1.98   Final Test: 45.98 ± 2.27   Final old_old: 49.51 ± 2.45   Final old_new: 31.86 ± 1.83   Final new_new: 11.82 ± 5.43
Hits@50:
  Final val: 71.88 ± 0.87   Final Test: 56.71 ± 3.73   Final old_old: 60.35 ± 3.91   Final old_new: 42.32 ± 3.53   Final new_new: 19.09 ± 4.71
AUC:
  Final val: 96.59 ± 0.09   Final Test: 95.97 ± 0.34   Final old_old: 96.60 ± 0.31   Final old_new: 93.42 ± 0.64   Final new_new: 90.18 ± 1.18
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.06 ± 4.90   Final Test: 28.77 ± 3.46   Final old_old: 31.25 ± 3.71   Final old_new: 18.92 ± 2.83   Final new_new: 4.55 ± 4.25
Hits@20:
  Final val: 59.28 ± 3.43   Final Test: 40.90 ± 3.06   Final old_old: 43.97 ± 3.36   Final old_new: 28.74 ± 2.41   Final new_new: 9.55 ± 6.31
Hits@30:
  Final val: 66.95 ± 1.93   Final Test: 46.95 ± 3.69   Final old_old: 50.26 ± 4.04   Final old_new: 33.98 ± 2.65   Final new_new: 10.45 ± 4.43
Hits@50:
  Final val: 74.08 ± 0.97   Final Test: 57.32 ± 2.92   Final old_old: 61.12 ± 2.78   Final old_new: 42.39 ± 4.17   Final new_new: 15.91 ± 4.25
AUC:
  Final val: 96.76 ± 0.14   Final Test: 96.18 ± 0.22   Final old_old: 96.79 ± 0.21   Final old_new: 93.70 ± 0.55   Final new_new: 90.98 ± 1.26
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.36 ± 3.00   Final Test: 33.04 ± 3.91   Final old_old: 36.21 ± 3.91   Final old_new: 20.15 ± 3.93   Final new_new: 6.82 ± 4.82
Hits@20:
  Final val: 59.71 ± 1.94   Final Test: 40.04 ± 3.66   Final old_old: 43.50 ± 3.92   Final old_new: 26.15 ± 2.79   Final new_new: 7.73 ± 4.43
Hits@30:
  Final val: 67.07 ± 1.86   Final Test: 48.22 ± 2.65   Final old_old: 52.14 ± 2.67   Final old_new: 32.57 ± 2.68   Final new_new: 10.45 ± 6.14
Hits@50:
  Final val: 73.60 ± 0.87   Final Test: 55.90 ± 4.26   Final old_old: 59.98 ± 4.44   Final old_new: 39.85 ± 3.59   Final new_new: 12.73 ± 3.45
AUC:
  Final val: 97.22 ± 0.10   Final Test: 96.77 ± 0.10   Final old_old: 97.37 ± 0.09   Final old_new: 94.41 ± 0.28   Final new_new: 90.20 ± 0.29
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.49 ± 3.91   Final Test: 31.10 ± 8.53   Final old_old: 34.21 ± 9.15   Final old_new: 18.44 ± 6.19   Final new_new: 5.45 ± 2.03
Hits@20:
  Final val: 59.21 ± 3.51   Final Test: 41.16 ± 4.86   Final old_old: 44.81 ± 5.30   Final old_new: 26.60 ± 3.43   Final new_new: 6.36 ± 2.96
Hits@30:
  Final val: 66.20 ± 1.88   Final Test: 47.48 ± 2.32   Final old_old: 51.34 ± 2.59   Final old_new: 32.14 ± 1.83   Final new_new: 9.09 ± 5.57
Hits@50:
  Final val: 73.67 ± 1.43   Final Test: 58.43 ± 2.20   Final old_old: 62.72 ± 2.37   Final old_new: 41.59 ± 2.13   Final new_new: 11.82 ± 6.89
AUC:
  Final val: 97.21 ± 0.14   Final Test: 96.73 ± 0.13   Final old_old: 97.37 ± 0.13   Final old_new: 94.20 ± 0.29   Final new_new: 89.56 ± 1.12
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.19 ± 2.41   Final Test: 30.44 ± 1.47   Final old_old: 33.48 ± 1.48   Final old_new: 18.06 ± 1.70   Final new_new: 5.91 ± 2.59
Hits@20:
  Final val: 62.67 ± 2.21   Final Test: 43.90 ± 0.80   Final old_old: 47.89 ± 0.79   Final old_new: 27.83 ± 1.98   Final new_new: 7.73 ± 4.13
Hits@30:
  Final val: 68.63 ± 2.04   Final Test: 48.17 ± 2.12   Final old_old: 52.06 ± 2.19   Final old_new: 32.67 ± 2.42   Final new_new: 10.45 ± 3.80
Hits@50:
  Final val: 76.23 ± 0.90   Final Test: 59.06 ± 2.98   Final old_old: 63.32 ± 3.22   Final old_new: 42.19 ± 2.45   Final new_new: 15.91 ± 5.33
AUC:
  Final val: 97.36 ± 0.06   Final Test: 96.80 ± 0.13   Final old_old: 97.42 ± 0.16   Final old_new: 94.33 ± 0.12   Final new_new: 90.28 ± 0.98
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 40.15 ± 5.61   Final Test: 27.85 ± 2.95   Final old_old: 30.21 ± 3.28   Final old_new: 18.39 ± 1.95   Final new_new: 5.45 ± 1.24
Hits@20:
  Final val: 51.20 ± 6.57   Final Test: 34.93 ± 6.07   Final old_old: 37.65 ± 6.44   Final old_new: 24.06 ± 4.96   Final new_new: 9.09 ± 4.25
Hits@30:
  Final val: 60.10 ± 5.47   Final Test: 43.37 ± 1.41   Final old_old: 46.51 ± 1.54   Final old_new: 31.08 ± 1.77   Final new_new: 8.64 ± 2.96
Hits@50:
  Final val: 69.93 ± 3.88   Final Test: 55.59 ± 3.07   Final old_old: 59.54 ± 3.66   Final old_new: 39.97 ± 1.69   Final new_new: 15.45 ± 5.88
AUC:
  Final val: 96.49 ± 0.30   Final Test: 95.94 ± 0.29   Final old_old: 96.56 ± 0.22   Final old_new: 93.45 ± 0.71   Final new_new: 89.71 ± 0.84
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 53.57 ± 1.99   Final Test: 32.16 ± 5.27   Final old_old: 35.42 ± 5.61   Final old_new: 18.94 ± 4.02   Final new_new: 4.09 ± 2.96
Hits@20:
  Final val: 65.57 ± 1.37   Final Test: 46.55 ± 5.57   Final old_old: 50.32 ± 5.93   Final old_new: 31.61 ± 4.38   Final new_new: 8.64 ± 2.49
Hits@30:
  Final val: 71.96 ± 1.29   Final Test: 53.19 ± 2.06   Final old_old: 57.22 ± 2.27   Final old_new: 37.20 ± 1.71   Final new_new: 13.18 ± 3.73
Hits@50:
  Final val: 79.60 ± 0.88   Final Test: 62.45 ± 3.11   Final old_old: 66.66 ± 3.38   Final old_new: 45.84 ± 2.46   Final new_new: 18.18 ± 5.57
AUC:
  Final val: 97.58 ± 0.13   Final Test: 97.04 ± 0.15   Final old_old: 97.60 ± 0.14   Final old_new: 94.84 ± 0.26   Final new_new: 91.07 ± 0.94
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.47 ± 2.76   Final Test: 31.58 ± 3.85   Final old_old: 34.39 ± 3.86   Final old_new: 20.20 ± 3.89   Final new_new: 7.27 ± 4.37
Hits@20:
  Final val: 55.32 ± 2.25   Final Test: 40.97 ± 4.92   Final old_old: 44.29 ± 5.15   Final old_new: 27.81 ± 4.31   Final new_new: 7.73 ± 3.45
Hits@30:
  Final val: 63.40 ± 1.85   Final Test: 47.24 ± 1.30   Final old_old: 50.64 ± 1.17   Final old_new: 33.90 ± 2.16   Final new_new: 10.45 ± 4.13
Hits@50:
  Final val: 71.99 ± 1.08   Final Test: 56.82 ± 1.44   Final old_old: 60.62 ± 1.44   Final old_new: 41.91 ± 2.57   Final new_new: 15.45 ± 7.94
AUC:
  Final val: 96.64 ± 0.14   Final Test: 95.92 ± 0.29   Final old_old: 96.60 ± 0.28   Final old_new: 93.20 ± 0.69   Final new_new: 88.97 ± 1.71
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.15 ± 3.43   Final Test: 30.72 ± 5.69   Final old_old: 33.79 ± 6.30   Final old_new: 18.34 ± 3.47   Final new_new: 3.64 ± 2.03
Hits@20:
  Final val: 59.49 ± 2.91   Final Test: 40.83 ± 4.79   Final old_old: 44.54 ± 5.41   Final old_new: 25.94 ± 2.38   Final new_new: 6.36 ± 1.90
Hits@30:
  Final val: 66.50 ± 2.46   Final Test: 47.71 ± 4.77   Final old_old: 51.78 ± 5.32   Final old_new: 31.49 ± 2.67   Final new_new: 8.18 ± 3.45
Hits@50:
  Final val: 74.29 ± 1.37   Final Test: 59.42 ± 0.95   Final old_old: 63.68 ± 1.26   Final old_new: 42.59 ± 0.50   Final new_new: 15.00 ± 3.80
AUC:
  Final val: 97.28 ± 0.10   Final Test: 96.73 ± 0.17   Final old_old: 97.33 ± 0.18   Final old_new: 94.36 ± 0.27   Final new_new: 90.36 ± 0.96
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.94 ± 3.23   Final Test: 33.18 ± 3.60   Final old_old: 36.35 ± 3.71   Final old_new: 20.40 ± 3.29   Final new_new: 5.00 ± 1.90
Hits@20:
  Final val: 62.19 ± 2.31   Final Test: 41.25 ± 2.33   Final old_old: 45.00 ± 2.78   Final old_new: 26.20 ± 1.42   Final new_new: 7.27 ± 2.49
Hits@30:
  Final val: 68.98 ± 1.74   Final Test: 51.78 ± 2.40   Final old_old: 55.84 ± 2.48   Final old_new: 35.77 ± 2.90   Final new_new: 9.09 ± 1.61
Hits@50:
  Final val: 75.90 ± 1.19   Final Test: 60.93 ± 2.57   Final old_old: 65.12 ± 2.58   Final old_new: 44.56 ± 2.68   Final new_new: 13.64 ± 3.59
AUC:
  Final val: 97.33 ± 0.07   Final Test: 96.75 ± 0.09   Final old_old: 97.38 ± 0.11   Final old_new: 94.28 ± 0.29   Final new_new: 89.98 ± 0.96
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.21 ± 2.89   Final Test: 31.57 ± 2.48   Final old_old: 34.61 ± 2.60   Final old_new: 19.40 ± 2.33   Final new_new: 3.18 ± 1.24
Hits@20:
  Final val: 60.60 ± 1.07   Final Test: 42.22 ± 4.00   Final old_old: 45.62 ± 4.19   Final old_new: 28.79 ± 3.28   Final new_new: 7.27 ± 2.96
Hits@30:
  Final val: 67.64 ± 0.97   Final Test: 49.84 ± 2.57   Final old_old: 53.68 ± 2.76   Final old_new: 34.84 ± 1.97   Final new_new: 7.27 ± 4.37
Hits@50:
  Final val: 76.02 ± 1.12   Final Test: 61.35 ± 2.71   Final old_old: 65.44 ± 3.11   Final old_new: 45.39 ± 1.20   Final new_new: 15.45 ± 2.49
AUC:
  Final val: 97.42 ± 0.09   Final Test: 97.04 ± 0.08   Final old_old: 97.57 ± 0.11   Final old_new: 94.95 ± 0.14   Final new_new: 91.01 ± 0.61
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.27 ± 3.24   Final Test: 32.64 ± 5.76   Final old_old: 35.90 ± 6.25   Final old_new: 19.42 ± 3.81   Final new_new: 5.00 ± 1.90
Hits@20:
  Final val: 63.67 ± 3.11   Final Test: 45.24 ± 4.62   Final old_old: 49.02 ± 5.03   Final old_new: 30.23 ± 3.13   Final new_new: 7.27 ± 1.90
Hits@30:
  Final val: 69.40 ± 2.56   Final Test: 52.24 ± 6.06   Final old_old: 56.38 ± 6.48   Final old_new: 35.87 ± 4.41   Final new_new: 9.55 ± 2.96
Hits@50:
  Final val: 76.33 ± 1.70   Final Test: 62.20 ± 2.62   Final old_old: 66.56 ± 2.82   Final old_new: 45.11 ± 1.99   Final new_new: 14.09 ± 1.90
AUC:
  Final val: 97.33 ± 0.09   Final Test: 96.83 ± 0.17   Final old_old: 97.47 ± 0.18   Final old_new: 94.30 ± 0.23   Final new_new: 90.32 ± 0.54
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 40.59 ± 3.75   Final Test: 23.34 ± 5.98   Final old_old: 25.59 ± 6.51   Final old_new: 14.13 ± 3.90   Final new_new: 5.91 ± 3.45
Hits@20:
  Final val: 54.84 ± 3.00   Final Test: 36.88 ± 2.99   Final old_old: 39.96 ± 3.05   Final old_new: 24.38 ± 2.99   Final new_new: 10.91 ± 7.08
Hits@30:
  Final val: 61.03 ± 1.61   Final Test: 42.63 ± 1.30   Final old_old: 45.96 ± 1.06   Final old_new: 29.32 ± 2.46   Final new_new: 10.45 ± 6.14
Hits@50:
  Final val: 69.81 ± 1.95   Final Test: 54.66 ± 2.14   Final old_old: 58.26 ± 2.19   Final old_new: 40.55 ± 2.04   Final new_new: 14.55 ± 6.35
AUC:
  Final val: 96.54 ± 0.14   Final Test: 95.96 ± 0.22   Final old_old: 96.58 ± 0.17   Final old_new: 93.49 ± 0.48   Final new_new: 89.29 ± 1.60
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.19 ± 2.96   Final Test: 31.23 ± 3.51   Final old_old: 34.34 ± 3.69   Final old_new: 18.64 ± 2.88   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 60.54 ± 2.76   Final Test: 40.45 ± 2.78   Final old_old: 43.94 ± 2.81   Final old_new: 26.45 ± 2.81   Final new_new: 7.73 ± 3.80
Hits@30:
  Final val: 66.37 ± 1.45   Final Test: 49.95 ± 2.43   Final old_old: 53.81 ± 2.49   Final old_new: 34.61 ± 2.34   Final new_new: 11.36 ± 3.21
Hits@50:
  Final val: 73.24 ± 1.36   Final Test: 57.17 ± 2.62   Final old_old: 61.29 ± 2.75   Final old_new: 40.88 ± 2.22   Final new_new: 14.55 ± 3.45
AUC:
  Final val: 97.20 ± 0.10   Final Test: 96.68 ± 0.10   Final old_old: 97.29 ± 0.09   Final old_new: 94.29 ± 0.19   Final new_new: 90.23 ± 1.83
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.59 ± 5.45   Final Test: 27.66 ± 7.23   Final old_old: 30.06 ± 7.85   Final old_new: 17.93 ± 4.95   Final new_new: 7.27 ± 1.90
Hits@20:
  Final val: 57.26 ± 3.93   Final Test: 37.11 ± 3.83   Final old_old: 40.31 ± 4.23   Final old_new: 24.16 ± 2.99   Final new_new: 10.00 ± 2.03
Hits@30:
  Final val: 64.83 ± 3.34   Final Test: 50.91 ± 2.55   Final old_old: 54.77 ± 2.86   Final old_new: 35.49 ± 2.02   Final new_new: 14.09 ± 3.37
Hits@50:
  Final val: 74.38 ± 1.71   Final Test: 58.94 ± 3.44   Final old_old: 62.97 ± 3.62   Final old_new: 42.97 ± 3.31   Final new_new: 18.18 ± 3.94
AUC:
  Final val: 96.84 ± 0.08   Final Test: 96.27 ± 0.18   Final old_old: 96.92 ± 0.21   Final old_new: 93.69 ± 0.27   Final new_new: 89.92 ± 1.85
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.26 ± 2.21   Final Test: 29.54 ± 3.53   Final old_old: 32.62 ± 3.83   Final old_new: 17.03 ± 2.38   Final new_new: 3.64 ± 2.03
Hits@20:
  Final val: 57.86 ± 3.04   Final Test: 38.78 ± 4.54   Final old_old: 42.24 ± 4.85   Final old_new: 24.86 ± 3.36   Final new_new: 7.27 ± 2.49
Hits@30:
  Final val: 65.98 ± 1.48   Final Test: 46.17 ± 2.89   Final old_old: 49.92 ± 3.18   Final old_new: 31.13 ± 2.26   Final new_new: 10.45 ± 1.24
Hits@50:
  Final val: 73.40 ± 1.07   Final Test: 57.26 ± 2.43   Final old_old: 61.33 ± 2.36   Final old_new: 41.03 ± 2.82   Final new_new: 17.27 ± 4.13
AUC:
  Final val: 97.21 ± 0.07   Final Test: 96.70 ± 0.14   Final old_old: 97.29 ± 0.13   Final old_new: 94.37 ± 0.25   Final new_new: 89.82 ± 0.70
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.78 ± 4.54   Final Test: 32.74 ± 7.82   Final old_old: 35.92 ± 8.32   Final old_new: 19.97 ± 6.03   Final new_new: 3.18 ± 2.59
Hits@20:
  Final val: 62.41 ± 3.13   Final Test: 42.20 ± 3.54   Final old_old: 45.93 ± 3.75   Final old_new: 27.38 ± 2.73   Final new_new: 5.45 ± 2.59
Hits@30:
  Final val: 68.91 ± 2.05   Final Test: 51.45 ± 3.26   Final old_old: 55.41 ± 3.30   Final old_new: 35.97 ± 3.42   Final new_new: 7.27 ± 1.90
Hits@50:
  Final val: 76.32 ± 1.70   Final Test: 61.44 ± 3.17   Final old_old: 65.50 ± 2.99   Final old_new: 45.67 ± 4.32   Final new_new: 14.55 ± 3.80
AUC:
  Final val: 97.35 ± 0.12   Final Test: 96.79 ± 0.14   Final old_old: 97.41 ± 0.10   Final old_new: 94.29 ± 0.30   Final new_new: 90.99 ± 1.12
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.02 ± 0.99   Final Test: 27.88 ± 4.91   Final old_old: 30.30 ± 5.09   Final old_new: 18.21 ± 4.56   Final new_new: 5.00 ± 2.96
Hits@20:
  Final val: 55.60 ± 1.75   Final Test: 38.58 ± 5.03   Final old_old: 41.69 ± 5.15   Final old_new: 26.27 ± 4.82   Final new_new: 6.82 ± 5.08
Hits@30:
  Final val: 63.76 ± 1.91   Final Test: 45.05 ± 6.73   Final old_old: 48.39 ± 7.25   Final old_new: 31.91 ± 5.01   Final new_new: 9.55 ± 6.31
Hits@50:
  Final val: 71.64 ± 0.99   Final Test: 54.83 ± 2.98   Final old_old: 58.64 ± 3.22   Final old_new: 39.90 ± 2.50   Final new_new: 13.64 ± 4.25
AUC:
  Final val: 96.55 ± 0.21   Final Test: 96.03 ± 0.39   Final old_old: 96.60 ± 0.37   Final old_new: 93.83 ± 0.52   Final new_new: 89.87 ± 1.80
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.36 ± 5.37   Final Test: 31.09 ± 8.54   Final old_old: 33.80 ± 9.16   Final old_new: 20.05 ± 6.35   Final new_new: 8.64 ± 1.90
Hits@20:
  Final val: 56.88 ± 3.36   Final Test: 38.23 ± 3.94   Final old_old: 41.14 ± 4.35   Final old_new: 26.52 ± 2.83   Final new_new: 11.82 ± 2.96
Hits@30:
  Final val: 64.55 ± 2.90   Final Test: 47.78 ± 3.78   Final old_old: 51.14 ± 4.11   Final old_new: 34.41 ± 3.07   Final new_new: 14.55 ± 4.71
Hits@50:
  Final val: 72.21 ± 2.13   Final Test: 55.16 ± 2.93   Final old_old: 58.75 ± 3.19   Final old_new: 40.98 ± 2.33   Final new_new: 18.18 ± 4.55
AUC:
  Final val: 96.61 ± 0.19   Final Test: 95.98 ± 0.21   Final old_old: 96.64 ± 0.19   Final old_new: 93.32 ± 0.63   Final new_new: 89.64 ± 1.28
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.49 ± 2.51   Final Test: 30.93 ± 3.73   Final old_old: 34.21 ± 4.02   Final old_new: 17.61 ± 2.59   Final new_new: 4.09 ± 2.49
Hits@20:
  Final val: 60.99 ± 2.36   Final Test: 41.09 ± 3.86   Final old_old: 44.57 ± 4.19   Final old_new: 27.13 ± 2.88   Final new_new: 8.64 ± 2.96
Hits@30:
  Final val: 66.46 ± 1.73   Final Test: 49.11 ± 1.87   Final old_old: 52.91 ± 2.06   Final old_new: 34.03 ± 1.50   Final new_new: 10.91 ± 4.07
Hits@50:
  Final val: 73.79 ± 0.54   Final Test: 57.14 ± 2.12   Final old_old: 61.50 ± 2.09   Final old_new: 39.82 ± 2.52   Final new_new: 13.18 ± 3.37
AUC:
  Final val: 97.23 ± 0.11   Final Test: 96.65 ± 0.04   Final old_old: 97.29 ± 0.05   Final old_new: 94.15 ± 0.36   Final new_new: 89.74 ± 0.47
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.49 ± 4.31   Final Test: 38.03 ± 5.00   Final old_old: 41.68 ± 5.17   Final old_new: 23.35 ± 4.61   Final new_new: 5.45 ± 2.03
Hits@20:
  Final val: 65.65 ± 0.93   Final Test: 48.27 ± 3.60   Final old_old: 52.22 ± 3.53   Final old_new: 32.64 ± 4.25   Final new_new: 7.73 ± 2.03
Hits@30:
  Final val: 70.93 ± 0.56   Final Test: 54.46 ± 1.07   Final old_old: 58.59 ± 1.22   Final old_new: 38.29 ± 1.01   Final new_new: 9.09 ± 1.61
Hits@50:
  Final val: 78.64 ± 0.78   Final Test: 64.95 ± 1.87   Final old_old: 69.21 ± 1.76   Final old_new: 48.31 ± 2.58   Final new_new: 16.82 ± 2.59
AUC:
  Final val: 97.54 ± 0.11   Final Test: 97.09 ± 0.07   Final old_old: 97.65 ± 0.09   Final old_new: 94.90 ± 0.13   Final new_new: 91.56 ± 0.91
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.42 ± 3.54   Final Test: 29.19 ± 6.89   Final old_old: 32.37 ± 7.45   Final old_new: 16.20 ± 4.64   Final new_new: 4.09 ± 2.96
Hits@20:
  Final val: 62.83 ± 1.85   Final Test: 42.77 ± 6.14   Final old_old: 46.49 ± 6.42   Final old_new: 27.86 ± 5.26   Final new_new: 8.18 ± 3.80
Hits@30:
  Final val: 69.21 ± 0.57   Final Test: 51.08 ± 5.67   Final old_old: 55.23 ± 5.60   Final old_new: 34.51 ± 6.27   Final new_new: 11.36 ± 3.94
Hits@50:
  Final val: 76.04 ± 0.71   Final Test: 60.97 ± 1.76   Final old_old: 65.19 ± 1.78   Final old_new: 44.33 ± 2.20   Final new_new: 16.36 ± 4.07
AUC:
  Final val: 97.38 ± 0.12   Final Test: 96.84 ± 0.10   Final old_old: 97.46 ± 0.12   Final old_new: 94.39 ± 0.26   Final new_new: 90.56 ± 1.15
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=0.1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.03 ± 8.33   Final Test: 27.22 ± 4.35   Final old_old: 29.63 ± 4.71   Final old_new: 17.51 ± 3.25   Final new_new: 5.91 ± 2.59
Hits@20:
  Final val: 54.35 ± 6.02   Final Test: 36.65 ± 4.18   Final old_old: 39.60 ± 4.61   Final old_new: 24.74 ± 2.51   Final new_new: 10.91 ± 5.88
Hits@30:
  Final val: 62.51 ± 4.23   Final Test: 44.36 ± 2.45   Final old_old: 47.60 ± 2.57   Final old_new: 31.39 ± 2.38   Final new_new: 14.09 ± 7.08
Hits@50:
  Final val: 70.82 ± 3.15   Final Test: 54.35 ± 2.75   Final old_old: 57.91 ± 3.03   Final old_new: 40.23 ± 1.92   Final new_new: 18.64 ± 6.31
AUC:
  Final val: 96.58 ± 0.23   Final Test: 95.90 ± 0.38   Final old_old: 96.58 ± 0.33   Final old_new: 93.20 ± 0.75   Final new_new: 89.24 ± 0.90
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.46 ± 3.59   Final Test: 34.48 ± 5.31   Final old_old: 37.95 ± 5.83   Final old_new: 20.45 ± 3.31   Final new_new: 4.09 ± 1.02
Hits@20:
  Final val: 62.22 ± 1.56   Final Test: 42.15 ± 3.31   Final old_old: 45.81 ± 3.68   Final old_new: 27.61 ± 1.95   Final new_new: 5.91 ± 2.03
Hits@30:
  Final val: 68.23 ± 1.27   Final Test: 51.54 ± 3.59   Final old_old: 55.50 ± 3.61   Final old_new: 35.89 ± 3.85   Final new_new: 10.45 ± 3.80
Hits@50:
  Final val: 75.89 ± 0.37   Final Test: 61.29 ± 2.42   Final old_old: 65.53 ± 2.55   Final old_new: 44.74 ± 2.83   Final new_new: 13.64 ± 4.82
AUC:
  Final val: 97.32 ± 0.12   Final Test: 96.81 ± 0.11   Final old_old: 97.42 ± 0.17   Final old_new: 94.40 ± 0.16   Final new_new: 90.29 ± 0.89
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.64 ± 3.05   Final Test: 32.90 ± 8.52   Final old_old: 36.18 ± 9.17   Final old_new: 19.67 ± 6.08   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 62.62 ± 0.68   Final Test: 42.08 ± 3.87   Final old_old: 45.77 ± 4.05   Final old_new: 27.41 ± 3.42   Final new_new: 5.00 ± 2.49
Hits@30:
  Final val: 68.52 ± 1.01   Final Test: 48.08 ± 0.74   Final old_old: 51.83 ± 0.65   Final old_new: 33.22 ± 1.72   Final new_new: 10.00 ± 4.13
Hits@50:
  Final val: 75.75 ± 0.97   Final Test: 60.20 ± 2.58   Final old_old: 64.14 ± 2.67   Final old_new: 44.96 ± 2.56   Final new_new: 14.09 ± 1.90
AUC:
  Final val: 97.50 ± 0.09   Final Test: 97.01 ± 0.09   Final old_old: 97.54 ± 0.12   Final old_new: 94.94 ± 0.22   Final new_new: 90.97 ± 0.72
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.07 ± 3.77   Final Test: 34.09 ± 2.62   Final old_old: 37.47 ± 2.86   Final old_new: 20.45 ± 2.06   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 60.30 ± 3.31   Final Test: 42.60 ± 1.87   Final old_old: 46.43 ± 2.31   Final old_new: 27.30 ± 2.13   Final new_new: 5.45 ± 2.59
Hits@30:
  Final val: 66.83 ± 1.60   Final Test: 48.04 ± 2.76   Final old_old: 51.91 ± 3.03   Final old_new: 32.77 ± 1.86   Final new_new: 8.18 ± 3.45
Hits@50:
  Final val: 74.25 ± 1.40   Final Test: 57.06 ± 2.54   Final old_old: 61.54 ± 2.64   Final old_new: 39.32 ± 2.43   Final new_new: 11.36 ± 4.25
AUC:
  Final val: 97.29 ± 0.07   Final Test: 96.71 ± 0.09   Final old_old: 97.35 ± 0.16   Final old_new: 94.21 ± 0.28   Final new_new: 90.00 ± 0.79
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.33 ± 3.40   Final Test: 31.47 ± 2.01   Final old_old: 34.46 ± 2.19   Final old_new: 19.40 ± 1.46   Final new_new: 5.91 ± 3.45
Hits@20:
  Final val: 58.18 ± 2.75   Final Test: 39.86 ± 4.23   Final old_old: 43.37 ± 4.54   Final old_new: 25.79 ± 3.40   Final new_new: 7.27 ± 2.96
Hits@30:
  Final val: 65.23 ± 2.67   Final Test: 46.27 ± 3.99   Final old_old: 49.89 ± 4.15   Final old_new: 31.89 ± 3.66   Final new_new: 10.00 ± 3.45
Hits@50:
  Final val: 73.17 ± 1.11   Final Test: 56.40 ± 4.26   Final old_old: 60.52 ± 4.35   Final old_new: 40.03 ± 4.06   Final new_new: 15.00 ± 4.43
AUC:
  Final val: 97.19 ± 0.10   Final Test: 96.67 ± 0.11   Final old_old: 97.23 ± 0.10   Final old_new: 94.44 ± 0.35   Final new_new: 90.64 ± 0.61
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.42 ± 1.97   Final Test: 29.50 ± 2.87   Final old_old: 32.44 ± 2.90   Final old_new: 17.71 ± 2.96   Final new_new: 2.73 ± 1.02
Hits@20:
  Final val: 63.36 ± 2.11   Final Test: 43.51 ± 3.63   Final old_old: 47.40 ± 3.86   Final old_new: 27.96 ± 3.00   Final new_new: 6.36 ± 1.90
Hits@30:
  Final val: 69.38 ± 2.34   Final Test: 49.14 ± 4.44   Final old_old: 53.15 ± 4.69   Final old_new: 33.35 ± 3.57   Final new_new: 6.82 ± 3.59
Hits@50:
  Final val: 76.07 ± 1.49   Final Test: 60.50 ± 2.14   Final old_old: 64.76 ± 2.24   Final old_new: 43.93 ± 1.88   Final new_new: 11.82 ± 3.73
AUC:
  Final val: 97.35 ± 0.12   Final Test: 96.81 ± 0.16   Final old_old: 97.42 ± 0.16   Final old_new: 94.36 ± 0.31   Final new_new: 90.53 ± 0.73
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 40.22 ± 4.87   Final Test: 27.33 ± 6.64   Final old_old: 29.64 ± 7.21   Final old_new: 18.01 ± 4.59   Final new_new: 6.82 ± 2.78
Hits@20:
  Final val: 54.52 ± 6.11   Final Test: 35.86 ± 6.39   Final old_old: 38.68 ± 6.95   Final old_new: 24.66 ± 4.21   Final new_new: 7.73 ± 5.23
Hits@30:
  Final val: 63.95 ± 4.20   Final Test: 44.49 ± 2.52   Final old_old: 47.72 ± 2.79   Final old_new: 31.71 ± 1.41   Final new_new: 11.82 ± 4.07
Hits@50:
  Final val: 72.92 ± 2.61   Final Test: 55.79 ± 1.65   Final old_old: 59.35 ± 1.66   Final old_new: 41.66 ± 1.87   Final new_new: 20.00 ± 3.37
AUC:
  Final val: 96.78 ± 0.18   Final Test: 96.20 ± 0.23   Final old_old: 96.87 ± 0.21   Final old_new: 93.54 ± 0.27   Final new_new: 89.75 ± 1.77
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.71 ± 3.06   Final Test: 28.06 ± 2.49   Final old_old: 30.66 ± 2.60   Final old_new: 17.56 ± 2.39   Final new_new: 5.45 ± 2.59
Hits@20:
  Final val: 55.63 ± 3.92   Final Test: 40.25 ± 3.19   Final old_old: 43.28 ± 3.43   Final old_new: 28.34 ± 2.62   Final new_new: 7.73 ± 2.59
Hits@30:
  Final val: 62.76 ± 3.59   Final Test: 45.83 ± 1.88   Final old_old: 49.21 ± 2.09   Final old_new: 32.57 ± 1.55   Final new_new: 9.55 ± 4.07
Hits@50:
  Final val: 71.68 ± 1.85   Final Test: 54.73 ± 2.38   Final old_old: 58.47 ± 2.62   Final old_new: 40.15 ± 2.13   Final new_new: 12.27 ± 4.98
AUC:
  Final val: 96.53 ± 0.28   Final Test: 96.00 ± 0.30   Final old_old: 96.65 ± 0.32   Final old_new: 93.39 ± 0.44   Final new_new: 89.28 ± 2.15
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.82 ± 5.91   Final Test: 29.26 ± 2.50   Final old_old: 31.89 ± 2.47   Final old_new: 18.54 ± 2.81   Final new_new: 8.18 ± 5.23
Hits@20:
  Final val: 55.65 ± 3.94   Final Test: 38.05 ± 4.23   Final old_old: 41.29 ± 4.51   Final old_new: 24.89 ± 3.27   Final new_new: 11.36 ± 1.61
Hits@30:
  Final val: 62.68 ± 2.99   Final Test: 45.93 ± 2.80   Final old_old: 49.28 ± 2.99   Final old_new: 32.57 ± 2.45   Final new_new: 13.18 ± 3.73
Hits@50:
  Final val: 71.12 ± 1.55   Final Test: 54.58 ± 3.00   Final old_old: 58.25 ± 3.29   Final old_new: 40.05 ± 2.17   Final new_new: 17.27 ± 6.93
AUC:
  Final val: 96.56 ± 0.18   Final Test: 95.94 ± 0.32   Final old_old: 96.63 ± 0.27   Final old_new: 93.16 ± 0.60   Final new_new: 88.94 ± 1.54
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.46 ± 2.21   Final Test: 29.86 ± 4.29   Final old_old: 32.99 ± 4.73   Final old_new: 17.18 ± 2.76   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 61.15 ± 3.24   Final Test: 41.09 ± 3.99   Final old_old: 44.73 ± 4.48   Final old_new: 26.45 ± 2.50   Final new_new: 7.73 ± 3.45
Hits@30:
  Final val: 67.37 ± 1.87   Final Test: 45.23 ± 3.68   Final old_old: 49.06 ± 3.98   Final old_new: 29.90 ± 2.92   Final new_new: 9.09 ± 4.82
Hits@50:
  Final val: 74.70 ± 1.01   Final Test: 56.58 ± 2.05   Final old_old: 60.77 ± 2.10   Final old_new: 40.03 ± 2.52   Final new_new: 13.64 ± 7.01
AUC:
  Final val: 97.27 ± 0.06   Final Test: 96.71 ± 0.07   Final old_old: 97.31 ± 0.12   Final old_new: 94.31 ± 0.29   Final new_new: 90.65 ± 0.58
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.46 ± 3.78   Final Test: 34.12 ± 10.05   Final old_old: 37.46 ± 10.74   Final old_new: 20.65 ± 7.54   Final new_new: 5.00 ± 2.49
Hits@20:
  Final val: 63.27 ± 2.92   Final Test: 44.28 ± 5.84   Final old_old: 48.09 ± 6.28   Final old_new: 29.09 ± 4.30   Final new_new: 7.73 ± 5.47
Hits@30:
  Final val: 69.86 ± 1.63   Final Test: 52.34 ± 3.12   Final old_old: 56.54 ± 3.23   Final old_new: 35.54 ± 2.90   Final new_new: 11.82 ± 2.96
Hits@50:
  Final val: 77.17 ± 1.66   Final Test: 63.41 ± 2.72   Final old_old: 67.60 ± 2.94   Final old_new: 46.88 ± 1.93   Final new_new: 19.55 ± 4.98
AUC:
  Final val: 97.49 ± 0.11   Final Test: 97.03 ± 0.13   Final old_old: 97.60 ± 0.14   Final old_new: 94.76 ± 0.16   Final new_new: 90.99 ± 0.71
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.13 ± 3.77   Final Test: 31.05 ± 5.58   Final old_old: 34.22 ± 6.02   Final old_new: 18.24 ± 4.05   Final new_new: 3.18 ± 2.03
Hits@20:
  Final val: 60.88 ± 2.14   Final Test: 41.71 ± 2.76   Final old_old: 45.52 ± 2.96   Final old_new: 26.55 ± 2.13   Final new_new: 4.55 ± 1.61
Hits@30:
  Final val: 67.86 ± 2.68   Final Test: 50.58 ± 2.18   Final old_old: 54.67 ± 2.11   Final old_new: 34.48 ± 2.75   Final new_new: 6.36 ± 1.90
Hits@50:
  Final val: 76.18 ± 1.73   Final Test: 60.24 ± 3.10   Final old_old: 64.46 ± 3.11   Final old_new: 43.80 ± 3.20   Final new_new: 12.73 ± 3.45
AUC:
  Final val: 97.33 ± 0.17   Final Test: 96.81 ± 0.19   Final old_old: 97.43 ± 0.19   Final old_new: 94.31 ± 0.35   Final new_new: 91.15 ± 0.52
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.86 ± 4.39   Final Test: 28.68 ± 4.12   Final old_old: 31.26 ± 4.50   Final old_new: 18.29 ± 3.16   Final new_new: 5.45 ± 2.03
Hits@20:
  Final val: 55.91 ± 3.66   Final Test: 36.66 ± 4.54   Final old_old: 39.59 ± 4.75   Final old_new: 25.04 ± 4.17   Final new_new: 6.82 ± 1.61
Hits@30:
  Final val: 63.67 ± 2.79   Final Test: 42.95 ± 5.51   Final old_old: 46.35 ± 5.86   Final old_new: 29.45 ± 4.16   Final new_new: 9.55 ± 2.96
Hits@50:
  Final val: 72.05 ± 1.74   Final Test: 54.10 ± 4.02   Final old_old: 57.64 ± 4.26   Final old_new: 40.33 ± 3.21   Final new_new: 13.18 ± 2.96
AUC:
  Final val: 96.64 ± 0.24   Final Test: 96.01 ± 0.27   Final old_old: 96.70 ± 0.27   Final old_new: 93.30 ± 0.38   Final new_new: 89.06 ± 1.67
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.73 ± 3.35   Final Test: 30.90 ± 4.19   Final old_old: 34.11 ± 4.45   Final old_new: 17.88 ± 3.47   Final new_new: 4.09 ± 1.02
Hits@20:
  Final val: 59.51 ± 2.50   Final Test: 42.17 ± 1.81   Final old_old: 45.80 ± 1.93   Final old_new: 27.66 ± 1.95   Final new_new: 7.27 ± 1.90
Hits@30:
  Final val: 65.51 ± 1.68   Final Test: 50.07 ± 1.48   Final old_old: 54.04 ± 1.61   Final old_new: 34.36 ± 1.75   Final new_new: 9.55 ± 2.49
Hits@50:
  Final val: 73.91 ± 1.20   Final Test: 57.64 ± 2.69   Final old_old: 61.84 ± 2.83   Final old_new: 41.18 ± 2.34   Final new_new: 12.27 ± 3.80
AUC:
  Final val: 97.23 ± 0.15   Final Test: 96.70 ± 0.09   Final old_old: 97.32 ± 0.08   Final old_new: 94.22 ± 0.38   Final new_new: 89.92 ± 0.87
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.85 ± 3.55   Final Test: 31.96 ± 2.01   Final old_old: 35.17 ± 2.30   Final old_new: 18.97 ± 0.86   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 62.68 ± 1.44   Final Test: 40.41 ± 2.31   Final old_old: 44.17 ± 2.26   Final old_new: 25.31 ± 2.65   Final new_new: 5.91 ± 1.24
Hits@30:
  Final val: 68.30 ± 1.08   Final Test: 50.01 ± 3.13   Final old_old: 54.04 ± 3.23   Final old_new: 34.08 ± 2.90   Final new_new: 8.64 ± 3.37
Hits@50:
  Final val: 75.82 ± 1.31   Final Test: 58.53 ± 0.92   Final old_old: 62.56 ± 0.77   Final old_new: 42.82 ± 1.78   Final new_new: 12.73 ± 1.24
AUC:
  Final val: 97.47 ± 0.15   Final Test: 97.02 ± 0.06   Final old_old: 97.54 ± 0.06   Final old_new: 94.97 ± 0.20   Final new_new: 91.43 ± 0.84
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.47 ± 4.94   Final Test: 35.46 ± 4.04   Final old_old: 39.08 ± 4.27   Final old_new: 20.88 ± 3.42   Final new_new: 2.73 ± 1.02
Hits@20:
  Final val: 63.95 ± 2.37   Final Test: 45.38 ± 4.28   Final old_old: 49.58 ± 4.28   Final old_new: 28.54 ± 4.56   Final new_new: 6.82 ± 2.78
Hits@30:
  Final val: 69.92 ± 1.62   Final Test: 53.93 ± 3.08   Final old_old: 58.20 ± 3.03   Final old_new: 37.00 ± 3.55   Final new_new: 10.45 ± 4.71
Hits@50:
  Final val: 76.86 ± 1.07   Final Test: 61.72 ± 2.45   Final old_old: 66.07 ± 2.33   Final old_new: 44.69 ± 3.03   Final new_new: 13.64 ± 7.54
AUC:
  Final val: 97.37 ± 0.12   Final Test: 96.87 ± 0.10   Final old_old: 97.52 ± 0.10   Final old_new: 94.26 ± 0.10   Final new_new: 90.18 ± 1.44
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.16 ± 1.52   Final Test: 33.72 ± 2.36   Final old_old: 37.09 ± 2.43   Final old_new: 20.15 ± 2.62   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 63.75 ± 1.64   Final Test: 45.20 ± 1.52   Final old_old: 48.98 ± 1.70   Final old_new: 30.23 ± 2.09   Final new_new: 6.36 ± 2.49
Hits@30:
  Final val: 70.00 ± 1.91   Final Test: 48.93 ± 1.34   Final old_old: 52.77 ± 1.32   Final old_new: 33.93 ± 1.96   Final new_new: 6.36 ± 2.96
Hits@50:
  Final val: 77.08 ± 0.58   Final Test: 61.07 ± 1.74   Final old_old: 65.23 ± 1.72   Final old_new: 45.01 ± 2.60   Final new_new: 10.91 ± 2.96
AUC:
  Final val: 97.37 ± 0.11   Final Test: 96.75 ± 0.09   Final old_old: 97.38 ± 0.08   Final old_new: 94.24 ± 0.20   Final new_new: 90.10 ± 1.67
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.88 ± 4.79   Final Test: 26.67 ± 4.03   Final old_old: 29.02 ± 4.28   Final old_new: 17.08 ± 3.34   Final new_new: 8.18 ± 5.70
Hits@20:
  Final val: 56.09 ± 5.04   Final Test: 39.56 ± 6.04   Final old_old: 42.55 ± 6.29   Final old_new: 27.63 ± 5.48   Final new_new: 10.91 ± 6.31
Hits@30:
  Final val: 62.79 ± 2.57   Final Test: 43.68 ± 4.91   Final old_old: 47.03 ± 5.01   Final old_new: 30.33 ± 5.19   Final new_new: 11.36 ± 7.01
Hits@50:
  Final val: 71.17 ± 1.27   Final Test: 54.94 ± 4.41   Final old_old: 58.66 ± 4.29   Final old_new: 40.30 ± 5.14   Final new_new: 15.91 ± 7.19
AUC:
  Final val: 96.60 ± 0.17   Final Test: 96.04 ± 0.20   Final old_old: 96.72 ± 0.21   Final old_new: 93.38 ± 0.35   Final new_new: 88.38 ± 2.15
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.58 ± 2.05   Final Test: 30.63 ± 4.72   Final old_old: 33.75 ± 5.10   Final old_new: 17.96 ± 3.20   Final new_new: 4.55 ± 2.27
Hits@20:
  Final val: 60.94 ± 2.08   Final Test: 40.57 ± 2.06   Final old_old: 44.20 ± 2.13   Final old_new: 26.07 ± 2.34   Final new_new: 5.45 ± 1.24
Hits@30:
  Final val: 67.19 ± 0.83   Final Test: 48.15 ± 3.72   Final old_old: 52.05 ± 3.97   Final old_new: 32.62 ± 2.72   Final new_new: 10.00 ± 3.45
Hits@50:
  Final val: 74.36 ± 0.44   Final Test: 58.27 ± 1.92   Final old_old: 62.52 ± 2.18   Final old_new: 41.51 ± 1.26   Final new_new: 13.64 ± 4.55
AUC:
  Final val: 97.25 ± 0.08   Final Test: 96.71 ± 0.08   Final old_old: 97.35 ± 0.09   Final old_new: 94.16 ± 0.37   Final new_new: 89.70 ± 1.22
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.16 ± 1.67   Final Test: 28.72 ± 4.93   Final old_old: 31.30 ± 5.27   Final old_new: 18.41 ± 3.63   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 60.26 ± 3.22   Final Test: 41.28 ± 6.50   Final old_old: 44.58 ± 6.94   Final old_new: 27.98 ± 4.65   Final new_new: 11.36 ± 4.55
Hits@30:
  Final val: 67.43 ± 2.06   Final Test: 47.33 ± 7.33   Final old_old: 50.90 ± 7.86   Final old_new: 33.05 ± 5.13   Final new_new: 12.73 ± 5.93
Hits@50:
  Final val: 74.59 ± 2.13   Final Test: 57.29 ± 2.41   Final old_old: 61.20 ± 2.39   Final old_new: 41.74 ± 2.89   Final new_new: 18.64 ± 3.73
AUC:
  Final val: 96.86 ± 0.19   Final Test: 96.22 ± 0.22   Final old_old: 96.83 ± 0.22   Final old_new: 93.72 ± 0.27   Final new_new: 91.07 ± 1.15
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.64 ± 2.13   Final Test: 31.75 ± 2.51   Final old_old: 35.10 ± 2.61   Final old_new: 18.14 ± 2.34   Final new_new: 3.64 ± 2.03
Hits@20:
  Final val: 58.41 ± 2.68   Final Test: 41.15 ± 4.32   Final old_old: 44.71 ± 4.52   Final old_new: 26.83 ± 3.55   Final new_new: 8.64 ± 3.37
Hits@30:
  Final val: 65.52 ± 1.64   Final Test: 47.22 ± 2.65   Final old_old: 51.08 ± 2.83   Final old_new: 31.81 ± 2.01   Final new_new: 10.00 ± 1.24
Hits@50:
  Final val: 73.70 ± 0.84   Final Test: 57.73 ± 1.77   Final old_old: 61.99 ± 1.89   Final old_new: 40.91 ± 1.18   Final new_new: 13.18 ± 3.73
AUC:
  Final val: 97.18 ± 0.11   Final Test: 96.72 ± 0.03   Final old_old: 97.33 ± 0.06   Final old_new: 94.34 ± 0.34   Final new_new: 90.21 ± 0.90
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.65 ± 3.17   Final Test: 34.89 ± 4.36   Final old_old: 38.21 ± 4.67   Final old_new: 21.64 ± 3.29   Final new_new: 3.18 ± 1.24
Hits@20:
  Final val: 64.30 ± 2.61   Final Test: 45.29 ± 3.27   Final old_old: 49.08 ± 3.23   Final old_new: 30.33 ± 3.54   Final new_new: 5.45 ± 2.59
Hits@30:
  Final val: 70.68 ± 2.31   Final Test: 53.32 ± 2.73   Final old_old: 57.31 ± 2.67   Final old_new: 37.73 ± 3.23   Final new_new: 9.09 ± 2.27
Hits@50:
  Final val: 77.63 ± 1.75   Final Test: 61.72 ± 2.44   Final old_old: 65.95 ± 2.20   Final old_new: 45.37 ± 3.64   Final new_new: 11.82 ± 4.93
AUC:
  Final val: 97.40 ± 0.11   Final Test: 96.89 ± 0.16   Final old_old: 97.51 ± 0.14   Final old_new: 94.45 ± 0.36   Final new_new: 89.70 ± 1.18
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.10 ± 2.60   Final Test: 30.90 ± 4.30   Final old_old: 33.74 ± 4.73   Final old_new: 19.47 ± 2.73   Final new_new: 5.91 ± 3.45
Hits@20:
  Final val: 55.37 ± 3.20   Final Test: 38.77 ± 3.86   Final old_old: 41.98 ± 4.01   Final old_new: 25.92 ± 3.90   Final new_new: 8.64 ± 3.73
Hits@30:
  Final val: 62.49 ± 3.15   Final Test: 45.85 ± 3.29   Final old_old: 49.30 ± 3.30   Final old_new: 32.09 ± 4.00   Final new_new: 12.27 ± 5.23
Hits@50:
  Final val: 71.46 ± 1.76   Final Test: 55.69 ± 2.28   Final old_old: 59.34 ± 2.55   Final old_new: 41.31 ± 2.16   Final new_new: 16.82 ± 9.72
AUC:
  Final val: 96.63 ± 0.24   Final Test: 95.94 ± 0.33   Final old_old: 96.63 ± 0.26   Final old_new: 93.22 ± 0.72   Final new_new: 88.38 ± 2.95
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 39.34 ± 3.91   Final Test: 29.80 ± 9.11   Final old_old: 32.50 ± 9.88   Final old_new: 18.82 ± 6.05   Final new_new: 7.73 ± 2.59
Hits@20:
  Final val: 51.82 ± 3.19   Final Test: 36.38 ± 6.38   Final old_old: 39.09 ± 7.02   Final old_new: 25.54 ± 4.12   Final new_new: 10.00 ± 1.24
Hits@30:
  Final val: 60.27 ± 2.30   Final Test: 43.85 ± 4.62   Final old_old: 47.09 ± 5.00   Final old_new: 30.81 ± 3.24   Final new_new: 15.45 ± 3.37
Hits@50:
  Final val: 70.70 ± 2.31   Final Test: 53.31 ± 3.78   Final old_old: 56.92 ± 3.96   Final old_new: 38.92 ± 3.87   Final new_new: 17.73 ± 3.73
AUC:
  Final val: 96.58 ± 0.17   Final Test: 95.95 ± 0.16   Final old_old: 96.62 ± 0.16   Final old_new: 93.25 ± 0.43   Final new_new: 89.64 ± 0.71
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 53.78 ± 3.27   Final Test: 35.08 ± 4.78   Final old_old: 38.40 ± 5.14   Final old_new: 21.79 ± 3.66   Final new_new: 3.64 ± 3.45
Hits@20:
  Final val: 66.49 ± 1.54   Final Test: 46.61 ± 3.51   Final old_old: 50.52 ± 3.73   Final old_new: 31.18 ± 2.94   Final new_new: 5.45 ± 4.71
Hits@30:
  Final val: 71.73 ± 0.99   Final Test: 53.38 ± 3.27   Final old_old: 57.43 ± 3.59   Final old_new: 37.51 ± 2.09   Final new_new: 8.64 ± 5.18
Hits@50:
  Final val: 78.57 ± 1.43   Final Test: 64.26 ± 2.97   Final old_old: 68.42 ± 3.15   Final old_new: 48.01 ± 2.55   Final new_new: 18.18 ± 4.25
AUC:
  Final val: 97.63 ± 0.10   Final Test: 97.14 ± 0.06   Final old_old: 97.67 ± 0.11   Final old_new: 95.07 ± 0.17   Final new_new: 91.30 ± 1.43
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.30 ± 1.29   Final Test: 33.01 ± 3.96   Final old_old: 36.33 ± 4.19   Final old_new: 19.50 ± 3.14   Final new_new: 5.45 ± 2.03
Hits@20:
  Final val: 59.59 ± 1.99   Final Test: 40.09 ± 0.85   Final old_old: 43.52 ± 1.08   Final old_new: 26.45 ± 1.14   Final new_new: 5.91 ± 1.24
Hits@30:
  Final val: 65.55 ± 1.98   Final Test: 46.31 ± 3.46   Final old_old: 49.97 ± 3.78   Final old_new: 31.81 ± 2.34   Final new_new: 8.64 ± 4.37
Hits@50:
  Final val: 73.41 ± 0.96   Final Test: 57.43 ± 1.90   Final old_old: 61.50 ± 1.83   Final old_new: 41.41 ± 2.14   Final new_new: 14.09 ± 6.10
AUC:
  Final val: 97.21 ± 0.08   Final Test: 96.72 ± 0.08   Final old_old: 97.32 ± 0.06   Final old_new: 94.40 ± 0.25   Final new_new: 89.93 ± 0.46
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.60 ± 1.71   Final Test: 30.63 ± 5.05   Final old_old: 33.51 ± 5.28   Final old_new: 19.14 ± 4.56   Final new_new: 3.18 ± 1.24
Hits@20:
  Final val: 64.16 ± 1.66   Final Test: 39.25 ± 2.10   Final old_old: 42.83 ± 1.93   Final old_new: 24.94 ± 3.27   Final new_new: 5.00 ± 1.90
Hits@30:
  Final val: 70.08 ± 1.01   Final Test: 46.07 ± 2.85   Final old_old: 50.00 ± 2.99   Final old_new: 30.43 ± 2.55   Final new_new: 6.82 ± 2.78
Hits@50:
  Final val: 76.88 ± 0.45   Final Test: 61.32 ± 1.63   Final old_old: 65.45 ± 1.82   Final old_new: 45.16 ± 1.54   Final new_new: 15.45 ± 2.96
AUC:
  Final val: 97.37 ± 0.09   Final Test: 96.79 ± 0.07   Final old_old: 97.44 ± 0.10   Final old_new: 94.23 ± 0.21   Final new_new: 90.08 ± 1.11
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.62 ± 3.33   Final Test: 29.25 ± 1.74   Final old_old: 32.24 ± 1.81   Final old_new: 17.23 ± 1.63   Final new_new: 2.27 ± 0.00
Hits@20:
  Final val: 62.15 ± 2.07   Final Test: 39.47 ± 4.15   Final old_old: 42.96 ± 4.25   Final old_new: 25.54 ± 3.91   Final new_new: 5.45 ± 2.59
Hits@30:
  Final val: 68.05 ± 2.28   Final Test: 49.40 ± 3.33   Final old_old: 53.30 ± 3.25   Final old_new: 33.90 ± 3.97   Final new_new: 10.00 ± 3.80
Hits@50:
  Final val: 75.32 ± 1.26   Final Test: 57.44 ± 3.83   Final old_old: 61.44 ± 3.74   Final old_new: 41.79 ± 4.41   Final new_new: 13.64 ± 3.59
AUC:
  Final val: 97.47 ± 0.03   Final Test: 97.04 ± 0.10   Final old_old: 97.52 ± 0.10   Final old_new: 95.17 ± 0.18   Final new_new: 91.49 ± 0.45
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.65 ± 3.12   Final Test: 31.91 ± 4.33   Final old_old: 35.12 ± 4.72   Final old_new: 19.02 ± 2.83   Final new_new: 2.73 ± 1.90
Hits@20:
  Final val: 62.71 ± 1.79   Final Test: 42.42 ± 4.55   Final old_old: 46.22 ± 4.94   Final old_new: 27.38 ± 3.30   Final new_new: 4.09 ± 1.90
Hits@30:
  Final val: 69.55 ± 2.26   Final Test: 51.33 ± 3.18   Final old_old: 55.39 ± 3.56   Final old_new: 35.31 ± 1.86   Final new_new: 9.09 ± 2.78
Hits@50:
  Final val: 76.31 ± 2.07   Final Test: 61.08 ± 2.80   Final old_old: 65.25 ± 2.99   Final old_new: 44.84 ± 2.30   Final new_new: 14.09 ± 2.49
AUC:
  Final val: 97.33 ± 0.13   Final Test: 96.81 ± 0.11   Final old_old: 97.44 ± 0.15   Final old_new: 94.34 ± 0.17   Final new_new: 90.18 ± 0.95
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.09 ± 2.36   Final Test: 25.06 ± 5.67   Final old_old: 27.37 ± 6.11   Final old_new: 15.72 ± 4.13   Final new_new: 4.55 ± 3.21
Hits@20:
  Final val: 54.93 ± 1.94   Final Test: 35.97 ± 4.61   Final old_old: 38.81 ± 4.95   Final old_new: 24.69 ± 3.70   Final new_new: 7.73 ± 2.59
Hits@30:
  Final val: 63.61 ± 1.20   Final Test: 43.86 ± 4.03   Final old_old: 47.26 ± 4.44   Final old_new: 30.40 ± 2.75   Final new_new: 9.09 ± 2.78
Hits@50:
  Final val: 71.27 ± 2.24   Final Test: 55.78 ± 2.97   Final old_old: 59.48 ± 3.22   Final old_new: 41.28 ± 2.28   Final new_new: 14.55 ± 3.45
AUC:
  Final val: 96.65 ± 0.21   Final Test: 95.94 ± 0.31   Final old_old: 96.60 ± 0.29   Final old_new: 93.31 ± 0.52   Final new_new: 89.29 ± 2.33
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.72 ± 2.69   Final Test: 30.63 ± 5.52   Final old_old: 33.75 ± 5.89   Final old_new: 17.93 ± 4.17   Final new_new: 5.00 ± 2.49
Hits@20:
  Final val: 59.57 ± 2.19   Final Test: 38.50 ± 3.17   Final old_old: 41.90 ± 3.28   Final old_new: 24.84 ± 3.06   Final new_new: 7.27 ± 1.02
Hits@30:
  Final val: 65.99 ± 2.50   Final Test: 46.63 ± 2.64   Final old_old: 50.53 ± 2.74   Final old_new: 31.01 ± 2.52   Final new_new: 10.45 ± 2.03
Hits@50:
  Final val: 73.47 ± 1.97   Final Test: 56.53 ± 2.78   Final old_old: 60.61 ± 2.87   Final old_new: 40.40 ± 2.63   Final new_new: 14.55 ± 2.59
AUC:
  Final val: 97.21 ± 0.10   Final Test: 96.64 ± 0.15   Final old_old: 97.25 ± 0.16   Final old_new: 94.25 ± 0.26   Final new_new: 89.70 ± 0.50
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.32 ± 2.37   Final Test: 28.32 ± 3.58   Final old_old: 30.94 ± 3.72   Final old_new: 17.71 ± 3.26   Final new_new: 5.91 ± 2.03
Hits@20:
  Final val: 59.75 ± 1.63   Final Test: 38.83 ± 3.39   Final old_old: 42.25 ± 3.67   Final old_new: 24.94 ± 2.69   Final new_new: 10.45 ± 3.45
Hits@30:
  Final val: 65.39 ± 1.82   Final Test: 47.24 ± 6.38   Final old_old: 50.93 ± 6.42   Final old_new: 32.44 ± 6.57   Final new_new: 13.18 ± 4.37
Hits@50:
  Final val: 72.28 ± 0.61   Final Test: 56.36 ± 3.00   Final old_old: 60.38 ± 2.90   Final old_new: 40.35 ± 3.88   Final new_new: 17.27 ± 6.14
AUC:
  Final val: 95.66 ± 0.30   Final Test: 95.13 ± 0.32   Final old_old: 95.83 ± 0.30   Final old_new: 92.49 ± 0.48   Final new_new: 85.06 ± 1.34
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.14 ± 2.14   Final Test: 28.13 ± 4.36   Final old_old: 30.99 ± 4.63   Final old_new: 16.57 ± 3.41   Final new_new: 3.18 ± 1.24
Hits@20:
  Final val: 59.12 ± 1.54   Final Test: 41.64 ± 3.10   Final old_old: 45.10 ± 3.20   Final old_new: 27.93 ± 3.34   Final new_new: 6.82 ± 2.27
Hits@30:
  Final val: 66.18 ± 2.50   Final Test: 47.44 ± 3.97   Final old_old: 51.10 ± 4.02   Final old_new: 32.97 ± 3.83   Final new_new: 9.09 ± 4.25
Hits@50:
  Final val: 74.22 ± 1.80   Final Test: 58.82 ± 1.99   Final old_old: 62.87 ± 1.95   Final old_new: 43.07 ± 2.27   Final new_new: 12.27 ± 2.03
AUC:
  Final val: 97.21 ± 0.12   Final Test: 96.70 ± 0.09   Final old_old: 97.31 ± 0.08   Final old_new: 94.32 ± 0.31   Final new_new: 90.03 ± 1.27
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.22 ± 0.77   Final Test: 33.69 ± 4.72   Final old_old: 36.87 ± 4.94   Final old_new: 20.91 ± 4.11   Final new_new: 5.00 ± 2.49
Hits@20:
  Final val: 61.87 ± 0.89   Final Test: 40.66 ± 3.02   Final old_old: 44.25 ± 3.44   Final old_new: 26.37 ± 1.55   Final new_new: 5.45 ± 3.45
Hits@30:
  Final val: 67.84 ± 0.98   Final Test: 49.28 ± 1.80   Final old_old: 53.25 ± 1.98   Final old_new: 33.50 ± 1.46   Final new_new: 9.09 ± 2.78
Hits@50:
  Final val: 76.49 ± 1.08   Final Test: 61.06 ± 1.75   Final old_old: 65.33 ± 1.75   Final old_new: 44.36 ± 2.71   Final new_new: 13.18 ± 3.37
AUC:
  Final val: 97.32 ± 0.10   Final Test: 96.78 ± 0.11   Final old_old: 97.40 ± 0.15   Final old_new: 94.35 ± 0.12   Final new_new: 90.17 ± 0.73
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 53.56 ± 2.42   Final Test: 32.08 ± 5.93   Final old_old: 35.36 ± 6.22   Final old_new: 18.74 ± 5.34   Final new_new: 5.00 ± 1.90
Hits@20:
  Final val: 63.76 ± 0.98   Final Test: 42.70 ± 2.66   Final old_old: 46.66 ± 2.51   Final old_new: 26.70 ± 3.57   Final new_new: 8.18 ± 3.45
Hits@30:
  Final val: 68.66 ± 0.54   Final Test: 49.42 ± 2.89   Final old_old: 53.31 ± 2.83   Final old_new: 33.98 ± 3.26   Final new_new: 10.00 ± 4.13
Hits@50:
  Final val: 74.29 ± 0.82   Final Test: 60.29 ± 2.02   Final old_old: 64.01 ± 2.20   Final old_new: 45.79 ± 1.56   Final new_new: 18.64 ± 4.93
AUC:
  Final val: 95.60 ± 0.25   Final Test: 95.16 ± 0.20   Final old_old: 95.86 ± 0.15   Final old_new: 92.44 ± 0.52   Final new_new: 86.42 ± 1.24
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.36 ± 6.52   Final Test: 25.47 ± 6.53   Final old_old: 28.01 ± 7.09   Final old_new: 15.06 ± 4.32   Final new_new: 5.91 ± 3.05
Hits@20:
  Final val: 52.71 ± 5.00   Final Test: 37.73 ± 7.14   Final old_old: 40.89 ± 7.42   Final old_new: 25.01 ± 6.26   Final new_new: 9.55 ± 5.43
Hits@30:
  Final val: 61.96 ± 3.61   Final Test: 44.19 ± 3.92   Final old_old: 47.51 ± 4.04   Final old_new: 30.98 ± 3.90   Final new_new: 11.36 ± 2.78
Hits@50:
  Final val: 70.90 ± 3.41   Final Test: 56.24 ± 2.81   Final old_old: 59.85 ± 3.03   Final old_new: 41.99 ± 3.23   Final new_new: 18.18 ± 2.78
AUC:
  Final val: 96.57 ± 0.11   Final Test: 95.88 ± 0.06   Final old_old: 96.56 ± 0.06   Final old_new: 93.17 ± 0.35   Final new_new: 89.31 ± 1.28
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.07 ± 3.63   Final Test: 28.60 ± 6.08   Final old_old: 31.10 ± 6.34   Final old_new: 18.54 ± 5.24   Final new_new: 5.91 ± 2.59
Hits@20:
  Final val: 57.22 ± 2.57   Final Test: 38.94 ± 4.05   Final old_old: 42.16 ± 4.23   Final old_new: 25.99 ± 3.91   Final new_new: 10.00 ± 3.45
Hits@30:
  Final val: 64.57 ± 1.98   Final Test: 44.82 ± 2.41   Final old_old: 48.24 ± 2.30   Final old_new: 31.18 ± 3.49   Final new_new: 11.36 ± 1.61
Hits@50:
  Final val: 72.67 ± 1.11   Final Test: 54.66 ± 2.03   Final old_old: 58.46 ± 2.06   Final old_new: 39.70 ± 2.76   Final new_new: 14.55 ± 2.59
AUC:
  Final val: 96.66 ± 0.25   Final Test: 95.95 ± 0.23   Final old_old: 96.65 ± 0.19   Final old_new: 93.15 ± 0.56   Final new_new: 89.02 ± 0.85
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.46 ± 3.20   Final Test: 31.77 ± 4.08   Final old_old: 34.85 ± 4.42   Final old_new: 19.42 ± 3.02   Final new_new: 3.18 ± 1.24
Hits@20:
  Final val: 59.39 ± 2.08   Final Test: 40.72 ± 5.72   Final old_old: 44.33 ± 5.92   Final old_new: 26.35 ± 5.25   Final new_new: 5.00 ± 3.73
Hits@30:
  Final val: 66.88 ± 0.78   Final Test: 49.46 ± 2.14   Final old_old: 53.39 ± 1.98   Final old_new: 34.01 ± 3.07   Final new_new: 7.27 ± 4.07
Hits@50:
  Final val: 74.80 ± 0.85   Final Test: 58.43 ± 2.73   Final old_old: 62.53 ± 2.82   Final old_new: 42.49 ± 2.54   Final new_new: 11.82 ± 3.73
AUC:
  Final val: 97.25 ± 0.13   Final Test: 96.72 ± 0.05   Final old_old: 97.35 ± 0.13   Final old_new: 94.24 ± 0.31   Final new_new: 90.54 ± 1.16
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.22 ± 1.44   Final Test: 31.21 ± 5.91   Final old_old: 34.42 ± 6.24   Final old_new: 18.11 ± 4.70   Final new_new: 5.45 ± 2.59
Hits@20:
  Final val: 60.14 ± 1.56   Final Test: 41.90 ± 3.10   Final old_old: 45.47 ± 3.08   Final old_new: 27.63 ± 3.48   Final new_new: 7.73 ± 4.71
Hits@30:
  Final val: 66.17 ± 1.30   Final Test: 51.50 ± 2.14   Final old_old: 55.05 ± 2.22   Final old_new: 37.46 ± 1.94   Final new_new: 15.45 ± 7.61
Hits@50:
  Final val: 72.87 ± 0.95   Final Test: 60.16 ± 1.45   Final old_old: 63.75 ± 1.09   Final old_new: 46.07 ± 3.00   Final new_new: 21.36 ± 11.66
AUC:
  Final val: 95.49 ± 0.12   Final Test: 95.09 ± 0.11   Final old_old: 95.79 ± 0.10   Final old_new: 92.31 ± 0.26   Final new_new: 87.39 ± 1.45
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.32 ± 3.79   Final Test: 34.80 ± 3.43   Final old_old: 38.00 ± 3.60   Final old_new: 21.96 ± 2.93   Final new_new: 5.00 ± 1.02
Hits@20:
  Final val: 61.87 ± 4.04   Final Test: 41.90 ± 3.25   Final old_old: 45.33 ± 3.54   Final old_new: 28.29 ± 2.11   Final new_new: 8.18 ± 5.23
Hits@30:
  Final val: 68.11 ± 3.02   Final Test: 51.23 ± 3.76   Final old_old: 55.11 ± 3.85   Final old_new: 35.92 ± 3.34   Final new_new: 10.45 ± 5.23
Hits@50:
  Final val: 75.36 ± 1.51   Final Test: 60.27 ± 2.30   Final old_old: 64.31 ± 2.34   Final old_new: 44.41 ± 2.36   Final new_new: 16.82 ± 1.24
AUC:
  Final val: 97.32 ± 0.06   Final Test: 96.74 ± 0.11   Final old_old: 97.35 ± 0.11   Final old_new: 94.38 ± 0.21   Final new_new: 89.62 ± 1.28
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.59 ± 5.36   Final Test: 31.32 ± 5.00   Final old_old: 34.26 ± 5.14   Final old_new: 19.50 ± 4.67   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 60.33 ± 5.31   Final Test: 42.69 ± 1.95   Final old_old: 46.32 ± 1.80   Final old_new: 28.24 ± 2.82   Final new_new: 6.82 ± 1.61
Hits@30:
  Final val: 67.62 ± 3.44   Final Test: 51.74 ± 2.63   Final old_old: 55.74 ± 2.83   Final old_new: 35.94 ± 1.99   Final new_new: 9.55 ± 1.90
Hits@50:
  Final val: 75.56 ± 1.34   Final Test: 61.87 ± 3.46   Final old_old: 66.02 ± 3.42   Final old_new: 45.69 ± 4.09   Final new_new: 15.45 ± 4.93
AUC:
  Final val: 97.30 ± 0.08   Final Test: 96.79 ± 0.04   Final old_old: 97.39 ± 0.04   Final old_new: 94.36 ± 0.32   Final new_new: 91.50 ± 0.92
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.48 ± 3.91   Final Test: 23.61 ± 3.14   Final old_old: 25.90 ± 3.24   Final old_new: 14.36 ± 2.90   Final new_new: 3.64 ± 3.05
Hits@20:
  Final val: 53.27 ± 3.77   Final Test: 33.41 ± 6.05   Final old_old: 36.25 ± 6.63   Final old_new: 21.91 ± 3.76   Final new_new: 8.18 ± 2.03
Hits@30:
  Final val: 61.80 ± 4.07   Final Test: 45.81 ± 2.81   Final old_old: 49.20 ± 3.03   Final old_new: 32.34 ± 2.57   Final new_new: 12.73 ± 4.71
Hits@50:
  Final val: 70.90 ± 1.15   Final Test: 55.57 ± 1.95   Final old_old: 59.34 ± 2.10   Final old_new: 40.53 ± 1.73   Final new_new: 18.64 ± 7.43
AUC:
  Final val: 96.59 ± 0.18   Final Test: 95.90 ± 0.15   Final old_old: 96.57 ± 0.16   Final old_new: 93.21 ± 0.40   Final new_new: 89.59 ± 1.43
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.31 ± 5.37   Final Test: 27.84 ± 5.08   Final old_old: 30.43 ± 5.66   Final old_new: 17.30 ± 2.64   Final new_new: 6.36 ± 4.37
Hits@20:
  Final val: 55.38 ± 4.51   Final Test: 39.90 ± 2.78   Final old_old: 43.07 ± 3.13   Final old_new: 27.13 ± 2.15   Final new_new: 10.91 ± 2.96
Hits@30:
  Final val: 62.87 ± 1.22   Final Test: 43.06 ± 2.52   Final old_old: 46.32 ± 2.54   Final old_new: 29.82 ± 2.69   Final new_new: 16.36 ± 4.93
Hits@50:
  Final val: 69.74 ± 0.70   Final Test: 53.11 ± 3.22   Final old_old: 56.64 ± 3.33   Final old_new: 39.02 ± 3.30   Final new_new: 18.64 ± 4.07
AUC:
  Final val: 95.03 ± 0.25   Final Test: 94.53 ± 0.45   Final old_old: 95.20 ± 0.41   Final old_new: 91.98 ± 0.83   Final new_new: 85.80 ± 3.27
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.08 ± 3.27   Final Test: 31.93 ± 3.28   Final old_old: 35.08 ± 3.39   Final old_new: 19.17 ± 3.15   Final new_new: 4.55 ± 3.21
Hits@20:
  Final val: 57.00 ± 3.39   Final Test: 38.97 ± 4.35   Final old_old: 42.47 ± 4.74   Final old_new: 24.91 ± 3.22   Final new_new: 6.36 ± 4.07
Hits@30:
  Final val: 64.81 ± 1.66   Final Test: 47.74 ± 5.39   Final old_old: 51.44 ± 5.53   Final old_new: 33.07 ± 4.92   Final new_new: 10.00 ± 3.05
Hits@50:
  Final val: 73.41 ± 1.33   Final Test: 57.49 ± 1.27   Final old_old: 61.46 ± 1.12   Final old_new: 41.86 ± 2.01   Final new_new: 15.00 ± 3.45
AUC:
  Final val: 97.13 ± 0.09   Final Test: 96.65 ± 0.07   Final old_old: 97.24 ± 0.08   Final old_new: 94.31 ± 0.16   Final new_new: 90.32 ± 1.47
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.09 ± 3.53   Final Test: 29.83 ± 2.49   Final old_old: 33.07 ± 2.82   Final old_new: 16.60 ± 1.36   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 59.92 ± 1.95   Final Test: 39.36 ± 2.41   Final old_old: 42.86 ± 2.38   Final old_new: 25.31 ± 2.75   Final new_new: 7.27 ± 2.96
Hits@30:
  Final val: 65.80 ± 0.93   Final Test: 49.23 ± 4.00   Final old_old: 52.88 ± 4.13   Final old_new: 34.79 ± 3.96   Final new_new: 11.82 ± 1.90
Hits@50:
  Final val: 71.95 ± 0.29   Final Test: 59.95 ± 3.62   Final old_old: 63.51 ± 3.76   Final old_new: 46.20 ± 3.65   Final new_new: 17.73 ± 5.88
AUC:
  Final val: 94.87 ± 0.30   Final Test: 94.57 ± 0.25   Final old_old: 95.21 ± 0.31   Final old_new: 92.08 ± 0.17   Final new_new: 87.61 ± 0.69
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.03 ± 3.37   Final Test: 30.40 ± 4.45   Final old_old: 33.64 ± 4.90   Final old_new: 17.23 ± 2.78   Final new_new: 3.64 ± 2.03
Hits@20:
  Final val: 59.07 ± 2.26   Final Test: 39.99 ± 2.39   Final old_old: 43.47 ± 2.56   Final old_new: 26.10 ± 1.86   Final new_new: 6.36 ± 3.37
Hits@30:
  Final val: 66.78 ± 2.33   Final Test: 47.93 ± 2.20   Final old_old: 51.71 ± 2.52   Final old_new: 33.02 ± 1.55   Final new_new: 8.18 ± 2.59
Hits@50:
  Final val: 74.32 ± 2.23   Final Test: 58.77 ± 1.60   Final old_old: 62.92 ± 1.76   Final old_new: 42.57 ± 1.86   Final new_new: 12.27 ± 6.55
AUC:
  Final val: 97.25 ± 0.09   Final Test: 96.70 ± 0.05   Final old_old: 97.29 ± 0.10   Final old_new: 94.42 ± 0.22   Final new_new: 90.12 ± 0.84
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.51 ± 2.55   Final Test: 37.12 ± 4.54   Final old_old: 40.61 ± 5.09   Final old_new: 23.20 ± 2.48   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 62.95 ± 1.14   Final Test: 45.11 ± 4.22   Final old_old: 48.93 ± 4.80   Final old_new: 30.05 ± 2.32   Final new_new: 4.55 ± 2.27
Hits@30:
  Final val: 69.35 ± 1.65   Final Test: 53.30 ± 4.60   Final old_old: 57.43 ± 4.90   Final old_new: 37.03 ± 3.80   Final new_new: 9.55 ± 3.73
Hits@50:
  Final val: 76.23 ± 1.30   Final Test: 61.40 ± 4.07   Final old_old: 65.64 ± 4.41   Final old_new: 44.86 ± 3.13   Final new_new: 13.18 ± 5.18
AUC:
  Final val: 97.37 ± 0.13   Final Test: 96.80 ± 0.18   Final old_old: 97.41 ± 0.23   Final old_new: 94.38 ± 0.15   Final new_new: 90.09 ± 1.05
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.37 ± 3.39   Final Test: 25.41 ± 6.83   Final old_old: 27.70 ± 7.45   Final old_new: 16.12 ± 4.44   Final new_new: 5.91 ± 2.03
Hits@20:
  Final val: 54.28 ± 4.18   Final Test: 38.10 ± 4.85   Final old_old: 41.15 ± 5.33   Final old_new: 25.77 ± 3.16   Final new_new: 10.91 ± 4.07
Hits@30:
  Final val: 61.61 ± 3.14   Final Test: 44.72 ± 2.13   Final old_old: 48.05 ± 2.42   Final old_new: 31.44 ± 1.12   Final new_new: 11.82 ± 1.90
Hits@50:
  Final val: 71.61 ± 3.00   Final Test: 54.58 ± 4.95   Final old_old: 58.26 ± 5.33   Final old_new: 40.03 ± 3.66   Final new_new: 16.36 ± 4.37
AUC:
  Final val: 96.61 ± 0.18   Final Test: 95.97 ± 0.22   Final old_old: 96.66 ± 0.29   Final old_new: 93.23 ± 0.17   Final new_new: 89.61 ± 1.49
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.95 ± 2.29   Final Test: 30.13 ± 3.26   Final old_old: 33.28 ± 3.45   Final old_new: 17.23 ± 2.66   Final new_new: 5.91 ± 2.03
Hits@20:
  Final val: 57.60 ± 1.51   Final Test: 40.91 ± 5.96   Final old_old: 44.48 ± 6.11   Final old_new: 26.52 ± 5.64   Final new_new: 9.09 ± 3.21
Hits@30:
  Final val: 63.29 ± 0.99   Final Test: 49.67 ± 2.34   Final old_old: 53.33 ± 2.49   Final old_new: 35.11 ± 2.14   Final new_new: 13.64 ± 1.61
Hits@50:
  Final val: 71.03 ± 0.87   Final Test: 56.65 ± 1.07   Final old_old: 60.38 ± 1.08   Final old_new: 41.91 ± 1.99   Final new_new: 17.73 ± 2.96
AUC:
  Final val: 94.76 ± 0.22   Final Test: 94.36 ± 0.34   Final old_old: 95.07 ± 0.24   Final old_new: 91.65 ± 0.76   Final new_new: 85.28 ± 2.93
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 38.90 ± 6.09   Final Test: 20.42 ± 3.57   Final old_old: 22.44 ± 3.93   Final old_new: 12.27 ± 2.21   Final new_new: 2.73 ± 2.49
Hits@20:
  Final val: 50.71 ± 4.27   Final Test: 34.63 ± 6.08   Final old_old: 37.46 ± 6.47   Final old_new: 23.27 ± 4.61   Final new_new: 9.09 ± 4.82
Hits@30:
  Final val: 58.88 ± 3.81   Final Test: 40.46 ± 1.91   Final old_old: 43.66 ± 2.20   Final old_new: 27.66 ± 0.83   Final new_new: 10.00 ± 3.05
Hits@50:
  Final val: 68.99 ± 3.10   Final Test: 52.10 ± 3.48   Final old_old: 55.57 ± 3.55   Final old_new: 38.46 ± 3.21   Final new_new: 14.55 ± 7.64
AUC:
  Final val: 96.53 ± 0.14   Final Test: 95.92 ± 0.13   Final old_old: 96.55 ± 0.13   Final old_new: 93.40 ± 0.27   Final new_new: 89.26 ± 2.06
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.72 ± 2.84   Final Test: 29.53 ± 1.43   Final old_old: 32.64 ± 1.71   Final old_new: 16.90 ± 0.78   Final new_new: 3.64 ± 2.03
Hits@20:
  Final val: 60.39 ± 2.01   Final Test: 39.14 ± 2.95   Final old_old: 42.64 ± 3.16   Final old_new: 25.04 ± 2.35   Final new_new: 7.27 ± 3.73
Hits@30:
  Final val: 66.49 ± 2.03   Final Test: 47.74 ± 1.78   Final old_old: 51.65 ± 1.84   Final old_new: 32.09 ± 1.80   Final new_new: 10.45 ± 3.80
Hits@50:
  Final val: 74.18 ± 1.64   Final Test: 57.59 ± 1.73   Final old_old: 61.75 ± 1.71   Final old_new: 41.06 ± 2.46   Final new_new: 15.45 ± 4.07
AUC:
  Final val: 97.29 ± 0.09   Final Test: 96.71 ± 0.10   Final old_old: 97.30 ± 0.16   Final old_new: 94.37 ± 0.25   Final new_new: 90.61 ± 0.96
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.06 ± 3.89   Final Test: 31.68 ± 5.30   Final old_old: 34.99 ± 5.83   Final old_new: 18.26 ± 3.44   Final new_new: 3.64 ± 2.59
Hits@20:
  Final val: 63.31 ± 3.30   Final Test: 43.12 ± 4.54   Final old_old: 47.15 ± 4.66   Final old_new: 26.93 ± 4.41   Final new_new: 6.36 ± 1.90
Hits@30:
  Final val: 69.20 ± 2.08   Final Test: 52.87 ± 3.25   Final old_old: 57.33 ± 3.49   Final old_new: 35.06 ± 3.01   Final new_new: 10.45 ± 2.59
Hits@50:
  Final val: 76.24 ± 1.94   Final Test: 62.39 ± 2.26   Final old_old: 66.79 ± 2.48   Final old_new: 45.01 ± 2.10   Final new_new: 16.36 ± 4.66
AUC:
  Final val: 97.36 ± 0.19   Final Test: 96.80 ± 0.15   Final old_old: 97.43 ± 0.16   Final old_new: 94.35 ± 0.23   Final new_new: 89.87 ± 0.68
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.06 ± 4.11   Final Test: 33.64 ± 7.25   Final old_old: 36.42 ± 7.83   Final old_new: 22.47 ± 4.90   Final new_new: 9.09 ± 4.55
Hits@20:
  Final val: 62.28 ± 2.49   Final Test: 41.44 ± 6.15   Final old_old: 44.72 ± 6.61   Final old_new: 28.29 ± 4.25   Final new_new: 11.36 ± 5.57
Hits@30:
  Final val: 68.79 ± 1.27   Final Test: 48.58 ± 5.88   Final old_old: 52.17 ± 6.16   Final old_new: 34.26 ± 4.99   Final new_new: 14.55 ± 6.14
Hits@50:
  Final val: 75.39 ± 1.08   Final Test: 57.92 ± 2.54   Final old_old: 61.80 ± 2.60   Final old_new: 42.44 ± 3.02   Final new_new: 20.00 ± 4.37
AUC:
  Final val: 96.76 ± 0.10   Final Test: 96.24 ± 0.14   Final old_old: 96.85 ± 0.15   Final old_new: 93.78 ± 0.27   Final new_new: 90.52 ± 0.98
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.57 ± 5.96   Final Test: 32.49 ± 3.83   Final old_old: 35.67 ± 4.01   Final old_new: 19.72 ± 3.62   Final new_new: 3.64 ± 3.45
Hits@20:
  Final val: 63.75 ± 3.57   Final Test: 44.02 ± 3.39   Final old_old: 47.88 ± 3.59   Final old_new: 28.66 ± 3.21   Final new_new: 5.91 ± 2.59
Hits@30:
  Final val: 70.67 ± 2.84   Final Test: 50.93 ± 1.69   Final old_old: 55.10 ± 1.68   Final old_new: 34.38 ± 2.15   Final new_new: 8.64 ± 2.96
Hits@50:
  Final val: 77.43 ± 1.71   Final Test: 60.33 ± 2.62   Final old_old: 64.68 ± 2.86   Final old_new: 43.32 ± 1.89   Final new_new: 12.27 ± 3.80
AUC:
  Final val: 97.37 ± 0.16   Final Test: 96.81 ± 0.07   Final old_old: 97.47 ± 0.05   Final old_new: 94.15 ± 0.24   Final new_new: 90.64 ± 0.99
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.55 ± 4.09   Final Test: 29.64 ± 5.79   Final old_old: 32.30 ± 6.29   Final old_new: 18.89 ± 3.81   Final new_new: 5.91 ± 3.05
Hits@20:
  Final val: 55.15 ± 2.84   Final Test: 38.90 ± 7.00   Final old_old: 41.99 ± 7.46   Final old_new: 26.60 ± 5.24   Final new_new: 9.09 ± 2.78
Hits@30:
  Final val: 62.07 ± 2.73   Final Test: 45.64 ± 3.64   Final old_old: 49.03 ± 3.87   Final old_new: 32.17 ± 2.96   Final new_new: 11.82 ± 4.37
Hits@50:
  Final val: 71.91 ± 1.74   Final Test: 55.40 ± 2.05   Final old_old: 59.14 ± 1.95   Final old_new: 40.65 ± 2.93   Final new_new: 15.91 ± 4.55
AUC:
  Final val: 96.62 ± 0.16   Final Test: 95.85 ± 0.19   Final old_old: 96.54 ± 0.18   Final old_new: 93.13 ± 0.38   Final new_new: 89.12 ± 1.60
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.67 ± 1.56   Final Test: 32.35 ± 6.27   Final old_old: 35.68 ± 6.70   Final old_new: 18.82 ± 4.64   Final new_new: 4.55 ± 2.27
Hits@20:
  Final val: 60.61 ± 1.49   Final Test: 40.16 ± 3.87   Final old_old: 43.89 ± 4.09   Final old_new: 25.24 ± 3.42   Final new_new: 5.00 ± 2.49
Hits@30:
  Final val: 66.29 ± 1.14   Final Test: 46.97 ± 4.03   Final old_old: 50.91 ± 4.19   Final old_new: 31.23 ± 3.42   Final new_new: 9.55 ± 2.96
Hits@50:
  Final val: 73.99 ± 0.61   Final Test: 57.90 ± 2.32   Final old_old: 62.02 ± 2.05   Final old_new: 41.76 ± 3.56   Final new_new: 12.27 ± 4.71
AUC:
  Final val: 97.20 ± 0.09   Final Test: 96.71 ± 0.14   Final old_old: 97.32 ± 0.10   Final old_new: 94.30 ± 0.35   Final new_new: 90.06 ± 1.52
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.98 ± 5.44   Final Test: 33.41 ± 7.77   Final old_old: 36.44 ± 8.31   Final old_new: 21.44 ± 5.90   Final new_new: 2.73 ± 1.90
Hits@20:
  Final val: 63.85 ± 1.64   Final Test: 41.59 ± 3.10   Final old_old: 44.97 ± 3.20   Final old_new: 28.31 ± 3.49   Final new_new: 5.91 ± 1.24
Hits@30:
  Final val: 70.65 ± 0.86   Final Test: 51.20 ± 2.94   Final old_old: 54.88 ± 2.95   Final old_new: 36.88 ± 2.86   Final new_new: 8.64 ± 4.07
Hits@50:
  Final val: 77.36 ± 0.43   Final Test: 61.28 ± 3.38   Final old_old: 65.03 ± 3.43   Final old_new: 46.83 ± 3.21   Final new_new: 16.36 ± 6.31
AUC:
  Final val: 97.30 ± 0.08   Final Test: 96.79 ± 0.09   Final old_old: 97.26 ± 0.08   Final old_new: 94.94 ± 0.30   Final new_new: 91.42 ± 0.95
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.06 ± 1.75   Final Test: 31.38 ± 3.60   Final old_old: 34.54 ± 3.87   Final old_new: 18.59 ± 2.44   Final new_new: 3.64 ± 3.80
Hits@20:
  Final val: 63.10 ± 1.90   Final Test: 41.09 ± 4.47   Final old_old: 44.74 ± 4.84   Final old_new: 26.55 ± 3.56   Final new_new: 5.45 ± 2.59
Hits@30:
  Final val: 69.23 ± 1.55   Final Test: 49.39 ± 4.68   Final old_old: 53.43 ± 5.00   Final old_new: 33.38 ± 3.68   Final new_new: 8.64 ± 4.07
Hits@50:
  Final val: 76.26 ± 1.33   Final Test: 60.39 ± 3.39   Final old_old: 64.53 ± 3.76   Final old_new: 44.16 ± 2.55   Final new_new: 15.45 ± 5.43
AUC:
  Final val: 97.36 ± 0.10   Final Test: 96.79 ± 0.12   Final old_old: 97.42 ± 0.14   Final old_new: 94.31 ± 0.16   Final new_new: 90.03 ± 1.03
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.45 ± 3.24   Final Test: 32.20 ± 1.83   Final old_old: 35.41 ± 2.00   Final old_new: 19.14 ± 1.98   Final new_new: 5.91 ± 1.24
Hits@20:
  Final val: 59.99 ± 3.01   Final Test: 41.33 ± 1.90   Final old_old: 44.99 ± 2.18   Final old_new: 26.73 ± 1.23   Final new_new: 6.36 ± 1.90
Hits@30:
  Final val: 67.22 ± 2.51   Final Test: 51.55 ± 3.43   Final old_old: 55.60 ± 3.53   Final old_new: 35.57 ± 3.23   Final new_new: 9.09 ± 1.61
Hits@50:
  Final val: 75.34 ± 2.30   Final Test: 58.95 ± 3.58   Final old_old: 63.29 ± 3.58   Final old_new: 41.81 ± 3.79   Final new_new: 14.09 ± 4.93
AUC:
  Final val: 97.30 ± 0.15   Final Test: 96.76 ± 0.11   Final old_old: 97.38 ± 0.12   Final old_new: 94.30 ± 0.17   Final new_new: 90.36 ± 0.46
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.96 ± 5.44   Final Test: 25.83 ± 2.40   Final old_old: 28.19 ± 2.54   Final old_new: 16.20 ± 2.02   Final new_new: 6.82 ± 5.79
Hits@20:
  Final val: 52.69 ± 4.82   Final Test: 35.60 ± 7.70   Final old_old: 38.45 ± 8.40   Final old_new: 24.13 ± 5.13   Final new_new: 10.45 ± 4.13
Hits@30:
  Final val: 59.81 ± 3.61   Final Test: 43.24 ± 3.03   Final old_old: 46.42 ± 3.51   Final old_new: 30.53 ± 1.56   Final new_new: 12.27 ± 6.35
Hits@50:
  Final val: 68.25 ± 2.74   Final Test: 53.13 ± 4.37   Final old_old: 56.82 ± 4.66   Final old_new: 38.49 ± 4.23   Final new_new: 16.36 ± 10.09
AUC:
  Final val: 96.39 ± 0.26   Final Test: 95.70 ± 0.30   Final old_old: 96.40 ± 0.33   Final old_new: 92.84 ± 0.55   Final new_new: 90.01 ± 0.60
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.27 ± 3.80   Final Test: 33.88 ± 2.58   Final old_old: 36.98 ± 2.94   Final old_new: 21.56 ± 1.38   Final new_new: 3.64 ± 2.03
Hits@20:
  Final val: 63.07 ± 2.23   Final Test: 43.00 ± 3.12   Final old_old: 46.34 ± 3.37   Final old_new: 29.90 ± 2.52   Final new_new: 6.82 ± 4.82
Hits@30:
  Final val: 69.50 ± 1.95   Final Test: 49.01 ± 3.94   Final old_old: 52.77 ± 4.20   Final old_new: 34.16 ± 2.99   Final new_new: 10.45 ± 2.59
Hits@50:
  Final val: 76.17 ± 1.25   Final Test: 59.89 ± 1.42   Final old_old: 63.75 ± 1.51   Final old_new: 44.89 ± 1.49   Final new_new: 15.45 ± 5.88
AUC:
  Final val: 97.39 ± 0.08   Final Test: 96.99 ± 0.06   Final old_old: 97.48 ± 0.06   Final old_new: 95.10 ± 0.15   Final new_new: 91.36 ± 0.47
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.81 ± 3.32   Final Test: 33.50 ± 5.95   Final old_old: 36.72 ± 6.27   Final old_new: 20.50 ± 5.16   Final new_new: 5.00 ± 2.96
Hits@20:
  Final val: 59.48 ± 0.98   Final Test: 41.82 ± 3.49   Final old_old: 45.54 ± 3.87   Final old_new: 26.90 ± 2.39   Final new_new: 7.27 ± 2.96
Hits@30:
  Final val: 66.40 ± 1.08   Final Test: 49.32 ± 3.81   Final old_old: 53.06 ± 3.97   Final old_new: 34.48 ± 3.47   Final new_new: 10.91 ± 2.96
Hits@50:
  Final val: 74.36 ± 0.65   Final Test: 58.52 ± 1.66   Final old_old: 62.67 ± 1.88   Final old_new: 42.17 ± 1.10   Final new_new: 14.55 ± 3.80
AUC:
  Final val: 97.21 ± 0.08   Final Test: 96.65 ± 0.06   Final old_old: 97.27 ± 0.05   Final old_new: 94.24 ± 0.34   Final new_new: 90.00 ± 1.67
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.57 ± 5.63   Final Test: 22.50 ± 7.66   Final old_old: 24.62 ± 8.29   Final old_new: 13.95 ± 5.29   Final new_new: 3.64 ± 2.03
Hits@20:
  Final val: 55.05 ± 3.67   Final Test: 36.65 ± 6.05   Final old_old: 39.56 ± 6.21   Final old_new: 25.01 ± 5.74   Final new_new: 8.64 ± 3.37
Hits@30:
  Final val: 62.04 ± 2.95   Final Test: 45.52 ± 3.75   Final old_old: 49.02 ± 4.04   Final old_new: 31.59 ± 2.82   Final new_new: 11.82 ± 6.10
Hits@50:
  Final val: 71.22 ± 1.53   Final Test: 54.03 ± 3.02   Final old_old: 57.59 ± 3.24   Final old_new: 40.03 ± 2.61   Final new_new: 16.36 ± 8.10
AUC:
  Final val: 96.57 ± 0.29   Final Test: 95.98 ± 0.26   Final old_old: 96.62 ± 0.24   Final old_new: 93.41 ± 0.54   Final new_new: 89.93 ± 1.26
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.28 ± 2.09   Final Test: 32.38 ± 4.35   Final old_old: 35.52 ± 4.60   Final old_new: 19.72 ± 3.50   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 61.32 ± 2.19   Final Test: 44.43 ± 4.11   Final old_old: 48.26 ± 4.46   Final old_new: 29.19 ± 2.90   Final new_new: 6.82 ± 0.00
Hits@30:
  Final val: 68.34 ± 1.56   Final Test: 51.12 ± 2.21   Final old_old: 55.30 ± 2.11   Final old_new: 34.48 ± 2.74   Final new_new: 10.00 ± 4.13
Hits@50:
  Final val: 75.62 ± 1.58   Final Test: 60.67 ± 2.72   Final old_old: 64.97 ± 2.83   Final old_new: 43.73 ± 2.72   Final new_new: 15.91 ± 3.59
AUC:
  Final val: 97.22 ± 0.24   Final Test: 96.67 ± 0.17   Final old_old: 97.39 ± 0.16   Final old_new: 93.78 ± 0.28   Final new_new: 89.81 ± 1.26
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.83 ± 6.98   Final Test: 33.51 ± 6.00   Final old_old: 36.30 ± 6.43   Final old_new: 22.12 ± 4.55   Final new_new: 11.36 ± 3.21
Hits@20:
  Final val: 57.44 ± 5.49   Final Test: 40.24 ± 8.19   Final old_old: 43.52 ± 8.75   Final old_new: 27.03 ± 6.13   Final new_new: 11.36 ± 1.61
Hits@30:
  Final val: 65.25 ± 3.40   Final Test: 49.59 ± 6.00   Final old_old: 53.06 ± 6.37   Final old_new: 35.77 ± 4.74   Final new_new: 15.45 ± 1.90
Hits@50:
  Final val: 74.03 ± 2.16   Final Test: 58.53 ± 4.39   Final old_old: 62.31 ± 4.52   Final old_new: 43.50 ± 4.08   Final new_new: 21.36 ± 3.45
AUC:
  Final val: 96.87 ± 0.21   Final Test: 96.32 ± 0.26   Final old_old: 96.91 ± 0.24   Final old_new: 93.96 ± 0.54   Final new_new: 90.98 ± 0.68
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.01 ± 1.87   Final Test: 29.82 ± 3.65   Final old_old: 32.55 ± 3.82   Final old_new: 18.77 ± 3.06   Final new_new: 6.82 ± 1.61
Hits@20:
  Final val: 58.01 ± 2.53   Final Test: 39.65 ± 2.26   Final old_old: 42.82 ± 2.41   Final old_new: 27.00 ± 1.78   Final new_new: 9.09 ± 3.21
Hits@30:
  Final val: 64.39 ± 1.19   Final Test: 47.22 ± 1.57   Final old_old: 50.87 ± 1.51   Final old_new: 32.72 ± 1.87   Final new_new: 10.91 ± 3.37
Hits@50:
  Final val: 72.81 ± 1.07   Final Test: 54.48 ± 1.61   Final old_old: 58.21 ± 1.84   Final old_new: 39.70 ± 1.16   Final new_new: 16.82 ± 2.59
AUC:
  Final val: 96.65 ± 0.17   Final Test: 95.97 ± 0.22   Final old_old: 96.64 ± 0.21   Final old_new: 93.29 ± 0.36   Final new_new: 90.00 ± 1.67
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.93 ± 2.27   Final Test: 32.23 ± 2.44   Final old_old: 35.51 ± 2.65   Final old_new: 18.92 ± 1.78   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 58.98 ± 1.39   Final Test: 40.93 ± 4.52   Final old_old: 44.57 ± 4.75   Final old_new: 26.40 ± 3.66   Final new_new: 5.91 ± 3.45
Hits@30:
  Final val: 65.91 ± 1.87   Final Test: 48.11 ± 3.86   Final old_old: 52.22 ± 3.98   Final old_new: 31.61 ± 3.62   Final new_new: 9.55 ± 2.96
Hits@50:
  Final val: 73.44 ± 1.32   Final Test: 56.93 ± 2.99   Final old_old: 61.13 ± 3.09   Final old_new: 40.30 ± 2.72   Final new_new: 13.64 ± 2.78
AUC:
  Final val: 97.16 ± 0.18   Final Test: 96.52 ± 0.03   Final old_old: 97.22 ± 0.07   Final old_new: 93.83 ± 0.20   Final new_new: 88.38 ± 1.27
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.46 ± 4.31   Final Test: 33.50 ± 3.31   Final old_old: 36.71 ± 3.53   Final old_new: 20.55 ± 3.18   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 63.40 ± 1.65   Final Test: 46.03 ± 3.54   Final old_old: 49.99 ± 3.70   Final old_new: 30.38 ± 3.30   Final new_new: 5.45 ± 1.24
Hits@30:
  Final val: 70.08 ± 1.37   Final Test: 54.09 ± 1.69   Final old_old: 58.16 ± 1.84   Final old_new: 38.11 ± 2.08   Final new_new: 10.00 ± 2.59
Hits@50:
  Final val: 76.66 ± 1.41   Final Test: 62.42 ± 3.38   Final old_old: 66.71 ± 3.64   Final old_new: 45.72 ± 3.11   Final new_new: 13.18 ± 1.90
AUC:
  Final val: 97.35 ± 0.19   Final Test: 96.82 ± 0.11   Final old_old: 97.48 ± 0.16   Final old_new: 94.19 ± 0.14   Final new_new: 89.94 ± 0.71
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.62 ± 3.48   Final Test: 34.06 ± 2.31   Final old_old: 37.36 ± 2.53   Final old_new: 20.81 ± 1.89   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 59.55 ± 2.81   Final Test: 40.71 ± 3.97   Final old_old: 44.20 ± 4.29   Final old_new: 26.83 ± 2.97   Final new_new: 6.36 ± 4.37
Hits@30:
  Final val: 67.18 ± 1.82   Final Test: 48.93 ± 5.13   Final old_old: 52.86 ± 5.32   Final old_new: 33.35 ± 4.78   Final new_new: 8.64 ± 3.37
Hits@50:
  Final val: 75.29 ± 0.66   Final Test: 59.56 ± 4.01   Final old_old: 63.60 ± 4.25   Final old_new: 43.55 ± 3.27   Final new_new: 18.18 ± 5.08
AUC:
  Final val: 97.24 ± 0.17   Final Test: 96.73 ± 0.08   Final old_old: 97.34 ± 0.13   Final old_new: 94.31 ± 0.24   Final new_new: 90.44 ± 0.63
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.1, LLP_R=10, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 57.28 ± 4.76   Final Test: 33.85 ± 4.83   Final old_old: 37.08 ± 5.15   Final old_new: 20.73 ± 3.95   Final new_new: 6.36 ± 1.90
Hits@20:
  Final val: 67.42 ± 2.47   Final Test: 47.63 ± 4.10   Final old_old: 51.33 ± 4.31   Final old_new: 33.05 ± 3.60   Final new_new: 8.64 ± 1.90
Hits@30:
  Final val: 73.26 ± 2.19   Final Test: 55.75 ± 3.59   Final old_old: 60.03 ± 3.77   Final old_new: 38.74 ± 3.04   Final new_new: 13.18 ± 3.73
Hits@50:
  Final val: 79.27 ± 1.36   Final Test: 66.36 ± 1.20   Final old_old: 70.36 ± 1.26   Final old_new: 50.63 ± 1.06   Final new_new: 23.18 ± 4.37
AUC:
  Final val: 97.67 ± 0.09   Final Test: 97.23 ± 0.16   Final old_old: 97.76 ± 0.15   Final old_new: 95.17 ± 0.23   Final new_new: 90.69 ± 0.83
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.50 ± 5.85   Final Test: 25.96 ± 3.57   Final old_old: 28.32 ± 3.85   Final old_new: 16.37 ± 2.55   Final new_new: 6.36 ± 4.66
Hits@20:
  Final val: 53.12 ± 4.92   Final Test: 33.05 ± 4.61   Final old_old: 35.83 ± 4.92   Final old_new: 21.91 ± 3.62   Final new_new: 7.73 ± 6.14
Hits@30:
  Final val: 60.22 ± 4.33   Final Test: 39.99 ± 2.21   Final old_old: 42.92 ± 2.55   Final old_new: 28.24 ± 1.13   Final new_new: 12.73 ± 5.47
Hits@50:
  Final val: 68.84 ± 3.01   Final Test: 52.93 ± 4.98   Final old_old: 56.32 ± 5.27   Final old_new: 39.50 ± 3.93   Final new_new: 18.64 ± 5.18
AUC:
  Final val: 96.40 ± 0.27   Final Test: 95.64 ± 0.33   Final old_old: 96.39 ± 0.33   Final old_new: 92.60 ± 0.56   Final new_new: 89.16 ± 0.90
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.75 ± 4.81   Final Test: 34.64 ± 2.87   Final old_old: 37.86 ± 3.29   Final old_new: 21.64 ± 1.41   Final new_new: 6.36 ± 4.66
Hits@20:
  Final val: 60.35 ± 2.90   Final Test: 43.35 ± 2.38   Final old_old: 46.95 ± 2.33   Final old_new: 28.94 ± 2.92   Final new_new: 10.00 ± 4.43
Hits@30:
  Final val: 67.29 ± 2.12   Final Test: 51.24 ± 0.92   Final old_old: 55.30 ± 1.04   Final old_new: 35.04 ± 1.65   Final new_new: 12.27 ± 5.23
Hits@50:
  Final val: 74.83 ± 1.40   Final Test: 60.40 ± 3.04   Final old_old: 64.58 ± 2.84   Final old_new: 43.83 ± 4.41   Final new_new: 17.73 ± 6.70
AUC:
  Final val: 97.29 ± 0.15   Final Test: 96.75 ± 0.13   Final old_old: 97.37 ± 0.14   Final old_new: 94.30 ± 0.42   Final new_new: 90.18 ± 1.82
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.10 ± 2.62   Final Test: 29.60 ± 5.36   Final old_old: 32.76 ± 5.65   Final old_new: 16.75 ± 4.41   Final new_new: 3.18 ± 2.03
Hits@20:
  Final val: 61.41 ± 2.78   Final Test: 43.45 ± 2.12   Final old_old: 47.38 ± 2.22   Final old_new: 27.76 ± 1.78   Final new_new: 5.91 ± 2.03
Hits@30:
  Final val: 67.63 ± 1.71   Final Test: 50.14 ± 3.80   Final old_old: 54.31 ± 3.83   Final old_new: 33.55 ± 3.86   Final new_new: 9.09 ± 5.33
Hits@50:
  Final val: 75.11 ± 1.00   Final Test: 57.60 ± 2.01   Final old_old: 62.05 ± 2.45   Final old_new: 39.97 ± 1.11   Final new_new: 12.27 ± 3.45
AUC:
  Final val: 97.31 ± 0.08   Final Test: 96.80 ± 0.04   Final old_old: 97.43 ± 0.08   Final old_new: 94.31 ± 0.29   Final new_new: 89.93 ± 1.43
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.55 ± 4.03   Final Test: 31.60 ± 5.44   Final old_old: 34.77 ± 5.88   Final old_new: 18.77 ± 3.71   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 64.75 ± 1.89   Final Test: 46.57 ± 2.79   Final old_old: 50.16 ± 2.97   Final old_new: 32.42 ± 2.47   Final new_new: 8.18 ± 4.43
Hits@30:
  Final val: 71.48 ± 0.98   Final Test: 52.45 ± 1.87   Final old_old: 56.29 ± 1.94   Final old_new: 37.36 ± 1.65   Final new_new: 11.36 ± 4.82
Hits@50:
  Final val: 77.31 ± 0.58   Final Test: 64.26 ± 2.41   Final old_old: 68.24 ± 2.43   Final old_new: 48.72 ± 2.62   Final new_new: 20.00 ± 5.18
AUC:
  Final val: 97.63 ± 0.08   Final Test: 97.30 ± 0.09   Final old_old: 97.77 ± 0.12   Final old_new: 95.45 ± 0.17   Final new_new: 92.17 ± 0.55
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.58 ± 3.35   Final Test: 30.32 ± 4.61   Final old_old: 33.47 ± 4.88   Final old_new: 17.61 ± 3.66   Final new_new: 2.73 ± 1.02
Hits@20:
  Final val: 62.82 ± 2.21   Final Test: 43.78 ± 3.98   Final old_old: 47.69 ± 4.24   Final old_new: 28.21 ± 3.35   Final new_new: 5.91 ± 2.03
Hits@30:
  Final val: 68.98 ± 1.40   Final Test: 53.09 ± 3.38   Final old_old: 57.23 ± 3.68   Final old_new: 36.75 ± 2.44   Final new_new: 10.45 ± 3.45
Hits@50:
  Final val: 76.85 ± 1.36   Final Test: 61.02 ± 3.78   Final old_old: 65.43 ± 4.03   Final old_new: 43.68 ± 2.86   Final new_new: 14.55 ± 5.70
AUC:
  Final val: 97.28 ± 0.15   Final Test: 96.69 ± 0.14   Final old_old: 97.43 ± 0.20   Final old_new: 93.73 ± 0.42   Final new_new: 89.48 ± 1.13
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 39.23 ± 2.92   Final Test: 21.84 ± 9.13   Final old_old: 23.70 ± 9.85   Final old_new: 14.13 ± 6.27   Final new_new: 9.09 ± 4.25
Hits@20:
  Final val: 51.66 ± 3.39   Final Test: 33.50 ± 2.66   Final old_old: 36.32 ± 2.90   Final old_new: 22.17 ± 2.25   Final new_new: 8.18 ± 3.80
Hits@30:
  Final val: 58.99 ± 2.48   Final Test: 43.16 ± 4.36   Final old_old: 46.33 ± 4.45   Final old_new: 30.65 ± 4.20   Final new_new: 10.00 ± 4.98
Hits@50:
  Final val: 68.39 ± 2.00   Final Test: 51.85 ± 4.08   Final old_old: 55.45 ± 4.11   Final old_new: 37.58 ± 4.53   Final new_new: 15.00 ± 3.45
AUC:
  Final val: 96.46 ± 0.24   Final Test: 95.82 ± 0.24   Final old_old: 96.50 ± 0.20   Final old_new: 93.08 ± 0.53   Final new_new: 89.35 ± 1.85
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 38.33 ± 2.59   Final Test: 26.94 ± 7.52   Final old_old: 29.46 ± 8.10   Final old_new: 16.68 ± 5.27   Final new_new: 6.36 ± 2.49
Hits@20:
  Final val: 50.77 ± 4.37   Final Test: 34.26 ± 4.41   Final old_old: 36.98 ± 4.79   Final old_new: 23.35 ± 3.13   Final new_new: 8.64 ± 4.07
Hits@30:
  Final val: 57.94 ± 3.64   Final Test: 40.49 ± 4.04   Final old_old: 43.65 ± 4.43   Final old_new: 27.88 ± 2.72   Final new_new: 10.45 ± 3.80
Hits@50:
  Final val: 68.36 ± 3.55   Final Test: 52.84 ± 5.36   Final old_old: 56.42 ± 5.83   Final old_new: 38.74 ± 3.77   Final new_new: 15.45 ± 4.93
AUC:
  Final val: 96.38 ± 0.27   Final Test: 95.73 ± 0.17   Final old_old: 96.45 ± 0.21   Final old_new: 92.87 ± 0.22   Final new_new: 88.42 ± 1.35
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.24 ± 3.61   Final Test: 32.50 ± 3.43   Final old_old: 35.83 ± 3.70   Final old_new: 18.99 ± 2.53   Final new_new: 4.55 ± 0.00
Hits@20:
  Final val: 59.23 ± 2.33   Final Test: 44.13 ± 2.28   Final old_old: 47.88 ± 2.40   Final old_new: 29.27 ± 1.92   Final new_new: 6.82 ± 2.27
Hits@30:
  Final val: 65.70 ± 1.81   Final Test: 49.51 ± 2.44   Final old_old: 53.70 ± 2.56   Final old_new: 32.82 ± 2.04   Final new_new: 8.64 ± 2.96
Hits@50:
  Final val: 73.63 ± 0.97   Final Test: 58.52 ± 1.92   Final old_old: 62.89 ± 2.09   Final old_new: 41.26 ± 1.38   Final new_new: 12.73 ± 4.98
AUC:
  Final val: 97.15 ± 0.18   Final Test: 96.59 ± 0.13   Final old_old: 97.26 ± 0.18   Final old_new: 93.97 ± 0.19   Final new_new: 89.04 ± 0.84
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.21 ± 3.47   Final Test: 30.52 ± 2.99   Final old_old: 33.14 ± 3.05   Final old_new: 19.77 ± 3.19   Final new_new: 10.45 ± 3.80
Hits@20:
  Final val: 62.29 ± 2.98   Final Test: 41.87 ± 7.33   Final old_old: 44.90 ± 7.84   Final old_new: 29.62 ± 5.37   Final new_new: 15.45 ± 5.88
Hits@30:
  Final val: 69.18 ± 2.11   Final Test: 49.26 ± 5.13   Final old_old: 52.60 ± 5.54   Final old_new: 35.92 ± 3.34   Final new_new: 17.27 ± 7.47
Hits@50:
  Final val: 75.67 ± 1.57   Final Test: 59.86 ± 4.30   Final old_old: 63.47 ± 4.60   Final old_new: 45.59 ± 3.05   Final new_new: 22.27 ± 7.08
AUC:
  Final val: 96.87 ± 0.17   Final Test: 96.34 ± 0.27   Final old_old: 96.94 ± 0.25   Final old_new: 93.91 ± 0.38   Final new_new: 90.62 ± 1.57
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.80 ± 4.42   Final Test: 32.54 ± 6.69   Final old_old: 35.94 ± 7.26   Final old_new: 18.87 ± 4.48   Final new_new: 2.27 ± 1.61
Hits@20:
  Final val: 60.15 ± 3.55   Final Test: 43.42 ± 2.83   Final old_old: 47.31 ± 3.10   Final old_new: 27.91 ± 2.38   Final new_new: 5.45 ± 3.45
Hits@30:
  Final val: 67.29 ± 2.84   Final Test: 51.67 ± 3.75   Final old_old: 55.83 ± 4.12   Final old_new: 35.29 ± 2.58   Final new_new: 7.27 ± 2.96
Hits@50:
  Final val: 75.21 ± 2.48   Final Test: 59.99 ± 2.97   Final old_old: 64.29 ± 3.21   Final old_new: 43.07 ± 2.60   Final new_new: 13.64 ± 5.57
AUC:
  Final val: 97.25 ± 0.31   Final Test: 96.60 ± 0.26   Final old_old: 97.34 ± 0.30   Final old_new: 93.64 ± 0.16   Final new_new: 88.96 ± 1.34
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.94 ± 3.75   Final Test: 33.81 ± 2.25   Final old_old: 37.31 ± 2.54   Final old_new: 19.67 ± 1.67   Final new_new: 3.18 ± 3.45
Hits@20:
  Final val: 61.31 ± 1.73   Final Test: 42.75 ± 3.45   Final old_old: 46.58 ± 3.37   Final old_new: 27.46 ± 4.00   Final new_new: 5.91 ± 3.80
Hits@30:
  Final val: 68.68 ± 2.57   Final Test: 50.35 ± 2.69   Final old_old: 54.72 ± 2.71   Final old_new: 32.92 ± 2.87   Final new_new: 8.64 ± 4.07
Hits@50:
  Final val: 76.37 ± 2.28   Final Test: 59.40 ± 1.91   Final old_old: 64.05 ± 2.00   Final old_new: 41.01 ± 2.09   Final new_new: 12.27 ± 4.13
AUC:
  Final val: 97.30 ± 0.22   Final Test: 96.64 ± 0.16   Final old_old: 97.40 ± 0.15   Final old_new: 93.62 ± 0.18   Final new_new: 89.17 ± 1.86
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.90 ± 4.43   Final Test: 28.52 ± 3.82   Final old_old: 30.98 ± 4.02   Final old_new: 18.39 ± 3.13   Final new_new: 10.45 ± 7.29
Hits@20:
  Final val: 55.11 ± 4.80   Final Test: 36.89 ± 3.05   Final old_old: 39.92 ± 3.21   Final old_new: 24.61 ± 2.91   Final new_new: 10.45 ± 6.14
Hits@30:
  Final val: 61.84 ± 4.74   Final Test: 44.64 ± 4.26   Final old_old: 47.92 ± 4.47   Final old_new: 31.44 ± 3.75   Final new_new: 15.45 ± 2.96
Hits@50:
  Final val: 71.41 ± 3.16   Final Test: 56.07 ± 4.05   Final old_old: 59.76 ± 4.32   Final old_new: 41.39 ± 3.14   Final new_new: 20.00 ± 5.43
AUC:
  Final val: 96.46 ± 0.30   Final Test: 95.80 ± 0.32   Final old_old: 96.54 ± 0.35   Final old_new: 92.80 ± 0.37   Final new_new: 89.71 ± 0.38
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.47 ± 2.37   Final Test: 27.59 ± 6.22   Final old_old: 30.70 ± 6.51   Final old_new: 14.86 ± 5.32   Final new_new: 3.64 ± 2.03
Hits@20:
  Final val: 59.62 ± 2.87   Final Test: 41.92 ± 2.79   Final old_old: 45.93 ± 2.79   Final old_new: 25.72 ± 3.06   Final new_new: 6.82 ± 3.59
Hits@30:
  Final val: 66.06 ± 2.93   Final Test: 49.00 ± 2.70   Final old_old: 53.13 ± 2.82   Final old_new: 32.54 ± 2.60   Final new_new: 9.09 ± 3.59
Hits@50:
  Final val: 74.43 ± 1.62   Final Test: 58.81 ± 2.58   Final old_old: 63.35 ± 2.83   Final old_new: 40.86 ± 2.23   Final new_new: 11.82 ± 1.90
AUC:
  Final val: 97.19 ± 0.19   Final Test: 96.53 ± 0.10   Final old_old: 97.27 ± 0.12   Final old_new: 93.64 ± 0.31   Final new_new: 88.37 ± 1.57
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 55.67 ± 3.99   Final Test: 37.77 ± 5.23   Final old_old: 41.27 ± 5.80   Final old_new: 23.73 ± 3.19   Final new_new: 5.00 ± 2.96
Hits@20:
  Final val: 68.44 ± 1.97   Final Test: 47.19 ± 3.22   Final old_old: 51.22 ± 3.41   Final old_new: 31.11 ± 3.40   Final new_new: 8.18 ± 5.23
Hits@30:
  Final val: 73.72 ± 1.23   Final Test: 57.47 ± 1.03   Final old_old: 61.61 ± 0.96   Final old_new: 41.08 ± 1.81   Final new_new: 15.00 ± 8.44
Hits@50:
  Final val: 79.62 ± 0.63   Final Test: 65.09 ± 1.06   Final old_old: 69.15 ± 1.12   Final old_new: 49.09 ± 1.22   Final new_new: 22.27 ± 7.08
AUC:
  Final val: 97.65 ± 0.09   Final Test: 97.24 ± 0.09   Final old_old: 97.76 ± 0.08   Final old_new: 95.18 ± 0.19   Final new_new: 91.63 ± 1.02
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.80 ± 2.87   Final Test: 29.02 ± 7.10   Final old_old: 31.97 ± 7.69   Final old_new: 17.03 ± 4.98   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 58.50 ± 2.71   Final Test: 40.34 ± 2.66   Final old_old: 43.96 ± 3.00   Final old_new: 25.79 ± 2.06   Final new_new: 7.73 ± 5.23
Hits@30:
  Final val: 65.50 ± 1.94   Final Test: 46.66 ± 2.20   Final old_old: 50.72 ± 2.46   Final old_new: 30.33 ± 1.62   Final new_new: 10.00 ± 3.05
Hits@50:
  Final val: 73.55 ± 0.95   Final Test: 55.73 ± 1.58   Final old_old: 60.14 ± 1.87   Final old_new: 38.19 ± 1.10   Final new_new: 12.27 ± 4.71
AUC:
  Final val: 97.19 ± 0.21   Final Test: 96.59 ± 0.10   Final old_old: 97.31 ± 0.18   Final old_new: 93.74 ± 0.27   Final new_new: 89.22 ± 1.28
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.62 ± 2.32   Final Test: 31.67 ± 4.45   Final old_old: 34.76 ± 4.57   Final old_new: 19.24 ± 4.31   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 62.46 ± 2.81   Final Test: 43.90 ± 4.11   Final old_old: 47.92 ± 4.27   Final old_new: 27.81 ± 3.61   Final new_new: 6.36 ± 3.37
Hits@30:
  Final val: 68.80 ± 2.22   Final Test: 50.27 ± 1.39   Final old_old: 54.42 ± 1.29   Final old_new: 33.85 ± 1.95   Final new_new: 7.73 ± 2.59
Hits@50:
  Final val: 76.62 ± 1.18   Final Test: 61.20 ± 1.54   Final old_old: 65.67 ± 1.86   Final old_new: 43.53 ± 1.03   Final new_new: 15.45 ± 2.49
AUC:
  Final val: 97.25 ± 0.22   Final Test: 96.63 ± 0.12   Final old_old: 97.42 ± 0.15   Final old_new: 93.50 ± 0.28   Final new_new: 89.19 ± 1.30
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.57 ± 3.82   Final Test: 25.87 ± 1.89   Final old_old: 28.28 ± 2.14   Final old_new: 16.07 ± 1.08   Final new_new: 5.45 ± 2.03
Hits@20:
  Final val: 54.95 ± 4.19   Final Test: 34.96 ± 2.95   Final old_old: 37.86 ± 3.19   Final old_new: 23.32 ± 2.24   Final new_new: 8.64 ± 2.96
Hits@30:
  Final val: 61.24 ± 3.71   Final Test: 46.37 ± 3.19   Final old_old: 49.70 ± 3.36   Final old_new: 33.17 ± 3.06   Final new_new: 12.27 ± 4.71
Hits@50:
  Final val: 69.38 ± 2.82   Final Test: 54.09 ± 3.51   Final old_old: 57.50 ± 3.83   Final old_new: 40.71 ± 3.00   Final new_new: 17.27 ± 2.59
AUC:
  Final val: 96.39 ± 0.35   Final Test: 95.72 ± 0.26   Final old_old: 96.37 ± 0.29   Final old_new: 93.13 ± 0.20   Final new_new: 89.76 ± 0.67
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.20 ± 3.14   Final Test: 36.39 ± 2.41   Final old_old: 39.63 ± 2.60   Final old_new: 23.50 ± 1.86   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 63.95 ± 1.81   Final Test: 48.21 ± 3.72   Final old_old: 51.86 ± 3.77   Final old_new: 33.80 ± 3.57   Final new_new: 10.45 ± 7.97
Hits@30:
  Final val: 70.28 ± 1.20   Final Test: 55.03 ± 1.34   Final old_old: 58.88 ± 1.45   Final old_new: 40.03 ± 0.88   Final new_new: 11.82 ± 6.51
Hits@50:
  Final val: 77.20 ± 1.47   Final Test: 64.78 ± 0.84   Final old_old: 68.61 ± 0.82   Final old_new: 49.77 ± 1.08   Final new_new: 23.64 ± 4.43
AUC:
  Final val: 97.61 ± 0.10   Final Test: 97.30 ± 0.06   Final old_old: 97.75 ± 0.05   Final old_new: 95.51 ± 0.17   Final new_new: 92.84 ± 1.25
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 37.91 ± 5.91   Final Test: 21.09 ± 5.53   Final old_old: 23.07 ± 6.10   Final old_new: 12.97 ± 3.49   Final new_new: 6.36 ± 3.73
Hits@20:
  Final val: 49.38 ± 5.14   Final Test: 33.87 ± 4.06   Final old_old: 36.74 ± 4.40   Final old_new: 22.24 ± 3.18   Final new_new: 10.00 ± 4.71
Hits@30:
  Final val: 57.65 ± 4.70   Final Test: 40.19 ± 3.01   Final old_old: 43.18 ± 3.04   Final old_new: 28.11 ± 3.69   Final new_new: 13.18 ± 3.37
Hits@50:
  Final val: 67.61 ± 3.14   Final Test: 50.23 ± 3.88   Final old_old: 53.76 ± 3.97   Final old_new: 36.07 ± 3.89   Final new_new: 17.73 ± 3.73
AUC:
  Final val: 96.34 ± 0.29   Final Test: 95.67 ± 0.25   Final old_old: 96.32 ± 0.23   Final old_new: 93.03 ± 0.42   Final new_new: 90.15 ± 0.96
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.86 ± 3.09   Final Test: 32.73 ± 3.86   Final old_old: 36.11 ± 4.17   Final old_new: 18.99 ± 3.10   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 59.13 ± 1.82   Final Test: 40.37 ± 1.75   Final old_old: 44.09 ± 1.99   Final old_new: 25.39 ± 2.33   Final new_new: 6.82 ± 4.25
Hits@30:
  Final val: 65.73 ± 1.42   Final Test: 47.76 ± 3.09   Final old_old: 51.78 ± 3.02   Final old_new: 31.79 ± 3.47   Final new_new: 7.73 ± 5.93
Hits@50:
  Final val: 73.80 ± 0.42   Final Test: 57.71 ± 3.25   Final old_old: 62.01 ± 3.22   Final old_new: 40.81 ± 3.24   Final new_new: 11.82 ± 6.89
AUC:
  Final val: 97.18 ± 0.15   Final Test: 96.58 ± 0.08   Final old_old: 97.27 ± 0.13   Final old_new: 93.85 ± 0.18   Final new_new: 89.09 ± 1.02
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.18 ± 2.78   Final Test: 35.86 ± 2.89   Final old_old: 39.43 ± 2.92   Final old_new: 21.51 ± 3.27   Final new_new: 3.18 ± 2.03
Hits@20:
  Final val: 60.01 ± 2.16   Final Test: 44.59 ± 2.22   Final old_old: 48.52 ± 2.42   Final old_new: 28.87 ± 1.34   Final new_new: 7.27 ± 2.49
Hits@30:
  Final val: 68.22 ± 1.87   Final Test: 53.42 ± 2.84   Final old_old: 57.72 ± 2.90   Final old_new: 36.35 ± 2.69   Final new_new: 10.45 ± 4.13
Hits@50:
  Final val: 75.94 ± 1.28   Final Test: 61.41 ± 2.64   Final old_old: 65.88 ± 2.51   Final old_new: 43.75 ± 3.20   Final new_new: 15.00 ± 4.71
AUC:
  Final val: 97.27 ± 0.27   Final Test: 96.67 ± 0.18   Final old_old: 97.42 ± 0.21   Final old_new: 93.67 ± 0.25   Final new_new: 89.44 ± 0.94
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.43 ± 5.63   Final Test: 32.45 ± 3.38   Final old_old: 35.32 ± 3.60   Final old_new: 20.71 ± 2.66   Final new_new: 10.45 ± 2.59
Hits@20:
  Final val: 61.54 ± 4.06   Final Test: 38.96 ± 3.96   Final old_old: 42.03 ± 4.37   Final old_new: 26.45 ± 2.38   Final new_new: 14.09 ± 1.90
Hits@30:
  Final val: 68.10 ± 3.41   Final Test: 49.63 ± 3.43   Final old_old: 53.01 ± 3.81   Final old_new: 36.17 ± 2.28   Final new_new: 16.36 ± 2.96
Hits@50:
  Final val: 75.73 ± 2.09   Final Test: 60.29 ± 4.41   Final old_old: 63.94 ± 4.47   Final old_new: 45.87 ± 4.44   Final new_new: 22.73 ± 4.55
AUC:
  Final val: 96.86 ± 0.19   Final Test: 96.33 ± 0.26   Final old_old: 96.93 ± 0.25   Final old_new: 93.93 ± 0.39   Final new_new: 90.88 ± 0.76
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 40.39 ± 5.44   Final Test: 29.15 ± 2.29   Final old_old: 31.62 ± 2.67   Final old_new: 19.04 ± 1.08   Final new_new: 10.00 ± 4.13
Hits@20:
  Final val: 52.31 ± 5.13   Final Test: 36.05 ± 5.34   Final old_old: 38.69 ± 5.45   Final old_new: 25.52 ± 5.17   Final new_new: 10.91 ± 6.51
Hits@30:
  Final val: 59.88 ± 3.94   Final Test: 43.75 ± 3.98   Final old_old: 46.93 ± 4.23   Final old_new: 30.96 ± 3.32   Final new_new: 14.55 ± 4.13
Hits@50:
  Final val: 69.28 ± 2.43   Final Test: 54.35 ± 4.19   Final old_old: 57.94 ± 4.78   Final old_new: 40.00 ± 2.23   Final new_new: 20.00 ± 5.88
AUC:
  Final val: 96.39 ± 0.34   Final Test: 95.67 ± 0.24   Final old_old: 96.42 ± 0.35   Final old_new: 92.60 ± 0.55   Final new_new: 89.08 ± 0.70
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.18 ± 2.19   Final Test: 32.08 ± 4.11   Final old_old: 35.49 ± 4.27   Final old_new: 18.31 ± 3.70   Final new_new: 2.27 ± 0.00
Hits@20:
  Final val: 64.57 ± 1.88   Final Test: 44.33 ± 1.55   Final old_old: 48.59 ± 1.89   Final old_new: 27.20 ± 1.05   Final new_new: 5.45 ± 2.59
Hits@30:
  Final val: 70.46 ± 1.55   Final Test: 51.95 ± 4.39   Final old_old: 56.43 ± 4.69   Final old_new: 34.13 ± 3.74   Final new_new: 8.18 ± 4.13
Hits@50:
  Final val: 77.20 ± 1.07   Final Test: 62.27 ± 2.87   Final old_old: 66.99 ± 2.95   Final old_new: 43.53 ± 2.69   Final new_new: 14.55 ± 4.71
AUC:
  Final val: 97.37 ± 0.13   Final Test: 96.73 ± 0.12   Final old_old: 97.51 ± 0.10   Final old_new: 93.68 ± 0.28   Final new_new: 88.37 ± 1.88
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.53 ± 4.44   Final Test: 27.01 ± 3.95   Final old_old: 30.10 ± 4.22   Final old_new: 14.31 ± 3.04   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 58.52 ± 3.90   Final Test: 37.47 ± 3.56   Final old_old: 41.20 ± 3.93   Final old_new: 22.29 ± 2.15   Final new_new: 6.36 ± 2.96
Hits@30:
  Final val: 65.52 ± 3.73   Final Test: 48.28 ± 2.38   Final old_old: 52.36 ± 2.35   Final old_new: 31.99 ± 2.79   Final new_new: 9.09 ± 0.00
Hits@50:
  Final val: 73.48 ± 2.55   Final Test: 58.41 ± 2.19   Final old_old: 62.79 ± 2.48   Final old_new: 41.03 ± 1.31   Final new_new: 14.55 ± 2.59
AUC:
  Final val: 97.17 ± 0.26   Final Test: 96.55 ± 0.13   Final old_old: 97.24 ± 0.19   Final old_new: 93.85 ± 0.17   Final new_new: 89.22 ± 1.42
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.54 ± 3.10   Final Test: 31.01 ± 5.50   Final old_old: 34.18 ± 5.89   Final old_new: 18.21 ± 4.09   Final new_new: 3.18 ± 1.24
Hits@20:
  Final val: 59.26 ± 3.68   Final Test: 42.04 ± 3.19   Final old_old: 45.88 ± 3.59   Final old_new: 26.62 ± 1.86   Final new_new: 6.36 ± 3.73
Hits@30:
  Final val: 66.95 ± 2.79   Final Test: 52.88 ± 2.71   Final old_old: 57.24 ± 3.29   Final old_new: 35.57 ± 0.77   Final new_new: 9.09 ± 1.61
Hits@50:
  Final val: 75.18 ± 2.59   Final Test: 60.71 ± 4.10   Final old_old: 64.99 ± 4.27   Final old_new: 43.75 ± 3.77   Final new_new: 17.73 ± 2.96
AUC:
  Final val: 97.19 ± 0.29   Final Test: 96.60 ± 0.27   Final old_old: 97.32 ± 0.30   Final old_new: 93.73 ± 0.25   Final new_new: 90.14 ± 0.90
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 53.56 ± 3.28   Final Test: 34.57 ± 4.34   Final old_old: 37.88 ± 4.53   Final old_new: 21.13 ± 3.64   Final new_new: 6.36 ± 2.96
Hits@20:
  Final val: 65.48 ± 3.38   Final Test: 44.70 ± 2.73   Final old_old: 48.49 ± 2.76   Final old_new: 29.57 ± 3.06   Final new_new: 8.18 ± 4.71
Hits@30:
  Final val: 71.93 ± 2.95   Final Test: 53.19 ± 1.94   Final old_old: 57.19 ± 1.85   Final old_new: 37.33 ± 2.73   Final new_new: 12.27 ± 2.59
Hits@50:
  Final val: 78.60 ± 1.52   Final Test: 63.85 ± 2.76   Final old_old: 67.92 ± 2.97   Final old_new: 48.14 ± 2.24   Final new_new: 15.91 ± 1.61
AUC:
  Final val: 97.65 ± 0.07   Final Test: 97.14 ± 0.10   Final old_old: 97.65 ± 0.09   Final old_new: 95.15 ± 0.18   Final new_new: 91.71 ± 1.08
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.78 ± 4.59   Final Test: 27.69 ± 2.38   Final old_old: 30.68 ± 2.51   Final old_new: 15.49 ± 2.11   Final new_new: 4.09 ± 1.02
Hits@20:
  Final val: 59.59 ± 2.99   Final Test: 37.82 ± 2.01   Final old_old: 41.31 ± 2.11   Final old_new: 23.68 ± 2.14   Final new_new: 7.73 ± 2.59
Hits@30:
  Final val: 65.71 ± 2.75   Final Test: 45.52 ± 2.27   Final old_old: 49.53 ± 2.79   Final old_new: 29.47 ± 1.53   Final new_new: 7.27 ± 2.96
Hits@50:
  Final val: 73.75 ± 1.82   Final Test: 58.14 ± 2.73   Final old_old: 62.45 ± 2.98   Final old_new: 40.96 ± 1.84   Final new_new: 15.91 ± 3.94
AUC:
  Final val: 97.18 ± 0.17   Final Test: 96.55 ± 0.12   Final old_old: 97.27 ± 0.18   Final old_new: 93.74 ± 0.13   Final new_new: 88.69 ± 1.51
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.49 ± 3.38   Final Test: 25.54 ± 3.80   Final old_old: 27.90 ± 4.34   Final old_new: 15.97 ± 1.67   Final new_new: 5.91 ± 2.03
Hits@20:
  Final val: 54.36 ± 2.64   Final Test: 31.97 ± 3.79   Final old_old: 34.73 ± 3.92   Final old_new: 20.81 ± 3.35   Final new_new: 8.18 ± 4.71
Hits@30:
  Final val: 60.32 ± 2.33   Final Test: 43.73 ± 1.97   Final old_old: 46.96 ± 2.08   Final old_new: 30.88 ± 1.76   Final new_new: 11.82 ± 4.37
Hits@50:
  Final val: 69.15 ± 2.20   Final Test: 52.99 ± 2.17   Final old_old: 56.40 ± 2.14   Final old_new: 39.75 ± 2.60   Final new_new: 14.09 ± 4.93
AUC:
  Final val: 96.40 ± 0.32   Final Test: 95.72 ± 0.26   Final old_old: 96.43 ± 0.27   Final old_new: 92.84 ± 0.30   Final new_new: 90.04 ± 0.97
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.30 ± 2.66   Final Test: 29.00 ± 4.58   Final old_old: 31.96 ± 4.85   Final old_new: 17.05 ± 3.69   Final new_new: 2.73 ± 1.02
Hits@20:
  Final val: 59.35 ± 3.32   Final Test: 40.05 ± 1.78   Final old_old: 43.58 ± 1.72   Final old_new: 25.89 ± 2.27   Final new_new: 6.82 ± 3.21
Hits@30:
  Final val: 65.17 ± 0.97   Final Test: 47.90 ± 1.05   Final old_old: 51.94 ± 1.14   Final old_new: 31.91 ± 1.30   Final new_new: 6.82 ± 2.78
Hits@50:
  Final val: 74.08 ± 1.22   Final Test: 57.73 ± 2.02   Final old_old: 62.18 ± 2.07   Final old_new: 40.08 ± 1.87   Final new_new: 13.64 ± 2.78
AUC:
  Final val: 97.19 ± 0.19   Final Test: 96.47 ± 0.09   Final old_old: 97.19 ± 0.15   Final old_new: 93.67 ± 0.16   Final new_new: 88.40 ± 1.01
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.38 ± 4.07   Final Test: 35.35 ± 3.24   Final old_old: 38.54 ± 3.62   Final old_new: 22.62 ± 2.14   Final new_new: 4.55 ± 2.78
Hits@20:
  Final val: 64.35 ± 2.38   Final Test: 43.74 ± 4.33   Final old_old: 47.30 ± 4.70   Final old_new: 29.52 ± 2.91   Final new_new: 9.09 ± 5.79
Hits@30:
  Final val: 70.24 ± 2.09   Final Test: 50.42 ± 0.79   Final old_old: 53.97 ± 1.04   Final old_new: 36.68 ± 1.24   Final new_new: 8.64 ± 4.07
Hits@50:
  Final val: 77.40 ± 1.37   Final Test: 63.50 ± 2.33   Final old_old: 67.34 ± 2.02   Final old_new: 48.61 ± 3.87   Final new_new: 18.64 ± 4.93
AUC:
  Final val: 97.61 ± 0.08   Final Test: 97.27 ± 0.07   Final old_old: 97.74 ± 0.08   Final old_new: 95.43 ± 0.14   Final new_new: 92.03 ± 0.81
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 40.51 ± 3.14   Final Test: 27.23 ± 4.34   Final old_old: 29.73 ± 4.71   Final old_new: 17.03 ± 2.96   Final new_new: 7.27 ± 2.96
Hits@20:
  Final val: 51.36 ± 3.94   Final Test: 35.34 ± 6.14   Final old_old: 38.14 ± 6.67   Final old_new: 24.06 ± 4.53   Final new_new: 10.45 ± 8.29
Hits@30:
  Final val: 58.24 ± 3.84   Final Test: 41.29 ± 7.04   Final old_old: 44.36 ± 7.28   Final old_new: 29.12 ± 6.30   Final new_new: 10.45 ± 4.13
Hits@50:
  Final val: 68.47 ± 3.22   Final Test: 53.36 ± 3.83   Final old_old: 56.82 ± 4.14   Final old_new: 39.60 ± 2.89   Final new_new: 19.55 ± 5.23
AUC:
  Final val: 96.36 ± 0.31   Final Test: 95.75 ± 0.21   Final old_old: 96.44 ± 0.23   Final old_new: 92.95 ± 0.23   Final new_new: 90.10 ± 0.50
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.26 ± 4.69   Final Test: 31.77 ± 4.51   Final old_old: 34.90 ± 4.87   Final old_new: 19.14 ± 3.23   Final new_new: 3.64 ± 2.03
Hits@20:
  Final val: 60.31 ± 2.67   Final Test: 40.71 ± 1.58   Final old_old: 44.22 ± 1.58   Final old_new: 26.78 ± 1.59   Final new_new: 5.91 ± 3.05
Hits@30:
  Final val: 67.28 ± 1.67   Final Test: 52.35 ± 3.45   Final old_old: 56.51 ± 3.57   Final old_new: 35.89 ± 3.30   Final new_new: 9.55 ± 2.49
Hits@50:
  Final val: 75.31 ± 2.54   Final Test: 60.34 ± 3.18   Final old_old: 64.65 ± 3.46   Final old_new: 43.32 ± 2.34   Final new_new: 15.00 ± 3.45
AUC:
  Final val: 97.23 ± 0.31   Final Test: 96.61 ± 0.23   Final old_old: 97.32 ± 0.28   Final old_new: 93.80 ± 0.15   Final new_new: 89.32 ± 1.73
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 37.65 ± 4.22   Final Test: 23.05 ± 4.89   Final old_old: 25.09 ± 5.58   Final old_new: 14.66 ± 2.26   Final new_new: 7.27 ± 3.37
Hits@20:
  Final val: 50.39 ± 3.31   Final Test: 31.86 ± 3.10   Final old_old: 34.56 ± 3.21   Final old_new: 20.91 ± 3.24   Final new_new: 9.09 ± 5.08
Hits@30:
  Final val: 59.01 ± 2.53   Final Test: 39.35 ± 3.06   Final old_old: 42.25 ± 3.29   Final old_new: 27.53 ± 2.86   Final new_new: 15.00 ± 6.14
Hits@50:
  Final val: 68.82 ± 2.36   Final Test: 53.53 ± 1.68   Final old_old: 57.00 ± 1.69   Final old_new: 39.75 ± 2.01   Final new_new: 18.64 ± 5.18
AUC:
  Final val: 96.43 ± 0.27   Final Test: 95.76 ± 0.19   Final old_old: 96.47 ± 0.19   Final old_new: 92.86 ± 0.45   Final new_new: 89.89 ± 1.12
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.92 ± 3.00   Final Test: 30.12 ± 4.00   Final old_old: 33.33 ± 4.28   Final old_new: 17.13 ± 3.05   Final new_new: 2.73 ± 1.02
Hits@20:
  Final val: 59.06 ± 4.64   Final Test: 38.13 ± 2.15   Final old_old: 41.71 ± 2.19   Final old_new: 23.80 ± 2.06   Final new_new: 4.55 ± 3.94
Hits@30:
  Final val: 65.14 ± 3.37   Final Test: 46.97 ± 3.87   Final old_old: 51.02 ± 4.13   Final old_new: 30.88 ± 2.90   Final new_new: 6.82 ± 3.59
Hits@50:
  Final val: 72.94 ± 2.40   Final Test: 56.38 ± 2.74   Final old_old: 60.55 ± 3.11   Final old_new: 39.92 ± 1.67   Final new_new: 12.73 ± 3.45
AUC:
  Final val: 97.16 ± 0.17   Final Test: 96.55 ± 0.09   Final old_old: 97.25 ± 0.13   Final old_new: 93.83 ± 0.22   Final new_new: 89.22 ± 1.09
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.50 ± 2.51   Final Test: 32.22 ± 8.99   Final old_old: 34.91 ± 9.38   Final old_new: 21.44 ± 7.79   Final new_new: 7.27 ± 3.73
Hits@20:
  Final val: 61.15 ± 1.22   Final Test: 43.67 ± 5.59   Final old_old: 46.80 ± 5.92   Final old_new: 31.28 ± 4.41   Final new_new: 11.36 ± 2.78
Hits@30:
  Final val: 66.95 ± 0.98   Final Test: 47.55 ± 4.54   Final old_old: 50.98 ± 4.66   Final old_new: 34.06 ± 4.26   Final new_new: 10.91 ± 3.73
Hits@50:
  Final val: 75.53 ± 1.69   Final Test: 57.66 ± 5.11   Final old_old: 61.28 ± 5.20   Final old_new: 43.50 ± 5.03   Final new_new: 17.27 ± 4.13
AUC:
  Final val: 96.96 ± 0.22   Final Test: 96.53 ± 0.32   Final old_old: 97.11 ± 0.29   Final old_new: 94.23 ± 0.53   Final new_new: 90.54 ± 1.46
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.80 ± 3.42   Final Test: 34.00 ± 5.87   Final old_old: 37.44 ± 6.18   Final old_new: 20.05 ± 4.76   Final new_new: 4.55 ± 2.27
Hits@20:
  Final val: 60.79 ± 2.85   Final Test: 46.62 ± 3.83   Final old_old: 50.53 ± 4.13   Final old_new: 31.13 ± 2.85   Final new_new: 7.27 ± 2.96
Hits@30:
  Final val: 68.07 ± 2.04   Final Test: 53.85 ± 2.44   Final old_old: 58.13 ± 2.55   Final old_new: 37.05 ± 2.11   Final new_new: 7.73 ± 2.59
Hits@50:
  Final val: 75.37 ± 2.13   Final Test: 62.69 ± 2.54   Final old_old: 67.14 ± 2.64   Final old_new: 45.39 ± 2.31   Final new_new: 12.27 ± 4.43
AUC:
  Final val: 97.25 ± 0.26   Final Test: 96.57 ± 0.25   Final old_old: 97.32 ± 0.22   Final old_new: 93.54 ± 0.50   Final new_new: 90.20 ± 1.06
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.93 ± 4.26   Final Test: 34.21 ± 1.14   Final old_old: 37.72 ± 1.12   Final old_new: 20.10 ± 1.89   Final new_new: 2.73 ± 2.49
Hits@20:
  Final val: 63.05 ± 1.02   Final Test: 43.00 ± 2.76   Final old_old: 47.15 ± 3.00   Final old_new: 26.35 ± 2.59   Final new_new: 5.00 ± 3.73
Hits@30:
  Final val: 69.56 ± 0.81   Final Test: 51.07 ± 4.45   Final old_old: 55.39 ± 4.74   Final old_new: 33.85 ± 3.96   Final new_new: 8.64 ± 3.73
Hits@50:
  Final val: 77.40 ± 1.19   Final Test: 61.55 ± 2.93   Final old_old: 66.18 ± 3.04   Final old_new: 43.30 ± 2.95   Final new_new: 12.73 ± 5.47
AUC:
  Final val: 97.31 ± 0.22   Final Test: 96.70 ± 0.13   Final old_old: 97.50 ± 0.17   Final old_new: 93.51 ± 0.15   Final new_new: 88.85 ± 1.01
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.62 ± 3.44   Final Test: 26.55 ± 4.37   Final old_old: 29.10 ± 4.80   Final old_new: 16.17 ± 2.83   Final new_new: 6.36 ± 1.90
Hits@20:
  Final val: 54.24 ± 3.22   Final Test: 33.98 ± 4.46   Final old_old: 36.75 ± 4.53   Final old_new: 22.77 ± 4.26   Final new_new: 9.55 ± 3.37
Hits@30:
  Final val: 61.36 ± 3.28   Final Test: 43.10 ± 3.49   Final old_old: 46.10 ± 3.87   Final old_new: 31.28 ± 2.79   Final new_new: 11.36 ± 2.78
Hits@50:
  Final val: 68.66 ± 2.74   Final Test: 52.55 ± 3.94   Final old_old: 55.96 ± 4.53   Final old_new: 39.17 ± 1.95   Final new_new: 15.45 ± 4.07
AUC:
  Final val: 96.39 ± 0.32   Final Test: 95.73 ± 0.27   Final old_old: 96.42 ± 0.29   Final old_new: 92.90 ± 0.38   Final new_new: 89.71 ± 1.08
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 54.02 ± 1.84   Final Test: 35.81 ± 5.97   Final old_old: 38.98 ± 6.55   Final old_new: 23.02 ± 3.72   Final new_new: 7.73 ± 6.14
Hits@20:
  Final val: 66.05 ± 3.53   Final Test: 45.78 ± 4.67   Final old_old: 49.36 ± 4.80   Final old_new: 31.66 ± 4.62   Final new_new: 8.18 ± 3.45
Hits@30:
  Final val: 72.89 ± 2.71   Final Test: 56.83 ± 1.39   Final old_old: 60.80 ± 1.58   Final old_new: 41.39 ± 1.64   Final new_new: 11.82 ± 3.73
Hits@50:
  Final val: 78.32 ± 2.42   Final Test: 64.48 ± 1.47   Final old_old: 68.37 ± 1.52   Final old_new: 49.50 ± 2.40   Final new_new: 17.27 ± 4.13
AUC:
  Final val: 97.61 ± 0.16   Final Test: 97.19 ± 0.13   Final old_old: 97.68 ± 0.14   Final old_new: 95.27 ± 0.08   Final new_new: 91.70 ± 0.85
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.85 ± 3.21   Final Test: 29.57 ± 4.22   Final old_old: 32.49 ± 4.30   Final old_new: 17.68 ± 4.14   Final new_new: 5.45 ± 2.59
Hits@20:
  Final val: 58.31 ± 2.22   Final Test: 37.22 ± 3.91   Final old_old: 40.61 ± 4.11   Final old_new: 23.63 ± 3.26   Final new_new: 5.91 ± 2.59
Hits@30:
  Final val: 65.32 ± 2.39   Final Test: 45.94 ± 3.71   Final old_old: 49.67 ± 3.86   Final old_new: 31.13 ± 3.29   Final new_new: 8.64 ± 1.90
Hits@50:
  Final val: 72.81 ± 2.10   Final Test: 55.60 ± 1.73   Final old_old: 59.79 ± 1.95   Final old_new: 38.99 ± 1.00   Final new_new: 13.18 ± 5.43
AUC:
  Final val: 97.15 ± 0.20   Final Test: 96.50 ± 0.10   Final old_old: 97.18 ± 0.17   Final old_new: 93.83 ± 0.24   Final new_new: 88.82 ± 1.22
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.27 ± 4.05   Final Test: 29.63 ± 4.40   Final old_old: 32.77 ± 4.50   Final old_new: 16.83 ± 4.25   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 58.63 ± 2.42   Final Test: 41.36 ± 2.26   Final old_old: 45.02 ± 2.37   Final old_new: 26.68 ± 1.88   Final new_new: 7.73 ± 3.80
Hits@30:
  Final val: 66.21 ± 2.02   Final Test: 46.21 ± 2.70   Final old_old: 50.31 ± 2.71   Final old_new: 29.72 ± 2.83   Final new_new: 9.09 ± 3.21
Hits@50:
  Final val: 74.29 ± 1.77   Final Test: 57.48 ± 4.00   Final old_old: 61.96 ± 4.16   Final old_new: 39.62 ± 3.57   Final new_new: 13.18 ± 2.49
AUC:
  Final val: 97.20 ± 0.18   Final Test: 96.57 ± 0.11   Final old_old: 97.32 ± 0.19   Final old_new: 93.61 ± 0.40   Final new_new: 88.95 ± 1.40
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.41 ± 4.20   Final Test: 35.16 ± 6.15   Final old_old: 38.72 ± 6.64   Final old_new: 20.81 ± 4.44   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 61.29 ± 3.61   Final Test: 41.85 ± 2.60   Final old_old: 45.79 ± 3.08   Final old_new: 25.99 ± 1.29   Final new_new: 5.91 ± 2.03
Hits@30:
  Final val: 68.38 ± 2.90   Final Test: 51.58 ± 3.77   Final old_old: 56.04 ± 4.11   Final old_new: 33.78 ± 2.57   Final new_new: 8.64 ± 1.90
Hits@50:
  Final val: 75.67 ± 2.10   Final Test: 61.31 ± 3.43   Final old_old: 65.71 ± 3.76   Final old_new: 44.08 ± 2.51   Final new_new: 12.73 ± 3.45
AUC:
  Final val: 97.22 ± 0.22   Final Test: 96.62 ± 0.19   Final old_old: 97.33 ± 0.23   Final old_new: 93.81 ± 0.39   Final new_new: 89.69 ± 1.08
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 40.15 ± 3.59   Final Test: 23.99 ± 4.16   Final old_old: 26.26 ± 4.66   Final old_new: 14.61 ± 2.26   Final new_new: 7.27 ± 3.37
Hits@20:
  Final val: 51.99 ± 3.29   Final Test: 34.92 ± 4.92   Final old_old: 37.64 ± 5.24   Final old_new: 23.90 ± 3.85   Final new_new: 11.36 ± 2.78
Hits@30:
  Final val: 59.38 ± 2.25   Final Test: 41.96 ± 5.05   Final old_old: 45.12 ± 5.33   Final old_new: 29.12 ± 3.97   Final new_new: 16.36 ± 2.49
Hits@50:
  Final val: 69.01 ± 0.84   Final Test: 53.12 ± 3.74   Final old_old: 56.79 ± 3.96   Final old_new: 38.34 ± 3.20   Final new_new: 20.45 ± 4.82
AUC:
  Final val: 96.46 ± 0.32   Final Test: 95.66 ± 0.21   Final old_old: 96.42 ± 0.26   Final old_new: 92.52 ± 0.25   Final new_new: 89.76 ± 0.73
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.05 ± 2.75   Final Test: 28.29 ± 9.81   Final old_old: 31.17 ± 10.43   Final old_new: 16.55 ± 7.47   Final new_new: 5.00 ± 2.96
Hits@20:
  Final val: 63.95 ± 1.78   Final Test: 44.49 ± 5.02   Final old_old: 48.14 ± 5.27   Final old_new: 30.00 ± 4.10   Final new_new: 7.73 ± 3.45
Hits@30:
  Final val: 71.04 ± 1.16   Final Test: 53.78 ± 3.54   Final old_old: 57.85 ± 3.68   Final old_new: 37.61 ± 3.14   Final new_new: 13.18 ± 4.07
Hits@50:
  Final val: 77.72 ± 1.10   Final Test: 63.46 ± 6.10   Final old_old: 67.28 ± 6.13   Final old_new: 48.61 ± 6.24   Final new_new: 19.09 ± 5.23
AUC:
  Final val: 97.67 ± 0.09   Final Test: 97.30 ± 0.08   Final old_old: 97.75 ± 0.09   Final old_new: 95.57 ± 0.18   Final new_new: 91.74 ± 0.30
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 40.19 ± 4.17   Final Test: 25.32 ± 5.68   Final old_old: 27.54 ± 6.18   Final old_new: 16.22 ± 3.73   Final new_new: 7.73 ± 3.45
Hits@20:
  Final val: 51.97 ± 3.53   Final Test: 32.53 ± 6.05   Final old_old: 35.31 ± 6.50   Final old_new: 21.21 ± 4.41   Final new_new: 10.45 ± 4.43
Hits@30:
  Final val: 59.75 ± 4.78   Final Test: 42.98 ± 5.95   Final old_old: 46.13 ± 6.16   Final old_new: 30.33 ± 5.09   Final new_new: 14.55 ± 6.35
Hits@50:
  Final val: 69.12 ± 4.30   Final Test: 52.35 ± 2.83   Final old_old: 55.76 ± 3.40   Final old_new: 38.77 ± 0.69   Final new_new: 18.64 ± 2.96
AUC:
  Final val: 96.39 ± 0.36   Final Test: 95.63 ± 0.22   Final old_old: 96.37 ± 0.25   Final old_new: 92.68 ± 0.19   Final new_new: 88.54 ± 1.36
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.20 ± 2.31   Final Test: 26.01 ± 4.34   Final old_old: 28.97 ± 4.73   Final old_new: 13.90 ± 3.05   Final new_new: 3.18 ± 1.24
Hits@20:
  Final val: 57.31 ± 3.30   Final Test: 38.99 ± 2.98   Final old_old: 42.52 ± 3.41   Final old_new: 24.76 ± 1.53   Final new_new: 7.27 ± 1.90
Hits@30:
  Final val: 65.13 ± 1.64   Final Test: 47.00 ± 4.10   Final old_old: 50.94 ± 4.36   Final old_new: 31.31 ± 3.52   Final new_new: 8.18 ± 3.80
Hits@50:
  Final val: 73.32 ± 1.08   Final Test: 56.29 ± 1.60   Final old_old: 60.67 ± 1.84   Final old_new: 38.87 ± 1.18   Final new_new: 12.27 ± 1.24
AUC:
  Final val: 97.13 ± 0.11   Final Test: 96.53 ± 0.08   Final old_old: 97.25 ± 0.09   Final old_new: 93.67 ± 0.09   Final new_new: 89.11 ± 0.57
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.68 ± 4.88   Final Test: 32.73 ± 5.75   Final old_old: 35.88 ± 6.20   Final old_new: 19.97 ± 4.35   Final new_new: 5.91 ± 2.03
Hits@20:
  Final val: 61.63 ± 3.99   Final Test: 42.70 ± 3.02   Final old_old: 46.51 ± 3.21   Final old_new: 27.43 ± 2.48   Final new_new: 6.82 ± 2.78
Hits@30:
  Final val: 67.93 ± 2.87   Final Test: 52.16 ± 2.35   Final old_old: 56.45 ± 2.39   Final old_new: 34.96 ± 2.31   Final new_new: 12.27 ± 2.59
Hits@50:
  Final val: 75.54 ± 2.22   Final Test: 62.59 ± 2.96   Final old_old: 67.14 ± 3.12   Final old_new: 44.51 ± 2.39   Final new_new: 16.82 ± 3.05
AUC:
  Final val: 97.21 ± 0.31   Final Test: 96.56 ± 0.18   Final old_old: 97.31 ± 0.20   Final old_new: 93.58 ± 0.35   Final new_new: 88.85 ± 1.25
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.37 ± 5.95   Final Test: 33.74 ± 7.34   Final old_old: 36.44 ± 7.66   Final old_new: 22.92 ± 6.30   Final new_new: 9.09 ± 3.21
Hits@20:
  Final val: 59.04 ± 4.96   Final Test: 42.85 ± 4.32   Final old_old: 46.08 ± 4.70   Final old_new: 29.90 ± 3.40   Final new_new: 12.27 ± 3.45
Hits@30:
  Final val: 65.89 ± 4.22   Final Test: 46.54 ± 3.87   Final old_old: 49.69 ± 4.17   Final old_new: 34.11 ± 3.59   Final new_new: 14.55 ± 4.98
Hits@50:
  Final val: 74.63 ± 2.87   Final Test: 59.26 ± 3.67   Final old_old: 62.93 ± 3.77   Final old_new: 44.86 ± 4.00   Final new_new: 19.09 ± 4.13
AUC:
  Final val: 96.89 ± 0.29   Final Test: 96.40 ± 0.16   Final old_old: 96.98 ± 0.15   Final old_new: 94.08 ± 0.48   Final new_new: 90.65 ± 1.05
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 40.05 ± 7.54   Final Test: 26.10 ± 8.71   Final old_old: 28.27 ± 9.33   Final old_new: 17.25 ± 6.53   Final new_new: 8.18 ± 2.03
Hits@20:
  Final val: 51.18 ± 5.55   Final Test: 36.31 ± 3.58   Final old_old: 39.08 ± 3.76   Final old_new: 25.19 ± 3.15   Final new_new: 10.91 ± 2.96
Hits@30:
  Final val: 58.80 ± 4.60   Final Test: 41.22 ± 5.15   Final old_old: 44.19 ± 5.38   Final old_new: 29.40 ± 4.84   Final new_new: 12.73 ± 2.59
Hits@50:
  Final val: 68.28 ± 3.45   Final Test: 53.33 ± 1.60   Final old_old: 56.72 ± 2.08   Final old_new: 39.92 ± 2.44   Final new_new: 18.18 ± 4.25
AUC:
  Final val: 96.38 ± 0.26   Final Test: 95.74 ± 0.15   Final old_old: 96.41 ± 0.20   Final old_new: 93.05 ± 0.27   Final new_new: 89.71 ± 0.38
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.96 ± 3.00   Final Test: 31.93 ± 2.01   Final old_old: 35.16 ± 2.27   Final old_new: 18.89 ± 1.83   Final new_new: 3.18 ± 1.24
Hits@20:
  Final val: 61.61 ± 3.58   Final Test: 43.70 ± 3.49   Final old_old: 47.60 ± 4.02   Final old_new: 28.09 ± 1.96   Final new_new: 6.36 ± 4.07
Hits@30:
  Final val: 67.65 ± 2.19   Final Test: 52.02 ± 2.27   Final old_old: 56.32 ± 2.78   Final old_new: 34.99 ± 1.67   Final new_new: 8.18 ± 3.45
Hits@50:
  Final val: 74.75 ± 2.30   Final Test: 59.54 ± 3.16   Final old_old: 63.95 ± 3.45   Final old_new: 42.14 ± 2.41   Final new_new: 13.18 ± 4.37
AUC:
  Final val: 97.23 ± 0.27   Final Test: 96.64 ± 0.22   Final old_old: 97.37 ± 0.27   Final old_new: 93.69 ± 0.23   Final new_new: 89.90 ± 0.99
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.33 ± 2.38   Final Test: 30.74 ± 3.50   Final old_old: 33.83 ± 3.48   Final old_new: 18.19 ± 3.82   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 58.37 ± 1.61   Final Test: 41.45 ± 3.17   Final old_old: 45.28 ± 3.28   Final old_new: 26.12 ± 3.08   Final new_new: 5.91 ± 2.03
Hits@30:
  Final val: 65.05 ± 1.44   Final Test: 45.58 ± 5.17   Final old_old: 49.40 ± 5.48   Final old_new: 30.35 ± 4.10   Final new_new: 9.09 ± 2.78
Hits@50:
  Final val: 73.15 ± 1.93   Final Test: 57.12 ± 2.26   Final old_old: 61.36 ± 2.09   Final old_new: 40.38 ± 3.06   Final new_new: 13.64 ± 7.01
AUC:
  Final val: 97.17 ± 0.17   Final Test: 96.54 ± 0.04   Final old_old: 97.25 ± 0.11   Final old_new: 93.79 ± 0.24   Final new_new: 88.94 ± 1.16
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.40 ± 3.36   Final Test: 32.01 ± 5.52   Final old_old: 35.25 ± 6.05   Final old_new: 18.99 ± 3.47   Final new_new: 2.27 ± 1.61
Hits@20:
  Final val: 62.17 ± 3.87   Final Test: 41.28 ± 3.85   Final old_old: 45.02 ± 4.07   Final old_new: 26.32 ± 3.32   Final new_new: 5.45 ± 3.05
Hits@30:
  Final val: 69.11 ± 3.33   Final Test: 52.33 ± 3.80   Final old_old: 56.58 ± 3.94   Final old_new: 35.47 ± 3.85   Final new_new: 9.55 ± 4.93
Hits@50:
  Final val: 76.36 ± 2.50   Final Test: 61.13 ± 3.45   Final old_old: 65.65 ± 3.67   Final old_new: 43.22 ± 3.02   Final new_new: 15.00 ± 3.05
AUC:
  Final val: 97.28 ± 0.26   Final Test: 96.61 ± 0.20   Final old_old: 97.33 ± 0.27   Final old_new: 93.73 ± 0.36   Final new_new: 90.01 ± 1.01
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.00 ± 2.45   Final Test: 25.63 ± 1.70   Final old_old: 28.42 ± 1.88   Final old_new: 14.26 ± 1.81   Final new_new: 3.18 ± 1.24
Hits@20:
  Final val: 58.36 ± 1.26   Final Test: 41.31 ± 2.43   Final old_old: 44.92 ± 2.70   Final old_new: 26.75 ± 1.61   Final new_new: 9.55 ± 1.90
Hits@30:
  Final val: 66.50 ± 1.58   Final Test: 46.98 ± 0.84   Final old_old: 50.96 ± 1.13   Final old_new: 31.01 ± 0.55   Final new_new: 10.00 ± 5.70
Hits@50:
  Final val: 74.06 ± 1.58   Final Test: 53.83 ± 2.90   Final old_old: 58.22 ± 3.18   Final old_new: 36.22 ± 1.83   Final new_new: 12.73 ± 2.59
AUC:
  Final val: 97.19 ± 0.16   Final Test: 96.55 ± 0.08   Final old_old: 97.22 ± 0.14   Final old_new: 93.93 ± 0.25   Final new_new: 89.24 ± 1.36
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 55.12 ± 5.46   Final Test: 34.45 ± 3.24   Final old_old: 37.60 ± 3.56   Final old_new: 21.86 ± 1.95   Final new_new: 4.55 ± 2.27
Hits@20:
  Final val: 66.65 ± 2.32   Final Test: 46.56 ± 5.10   Final old_old: 50.24 ± 5.38   Final old_new: 31.96 ± 4.10   Final new_new: 9.09 ± 5.57
Hits@30:
  Final val: 71.91 ± 2.03   Final Test: 54.73 ± 3.85   Final old_old: 58.62 ± 4.01   Final old_new: 39.55 ± 3.53   Final new_new: 11.36 ± 6.63
Hits@50:
  Final val: 79.08 ± 1.44   Final Test: 64.13 ± 2.01   Final old_old: 68.28 ± 2.10   Final old_new: 47.83 ± 2.17   Final new_new: 18.64 ± 4.66
AUC:
  Final val: 97.61 ± 0.10   Final Test: 97.12 ± 0.13   Final old_old: 97.66 ± 0.11   Final old_new: 95.02 ± 0.24   Final new_new: 90.48 ± 1.69
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.67 ± 3.98   Final Test: 26.82 ± 3.14   Final old_old: 29.47 ± 3.49   Final old_new: 15.92 ± 1.92   Final new_new: 6.36 ± 3.73
Hits@20:
  Final val: 53.86 ± 4.39   Final Test: 37.54 ± 5.16   Final old_old: 40.66 ± 5.36   Final old_new: 24.89 ± 4.47   Final new_new: 10.91 ± 4.37
Hits@30:
  Final val: 61.37 ± 4.36   Final Test: 43.94 ± 3.49   Final old_old: 47.19 ± 3.91   Final old_new: 31.03 ± 1.81   Final new_new: 12.27 ± 5.23
Hits@50:
  Final val: 69.94 ± 3.41   Final Test: 54.85 ± 2.90   Final old_old: 58.58 ± 3.17   Final old_new: 40.20 ± 2.11   Final new_new: 15.00 ± 2.59
AUC:
  Final val: 96.34 ± 0.38   Final Test: 95.79 ± 0.30   Final old_old: 96.44 ± 0.32   Final old_new: 93.15 ± 0.26   Final new_new: 89.92 ± 0.63
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.70 ± 2.96   Final Test: 27.04 ± 4.56   Final old_old: 29.90 ± 5.07   Final old_new: 15.42 ± 2.55   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 57.29 ± 2.03   Final Test: 38.46 ± 5.15   Final old_old: 41.96 ± 5.70   Final old_new: 24.43 ± 2.97   Final new_new: 5.91 ± 3.45
Hits@30:
  Final val: 64.76 ± 1.48   Final Test: 44.97 ± 3.81   Final old_old: 48.77 ± 4.25   Final old_new: 29.87 ± 2.01   Final new_new: 7.27 ± 1.02
Hits@50:
  Final val: 72.55 ± 1.21   Final Test: 56.00 ± 4.92   Final old_old: 60.08 ± 5.19   Final old_new: 40.00 ± 3.83   Final new_new: 11.36 ± 3.59
AUC:
  Final val: 97.10 ± 0.13   Final Test: 96.52 ± 0.10   Final old_old: 97.21 ± 0.13   Final old_new: 93.81 ± 0.15   Final new_new: 89.06 ± 1.12
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.39 ± 2.97   Final Test: 35.00 ± 7.98   Final old_old: 38.25 ± 8.46   Final old_new: 21.96 ± 6.35   Final new_new: 5.45 ± 1.24
Hits@20:
  Final val: 63.15 ± 3.17   Final Test: 44.44 ± 3.49   Final old_old: 48.12 ± 3.89   Final old_new: 29.85 ± 1.87   Final new_new: 7.73 ± 3.45
Hits@30:
  Final val: 70.63 ± 1.98   Final Test: 53.00 ± 2.30   Final old_old: 56.84 ± 2.53   Final old_new: 37.93 ± 1.90   Final new_new: 10.91 ± 1.90
Hits@50:
  Final val: 77.56 ± 0.70   Final Test: 64.45 ± 2.51   Final old_old: 68.24 ± 2.71   Final old_new: 49.57 ± 1.96   Final new_new: 23.18 ± 4.37
AUC:
  Final val: 97.61 ± 0.11   Final Test: 97.25 ± 0.08   Final old_old: 97.70 ± 0.13   Final old_new: 95.51 ± 0.16   Final new_new: 91.81 ± 0.87
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.53 ± 2.38   Final Test: 28.59 ± 5.49   Final old_old: 31.21 ± 5.91   Final old_new: 17.88 ± 3.95   Final new_new: 7.73 ± 4.98
Hits@20:
  Final val: 53.98 ± 2.04   Final Test: 34.93 ± 5.20   Final old_old: 37.76 ± 5.62   Final old_new: 23.55 ± 3.87   Final new_new: 9.55 ± 3.37
Hits@30:
  Final val: 61.08 ± 2.71   Final Test: 43.96 ± 3.29   Final old_old: 47.28 ± 3.31   Final old_new: 30.65 ± 3.87   Final new_new: 13.64 ± 3.59
Hits@50:
  Final val: 70.11 ± 2.26   Final Test: 55.92 ± 1.89   Final old_old: 59.70 ± 1.88   Final old_new: 41.06 ± 2.78   Final new_new: 15.91 ± 4.55
AUC:
  Final val: 96.45 ± 0.22   Final Test: 95.76 ± 0.19   Final old_old: 96.47 ± 0.20   Final old_new: 92.81 ± 0.32   Final new_new: 90.35 ± 0.79
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.17 ± 5.55   Final Test: 30.14 ± 3.98   Final old_old: 33.32 ± 4.30   Final old_new: 17.18 ± 2.86   Final new_new: 4.55 ± 0.00
Hits@20:
  Final val: 60.30 ± 2.77   Final Test: 43.32 ± 4.28   Final old_old: 47.20 ± 4.53   Final old_new: 27.76 ± 3.51   Final new_new: 7.27 ± 1.90
Hits@30:
  Final val: 68.11 ± 2.21   Final Test: 51.50 ± 3.47   Final old_old: 55.72 ± 3.63   Final old_new: 34.74 ± 2.90   Final new_new: 9.09 ± 3.21
Hits@50:
  Final val: 75.30 ± 2.38   Final Test: 58.70 ± 4.35   Final old_old: 63.01 ± 4.45   Final old_new: 41.66 ± 4.03   Final new_new: 14.09 ± 3.73
AUC:
  Final val: 97.26 ± 0.22   Final Test: 96.58 ± 0.22   Final old_old: 97.32 ± 0.26   Final old_new: 93.58 ± 0.18   Final new_new: 89.83 ± 0.68
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.81 ± 5.74   Final Test: 28.36 ± 6.72   Final old_old: 30.96 ± 7.12   Final old_new: 17.66 ± 5.07   Final new_new: 9.09 ± 4.82
Hits@20:
  Final val: 54.80 ± 4.82   Final Test: 37.70 ± 2.38   Final old_old: 40.69 ± 2.70   Final old_new: 25.69 ± 1.52   Final new_new: 10.45 ± 1.24
Hits@30:
  Final val: 61.89 ± 3.48   Final Test: 43.96 ± 3.46   Final old_old: 47.20 ± 4.00   Final old_new: 31.01 ± 1.31   Final new_new: 13.18 ± 2.96
Hits@50:
  Final val: 69.56 ± 2.50   Final Test: 51.95 ± 5.00   Final old_old: 55.54 ± 5.42   Final old_new: 37.61 ± 3.38   Final new_new: 18.18 ± 1.61
AUC:
  Final val: 96.45 ± 0.39   Final Test: 95.78 ± 0.30   Final old_old: 96.44 ± 0.35   Final old_new: 93.07 ± 0.26   Final new_new: 90.12 ± 1.26
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.31 ± 4.22   Final Test: 30.71 ± 3.60   Final old_old: 33.92 ± 3.60   Final old_new: 17.63 ± 3.90   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 57.34 ± 3.12   Final Test: 38.72 ± 1.23   Final old_old: 42.18 ± 1.42   Final old_new: 24.86 ± 1.07   Final new_new: 6.36 ± 1.90
Hits@30:
  Final val: 64.41 ± 2.24   Final Test: 46.62 ± 2.59   Final old_old: 50.53 ± 2.90   Final old_new: 31.08 ± 1.84   Final new_new: 8.18 ± 2.03
Hits@50:
  Final val: 72.42 ± 2.07   Final Test: 56.31 ± 4.12   Final old_old: 60.42 ± 4.33   Final old_new: 40.10 ± 3.47   Final new_new: 13.64 ± 2.78
AUC:
  Final val: 97.10 ± 0.13   Final Test: 96.55 ± 0.18   Final old_old: 97.24 ± 0.23   Final old_new: 93.83 ± 0.26   Final new_new: 89.40 ± 0.82
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.27 ± 2.54   Final Test: 32.81 ± 3.47   Final old_old: 36.21 ± 3.66   Final old_new: 19.07 ± 2.77   Final new_new: 3.18 ± 3.05
Hits@20:
  Final val: 61.77 ± 2.38   Final Test: 40.91 ± 4.45   Final old_old: 44.72 ± 4.68   Final old_new: 25.64 ± 3.61   Final new_new: 5.00 ± 3.37
Hits@30:
  Final val: 68.78 ± 2.32   Final Test: 51.04 ± 3.22   Final old_old: 55.42 ± 3.42   Final old_new: 33.68 ± 2.38   Final new_new: 6.36 ± 4.93
Hits@50:
  Final val: 76.92 ± 2.06   Final Test: 59.35 ± 3.13   Final old_old: 63.83 ± 3.42   Final old_new: 41.79 ± 2.08   Final new_new: 10.91 ± 1.90
AUC:
  Final val: 97.27 ± 0.21   Final Test: 96.62 ± 0.20   Final old_old: 97.36 ± 0.21   Final old_new: 93.62 ± 0.21   Final new_new: 89.98 ± 0.82
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.72 ± 4.52   Final Test: 28.76 ± 3.09   Final old_old: 31.35 ± 3.41   Final old_new: 18.16 ± 2.27   Final new_new: 8.18 ± 1.24
Hits@20:
  Final val: 62.05 ± 3.42   Final Test: 44.53 ± 3.29   Final old_old: 47.89 ± 3.44   Final old_new: 31.18 ± 2.99   Final new_new: 11.36 ± 1.61
Hits@30:
  Final val: 67.89 ± 1.35   Final Test: 51.46 ± 1.78   Final old_old: 55.29 ± 1.82   Final old_new: 36.22 ± 1.76   Final new_new: 13.64 ± 1.61
Hits@50:
  Final val: 74.71 ± 1.17   Final Test: 60.95 ± 2.94   Final old_old: 64.80 ± 3.05   Final old_new: 45.89 ± 2.65   Final new_new: 18.18 ± 3.21
AUC:
  Final val: 96.83 ± 0.21   Final Test: 96.40 ± 0.15   Final old_old: 97.00 ± 0.19   Final old_new: 93.98 ± 0.23   Final new_new: 90.44 ± 1.00
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.53 ± 6.25   Final Test: 34.74 ± 4.83   Final old_old: 38.27 ± 5.27   Final old_new: 20.40 ± 3.45   Final new_new: 5.00 ± 1.02
Hits@20:
  Final val: 62.55 ± 3.74   Final Test: 43.06 ± 3.38   Final old_old: 47.03 ± 3.90   Final old_new: 27.13 ± 1.52   Final new_new: 5.91 ± 2.03
Hits@30:
  Final val: 68.02 ± 3.13   Final Test: 53.26 ± 3.76   Final old_old: 57.62 ± 4.10   Final old_new: 35.97 ± 2.41   Final new_new: 9.09 ± 1.61
Hits@50:
  Final val: 76.11 ± 2.51   Final Test: 60.32 ± 3.88   Final old_old: 64.71 ± 4.16   Final old_new: 43.15 ± 3.22   Final new_new: 11.82 ± 2.49
AUC:
  Final val: 97.25 ± 0.33   Final Test: 96.62 ± 0.23   Final old_old: 97.35 ± 0.33   Final old_new: 93.73 ± 0.19   Final new_new: 89.40 ± 1.57
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 40.35 ± 4.98   Final Test: 28.45 ± 5.27   Final old_old: 31.02 ± 5.67   Final old_new: 17.93 ± 4.03   Final new_new: 8.18 ± 2.59
Hits@20:
  Final val: 51.97 ± 4.90   Final Test: 35.11 ± 3.42   Final old_old: 37.82 ± 3.84   Final old_new: 24.13 ± 2.23   Final new_new: 11.82 ± 6.31
Hits@30:
  Final val: 60.05 ± 5.29   Final Test: 43.48 ± 3.32   Final old_old: 46.80 ± 3.62   Final old_new: 30.15 ± 2.44   Final new_new: 12.27 ± 5.47
Hits@50:
  Final val: 69.32 ± 4.49   Final Test: 54.39 ± 5.55   Final old_old: 57.75 ± 6.01   Final old_new: 41.06 ± 3.80   Final new_new: 20.00 ± 8.10
AUC:
  Final val: 96.40 ± 0.38   Final Test: 95.80 ± 0.28   Final old_old: 96.55 ± 0.27   Final old_new: 92.77 ± 0.37   Final new_new: 88.93 ± 0.75
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.93 ± 3.16   Final Test: 26.83 ± 3.80   Final old_old: 29.77 ± 4.27   Final old_new: 14.84 ± 1.92   Final new_new: 2.73 ± 1.90
Hits@20:
  Final val: 57.17 ± 4.23   Final Test: 39.27 ± 1.72   Final old_old: 42.72 ± 1.76   Final old_new: 25.42 ± 2.13   Final new_new: 8.18 ± 2.59
Hits@30:
  Final val: 64.76 ± 2.18   Final Test: 45.80 ± 2.27   Final old_old: 49.75 ± 2.12   Final old_new: 29.90 ± 3.06   Final new_new: 10.00 ± 2.03
Hits@50:
  Final val: 73.27 ± 2.26   Final Test: 55.80 ± 2.91   Final old_old: 59.91 ± 3.15   Final old_new: 39.45 ± 2.31   Final new_new: 15.00 ± 2.59
AUC:
  Final val: 97.15 ± 0.14   Final Test: 96.48 ± 0.09   Final old_old: 97.15 ± 0.14   Final old_new: 93.83 ± 0.30   Final new_new: 89.54 ± 0.96
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.68 ± 2.77   Final Test: 27.77 ± 2.80   Final old_old: 30.66 ± 2.84   Final old_new: 16.05 ± 2.88   Final new_new: 3.64 ± 2.59
Hits@20:
  Final val: 59.45 ± 2.62   Final Test: 37.84 ± 2.53   Final old_old: 41.37 ± 2.50   Final old_new: 23.65 ± 2.77   Final new_new: 5.00 ± 2.96
Hits@30:
  Final val: 65.93 ± 1.49   Final Test: 48.12 ± 1.68   Final old_old: 52.10 ± 1.65   Final old_new: 32.29 ± 2.15   Final new_new: 8.18 ± 2.03
Hits@50:
  Final val: 73.44 ± 1.65   Final Test: 58.97 ± 2.68   Final old_old: 63.20 ± 2.67   Final old_new: 42.39 ± 2.97   Final new_new: 12.73 ± 2.59
AUC:
  Final val: 97.15 ± 0.13   Final Test: 96.59 ± 0.09   Final old_old: 97.28 ± 0.12   Final old_new: 93.88 ± 0.09   Final new_new: 89.47 ± 0.79
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 55.63 ± 2.50   Final Test: 36.74 ± 4.61   Final old_old: 40.06 ± 4.97   Final old_new: 23.30 ± 3.36   Final new_new: 7.73 ± 3.45
Hits@20:
  Final val: 67.51 ± 3.46   Final Test: 52.49 ± 4.05   Final old_old: 56.51 ± 4.32   Final old_new: 36.55 ± 3.34   Final new_new: 12.27 ± 2.59
Hits@30:
  Final val: 73.35 ± 2.24   Final Test: 58.31 ± 3.55   Final old_old: 62.53 ± 3.64   Final old_new: 41.61 ± 3.82   Final new_new: 15.45 ± 2.96
Hits@50:
  Final val: 79.53 ± 0.95   Final Test: 66.06 ± 3.01   Final old_old: 70.34 ± 2.99   Final old_new: 49.29 ± 3.51   Final new_new: 19.09 ± 4.13
AUC:
  Final val: 97.70 ± 0.09   Final Test: 97.21 ± 0.17   Final old_old: 97.74 ± 0.21   Final old_new: 95.08 ± 0.16   Final new_new: 91.67 ± 1.28
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 38.23 ± 3.72   Final Test: 26.72 ± 8.79   Final old_old: 29.05 ± 9.57   Final old_new: 17.15 ± 5.71   Final new_new: 9.09 ± 4.25
Hits@20:
  Final val: 50.29 ± 4.27   Final Test: 34.13 ± 2.55   Final old_old: 36.79 ± 2.48   Final old_new: 23.38 ± 3.04   Final new_new: 10.45 ± 2.03
Hits@30:
  Final val: 58.62 ± 4.33   Final Test: 38.81 ± 3.95   Final old_old: 41.78 ± 4.34   Final old_new: 26.83 ± 2.51   Final new_new: 12.73 ± 4.71
Hits@50:
  Final val: 67.56 ± 3.37   Final Test: 53.53 ± 4.07   Final old_old: 56.98 ± 4.51   Final old_new: 39.92 ± 3.73   Final new_new: 17.73 ± 4.93
AUC:
  Final val: 96.35 ± 0.30   Final Test: 95.67 ± 0.17   Final old_old: 96.37 ± 0.22   Final old_new: 92.87 ± 0.65   Final new_new: 89.25 ± 0.78
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.91 ± 2.79   Final Test: 33.92 ± 5.60   Final old_old: 37.31 ± 5.96   Final old_new: 20.23 ± 4.22   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 60.17 ± 3.18   Final Test: 42.76 ± 4.07   Final old_old: 46.50 ± 4.32   Final old_new: 27.83 ± 3.22   Final new_new: 7.27 ± 3.37
Hits@30:
  Final val: 66.24 ± 3.27   Final Test: 48.24 ± 2.31   Final old_old: 52.37 ± 2.43   Final old_new: 31.74 ± 2.08   Final new_new: 8.18 ± 2.03
Hits@50:
  Final val: 74.42 ± 2.16   Final Test: 58.88 ± 4.51   Final old_old: 63.17 ± 4.74   Final old_new: 41.86 ± 3.57   Final new_new: 15.45 ± 5.88
AUC:
  Final val: 97.21 ± 0.19   Final Test: 96.59 ± 0.24   Final old_old: 97.30 ± 0.26   Final old_new: 93.71 ± 0.45   Final new_new: 90.29 ± 1.03
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.21 ± 8.00   Final Test: 25.88 ± 4.49   Final old_old: 28.20 ± 4.72   Final old_new: 16.42 ± 3.75   Final new_new: 6.82 ± 3.21
Hits@20:
  Final val: 55.60 ± 4.01   Final Test: 35.41 ± 7.43   Final old_old: 38.06 ± 7.70   Final old_new: 24.86 ± 6.69   Final new_new: 9.09 ± 3.94
Hits@30:
  Final val: 61.84 ± 4.12   Final Test: 42.19 ± 6.97   Final old_old: 45.31 ± 7.61   Final old_new: 29.82 ± 4.62   Final new_new: 10.45 ± 2.03
Hits@50:
  Final val: 70.78 ± 3.90   Final Test: 53.97 ± 5.98   Final old_old: 57.77 ± 6.29   Final old_new: 38.99 ± 5.25   Final new_new: 14.09 ± 3.73
AUC:
  Final val: 96.41 ± 0.42   Final Test: 95.73 ± 0.23   Final old_old: 96.45 ± 0.31   Final old_new: 92.79 ± 0.36   Final new_new: 89.58 ± 1.97
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.94 ± 4.04   Final Test: 31.84 ± 1.14   Final old_old: 35.08 ± 1.20   Final old_new: 18.69 ± 1.36   Final new_new: 4.55 ± 0.00
Hits@20:
  Final val: 58.59 ± 2.22   Final Test: 40.69 ± 2.26   Final old_old: 44.22 ± 2.42   Final old_new: 26.42 ± 1.84   Final new_new: 9.09 ± 1.61
Hits@30:
  Final val: 65.43 ± 2.38   Final Test: 48.38 ± 1.59   Final old_old: 52.42 ± 1.68   Final old_new: 32.12 ± 1.46   Final new_new: 11.36 ± 1.61
Hits@50:
  Final val: 72.06 ± 1.37   Final Test: 57.51 ± 3.25   Final old_old: 61.88 ± 3.51   Final old_new: 40.08 ± 2.41   Final new_new: 15.45 ± 3.37
AUC:
  Final val: 97.14 ± 0.15   Final Test: 96.57 ± 0.07   Final old_old: 97.27 ± 0.14   Final old_new: 93.81 ± 0.19   Final new_new: 88.76 ± 2.05
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.88 ± 4.96   Final Test: 36.00 ± 4.43   Final old_old: 39.29 ± 4.64   Final old_new: 22.85 ± 3.66   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 65.51 ± 1.76   Final Test: 47.56 ± 3.87   Final old_old: 51.22 ± 3.96   Final old_new: 33.10 ± 3.86   Final new_new: 9.55 ± 2.96
Hits@30:
  Final val: 71.40 ± 0.80   Final Test: 55.11 ± 3.70   Final old_old: 59.08 ± 3.55   Final old_new: 39.57 ± 4.52   Final new_new: 10.91 ± 4.66
Hits@50:
  Final val: 77.27 ± 0.69   Final Test: 64.07 ± 1.75   Final old_old: 67.99 ± 2.02   Final old_new: 48.87 ± 0.91   Final new_new: 17.73 ± 4.93
AUC:
  Final val: 97.63 ± 0.06   Final Test: 97.33 ± 0.08   Final old_old: 97.78 ± 0.08   Final old_new: 95.56 ± 0.14   Final new_new: 92.54 ± 0.34
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.15 ± 5.64   Final Test: 32.16 ± 3.38   Final old_old: 35.62 ± 3.81   Final old_new: 17.98 ± 2.79   Final new_new: 4.55 ± 4.25
Hits@20:
  Final val: 62.07 ± 3.84   Final Test: 44.55 ± 1.71   Final old_old: 48.74 ± 2.08   Final old_new: 27.71 ± 1.34   Final new_new: 5.91 ± 3.80
Hits@30:
  Final val: 68.49 ± 2.58   Final Test: 49.93 ± 4.91   Final old_old: 54.28 ± 5.36   Final old_new: 32.59 ± 3.24   Final new_new: 7.27 ± 3.73
Hits@50:
  Final val: 75.91 ± 1.57   Final Test: 60.37 ± 2.68   Final old_old: 64.94 ± 2.66   Final old_new: 42.47 ± 3.12   Final new_new: 10.45 ± 1.24
AUC:
  Final val: 97.28 ± 0.22   Final Test: 96.56 ± 0.09   Final old_old: 97.35 ± 0.15   Final old_new: 93.45 ± 0.20   Final new_new: 88.23 ± 1.79
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.02 ± 2.12   Final Test: 31.45 ± 4.91   Final old_old: 34.78 ± 5.22   Final old_new: 17.98 ± 3.90   Final new_new: 2.73 ± 1.90
Hits@20:
  Final val: 58.48 ± 2.63   Final Test: 39.95 ± 2.71   Final old_old: 43.78 ± 3.13   Final old_new: 24.61 ± 1.31   Final new_new: 4.55 ± 3.59
Hits@30:
  Final val: 66.44 ± 1.32   Final Test: 49.39 ± 4.64   Final old_old: 53.57 ± 5.11   Final old_new: 32.87 ± 2.93   Final new_new: 6.36 ± 1.90
Hits@50:
  Final val: 74.48 ± 1.91   Final Test: 59.10 ± 1.81   Final old_old: 63.54 ± 1.92   Final old_new: 41.54 ± 1.91   Final new_new: 13.18 ± 1.90
AUC:
  Final val: 97.19 ± 0.29   Final Test: 96.57 ± 0.18   Final old_old: 97.33 ± 0.26   Final old_new: 93.48 ± 0.31   Final new_new: 89.47 ± 0.81
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 39.75 ± 4.87   Final Test: 25.62 ± 5.91   Final old_old: 28.13 ± 6.35   Final old_new: 15.44 ± 4.35   Final new_new: 5.00 ± 3.73
Hits@20:
  Final val: 52.46 ± 3.49   Final Test: 33.87 ± 3.91   Final old_old: 36.75 ± 4.09   Final old_new: 22.27 ± 3.40   Final new_new: 8.64 ± 4.93
Hits@30:
  Final val: 60.24 ± 3.19   Final Test: 44.66 ± 4.91   Final old_old: 47.78 ± 5.41   Final old_new: 32.34 ± 3.24   Final new_new: 12.73 ± 8.29
Hits@50:
  Final val: 68.73 ± 2.27   Final Test: 53.66 ± 2.21   Final old_old: 57.07 ± 2.69   Final old_new: 40.18 ± 1.67   Final new_new: 18.64 ± 7.94
AUC:
  Final val: 96.41 ± 0.26   Final Test: 95.74 ± 0.24   Final old_old: 96.43 ± 0.23   Final old_new: 92.93 ± 0.42   Final new_new: 89.97 ± 1.34
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.89 ± 6.32   Final Test: 26.15 ± 1.86   Final old_old: 28.48 ± 1.91   Final old_new: 16.73 ± 1.88   Final new_new: 5.91 ± 1.24
Hits@20:
  Final val: 63.35 ± 4.21   Final Test: 39.75 ± 6.09   Final old_old: 43.03 ± 6.42   Final old_new: 26.65 ± 4.88   Final new_new: 8.18 ± 4.43
Hits@30:
  Final val: 69.53 ± 2.79   Final Test: 46.80 ± 5.18   Final old_old: 50.30 ± 5.15   Final old_new: 32.87 ± 5.40   Final new_new: 12.27 ± 6.93
Hits@50:
  Final val: 76.12 ± 1.20   Final Test: 59.57 ± 1.26   Final old_old: 63.54 ± 1.45   Final old_new: 43.83 ± 2.24   Final new_new: 19.09 ± 4.71
AUC:
  Final val: 96.79 ± 0.25   Final Test: 96.26 ± 0.34   Final old_old: 96.88 ± 0.32   Final old_new: 93.74 ± 0.45   Final new_new: 90.98 ± 1.89
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.84 ± 2.63   Final Test: 29.57 ± 2.30   Final old_old: 32.70 ± 2.33   Final old_new: 16.85 ± 2.50   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 57.55 ± 2.86   Final Test: 40.05 ± 2.24   Final old_old: 43.91 ± 2.36   Final old_new: 24.46 ± 2.54   Final new_new: 5.91 ± 1.24
Hits@30:
  Final val: 63.86 ± 1.85   Final Test: 48.27 ± 2.35   Final old_old: 52.31 ± 2.36   Final old_new: 32.19 ± 2.64   Final new_new: 8.64 ± 2.96
Hits@50:
  Final val: 72.44 ± 1.78   Final Test: 57.43 ± 1.89   Final old_old: 61.72 ± 2.10   Final old_new: 40.55 ± 1.12   Final new_new: 11.82 ± 4.37
AUC:
  Final val: 97.11 ± 0.18   Final Test: 96.56 ± 0.14   Final old_old: 97.24 ± 0.19   Final old_new: 93.92 ± 0.07   Final new_new: 88.87 ± 1.10
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.32 ± 4.55   Final Test: 30.47 ± 3.51   Final old_old: 33.81 ± 3.80   Final old_new: 16.75 ± 2.55   Final new_new: 5.00 ± 1.02
Hits@20:
  Final val: 58.62 ± 3.20   Final Test: 37.49 ± 5.36   Final old_old: 41.12 ± 5.51   Final old_new: 22.80 ± 5.02   Final new_new: 6.82 ± 2.27
Hits@30:
  Final val: 65.85 ± 1.96   Final Test: 44.32 ± 4.35   Final old_old: 48.24 ± 4.62   Final old_new: 28.54 ± 3.43   Final new_new: 9.55 ± 3.73
Hits@50:
  Final val: 73.59 ± 1.60   Final Test: 56.22 ± 2.84   Final old_old: 60.62 ± 2.92   Final old_new: 38.66 ± 2.47   Final new_new: 13.18 ± 5.18
AUC:
  Final val: 97.15 ± 0.22   Final Test: 96.53 ± 0.13   Final old_old: 97.25 ± 0.16   Final old_new: 93.68 ± 0.29   Final new_new: 89.10 ± 0.85
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.93 ± 5.22   Final Test: 30.11 ± 2.04   Final old_old: 33.28 ± 2.28   Final old_new: 17.18 ± 1.27   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 60.63 ± 5.00   Final Test: 43.25 ± 2.55   Final old_old: 47.21 ± 2.70   Final old_new: 27.36 ± 2.37   Final new_new: 7.27 ± 2.96
Hits@30:
  Final val: 67.68 ± 2.56   Final Test: 49.38 ± 1.72   Final old_old: 53.63 ± 1.91   Final old_new: 32.42 ± 1.21   Final new_new: 8.18 ± 2.59
Hits@50:
  Final val: 75.96 ± 0.99   Final Test: 61.25 ± 0.59   Final old_old: 65.67 ± 0.67   Final old_new: 43.78 ± 0.82   Final new_new: 16.36 ± 3.73
AUC:
  Final val: 97.22 ± 0.12   Final Test: 96.53 ± 0.16   Final old_old: 97.27 ± 0.17   Final old_new: 93.54 ± 0.25   Final new_new: 89.22 ± 1.11
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 40.19 ± 2.23   Final Test: 25.80 ± 3.11   Final old_old: 27.99 ± 3.38   Final old_new: 17.00 ± 2.64   Final new_new: 6.36 ± 2.96
Hits@20:
  Final val: 52.07 ± 2.14   Final Test: 36.05 ± 2.55   Final old_old: 38.96 ± 2.80   Final old_new: 24.48 ± 1.89   Final new_new: 6.82 ± 2.27
Hits@30:
  Final val: 59.71 ± 2.54   Final Test: 43.60 ± 1.42   Final old_old: 46.78 ± 1.44   Final old_new: 30.93 ± 1.62   Final new_new: 12.27 ± 1.24
Hits@50:
  Final val: 68.30 ± 1.35   Final Test: 51.68 ± 1.66   Final old_old: 55.16 ± 1.76   Final old_new: 38.01 ± 1.66   Final new_new: 14.55 ± 2.59
AUC:
  Final val: 96.47 ± 0.29   Final Test: 95.84 ± 0.21   Final old_old: 96.56 ± 0.25   Final old_new: 92.98 ± 0.18   Final new_new: 88.91 ± 0.91
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 54.21 ± 2.68   Final Test: 35.71 ± 6.15   Final old_old: 39.18 ± 6.48   Final old_new: 21.69 ± 4.98   Final new_new: 5.91 ± 3.80
Hits@20:
  Final val: 67.04 ± 2.23   Final Test: 48.13 ± 4.51   Final old_old: 52.13 ± 4.77   Final old_new: 32.19 ± 3.70   Final new_new: 9.09 ± 2.78
Hits@30:
  Final val: 72.43 ± 2.11   Final Test: 56.19 ± 2.08   Final old_old: 60.40 ± 2.23   Final old_new: 39.40 ± 1.80   Final new_new: 15.45 ± 4.37
Hits@50:
  Final val: 78.64 ± 1.43   Final Test: 64.62 ± 3.21   Final old_old: 68.89 ± 3.34   Final old_new: 47.68 ± 2.82   Final new_new: 21.82 ± 2.03
AUC:
  Final val: 97.59 ± 0.04   Final Test: 97.15 ± 0.08   Final old_old: 97.68 ± 0.09   Final old_new: 95.04 ± 0.19   Final new_new: 91.86 ± 1.39
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.89 ± 6.67   Final Test: 24.30 ± 8.30   Final old_old: 26.52 ± 9.10   Final old_new: 15.16 ± 5.00   Final new_new: 7.73 ± 4.13
Hits@20:
  Final val: 52.36 ± 5.29   Final Test: 35.00 ± 7.78   Final old_old: 37.87 ± 8.34   Final old_new: 23.38 ± 5.49   Final new_new: 10.45 ± 4.71
Hits@30:
  Final val: 59.47 ± 4.25   Final Test: 43.62 ± 6.07   Final old_old: 46.98 ± 6.26   Final old_new: 30.10 ± 5.50   Final new_new: 13.18 ± 5.88
Hits@50:
  Final val: 68.87 ± 3.37   Final Test: 52.58 ± 5.35   Final old_old: 55.96 ± 5.81   Final old_new: 39.17 ± 3.46   Final new_new: 17.73 ± 6.70
AUC:
  Final val: 96.44 ± 0.30   Final Test: 95.74 ± 0.33   Final old_old: 96.48 ± 0.35   Final old_new: 92.78 ± 0.30   Final new_new: 89.42 ± 0.45
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.33 ± 3.36   Final Test: 29.21 ± 6.30   Final old_old: 32.24 ± 6.92   Final old_new: 16.88 ± 4.10   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 57.28 ± 2.02   Final Test: 38.62 ± 4.32   Final old_old: 42.15 ± 4.64   Final old_new: 24.38 ± 3.17   Final new_new: 6.82 ± 3.59
Hits@30:
  Final val: 63.97 ± 2.13   Final Test: 46.58 ± 3.80   Final old_old: 50.42 ± 4.15   Final old_new: 31.21 ± 2.40   Final new_new: 10.00 ± 5.70
Hits@50:
  Final val: 71.91 ± 1.54   Final Test: 55.70 ± 4.54   Final old_old: 59.93 ± 4.70   Final old_new: 38.94 ± 3.97   Final new_new: 12.27 ± 7.29
AUC:
  Final val: 97.08 ± 0.16   Final Test: 96.51 ± 0.11   Final old_old: 97.19 ± 0.20   Final old_new: 93.80 ± 0.43   Final new_new: 89.81 ± 1.42
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.34 ± 3.86   Final Test: 30.67 ± 4.40   Final old_old: 33.73 ± 4.82   Final old_new: 18.31 ± 3.10   Final new_new: 4.09 ± 1.02
Hits@20:
  Final val: 60.19 ± 3.25   Final Test: 44.73 ± 3.92   Final old_old: 48.64 ± 4.12   Final old_new: 29.09 ± 3.30   Final new_new: 6.82 ± 2.78
Hits@30:
  Final val: 67.87 ± 2.64   Final Test: 51.33 ± 2.50   Final old_old: 55.48 ± 2.60   Final old_new: 34.99 ± 2.30   Final new_new: 7.73 ± 1.24
Hits@50:
  Final val: 75.50 ± 1.96   Final Test: 60.54 ± 1.43   Final old_old: 65.04 ± 1.55   Final old_new: 42.95 ± 2.52   Final new_new: 10.91 ± 1.02
AUC:
  Final val: 97.21 ± 0.25   Final Test: 96.56 ± 0.21   Final old_old: 97.34 ± 0.27   Final old_new: 93.41 ± 0.36   Final new_new: 88.94 ± 1.31
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.99 ± 5.74   Final Test: 33.70 ± 3.43   Final old_old: 37.05 ± 3.60   Final old_new: 20.15 ± 2.91   Final new_new: 5.00 ± 1.90
Hits@20:
  Final val: 60.65 ± 2.51   Final Test: 43.65 ± 4.45   Final old_old: 47.41 ± 4.73   Final old_new: 28.72 ± 3.41   Final new_new: 5.91 ± 1.24
Hits@30:
  Final val: 67.59 ± 2.15   Final Test: 50.15 ± 4.07   Final old_old: 54.37 ± 4.35   Final old_new: 33.38 ± 3.11   Final new_new: 8.18 ± 2.59
Hits@50:
  Final val: 75.36 ± 2.11   Final Test: 59.80 ± 3.71   Final old_old: 64.19 ± 3.95   Final old_new: 42.49 ± 2.92   Final new_new: 13.64 ± 1.61
AUC:
  Final val: 97.20 ± 0.27   Final Test: 96.60 ± 0.20   Final old_old: 97.34 ± 0.22   Final old_new: 93.63 ± 0.28   Final new_new: 89.84 ± 0.69
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.88 ± 2.30   Final Test: 34.08 ± 1.67   Final old_old: 37.31 ± 1.78   Final old_new: 21.11 ± 1.47   Final new_new: 5.00 ± 1.02
Hits@20:
  Final val: 64.15 ± 1.98   Final Test: 45.24 ± 3.65   Final old_old: 48.98 ± 3.75   Final old_new: 30.33 ± 3.33   Final new_new: 9.09 ± 3.94
Hits@30:
  Final val: 71.41 ± 1.29   Final Test: 55.11 ± 2.71   Final old_old: 59.28 ± 2.79   Final old_new: 38.69 ± 2.57   Final new_new: 11.36 ± 1.61
Hits@50:
  Final val: 77.86 ± 0.80   Final Test: 63.70 ± 2.24   Final old_old: 67.66 ± 2.18   Final old_new: 48.16 ± 2.85   Final new_new: 20.00 ± 6.31
AUC:
  Final val: 97.62 ± 0.07   Final Test: 97.25 ± 0.07   Final old_old: 97.75 ± 0.09   Final old_new: 95.23 ± 0.12   Final new_new: 92.55 ± 0.60
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.81 ± 7.51   Final Test: 27.82 ± 6.51   Final old_old: 30.12 ± 7.10   Final old_new: 18.54 ± 4.44   Final new_new: 7.27 ± 1.02
Hits@20:
  Final val: 53.00 ± 5.86   Final Test: 35.52 ± 3.54   Final old_old: 38.26 ± 3.99   Final old_new: 24.43 ± 2.36   Final new_new: 11.82 ± 3.73
Hits@30:
  Final val: 60.38 ± 5.96   Final Test: 43.98 ± 5.69   Final old_old: 47.31 ± 6.10   Final old_new: 30.63 ± 4.30   Final new_new: 12.73 ± 2.59
Hits@50:
  Final val: 69.33 ± 4.83   Final Test: 54.58 ± 3.90   Final old_old: 58.02 ± 4.30   Final old_new: 40.91 ± 3.10   Final new_new: 20.00 ± 6.31
AUC:
  Final val: 96.38 ± 0.46   Final Test: 95.75 ± 0.40   Final old_old: 96.46 ± 0.44   Final old_new: 92.84 ± 0.46   Final new_new: 90.53 ± 1.48
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.29 ± 1.60   Final Test: 28.99 ± 5.77   Final old_old: 32.05 ± 6.23   Final old_new: 16.55 ± 4.20   Final new_new: 3.64 ± 2.03
Hits@20:
  Final val: 56.60 ± 1.91   Final Test: 38.69 ± 2.73   Final old_old: 42.22 ± 3.04   Final old_new: 24.46 ± 2.09   Final new_new: 7.27 ± 2.96
Hits@30:
  Final val: 64.29 ± 1.74   Final Test: 44.69 ± 3.34   Final old_old: 48.50 ± 3.64   Final old_new: 29.57 ± 2.37   Final new_new: 6.36 ± 3.37
Hits@50:
  Final val: 72.28 ± 0.45   Final Test: 56.60 ± 1.50   Final old_old: 60.77 ± 1.83   Final old_new: 40.23 ± 0.69   Final new_new: 11.36 ± 2.27
AUC:
  Final val: 97.12 ± 0.16   Final Test: 96.50 ± 0.09   Final old_old: 97.17 ± 0.13   Final old_new: 93.85 ± 0.22   Final new_new: 89.73 ± 1.28
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.32 ± 2.71   Final Test: 28.66 ± 3.29   Final old_old: 31.83 ± 3.45   Final old_new: 15.69 ± 2.88   Final new_new: 3.18 ± 1.24
Hits@20:
  Final val: 58.08 ± 3.67   Final Test: 38.92 ± 2.75   Final old_old: 42.54 ± 2.68   Final old_new: 24.41 ± 3.28   Final new_new: 4.55 ± 0.00
Hits@30:
  Final val: 65.98 ± 1.43   Final Test: 46.40 ± 1.47   Final old_old: 50.54 ± 1.47   Final old_new: 29.92 ± 2.12   Final new_new: 5.45 ± 1.24
Hits@50:
  Final val: 73.60 ± 1.18   Final Test: 58.90 ± 1.76   Final old_old: 63.29 ± 1.65   Final old_new: 41.64 ± 2.43   Final new_new: 12.27 ± 4.43
AUC:
  Final val: 97.17 ± 0.16   Final Test: 96.55 ± 0.06   Final old_old: 97.26 ± 0.13   Final old_new: 93.76 ± 0.19   Final new_new: 88.89 ± 2.16
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.80 ± 1.54   Final Test: 29.24 ± 5.98   Final old_old: 32.28 ± 6.44   Final old_new: 16.95 ± 4.41   Final new_new: 2.73 ± 1.90
Hits@20:
  Final val: 57.70 ± 2.96   Final Test: 39.74 ± 6.34   Final old_old: 43.38 ± 7.00   Final old_new: 25.24 ± 3.97   Final new_new: 3.64 ± 2.03
Hits@30:
  Final val: 66.09 ± 2.23   Final Test: 49.69 ± 3.38   Final old_old: 53.95 ± 3.63   Final old_new: 32.80 ± 2.59   Final new_new: 6.36 ± 2.96
Hits@50:
  Final val: 74.80 ± 1.60   Final Test: 59.41 ± 3.68   Final old_old: 63.79 ± 3.75   Final old_new: 42.24 ± 3.82   Final new_new: 11.36 ± 2.78
AUC:
  Final val: 97.16 ± 0.20   Final Test: 96.55 ± 0.19   Final old_old: 97.27 ± 0.21   Final old_new: 93.69 ± 0.28   Final new_new: 89.51 ± 1.24
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.04 ± 2.40   Final Test: 30.45 ± 7.69   Final old_old: 33.09 ± 8.18   Final old_new: 19.80 ± 5.97   Final new_new: 6.82 ± 4.55
Hits@20:
  Final val: 62.63 ± 2.07   Final Test: 42.53 ± 2.92   Final old_old: 46.02 ± 3.17   Final old_new: 28.59 ± 2.44   Final new_new: 9.55 ± 6.31
Hits@30:
  Final val: 68.71 ± 1.32   Final Test: 48.97 ± 3.25   Final old_old: 52.71 ± 3.32   Final old_new: 34.11 ± 3.17   Final new_new: 11.82 ± 2.49
Hits@50:
  Final val: 75.30 ± 1.68   Final Test: 60.90 ± 1.93   Final old_old: 64.85 ± 1.95   Final old_new: 45.21 ± 2.12   Final new_new: 21.36 ± 7.81
AUC:
  Final val: 96.86 ± 0.35   Final Test: 96.28 ± 0.28   Final old_old: 96.89 ± 0.24   Final old_new: 93.88 ± 0.48   Final new_new: 90.00 ± 1.06
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 39.76 ± 5.24   Final Test: 26.45 ± 4.24   Final old_old: 28.64 ± 4.42   Final old_new: 17.66 ± 3.85   Final new_new: 6.82 ± 2.78
Hits@20:
  Final val: 52.90 ± 3.54   Final Test: 34.19 ± 4.62   Final old_old: 36.90 ± 4.73   Final old_new: 23.40 ± 4.22   Final new_new: 7.73 ± 4.98
Hits@30:
  Final val: 60.07 ± 3.30   Final Test: 41.84 ± 1.27   Final old_old: 44.89 ± 1.10   Final old_new: 29.75 ± 3.07   Final new_new: 10.91 ± 2.96
Hits@50:
  Final val: 69.11 ± 2.88   Final Test: 53.93 ± 2.30   Final old_old: 57.51 ± 2.68   Final old_new: 39.82 ± 1.88   Final new_new: 16.82 ± 4.71
AUC:
  Final val: 96.48 ± 0.29   Final Test: 95.74 ± 0.20   Final old_old: 96.44 ± 0.23   Final old_new: 92.94 ± 0.08   Final new_new: 89.24 ± 1.18
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.98 ± 2.81   Final Test: 26.33 ± 3.80   Final old_old: 28.62 ± 4.22   Final old_new: 17.13 ± 2.50   Final new_new: 5.45 ± 2.59
Hits@20:
  Final val: 54.40 ± 0.46   Final Test: 33.78 ± 3.63   Final old_old: 36.60 ± 3.84   Final old_new: 22.39 ± 2.99   Final new_new: 8.64 ± 1.90
Hits@30:
  Final val: 61.50 ± 1.34   Final Test: 42.27 ± 2.46   Final old_old: 45.44 ± 2.62   Final old_new: 29.67 ± 2.29   Final new_new: 10.91 ± 2.96
Hits@50:
  Final val: 71.16 ± 0.92   Final Test: 54.83 ± 2.95   Final old_old: 58.61 ± 2.97   Final old_new: 39.87 ± 2.97   Final new_new: 15.91 ± 4.82
AUC:
  Final val: 96.50 ± 0.25   Final Test: 95.77 ± 0.19   Final old_old: 96.48 ± 0.18   Final old_new: 92.90 ± 0.24   Final new_new: 89.70 ± 1.39
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.18 ± 2.41   Final Test: 29.51 ± 1.78   Final old_old: 32.47 ± 1.80   Final old_new: 17.53 ± 2.03   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 56.54 ± 2.95   Final Test: 39.83 ± 5.64   Final old_old: 43.33 ± 5.98   Final old_new: 25.84 ± 4.59   Final new_new: 6.36 ± 3.37
Hits@30:
  Final val: 63.67 ± 2.22   Final Test: 46.54 ± 4.09   Final old_old: 50.49 ± 4.28   Final old_new: 30.73 ± 3.35   Final new_new: 9.09 ± 4.82
Hits@50:
  Final val: 72.38 ± 1.62   Final Test: 56.70 ± 2.67   Final old_old: 60.90 ± 2.66   Final old_new: 40.10 ± 2.79   Final new_new: 13.18 ± 2.96
AUC:
  Final val: 97.11 ± 0.16   Final Test: 96.46 ± 0.21   Final old_old: 97.17 ± 0.21   Final old_new: 93.67 ± 0.31   Final new_new: 89.02 ± 0.58
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 38.51 ± 3.15   Final Test: 26.24 ± 4.49   Final old_old: 28.37 ± 4.84   Final old_new: 17.76 ± 3.45   Final new_new: 5.91 ± 3.05
Hits@20:
  Final val: 50.30 ± 2.10   Final Test: 35.87 ± 2.36   Final old_old: 38.70 ± 2.27   Final old_new: 24.61 ± 2.91   Final new_new: 8.64 ± 4.37
Hits@30:
  Final val: 58.25 ± 2.69   Final Test: 40.45 ± 2.85   Final old_old: 43.54 ± 3.15   Final old_new: 28.29 ± 2.87   Final new_new: 7.27 ± 4.37
Hits@50:
  Final val: 68.64 ± 2.06   Final Test: 52.86 ± 3.36   Final old_old: 56.42 ± 3.61   Final old_new: 38.77 ± 2.37   Final new_new: 16.82 ± 6.14
AUC:
  Final val: 96.41 ± 0.32   Final Test: 95.82 ± 0.30   Final old_old: 96.45 ± 0.33   Final old_new: 93.28 ± 0.27   Final new_new: 90.03 ± 1.60
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.27 ± 3.27   Final Test: 35.42 ± 7.42   Final old_old: 38.77 ± 8.01   Final old_new: 21.99 ± 5.09   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 61.31 ± 3.62   Final Test: 45.29 ± 4.18   Final old_old: 49.07 ± 4.41   Final old_new: 30.25 ± 3.36   Final new_new: 7.73 ± 2.59
Hits@30:
  Final val: 68.25 ± 2.32   Final Test: 53.16 ± 2.07   Final old_old: 57.28 ± 2.36   Final old_new: 36.98 ± 1.52   Final new_new: 8.64 ± 2.96
Hits@50:
  Final val: 75.03 ± 2.23   Final Test: 60.21 ± 0.95   Final old_old: 64.38 ± 0.68   Final old_new: 43.90 ± 2.15   Final new_new: 13.64 ± 4.55
AUC:
  Final val: 97.22 ± 0.29   Final Test: 96.66 ± 0.22   Final old_old: 97.38 ± 0.27   Final old_new: 93.79 ± 0.14   Final new_new: 89.99 ± 0.66
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 53.72 ± 6.12   Final Test: 33.84 ± 9.10   Final old_old: 37.10 ± 9.68   Final old_new: 20.76 ± 6.96   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 66.35 ± 3.29   Final Test: 50.46 ± 4.86   Final old_old: 54.51 ± 5.10   Final old_new: 34.38 ± 4.01   Final new_new: 10.00 ± 4.71
Hits@30:
  Final val: 72.35 ± 2.24   Final Test: 54.74 ± 3.13   Final old_old: 58.87 ± 3.36   Final old_new: 38.39 ± 2.38   Final new_new: 12.27 ± 5.93
Hits@50:
  Final val: 78.37 ± 1.45   Final Test: 64.40 ± 1.98   Final old_old: 68.44 ± 2.03   Final old_new: 48.64 ± 2.17   Final new_new: 19.09 ± 6.55
AUC:
  Final val: 97.51 ± 0.15   Final Test: 97.08 ± 0.11   Final old_old: 97.63 ± 0.11   Final old_new: 94.94 ± 0.20   Final new_new: 91.14 ± 1.12
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.70 ± 3.32   Final Test: 33.41 ± 4.88   Final old_old: 36.64 ± 5.26   Final old_new: 20.40 ± 3.55   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 59.28 ± 3.28   Final Test: 42.12 ± 1.85   Final old_old: 45.79 ± 2.12   Final old_new: 27.51 ± 1.41   Final new_new: 6.36 ± 3.37
Hits@30:
  Final val: 66.46 ± 2.79   Final Test: 49.88 ± 2.41   Final old_old: 53.92 ± 2.72   Final old_new: 33.83 ± 1.45   Final new_new: 10.00 ± 3.05
Hits@50:
  Final val: 74.47 ± 2.40   Final Test: 59.90 ± 2.41   Final old_old: 64.18 ± 2.49   Final old_new: 42.97 ± 2.17   Final new_new: 15.91 ± 5.08
AUC:
  Final val: 97.12 ± 0.16   Final Test: 96.54 ± 0.12   Final old_old: 97.27 ± 0.15   Final old_new: 93.61 ± 0.20   Final new_new: 89.75 ± 1.23
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.07 ± 4.93   Final Test: 31.04 ± 2.97   Final old_old: 34.23 ± 3.33   Final old_new: 18.06 ± 2.31   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 55.98 ± 5.38   Final Test: 38.44 ± 4.32   Final old_old: 41.95 ± 4.66   Final old_new: 24.43 ± 3.59   Final new_new: 4.55 ± 1.61
Hits@30:
  Final val: 63.31 ± 4.07   Final Test: 46.13 ± 1.74   Final old_old: 50.04 ± 1.90   Final old_new: 30.45 ± 1.86   Final new_new: 9.55 ± 2.49
Hits@50:
  Final val: 72.37 ± 1.82   Final Test: 55.67 ± 1.99   Final old_old: 59.68 ± 2.21   Final old_new: 39.87 ± 1.39   Final new_new: 12.73 ± 4.13
AUC:
  Final val: 97.13 ± 0.20   Final Test: 96.54 ± 0.10   Final old_old: 97.19 ± 0.14   Final old_new: 93.99 ± 0.24   Final new_new: 89.26 ± 1.79
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.56 ± 2.70   Final Test: 32.13 ± 2.90   Final old_old: 35.17 ± 3.05   Final old_new: 19.90 ± 2.41   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 62.12 ± 3.01   Final Test: 45.61 ± 4.80   Final old_old: 49.25 ± 4.96   Final old_new: 31.23 ± 4.25   Final new_new: 7.73 ± 3.05
Hits@30:
  Final val: 68.52 ± 2.08   Final Test: 53.55 ± 1.64   Final old_old: 57.34 ± 1.69   Final old_new: 38.72 ± 1.75   Final new_new: 10.91 ± 2.49
Hits@50:
  Final val: 76.43 ± 0.40   Final Test: 62.51 ± 3.10   Final old_old: 66.33 ± 3.18   Final old_new: 47.81 ± 2.92   Final new_new: 15.00 ± 3.45
AUC:
  Final val: 97.55 ± 0.07   Final Test: 97.22 ± 0.06   Final old_old: 97.70 ± 0.08   Final old_new: 95.37 ± 0.12   Final new_new: 91.88 ± 0.82
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.22 ± 3.38   Final Test: 28.33 ± 2.82   Final old_old: 31.34 ± 3.08   Final old_new: 16.05 ± 2.21   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 59.15 ± 2.98   Final Test: 40.81 ± 3.16   Final old_old: 44.58 ± 3.37   Final old_new: 25.64 ± 2.62   Final new_new: 6.36 ± 1.90
Hits@30:
  Final val: 65.88 ± 1.21   Final Test: 49.35 ± 1.90   Final old_old: 53.36 ± 2.19   Final old_new: 33.30 ± 0.92   Final new_new: 10.91 ± 2.96
Hits@50:
  Final val: 73.29 ± 1.82   Final Test: 58.08 ± 1.51   Final old_old: 62.27 ± 1.72   Final old_new: 41.51 ± 1.31   Final new_new: 15.45 ± 3.73
AUC:
  Final val: 97.18 ± 0.16   Final Test: 96.60 ± 0.08   Final old_old: 97.31 ± 0.12   Final old_new: 93.84 ± 0.15   Final new_new: 88.87 ± 0.91
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.07 ± 3.95   Final Test: 34.62 ± 3.99   Final old_old: 38.37 ± 4.31   Final old_new: 19.40 ± 2.71   Final new_new: 3.18 ± 2.59
Hits@20:
  Final val: 61.54 ± 2.94   Final Test: 44.69 ± 4.37   Final old_old: 48.83 ± 4.86   Final old_new: 28.19 ± 2.49   Final new_new: 4.09 ± 1.90
Hits@30:
  Final val: 68.09 ± 1.99   Final Test: 54.28 ± 2.57   Final old_old: 58.89 ± 2.71   Final old_new: 35.99 ± 2.28   Final new_new: 7.73 ± 2.59
Hits@50:
  Final val: 75.50 ± 1.73   Final Test: 61.22 ± 1.71   Final old_old: 65.72 ± 1.78   Final old_new: 43.50 ± 1.88   Final new_new: 14.09 ± 8.71
AUC:
  Final val: 97.23 ± 0.22   Final Test: 96.61 ± 0.18   Final old_old: 97.35 ± 0.22   Final old_new: 93.72 ± 0.23   Final new_new: 89.02 ± 1.94
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.35 ± 1.59   Final Test: 28.44 ± 3.59   Final old_old: 31.45 ± 3.89   Final old_new: 16.22 ± 2.43   Final new_new: 2.73 ± 1.02
Hits@20:
  Final val: 57.36 ± 1.01   Final Test: 38.33 ± 1.55   Final old_old: 41.76 ± 1.81   Final old_new: 24.61 ± 1.55   Final new_new: 5.91 ± 3.45
Hits@30:
  Final val: 64.99 ± 0.65   Final Test: 47.53 ± 2.55   Final old_old: 51.52 ± 2.88   Final old_new: 31.69 ± 1.52   Final new_new: 8.18 ± 2.59
Hits@50:
  Final val: 73.41 ± 1.16   Final Test: 58.50 ± 3.38   Final old_old: 62.78 ± 3.55   Final old_new: 41.79 ± 2.90   Final new_new: 10.91 ± 2.96
AUC:
  Final val: 97.11 ± 0.19   Final Test: 96.49 ± 0.12   Final old_old: 97.21 ± 0.15   Final old_new: 93.66 ± 0.22   Final new_new: 88.93 ± 0.94
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.57 ± 5.38   Final Test: 27.57 ± 3.36   Final old_old: 29.89 ± 3.67   Final old_new: 18.24 ± 2.59   Final new_new: 7.27 ± 4.93
Hits@20:
  Final val: 51.43 ± 3.43   Final Test: 38.37 ± 5.73   Final old_old: 41.37 ± 6.25   Final old_new: 26.27 ± 3.75   Final new_new: 11.36 ± 3.59
Hits@30:
  Final val: 59.60 ± 2.67   Final Test: 45.33 ± 6.07   Final old_old: 48.75 ± 6.39   Final old_new: 31.64 ± 4.97   Final new_new: 13.18 ± 6.51
Hits@50:
  Final val: 69.40 ± 2.25   Final Test: 57.35 ± 3.43   Final old_old: 61.24 ± 3.94   Final old_new: 41.99 ± 1.45   Final new_new: 17.27 ± 4.43
AUC:
  Final val: 96.35 ± 0.31   Final Test: 95.78 ± 0.28   Final old_old: 96.52 ± 0.34   Final old_new: 92.80 ± 0.33   Final new_new: 88.93 ± 1.54
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.89 ± 5.26   Final Test: 26.50 ± 3.31   Final old_old: 28.96 ± 3.52   Final old_new: 16.50 ± 2.59   Final new_new: 5.45 ± 3.80
Hits@20:
  Final val: 54.13 ± 5.71   Final Test: 34.50 ± 3.70   Final old_old: 37.30 ± 3.74   Final old_new: 23.32 ± 3.70   Final new_new: 7.27 ± 3.37
Hits@30:
  Final val: 60.16 ± 5.43   Final Test: 42.36 ± 5.06   Final old_old: 45.51 ± 5.54   Final old_new: 29.80 ± 3.54   Final new_new: 12.27 ± 4.71
Hits@50:
  Final val: 69.11 ± 4.34   Final Test: 52.85 ± 1.44   Final old_old: 56.37 ± 1.82   Final old_new: 38.89 ± 1.02   Final new_new: 16.82 ± 1.24
AUC:
  Final val: 96.44 ± 0.27   Final Test: 95.71 ± 0.27   Final old_old: 96.44 ± 0.32   Final old_new: 92.71 ± 0.41   Final new_new: 89.48 ± 0.69
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.82 ± 4.18   Final Test: 36.46 ± 0.83   Final old_old: 39.28 ± 0.98   Final old_new: 25.26 ± 1.94   Final new_new: 7.73 ± 2.59
Hits@20:
  Final val: 59.81 ± 2.06   Final Test: 42.64 ± 4.32   Final old_old: 45.67 ± 4.56   Final old_new: 30.86 ± 3.76   Final new_new: 8.64 ± 1.90
Hits@30:
  Final val: 67.56 ± 1.47   Final Test: 52.12 ± 1.11   Final old_old: 55.70 ± 1.14   Final old_new: 38.04 ± 1.24   Final new_new: 13.64 ± 5.33
Hits@50:
  Final val: 75.39 ± 1.07   Final Test: 59.91 ± 2.68   Final old_old: 63.63 ± 2.76   Final old_new: 45.37 ± 2.62   Final new_new: 18.18 ± 4.25
AUC:
  Final val: 96.88 ± 0.18   Final Test: 96.42 ± 0.20   Final old_old: 97.03 ± 0.15   Final old_new: 94.06 ± 0.50   Final new_new: 89.75 ± 1.05
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.21 ± 4.54   Final Test: 26.12 ± 3.23   Final old_old: 28.28 ± 3.54   Final old_new: 17.46 ± 2.61   Final new_new: 5.91 ± 2.03
Hits@20:
  Final val: 53.01 ± 3.70   Final Test: 35.67 ± 2.35   Final old_old: 38.34 ± 2.74   Final old_new: 25.01 ± 1.75   Final new_new: 9.55 ± 5.18
Hits@30:
  Final val: 60.21 ± 3.92   Final Test: 44.31 ± 3.85   Final old_old: 47.47 ± 4.08   Final old_new: 31.76 ± 3.55   Final new_new: 12.73 ± 4.13
Hits@50:
  Final val: 69.30 ± 3.75   Final Test: 55.45 ± 4.06   Final old_old: 59.03 ± 4.26   Final old_new: 41.31 ± 4.08   Final new_new: 18.18 ± 3.59
AUC:
  Final val: 96.36 ± 0.45   Final Test: 95.90 ± 0.39   Final old_old: 96.52 ± 0.38   Final old_new: 93.40 ± 0.53   Final new_new: 90.92 ± 1.40
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.58 ± 5.22   Final Test: 31.33 ± 1.67   Final old_old: 34.61 ± 1.67   Final old_new: 18.09 ± 1.70   Final new_new: 2.73 ± 2.49
Hits@20:
  Final val: 59.94 ± 4.08   Final Test: 39.57 ± 3.02   Final old_old: 43.24 ± 3.30   Final old_new: 24.94 ± 2.05   Final new_new: 3.64 ± 2.03
Hits@30:
  Final val: 67.36 ± 3.16   Final Test: 50.09 ± 2.59   Final old_old: 54.25 ± 2.81   Final old_new: 33.63 ± 2.42   Final new_new: 7.27 ± 3.73
Hits@50:
  Final val: 75.18 ± 2.06   Final Test: 59.65 ± 1.08   Final old_old: 63.94 ± 1.22   Final old_new: 42.85 ± 0.76   Final new_new: 12.73 ± 5.47
AUC:
  Final val: 97.22 ± 0.28   Final Test: 96.58 ± 0.11   Final old_old: 97.35 ± 0.20   Final old_new: 93.53 ± 0.36   Final new_new: 89.30 ± 1.43
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.42 ± 4.91   Final Test: 30.45 ± 2.57   Final old_old: 33.82 ± 2.96   Final old_new: 16.75 ± 1.39   Final new_new: 2.73 ± 1.02
Hits@20:
  Final val: 61.94 ± 1.23   Final Test: 44.61 ± 4.28   Final old_old: 48.72 ± 4.45   Final old_new: 28.14 ± 4.30   Final new_new: 6.36 ± 1.90
Hits@30:
  Final val: 68.03 ± 0.49   Final Test: 52.16 ± 2.61   Final old_old: 56.50 ± 2.95   Final old_new: 35.04 ± 2.13   Final new_new: 6.82 ± 3.94
Hits@50:
  Final val: 76.01 ± 1.74   Final Test: 60.77 ± 3.79   Final old_old: 65.34 ± 3.98   Final old_new: 42.75 ± 3.35   Final new_new: 12.27 ± 2.03
AUC:
  Final val: 97.23 ± 0.25   Final Test: 96.62 ± 0.21   Final old_old: 97.38 ± 0.25   Final old_new: 93.59 ± 0.15   Final new_new: 88.97 ± 1.04
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 54.12 ± 2.76   Final Test: 41.48 ± 3.01   Final old_old: 45.06 ± 3.10   Final old_new: 27.18 ± 3.17   Final new_new: 7.73 ± 4.13
Hits@20:
  Final val: 68.89 ± 1.67   Final Test: 49.06 ± 3.82   Final old_old: 53.07 ± 3.90   Final old_new: 33.10 ± 3.47   Final new_new: 9.55 ± 5.88
Hits@30:
  Final val: 73.72 ± 1.35   Final Test: 56.81 ± 3.06   Final old_old: 61.01 ± 3.25   Final old_new: 40.23 ± 2.48   Final new_new: 12.73 ± 2.59
Hits@50:
  Final val: 79.36 ± 1.05   Final Test: 64.49 ± 3.14   Final old_old: 68.64 ± 3.03   Final old_new: 48.19 ± 3.88   Final new_new: 19.09 ± 3.45
AUC:
  Final val: 97.66 ± 0.09   Final Test: 97.21 ± 0.08   Final old_old: 97.72 ± 0.09   Final old_new: 95.16 ± 0.19   Final new_new: 91.86 ± 1.08
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.01 ± 2.28   Final Test: 33.22 ± 2.48   Final old_old: 36.50 ± 2.81   Final old_new: 20.00 ± 1.15   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 59.89 ± 2.54   Final Test: 45.68 ± 1.55   Final old_old: 49.51 ± 1.53   Final old_new: 30.50 ± 2.47   Final new_new: 7.27 ± 3.37
Hits@30:
  Final val: 65.94 ± 1.99   Final Test: 51.22 ± 2.02   Final old_old: 55.29 ± 2.24   Final old_new: 35.11 ± 1.83   Final new_new: 10.00 ± 3.80
Hits@50:
  Final val: 74.38 ± 1.22   Final Test: 60.55 ± 2.28   Final old_old: 64.76 ± 2.43   Final old_new: 43.88 ± 2.22   Final new_new: 17.27 ± 5.47
AUC:
  Final val: 97.12 ± 0.17   Final Test: 96.60 ± 0.13   Final old_old: 97.28 ± 0.18   Final old_new: 93.91 ± 0.09   Final new_new: 89.34 ± 1.94
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.42 ± 3.34   Final Test: 27.49 ± 2.69   Final old_old: 30.54 ± 2.85   Final old_new: 15.01 ± 2.13   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 57.87 ± 2.50   Final Test: 37.77 ± 5.08   Final old_old: 41.36 ± 5.53   Final old_new: 23.25 ± 3.45   Final new_new: 6.36 ± 2.96
Hits@30:
  Final val: 65.28 ± 1.60   Final Test: 44.58 ± 2.29   Final old_old: 48.37 ± 2.45   Final old_new: 29.47 ± 2.20   Final new_new: 7.73 ± 2.59
Hits@50:
  Final val: 72.63 ± 1.35   Final Test: 56.02 ± 3.00   Final old_old: 60.18 ± 2.99   Final old_new: 39.65 ± 3.24   Final new_new: 11.82 ± 3.73
AUC:
  Final val: 97.16 ± 0.15   Final Test: 96.49 ± 0.13   Final old_old: 97.17 ± 0.18   Final old_new: 93.77 ± 0.30   Final new_new: 89.42 ± 1.55
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.15 ± 3.44   Final Test: 27.59 ± 6.24   Final old_old: 30.69 ± 6.66   Final old_new: 14.99 ± 4.90   Final new_new: 2.27 ± 0.00
Hits@20:
  Final val: 58.24 ± 2.04   Final Test: 40.34 ± 5.80   Final old_old: 44.27 ± 6.43   Final old_new: 24.51 ± 3.38   Final new_new: 4.55 ± 1.61
Hits@30:
  Final val: 65.32 ± 1.63   Final Test: 46.96 ± 4.83   Final old_old: 51.02 ± 5.23   Final old_new: 30.86 ± 3.41   Final new_new: 6.82 ± 2.27
Hits@50:
  Final val: 73.81 ± 1.70   Final Test: 57.77 ± 3.40   Final old_old: 62.24 ± 3.49   Final old_new: 40.28 ± 3.21   Final new_new: 8.64 ± 2.49
AUC:
  Final val: 97.20 ± 0.18   Final Test: 96.53 ± 0.16   Final old_old: 97.25 ± 0.21   Final old_new: 93.69 ± 0.32   Final new_new: 88.95 ± 0.63
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.08 ± 3.68   Final Test: 32.60 ± 4.95   Final old_old: 35.77 ± 5.19   Final old_new: 19.82 ± 4.07   Final new_new: 4.55 ± 2.78
Hits@20:
  Final val: 65.48 ± 0.66   Final Test: 45.25 ± 4.31   Final old_old: 48.81 ± 4.49   Final old_new: 31.06 ± 3.62   Final new_new: 10.91 ± 4.07
Hits@30:
  Final val: 71.24 ± 0.74   Final Test: 53.91 ± 1.34   Final old_old: 57.73 ± 1.41   Final old_new: 38.77 ± 1.10   Final new_new: 15.00 ± 4.43
Hits@50:
  Final val: 77.47 ± 0.75   Final Test: 62.48 ± 2.72   Final old_old: 66.17 ± 2.74   Final old_new: 47.93 ± 2.99   Final new_new: 23.18 ± 6.89
AUC:
  Final val: 97.62 ± 0.06   Final Test: 97.24 ± 0.07   Final old_old: 97.68 ± 0.06   Final old_new: 95.49 ± 0.21   Final new_new: 92.24 ± 0.50
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.40 ± 2.53   Final Test: 26.72 ± 5.07   Final old_old: 29.70 ± 5.63   Final old_new: 14.53 ± 3.04   Final new_new: 3.18 ± 1.24
Hits@20:
  Final val: 57.91 ± 1.71   Final Test: 38.81 ± 5.32   Final old_old: 42.49 ± 5.73   Final old_new: 23.90 ± 3.98   Final new_new: 7.27 ± 2.49
Hits@30:
  Final val: 65.69 ± 1.31   Final Test: 44.68 ± 2.60   Final old_old: 48.53 ± 2.74   Final old_new: 29.35 ± 2.69   Final new_new: 6.36 ± 2.49
Hits@50:
  Final val: 73.10 ± 2.22   Final Test: 57.20 ± 1.84   Final old_old: 61.50 ± 2.06   Final old_new: 40.25 ± 1.82   Final new_new: 11.36 ± 5.33
AUC:
  Final val: 97.11 ± 0.09   Final Test: 96.47 ± 0.15   Final old_old: 97.17 ± 0.21   Final old_new: 93.72 ± 0.23   Final new_new: 88.48 ± 0.99
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 40.33 ± 2.79   Final Test: 27.57 ± 5.11   Final old_old: 30.04 ± 5.36   Final old_new: 17.66 ± 4.26   Final new_new: 5.00 ± 2.49
Hits@20:
  Final val: 51.75 ± 4.18   Final Test: 35.77 ± 2.57   Final old_old: 38.73 ± 2.88   Final old_new: 23.93 ± 1.59   Final new_new: 7.27 ± 2.96
Hits@30:
  Final val: 61.33 ± 3.46   Final Test: 44.46 ± 2.84   Final old_old: 47.58 ± 2.81   Final old_new: 32.19 ± 3.29   Final new_new: 11.36 ± 7.01
Hits@50:
  Final val: 69.85 ± 3.67   Final Test: 52.84 ± 1.00   Final old_old: 56.33 ± 0.93   Final old_new: 39.12 ± 2.14   Final new_new: 15.00 ± 4.13
AUC:
  Final val: 96.50 ± 0.31   Final Test: 95.74 ± 0.25   Final old_old: 96.43 ± 0.24   Final old_new: 92.95 ± 0.49   Final new_new: 88.99 ± 1.50
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.54 ± 4.48   Final Test: 30.06 ± 6.14   Final old_old: 32.51 ± 6.31   Final old_new: 20.20 ± 5.89   Final new_new: 7.27 ± 3.37
Hits@20:
  Final val: 53.19 ± 3.69   Final Test: 36.49 ± 5.21   Final old_old: 39.50 ± 5.51   Final old_new: 24.28 ± 4.17   Final new_new: 10.91 ± 5.88
Hits@30:
  Final val: 60.95 ± 4.55   Final Test: 42.17 ± 4.15   Final old_old: 45.37 ± 4.36   Final old_new: 29.29 ± 3.57   Final new_new: 12.73 ± 4.71
Hits@50:
  Final val: 69.58 ± 3.61   Final Test: 53.76 ± 4.15   Final old_old: 57.28 ± 4.33   Final old_new: 39.70 ± 4.03   Final new_new: 19.55 ± 6.55
AUC:
  Final val: 96.40 ± 0.41   Final Test: 95.81 ± 0.31   Final old_old: 96.52 ± 0.33   Final old_new: 93.01 ± 0.28   Final new_new: 88.83 ± 1.09
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.54 ± 4.15   Final Test: 30.82 ± 6.99   Final old_old: 33.46 ± 7.45   Final old_new: 20.23 ± 5.23   Final new_new: 6.36 ± 4.66
Hits@20:
  Final val: 60.08 ± 2.31   Final Test: 43.33 ± 3.52   Final old_old: 46.59 ± 3.66   Final old_new: 30.23 ± 3.00   Final new_new: 13.18 ± 4.07
Hits@30:
  Final val: 66.37 ± 0.78   Final Test: 50.46 ± 4.49   Final old_old: 54.05 ± 4.50   Final old_new: 36.05 ± 4.51   Final new_new: 17.27 ± 7.64
Hits@50:
  Final val: 73.68 ± 0.70   Final Test: 60.77 ± 3.42   Final old_old: 64.47 ± 3.39   Final old_new: 46.07 ± 3.69   Final new_new: 24.09 ± 5.47
AUC:
  Final val: 95.60 ± 0.16   Final Test: 95.03 ± 0.28   Final old_old: 95.63 ± 0.31   Final old_new: 92.76 ± 0.56   Final new_new: 86.55 ± 1.81
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 39.37 ± 5.25   Final Test: 27.29 ± 4.60   Final old_old: 29.85 ± 4.87   Final old_new: 16.83 ± 3.46   Final new_new: 7.27 ± 4.93
Hits@20:
  Final val: 51.15 ± 2.65   Final Test: 35.90 ± 2.78   Final old_old: 38.84 ± 3.25   Final old_new: 24.06 ± 1.45   Final new_new: 9.09 ± 4.25
Hits@30:
  Final val: 58.78 ± 1.92   Final Test: 43.59 ± 1.84   Final old_old: 46.86 ± 1.72   Final old_new: 30.58 ± 2.42   Final new_new: 11.82 ± 7.61
Hits@50:
  Final val: 69.63 ± 1.84   Final Test: 55.80 ± 2.08   Final old_old: 59.33 ± 2.05   Final old_new: 42.14 ± 2.62   Final new_new: 14.55 ± 3.80
AUC:
  Final val: 96.40 ± 0.22   Final Test: 95.80 ± 0.21   Final old_old: 96.52 ± 0.23   Final old_new: 92.92 ± 0.32   Final new_new: 89.26 ± 0.83
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.43 ± 2.15   Final Test: 30.75 ± 2.40   Final old_old: 33.94 ± 2.58   Final old_new: 17.76 ± 1.63   Final new_new: 4.55 ± 2.27
Hits@20:
  Final val: 61.27 ± 2.56   Final Test: 43.62 ± 4.95   Final old_old: 47.64 ± 5.29   Final old_new: 27.43 ± 3.71   Final new_new: 7.73 ± 2.59
Hits@30:
  Final val: 68.10 ± 2.04   Final Test: 50.17 ± 2.83   Final old_old: 54.48 ± 3.06   Final old_new: 32.95 ± 2.28   Final new_new: 9.09 ± 2.78
Hits@50:
  Final val: 75.13 ± 1.33   Final Test: 61.72 ± 2.88   Final old_old: 66.22 ± 3.16   Final old_new: 43.95 ± 2.37   Final new_new: 14.09 ± 2.49
AUC:
  Final val: 97.22 ± 0.14   Final Test: 96.60 ± 0.14   Final old_old: 97.38 ± 0.16   Final old_new: 93.47 ± 0.34   Final new_new: 89.21 ± 0.75
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.37 ± 3.56   Final Test: 33.33 ± 8.29   Final old_old: 36.65 ± 8.96   Final old_new: 19.92 ± 5.75   Final new_new: 3.64 ± 2.03
Hits@20:
  Final val: 60.71 ± 2.94   Final Test: 43.44 ± 6.89   Final old_old: 47.19 ± 7.42   Final old_new: 28.51 ± 4.88   Final new_new: 7.27 ± 1.90
Hits@30:
  Final val: 67.65 ± 2.20   Final Test: 49.68 ± 4.89   Final old_old: 53.79 ± 5.24   Final old_new: 33.38 ± 3.54   Final new_new: 7.73 ± 3.45
Hits@50:
  Final val: 75.61 ± 1.75   Final Test: 60.34 ± 4.40   Final old_old: 64.59 ± 4.69   Final old_new: 43.75 ± 3.54   Final new_new: 12.73 ± 3.45
AUC:
  Final val: 97.21 ± 0.17   Final Test: 96.58 ± 0.20   Final old_old: 97.32 ± 0.23   Final old_new: 93.61 ± 0.15   Final new_new: 89.23 ± 1.14
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 55.89 ± 3.47   Final Test: 35.37 ± 3.57   Final old_old: 38.68 ± 3.69   Final old_new: 21.94 ± 3.27   Final new_new: 7.73 ± 1.24
Hits@20:
  Final val: 65.23 ± 2.08   Final Test: 43.37 ± 2.54   Final old_old: 47.21 ± 2.55   Final old_new: 27.81 ± 2.72   Final new_new: 10.00 ± 3.80
Hits@30:
  Final val: 70.12 ± 1.88   Final Test: 50.31 ± 3.07   Final old_old: 54.12 ± 3.08   Final old_new: 35.19 ± 2.98   Final new_new: 12.73 ± 6.55
Hits@50:
  Final val: 75.76 ± 1.33   Final Test: 60.88 ± 2.30   Final old_old: 64.71 ± 2.30   Final old_new: 45.77 ± 2.75   Final new_new: 20.45 ± 6.63
AUC:
  Final val: 95.65 ± 0.17   Final Test: 95.22 ± 0.24   Final old_old: 95.85 ± 0.26   Final old_new: 92.79 ± 0.39   Final new_new: 87.59 ± 2.09
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.69 ± 2.43   Final Test: 33.13 ± 1.41   Final old_old: 36.66 ± 1.55   Final old_new: 18.77 ± 0.89   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 60.84 ± 1.47   Final Test: 44.02 ± 4.19   Final old_old: 47.89 ± 4.40   Final old_new: 28.54 ± 3.66   Final new_new: 7.73 ± 2.59
Hits@30:
  Final val: 67.61 ± 0.71   Final Test: 52.15 ± 0.49   Final old_old: 56.39 ± 0.49   Final old_new: 35.26 ± 1.64   Final new_new: 10.45 ± 2.59
Hits@50:
  Final val: 75.35 ± 1.19   Final Test: 62.33 ± 1.73   Final old_old: 66.81 ± 1.73   Final old_new: 44.69 ± 1.96   Final new_new: 15.45 ± 3.37
AUC:
  Final val: 97.26 ± 0.14   Final Test: 96.63 ± 0.10   Final old_old: 97.37 ± 0.12   Final old_new: 93.76 ± 0.19   Final new_new: 88.44 ± 1.31
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.42 ± 1.86   Final Test: 29.45 ± 3.58   Final old_old: 32.54 ± 3.92   Final old_new: 16.80 ± 2.79   Final new_new: 5.45 ± 2.03
Hits@20:
  Final val: 57.59 ± 2.47   Final Test: 38.69 ± 2.59   Final old_old: 42.29 ± 2.90   Final old_new: 24.18 ± 1.81   Final new_new: 5.91 ± 2.59
Hits@30:
  Final val: 64.59 ± 1.69   Final Test: 46.50 ± 1.77   Final old_old: 50.39 ± 1.92   Final old_new: 30.96 ± 1.70   Final new_new: 10.00 ± 2.03
Hits@50:
  Final val: 72.64 ± 1.71   Final Test: 56.56 ± 2.94   Final old_old: 60.78 ± 3.08   Final old_new: 39.70 ± 2.43   Final new_new: 15.45 ± 5.43
AUC:
  Final val: 97.17 ± 0.15   Final Test: 96.51 ± 0.12   Final old_old: 97.23 ± 0.19   Final old_new: 93.67 ± 0.25   Final new_new: 89.00 ± 1.48
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.05 ± 2.42   Final Test: 27.69 ± 3.35   Final old_old: 30.70 ± 3.60   Final old_new: 15.49 ± 2.47   Final new_new: 2.27 ± 1.61
Hits@20:
  Final val: 57.91 ± 3.34   Final Test: 39.90 ± 4.84   Final old_old: 43.55 ± 5.45   Final old_new: 25.34 ± 2.56   Final new_new: 4.55 ± 2.27
Hits@30:
  Final val: 65.31 ± 2.12   Final Test: 46.08 ± 5.10   Final old_old: 49.89 ± 5.40   Final old_new: 30.98 ± 4.05   Final new_new: 7.27 ± 1.90
Hits@50:
  Final val: 73.19 ± 1.68   Final Test: 57.22 ± 1.32   Final old_old: 61.53 ± 1.50   Final old_new: 40.05 ± 1.23   Final new_new: 15.00 ± 4.43
AUC:
  Final val: 97.15 ± 0.06   Final Test: 96.50 ± 0.13   Final old_old: 97.18 ± 0.15   Final old_new: 93.82 ± 0.22   Final new_new: 89.41 ± 1.44
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.78 ± 1.43   Final Test: 29.66 ± 3.61   Final old_old: 32.71 ± 3.78   Final old_new: 17.36 ± 3.18   Final new_new: 2.73 ± 2.49
Hits@20:
  Final val: 61.96 ± 1.70   Final Test: 40.95 ± 4.80   Final old_old: 44.26 ± 5.06   Final old_new: 27.78 ± 3.97   Final new_new: 7.73 ± 3.45
Hits@30:
  Final val: 67.21 ± 1.31   Final Test: 51.36 ± 2.63   Final old_old: 54.85 ± 2.66   Final old_new: 37.83 ± 2.76   Final new_new: 10.45 ± 4.98
Hits@50:
  Final val: 73.92 ± 1.35   Final Test: 60.33 ± 1.90   Final old_old: 63.73 ± 2.03   Final old_new: 47.30 ± 1.55   Final new_new: 17.73 ± 4.37
AUC:
  Final val: 95.31 ± 0.24   Final Test: 94.98 ± 0.06   Final old_old: 95.67 ± 0.18   Final old_new: 92.34 ± 0.53   Final new_new: 86.49 ± 3.04
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.17 ± 3.38   Final Test: 32.71 ± 4.20   Final old_old: 36.01 ± 4.51   Final old_new: 19.40 ± 3.05   Final new_new: 3.18 ± 2.59
Hits@20:
  Final val: 58.34 ± 2.93   Final Test: 42.71 ± 3.51   Final old_old: 46.47 ± 3.50   Final old_new: 27.66 ± 3.75   Final new_new: 6.82 ± 2.27
Hits@30:
  Final val: 65.05 ± 1.71   Final Test: 48.44 ± 2.96   Final old_old: 52.52 ± 3.15   Final old_new: 32.07 ± 2.55   Final new_new: 10.45 ± 3.05
Hits@50:
  Final val: 72.99 ± 1.44   Final Test: 55.61 ± 2.77   Final old_old: 59.87 ± 2.66   Final old_new: 38.69 ± 3.64   Final new_new: 13.18 ± 4.07
AUC:
  Final val: 97.18 ± 0.13   Final Test: 96.54 ± 0.09   Final old_old: 97.25 ± 0.10   Final old_new: 93.72 ± 0.19   Final new_new: 89.52 ± 0.73
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.02 ± 6.04   Final Test: 24.48 ± 1.89   Final old_old: 26.69 ± 2.08   Final old_new: 15.59 ± 1.75   Final new_new: 4.55 ± 2.27
Hits@20:
  Final val: 52.68 ± 4.81   Final Test: 32.97 ± 3.74   Final old_old: 35.83 ± 3.94   Final old_new: 21.44 ± 3.28   Final new_new: 7.27 ± 2.96
Hits@30:
  Final val: 60.30 ± 4.03   Final Test: 41.18 ± 4.86   Final old_old: 44.29 ± 5.08   Final old_new: 28.89 ± 4.09   Final new_new: 9.55 ± 3.37
Hits@50:
  Final val: 69.43 ± 2.51   Final Test: 55.21 ± 3.74   Final old_old: 58.73 ± 4.10   Final old_new: 41.28 ± 2.68   Final new_new: 19.09 ± 6.14
AUC:
  Final val: 96.42 ± 0.34   Final Test: 95.73 ± 0.28   Final old_old: 96.44 ± 0.27   Final old_new: 92.86 ± 0.44   Final new_new: 89.43 ± 1.37
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.63 ± 6.00   Final Test: 26.05 ± 3.67   Final old_old: 28.51 ± 3.99   Final old_new: 15.99 ± 2.52   Final new_new: 6.82 ± 1.61
Hits@20:
  Final val: 53.56 ± 5.46   Final Test: 39.12 ± 2.67   Final old_old: 42.42 ± 2.85   Final old_new: 25.77 ± 2.13   Final new_new: 10.45 ± 2.59
Hits@30:
  Final val: 60.95 ± 3.60   Final Test: 43.52 ± 3.74   Final old_old: 46.97 ± 3.88   Final old_new: 29.65 ± 3.34   Final new_new: 11.36 ± 3.21
Hits@50:
  Final val: 69.57 ± 2.91   Final Test: 56.49 ± 2.27   Final old_old: 60.24 ± 2.34   Final old_new: 41.61 ± 2.44   Final new_new: 18.64 ± 1.90
AUC:
  Final val: 96.50 ± 0.33   Final Test: 95.81 ± 0.21   Final old_old: 96.49 ± 0.27   Final old_new: 93.13 ± 0.18   Final new_new: 88.88 ± 1.12
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 39.21 ± 1.81   Final Test: 25.59 ± 4.07   Final old_old: 27.79 ± 4.31   Final old_new: 16.68 ± 3.38   Final new_new: 6.82 ± 1.61
Hits@20:
  Final val: 50.68 ± 1.26   Final Test: 37.57 ± 2.01   Final old_old: 40.47 ± 2.25   Final old_new: 25.99 ± 1.23   Final new_new: 9.55 ± 6.31
Hits@30:
  Final val: 58.70 ± 1.62   Final Test: 43.18 ± 5.50   Final old_old: 46.40 ± 5.65   Final old_new: 30.45 ± 5.01   Final new_new: 10.45 ± 4.71
Hits@50:
  Final val: 69.14 ± 1.32   Final Test: 53.16 ± 2.36   Final old_old: 56.74 ± 2.36   Final old_new: 39.09 ± 2.62   Final new_new: 15.00 ± 1.24
AUC:
  Final val: 96.41 ± 0.26   Final Test: 95.58 ± 0.22   Final old_old: 96.37 ± 0.24   Final old_new: 92.38 ± 0.31   Final new_new: 88.47 ± 0.70
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.42 ± 4.20   Final Test: 33.74 ± 4.52   Final old_old: 37.14 ± 5.07   Final old_new: 19.92 ± 2.36   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 58.65 ± 2.96   Final Test: 41.78 ± 2.83   Final old_old: 45.34 ± 3.04   Final old_new: 27.38 ± 2.08   Final new_new: 10.45 ± 3.80
Hits@30:
  Final val: 66.20 ± 2.56   Final Test: 49.10 ± 1.43   Final old_old: 52.95 ± 1.52   Final old_new: 33.73 ± 1.62   Final new_new: 11.82 ± 1.90
Hits@50:
  Final val: 75.27 ± 2.10   Final Test: 59.91 ± 1.92   Final old_old: 64.15 ± 2.21   Final old_new: 43.07 ± 1.35   Final new_new: 17.27 ± 6.14
AUC:
  Final val: 97.18 ± 0.26   Final Test: 96.58 ± 0.26   Final old_old: 97.31 ± 0.31   Final old_new: 93.62 ± 0.28   Final new_new: 90.42 ± 2.07
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.69 ± 1.43   Final Test: 32.35 ± 4.02   Final old_old: 35.24 ± 4.55   Final old_new: 20.63 ± 2.10   Final new_new: 8.18 ± 2.03
Hits@20:
  Final val: 59.21 ± 1.97   Final Test: 39.12 ± 4.46   Final old_old: 42.52 ± 4.80   Final old_new: 25.26 ± 3.26   Final new_new: 11.82 ± 2.49
Hits@30:
  Final val: 64.71 ± 1.75   Final Test: 46.89 ± 4.03   Final old_old: 50.38 ± 4.07   Final old_new: 32.92 ± 4.00   Final new_new: 14.09 ± 2.96
Hits@50:
  Final val: 70.77 ± 0.89   Final Test: 55.24 ± 3.22   Final old_old: 58.86 ± 3.06   Final old_new: 40.88 ± 4.10   Final new_new: 18.64 ± 3.37
AUC:
  Final val: 94.90 ± 0.39   Final Test: 94.48 ± 0.36   Final old_old: 95.14 ± 0.39   Final old_new: 91.96 ± 0.46   Final new_new: 85.87 ± 2.13
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.65 ± 4.68   Final Test: 30.89 ± 2.96   Final old_old: 34.23 ± 3.31   Final old_new: 17.38 ± 1.79   Final new_new: 2.27 ± 1.61
Hits@20:
  Final val: 61.67 ± 3.57   Final Test: 44.43 ± 5.40   Final old_old: 48.49 ± 5.96   Final old_new: 28.24 ± 3.55   Final new_new: 4.55 ± 1.61
Hits@30:
  Final val: 67.88 ± 2.91   Final Test: 51.61 ± 3.79   Final old_old: 55.86 ± 4.40   Final old_new: 34.66 ± 1.96   Final new_new: 10.00 ± 4.13
Hits@50:
  Final val: 74.93 ± 2.75   Final Test: 58.91 ± 5.28   Final old_old: 63.26 ± 5.74   Final old_new: 41.69 ± 3.58   Final new_new: 14.09 ± 4.93
AUC:
  Final val: 97.16 ± 0.23   Final Test: 96.51 ± 0.27   Final old_old: 97.23 ± 0.32   Final old_new: 93.66 ± 0.28   Final new_new: 89.93 ± 0.73
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.05 ± 2.16   Final Test: 30.38 ± 4.89   Final old_old: 33.33 ± 5.35   Final old_new: 18.54 ± 3.15   Final new_new: 2.73 ± 2.96
Hits@20:
  Final val: 62.05 ± 1.49   Final Test: 44.83 ± 2.72   Final old_old: 48.26 ± 2.74   Final old_new: 31.26 ± 3.08   Final new_new: 9.55 ± 3.37
Hits@30:
  Final val: 66.99 ± 1.09   Final Test: 51.80 ± 2.80   Final old_old: 55.43 ± 2.79   Final old_new: 37.43 ± 2.93   Final new_new: 15.00 ± 5.23
Hits@50:
  Final val: 72.79 ± 0.58   Final Test: 61.63 ± 1.94   Final old_old: 65.27 ± 2.01   Final old_new: 47.43 ± 1.94   Final new_new: 20.91 ± 2.49
AUC:
  Final val: 94.88 ± 0.17   Final Test: 94.60 ± 0.17   Final old_old: 95.23 ± 0.15   Final old_new: 92.14 ± 0.47   Final new_new: 87.33 ± 0.75
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.43 ± 1.69   Final Test: 28.45 ± 5.63   Final old_old: 31.52 ± 6.03   Final old_new: 15.94 ± 4.19   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 58.66 ± 2.11   Final Test: 39.80 ± 2.31   Final old_old: 43.23 ± 2.21   Final old_new: 26.07 ± 2.88   Final new_new: 7.27 ± 2.49
Hits@30:
  Final val: 65.42 ± 2.53   Final Test: 46.09 ± 2.98   Final old_old: 49.86 ± 3.17   Final old_new: 31.06 ± 2.47   Final new_new: 9.09 ± 1.61
Hits@50:
  Final val: 73.30 ± 1.29   Final Test: 56.30 ± 4.25   Final old_old: 60.38 ± 4.61   Final old_new: 40.03 ± 3.07   Final new_new: 16.82 ± 4.98
AUC:
  Final val: 97.13 ± 0.16   Final Test: 96.54 ± 0.12   Final old_old: 97.21 ± 0.18   Final old_new: 93.94 ± 0.24   Final new_new: 89.18 ± 0.97
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.19 ± 2.65   Final Test: 34.32 ± 4.89   Final old_old: 37.93 ± 5.19   Final old_new: 19.65 ± 4.10   Final new_new: 4.55 ± 2.78
Hits@20:
  Final val: 62.72 ± 3.63   Final Test: 45.89 ± 6.21   Final old_old: 49.86 ± 6.65   Final old_new: 29.97 ± 4.90   Final new_new: 8.18 ± 4.43
Hits@30:
  Final val: 67.89 ± 2.98   Final Test: 53.69 ± 2.65   Final old_old: 57.97 ± 3.12   Final old_new: 36.68 ± 1.06   Final new_new: 11.36 ± 4.25
Hits@50:
  Final val: 75.52 ± 2.09   Final Test: 62.36 ± 4.09   Final old_old: 66.77 ± 4.27   Final old_new: 44.89 ± 3.58   Final new_new: 17.27 ± 4.71
AUC:
  Final val: 97.28 ± 0.16   Final Test: 96.61 ± 0.11   Final old_old: 97.39 ± 0.21   Final old_new: 93.55 ± 0.32   Final new_new: 87.79 ± 1.75
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.88 ± 3.25   Final Test: 30.58 ± 2.61   Final old_old: 33.69 ± 2.99   Final old_new: 18.01 ± 1.14   Final new_new: 3.64 ± 2.03
Hits@20:
  Final val: 60.09 ± 1.60   Final Test: 40.40 ± 1.75   Final old_old: 44.14 ± 1.84   Final old_new: 25.31 ± 1.79   Final new_new: 6.82 ± 2.78
Hits@30:
  Final val: 66.30 ± 1.13   Final Test: 47.05 ± 1.82   Final old_old: 50.98 ± 1.98   Final old_new: 31.44 ± 1.98   Final new_new: 8.18 ± 3.05
Hits@50:
  Final val: 73.43 ± 0.85   Final Test: 57.32 ± 1.33   Final old_old: 61.66 ± 1.35   Final old_new: 40.23 ± 1.80   Final new_new: 11.36 ± 4.25
AUC:
  Final val: 97.22 ± 0.10   Final Test: 96.58 ± 0.07   Final old_old: 97.30 ± 0.10   Final old_new: 93.72 ± 0.14   Final new_new: 89.01 ± 1.47
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.28 ± 2.01   Final Test: 31.75 ± 3.46   Final old_old: 35.05 ± 3.48   Final old_new: 18.34 ± 3.63   Final new_new: 4.09 ± 2.96
Hits@20:
  Final val: 58.63 ± 1.69   Final Test: 43.77 ± 2.79   Final old_old: 47.23 ± 2.78   Final old_new: 30.03 ± 2.86   Final new_new: 9.55 ± 5.43
Hits@30:
  Final val: 65.04 ± 1.26   Final Test: 50.49 ± 3.73   Final old_old: 54.11 ± 3.67   Final old_new: 36.10 ± 4.06   Final new_new: 14.55 ± 7.29
Hits@50:
  Final val: 71.99 ± 0.64   Final Test: 60.02 ± 1.41   Final old_old: 63.48 ± 1.35   Final old_new: 46.52 ± 2.46   Final new_new: 21.36 ± 2.03
AUC:
  Final val: 94.66 ± 0.13   Final Test: 94.35 ± 0.13   Final old_old: 95.02 ± 0.14   Final old_new: 91.77 ± 0.60   Final new_new: 85.60 ± 1.14
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 39.68 ± 4.65   Final Test: 24.06 ± 5.55   Final old_old: 26.22 ± 5.98   Final old_new: 15.29 ± 4.13   Final new_new: 5.45 ± 2.59
Hits@20:
  Final val: 51.17 ± 5.00   Final Test: 36.61 ± 2.68   Final old_old: 39.57 ± 2.90   Final old_new: 24.79 ± 2.92   Final new_new: 8.18 ± 4.43
Hits@30:
  Final val: 58.67 ± 3.96   Final Test: 43.65 ± 3.94   Final old_old: 47.12 ± 4.16   Final old_new: 29.92 ± 3.57   Final new_new: 7.73 ± 1.24
Hits@50:
  Final val: 68.78 ± 2.99   Final Test: 54.38 ± 1.21   Final old_old: 58.02 ± 1.17   Final old_new: 40.18 ± 2.31   Final new_new: 13.18 ± 5.88
AUC:
  Final val: 96.46 ± 0.42   Final Test: 95.77 ± 0.24   Final old_old: 96.52 ± 0.29   Final old_new: 92.75 ± 0.24   Final new_new: 89.53 ± 1.46
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.68 ± 2.13   Final Test: 33.27 ± 3.61   Final old_old: 36.62 ± 3.88   Final old_new: 19.70 ± 2.66   Final new_new: 4.55 ± 2.27
Hits@20:
  Final val: 58.49 ± 2.40   Final Test: 42.72 ± 4.24   Final old_old: 46.41 ± 4.31   Final old_new: 27.93 ± 4.15   Final new_new: 8.18 ± 3.80
Hits@30:
  Final val: 65.55 ± 2.30   Final Test: 49.22 ± 3.02   Final old_old: 53.15 ± 3.16   Final old_new: 33.63 ± 2.65   Final new_new: 10.00 ± 3.45
Hits@50:
  Final val: 73.62 ± 1.74   Final Test: 57.27 ± 2.93   Final old_old: 61.44 ± 2.81   Final old_new: 40.68 ± 3.43   Final new_new: 15.91 ± 5.57
AUC:
  Final val: 97.18 ± 0.10   Final Test: 96.60 ± 0.10   Final old_old: 97.29 ± 0.11   Final old_new: 93.89 ± 0.22   Final new_new: 89.18 ± 1.54
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 40.61 ± 5.50   Final Test: 25.12 ± 5.80   Final old_old: 27.31 ± 5.94   Final old_new: 16.35 ± 5.49   Final new_new: 4.55 ± 3.21
Hits@20:
  Final val: 51.83 ± 4.96   Final Test: 33.51 ± 6.00   Final old_old: 36.14 ± 6.22   Final old_new: 23.02 ± 5.38   Final new_new: 7.73 ± 3.45
Hits@30:
  Final val: 59.69 ± 3.43   Final Test: 41.31 ± 4.53   Final old_old: 44.50 ± 4.84   Final old_new: 28.64 ± 3.53   Final new_new: 9.55 ± 6.10
Hits@50:
  Final val: 68.91 ± 2.31   Final Test: 51.76 ± 1.17   Final old_old: 55.37 ± 1.30   Final old_new: 37.48 ± 1.22   Final new_new: 14.55 ± 5.93
AUC:
  Final val: 96.47 ± 0.23   Final Test: 95.82 ± 0.10   Final old_old: 96.50 ± 0.12   Final old_new: 93.08 ± 0.09   Final new_new: 89.94 ± 1.45
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.52 ± 1.58   Final Test: 30.82 ± 3.54   Final old_old: 33.55 ± 3.88   Final old_new: 19.75 ± 2.27   Final new_new: 7.73 ± 3.45
Hits@20:
  Final val: 62.79 ± 1.08   Final Test: 41.98 ± 4.40   Final old_old: 45.32 ± 4.67   Final old_new: 28.44 ± 3.34   Final new_new: 13.64 ± 4.55
Hits@30:
  Final val: 68.64 ± 1.29   Final Test: 52.29 ± 1.44   Final old_old: 55.93 ± 1.37   Final old_new: 37.81 ± 2.37   Final new_new: 16.36 ± 5.66
Hits@50:
  Final val: 75.70 ± 0.72   Final Test: 57.78 ± 2.95   Final old_old: 61.65 ± 2.98   Final old_new: 42.54 ± 3.23   Final new_new: 17.27 ± 2.59
AUC:
  Final val: 96.65 ± 0.17   Final Test: 96.17 ± 0.24   Final old_old: 96.74 ± 0.25   Final old_new: 93.97 ± 0.37   Final new_new: 89.94 ± 0.94
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 38.41 ± 2.82   Final Test: 24.49 ± 4.57   Final old_old: 26.47 ± 4.71   Final old_new: 16.47 ± 4.19   Final new_new: 7.27 ± 4.37
Hits@20:
  Final val: 50.72 ± 2.65   Final Test: 33.95 ± 5.52   Final old_old: 36.74 ± 5.73   Final old_new: 22.77 ± 4.89   Final new_new: 8.18 ± 3.05
Hits@30:
  Final val: 58.69 ± 1.64   Final Test: 40.97 ± 3.16   Final old_old: 44.17 ± 3.26   Final old_new: 28.21 ± 2.98   Final new_new: 10.00 ± 1.24
Hits@50:
  Final val: 68.20 ± 1.52   Final Test: 51.44 ± 2.56   Final old_old: 55.09 ± 2.90   Final old_new: 37.05 ± 1.22   Final new_new: 13.18 ± 1.90
AUC:
  Final val: 96.32 ± 0.25   Final Test: 95.62 ± 0.18   Final old_old: 96.35 ± 0.20   Final old_new: 92.69 ± 0.40   Final new_new: 88.60 ± 1.45
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.06 ± 5.54   Final Test: 34.75 ± 4.26   Final old_old: 38.37 ± 4.72   Final old_new: 20.03 ± 2.67   Final new_new: 4.09 ± 1.02
Hits@20:
  Final val: 61.50 ± 4.12   Final Test: 45.37 ± 4.46   Final old_old: 49.38 ± 4.84   Final old_new: 29.29 ± 3.29   Final new_new: 7.73 ± 2.59
Hits@30:
  Final val: 67.65 ± 2.39   Final Test: 51.50 ± 3.48   Final old_old: 55.65 ± 3.71   Final old_new: 35.04 ± 2.72   Final new_new: 10.00 ± 1.24
Hits@50:
  Final val: 75.46 ± 2.01   Final Test: 61.15 ± 2.22   Final old_old: 65.48 ± 2.44   Final old_new: 44.11 ± 1.50   Final new_new: 14.55 ± 3.80
AUC:
  Final val: 97.30 ± 0.23   Final Test: 96.64 ± 0.23   Final old_old: 97.40 ± 0.23   Final old_new: 93.63 ± 0.36   Final new_new: 88.87 ± 0.58
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.92 ± 6.64   Final Test: 31.23 ± 2.10   Final old_old: 34.34 ± 1.94   Final old_new: 18.69 ± 3.22   Final new_new: 3.18 ± 2.03
Hits@20:
  Final val: 60.08 ± 5.56   Final Test: 41.93 ± 5.35   Final old_old: 45.92 ± 5.90   Final old_new: 25.89 ± 3.61   Final new_new: 5.91 ± 3.05
Hits@30:
  Final val: 66.91 ± 4.19   Final Test: 51.99 ± 4.30   Final old_old: 56.19 ± 4.77   Final old_new: 35.31 ± 2.59   Final new_new: 10.45 ± 3.05
Hits@50:
  Final val: 74.65 ± 2.70   Final Test: 60.50 ± 3.89   Final old_old: 64.83 ± 4.30   Final old_new: 43.35 ± 2.35   Final new_new: 16.36 ± 2.96
AUC:
  Final val: 97.18 ± 0.22   Final Test: 96.59 ± 0.11   Final old_old: 97.34 ± 0.22   Final old_new: 93.59 ± 0.42   Final new_new: 88.97 ± 2.63
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.04 ± 3.06   Final Test: 32.94 ± 2.35   Final old_old: 35.99 ± 2.53   Final old_new: 20.65 ± 1.82   Final new_new: 5.00 ± 1.90
Hits@20:
  Final val: 64.97 ± 2.19   Final Test: 43.80 ± 1.68   Final old_old: 47.13 ± 1.73   Final old_new: 30.71 ± 1.74   Final new_new: 8.18 ± 2.59
Hits@30:
  Final val: 71.45 ± 1.87   Final Test: 51.88 ± 2.13   Final old_old: 55.46 ± 2.35   Final old_new: 37.83 ± 1.55   Final new_new: 12.27 ± 4.13
Hits@50:
  Final val: 78.02 ± 1.87   Final Test: 61.97 ± 2.17   Final old_old: 65.56 ± 2.22   Final old_new: 48.09 ± 2.13   Final new_new: 19.55 ± 3.05
AUC:
  Final val: 97.25 ± 0.10   Final Test: 96.86 ± 0.11   Final old_old: 97.33 ± 0.10   Final old_new: 95.06 ± 0.30   Final new_new: 90.74 ± 0.83
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.51 ± 2.56   Final Test: 30.57 ± 5.26   Final old_old: 33.55 ± 5.75   Final old_new: 18.46 ± 3.48   Final new_new: 5.45 ± 1.24
Hits@20:
  Final val: 58.22 ± 1.37   Final Test: 37.91 ± 2.84   Final old_old: 41.36 ± 3.08   Final old_new: 24.13 ± 2.00   Final new_new: 5.00 ± 2.49
Hits@30:
  Final val: 64.64 ± 1.58   Final Test: 44.92 ± 1.80   Final old_old: 48.59 ± 1.77   Final old_new: 30.30 ± 2.15   Final new_new: 8.64 ± 4.93
Hits@50:
  Final val: 72.62 ± 1.27   Final Test: 55.39 ± 1.68   Final old_old: 59.52 ± 1.86   Final old_new: 39.07 ± 1.16   Final new_new: 12.27 ± 3.45
AUC:
  Final val: 97.04 ± 0.14   Final Test: 96.51 ± 0.10   Final old_old: 97.19 ± 0.12   Final old_new: 93.81 ± 0.14   Final new_new: 89.43 ± 0.99
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.35 ± 2.69   Final Test: 34.00 ± 5.75   Final old_old: 37.56 ± 6.10   Final old_new: 19.50 ± 4.50   Final new_new: 5.00 ± 3.73
Hits@20:
  Final val: 61.31 ± 1.26   Final Test: 46.17 ± 3.46   Final old_old: 50.39 ± 3.71   Final old_new: 29.19 ± 2.85   Final new_new: 7.27 ± 2.96
Hits@30:
  Final val: 67.31 ± 0.53   Final Test: 50.09 ± 2.75   Final old_old: 54.43 ± 2.73   Final old_new: 32.72 ± 2.98   Final new_new: 9.55 ± 5.18
Hits@50:
  Final val: 75.78 ± 1.77   Final Test: 62.27 ± 0.81   Final old_old: 66.90 ± 0.99   Final old_new: 43.88 ± 0.68   Final new_new: 15.91 ± 5.33
AUC:
  Final val: 97.29 ± 0.17   Final Test: 96.69 ± 0.11   Final old_old: 97.45 ± 0.12   Final old_new: 93.77 ± 0.16   Final new_new: 87.90 ± 1.35
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.70 ± 3.24   Final Test: 33.93 ± 3.81   Final old_old: 36.92 ± 3.97   Final old_new: 22.12 ± 3.33   Final new_new: 3.64 ± 3.45
Hits@20:
  Final val: 62.81 ± 3.58   Final Test: 46.13 ± 3.55   Final old_old: 49.44 ± 3.59   Final old_new: 33.05 ± 3.64   Final new_new: 11.36 ± 4.25
Hits@30:
  Final val: 69.09 ± 1.86   Final Test: 54.19 ± 3.04   Final old_old: 57.57 ± 2.96   Final old_new: 41.08 ± 3.39   Final new_new: 15.00 ± 5.47
Hits@50:
  Final val: 76.54 ± 1.05   Final Test: 62.04 ± 1.15   Final old_old: 65.80 ± 1.11   Final old_new: 47.43 ± 1.84   Final new_new: 18.64 ± 4.66
AUC:
  Final val: 97.40 ± 0.13   Final Test: 97.03 ± 0.02   Final old_old: 97.50 ± 0.05   Final old_new: 95.19 ± 0.20   Final new_new: 91.78 ± 0.73
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.15 ± 3.81   Final Test: 31.78 ± 0.88   Final old_old: 35.08 ± 0.86   Final old_new: 18.29 ± 1.83   Final new_new: 5.45 ± 3.80
Hits@20:
  Final val: 60.29 ± 1.52   Final Test: 42.14 ± 2.15   Final old_old: 45.89 ± 2.09   Final old_new: 27.03 ± 2.90   Final new_new: 8.64 ± 5.88
Hits@30:
  Final val: 65.97 ± 1.81   Final Test: 47.42 ± 1.15   Final old_old: 51.68 ± 1.15   Final old_new: 30.30 ± 2.06   Final new_new: 9.09 ± 5.79
Hits@50:
  Final val: 73.84 ± 1.50   Final Test: 59.69 ± 1.82   Final old_old: 64.10 ± 1.81   Final old_new: 42.07 ± 2.81   Final new_new: 17.27 ± 3.80
AUC:
  Final val: 97.17 ± 0.15   Final Test: 96.63 ± 0.06   Final old_old: 97.31 ± 0.09   Final old_new: 93.89 ± 0.17   Final new_new: 89.94 ± 1.36
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 39.81 ± 3.81   Final Test: 30.56 ± 5.11   Final old_old: 33.06 ± 5.59   Final old_new: 20.55 ± 3.75   Final new_new: 6.82 ± 3.94
Hits@20:
  Final val: 52.55 ± 2.58   Final Test: 36.11 ± 4.54   Final old_old: 38.89 ± 5.09   Final old_new: 25.06 ± 2.78   Final new_new: 8.18 ± 5.47
Hits@30:
  Final val: 59.98 ± 2.68   Final Test: 43.60 ± 5.15   Final old_old: 46.72 ± 5.59   Final old_new: 31.26 ± 3.74   Final new_new: 11.82 ± 5.66
Hits@50:
  Final val: 69.23 ± 1.61   Final Test: 55.06 ± 3.89   Final old_old: 58.53 ± 4.25   Final old_new: 41.49 ± 2.37   Final new_new: 16.82 ± 6.55
AUC:
  Final val: 96.35 ± 0.34   Final Test: 95.77 ± 0.25   Final old_old: 96.49 ± 0.29   Final old_new: 92.85 ± 0.20   Final new_new: 89.91 ± 1.06
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.80 ± 3.66   Final Test: 29.56 ± 5.17   Final old_old: 32.45 ± 5.47   Final old_new: 17.86 ± 4.15   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 58.23 ± 3.36   Final Test: 40.05 ± 3.52   Final old_old: 43.65 ± 3.71   Final old_new: 25.69 ± 3.09   Final new_new: 5.91 ± 2.59
Hits@30:
  Final val: 65.54 ± 1.80   Final Test: 48.03 ± 2.65   Final old_old: 52.02 ± 2.56   Final old_new: 32.09 ± 3.14   Final new_new: 9.55 ± 4.07
Hits@50:
  Final val: 72.90 ± 1.09   Final Test: 56.36 ± 1.54   Final old_old: 60.56 ± 1.62   Final old_new: 39.77 ± 1.58   Final new_new: 12.73 ± 3.80
AUC:
  Final val: 97.18 ± 0.15   Final Test: 96.55 ± 0.04   Final old_old: 97.21 ± 0.09   Final old_new: 93.91 ± 0.24   Final new_new: 89.69 ± 0.80
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 37.61 ± 5.08   Final Test: 24.59 ± 4.61   Final old_old: 26.62 ± 4.84   Final old_new: 16.35 ± 3.92   Final new_new: 7.27 ± 2.96
Hits@20:
  Final val: 48.91 ± 3.45   Final Test: 32.31 ± 3.86   Final old_old: 34.91 ± 3.97   Final old_new: 21.91 ± 3.69   Final new_new: 8.18 ± 4.13
Hits@30:
  Final val: 58.04 ± 2.80   Final Test: 39.44 ± 3.15   Final old_old: 42.47 ± 3.20   Final old_new: 27.36 ± 3.50   Final new_new: 10.00 ± 3.45
Hits@50:
  Final val: 67.64 ± 2.69   Final Test: 52.06 ± 3.42   Final old_old: 55.71 ± 3.52   Final old_new: 37.53 ± 3.82   Final new_new: 16.82 ± 6.93
AUC:
  Final val: 96.35 ± 0.26   Final Test: 95.71 ± 0.19   Final old_old: 96.40 ± 0.19   Final old_new: 92.93 ± 0.52   Final new_new: 89.75 ± 0.92
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.78 ± 3.87   Final Test: 25.09 ± 6.27   Final old_old: 27.14 ± 6.54   Final old_new: 16.78 ± 5.46   Final new_new: 7.73 ± 2.03
Hits@20:
  Final val: 61.77 ± 2.87   Final Test: 37.63 ± 3.55   Final old_old: 40.55 ± 3.74   Final old_new: 25.94 ± 3.07   Final new_new: 10.00 ± 3.05
Hits@30:
  Final val: 68.11 ± 2.45   Final Test: 48.45 ± 2.06   Final old_old: 51.91 ± 2.10   Final old_new: 34.66 ± 2.00   Final new_new: 14.55 ± 4.71
Hits@50:
  Final val: 75.79 ± 1.68   Final Test: 60.06 ± 4.77   Final old_old: 63.82 ± 4.86   Final old_new: 45.14 ± 4.48   Final new_new: 22.27 ± 4.07
AUC:
  Final val: 96.88 ± 0.06   Final Test: 96.28 ± 0.28   Final old_old: 96.86 ± 0.27   Final old_new: 94.02 ± 0.40   Final new_new: 90.29 ± 0.93
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.53 ± 4.81   Final Test: 36.49 ± 4.76   Final old_old: 40.22 ± 5.22   Final old_new: 21.41 ± 3.04   Final new_new: 4.55 ± 3.59
Hits@20:
  Final val: 61.16 ± 3.54   Final Test: 45.15 ± 6.11   Final old_old: 49.18 ± 6.52   Final old_new: 28.97 ± 4.63   Final new_new: 8.18 ± 4.13
Hits@30:
  Final val: 66.64 ± 2.76   Final Test: 51.65 ± 4.95   Final old_old: 55.94 ± 5.06   Final old_new: 34.56 ± 4.65   Final new_new: 10.00 ± 5.23
Hits@50:
  Final val: 75.14 ± 2.20   Final Test: 61.41 ± 2.01   Final old_old: 65.95 ± 2.11   Final old_new: 43.43 ± 2.05   Final new_new: 15.45 ± 4.07
AUC:
  Final val: 97.21 ± 0.31   Final Test: 96.59 ± 0.26   Final old_old: 97.33 ± 0.33   Final old_new: 93.67 ± 0.25   Final new_new: 89.52 ± 0.82
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 40.88 ± 2.79   Final Test: 28.21 ± 1.95   Final old_old: 30.62 ± 2.22   Final old_new: 18.56 ± 1.20   Final new_new: 5.00 ± 2.49
Hits@20:
  Final val: 53.10 ± 3.16   Final Test: 33.37 ± 1.47   Final old_old: 36.20 ± 1.41   Final old_new: 21.91 ± 2.27   Final new_new: 9.09 ± 2.78
Hits@30:
  Final val: 60.50 ± 4.12   Final Test: 44.45 ± 3.78   Final old_old: 47.90 ± 4.12   Final old_new: 30.71 ± 2.90   Final new_new: 10.91 ± 1.90
Hits@50:
  Final val: 68.94 ± 2.53   Final Test: 53.38 ± 2.16   Final old_old: 57.21 ± 2.34   Final old_new: 38.16 ± 1.97   Final new_new: 15.45 ± 4.93
AUC:
  Final val: 96.36 ± 0.32   Final Test: 95.81 ± 0.25   Final old_old: 96.50 ± 0.25   Final old_new: 93.02 ± 0.37   Final new_new: 89.61 ± 1.01
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.11 ± 4.17   Final Test: 31.72 ± 2.98   Final old_old: 35.02 ± 3.29   Final old_new: 18.36 ± 1.88   Final new_new: 3.64 ± 2.03
Hits@20:
  Final val: 60.40 ± 3.65   Final Test: 43.42 ± 4.94   Final old_old: 47.21 ± 5.24   Final old_new: 28.29 ± 3.85   Final new_new: 6.82 ± 4.25
Hits@30:
  Final val: 66.87 ± 2.21   Final Test: 51.52 ± 2.40   Final old_old: 55.74 ± 2.63   Final old_new: 34.63 ± 1.77   Final new_new: 11.82 ± 2.96
Hits@50:
  Final val: 74.82 ± 1.45   Final Test: 60.94 ± 2.02   Final old_old: 65.52 ± 2.12   Final old_new: 42.87 ± 1.99   Final new_new: 13.18 ± 5.18
AUC:
  Final val: 97.18 ± 0.16   Final Test: 96.58 ± 0.13   Final old_old: 97.33 ± 0.13   Final old_new: 93.61 ± 0.26   Final new_new: 89.36 ± 0.65
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.51 ± 5.32   Final Test: 35.46 ± 6.23   Final old_old: 38.79 ± 6.57   Final old_new: 22.02 ± 5.04   Final new_new: 6.82 ± 2.27
Hits@20:
  Final val: 66.18 ± 3.34   Final Test: 47.57 ± 4.15   Final old_old: 51.47 ± 4.22   Final old_new: 31.96 ± 4.22   Final new_new: 10.45 ± 4.43
Hits@30:
  Final val: 73.20 ± 2.70   Final Test: 56.57 ± 4.13   Final old_old: 60.63 ± 4.40   Final old_new: 40.55 ± 3.33   Final new_new: 13.64 ± 3.59
Hits@50:
  Final val: 79.18 ± 1.39   Final Test: 64.17 ± 3.67   Final old_old: 68.19 ± 3.66   Final old_new: 48.36 ± 3.73   Final new_new: 20.91 ± 5.43
AUC:
  Final val: 97.63 ± 0.13   Final Test: 97.20 ± 0.10   Final old_old: 97.70 ± 0.09   Final old_new: 95.25 ± 0.16   Final new_new: 91.58 ± 0.55
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.69 ± 1.62   Final Test: 32.55 ± 3.76   Final old_old: 35.86 ± 3.90   Final old_new: 19.17 ± 3.46   Final new_new: 4.09 ± 2.96
Hits@20:
  Final val: 58.41 ± 1.60   Final Test: 39.04 ± 2.45   Final old_old: 42.55 ± 2.48   Final old_new: 25.01 ± 2.52   Final new_new: 5.45 ± 3.80
Hits@30:
  Final val: 65.68 ± 2.17   Final Test: 48.35 ± 2.23   Final old_old: 52.32 ± 2.11   Final old_new: 32.54 ± 2.89   Final new_new: 9.09 ± 3.59
Hits@50:
  Final val: 72.58 ± 1.16   Final Test: 58.84 ± 2.56   Final old_old: 63.07 ± 2.31   Final old_new: 42.14 ± 3.78   Final new_new: 15.45 ± 4.37
AUC:
  Final val: 97.12 ± 0.16   Final Test: 96.58 ± 0.06   Final old_old: 97.23 ± 0.10   Final old_new: 93.99 ± 0.23   Final new_new: 90.27 ± 0.40
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.10 ± 2.91   Final Test: 35.80 ± 5.06   Final old_old: 39.42 ± 5.39   Final old_new: 21.16 ± 4.04   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 59.88 ± 1.94   Final Test: 42.77 ± 3.23   Final old_old: 46.70 ± 3.45   Final old_new: 27.08 ± 3.13   Final new_new: 5.00 ± 2.96
Hits@30:
  Final val: 67.29 ± 2.23   Final Test: 51.42 ± 6.04   Final old_old: 55.65 ± 6.45   Final old_new: 34.74 ± 4.46   Final new_new: 7.73 ± 4.71
Hits@50:
  Final val: 75.09 ± 2.45   Final Test: 62.95 ± 4.01   Final old_old: 67.34 ± 4.31   Final old_new: 45.72 ± 2.83   Final new_new: 15.91 ± 4.25
AUC:
  Final val: 97.26 ± 0.20   Final Test: 96.64 ± 0.16   Final old_old: 97.39 ± 0.18   Final old_new: 93.67 ± 0.20   Final new_new: 88.67 ± 1.67
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.60 ± 3.04   Final Test: 33.37 ± 6.37   Final old_old: 36.48 ± 6.85   Final old_new: 20.78 ± 4.74   Final new_new: 6.36 ± 1.02
Hits@20:
  Final val: 63.56 ± 2.04   Final Test: 46.07 ± 4.71   Final old_old: 49.37 ± 4.88   Final old_new: 33.07 ± 4.18   Final new_new: 10.91 ± 2.96
Hits@30:
  Final val: 69.60 ± 1.71   Final Test: 54.11 ± 2.96   Final old_old: 57.63 ± 2.96   Final old_new: 40.33 ± 3.15   Final new_new: 15.91 ± 2.27
Hits@50:
  Final val: 76.71 ± 1.06   Final Test: 62.64 ± 3.61   Final old_old: 66.43 ± 3.77   Final old_new: 47.78 ± 3.09   Final new_new: 20.45 ± 2.78
AUC:
  Final val: 97.57 ± 0.10   Final Test: 97.23 ± 0.15   Final old_old: 97.71 ± 0.16   Final old_new: 95.37 ± 0.20   Final new_new: 91.56 ± 0.59
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.16 ± 2.82   Final Test: 29.56 ± 5.80   Final old_old: 32.66 ± 6.25   Final old_new: 16.98 ± 4.17   Final new_new: 3.64 ± 3.05
Hits@20:
  Final val: 60.10 ± 2.26   Final Test: 40.28 ± 2.12   Final old_old: 44.02 ± 2.31   Final old_new: 25.26 ± 1.95   Final new_new: 5.91 ± 3.45
Hits@30:
  Final val: 65.62 ± 1.69   Final Test: 46.11 ± 3.07   Final old_old: 50.10 ± 3.11   Final old_new: 30.15 ± 3.16   Final new_new: 7.73 ± 4.71
Hits@50:
  Final val: 73.33 ± 1.07   Final Test: 58.67 ± 1.91   Final old_old: 63.11 ± 1.93   Final old_new: 41.18 ± 2.41   Final new_new: 11.82 ± 2.49
AUC:
  Final val: 97.21 ± 0.22   Final Test: 96.61 ± 0.07   Final old_old: 97.30 ± 0.08   Final old_new: 93.91 ± 0.31   Final new_new: 89.62 ± 0.85
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 40.10 ± 5.01   Final Test: 25.14 ± 6.63   Final old_old: 27.14 ± 7.04   Final old_new: 17.13 ± 5.28   Final new_new: 5.91 ± 3.45
Hits@20:
  Final val: 51.99 ± 3.23   Final Test: 35.94 ± 4.49   Final old_old: 38.54 ± 4.66   Final old_new: 25.59 ± 3.86   Final new_new: 10.45 ± 6.93
Hits@30:
  Final val: 59.74 ± 2.28   Final Test: 44.39 ± 2.61   Final old_old: 47.48 ± 2.83   Final old_new: 31.89 ± 2.30   Final new_new: 17.27 ± 5.93
Hits@50:
  Final val: 69.12 ± 2.46   Final Test: 54.01 ± 2.93   Final old_old: 57.42 ± 3.14   Final old_new: 40.63 ± 2.68   Final new_new: 16.82 ± 4.71
AUC:
  Final val: 96.39 ± 0.21   Final Test: 95.81 ± 0.18   Final old_old: 96.47 ± 0.23   Final old_new: 93.13 ± 0.25   Final new_new: 90.32 ± 1.16
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.97 ± 2.31   Final Test: 29.55 ± 6.47   Final old_old: 32.68 ± 6.91   Final old_new: 16.88 ± 5.03   Final new_new: 2.73 ± 1.02
Hits@20:
  Final val: 58.16 ± 1.68   Final Test: 40.97 ± 4.16   Final old_old: 44.75 ± 4.33   Final old_new: 25.87 ± 3.92   Final new_new: 5.00 ± 2.96
Hits@30:
  Final val: 65.36 ± 0.79   Final Test: 49.48 ± 4.26   Final old_old: 53.45 ± 4.32   Final old_new: 33.70 ± 4.55   Final new_new: 10.00 ± 4.43
Hits@50:
  Final val: 72.75 ± 1.16   Final Test: 57.85 ± 1.35   Final old_old: 62.28 ± 1.46   Final old_new: 40.40 ± 1.62   Final new_new: 11.36 ± 5.33
AUC:
  Final val: 97.11 ± 0.15   Final Test: 96.53 ± 0.03   Final old_old: 97.28 ± 0.07   Final old_new: 93.62 ± 0.22   Final new_new: 88.30 ± 1.14
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 38.92 ± 6.22   Final Test: 26.81 ± 2.01   Final old_old: 29.11 ± 2.19   Final old_new: 17.51 ± 2.29   Final new_new: 6.82 ± 2.78
Hits@20:
  Final val: 51.32 ± 5.37   Final Test: 33.81 ± 2.51   Final old_old: 36.44 ± 2.75   Final old_new: 23.30 ± 2.14   Final new_new: 8.18 ± 6.93
Hits@30:
  Final val: 59.62 ± 4.58   Final Test: 41.76 ± 4.63   Final old_old: 44.89 ± 4.97   Final old_new: 29.22 ± 3.35   Final new_new: 11.82 ± 3.37
Hits@50:
  Final val: 68.83 ± 2.59   Final Test: 52.53 ± 3.66   Final old_old: 56.09 ± 4.20   Final old_new: 38.49 ± 2.20   Final new_new: 14.55 ± 3.80
AUC:
  Final val: 96.39 ± 0.32   Final Test: 95.85 ± 0.24   Final old_old: 96.55 ± 0.25   Final old_new: 93.03 ± 0.39   Final new_new: 90.21 ± 1.27
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.48 ± 4.96   Final Test: 28.49 ± 2.66   Final old_old: 30.90 ± 2.73   Final old_new: 18.82 ± 2.59   Final new_new: 6.82 ± 4.82
Hits@20:
  Final val: 61.06 ± 3.55   Final Test: 37.32 ± 4.32   Final old_old: 40.24 ± 4.63   Final old_new: 25.49 ± 3.23   Final new_new: 11.82 ± 3.37
Hits@30:
  Final val: 68.59 ± 2.31   Final Test: 48.93 ± 3.69   Final old_old: 52.57 ± 3.95   Final old_new: 34.36 ± 3.12   Final new_new: 14.55 ± 3.05
Hits@50:
  Final val: 75.07 ± 1.35   Final Test: 58.38 ± 4.46   Final old_old: 62.15 ± 4.47   Final old_new: 43.48 ± 4.91   Final new_new: 19.09 ± 5.23
AUC:
  Final val: 96.88 ± 0.10   Final Test: 96.33 ± 0.19   Final old_old: 96.93 ± 0.17   Final old_new: 93.90 ± 0.28   Final new_new: 90.99 ± 1.02
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.81 ± 4.96   Final Test: 30.99 ± 4.26   Final old_old: 34.13 ± 4.82   Final old_new: 18.16 ± 2.44   Final new_new: 5.45 ± 1.24
Hits@20:
  Final val: 61.22 ± 3.16   Final Test: 42.27 ± 2.29   Final old_old: 45.90 ± 2.37   Final old_new: 27.78 ± 2.34   Final new_new: 6.82 ± 1.61
Hits@30:
  Final val: 66.96 ± 2.84   Final Test: 50.70 ± 1.68   Final old_old: 54.73 ± 1.72   Final old_new: 34.74 ± 2.17   Final new_new: 9.55 ± 2.96
Hits@50:
  Final val: 74.34 ± 2.16   Final Test: 61.13 ± 3.34   Final old_old: 65.41 ± 3.43   Final old_new: 44.31 ± 3.08   Final new_new: 15.45 ± 4.37
AUC:
  Final val: 97.19 ± 0.28   Final Test: 96.56 ± 0.19   Final old_old: 97.27 ± 0.21   Final old_new: 93.76 ± 0.17   Final new_new: 89.44 ± 0.65
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.71 ± 2.25   Final Test: 27.59 ± 6.03   Final old_old: 30.14 ± 6.69   Final old_new: 17.38 ± 3.61   Final new_new: 3.64 ± 2.59
Hits@20:
  Final val: 54.78 ± 3.11   Final Test: 40.08 ± 4.44   Final old_old: 43.34 ± 4.69   Final old_new: 27.05 ± 3.85   Final new_new: 9.09 ± 4.25
Hits@30:
  Final val: 61.75 ± 2.72   Final Test: 43.58 ± 2.51   Final old_old: 46.78 ± 2.56   Final old_new: 30.83 ± 2.87   Final new_new: 12.73 ± 5.93
Hits@50:
  Final val: 70.07 ± 1.26   Final Test: 53.13 ± 3.63   Final old_old: 56.62 ± 3.84   Final old_new: 39.55 ± 2.95   Final new_new: 13.18 ± 5.88
AUC:
  Final val: 96.43 ± 0.23   Final Test: 95.66 ± 0.22   Final old_old: 96.40 ± 0.22   Final old_new: 92.69 ± 0.41   Final new_new: 88.70 ± 0.80
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.08 ± 2.29   Final Test: 31.31 ± 4.01   Final old_old: 34.73 ± 4.28   Final old_new: 17.46 ± 3.35   Final new_new: 1.82 ± 1.02
Hits@20:
  Final val: 62.99 ± 1.74   Final Test: 44.87 ± 2.81   Final old_old: 48.99 ± 3.18   Final old_new: 28.46 ± 1.61   Final new_new: 5.00 ± 2.49
Hits@30:
  Final val: 68.55 ± 1.95   Final Test: 53.96 ± 2.26   Final old_old: 58.35 ± 2.18   Final old_new: 36.62 ± 2.55   Final new_new: 8.64 ± 7.08
Hits@50:
  Final val: 75.52 ± 1.31   Final Test: 62.27 ± 2.53   Final old_old: 66.79 ± 2.63   Final old_new: 44.36 ± 2.38   Final new_new: 15.91 ± 6.43
AUC:
  Final val: 97.23 ± 0.20   Final Test: 96.63 ± 0.14   Final old_old: 97.39 ± 0.13   Final old_new: 93.63 ± 0.40   Final new_new: 88.79 ± 1.15
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 56.51 ± 4.25   Final Test: 37.17 ± 7.05   Final old_old: 40.60 ± 7.52   Final old_new: 23.43 ± 5.29   Final new_new: 5.00 ± 1.90
Hits@20:
  Final val: 66.87 ± 2.76   Final Test: 48.73 ± 5.63   Final old_old: 52.53 ± 5.75   Final old_new: 33.68 ± 5.13   Final new_new: 10.45 ± 5.47
Hits@30:
  Final val: 73.35 ± 2.20   Final Test: 53.78 ± 4.36   Final old_old: 57.91 ± 4.41   Final old_new: 37.46 ± 4.29   Final new_new: 10.45 ± 5.70
Hits@50:
  Final val: 79.15 ± 1.28   Final Test: 63.39 ± 3.11   Final old_old: 67.48 ± 2.95   Final old_new: 47.43 ± 4.04   Final new_new: 17.27 ± 5.47
AUC:
  Final val: 97.67 ± 0.07   Final Test: 97.21 ± 0.05   Final old_old: 97.75 ± 0.03   Final old_new: 95.13 ± 0.16   Final new_new: 91.00 ± 0.66
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.76 ± 3.82   Final Test: 29.56 ± 5.82   Final old_old: 32.86 ± 6.17   Final old_new: 16.05 ± 4.60   Final new_new: 3.18 ± 1.24
Hits@20:
  Final val: 57.49 ± 2.85   Final Test: 38.55 ± 6.36   Final old_old: 42.17 ± 6.73   Final old_new: 24.01 ± 5.28   Final new_new: 5.45 ± 3.80
Hits@30:
  Final val: 64.19 ± 1.35   Final Test: 46.60 ± 2.34   Final old_old: 50.41 ± 2.40   Final old_new: 31.54 ± 2.54   Final new_new: 7.73 ± 4.43
Hits@50:
  Final val: 73.12 ± 1.47   Final Test: 56.24 ± 2.55   Final old_old: 60.46 ± 2.47   Final old_new: 39.67 ± 3.37   Final new_new: 10.45 ± 5.23
AUC:
  Final val: 97.11 ± 0.19   Final Test: 96.51 ± 0.11   Final old_old: 97.19 ± 0.20   Final old_new: 93.80 ± 0.37   Final new_new: 89.54 ± 1.05
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.66 ± 6.44   Final Test: 29.79 ± 4.62   Final old_old: 32.94 ± 4.98   Final old_new: 16.93 ± 3.37   Final new_new: 5.00 ± 1.90
Hits@20:
  Final val: 60.62 ± 3.40   Final Test: 40.37 ± 4.89   Final old_old: 44.25 ± 5.30   Final old_new: 24.66 ± 3.50   Final new_new: 6.82 ± 2.27
Hits@30:
  Final val: 66.37 ± 2.71   Final Test: 51.18 ± 3.94   Final old_old: 55.49 ± 4.24   Final old_new: 34.06 ± 2.85   Final new_new: 8.64 ± 2.96
Hits@50:
  Final val: 74.61 ± 2.40   Final Test: 59.47 ± 4.88   Final old_old: 63.87 ± 5.10   Final old_new: 42.24 ± 4.06   Final new_new: 11.36 ± 3.59
AUC:
  Final val: 97.20 ± 0.22   Final Test: 96.56 ± 0.23   Final old_old: 97.32 ± 0.26   Final old_new: 93.48 ± 0.33   Final new_new: 89.63 ± 1.28
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 39.05 ± 5.75   Final Test: 25.40 ± 6.69   Final old_old: 27.60 ± 7.19   Final old_new: 16.37 ± 5.52   Final new_new: 8.64 ± 1.90
Hits@20:
  Final val: 50.92 ± 4.32   Final Test: 36.33 ± 3.03   Final old_old: 39.19 ± 3.30   Final old_new: 24.76 ± 2.68   Final new_new: 11.36 ± 6.22
Hits@30:
  Final val: 59.06 ± 3.59   Final Test: 45.02 ± 4.01   Final old_old: 48.28 ± 4.36   Final old_new: 31.99 ± 2.61   Final new_new: 14.09 ± 4.37
Hits@50:
  Final val: 68.47 ± 3.19   Final Test: 55.39 ± 3.87   Final old_old: 58.95 ± 4.40   Final old_new: 41.26 ± 2.00   Final new_new: 19.55 ± 5.47
AUC:
  Final val: 96.36 ± 0.33   Final Test: 95.75 ± 0.19   Final old_old: 96.44 ± 0.23   Final old_new: 92.97 ± 0.39   Final new_new: 89.59 ± 1.17
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.22 ± 2.88   Final Test: 30.64 ± 4.02   Final old_old: 33.79 ± 4.27   Final old_new: 17.91 ± 3.18   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 58.37 ± 2.21   Final Test: 41.28 ± 3.90   Final old_old: 44.87 ± 4.12   Final old_new: 26.75 ± 3.20   Final new_new: 10.00 ± 4.71
Hits@30:
  Final val: 65.39 ± 1.04   Final Test: 45.88 ± 2.92   Final old_old: 49.68 ± 3.16   Final old_new: 30.71 ± 2.12   Final new_new: 9.09 ± 2.78
Hits@50:
  Final val: 72.89 ± 1.16   Final Test: 57.45 ± 3.23   Final old_old: 61.71 ± 3.27   Final old_new: 40.55 ± 3.45   Final new_new: 14.55 ± 6.14
AUC:
  Final val: 97.12 ± 0.14   Final Test: 96.55 ± 0.09   Final old_old: 97.26 ± 0.07   Final old_new: 93.73 ± 0.31   Final new_new: 89.63 ± 1.33
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.47 ± 4.69   Final Test: 37.86 ± 5.17   Final old_old: 41.13 ± 5.51   Final old_new: 24.81 ± 3.94   Final new_new: 6.36 ± 1.90
Hits@20:
  Final val: 64.79 ± 3.13   Final Test: 44.30 ± 3.91   Final old_old: 47.69 ± 4.15   Final old_new: 30.96 ± 3.38   Final new_new: 8.18 ± 3.05
Hits@30:
  Final val: 70.63 ± 2.39   Final Test: 54.35 ± 2.86   Final old_old: 58.15 ± 2.92   Final old_new: 39.52 ± 3.02   Final new_new: 12.27 ± 4.13
Hits@50:
  Final val: 77.43 ± 1.36   Final Test: 63.29 ± 1.40   Final old_old: 67.21 ± 1.45   Final old_new: 47.96 ± 1.71   Final new_new: 19.09 ± 4.13
AUC:
  Final val: 97.57 ± 0.17   Final Test: 97.25 ± 0.11   Final old_old: 97.72 ± 0.12   Final old_new: 95.44 ± 0.15   Final new_new: 91.93 ± 0.37
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.87 ± 2.39   Final Test: 33.85 ± 3.80   Final old_old: 37.30 ± 4.04   Final old_new: 19.77 ± 2.94   Final new_new: 5.91 ± 4.71
Hits@20:
  Final val: 58.31 ± 2.56   Final Test: 43.17 ± 5.91   Final old_old: 46.98 ± 6.33   Final old_new: 27.86 ± 4.66   Final new_new: 8.18 ± 3.80
Hits@30:
  Final val: 65.32 ± 1.95   Final Test: 49.23 ± 3.23   Final old_old: 53.38 ± 3.54   Final old_new: 32.64 ± 2.12   Final new_new: 9.55 ± 4.07
Hits@50:
  Final val: 73.32 ± 1.63   Final Test: 58.12 ± 2.30   Final old_old: 62.35 ± 2.48   Final old_new: 41.36 ± 2.49   Final new_new: 15.00 ± 5.93
AUC:
  Final val: 97.16 ± 0.13   Final Test: 96.59 ± 0.08   Final old_old: 97.28 ± 0.13   Final old_new: 93.86 ± 0.19   Final new_new: 88.79 ± 1.88
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 40.94 ± 3.74   Final Test: 22.63 ± 2.88   Final old_old: 24.71 ± 3.03   Final old_new: 14.18 ± 2.44   Final new_new: 5.45 ± 2.59
Hits@20:
  Final val: 52.77 ± 2.72   Final Test: 34.45 ± 4.81   Final old_old: 37.21 ± 4.98   Final old_new: 23.48 ± 4.50   Final new_new: 7.27 ± 2.49
Hits@30:
  Final val: 60.26 ± 1.57   Final Test: 40.58 ± 4.33   Final old_old: 43.84 ± 4.59   Final old_new: 27.53 ± 3.60   Final new_new: 10.00 ± 2.03
Hits@50:
  Final val: 68.59 ± 1.31   Final Test: 52.40 ± 3.57   Final old_old: 56.05 ± 3.90   Final old_new: 38.11 ± 2.79   Final new_new: 12.73 ± 3.80
AUC:
  Final val: 96.44 ± 0.30   Final Test: 95.74 ± 0.33   Final old_old: 96.45 ± 0.31   Final old_new: 92.86 ± 0.58   Final new_new: 89.73 ± 1.52
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.89 ± 2.38   Final Test: 36.15 ± 4.36   Final old_old: 39.72 ± 4.67   Final old_new: 21.74 ± 3.64   Final new_new: 4.55 ± 2.78
Hits@20:
  Final val: 61.41 ± 1.37   Final Test: 43.67 ± 5.52   Final old_old: 47.68 ± 5.90   Final old_new: 27.68 ± 4.29   Final new_new: 5.00 ± 2.96
Hits@30:
  Final val: 68.39 ± 1.78   Final Test: 51.16 ± 2.67   Final old_old: 55.46 ± 2.84   Final old_new: 34.11 ± 2.24   Final new_new: 8.18 ± 4.43
Hits@50:
  Final val: 75.63 ± 1.61   Final Test: 61.76 ± 4.33   Final old_old: 66.19 ± 4.51   Final old_new: 44.23 ± 3.86   Final new_new: 16.36 ± 6.10
AUC:
  Final val: 97.22 ± 0.26   Final Test: 96.64 ± 0.17   Final old_old: 97.38 ± 0.18   Final old_new: 93.72 ± 0.24   Final new_new: 89.33 ± 1.75
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.12 ± 5.60   Final Test: 28.82 ± 7.26   Final old_old: 31.22 ± 7.83   Final old_new: 19.22 ± 4.94   Final new_new: 6.36 ± 4.93
Hits@20:
  Final val: 61.00 ± 2.91   Final Test: 41.39 ± 7.27   Final old_old: 44.39 ± 7.81   Final old_new: 29.45 ± 5.34   Final new_new: 12.73 ± 5.70
Hits@30:
  Final val: 68.11 ± 1.76   Final Test: 48.44 ± 7.91   Final old_old: 51.86 ± 8.14   Final old_new: 34.79 ± 7.42   Final new_new: 15.45 ± 7.08
Hits@50:
  Final val: 75.62 ± 0.99   Final Test: 60.94 ± 3.23   Final old_old: 64.73 ± 3.27   Final old_new: 46.22 ± 3.40   Final new_new: 17.27 ± 6.74
AUC:
  Final val: 96.92 ± 0.09   Final Test: 96.27 ± 0.35   Final old_old: 96.89 ± 0.31   Final old_new: 93.78 ± 0.56   Final new_new: 90.56 ± 1.02
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.25 ± 4.65   Final Test: 26.22 ± 2.46   Final old_old: 28.61 ± 2.55   Final old_new: 16.47 ± 2.34   Final new_new: 6.36 ± 2.96
Hits@20:
  Final val: 55.24 ± 4.05   Final Test: 35.15 ± 5.05   Final old_old: 38.05 ± 5.26   Final old_new: 23.63 ± 4.60   Final new_new: 6.82 ± 1.61
Hits@30:
  Final val: 61.67 ± 4.15   Final Test: 44.77 ± 4.63   Final old_old: 48.15 ± 4.90   Final old_new: 31.46 ± 3.93   Final new_new: 8.64 ± 4.37
Hits@50:
  Final val: 70.75 ± 2.79   Final Test: 53.87 ± 3.14   Final old_old: 57.51 ± 3.34   Final old_new: 39.52 ± 2.54   Final new_new: 15.91 ± 3.59
AUC:
  Final val: 96.49 ± 0.33   Final Test: 95.87 ± 0.32   Final old_old: 96.58 ± 0.31   Final old_new: 93.00 ± 0.44   Final new_new: 89.48 ± 0.86
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.70 ± 3.46   Final Test: 30.77 ± 2.03   Final old_old: 33.84 ± 2.24   Final old_new: 18.36 ± 1.36   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 57.78 ± 2.43   Final Test: 40.09 ± 3.96   Final old_old: 43.60 ± 4.29   Final old_new: 26.07 ± 2.66   Final new_new: 5.91 ± 3.05
Hits@30:
  Final val: 64.35 ± 0.83   Final Test: 48.33 ± 4.30   Final old_old: 51.97 ± 4.73   Final old_new: 34.06 ± 2.69   Final new_new: 8.18 ± 4.43
Hits@50:
  Final val: 72.60 ± 1.23   Final Test: 57.82 ± 2.47   Final old_old: 61.84 ± 2.65   Final old_new: 42.07 ± 1.82   Final new_new: 14.09 ± 2.96
AUC:
  Final val: 97.08 ± 0.19   Final Test: 96.52 ± 0.10   Final old_old: 97.19 ± 0.11   Final old_new: 93.86 ± 0.15   Final new_new: 89.89 ± 0.57
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.45 ± 4.55   Final Test: 32.03 ± 6.36   Final old_old: 35.40 ± 6.80   Final old_new: 18.36 ± 4.93   Final new_new: 3.64 ± 3.05
Hits@20:
  Final val: 62.14 ± 5.39   Final Test: 45.27 ± 4.45   Final old_old: 49.12 ± 4.65   Final old_new: 29.97 ± 3.95   Final new_new: 6.36 ± 1.90
Hits@30:
  Final val: 67.91 ± 3.50   Final Test: 53.92 ± 3.31   Final old_old: 58.22 ± 3.49   Final old_new: 36.95 ± 2.99   Final new_new: 8.64 ± 4.37
Hits@50:
  Final val: 75.91 ± 2.56   Final Test: 62.41 ± 2.79   Final old_old: 66.88 ± 2.92   Final old_new: 44.99 ± 2.42   Final new_new: 12.27 ± 2.59
AUC:
  Final val: 97.28 ± 0.25   Final Test: 96.62 ± 0.19   Final old_old: 97.39 ± 0.22   Final old_new: 93.54 ± 0.28   Final new_new: 89.28 ± 2.70
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 56.52 ± 3.61   Final Test: 36.42 ± 9.87   Final old_old: 39.73 ± 10.46   Final old_new: 23.07 ± 7.58   Final new_new: 6.82 ± 4.25
Hits@20:
  Final val: 68.26 ± 2.50   Final Test: 50.72 ± 5.65   Final old_old: 54.63 ± 5.88   Final old_new: 35.29 ± 4.92   Final new_new: 9.55 ± 2.49
Hits@30:
  Final val: 74.49 ± 1.74   Final Test: 58.10 ± 3.28   Final old_old: 62.33 ± 3.31   Final old_new: 41.51 ± 3.32   Final new_new: 12.27 ± 4.43
Hits@50:
  Final val: 79.99 ± 1.55   Final Test: 66.49 ± 3.74   Final old_old: 70.62 ± 3.82   Final old_new: 50.30 ± 3.60   Final new_new: 21.36 ± 5.23
AUC:
  Final val: 97.71 ± 0.09   Final Test: 97.24 ± 0.09   Final old_old: 97.76 ± 0.11   Final old_new: 95.22 ± 0.09   Final new_new: 90.94 ± 1.05
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 40.27 ± 3.39   Final Test: 29.19 ± 4.82   Final old_old: 31.59 ± 5.30   Final old_new: 19.50 ± 3.02   Final new_new: 7.27 ± 2.96
Hits@20:
  Final val: 51.70 ± 2.84   Final Test: 36.06 ± 2.75   Final old_old: 38.90 ± 2.92   Final old_new: 24.61 ± 2.36   Final new_new: 10.91 ± 1.90
Hits@30:
  Final val: 59.17 ± 2.92   Final Test: 45.21 ± 4.07   Final old_old: 48.45 ± 4.09   Final old_new: 32.34 ± 4.79   Final new_new: 12.73 ± 3.45
Hits@50:
  Final val: 68.58 ± 1.86   Final Test: 54.14 ± 2.46   Final old_old: 57.61 ± 2.41   Final old_new: 40.43 ± 2.94   Final new_new: 18.18 ± 3.59
AUC:
  Final val: 96.34 ± 0.23   Final Test: 95.69 ± 0.15   Final old_old: 96.42 ± 0.18   Final old_new: 92.69 ± 0.09   Final new_new: 89.52 ± 1.83
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.19 ± 3.52   Final Test: 33.31 ± 5.17   Final old_old: 36.82 ± 5.41   Final old_new: 19.09 ± 4.37   Final new_new: 3.64 ± 2.59
Hits@20:
  Final val: 61.45 ± 3.02   Final Test: 43.33 ± 4.81   Final old_old: 47.46 ± 5.00   Final old_new: 26.70 ± 4.31   Final new_new: 5.45 ± 2.59
Hits@30:
  Final val: 67.52 ± 2.22   Final Test: 49.95 ± 2.59   Final old_old: 54.34 ± 2.72   Final old_new: 32.42 ± 2.42   Final new_new: 7.27 ± 3.73
Hits@50:
  Final val: 75.58 ± 2.48   Final Test: 60.20 ± 0.82   Final old_old: 64.70 ± 0.96   Final old_new: 42.47 ± 2.19   Final new_new: 12.73 ± 7.11
AUC:
  Final val: 97.23 ± 0.19   Final Test: 96.62 ± 0.06   Final old_old: 97.36 ± 0.11   Final old_new: 93.64 ± 0.25   Final new_new: 89.55 ± 1.69
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.76 ± 2.83   Final Test: 31.09 ± 2.48   Final old_old: 34.38 ± 2.56   Final old_new: 17.71 ± 2.48   Final new_new: 3.64 ± 2.03
Hits@20:
  Final val: 58.39 ± 1.39   Final Test: 41.15 ± 2.84   Final old_old: 44.85 ± 2.87   Final old_new: 26.37 ± 2.92   Final new_new: 5.91 ± 3.45
Hits@30:
  Final val: 65.18 ± 1.27   Final Test: 48.53 ± 2.60   Final old_old: 52.55 ± 2.92   Final old_new: 32.44 ± 2.20   Final new_new: 10.00 ± 4.71
Hits@50:
  Final val: 73.12 ± 2.10   Final Test: 56.32 ± 1.74   Final old_old: 60.44 ± 1.79   Final old_new: 40.10 ± 1.93   Final new_new: 12.73 ± 6.74
AUC:
  Final val: 97.17 ± 0.17   Final Test: 96.57 ± 0.11   Final old_old: 97.27 ± 0.22   Final old_new: 93.79 ± 0.42   Final new_new: 89.33 ± 1.54
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.35 ± 2.14   Final Test: 36.43 ± 3.12   Final old_old: 39.74 ± 3.34   Final old_new: 23.12 ± 3.25   Final new_new: 5.91 ± 2.03
Hits@20:
  Final val: 65.08 ± 1.63   Final Test: 47.78 ± 4.20   Final old_old: 51.45 ± 4.55   Final old_new: 33.30 ± 3.04   Final new_new: 9.55 ± 2.96
Hits@30:
  Final val: 71.20 ± 1.15   Final Test: 53.17 ± 3.51   Final old_old: 57.01 ± 3.72   Final old_new: 38.01 ± 2.81   Final new_new: 12.73 ± 3.45
Hits@50:
  Final val: 77.72 ± 1.30   Final Test: 63.17 ± 3.11   Final old_old: 67.11 ± 3.38   Final old_new: 47.73 ± 2.11   Final new_new: 20.00 ± 3.37
AUC:
  Final val: 97.64 ± 0.06   Final Test: 97.25 ± 0.13   Final old_old: 97.74 ± 0.15   Final old_new: 95.34 ± 0.18   Final new_new: 92.06 ± 0.79
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.72 ± 3.08   Final Test: 34.88 ± 8.52   Final old_old: 38.37 ± 9.00   Final old_new: 20.78 ± 7.00   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 59.51 ± 3.33   Final Test: 43.15 ± 5.10   Final old_old: 46.99 ± 5.51   Final old_new: 27.86 ± 3.85   Final new_new: 5.45 ± 2.59
Hits@30:
  Final val: 66.08 ± 2.94   Final Test: 48.13 ± 4.55   Final old_old: 52.30 ± 4.90   Final old_new: 31.44 ± 3.38   Final new_new: 9.09 ± 2.78
Hits@50:
  Final val: 74.29 ± 2.76   Final Test: 61.81 ± 2.92   Final old_old: 66.20 ± 2.91   Final old_new: 44.43 ± 3.59   Final new_new: 17.73 ± 7.08
AUC:
  Final val: 97.20 ± 0.23   Final Test: 96.60 ± 0.28   Final old_old: 97.32 ± 0.30   Final old_new: 93.75 ± 0.32   Final new_new: 89.51 ± 0.82
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.29 ± 2.11   Final Test: 31.58 ± 2.35   Final old_old: 34.81 ± 2.26   Final old_new: 18.49 ± 2.87   Final new_new: 3.64 ± 2.03
Hits@20:
  Final val: 58.73 ± 1.29   Final Test: 40.00 ± 4.71   Final old_old: 43.59 ± 4.79   Final old_new: 25.59 ± 4.74   Final new_new: 6.36 ± 3.73
Hits@30:
  Final val: 65.47 ± 1.06   Final Test: 46.10 ± 3.91   Final old_old: 50.05 ± 3.98   Final old_new: 30.33 ± 3.84   Final new_new: 8.18 ± 2.59
Hits@50:
  Final val: 72.45 ± 0.98   Final Test: 57.25 ± 1.76   Final old_old: 61.63 ± 1.81   Final old_new: 39.97 ± 1.91   Final new_new: 11.36 ± 3.21
AUC:
  Final val: 97.12 ± 0.15   Final Test: 96.62 ± 0.11   Final old_old: 97.29 ± 0.13   Final old_new: 93.99 ± 0.24   Final new_new: 89.30 ± 1.27
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 40.81 ± 3.05   Final Test: 28.17 ± 4.52   Final old_old: 30.61 ± 4.98   Final old_new: 18.31 ± 2.89   Final new_new: 7.27 ± 1.90
Hits@20:
  Final val: 53.73 ± 3.47   Final Test: 38.97 ± 3.83   Final old_old: 42.20 ± 4.19   Final old_new: 26.15 ± 2.68   Final new_new: 6.82 ± 3.21
Hits@30:
  Final val: 61.37 ± 3.41   Final Test: 44.83 ± 3.67   Final old_old: 48.31 ± 3.99   Final old_new: 31.08 ± 3.12   Final new_new: 9.09 ± 2.27
Hits@50:
  Final val: 69.86 ± 2.57   Final Test: 54.63 ± 5.23   Final old_old: 58.09 ± 5.46   Final old_new: 41.01 ± 4.79   Final new_new: 18.18 ± 4.25
AUC:
  Final val: 96.48 ± 0.23   Final Test: 95.80 ± 0.23   Final old_old: 96.51 ± 0.23   Final old_new: 92.96 ± 0.49   Final new_new: 88.90 ± 1.71
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.01 ± 1.50   Final Test: 32.76 ± 3.56   Final old_old: 35.88 ± 3.71   Final old_new: 20.25 ± 3.18   Final new_new: 4.09 ± 2.49
Hits@20:
  Final val: 58.92 ± 1.88   Final Test: 41.30 ± 2.42   Final old_old: 44.88 ± 2.35   Final old_new: 27.03 ± 3.21   Final new_new: 6.36 ± 3.37
Hits@30:
  Final val: 64.64 ± 2.16   Final Test: 47.66 ± 2.27   Final old_old: 51.50 ± 2.33   Final old_new: 32.54 ± 2.54   Final new_new: 7.27 ± 3.37
Hits@50:
  Final val: 72.61 ± 2.47   Final Test: 57.49 ± 2.12   Final old_old: 61.61 ± 2.14   Final old_new: 41.44 ± 2.68   Final new_new: 10.91 ± 4.93
AUC:
  Final val: 97.16 ± 0.16   Final Test: 96.57 ± 0.08   Final old_old: 97.23 ± 0.14   Final old_new: 93.98 ± 0.25   Final new_new: 89.43 ± 2.16
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.40 ± 3.32   Final Test: 29.43 ± 4.97   Final old_old: 31.95 ± 5.30   Final old_new: 19.24 ± 3.73   Final new_new: 7.27 ± 2.96
Hits@20:
  Final val: 61.23 ± 1.74   Final Test: 44.20 ± 4.90   Final old_old: 47.43 ± 5.17   Final old_new: 31.18 ± 4.12   Final new_new: 15.00 ± 3.45
Hits@30:
  Final val: 67.76 ± 2.27   Final Test: 48.17 ± 4.76   Final old_old: 51.53 ± 5.06   Final old_new: 34.74 ± 3.74   Final new_new: 15.91 ± 2.27
Hits@50:
  Final val: 74.66 ± 1.70   Final Test: 59.12 ± 3.86   Final old_old: 62.79 ± 3.91   Final old_new: 44.74 ± 4.07   Final new_new: 19.09 ± 4.43
AUC:
  Final val: 96.86 ± 0.20   Final Test: 96.40 ± 0.27   Final old_old: 96.96 ± 0.24   Final old_new: 94.13 ± 0.52   Final new_new: 91.23 ± 0.65
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.74 ± 2.24   Final Test: 24.92 ± 2.87   Final old_old: 27.15 ± 3.01   Final old_new: 16.02 ± 2.43   Final new_new: 3.64 ± 2.03
Hits@20:
  Final val: 54.70 ± 1.23   Final Test: 35.17 ± 3.02   Final old_old: 37.98 ± 2.98   Final old_new: 24.03 ± 3.51   Final new_new: 6.36 ± 1.90
Hits@30:
  Final val: 61.71 ± 0.88   Final Test: 43.43 ± 1.82   Final old_old: 46.74 ± 1.78   Final old_new: 30.33 ± 2.47   Final new_new: 10.00 ± 4.13
Hits@50:
  Final val: 70.79 ± 1.02   Final Test: 55.31 ± 2.13   Final old_old: 58.98 ± 2.12   Final old_new: 41.08 ± 3.04   Final new_new: 13.18 ± 2.49
AUC:
  Final val: 96.48 ± 0.25   Final Test: 95.81 ± 0.09   Final old_old: 96.50 ± 0.11   Final old_new: 93.02 ± 0.24   Final new_new: 89.57 ± 0.87
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.51 ± 3.64   Final Test: 34.36 ± 6.48   Final old_old: 37.85 ± 7.11   Final old_new: 20.28 ± 4.01   Final new_new: 4.09 ± 2.49
Hits@20:
  Final val: 61.00 ± 4.10   Final Test: 43.02 ± 4.84   Final old_old: 46.93 ± 5.30   Final old_new: 27.36 ± 3.00   Final new_new: 6.82 ± 3.21
Hits@30:
  Final val: 67.79 ± 3.04   Final Test: 50.35 ± 5.58   Final old_old: 54.44 ± 6.15   Final old_new: 34.06 ± 3.37   Final new_new: 10.45 ± 1.24
Hits@50:
  Final val: 75.36 ± 2.33   Final Test: 59.68 ± 2.75   Final old_old: 63.94 ± 3.06   Final old_new: 42.85 ± 1.90   Final new_new: 15.45 ± 4.07
AUC:
  Final val: 97.25 ± 0.23   Final Test: 96.56 ± 0.22   Final old_old: 97.33 ± 0.31   Final old_new: 93.51 ± 0.37   Final new_new: 88.76 ± 2.08
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.78 ± 3.08   Final Test: 27.31 ± 3.55   Final old_old: 29.84 ± 3.95   Final old_new: 17.10 ± 1.99   Final new_new: 5.45 ± 2.59
Hits@20:
  Final val: 53.30 ± 3.94   Final Test: 36.50 ± 3.34   Final old_old: 39.52 ± 3.62   Final old_new: 24.38 ± 2.48   Final new_new: 8.18 ± 1.24
Hits@30:
  Final val: 60.67 ± 3.66   Final Test: 42.99 ± 1.48   Final old_old: 46.29 ± 1.23   Final old_new: 29.92 ± 2.89   Final new_new: 9.55 ± 1.90
Hits@50:
  Final val: 70.00 ± 2.68   Final Test: 54.37 ± 4.60   Final old_old: 57.93 ± 4.65   Final old_new: 40.50 ± 4.78   Final new_new: 14.09 ± 4.07
AUC:
  Final val: 96.32 ± 0.31   Final Test: 95.77 ± 0.19   Final old_old: 96.46 ± 0.20   Final old_new: 92.99 ± 0.47   Final new_new: 89.95 ± 1.10
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 53.43 ± 4.16   Final Test: 36.85 ± 3.55   Final old_old: 40.19 ± 3.85   Final old_new: 23.45 ± 2.63   Final new_new: 6.36 ± 1.90
Hits@20:
  Final val: 66.53 ± 2.42   Final Test: 47.65 ± 2.74   Final old_old: 51.36 ± 2.96   Final old_new: 32.92 ± 1.93   Final new_new: 10.45 ± 3.80
Hits@30:
  Final val: 71.92 ± 2.55   Final Test: 52.37 ± 1.26   Final old_old: 56.35 ± 1.32   Final old_new: 36.62 ± 1.39   Final new_new: 11.36 ± 5.57
Hits@50:
  Final val: 78.78 ± 1.16   Final Test: 63.09 ± 3.83   Final old_old: 67.15 ± 3.97   Final old_new: 47.08 ± 3.77   Final new_new: 20.91 ± 1.02
AUC:
  Final val: 97.67 ± 0.04   Final Test: 97.20 ± 0.08   Final old_old: 97.71 ± 0.10   Final old_new: 95.20 ± 0.22   Final new_new: 91.68 ± 0.77
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.51 ± 5.76   Final Test: 33.34 ± 5.02   Final old_old: 36.76 ± 5.43   Final old_new: 19.45 ± 3.61   Final new_new: 4.55 ± 2.78
Hits@20:
  Final val: 58.65 ± 4.34   Final Test: 42.41 ± 4.92   Final old_old: 46.22 ± 5.41   Final old_new: 27.10 ± 3.10   Final new_new: 8.18 ± 3.45
Hits@30:
  Final val: 65.85 ± 3.20   Final Test: 50.17 ± 2.64   Final old_old: 54.27 ± 3.08   Final old_new: 33.93 ± 1.36   Final new_new: 9.09 ± 5.79
Hits@50:
  Final val: 73.88 ± 1.74   Final Test: 58.93 ± 2.65   Final old_old: 63.17 ± 3.04   Final old_new: 42.17 ± 1.56   Final new_new: 15.00 ± 3.80
AUC:
  Final val: 97.16 ± 0.20   Final Test: 96.58 ± 0.14   Final old_old: 97.30 ± 0.19   Final old_new: 93.71 ± 0.20   Final new_new: 89.66 ± 0.83
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.37 ± 5.19   Final Test: 32.95 ± 3.14   Final old_old: 36.33 ± 3.35   Final old_new: 19.12 ± 2.55   Final new_new: 6.36 ± 1.90
Hits@20:
  Final val: 59.71 ± 2.74   Final Test: 40.61 ± 1.80   Final old_old: 44.27 ± 2.12   Final old_new: 25.87 ± 1.45   Final new_new: 7.73 ± 2.59
Hits@30:
  Final val: 67.42 ± 1.82   Final Test: 49.73 ± 2.21   Final old_old: 53.79 ± 2.16   Final old_new: 33.60 ± 2.71   Final new_new: 9.09 ± 3.59
Hits@50:
  Final val: 75.16 ± 2.15   Final Test: 61.62 ± 0.96   Final old_old: 66.01 ± 1.12   Final old_new: 44.28 ± 1.21   Final new_new: 15.91 ± 3.59
AUC:
  Final val: 97.21 ± 0.26   Final Test: 96.63 ± 0.17   Final old_old: 97.36 ± 0.20   Final old_new: 93.74 ± 0.22   Final new_new: 89.30 ± 1.00
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.56 ± 4.53   Final Test: 30.82 ± 4.69   Final old_old: 34.11 ± 4.91   Final old_new: 17.43 ± 4.10   Final new_new: 3.64 ± 3.05
Hits@20:
  Final val: 61.01 ± 2.86   Final Test: 41.94 ± 3.76   Final old_old: 45.79 ± 3.78   Final old_new: 26.45 ± 4.41   Final new_new: 6.36 ± 1.90
Hits@30:
  Final val: 67.32 ± 1.69   Final Test: 49.24 ± 3.24   Final old_old: 53.16 ± 3.18   Final old_new: 33.65 ± 4.20   Final new_new: 10.45 ± 3.45
Hits@50:
  Final val: 74.13 ± 0.95   Final Test: 57.76 ± 2.51   Final old_old: 62.29 ± 2.87   Final old_new: 39.72 ± 1.38   Final new_new: 13.18 ± 4.37
AUC:
  Final val: 97.23 ± 0.15   Final Test: 96.56 ± 0.10   Final old_old: 97.30 ± 0.15   Final old_new: 93.63 ± 0.46   Final new_new: 89.47 ± 1.00
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 53.83 ± 3.94   Final Test: 40.62 ± 4.74   Final old_old: 43.91 ± 5.07   Final old_new: 27.58 ± 3.53   Final new_new: 7.27 ± 2.96
Hits@20:
  Final val: 66.79 ± 2.06   Final Test: 49.20 ± 2.02   Final old_old: 52.75 ± 2.29   Final old_new: 35.31 ± 1.36   Final new_new: 10.00 ± 3.45
Hits@30:
  Final val: 72.32 ± 1.07   Final Test: 55.73 ± 0.89   Final old_old: 59.50 ± 1.12   Final old_new: 40.96 ± 0.89   Final new_new: 14.55 ± 4.98
Hits@50:
  Final val: 78.45 ± 1.16   Final Test: 64.03 ± 2.95   Final old_old: 67.84 ± 3.10   Final old_new: 49.29 ± 2.73   Final new_new: 19.09 ± 5.23
AUC:
  Final val: 97.71 ± 0.12   Final Test: 97.30 ± 0.09   Final old_old: 97.78 ± 0.11   Final old_new: 95.44 ± 0.13   Final new_new: 91.85 ± 0.72
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.78 ± 3.41   Final Test: 34.98 ± 6.39   Final old_old: 38.32 ± 6.55   Final old_new: 21.46 ± 6.02   Final new_new: 5.45 ± 3.05
Hits@20:
  Final val: 57.68 ± 2.63   Final Test: 39.12 ± 4.40   Final old_old: 42.82 ± 4.58   Final old_new: 24.36 ± 4.00   Final new_new: 4.09 ± 1.02
Hits@30:
  Final val: 64.20 ± 2.81   Final Test: 48.16 ± 2.82   Final old_old: 52.25 ± 3.01   Final old_new: 31.96 ± 2.72   Final new_new: 6.36 ± 2.96
Hits@50:
  Final val: 72.45 ± 1.70   Final Test: 57.15 ± 1.36   Final old_old: 61.32 ± 1.64   Final old_new: 40.60 ± 1.15   Final new_new: 15.45 ± 3.73
AUC:
  Final val: 97.10 ± 0.16   Final Test: 96.59 ± 0.15   Final old_old: 97.23 ± 0.23   Final old_new: 94.06 ± 0.29   Final new_new: 89.81 ± 1.38
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.47 ± 4.24   Final Test: 27.25 ± 3.87   Final old_old: 29.97 ± 4.25   Final old_new: 16.15 ± 2.58   Final new_new: 5.00 ± 2.96
Hits@20:
  Final val: 57.40 ± 3.58   Final Test: 37.59 ± 4.48   Final old_old: 41.00 ± 4.95   Final old_new: 23.90 ± 2.68   Final new_new: 5.91 ± 2.03
Hits@30:
  Final val: 65.01 ± 0.84   Final Test: 44.37 ± 1.38   Final old_old: 48.19 ± 1.59   Final old_new: 29.09 ± 2.47   Final new_new: 7.73 ± 2.03
Hits@50:
  Final val: 72.75 ± 1.31   Final Test: 55.41 ± 2.43   Final old_old: 59.61 ± 2.45   Final old_new: 38.72 ± 2.63   Final new_new: 13.64 ± 6.63
AUC:
  Final val: 97.11 ± 0.21   Final Test: 96.51 ± 0.14   Final old_old: 97.18 ± 0.20   Final old_new: 93.86 ± 0.21   Final new_new: 89.97 ± 0.77
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 39.69 ± 4.42   Final Test: 28.91 ± 8.13   Final old_old: 31.33 ± 8.72   Final old_new: 19.27 ± 5.83   Final new_new: 5.45 ± 3.45
Hits@20:
  Final val: 52.08 ± 4.04   Final Test: 36.18 ± 6.21   Final old_old: 39.05 ± 6.63   Final old_new: 24.74 ± 4.74   Final new_new: 8.18 ± 1.24
Hits@30:
  Final val: 60.47 ± 2.70   Final Test: 44.52 ± 4.75   Final old_old: 47.88 ± 5.02   Final old_new: 31.13 ± 3.58   Final new_new: 11.36 ± 6.22
Hits@50:
  Final val: 69.03 ± 1.91   Final Test: 52.67 ± 3.77   Final old_old: 56.32 ± 3.79   Final old_new: 38.16 ± 3.62   Final new_new: 15.91 ± 6.22
AUC:
  Final val: 96.39 ± 0.28   Final Test: 95.73 ± 0.18   Final old_old: 96.46 ± 0.17   Final old_new: 92.79 ± 0.44   Final new_new: 89.21 ± 1.16
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 39.28 ± 4.74   Final Test: 25.82 ± 5.01   Final old_old: 28.12 ± 5.33   Final old_new: 16.47 ± 3.76   Final new_new: 6.36 ± 3.37
Hits@20:
  Final val: 55.05 ± 2.74   Final Test: 35.69 ± 2.78   Final old_old: 38.59 ± 2.77   Final old_new: 24.11 ± 3.16   Final new_new: 7.73 ± 3.80
Hits@30:
  Final val: 61.53 ± 3.07   Final Test: 42.59 ± 6.06   Final old_old: 45.77 ± 6.08   Final old_new: 30.00 ± 6.17   Final new_new: 10.00 ± 4.71
Hits@50:
  Final val: 69.67 ± 2.01   Final Test: 52.98 ± 4.74   Final old_old: 56.50 ± 4.49   Final old_new: 39.09 ± 6.14   Final new_new: 15.91 ± 6.63
AUC:
  Final val: 96.46 ± 0.27   Final Test: 95.68 ± 0.21   Final old_old: 96.38 ± 0.16   Final old_new: 92.88 ± 0.59   Final new_new: 89.34 ± 1.47
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.13 ± 5.68   Final Test: 29.53 ± 7.84   Final old_old: 32.10 ± 8.55   Final old_new: 19.24 ± 5.10   Final new_new: 5.45 ± 2.59
Hits@20:
  Final val: 59.05 ± 4.09   Final Test: 39.70 ± 3.36   Final old_old: 42.93 ± 3.41   Final old_new: 26.68 ± 3.32   Final new_new: 11.36 ± 5.33
Hits@30:
  Final val: 66.90 ± 2.77   Final Test: 48.82 ± 1.90   Final old_old: 52.36 ± 2.16   Final old_new: 34.81 ± 1.48   Final new_new: 12.73 ± 5.23
Hits@50:
  Final val: 75.36 ± 1.77   Final Test: 60.26 ± 2.55   Final old_old: 63.99 ± 2.63   Final old_new: 45.62 ± 2.41   Final new_new: 20.45 ± 7.01
AUC:
  Final val: 96.87 ± 0.27   Final Test: 96.37 ± 0.28   Final old_old: 96.94 ± 0.26   Final old_new: 94.10 ± 0.45   Final new_new: 90.92 ± 1.22
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 40.62 ± 5.32   Final Test: 25.80 ± 5.32   Final old_old: 27.93 ± 5.83   Final old_new: 17.25 ± 3.48   Final new_new: 6.82 ± 3.21
Hits@20:
  Final val: 51.81 ± 3.77   Final Test: 35.05 ± 1.58   Final old_old: 37.93 ± 1.82   Final old_new: 23.53 ± 1.43   Final new_new: 7.27 ± 2.96
Hits@30:
  Final val: 59.83 ± 3.06   Final Test: 45.38 ± 1.57   Final old_old: 48.77 ± 1.93   Final old_new: 31.84 ± 1.37   Final new_new: 12.73 ± 4.71
Hits@50:
  Final val: 69.23 ± 2.24   Final Test: 53.33 ± 1.93   Final old_old: 57.02 ± 1.90   Final old_new: 38.69 ± 2.84   Final new_new: 15.91 ± 8.50
AUC:
  Final val: 96.43 ± 0.31   Final Test: 95.73 ± 0.11   Final old_old: 96.47 ± 0.11   Final old_new: 92.78 ± 0.26   Final new_new: 88.95 ± 1.57
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.28 ± 3.96   Final Test: 31.66 ± 2.52   Final old_old: 34.78 ± 2.75   Final old_new: 19.07 ± 1.74   Final new_new: 4.55 ± 4.55
Hits@20:
  Final val: 55.23 ± 2.78   Final Test: 40.88 ± 4.60   Final old_old: 44.32 ± 5.08   Final old_new: 27.15 ± 2.77   Final new_new: 7.27 ± 4.66
Hits@30:
  Final val: 64.41 ± 3.17   Final Test: 46.84 ± 2.22   Final old_old: 50.65 ± 2.58   Final old_new: 31.61 ± 1.27   Final new_new: 10.00 ± 5.47
Hits@50:
  Final val: 73.45 ± 2.39   Final Test: 60.18 ± 2.68   Final old_old: 64.39 ± 2.91   Final old_new: 43.58 ± 2.18   Final new_new: 15.91 ± 6.01
AUC:
  Final val: 97.08 ± 0.22   Final Test: 96.49 ± 0.21   Final old_old: 97.17 ± 0.23   Final old_new: 93.82 ± 0.42   Final new_new: 89.17 ± 1.36
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.09 ± 4.62   Final Test: 31.53 ± 3.90   Final old_old: 34.77 ± 4.29   Final old_new: 18.39 ± 2.57   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 61.20 ± 3.58   Final Test: 42.57 ± 4.41   Final old_old: 46.36 ± 4.75   Final old_new: 27.43 ± 3.39   Final new_new: 6.82 ± 2.27
Hits@30:
  Final val: 67.17 ± 3.20   Final Test: 51.03 ± 5.05   Final old_old: 55.09 ± 5.43   Final old_new: 34.91 ± 3.65   Final new_new: 10.00 ± 3.80
Hits@50:
  Final val: 75.01 ± 3.00   Final Test: 59.92 ± 3.07   Final old_old: 64.15 ± 3.18   Final old_new: 43.17 ± 2.93   Final new_new: 16.82 ± 6.35
AUC:
  Final val: 97.20 ± 0.25   Final Test: 96.57 ± 0.19   Final old_old: 97.29 ± 0.24   Final old_new: 93.67 ± 0.35   Final new_new: 89.99 ± 1.20
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 53.05 ± 4.46   Final Test: 35.94 ± 8.21   Final old_old: 39.15 ± 8.60   Final old_new: 22.92 ± 6.80   Final new_new: 8.18 ± 4.43
Hits@20:
  Final val: 64.11 ± 3.02   Final Test: 47.99 ± 7.46   Final old_old: 51.91 ± 7.71   Final old_new: 32.44 ± 6.81   Final new_new: 8.64 ± 2.96
Hits@30:
  Final val: 72.03 ± 1.61   Final Test: 55.47 ± 3.61   Final old_old: 59.67 ± 3.81   Final old_new: 38.82 ± 2.89   Final new_new: 13.18 ± 4.37
Hits@50:
  Final val: 78.98 ± 1.23   Final Test: 65.30 ± 1.53   Final old_old: 69.45 ± 1.43   Final old_new: 49.02 ± 2.69   Final new_new: 20.45 ± 3.21
AUC:
  Final val: 97.63 ± 0.12   Final Test: 97.13 ± 0.13   Final old_old: 97.66 ± 0.12   Final old_new: 95.09 ± 0.19   Final new_new: 90.77 ± 0.46
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.45 ± 2.33   Final Test: 28.44 ± 5.81   Final old_old: 31.42 ± 6.39   Final old_new: 16.25 ± 3.68   Final new_new: 5.00 ± 1.90
Hits@20:
  Final val: 56.68 ± 2.27   Final Test: 39.58 ± 3.84   Final old_old: 43.19 ± 4.07   Final old_new: 25.11 ± 3.08   Final new_new: 6.36 ± 1.02
Hits@30:
  Final val: 64.15 ± 1.11   Final Test: 46.08 ± 3.83   Final old_old: 49.91 ± 4.01   Final old_new: 30.76 ± 3.11   Final new_new: 9.55 ± 4.07
Hits@50:
  Final val: 73.01 ± 0.90   Final Test: 58.03 ± 1.87   Final old_old: 62.37 ± 1.93   Final old_new: 40.71 ± 2.05   Final new_new: 15.91 ± 3.21
AUC:
  Final val: 97.11 ± 0.14   Final Test: 96.57 ± 0.09   Final old_old: 97.21 ± 0.12   Final old_new: 94.01 ± 0.28   Final new_new: 89.75 ± 0.94
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.01 ± 3.45   Final Test: 28.93 ± 3.09   Final old_old: 31.98 ± 3.33   Final old_new: 16.47 ± 2.49   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 57.20 ± 2.84   Final Test: 38.28 ± 3.13   Final old_old: 41.94 ± 3.34   Final old_new: 23.50 ± 2.49   Final new_new: 5.91 ± 2.59
Hits@30:
  Final val: 64.96 ± 1.84   Final Test: 46.50 ± 2.27   Final old_old: 50.31 ± 2.55   Final old_new: 31.46 ± 1.70   Final new_new: 6.36 ± 2.96
Hits@50:
  Final val: 73.28 ± 1.41   Final Test: 58.68 ± 2.49   Final old_old: 62.88 ± 2.50   Final old_new: 42.02 ± 2.92   Final new_new: 16.36 ± 2.96
AUC:
  Final val: 97.14 ± 0.14   Final Test: 96.52 ± 0.08   Final old_old: 97.23 ± 0.13   Final old_new: 93.75 ± 0.32   Final new_new: 88.91 ± 0.96
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.71 ± 4.29   Final Test: 34.15 ± 5.77   Final old_old: 37.66 ± 6.28   Final old_new: 19.95 ± 4.06   Final new_new: 3.64 ± 2.03
Hits@20:
  Final val: 61.89 ± 3.21   Final Test: 43.95 ± 7.28   Final old_old: 48.00 ± 7.65   Final old_new: 27.76 ± 6.07   Final new_new: 5.45 ± 3.45
Hits@30:
  Final val: 68.34 ± 3.14   Final Test: 54.18 ± 7.04   Final old_old: 58.47 ± 7.37   Final old_new: 37.18 ± 5.87   Final new_new: 10.00 ± 3.80
Hits@50:
  Final val: 75.07 ± 2.26   Final Test: 61.49 ± 4.64   Final old_old: 65.87 ± 4.90   Final old_new: 44.36 ± 3.71   Final new_new: 13.18 ± 1.90
AUC:
  Final val: 97.29 ± 0.25   Final Test: 96.75 ± 0.25   Final old_old: 97.48 ± 0.31   Final old_new: 93.83 ± 0.11   Final new_new: 89.55 ± 1.14
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.71 ± 3.09   Final Test: 33.26 ± 4.90   Final old_old: 36.33 ± 5.06   Final old_new: 20.91 ± 4.37   Final new_new: 5.00 ± 1.90
Hits@20:
  Final val: 63.99 ± 1.74   Final Test: 45.38 ± 1.75   Final old_old: 48.83 ± 1.73   Final old_new: 31.71 ± 2.03   Final new_new: 10.45 ± 3.80
Hits@30:
  Final val: 70.85 ± 1.29   Final Test: 54.79 ± 4.11   Final old_old: 58.44 ± 4.05   Final old_new: 40.63 ± 4.56   Final new_new: 12.73 ± 3.05
Hits@50:
  Final val: 77.68 ± 1.04   Final Test: 63.62 ± 1.09   Final old_old: 67.45 ± 1.06   Final old_new: 48.82 ± 1.61   Final new_new: 18.18 ± 7.19
AUC:
  Final val: 97.61 ± 0.04   Final Test: 97.28 ± 0.07   Final old_old: 97.73 ± 0.09   Final old_new: 95.56 ± 0.14   Final new_new: 91.80 ± 0.74
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 39.04 ± 2.85   Final Test: 23.29 ± 3.89   Final old_old: 25.29 ± 4.32   Final old_new: 15.24 ± 2.20   Final new_new: 5.45 ± 4.71
Hits@20:
  Final val: 51.65 ± 3.05   Final Test: 31.20 ± 2.24   Final old_old: 33.65 ± 2.25   Final old_new: 21.49 ± 2.79   Final new_new: 6.36 ± 2.49
Hits@30:
  Final val: 58.23 ± 2.76   Final Test: 40.11 ± 1.06   Final old_old: 43.11 ± 0.99   Final old_new: 28.26 ± 2.13   Final new_new: 9.09 ± 2.78
Hits@50:
  Final val: 67.79 ± 1.55   Final Test: 50.99 ± 3.34   Final old_old: 54.37 ± 3.49   Final old_new: 37.71 ± 3.62   Final new_new: 14.09 ± 3.73
AUC:
  Final val: 96.45 ± 0.31   Final Test: 95.84 ± 0.12   Final old_old: 96.47 ± 0.19   Final old_new: 93.33 ± 0.30   Final new_new: 89.77 ± 0.93
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.53 ± 2.26   Final Test: 31.95 ± 4.13   Final old_old: 35.30 ± 4.80   Final old_new: 18.24 ± 1.92   Final new_new: 5.91 ± 2.59
Hits@20:
  Final val: 58.43 ± 2.19   Final Test: 40.65 ± 2.42   Final old_old: 44.42 ± 2.54   Final old_new: 25.44 ± 2.43   Final new_new: 6.82 ± 7.71
Hits@30:
  Final val: 64.50 ± 1.88   Final Test: 44.90 ± 1.67   Final old_old: 48.81 ± 1.89   Final old_new: 29.27 ± 1.18   Final new_new: 7.27 ± 5.18
Hits@50:
  Final val: 72.48 ± 0.99   Final Test: 58.86 ± 2.46   Final old_old: 63.17 ± 2.59   Final old_new: 41.66 ± 2.16   Final new_new: 17.73 ± 5.18
AUC:
  Final val: 97.13 ± 0.09   Final Test: 96.63 ± 0.09   Final old_old: 97.29 ± 0.11   Final old_new: 93.96 ± 0.17   Final new_new: 90.25 ± 1.01
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.29 ± 3.34   Final Test: 28.15 ± 3.01   Final old_old: 30.64 ± 3.26   Final old_new: 18.16 ± 2.10   Final new_new: 5.00 ± 2.49
Hits@20:
  Final val: 53.73 ± 2.24   Final Test: 34.42 ± 5.47   Final old_old: 37.27 ± 5.60   Final old_new: 23.10 ± 5.09   Final new_new: 6.36 ± 3.73
Hits@30:
  Final val: 60.41 ± 1.58   Final Test: 42.27 ± 5.16   Final old_old: 45.50 ± 5.30   Final old_new: 29.55 ± 4.67   Final new_new: 7.73 ± 4.98
Hits@50:
  Final val: 69.21 ± 0.54   Final Test: 55.43 ± 3.78   Final old_old: 59.21 ± 3.50   Final old_new: 40.58 ± 5.18   Final new_new: 15.00 ± 5.70
AUC:
  Final val: 96.34 ± 0.30   Final Test: 95.74 ± 0.19   Final old_old: 96.43 ± 0.20   Final old_new: 92.95 ± 0.49   Final new_new: 89.38 ± 1.67
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.63 ± 5.00   Final Test: 29.70 ± 2.57   Final old_old: 32.78 ± 2.80   Final old_new: 17.20 ± 1.89   Final new_new: 4.09 ± 4.37
Hits@20:
  Final val: 57.22 ± 5.74   Final Test: 39.77 ± 2.58   Final old_old: 43.49 ± 2.98   Final old_new: 24.71 ± 1.82   Final new_new: 7.27 ± 4.93
Hits@30:
  Final val: 65.40 ± 2.93   Final Test: 48.55 ± 5.87   Final old_old: 52.62 ± 6.51   Final old_new: 32.29 ± 3.48   Final new_new: 9.55 ± 4.93
Hits@50:
  Final val: 73.84 ± 2.74   Final Test: 58.08 ± 5.23   Final old_old: 62.41 ± 5.74   Final old_new: 40.88 ± 3.58   Final new_new: 15.00 ± 3.80
AUC:
  Final val: 97.14 ± 0.25   Final Test: 96.50 ± 0.23   Final old_old: 97.21 ± 0.32   Final old_new: 93.68 ± 0.36   Final new_new: 89.32 ± 2.40
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.61 ± 0.94   Final Test: 28.83 ± 8.99   Final old_old: 31.21 ± 9.66   Final old_new: 19.35 ± 6.34   Final new_new: 6.36 ± 4.93
Hits@20:
  Final val: 62.39 ± 1.72   Final Test: 40.35 ± 3.99   Final old_old: 43.60 ± 4.28   Final old_new: 27.48 ± 2.85   Final new_new: 7.27 ± 5.18
Hits@30:
  Final val: 68.58 ± 1.02   Final Test: 47.62 ± 1.74   Final old_old: 51.25 ± 1.71   Final old_new: 33.22 ± 2.33   Final new_new: 11.36 ± 4.55
Hits@50:
  Final val: 75.36 ± 1.74   Final Test: 58.36 ± 2.98   Final old_old: 61.98 ± 2.96   Final old_new: 44.13 ± 3.30   Final new_new: 19.55 ± 8.29
AUC:
  Final val: 96.83 ± 0.13   Final Test: 96.29 ± 0.06   Final old_old: 96.94 ± 0.03   Final old_new: 93.76 ± 0.24   Final new_new: 89.11 ± 1.16
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 36.85 ± 4.02   Final Test: 25.75 ± 5.47   Final old_old: 27.85 ± 5.79   Final old_new: 17.28 ± 4.17   Final new_new: 6.36 ± 5.18
Hits@20:
  Final val: 49.16 ± 4.57   Final Test: 35.84 ± 4.53   Final old_old: 38.65 ± 4.95   Final old_new: 24.56 ± 2.93   Final new_new: 9.55 ± 4.66
Hits@30:
  Final val: 57.37 ± 3.69   Final Test: 41.76 ± 6.19   Final old_old: 44.81 ± 6.84   Final old_new: 29.75 ± 3.68   Final new_new: 9.55 ± 2.49
Hits@50:
  Final val: 67.95 ± 3.14   Final Test: 53.03 ± 2.77   Final old_old: 56.61 ± 3.26   Final old_new: 38.97 ± 0.92   Final new_new: 14.09 ± 3.37
AUC:
  Final val: 96.45 ± 0.33   Final Test: 95.82 ± 0.19   Final old_old: 96.54 ± 0.22   Final old_new: 92.94 ± 0.35   Final new_new: 89.40 ± 0.91
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.95 ± 3.68   Final Test: 33.65 ± 2.40   Final old_old: 37.16 ± 2.68   Final old_new: 19.45 ± 1.54   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 59.82 ± 2.24   Final Test: 43.91 ± 2.95   Final old_old: 48.05 ± 3.13   Final old_new: 27.28 ± 2.90   Final new_new: 5.91 ± 3.05
Hits@30:
  Final val: 66.84 ± 2.89   Final Test: 51.59 ± 4.05   Final old_old: 56.09 ± 4.32   Final old_new: 33.63 ± 3.12   Final new_new: 8.18 ± 2.03
Hits@50:
  Final val: 75.25 ± 1.85   Final Test: 61.01 ± 2.70   Final old_old: 65.48 ± 2.83   Final old_new: 43.40 ± 2.78   Final new_new: 13.64 ± 2.78
AUC:
  Final val: 97.21 ± 0.21   Final Test: 96.62 ± 0.14   Final old_old: 97.40 ± 0.18   Final old_new: 93.58 ± 0.05   Final new_new: 88.45 ± 2.86
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.52 ± 3.49   Final Test: 30.60 ± 2.62   Final old_old: 33.84 ± 2.83   Final old_new: 17.38 ± 2.35   Final new_new: 4.55 ± 3.94
Hits@20:
  Final val: 58.25 ± 1.96   Final Test: 39.62 ± 2.12   Final old_old: 43.21 ± 2.26   Final old_new: 25.16 ± 3.19   Final new_new: 6.82 ± 6.01
Hits@30:
  Final val: 64.24 ± 1.06   Final Test: 45.61 ± 4.54   Final old_old: 49.47 ± 4.64   Final old_new: 30.18 ± 4.66   Final new_new: 9.09 ± 4.25
Hits@50:
  Final val: 72.32 ± 1.40   Final Test: 55.44 ± 1.76   Final old_old: 59.60 ± 1.76   Final old_new: 39.02 ± 2.71   Final new_new: 11.82 ± 4.07
AUC:
  Final val: 97.08 ± 0.12   Final Test: 96.53 ± 0.09   Final old_old: 97.22 ± 0.10   Final old_new: 93.77 ± 0.27   Final new_new: 89.78 ± 0.58
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 56.47 ± 3.48   Final Test: 34.50 ± 6.76   Final old_old: 37.84 ± 7.31   Final old_new: 21.03 ± 4.50   Final new_new: 5.00 ± 2.96
Hits@20:
  Final val: 68.97 ± 2.02   Final Test: 47.63 ± 6.66   Final old_old: 51.59 ± 7.40   Final old_new: 31.76 ± 4.06   Final new_new: 10.00 ± 3.05
Hits@30:
  Final val: 73.90 ± 2.02   Final Test: 54.36 ± 6.40   Final old_old: 58.47 ± 6.87   Final old_new: 38.11 ± 4.73   Final new_new: 11.82 ± 4.66
Hits@50:
  Final val: 79.64 ± 1.68   Final Test: 64.69 ± 3.02   Final old_old: 68.84 ± 3.37   Final old_new: 48.46 ± 2.26   Final new_new: 18.18 ± 3.21
AUC:
  Final val: 97.71 ± 0.12   Final Test: 97.24 ± 0.08   Final old_old: 97.75 ± 0.12   Final old_new: 95.22 ± 0.19   Final new_new: 91.70 ± 0.56
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.27 ± 3.26   Final Test: 30.04 ± 4.37   Final old_old: 33.23 ± 4.81   Final old_new: 17.05 ± 2.83   Final new_new: 4.09 ± 2.49
Hits@20:
  Final val: 61.36 ± 2.84   Final Test: 44.32 ± 4.96   Final old_old: 48.41 ± 5.29   Final old_new: 27.91 ± 4.31   Final new_new: 6.36 ± 2.49
Hits@30:
  Final val: 67.49 ± 1.75   Final Test: 52.64 ± 4.52   Final old_old: 57.08 ± 4.50   Final old_new: 34.94 ± 4.97   Final new_new: 9.55 ± 2.96
Hits@50:
  Final val: 74.64 ± 1.34   Final Test: 63.24 ± 3.79   Final old_old: 67.69 ± 3.92   Final old_new: 45.54 ± 3.48   Final new_new: 18.64 ± 4.37
AUC:
  Final val: 97.24 ± 0.10   Final Test: 96.66 ± 0.10   Final old_old: 97.45 ± 0.13   Final old_new: 93.57 ± 0.40   Final new_new: 87.91 ± 1.50
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.69 ± 1.69   Final Test: 28.79 ± 2.70   Final old_old: 31.78 ± 2.81   Final old_new: 16.52 ± 2.37   Final new_new: 5.91 ± 3.45
Hits@20:
  Final val: 56.40 ± 2.61   Final Test: 38.88 ± 3.18   Final old_old: 42.40 ± 3.48   Final old_new: 24.56 ± 1.97   Final new_new: 9.09 ± 4.82
Hits@30:
  Final val: 63.92 ± 2.49   Final Test: 45.08 ± 1.59   Final old_old: 49.05 ± 1.83   Final old_new: 29.07 ± 1.01   Final new_new: 10.00 ± 3.45
Hits@50:
  Final val: 72.26 ± 1.57   Final Test: 57.72 ± 2.59   Final old_old: 62.00 ± 2.69   Final old_new: 40.55 ± 2.46   Final new_new: 17.73 ± 4.37
AUC:
  Final val: 97.06 ± 0.17   Final Test: 96.57 ± 0.13   Final old_old: 97.21 ± 0.15   Final old_new: 93.99 ± 0.12   Final new_new: 90.37 ± 0.63
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 38.17 ± 4.41   Final Test: 23.90 ± 2.90   Final old_old: 25.97 ± 3.01   Final old_new: 15.44 ± 2.60   Final new_new: 7.73 ± 2.59
Hits@20:
  Final val: 51.48 ± 2.98   Final Test: 33.65 ± 4.06   Final old_old: 36.33 ± 4.38   Final old_new: 22.72 ± 2.94   Final new_new: 11.82 ± 6.89
Hits@30:
  Final val: 59.42 ± 2.35   Final Test: 39.66 ± 6.27   Final old_old: 42.55 ± 6.69   Final old_new: 28.09 ± 4.66   Final new_new: 12.27 ± 3.80
Hits@50:
  Final val: 69.15 ± 2.01   Final Test: 51.65 ± 4.78   Final old_old: 55.00 ± 4.91   Final old_new: 38.34 ± 4.59   Final new_new: 18.64 ± 6.31
AUC:
  Final val: 96.44 ± 0.24   Final Test: 95.74 ± 0.25   Final old_old: 96.45 ± 0.24   Final old_new: 92.85 ± 0.32   Final new_new: 90.47 ± 1.52
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.78 ± 4.39   Final Test: 35.99 ± 2.47   Final old_old: 39.24 ± 2.70   Final old_new: 23.07 ± 1.99   Final new_new: 3.64 ± 2.03
Hits@20:
  Final val: 65.23 ± 1.68   Final Test: 48.47 ± 2.93   Final old_old: 52.26 ± 2.97   Final old_new: 33.55 ± 3.03   Final new_new: 8.64 ± 2.96
Hits@30:
  Final val: 71.32 ± 0.96   Final Test: 54.83 ± 1.02   Final old_old: 58.83 ± 0.85   Final old_new: 39.32 ± 1.97   Final new_new: 8.64 ± 1.90
Hits@50:
  Final val: 77.47 ± 0.93   Final Test: 63.05 ± 2.34   Final old_old: 67.01 ± 2.15   Final old_new: 47.71 ± 3.23   Final new_new: 16.36 ± 5.66
AUC:
  Final val: 97.64 ± 0.06   Final Test: 97.32 ± 0.09   Final old_old: 97.78 ± 0.10   Final old_new: 95.51 ± 0.22   Final new_new: 92.10 ± 0.29
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.38 ± 6.06   Final Test: 28.66 ± 3.34   Final old_old: 31.61 ± 3.59   Final old_new: 16.73 ± 2.59   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 56.04 ± 4.30   Final Test: 37.81 ± 4.71   Final old_old: 41.13 ± 5.16   Final old_new: 24.51 ± 3.17   Final new_new: 6.36 ± 3.37
Hits@30:
  Final val: 63.02 ± 3.15   Final Test: 45.34 ± 3.54   Final old_old: 49.06 ± 3.67   Final old_new: 30.53 ± 3.26   Final new_new: 8.64 ± 2.96
Hits@50:
  Final val: 71.75 ± 1.80   Final Test: 56.41 ± 2.77   Final old_old: 60.45 ± 2.94   Final old_new: 40.35 ± 2.35   Final new_new: 15.91 ± 1.61
AUC:
  Final val: 97.04 ± 0.22   Final Test: 96.49 ± 0.11   Final old_old: 97.14 ± 0.16   Final old_new: 93.93 ± 0.35   Final new_new: 89.83 ± 1.12
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 40.51 ± 1.48   Final Test: 26.31 ± 3.01   Final old_old: 28.74 ± 3.42   Final old_new: 16.55 ± 1.49   Final new_new: 4.09 ± 2.96
Hits@20:
  Final val: 51.78 ± 1.95   Final Test: 32.57 ± 2.90   Final old_old: 35.33 ± 2.94   Final old_new: 21.56 ± 2.84   Final new_new: 5.91 ± 3.05
Hits@30:
  Final val: 60.85 ± 2.31   Final Test: 38.91 ± 4.07   Final old_old: 41.84 ± 4.28   Final old_new: 27.28 ± 3.37   Final new_new: 9.55 ± 2.96
Hits@50:
  Final val: 69.27 ± 0.77   Final Test: 51.35 ± 2.25   Final old_old: 54.93 ± 2.31   Final old_new: 37.25 ± 2.69   Final new_new: 13.18 ± 1.90
AUC:
  Final val: 96.42 ± 0.25   Final Test: 95.67 ± 0.21   Final old_old: 96.43 ± 0.21   Final old_new: 92.59 ± 0.42   Final new_new: 88.53 ± 1.43
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.60 ± 4.84   Final Test: 32.69 ± 6.53   Final old_old: 35.91 ± 6.83   Final old_new: 19.77 ± 6.12   Final new_new: 3.18 ± 2.59
Hits@20:
  Final val: 61.74 ± 3.42   Final Test: 43.13 ± 5.24   Final old_old: 47.14 ± 5.43   Final old_new: 27.05 ± 5.05   Final new_new: 5.45 ± 4.43
Hits@30:
  Final val: 68.39 ± 2.27   Final Test: 53.88 ± 3.14   Final old_old: 58.05 ± 3.62   Final old_new: 37.56 ± 1.41   Final new_new: 7.73 ± 3.05
Hits@50:
  Final val: 75.81 ± 1.92   Final Test: 61.06 ± 4.53   Final old_old: 65.22 ± 4.63   Final old_new: 45.01 ± 4.68   Final new_new: 10.91 ± 2.96
AUC:
  Final val: 97.22 ± 0.19   Final Test: 96.59 ± 0.13   Final old_old: 97.34 ± 0.16   Final old_new: 93.61 ± 0.28   Final new_new: 88.94 ± 0.96
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 40.21 ± 1.95   Final Test: 30.58 ± 3.51   Final old_old: 33.25 ± 3.70   Final old_new: 19.87 ± 2.90   Final new_new: 6.36 ± 1.02
Hits@20:
  Final val: 53.04 ± 1.31   Final Test: 38.63 ± 5.12   Final old_old: 41.60 ± 5.47   Final old_new: 26.73 ± 3.91   Final new_new: 11.36 ± 3.94
Hits@30:
  Final val: 60.08 ± 1.34   Final Test: 44.96 ± 4.20   Final old_old: 48.22 ± 4.54   Final old_new: 31.89 ± 2.83   Final new_new: 14.09 ± 5.66
Hits@50:
  Final val: 68.82 ± 1.11   Final Test: 53.17 ± 3.42   Final old_old: 56.63 ± 3.58   Final old_new: 39.62 ± 2.96   Final new_new: 15.00 ± 4.13
AUC:
  Final val: 96.45 ± 0.29   Final Test: 95.74 ± 0.21   Final old_old: 96.46 ± 0.24   Final old_new: 92.83 ± 0.36   Final new_new: 89.47 ± 1.22
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.28 ± 3.63   Final Test: 31.41 ± 6.66   Final old_old: 34.08 ± 7.15   Final old_new: 20.73 ± 4.96   Final new_new: 6.36 ± 3.73
Hits@20:
  Final val: 62.55 ± 2.47   Final Test: 43.11 ± 5.49   Final old_old: 46.36 ± 5.87   Final old_new: 30.28 ± 4.20   Final new_new: 10.00 ± 4.71
Hits@30:
  Final val: 69.77 ± 1.63   Final Test: 53.15 ± 3.42   Final old_old: 56.86 ± 3.91   Final old_new: 38.51 ± 2.13   Final new_new: 14.55 ± 9.72
Hits@50:
  Final val: 76.14 ± 1.30   Final Test: 62.10 ± 1.08   Final old_old: 66.03 ± 1.29   Final old_new: 46.80 ± 0.69   Final new_new: 17.73 ± 8.10
AUC:
  Final val: 96.87 ± 0.13   Final Test: 96.27 ± 0.25   Final old_old: 96.95 ± 0.23   Final old_new: 93.56 ± 0.40   Final new_new: 89.82 ± 2.32
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.52 ± 4.52   Final Test: 32.67 ± 2.74   Final old_old: 36.04 ± 3.03   Final old_new: 19.07 ± 1.94   Final new_new: 3.18 ± 1.24
Hits@20:
  Final val: 61.45 ± 3.78   Final Test: 42.74 ± 4.80   Final old_old: 46.61 ± 5.06   Final old_new: 27.25 ± 3.90   Final new_new: 5.45 ± 2.59
Hits@30:
  Final val: 68.73 ± 1.92   Final Test: 51.74 ± 2.15   Final old_old: 56.08 ± 2.30   Final old_new: 34.43 ± 1.61   Final new_new: 10.00 ± 4.13
Hits@50:
  Final val: 75.98 ± 1.32   Final Test: 61.86 ± 2.27   Final old_old: 66.28 ± 2.55   Final old_new: 44.46 ± 1.68   Final new_new: 15.45 ± 4.37
AUC:
  Final val: 97.22 ± 0.23   Final Test: 96.58 ± 0.18   Final old_old: 97.34 ± 0.23   Final old_new: 93.58 ± 0.50   Final new_new: 88.30 ± 2.09
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.83 ± 1.68   Final Test: 31.72 ± 1.86   Final old_old: 35.03 ± 2.08   Final old_new: 18.36 ± 1.54   Final new_new: 2.73 ± 1.02
Hits@20:
  Final val: 58.21 ± 1.83   Final Test: 40.93 ± 2.87   Final old_old: 44.58 ± 2.78   Final old_new: 26.30 ± 3.39   Final new_new: 6.36 ± 4.07
Hits@30:
  Final val: 65.17 ± 2.09   Final Test: 47.27 ± 0.49   Final old_old: 51.32 ± 0.36   Final old_new: 31.21 ± 1.29   Final new_new: 5.91 ± 2.03
Hits@50:
  Final val: 72.77 ± 1.47   Final Test: 54.93 ± 2.78   Final old_old: 59.29 ± 2.92   Final old_new: 37.61 ± 2.66   Final new_new: 10.91 ± 3.73
AUC:
  Final val: 97.12 ± 0.13   Final Test: 96.46 ± 0.11   Final old_old: 97.13 ± 0.13   Final old_new: 93.78 ± 0.30   Final new_new: 89.77 ± 1.23
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 53.08 ± 2.52   Final Test: 36.56 ± 7.97   Final old_old: 39.99 ± 8.40   Final old_new: 22.75 ± 6.49   Final new_new: 5.45 ± 2.59
Hits@20:
  Final val: 66.46 ± 2.24   Final Test: 46.47 ± 3.78   Final old_old: 50.35 ± 3.84   Final old_new: 30.96 ± 3.69   Final new_new: 9.09 ± 3.21
Hits@30:
  Final val: 72.71 ± 1.30   Final Test: 54.72 ± 3.30   Final old_old: 58.75 ± 3.40   Final old_new: 38.89 ± 3.09   Final new_new: 11.36 ± 3.59
Hits@50:
  Final val: 79.12 ± 0.93   Final Test: 65.53 ± 2.72   Final old_old: 69.50 ± 2.77   Final old_new: 50.25 ± 2.61   Final new_new: 17.73 ± 4.37
AUC:
  Final val: 97.53 ± 0.09   Final Test: 97.15 ± 0.11   Final old_old: 97.67 ± 0.13   Final old_new: 95.06 ± 0.17   Final new_new: 92.21 ± 0.75
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.23 ± 3.84   Final Test: 33.84 ± 0.86   Final old_old: 37.23 ± 1.02   Final old_new: 20.13 ± 0.85   Final new_new: 4.55 ± 3.59
Hits@20:
  Final val: 61.13 ± 4.01   Final Test: 44.70 ± 5.33   Final old_old: 48.47 ± 5.82   Final old_new: 29.67 ± 3.33   Final new_new: 8.64 ± 3.73
Hits@30:
  Final val: 67.61 ± 2.98   Final Test: 50.98 ± 3.33   Final old_old: 55.12 ± 3.53   Final old_new: 34.53 ± 2.88   Final new_new: 9.55 ± 2.49
Hits@50:
  Final val: 75.11 ± 2.51   Final Test: 61.52 ± 3.14   Final old_old: 65.81 ± 3.49   Final old_new: 44.63 ± 1.87   Final new_new: 15.91 ± 1.61
AUC:
  Final val: 97.22 ± 0.22   Final Test: 96.59 ± 0.21   Final old_old: 97.32 ± 0.22   Final old_new: 93.69 ± 0.25   Final new_new: 89.18 ± 0.89
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.34 ± 1.69   Final Test: 27.20 ± 3.22   Final old_old: 30.23 ± 3.49   Final old_new: 14.84 ± 2.32   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 58.70 ± 1.80   Final Test: 37.87 ± 3.26   Final old_old: 41.48 ± 3.52   Final old_new: 23.35 ± 2.22   Final new_new: 5.91 ± 3.45
Hits@30:
  Final val: 65.48 ± 1.34   Final Test: 45.92 ± 1.58   Final old_old: 49.89 ± 1.69   Final old_new: 30.03 ± 1.25   Final new_new: 8.64 ± 2.96
Hits@50:
  Final val: 72.71 ± 1.27   Final Test: 56.64 ± 2.18   Final old_old: 60.96 ± 2.40   Final old_new: 39.52 ± 1.55   Final new_new: 12.73 ± 4.43
AUC:
  Final val: 97.13 ± 0.17   Final Test: 96.52 ± 0.12   Final old_old: 97.20 ± 0.14   Final old_new: 93.82 ± 0.07   Final new_new: 89.90 ± 0.98
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.40 ± 5.88   Final Test: 26.18 ± 4.83   Final old_old: 28.55 ± 5.14   Final old_new: 16.42 ± 3.70   Final new_new: 9.09 ± 5.57
Hits@20:
  Final val: 54.85 ± 4.96   Final Test: 34.81 ± 4.13   Final old_old: 37.57 ± 4.68   Final old_new: 23.78 ± 2.55   Final new_new: 9.09 ± 3.21
Hits@30:
  Final val: 61.23 ± 4.76   Final Test: 46.79 ± 4.23   Final old_old: 50.10 ± 4.80   Final old_new: 33.58 ± 2.02   Final new_new: 15.00 ± 6.55
Hits@50:
  Final val: 70.24 ± 3.59   Final Test: 53.80 ± 3.18   Final old_old: 57.58 ± 3.34   Final old_new: 38.82 ± 3.18   Final new_new: 15.91 ± 3.59
AUC:
  Final val: 96.47 ± 0.31   Final Test: 95.76 ± 0.22   Final old_old: 96.47 ± 0.22   Final old_new: 92.79 ± 0.46   Final new_new: 90.84 ± 0.56
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.08 ± 3.44   Final Test: 31.25 ± 3.72   Final old_old: 34.52 ± 3.92   Final old_new: 17.98 ± 3.08   Final new_new: 4.09 ± 2.49
Hits@20:
  Final val: 58.48 ± 2.76   Final Test: 39.65 ± 5.47   Final old_old: 43.28 ± 5.61   Final old_new: 25.06 ± 5.12   Final new_new: 6.36 ± 2.49
Hits@30:
  Final val: 65.21 ± 2.32   Final Test: 47.16 ± 3.53   Final old_old: 51.10 ± 3.65   Final old_new: 31.46 ± 3.21   Final new_new: 8.64 ± 4.07
Hits@50:
  Final val: 72.58 ± 1.73   Final Test: 56.74 ± 1.06   Final old_old: 60.94 ± 0.82   Final old_new: 40.23 ± 2.17   Final new_new: 11.36 ± 4.25
AUC:
  Final val: 97.18 ± 0.16   Final Test: 96.58 ± 0.09   Final old_old: 97.24 ± 0.17   Final old_new: 93.95 ± 0.24   Final new_new: 89.75 ± 0.88
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.00 ± 3.03   Final Test: 34.29 ± 3.52   Final old_old: 37.57 ± 3.63   Final old_new: 21.08 ± 3.25   Final new_new: 5.45 ± 2.59
Hits@20:
  Final val: 64.54 ± 2.81   Final Test: 45.62 ± 2.12   Final old_old: 49.08 ± 1.91   Final old_new: 31.94 ± 3.34   Final new_new: 9.55 ± 3.73
Hits@30:
  Final val: 70.96 ± 1.90   Final Test: 54.10 ± 3.03   Final old_old: 57.93 ± 3.20   Final old_new: 38.94 ± 2.49   Final new_new: 14.55 ± 6.14
Hits@50:
  Final val: 78.13 ± 1.22   Final Test: 62.88 ± 1.54   Final old_old: 66.82 ± 1.87   Final old_new: 47.51 ± 1.33   Final new_new: 18.18 ± 5.08
AUC:
  Final val: 97.63 ± 0.06   Final Test: 97.24 ± 0.04   Final old_old: 97.73 ± 0.09   Final old_new: 95.31 ± 0.20   Final new_new: 92.26 ± 0.48
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.18 ± 3.11   Final Test: 28.66 ± 3.72   Final old_old: 30.94 ± 3.86   Final old_new: 19.55 ± 3.74   Final new_new: 7.27 ± 4.93
Hits@20:
  Final val: 53.22 ± 3.05   Final Test: 35.23 ± 6.37   Final old_old: 37.86 ± 6.68   Final old_new: 24.76 ± 5.62   Final new_new: 9.55 ± 3.73
Hits@30:
  Final val: 60.44 ± 2.76   Final Test: 43.16 ± 4.07   Final old_old: 46.64 ± 4.16   Final old_new: 29.32 ± 4.16   Final new_new: 9.55 ± 3.37
Hits@50:
  Final val: 69.35 ± 1.95   Final Test: 54.30 ± 4.80   Final old_old: 58.03 ± 5.04   Final old_new: 39.60 ± 4.66   Final new_new: 15.45 ± 1.90
AUC:
  Final val: 96.36 ± 0.28   Final Test: 95.67 ± 0.19   Final old_old: 96.37 ± 0.21   Final old_new: 92.81 ± 0.36   Final new_new: 89.85 ± 0.49
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.75 ± 2.89   Final Test: 35.16 ± 2.56   Final old_old: 38.96 ± 2.71   Final old_new: 19.67 ± 2.44   Final new_new: 3.64 ± 2.03
Hits@20:
  Final val: 60.96 ± 2.93   Final Test: 44.98 ± 5.91   Final old_old: 49.01 ± 6.39   Final old_new: 28.94 ± 4.37   Final new_new: 5.45 ± 2.03
Hits@30:
  Final val: 67.36 ± 2.09   Final Test: 52.78 ± 4.93   Final old_old: 57.18 ± 5.22   Final old_new: 35.26 ± 4.09   Final new_new: 9.55 ± 2.49
Hits@50:
  Final val: 75.28 ± 1.42   Final Test: 60.22 ± 4.03   Final old_old: 64.61 ± 4.29   Final old_new: 42.85 ± 3.32   Final new_new: 14.55 ± 1.24
AUC:
  Final val: 97.19 ± 0.18   Final Test: 96.63 ± 0.15   Final old_old: 97.39 ± 0.18   Final old_new: 93.55 ± 0.29   Final new_new: 89.76 ± 1.09
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 40.13 ± 6.08   Final Test: 28.90 ± 3.86   Final old_old: 31.48 ± 4.24   Final old_new: 18.49 ± 2.37   Final new_new: 5.91 ± 2.59
Hits@20:
  Final val: 51.28 ± 3.92   Final Test: 35.27 ± 3.48   Final old_old: 38.06 ± 3.67   Final old_new: 23.93 ± 3.31   Final new_new: 11.36 ± 3.21
Hits@30:
  Final val: 58.94 ± 3.40   Final Test: 43.69 ± 3.48   Final old_old: 46.89 ± 3.94   Final old_new: 30.98 ± 2.61   Final new_new: 11.36 ± 4.82
Hits@50:
  Final val: 69.83 ± 3.13   Final Test: 54.02 ± 4.07   Final old_old: 57.57 ± 4.35   Final old_new: 39.97 ± 3.66   Final new_new: 17.27 ± 2.59
AUC:
  Final val: 96.40 ± 0.29   Final Test: 95.68 ± 0.26   Final old_old: 96.38 ± 0.21   Final old_new: 92.78 ± 0.58   Final new_new: 90.25 ± 1.43
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.96 ± 7.58   Final Test: 33.39 ± 5.88   Final old_old: 36.14 ± 5.98   Final old_new: 22.52 ± 5.76   Final new_new: 5.00 ± 2.49
Hits@20:
  Final val: 61.36 ± 5.82   Final Test: 44.10 ± 3.95   Final old_old: 47.41 ± 4.17   Final old_new: 31.08 ± 3.48   Final new_new: 9.09 ± 3.21
Hits@30:
  Final val: 67.38 ± 4.26   Final Test: 50.67 ± 3.57   Final old_old: 54.33 ± 3.82   Final old_new: 36.17 ± 2.84   Final new_new: 13.64 ± 4.25
Hits@50:
  Final val: 75.21 ± 1.36   Final Test: 57.00 ± 4.10   Final old_old: 60.81 ± 4.15   Final old_new: 42.04 ± 3.99   Final new_new: 16.36 ± 8.10
AUC:
  Final val: 96.80 ± 0.29   Final Test: 96.34 ± 0.22   Final old_old: 96.98 ± 0.18   Final old_new: 93.78 ± 0.49   Final new_new: 90.28 ± 1.54
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.34 ± 2.26   Final Test: 31.91 ± 3.83   Final old_old: 35.30 ± 4.15   Final old_new: 18.21 ± 3.00   Final new_new: 2.27 ± 1.61
Hits@20:
  Final val: 60.26 ± 2.19   Final Test: 43.02 ± 3.34   Final old_old: 46.92 ± 3.78   Final old_new: 27.53 ± 1.63   Final new_new: 3.64 ± 2.03
Hits@30:
  Final val: 66.62 ± 2.35   Final Test: 50.23 ± 2.48   Final old_old: 54.45 ± 2.78   Final old_new: 33.53 ± 1.71   Final new_new: 6.82 ± 1.61
Hits@50:
  Final val: 74.78 ± 1.96   Final Test: 59.68 ± 4.02   Final old_old: 63.87 ± 4.42   Final old_new: 43.35 ± 2.40   Final new_new: 12.73 ± 6.55
AUC:
  Final val: 97.20 ± 0.18   Final Test: 96.53 ± 0.13   Final old_old: 97.28 ± 0.20   Final old_new: 93.56 ± 0.28   Final new_new: 88.41 ± 0.77
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.18 ± 2.18   Final Test: 31.46 ± 5.38   Final old_old: 34.72 ± 5.66   Final old_new: 18.24 ± 4.35   Final new_new: 4.09 ± 3.37
Hits@20:
  Final val: 58.97 ± 0.87   Final Test: 40.72 ± 4.28   Final old_old: 44.47 ± 4.21   Final old_new: 25.69 ± 4.85   Final new_new: 5.91 ± 2.03
Hits@30:
  Final val: 65.02 ± 1.20   Final Test: 46.03 ± 3.48   Final old_old: 50.15 ± 3.49   Final old_new: 29.55 ± 3.60   Final new_new: 7.27 ± 1.90
Hits@50:
  Final val: 73.17 ± 0.71   Final Test: 57.29 ± 4.06   Final old_old: 61.58 ± 4.05   Final old_new: 40.40 ± 4.12   Final new_new: 11.36 ± 4.55
AUC:
  Final val: 97.16 ± 0.13   Final Test: 96.46 ± 0.12   Final old_old: 97.14 ± 0.17   Final old_new: 93.77 ± 0.16   Final new_new: 88.71 ± 1.20
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.05 ± 1.69   Final Test: 35.20 ± 4.27   Final old_old: 38.63 ± 4.56   Final old_new: 21.18 ± 3.11   Final new_new: 7.73 ± 3.45
Hits@20:
  Final val: 56.91 ± 2.46   Final Test: 41.81 ± 3.41   Final old_old: 45.46 ± 3.60   Final old_new: 27.13 ± 2.87   Final new_new: 9.55 ± 3.73
Hits@30:
  Final val: 65.63 ± 2.00   Final Test: 48.75 ± 1.99   Final old_old: 52.87 ± 2.21   Final old_new: 32.22 ± 1.30   Final new_new: 10.45 ± 4.13
Hits@50:
  Final val: 74.20 ± 1.70   Final Test: 57.35 ± 2.65   Final old_old: 61.51 ± 2.49   Final old_new: 40.83 ± 3.72   Final new_new: 15.45 ± 4.93
AUC:
  Final val: 97.14 ± 0.18   Final Test: 96.62 ± 0.18   Final old_old: 97.34 ± 0.17   Final old_new: 93.74 ± 0.29   Final new_new: 89.86 ± 1.09
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 53.77 ± 5.35   Final Test: 34.35 ± 5.35   Final old_old: 37.53 ± 5.65   Final old_new: 21.51 ± 4.31   Final new_new: 6.36 ± 2.96
Hits@20:
  Final val: 67.45 ± 2.17   Final Test: 48.87 ± 4.39   Final old_old: 52.66 ± 4.81   Final old_new: 33.90 ± 2.58   Final new_new: 9.55 ± 4.37
Hits@30:
  Final val: 72.76 ± 1.62   Final Test: 53.67 ± 2.82   Final old_old: 57.56 ± 3.03   Final old_new: 38.31 ± 2.07   Final new_new: 13.18 ± 4.07
Hits@50:
  Final val: 78.81 ± 1.09   Final Test: 63.87 ± 2.93   Final old_old: 67.91 ± 3.06   Final old_new: 47.93 ± 2.65   Final new_new: 21.36 ± 4.13
AUC:
  Final val: 97.55 ± 0.10   Final Test: 97.13 ± 0.08   Final old_old: 97.67 ± 0.10   Final old_new: 95.02 ± 0.13   Final new_new: 91.53 ± 0.79
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.57 ± 2.98   Final Test: 30.57 ± 2.95   Final old_old: 33.85 ± 3.34   Final old_new: 17.25 ± 1.96   Final new_new: 3.18 ± 1.24
Hits@20:
  Final val: 59.45 ± 2.82   Final Test: 39.01 ± 3.14   Final old_old: 42.55 ± 3.40   Final old_new: 24.76 ± 2.16   Final new_new: 6.82 ± 3.21
Hits@30:
  Final val: 64.44 ± 2.51   Final Test: 47.40 ± 2.91   Final old_old: 51.14 ± 3.07   Final old_new: 32.57 ± 2.32   Final new_new: 10.00 ± 3.80
Hits@50:
  Final val: 72.23 ± 1.52   Final Test: 55.03 ± 2.50   Final old_old: 59.03 ± 2.66   Final old_new: 39.24 ± 2.20   Final new_new: 14.09 ± 2.96
AUC:
  Final val: 97.12 ± 0.17   Final Test: 96.51 ± 0.11   Final old_old: 97.18 ± 0.18   Final old_new: 93.87 ± 0.23   Final new_new: 89.07 ± 0.88
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.59 ± 4.49   Final Test: 30.18 ± 2.27   Final old_old: 32.89 ± 2.39   Final old_new: 19.40 ± 1.95   Final new_new: 4.09 ± 1.02
Hits@20:
  Final val: 54.62 ± 3.32   Final Test: 37.43 ± 5.64   Final old_old: 40.47 ± 5.86   Final old_new: 25.11 ± 4.91   Final new_new: 11.36 ± 4.25
Hits@30:
  Final val: 61.96 ± 2.24   Final Test: 44.46 ± 3.05   Final old_old: 47.89 ± 3.10   Final old_new: 30.78 ± 2.91   Final new_new: 11.36 ± 6.43
Hits@50:
  Final val: 70.06 ± 1.18   Final Test: 54.03 ± 3.65   Final old_old: 57.76 ± 3.89   Final old_new: 39.09 ± 2.99   Final new_new: 18.64 ± 7.77
AUC:
  Final val: 96.41 ± 0.39   Final Test: 95.69 ± 0.29   Final old_old: 96.39 ± 0.31   Final old_new: 92.79 ± 0.33   Final new_new: 90.04 ± 0.47
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.94 ± 3.75   Final Test: 26.71 ± 5.73   Final old_old: 29.62 ± 6.05   Final old_new: 14.74 ± 4.51   Final new_new: 5.00 ± 2.49
Hits@20:
  Final val: 57.68 ± 2.25   Final Test: 38.46 ± 4.26   Final old_old: 42.03 ± 4.26   Final old_new: 24.01 ± 4.29   Final new_new: 7.73 ± 4.71
Hits@30:
  Final val: 65.46 ± 1.99   Final Test: 48.35 ± 3.66   Final old_old: 52.23 ± 3.54   Final old_new: 32.87 ± 4.15   Final new_new: 11.36 ± 6.43
Hits@50:
  Final val: 72.96 ± 1.35   Final Test: 58.08 ± 1.35   Final old_old: 62.26 ± 1.40   Final old_new: 41.54 ± 1.61   Final new_new: 15.00 ± 3.45
AUC:
  Final val: 97.14 ± 0.14   Final Test: 96.56 ± 0.09   Final old_old: 97.23 ± 0.10   Final old_new: 93.93 ± 0.28   Final new_new: 89.96 ± 1.14
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.02 ± 3.85   Final Test: 26.05 ± 3.58   Final old_old: 28.54 ± 3.88   Final old_new: 15.89 ± 2.65   Final new_new: 5.91 ± 2.59
Hits@20:
  Final val: 51.74 ± 3.61   Final Test: 34.93 ± 4.75   Final old_old: 37.89 ± 5.21   Final old_new: 23.02 ± 2.84   Final new_new: 7.73 ± 6.55
Hits@30:
  Final val: 59.92 ± 3.17   Final Test: 44.32 ± 5.64   Final old_old: 47.55 ± 5.98   Final old_new: 31.51 ± 4.45   Final new_new: 12.27 ± 6.14
Hits@50:
  Final val: 69.42 ± 2.52   Final Test: 55.73 ± 3.61   Final old_old: 59.35 ± 3.74   Final old_new: 41.36 ± 3.38   Final new_new: 19.09 ± 3.80
AUC:
  Final val: 96.50 ± 0.24   Final Test: 95.75 ± 0.21   Final old_old: 96.48 ± 0.22   Final old_new: 92.73 ± 0.41   Final new_new: 90.59 ± 1.66
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.57 ± 1.02   Final Test: 33.15 ± 6.89   Final old_old: 36.22 ± 7.17   Final old_new: 20.83 ± 5.89   Final new_new: 5.00 ± 4.07
Hits@20:
  Final val: 64.00 ± 3.05   Final Test: 47.46 ± 3.30   Final old_old: 51.28 ± 3.64   Final old_new: 32.32 ± 2.46   Final new_new: 8.64 ± 4.07
Hits@30:
  Final val: 70.31 ± 1.56   Final Test: 53.36 ± 2.73   Final old_old: 57.23 ± 3.05   Final old_new: 38.24 ± 2.22   Final new_new: 10.45 ± 4.13
Hits@50:
  Final val: 76.40 ± 1.34   Final Test: 62.38 ± 3.39   Final old_old: 66.36 ± 3.59   Final old_new: 46.93 ± 3.00   Final new_new: 15.91 ± 5.33
AUC:
  Final val: 97.55 ± 0.04   Final Test: 97.20 ± 0.08   Final old_old: 97.69 ± 0.12   Final old_new: 95.27 ± 0.19   Final new_new: 91.92 ± 0.67
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.82 ± 3.59   Final Test: 31.83 ± 4.06   Final old_old: 35.11 ± 4.35   Final old_new: 18.56 ± 3.27   Final new_new: 2.73 ± 2.96
Hits@20:
  Final val: 59.85 ± 2.76   Final Test: 42.52 ± 2.30   Final old_old: 46.25 ± 2.37   Final old_new: 27.63 ± 2.69   Final new_new: 6.36 ± 3.73
Hits@30:
  Final val: 67.14 ± 1.05   Final Test: 51.93 ± 2.56   Final old_old: 55.95 ± 2.88   Final old_new: 35.99 ± 2.02   Final new_new: 10.91 ± 3.73
Hits@50:
  Final val: 74.40 ± 1.29   Final Test: 60.12 ± 2.66   Final old_old: 64.36 ± 2.89   Final old_new: 43.43 ± 1.95   Final new_new: 14.55 ± 6.14
AUC:
  Final val: 97.15 ± 0.18   Final Test: 96.55 ± 0.17   Final old_old: 97.29 ± 0.18   Final old_new: 93.58 ± 0.39   Final new_new: 90.16 ± 1.73
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 38.35 ± 4.13   Final Test: 26.39 ± 4.34   Final old_old: 28.34 ± 4.13   Final old_new: 18.56 ± 5.44   Final new_new: 8.18 ± 3.45
Hits@20:
  Final val: 50.26 ± 4.30   Final Test: 29.57 ± 5.63   Final old_old: 31.91 ± 5.98   Final old_new: 20.10 ± 4.27   Final new_new: 9.09 ± 3.59
Hits@30:
  Final val: 58.41 ± 3.28   Final Test: 40.61 ± 3.29   Final old_old: 43.47 ± 3.76   Final old_new: 29.19 ± 1.90   Final new_new: 13.18 ± 3.37
Hits@50:
  Final val: 67.97 ± 2.24   Final Test: 50.64 ± 1.13   Final old_old: 53.90 ± 1.27   Final old_new: 37.78 ± 2.22   Final new_new: 16.82 ± 3.45
AUC:
  Final val: 96.42 ± 0.33   Final Test: 95.77 ± 0.29   Final old_old: 96.45 ± 0.25   Final old_new: 93.04 ± 0.61   Final new_new: 89.77 ± 2.22
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.00 ± 3.39   Final Test: 31.96 ± 5.08   Final old_old: 35.31 ± 5.41   Final old_new: 18.34 ± 4.04   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 59.67 ± 1.68   Final Test: 45.13 ± 5.30   Final old_old: 49.16 ± 5.61   Final old_new: 29.07 ± 4.24   Final new_new: 6.36 ± 2.49
Hits@30:
  Final val: 66.87 ± 3.03   Final Test: 52.72 ± 2.55   Final old_old: 57.04 ± 2.97   Final old_new: 35.52 ± 1.09   Final new_new: 10.45 ± 3.05
Hits@50:
  Final val: 74.83 ± 2.74   Final Test: 59.75 ± 4.98   Final old_old: 64.27 ± 5.15   Final old_new: 41.84 ± 4.68   Final new_new: 13.64 ± 3.59
AUC:
  Final val: 97.20 ± 0.26   Final Test: 96.62 ± 0.20   Final old_old: 97.35 ± 0.19   Final old_new: 93.78 ± 0.29   Final new_new: 88.80 ± 1.01
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.56 ± 2.74   Final Test: 27.81 ± 8.53   Final old_old: 30.05 ± 9.09   Final old_new: 18.92 ± 6.52   Final new_new: 5.91 ± 1.24
Hits@20:
  Final val: 59.78 ± 1.15   Final Test: 40.16 ± 6.09   Final old_old: 43.02 ± 6.35   Final old_new: 28.87 ± 5.31   Final new_new: 10.45 ± 2.59
Hits@30:
  Final val: 66.95 ± 2.75   Final Test: 50.43 ± 3.21   Final old_old: 53.91 ± 3.39   Final old_new: 36.75 ± 2.92   Final new_new: 13.18 ± 4.37
Hits@50:
  Final val: 74.29 ± 2.04   Final Test: 58.57 ± 2.10   Final old_old: 62.30 ± 1.95   Final old_new: 43.88 ± 2.94   Final new_new: 18.64 ± 6.89
AUC:
  Final val: 96.83 ± 0.19   Final Test: 96.31 ± 0.24   Final old_old: 96.91 ± 0.18   Final old_new: 93.91 ± 0.52   Final new_new: 90.00 ± 1.43
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.48 ± 3.83   Final Test: 34.39 ± 2.17   Final old_old: 37.96 ± 2.62   Final old_new: 19.87 ± 0.54   Final new_new: 4.09 ± 1.02
Hits@20:
  Final val: 59.71 ± 1.89   Final Test: 45.12 ± 3.68   Final old_old: 49.17 ± 3.84   Final old_new: 28.92 ± 3.34   Final new_new: 6.82 ± 2.78
Hits@30:
  Final val: 66.88 ± 2.94   Final Test: 51.18 ± 2.53   Final old_old: 55.32 ± 2.61   Final old_new: 34.81 ± 2.33   Final new_new: 8.64 ± 1.90
Hits@50:
  Final val: 74.89 ± 1.89   Final Test: 60.34 ± 3.33   Final old_old: 64.68 ± 3.58   Final old_new: 43.20 ± 2.70   Final new_new: 15.91 ± 4.25
AUC:
  Final val: 97.16 ± 0.22   Final Test: 96.62 ± 0.19   Final old_old: 97.34 ± 0.19   Final old_new: 93.74 ± 0.35   Final new_new: 89.93 ± 0.68
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.63 ± 3.50   Final Test: 29.81 ± 2.54   Final old_old: 32.90 ± 2.78   Final old_new: 17.33 ± 1.91   Final new_new: 2.73 ± 1.90
Hits@20:
  Final val: 60.93 ± 2.18   Final Test: 39.67 ± 2.17   Final old_old: 43.42 ± 2.18   Final old_new: 24.63 ± 2.38   Final new_new: 4.55 ± 1.61
Hits@30:
  Final val: 66.36 ± 1.43   Final Test: 48.54 ± 1.15   Final old_old: 52.61 ± 1.32   Final old_new: 32.37 ± 0.97   Final new_new: 7.73 ± 2.03
Hits@50:
  Final val: 72.99 ± 0.84   Final Test: 56.19 ± 3.07   Final old_old: 60.55 ± 3.24   Final old_new: 38.92 ± 2.52   Final new_new: 12.27 ± 3.05
AUC:
  Final val: 97.17 ± 0.11   Final Test: 96.52 ± 0.10   Final old_old: 97.22 ± 0.12   Final old_new: 93.77 ± 0.09   Final new_new: 89.06 ± 1.35
All runs:
Hits@10:
  Final val: 44.58 ± 1.85   Final Test: 27.66 ± 4.83   Final old_old: 30.57 ± 5.51   Final old_new: 15.82 ± 2.34   Final new_new: 4.09 ± 2.96
Hits@20:
  Final val: 57.57 ± 2.26   Final Test: 41.56 ± 3.98   Final old_old: 45.20 ± 4.05   Final old_new: 26.95 ± 4.18   Final new_new: 7.73 ± 2.59
Hits@30:
  Final val: 64.96 ± 2.07   Final Test: 46.11 ± 2.93   Final old_old: 49.94 ± 2.83   Final old_new: 30.88 ± 3.66   Final new_new: 8.18 ± 3.80
Hits@50:
  Final val: 72.52 ± 1.70   Final Test: 57.23 ± 2.08   Final old_old: 61.43 ± 2.41   Final old_new: 40.65 ± 2.12   Final new_new: 13.64 ± 6.01
AUC:
  Final val: 97.09 ± 0.19   Final Test: 96.51 ± 0.13   Final old_old: 97.19 ± 0.18   Final old_new: 93.84 ± 0.24   Final new_new: 89.19 ± 1.26
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 55.23 ± 3.22   Final Test: 34.43 ± 5.02   Final old_old: 37.52 ± 5.16   Final old_new: 22.14 ± 4.70   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 66.60 ± 3.00   Final Test: 47.47 ± 4.44   Final old_old: 51.10 ± 4.61   Final old_new: 33.00 ± 3.88   Final new_new: 12.27 ± 4.71
Hits@30:
  Final val: 73.24 ± 2.09   Final Test: 53.64 ± 3.84   Final old_old: 57.43 ± 3.87   Final old_new: 38.59 ± 3.86   Final new_new: 16.36 ± 5.88
Hits@50:
  Final val: 79.83 ± 1.15   Final Test: 65.37 ± 2.20   Final old_old: 69.34 ± 2.25   Final old_new: 49.67 ± 2.36   Final new_new: 24.09 ± 3.80
AUC:
  Final val: 97.69 ± 0.07   Final Test: 97.24 ± 0.03   Final old_old: 97.75 ± 0.03   Final old_new: 95.24 ± 0.10   Final new_new: 91.84 ± 0.84
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.04 ± 3.16   Final Test: 30.97 ± 3.79   Final old_old: 34.09 ± 4.12   Final old_new: 18.26 ± 2.65   Final new_new: 5.00 ± 1.02
Hits@20:
  Final val: 58.94 ± 2.30   Final Test: 41.41 ± 3.52   Final old_old: 45.03 ± 4.07   Final old_new: 26.98 ± 1.96   Final new_new: 6.36 ± 1.90
Hits@30:
  Final val: 65.76 ± 1.43   Final Test: 48.73 ± 1.68   Final old_old: 52.79 ± 1.70   Final old_new: 32.67 ± 2.05   Final new_new: 7.27 ± 3.37
Hits@50:
  Final val: 73.31 ± 0.79   Final Test: 58.40 ± 2.66   Final old_old: 62.96 ± 2.63   Final old_new: 40.43 ± 3.44   Final new_new: 10.45 ± 4.71
AUC:
  Final val: 97.13 ± 0.16   Final Test: 96.51 ± 0.09   Final old_old: 97.22 ± 0.13   Final old_new: 93.70 ± 0.11   Final new_new: 89.54 ± 0.97
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 39.09 ± 2.96   Final Test: 26.36 ± 4.20   Final old_old: 28.69 ± 4.45   Final old_new: 16.88 ± 3.47   Final new_new: 7.27 ± 4.37
Hits@20:
  Final val: 50.81 ± 3.25   Final Test: 36.75 ± 3.75   Final old_old: 39.70 ± 4.13   Final old_new: 24.86 ± 2.53   Final new_new: 10.00 ± 5.23
Hits@30:
  Final val: 59.53 ± 2.67   Final Test: 43.77 ± 3.65   Final old_old: 47.22 ± 3.93   Final old_new: 29.85 ± 2.53   Final new_new: 13.18 ± 7.94
Hits@50:
  Final val: 69.34 ± 3.04   Final Test: 54.04 ± 3.30   Final old_old: 57.83 ± 3.54   Final old_new: 38.92 ± 2.46   Final new_new: 17.27 ± 4.71
AUC:
  Final val: 96.47 ± 0.31   Final Test: 95.91 ± 0.21   Final old_old: 96.60 ± 0.20   Final old_new: 93.10 ± 0.42   Final new_new: 90.61 ± 0.76
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.51 ± 1.68   Final Test: 30.81 ± 5.74   Final old_old: 33.71 ± 6.18   Final old_new: 19.19 ± 4.18   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 64.13 ± 2.02   Final Test: 41.97 ± 5.26   Final old_old: 45.25 ± 5.34   Final old_new: 28.87 ± 5.13   Final new_new: 10.00 ± 3.45
Hits@30:
  Final val: 70.14 ± 1.13   Final Test: 54.74 ± 3.53   Final old_old: 58.45 ± 3.56   Final old_new: 40.20 ± 3.51   Final new_new: 14.09 ± 1.90
Hits@50:
  Final val: 77.24 ± 0.87   Final Test: 64.34 ± 0.41   Final old_old: 68.33 ± 0.49   Final old_new: 48.79 ± 0.39   Final new_new: 19.55 ± 2.59
AUC:
  Final val: 97.60 ± 0.09   Final Test: 97.22 ± 0.05   Final old_old: 97.68 ± 0.10   Final old_new: 95.43 ± 0.16   Final new_new: 92.46 ± 0.94
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 39.95 ± 3.81   Final Test: 28.99 ± 2.30   Final old_old: 31.53 ± 2.80   Final old_new: 18.66 ± 0.69   Final new_new: 7.73 ± 4.13
Hits@20:
  Final val: 49.99 ± 2.59   Final Test: 34.42 ± 6.34   Final old_old: 37.11 ± 6.70   Final old_new: 23.70 ± 5.23   Final new_new: 8.18 ± 4.43
Hits@30:
  Final val: 58.55 ± 3.44   Final Test: 42.78 ± 3.20   Final old_old: 46.07 ± 3.46   Final old_new: 29.75 ± 2.32   Final new_new: 8.64 ± 1.90
Hits@50:
  Final val: 68.01 ± 3.98   Final Test: 52.19 ± 4.19   Final old_old: 55.84 ± 4.56   Final old_new: 37.76 ± 3.02   Final new_new: 14.55 ± 4.13
AUC:
  Final val: 96.33 ± 0.42   Final Test: 95.66 ± 0.17   Final old_old: 96.37 ± 0.17   Final old_new: 92.77 ± 0.26   Final new_new: 89.26 ± 1.36
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.48 ± 4.08   Final Test: 33.60 ± 3.83   Final old_old: 37.17 ± 4.31   Final old_new: 19.19 ± 2.19   Final new_new: 2.73 ± 1.90
Hits@20:
  Final val: 62.19 ± 2.16   Final Test: 46.25 ± 6.24   Final old_old: 50.50 ± 6.53   Final old_new: 29.22 ± 5.53   Final new_new: 6.36 ± 1.90
Hits@30:
  Final val: 68.43 ± 1.73   Final Test: 52.16 ± 5.52   Final old_old: 56.49 ± 5.87   Final old_new: 35.01 ± 4.25   Final new_new: 8.18 ± 3.05
Hits@50:
  Final val: 75.32 ± 1.96   Final Test: 61.96 ± 3.55   Final old_old: 66.51 ± 3.67   Final old_new: 44.11 ± 3.54   Final new_new: 13.18 ± 1.90
AUC:
  Final val: 97.22 ± 0.27   Final Test: 96.61 ± 0.28   Final old_old: 97.36 ± 0.32   Final old_new: 93.62 ± 0.25   Final new_new: 89.53 ± 0.83
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.86 ± 3.05   Final Test: 31.43 ± 5.25   Final old_old: 34.34 ± 5.54   Final old_new: 19.75 ± 4.34   Final new_new: 5.00 ± 2.49
Hits@20:
  Final val: 62.43 ± 1.15   Final Test: 41.28 ± 5.58   Final old_old: 44.71 ± 5.91   Final old_new: 27.46 ± 4.42   Final new_new: 10.00 ± 2.59
Hits@30:
  Final val: 67.75 ± 1.11   Final Test: 47.42 ± 1.58   Final old_old: 51.04 ± 1.77   Final old_new: 32.87 ± 1.34   Final new_new: 14.55 ± 3.45
Hits@50:
  Final val: 73.38 ± 0.72   Final Test: 59.77 ± 1.81   Final old_old: 63.58 ± 1.87   Final old_new: 44.71 ± 1.97   Final new_new: 20.00 ± 4.07
AUC:
  Final val: 95.70 ± 0.20   Final Test: 95.35 ± 0.22   Final old_old: 95.86 ± 0.18   Final old_new: 93.48 ± 0.42   Final new_new: 87.38 ± 3.03
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.32 ± 2.53   Final Test: 35.71 ± 4.04   Final old_old: 39.34 ± 4.32   Final old_new: 21.01 ± 3.37   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 61.26 ± 3.63   Final Test: 44.06 ± 4.69   Final old_old: 48.13 ± 5.37   Final old_new: 27.71 ± 2.19   Final new_new: 7.27 ± 1.90
Hits@30:
  Final val: 68.27 ± 1.99   Final Test: 47.84 ± 3.89   Final old_old: 52.07 ± 4.28   Final old_new: 30.96 ± 3.17   Final new_new: 7.27 ± 2.49
Hits@50:
  Final val: 75.37 ± 2.00   Final Test: 58.98 ± 4.56   Final old_old: 63.42 ± 4.76   Final old_new: 41.44 ± 4.70   Final new_new: 12.73 ± 4.43
AUC:
  Final val: 97.22 ± 0.18   Final Test: 96.60 ± 0.15   Final old_old: 97.35 ± 0.19   Final old_new: 93.61 ± 0.09   Final new_new: 89.28 ± 0.75
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 53.78 ± 4.50   Final Test: 34.60 ± 2.44   Final old_old: 37.90 ± 2.55   Final old_new: 21.26 ± 2.20   Final new_new: 5.45 ± 2.59
Hits@20:
  Final val: 65.41 ± 1.79   Final Test: 47.90 ± 3.46   Final old_old: 51.63 ± 3.33   Final old_new: 33.12 ± 4.20   Final new_new: 10.45 ± 2.59
Hits@30:
  Final val: 69.90 ± 0.87   Final Test: 53.58 ± 3.75   Final old_old: 57.46 ± 3.72   Final old_new: 38.14 ± 4.06   Final new_new: 15.45 ± 5.43
Hits@50:
  Final val: 75.69 ± 0.34   Final Test: 63.11 ± 1.07   Final old_old: 66.81 ± 1.17   Final old_new: 48.49 ± 1.16   Final new_new: 24.55 ± 7.08
AUC:
  Final val: 95.67 ± 0.19   Final Test: 95.24 ± 0.16   Final old_old: 95.85 ± 0.15   Final old_new: 92.93 ± 0.36   Final new_new: 86.85 ± 2.90
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.05, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.09 ± 2.78   Final Test: 31.51 ± 3.36   Final old_old: 34.95 ± 3.53   Final old_new: 17.46 ± 2.82   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 56.67 ± 2.95   Final Test: 40.10 ± 1.70   Final old_old: 43.98 ± 1.64   Final old_new: 24.43 ± 2.48   Final new_new: 6.36 ± 3.73
Hits@30:
  Final val: 64.41 ± 2.33   Final Test: 48.84 ± 1.22   Final old_old: 52.96 ± 1.25   Final old_new: 32.42 ± 1.62   Final new_new: 9.09 ± 3.59
Hits@50:
  Final val: 72.87 ± 1.91   Final Test: 58.22 ± 2.84   Final old_old: 62.61 ± 3.04   Final old_new: 40.88 ± 2.27   Final new_new: 13.18 ± 5.88
AUC:
  Final val: 97.06 ± 0.23   Final Test: 96.55 ± 0.17   Final old_old: 97.25 ± 0.21   Final old_new: 93.77 ± 0.11   Final new_new: 89.54 ± 1.79
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.48 ± 3.22   Final Test: 31.15 ± 2.32   Final old_old: 34.35 ± 2.64   Final old_new: 18.11 ± 1.17   Final new_new: 5.00 ± 1.90
Hits@20:
  Final val: 57.72 ± 2.45   Final Test: 39.25 ± 2.86   Final old_old: 42.94 ± 3.07   Final old_new: 24.38 ± 2.22   Final new_new: 5.91 ± 2.59
Hits@30:
  Final val: 64.13 ± 1.46   Final Test: 45.19 ± 3.46   Final old_old: 49.19 ± 3.62   Final old_new: 29.22 ± 3.14   Final new_new: 6.82 ± 2.78
Hits@50:
  Final val: 72.16 ± 0.55   Final Test: 55.32 ± 2.83   Final old_old: 59.32 ± 2.92   Final old_new: 39.55 ± 2.66   Final new_new: 14.09 ± 3.73
AUC:
  Final val: 97.08 ± 0.06   Final Test: 96.55 ± 0.05   Final old_old: 97.23 ± 0.09   Final old_new: 93.87 ± 0.15   Final new_new: 89.35 ± 0.99
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.89 ± 1.57   Final Test: 32.57 ± 3.80   Final old_old: 35.83 ± 3.97   Final old_new: 19.37 ± 3.29   Final new_new: 4.55 ± 4.25
Hits@20:
  Final val: 63.25 ± 0.86   Final Test: 43.39 ± 3.03   Final old_old: 46.81 ± 3.05   Final old_new: 29.85 ± 3.18   Final new_new: 8.64 ± 3.37
Hits@30:
  Final val: 68.26 ± 0.74   Final Test: 52.70 ± 3.01   Final old_old: 56.36 ± 3.00   Final old_new: 38.31 ± 3.10   Final new_new: 13.64 ± 5.79
Hits@50:
  Final val: 74.28 ± 1.35   Final Test: 60.85 ± 1.60   Final old_old: 64.28 ± 1.75   Final old_new: 47.46 ± 1.43   Final new_new: 22.27 ± 4.07
AUC:
  Final val: 95.39 ± 0.18   Final Test: 95.05 ± 0.26   Final old_old: 95.72 ± 0.23   Final old_new: 92.42 ± 0.45   Final new_new: 87.37 ± 1.70
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 38.91 ± 4.81   Final Test: 26.08 ± 5.77   Final old_old: 28.39 ± 6.31   Final old_new: 16.83 ± 3.84   Final new_new: 4.09 ± 2.96
Hits@20:
  Final val: 52.48 ± 3.74   Final Test: 35.92 ± 6.72   Final old_old: 38.73 ± 7.07   Final old_new: 24.76 ± 5.80   Final new_new: 7.73 ± 3.45
Hits@30:
  Final val: 59.77 ± 3.70   Final Test: 44.68 ± 5.60   Final old_old: 47.94 ± 6.04   Final old_new: 31.71 ± 4.00   Final new_new: 12.27 ± 2.03
Hits@50:
  Final val: 69.29 ± 2.19   Final Test: 52.63 ± 3.36   Final old_old: 56.17 ± 3.59   Final old_new: 38.74 ± 2.59   Final new_new: 14.09 ± 5.43
AUC:
  Final val: 96.41 ± 0.21   Final Test: 95.75 ± 0.17   Final old_old: 96.44 ± 0.13   Final old_new: 92.99 ± 0.61   Final new_new: 88.97 ± 1.55
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 37.66 ± 5.64   Final Test: 24.45 ± 5.92   Final old_old: 26.62 ± 6.28   Final old_new: 15.62 ± 4.47   Final new_new: 6.82 ± 4.55
Hits@20:
  Final val: 50.09 ± 6.00   Final Test: 34.07 ± 6.02   Final old_old: 36.75 ± 6.57   Final old_new: 23.25 ± 3.86   Final new_new: 10.45 ± 2.03
Hits@30:
  Final val: 58.49 ± 4.68   Final Test: 41.11 ± 2.07   Final old_old: 44.45 ± 2.16   Final old_new: 27.63 ± 2.00   Final new_new: 12.27 ± 3.45
Hits@50:
  Final val: 67.98 ± 3.59   Final Test: 53.01 ± 2.58   Final old_old: 56.39 ± 3.01   Final old_new: 39.67 ± 1.42   Final new_new: 18.18 ± 3.21
AUC:
  Final val: 96.36 ± 0.32   Final Test: 95.68 ± 0.26   Final old_old: 96.35 ± 0.20   Final old_new: 92.93 ± 0.50   Final new_new: 90.44 ± 0.73
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.68 ± 4.27   Final Test: 26.63 ± 2.31   Final old_old: 29.03 ± 2.54   Final old_new: 16.85 ± 1.34   Final new_new: 6.36 ± 3.73
Hits@20:
  Final val: 58.59 ± 3.35   Final Test: 41.90 ± 1.78   Final old_old: 45.25 ± 1.77   Final old_new: 28.31 ± 2.07   Final new_new: 14.09 ± 4.07
Hits@30:
  Final val: 64.36 ± 1.85   Final Test: 45.34 ± 1.43   Final old_old: 48.72 ± 1.22   Final old_new: 31.79 ± 2.38   Final new_new: 13.64 ± 4.55
Hits@50:
  Final val: 71.59 ± 0.63   Final Test: 57.16 ± 2.09   Final old_old: 60.85 ± 2.26   Final old_new: 42.39 ± 1.76   Final new_new: 22.27 ± 4.66
AUC:
  Final val: 95.10 ± 0.23   Final Test: 94.65 ± 0.35   Final old_old: 95.34 ± 0.29   Final old_new: 91.99 ± 0.68   Final new_new: 85.98 ± 2.39
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.54 ± 3.07   Final Test: 32.36 ± 5.00   Final old_old: 35.66 ± 5.38   Final old_new: 18.92 ± 3.63   Final new_new: 5.00 ± 1.90
Hits@20:
  Final val: 60.00 ± 1.90   Final Test: 45.65 ± 3.31   Final old_old: 49.62 ± 3.51   Final old_new: 29.77 ± 2.57   Final new_new: 7.27 ± 2.96
Hits@30:
  Final val: 67.15 ± 2.56   Final Test: 52.83 ± 3.70   Final old_old: 57.14 ± 3.92   Final old_new: 35.72 ± 2.90   Final new_new: 9.55 ± 3.73
Hits@50:
  Final val: 75.18 ± 2.16   Final Test: 60.42 ± 5.46   Final old_old: 64.74 ± 5.38   Final old_new: 43.38 ± 5.78   Final new_new: 15.91 ± 7.01
AUC:
  Final val: 97.16 ± 0.25   Final Test: 96.67 ± 0.23   Final old_old: 97.36 ± 0.27   Final old_new: 93.94 ± 0.23   Final new_new: 89.88 ± 1.18
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.51 ± 3.30   Final Test: 33.33 ± 4.10   Final old_old: 36.77 ± 4.34   Final old_new: 19.45 ± 3.45   Final new_new: 2.73 ± 1.02
Hits@20:
  Final val: 60.87 ± 2.05   Final Test: 43.00 ± 4.72   Final old_old: 46.76 ± 5.05   Final old_new: 27.88 ± 3.50   Final new_new: 8.64 ± 2.96
Hits@30:
  Final val: 67.71 ± 1.48   Final Test: 49.71 ± 2.27   Final old_old: 53.82 ± 2.55   Final old_new: 33.27 ± 1.72   Final new_new: 10.45 ± 3.80
Hits@50:
  Final val: 74.95 ± 2.38   Final Test: 59.83 ± 3.55   Final old_old: 64.03 ± 3.91   Final old_new: 43.27 ± 2.13   Final new_new: 15.91 ± 2.78
AUC:
  Final val: 97.22 ± 0.26   Final Test: 96.67 ± 0.21   Final old_old: 97.39 ± 0.25   Final old_new: 93.82 ± 0.13   Final new_new: 89.68 ± 1.09
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 53.52 ± 2.28   Final Test: 34.57 ± 3.12   Final old_old: 37.91 ± 3.30   Final old_new: 21.08 ± 2.60   Final new_new: 5.91 ± 2.03
Hits@20:
  Final val: 61.39 ± 1.68   Final Test: 43.48 ± 2.47   Final old_old: 47.00 ± 2.46   Final old_new: 29.52 ± 3.12   Final new_new: 7.27 ± 2.96
Hits@30:
  Final val: 66.42 ± 1.60   Final Test: 51.51 ± 1.32   Final old_old: 55.35 ± 1.07   Final old_new: 36.32 ± 2.44   Final new_new: 12.27 ± 3.05
Hits@50:
  Final val: 73.13 ± 0.91   Final Test: 60.00 ± 0.94   Final old_old: 63.65 ± 0.91   Final old_new: 45.79 ± 1.49   Final new_new: 18.18 ± 2.78
AUC:
  Final val: 94.84 ± 0.31   Final Test: 94.60 ± 0.31   Final old_old: 95.22 ± 0.36   Final old_new: 92.26 ± 0.43   Final new_new: 86.04 ± 1.37
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.48 ± 2.77   Final Test: 30.01 ± 3.26   Final old_old: 33.14 ± 3.42   Final old_new: 17.30 ± 2.91   Final new_new: 3.64 ± 2.03
Hits@20:
  Final val: 58.02 ± 2.53   Final Test: 40.01 ± 5.22   Final old_old: 43.61 ± 5.45   Final old_new: 25.52 ± 4.52   Final new_new: 7.27 ± 3.37
Hits@30:
  Final val: 64.13 ± 2.15   Final Test: 47.40 ± 4.60   Final old_old: 51.23 ± 4.72   Final old_new: 32.12 ± 4.38   Final new_new: 10.00 ± 4.13
Hits@50:
  Final val: 72.07 ± 1.18   Final Test: 56.36 ± 1.39   Final old_old: 60.43 ± 1.58   Final old_new: 40.38 ± 1.26   Final new_new: 12.73 ± 2.59
AUC:
  Final val: 97.09 ± 0.18   Final Test: 96.52 ± 0.15   Final old_old: 97.19 ± 0.19   Final old_new: 93.87 ± 0.29   Final new_new: 90.19 ± 1.19
All runs:
Hits@10:
  Final val: 44.21 ± 2.51   Final Test: 26.89 ± 3.49   Final old_old: 29.76 ± 3.66   Final old_new: 15.11 ± 2.94   Final new_new: 5.00 ± 2.96
Hits@20:
  Final val: 58.11 ± 2.37   Final Test: 37.97 ± 2.73   Final old_old: 41.40 ± 2.96   Final old_new: 24.23 ± 2.52   Final new_new: 5.91 ± 5.23
Hits@30:
  Final val: 65.32 ± 2.22   Final Test: 49.69 ± 3.52   Final old_old: 53.70 ± 4.08   Final old_new: 33.73 ± 2.12   Final new_new: 9.55 ± 6.31
Hits@50:
  Final val: 72.19 ± 1.94   Final Test: 57.82 ± 1.95   Final old_old: 62.14 ± 2.22   Final old_new: 40.53 ± 1.19   Final new_new: 17.27 ± 3.05
AUC:
  Final val: 97.09 ± 0.16   Final Test: 96.48 ± 0.14   Final old_old: 97.17 ± 0.20   Final old_new: 93.79 ± 0.14   Final new_new: 89.05 ± 1.28
All runs:
Hits@10:
  Final val: 49.69 ± 1.65   Final Test: 28.90 ± 2.37   Final old_old: 32.09 ± 2.56   Final old_new: 15.84 ± 2.04   Final new_new: 4.09 ± 2.49
Hits@20:
  Final val: 60.01 ± 0.86   Final Test: 45.91 ± 5.59   Final old_old: 49.42 ± 5.30   Final old_new: 32.04 ± 6.89   Final new_new: 9.55 ± 7.08
Hits@30:
  Final val: 66.45 ± 1.16   Final Test: 51.36 ± 5.03   Final old_old: 54.92 ± 4.81   Final old_new: 37.33 ± 5.98   Final new_new: 13.64 ± 9.09
Hits@50:
  Final val: 72.32 ± 0.85   Final Test: 59.64 ± 3.48   Final old_old: 63.04 ± 3.26   Final old_new: 46.60 ± 4.61   Final new_new: 17.27 ± 6.74
AUC:
  Final val: 94.68 ± 0.11   Final Test: 94.25 ± 0.17   Final old_old: 94.95 ± 0.19   Final old_new: 91.53 ± 0.28   Final new_new: 85.76 ± 0.36
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.29 ± 2.88   Final Test: 29.68 ± 4.43   Final old_old: 32.12 ± 4.70   Final old_new: 19.90 ± 3.44   Final new_new: 6.82 ± 2.27
Hits@20:
  Final val: 61.28 ± 1.98   Final Test: 42.73 ± 2.52   Final old_old: 45.95 ± 2.55   Final old_new: 29.90 ± 2.64   Final new_new: 10.91 ± 3.73
Hits@30:
  Final val: 67.25 ± 2.03   Final Test: 49.37 ± 2.16   Final old_old: 52.76 ± 2.47   Final old_new: 35.97 ± 1.29   Final new_new: 14.55 ± 5.70
Hits@50:
  Final val: 74.49 ± 1.72   Final Test: 60.01 ± 3.37   Final old_old: 63.63 ± 3.32   Final old_new: 45.87 ± 4.47   Final new_new: 19.55 ± 4.98
AUC:
  Final val: 96.60 ± 0.32   Final Test: 96.29 ± 0.24   Final old_old: 96.82 ± 0.23   Final old_new: 94.21 ± 0.41   Final new_new: 90.42 ± 1.29
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.57 ± 4.30   Final Test: 33.56 ± 5.34   Final old_old: 36.60 ± 5.56   Final old_new: 21.39 ± 4.63   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 65.82 ± 2.00   Final Test: 43.38 ± 4.42   Final old_old: 46.91 ± 4.64   Final old_new: 29.37 ± 3.69   Final new_new: 7.73 ± 2.03
Hits@30:
  Final val: 72.38 ± 1.23   Final Test: 51.89 ± 5.66   Final old_old: 55.50 ± 5.52   Final old_new: 37.68 ± 6.34   Final new_new: 14.09 ± 6.31
Hits@50:
  Final val: 78.19 ± 0.84   Final Test: 61.12 ± 4.35   Final old_old: 64.83 ± 4.56   Final old_new: 46.68 ± 3.76   Final new_new: 19.09 ± 6.14
AUC:
  Final val: 97.30 ± 0.09   Final Test: 96.84 ± 0.06   Final old_old: 97.33 ± 0.08   Final old_new: 94.94 ± 0.16   Final new_new: 90.82 ± 0.94
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.2, l2=1e-07, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 53.48 ± 2.29   Final Test: 35.87 ± 4.64   Final old_old: 38.75 ± 4.76   Final old_new: 24.41 ± 4.30   Final new_new: 8.18 ± 3.80
Hits@20:
  Final val: 65.48 ± 1.97   Final Test: 45.67 ± 3.89   Final old_old: 48.92 ± 4.15   Final old_new: 32.95 ± 3.27   Final new_new: 9.55 ± 3.37
Hits@30:
  Final val: 71.04 ± 1.05   Final Test: 51.64 ± 2.16   Final old_old: 55.21 ± 2.12   Final old_new: 37.61 ± 2.64   Final new_new: 13.18 ± 3.73
Hits@50:
  Final val: 77.38 ± 0.81   Final Test: 62.49 ± 2.57   Final old_old: 66.29 ± 2.53   Final old_new: 47.73 ± 3.04   Final new_new: 18.64 ± 1.90
AUC:
  Final val: 97.48 ± 0.11   Final Test: 97.04 ± 0.06   Final old_old: 97.53 ± 0.06   Final old_new: 95.10 ± 0.19   Final new_new: 91.91 ± 0.83
Namespace(heuristic=False, device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=65536, lr=0.005, epochs=20000, eval_steps=5, runs=10, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=0.1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, rw_step=3, ns_rate=5, hops=3, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(heuristic=False, device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=65536, lr=0.005, epochs=20000, eval_steps=5, runs=10, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=0.1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, rw_step=3, ns_rate=5, hops=3, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(heuristic=False, device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=65536, lr=0.005, epochs=20000, eval_steps=5, runs=10, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=0.1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, rw_step=5, ns_rate=5, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=65536, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=65536, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=65536, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=65536, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=65536, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=65536, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=65536, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=65536, lr=0.001, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=65536, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=65536, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=65536, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=65536, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=65536, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=65536, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=65536, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 40.04 ± 2.53   Final Test: 25.11 ± 6.45   Final old_old: 27.59 ± 6.96   Final old_new: 14.99 ± 4.63   Final new_new: 5.91 ± 2.59
Hits@20:
  Final val: 53.55 ± 2.71   Final Test: 36.43 ± 2.66   Final old_old: 39.40 ± 2.98   Final old_new: 24.43 ± 1.53   Final new_new: 10.91 ± 5.18
Hits@30:
  Final val: 61.19 ± 2.57   Final Test: 42.85 ± 3.07   Final old_old: 46.18 ± 3.37   Final old_new: 29.52 ± 1.85   Final new_new: 11.36 ± 3.94
Hits@50:
  Final val: 71.62 ± 2.23   Final Test: 54.80 ± 5.56   Final old_old: 58.37 ± 5.99   Final old_new: 40.68 ± 4.14   Final new_new: 18.18 ± 4.25
AUC:
  Final val: 96.62 ± 0.27   Final Test: 95.88 ± 0.28   Final old_old: 96.51 ± 0.26   Final old_new: 93.31 ± 0.57   Final new_new: 90.44 ± 0.73
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 38.27 ± 3.07   Final Test: 26.03 ± 3.60   Final old_old: 28.44 ± 3.89   Final old_new: 16.17 ± 2.56   Final new_new: 6.82 ± 3.59
Hits@20:
  Final val: 52.76 ± 2.64   Final Test: 36.88 ± 2.57   Final old_old: 39.83 ± 2.73   Final old_new: 24.91 ± 2.44   Final new_new: 11.36 ± 4.55
Hits@30:
  Final val: 60.25 ± 3.67   Final Test: 45.85 ± 3.28   Final old_old: 49.05 ± 3.46   Final old_new: 33.25 ± 2.99   Final new_new: 11.82 ± 5.43
Hits@50:
  Final val: 70.77 ± 2.23   Final Test: 56.04 ± 4.41   Final old_old: 59.86 ± 4.41   Final old_new: 40.81 ± 4.63   Final new_new: 19.55 ± 8.59
AUC:
  Final val: 96.65 ± 0.33   Final Test: 96.03 ± 0.24   Final old_old: 96.67 ± 0.26   Final old_new: 93.48 ± 0.33   Final new_new: 90.02 ± 1.00
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=65536, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.78 ± 4.92   Final Test: 29.37 ± 2.88   Final old_old: 31.99 ± 3.28   Final old_new: 18.66 ± 1.59   Final new_new: 9.09 ± 3.59
Hits@20:
  Final val: 53.32 ± 2.93   Final Test: 36.64 ± 5.03   Final old_old: 39.65 ± 5.51   Final old_new: 24.46 ± 3.44   Final new_new: 10.91 ± 4.37
Hits@30:
  Final val: 62.65 ± 3.70   Final Test: 45.10 ± 3.14   Final old_old: 48.61 ± 3.49   Final old_new: 31.03 ± 1.88   Final new_new: 12.27 ± 3.80
Hits@50:
  Final val: 71.55 ± 3.59   Final Test: 56.62 ± 2.60   Final old_old: 60.22 ± 2.79   Final old_new: 42.42 ± 2.01   Final new_new: 19.09 ± 6.93
AUC:
  Final val: 96.67 ± 0.26   Final Test: 95.97 ± 0.29   Final old_old: 96.60 ± 0.30   Final old_new: 93.45 ± 0.33   Final new_new: 89.84 ± 2.53
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.08 ± 4.37   Final Test: 30.60 ± 4.99   Final old_old: 33.38 ± 5.49   Final old_new: 19.32 ± 3.15   Final new_new: 7.27 ± 4.37
Hits@20:
  Final val: 56.79 ± 3.47   Final Test: 37.72 ± 6.65   Final old_old: 40.97 ± 7.23   Final old_new: 24.53 ± 4.49   Final new_new: 10.45 ± 2.59
Hits@30:
  Final val: 65.39 ± 1.35   Final Test: 47.25 ± 5.99   Final old_old: 50.89 ± 6.22   Final old_new: 32.67 ± 5.21   Final new_new: 13.18 ± 3.37
Hits@50:
  Final val: 72.54 ± 1.23   Final Test: 57.03 ± 3.84   Final old_old: 60.95 ± 4.04   Final old_new: 41.56 ± 3.40   Final new_new: 16.82 ± 3.45
AUC:
  Final val: 96.70 ± 0.21   Final Test: 95.97 ± 0.20   Final old_old: 96.63 ± 0.22   Final old_new: 93.42 ± 0.24   Final new_new: 88.59 ± 1.05
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.09 ± 4.27   Final Test: 29.50 ± 5.50   Final old_old: 32.05 ± 6.17   Final old_new: 19.12 ± 2.94   Final new_new: 8.18 ± 2.03
Hits@20:
  Final val: 56.27 ± 3.69   Final Test: 37.56 ± 7.17   Final old_old: 40.67 ± 7.82   Final old_new: 24.99 ± 4.64   Final new_new: 10.45 ± 4.43
Hits@30:
  Final val: 63.46 ± 4.01   Final Test: 48.37 ± 2.81   Final old_old: 51.92 ± 2.89   Final old_new: 34.26 ± 2.63   Final new_new: 13.64 ± 6.22
Hits@50:
  Final val: 72.03 ± 3.02   Final Test: 57.93 ± 3.91   Final old_old: 61.74 ± 4.11   Final old_new: 42.87 ± 3.61   Final new_new: 19.09 ± 7.64
AUC:
  Final val: 96.68 ± 0.27   Final Test: 96.03 ± 0.24   Final old_old: 96.67 ± 0.29   Final old_new: 93.48 ± 0.17   Final new_new: 89.84 ± 1.64
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.28 ± 8.08   Final Test: 27.04 ± 3.79   Final old_old: 29.47 ± 4.05   Final old_new: 17.20 ± 3.03   Final new_new: 6.36 ± 3.73
Hits@20:
  Final val: 55.35 ± 6.23   Final Test: 36.76 ± 6.60   Final old_old: 39.79 ± 7.03   Final old_new: 24.63 ± 4.97   Final new_new: 8.18 ± 4.98
Hits@30:
  Final val: 63.01 ± 4.42   Final Test: 43.44 ± 5.83   Final old_old: 46.67 ± 6.11   Final old_new: 30.55 ± 4.97   Final new_new: 12.27 ± 4.13
Hits@50:
  Final val: 71.64 ± 2.69   Final Test: 56.02 ± 5.41   Final old_old: 59.76 ± 5.48   Final old_new: 41.23 ± 5.38   Final new_new: 17.73 ± 7.26
AUC:
  Final val: 96.58 ± 0.26   Final Test: 95.93 ± 0.27   Final old_old: 96.56 ± 0.26   Final old_new: 93.41 ± 0.30   Final new_new: 89.92 ± 0.43
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.55 ± 3.07   Final Test: 32.43 ± 4.28   Final old_old: 35.71 ± 4.47   Final old_new: 19.17 ± 4.01   Final new_new: 3.64 ± 3.45
Hits@20:
  Final val: 63.56 ± 1.69   Final Test: 44.58 ± 5.17   Final old_old: 48.56 ± 5.34   Final old_new: 28.74 ± 4.83   Final new_new: 5.91 ± 4.43
Hits@30:
  Final val: 69.47 ± 2.33   Final Test: 54.29 ± 4.12   Final old_old: 58.50 ± 4.30   Final old_new: 37.68 ± 3.79   Final new_new: 10.45 ± 7.47
Hits@50:
  Final val: 76.91 ± 2.14   Final Test: 62.53 ± 2.51   Final old_old: 66.90 ± 2.68   Final old_new: 45.29 ± 2.87   Final new_new: 16.36 ± 6.51
AUC:
  Final val: 97.43 ± 0.08   Final Test: 96.85 ± 0.16   Final old_old: 97.50 ± 0.23   Final old_new: 94.26 ± 0.47   Final new_new: 89.90 ± 1.08
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.67 ± 6.50   Final Test: 28.38 ± 6.28   Final old_old: 30.89 ± 6.56   Final old_new: 18.09 ± 5.16   Final new_new: 8.64 ± 6.89
Hits@20:
  Final val: 54.40 ± 4.64   Final Test: 33.60 ± 6.96   Final old_old: 36.45 ± 7.51   Final old_new: 21.91 ± 5.10   Final new_new: 10.91 ± 5.18
Hits@30:
  Final val: 62.15 ± 2.86   Final Test: 42.64 ± 1.32   Final old_old: 45.97 ± 1.56   Final old_new: 29.24 ± 0.80   Final new_new: 12.27 ± 4.71
Hits@50:
  Final val: 71.60 ± 2.03   Final Test: 54.91 ± 1.97   Final old_old: 58.61 ± 2.34   Final old_new: 40.15 ± 0.84   Final new_new: 19.09 ± 4.13
AUC:
  Final val: 96.67 ± 0.20   Final Test: 95.95 ± 0.06   Final old_old: 96.62 ± 0.11   Final old_new: 93.40 ± 0.30   Final new_new: 87.85 ± 2.36
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.04 ± 2.62   Final Test: 32.67 ± 5.48   Final old_old: 36.08 ± 5.82   Final old_new: 18.87 ± 4.20   Final new_new: 3.18 ± 3.45
Hits@20:
  Final val: 63.70 ± 2.82   Final Test: 44.19 ± 4.08   Final old_old: 48.15 ± 4.26   Final old_new: 28.46 ± 3.27   Final new_new: 4.55 ± 5.33
Hits@30:
  Final val: 70.58 ± 1.33   Final Test: 52.19 ± 2.96   Final old_old: 56.52 ± 3.16   Final old_new: 35.06 ± 2.59   Final new_new: 7.27 ± 5.18
Hits@50:
  Final val: 77.22 ± 0.68   Final Test: 61.25 ± 2.25   Final old_old: 65.58 ± 2.36   Final old_new: 44.36 ± 2.30   Final new_new: 12.73 ± 6.55
AUC:
  Final val: 97.43 ± 0.13   Final Test: 96.82 ± 0.10   Final old_old: 97.45 ± 0.13   Final old_new: 94.32 ± 0.40   Final new_new: 90.11 ± 1.32
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.03 ± 3.55   Final Test: 35.30 ± 6.92   Final old_old: 38.85 ± 7.46   Final old_new: 21.03 ± 4.95   Final new_new: 2.73 ± 1.90
Hits@20:
  Final val: 65.05 ± 3.09   Final Test: 42.72 ± 6.01   Final old_old: 46.61 ± 6.47   Final old_new: 27.10 ± 4.39   Final new_new: 6.36 ± 3.37
Hits@30:
  Final val: 70.73 ± 2.15   Final Test: 52.39 ± 3.67   Final old_old: 56.62 ± 3.91   Final old_new: 35.57 ± 2.77   Final new_new: 10.91 ± 4.37
Hits@50:
  Final val: 77.30 ± 2.14   Final Test: 62.79 ± 3.12   Final old_old: 66.92 ± 3.31   Final old_new: 46.70 ± 3.01   Final new_new: 15.91 ± 4.25
AUC:
  Final val: 97.44 ± 0.11   Final Test: 96.87 ± 0.15   Final old_old: 97.50 ± 0.22   Final old_new: 94.35 ± 0.40   Final new_new: 90.22 ± 2.09
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.44 ± 4.95   Final Test: 31.87 ± 3.01   Final old_old: 35.29 ± 3.31   Final old_new: 18.01 ± 2.06   Final new_new: 2.73 ± 1.90
Hits@20:
  Final val: 63.99 ± 4.01   Final Test: 42.25 ± 4.88   Final old_old: 46.07 ± 5.38   Final old_new: 27.03 ± 3.08   Final new_new: 5.00 ± 2.49
Hits@30:
  Final val: 70.31 ± 2.71   Final Test: 52.79 ± 5.88   Final old_old: 57.07 ± 6.09   Final old_new: 35.64 ± 5.28   Final new_new: 13.18 ± 1.90
Hits@50:
  Final val: 77.59 ± 1.68   Final Test: 61.20 ± 4.46   Final old_old: 65.48 ± 4.63   Final old_new: 44.31 ± 3.93   Final new_new: 15.91 ± 4.25
AUC:
  Final val: 97.45 ± 0.10   Final Test: 96.78 ± 0.21   Final old_old: 97.47 ± 0.22   Final old_new: 94.06 ± 0.36   Final new_new: 89.93 ± 1.32
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.33 ± 3.81   Final Test: 38.19 ± 5.38   Final old_old: 41.70 ± 5.72   Final old_new: 24.13 ± 4.07   Final new_new: 5.00 ± 2.49
Hits@20:
  Final val: 63.84 ± 4.27   Final Test: 45.97 ± 7.20   Final old_old: 49.96 ± 7.54   Final old_new: 30.05 ± 5.97   Final new_new: 7.27 ± 3.73
Hits@30:
  Final val: 69.97 ± 3.02   Final Test: 52.02 ± 4.19   Final old_old: 56.20 ± 4.34   Final old_new: 35.47 ± 3.69   Final new_new: 9.55 ± 2.96
Hits@50:
  Final val: 77.07 ± 1.43   Final Test: 60.82 ± 2.94   Final old_old: 65.16 ± 3.01   Final old_new: 43.65 ± 2.83   Final new_new: 16.36 ± 2.96
AUC:
  Final val: 97.47 ± 0.09   Final Test: 96.87 ± 0.09   Final old_old: 97.52 ± 0.09   Final old_new: 94.31 ± 0.32   Final new_new: 90.11 ± 1.47
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.42 ± 5.33   Final Test: 29.47 ± 4.70   Final old_old: 32.15 ± 5.12   Final old_new: 18.51 ± 3.16   Final new_new: 8.18 ± 2.59
Hits@20:
  Final val: 53.82 ± 4.39   Final Test: 36.66 ± 3.62   Final old_old: 39.72 ± 3.97   Final old_new: 24.21 ± 2.71   Final new_new: 10.91 ± 3.37
Hits@30:
  Final val: 61.63 ± 3.22   Final Test: 47.15 ± 3.64   Final old_old: 50.69 ± 4.08   Final old_new: 33.00 ± 2.29   Final new_new: 13.64 ± 4.82
Hits@50:
  Final val: 71.46 ± 2.83   Final Test: 54.74 ± 2.69   Final old_old: 58.39 ± 2.74   Final old_new: 40.30 ± 3.56   Final new_new: 17.27 ± 4.71
AUC:
  Final val: 96.64 ± 0.27   Final Test: 95.98 ± 0.27   Final old_old: 96.61 ± 0.27   Final old_new: 93.46 ± 0.36   Final new_new: 89.89 ± 1.34
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 54.14 ± 2.78   Final Test: 31.31 ± 5.89   Final old_old: 34.48 ± 6.34   Final old_new: 18.51 ± 4.30   Final new_new: 3.18 ± 2.59
Hits@20:
  Final val: 65.64 ± 3.28   Final Test: 46.26 ± 3.07   Final old_old: 50.38 ± 3.42   Final old_new: 29.80 ± 2.10   Final new_new: 6.82 ± 2.27
Hits@30:
  Final val: 71.59 ± 2.46   Final Test: 52.41 ± 4.47   Final old_old: 56.59 ± 4.78   Final old_new: 35.82 ± 3.58   Final new_new: 10.00 ± 4.13
Hits@50:
  Final val: 77.95 ± 1.46   Final Test: 62.65 ± 1.33   Final old_old: 67.05 ± 1.17   Final old_new: 45.31 ± 2.46   Final new_new: 15.91 ± 7.54
AUC:
  Final val: 97.49 ± 0.08   Final Test: 96.89 ± 0.12   Final old_old: 97.55 ± 0.16   Final old_new: 94.24 ± 0.17   Final new_new: 90.68 ± 1.43
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 39.00 ± 4.81   Final Test: 23.48 ± 4.14   Final old_old: 25.51 ± 4.33   Final old_new: 15.19 ± 3.60   Final new_new: 7.73 ± 2.59
Hits@20:
  Final val: 49.81 ± 3.59   Final Test: 33.55 ± 5.30   Final old_old: 36.20 ± 5.78   Final old_new: 22.87 ± 3.54   Final new_new: 9.55 ± 4.07
Hits@30:
  Final val: 59.35 ± 3.25   Final Test: 43.49 ± 2.40   Final old_old: 46.71 ± 2.39   Final old_new: 30.65 ± 2.48   Final new_new: 11.82 ± 6.31
Hits@50:
  Final val: 70.62 ± 2.76   Final Test: 53.13 ± 2.80   Final old_old: 56.58 ± 3.04   Final old_new: 39.47 ± 2.50   Final new_new: 17.73 ± 5.88
AUC:
  Final val: 96.57 ± 0.20   Final Test: 95.87 ± 0.30   Final old_old: 96.48 ± 0.29   Final old_new: 93.44 ± 0.37   Final new_new: 90.46 ± 0.95
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.61 ± 4.41   Final Test: 29.93 ± 5.50   Final old_old: 32.59 ± 6.01   Final old_new: 19.04 ± 3.46   Final new_new: 8.64 ± 2.96
Hits@20:
  Final val: 56.79 ± 4.06   Final Test: 40.62 ± 4.02   Final old_old: 43.82 ± 4.29   Final old_new: 27.66 ± 3.19   Final new_new: 12.73 ± 3.45
Hits@30:
  Final val: 63.94 ± 2.73   Final Test: 45.98 ± 3.85   Final old_old: 49.44 ± 4.03   Final old_new: 32.12 ± 3.39   Final new_new: 14.09 ± 4.37
Hits@50:
  Final val: 72.23 ± 2.06   Final Test: 59.16 ± 1.66   Final old_old: 63.02 ± 1.87   Final old_new: 43.75 ± 1.08   Final new_new: 21.36 ± 4.13
AUC:
  Final val: 96.71 ± 0.25   Final Test: 95.96 ± 0.22   Final old_old: 96.64 ± 0.22   Final old_new: 93.28 ± 0.25   Final new_new: 88.34 ± 1.44
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 53.40 ± 2.94   Final Test: 28.32 ± 3.69   Final old_old: 31.41 ± 4.04   Final old_new: 15.87 ± 2.48   Final new_new: 0.91 ± 1.24
Hits@20:
  Final val: 64.52 ± 4.82   Final Test: 42.43 ± 5.01   Final old_old: 46.24 ± 5.42   Final old_new: 27.33 ± 3.58   Final new_new: 3.64 ± 2.59
Hits@30:
  Final val: 70.99 ± 4.00   Final Test: 51.08 ± 4.78   Final old_old: 55.34 ± 4.86   Final old_new: 34.23 ± 4.62   Final new_new: 7.73 ± 2.03
Hits@50:
  Final val: 77.41 ± 2.90   Final Test: 61.04 ± 4.28   Final old_old: 65.33 ± 4.46   Final old_new: 44.18 ± 3.73   Final new_new: 14.55 ± 3.05
AUC:
  Final val: 97.43 ± 0.21   Final Test: 96.82 ± 0.25   Final old_old: 97.47 ± 0.28   Final old_new: 94.26 ± 0.33   Final new_new: 90.02 ± 1.73
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.30 ± 4.16   Final Test: 26.25 ± 6.18   Final old_old: 28.59 ± 6.65   Final old_new: 16.83 ± 4.26   Final new_new: 5.00 ± 4.66
Hits@20:
  Final val: 54.34 ± 3.87   Final Test: 39.87 ± 5.38   Final old_old: 43.18 ± 5.90   Final old_new: 26.55 ± 3.60   Final new_new: 10.45 ± 5.23
Hits@30:
  Final val: 61.84 ± 2.89   Final Test: 46.12 ± 5.33   Final old_old: 49.52 ± 5.59   Final old_new: 32.54 ± 4.62   Final new_new: 13.64 ± 5.79
Hits@50:
  Final val: 71.34 ± 2.59   Final Test: 57.30 ± 3.91   Final old_old: 61.00 ± 4.08   Final old_new: 42.85 ± 4.05   Final new_new: 15.91 ± 3.59
AUC:
  Final val: 96.64 ± 0.30   Final Test: 95.94 ± 0.25   Final old_old: 96.63 ± 0.26   Final old_new: 93.24 ± 0.31   Final new_new: 88.15 ± 2.16
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.46 ± 6.14   Final Test: 33.32 ± 6.44   Final old_old: 36.60 ± 6.86   Final old_new: 20.10 ± 4.81   Final new_new: 3.64 ± 2.59
Hits@20:
  Final val: 61.57 ± 6.78   Final Test: 41.02 ± 10.06   Final old_old: 44.49 ± 10.92   Final old_new: 27.13 ± 6.77   Final new_new: 7.73 ± 2.03
Hits@30:
  Final val: 67.89 ± 4.28   Final Test: 50.53 ± 4.81   Final old_old: 54.44 ± 5.12   Final old_new: 35.09 ± 3.74   Final new_new: 10.00 ± 1.24
Hits@50:
  Final val: 75.85 ± 2.85   Final Test: 61.11 ± 3.26   Final old_old: 65.44 ± 3.48   Final old_new: 43.98 ± 2.67   Final new_new: 17.27 ± 4.43
AUC:
  Final val: 97.35 ± 0.18   Final Test: 96.80 ± 0.16   Final old_old: 97.42 ± 0.19   Final old_new: 94.34 ± 0.35   Final new_new: 90.78 ± 1.56
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.27 ± 4.06   Final Test: 26.65 ± 11.90   Final old_old: 28.91 ± 12.78   Final old_new: 17.43 ± 8.45   Final new_new: 8.18 ± 5.70
Hits@20:
  Final val: 55.13 ± 3.04   Final Test: 37.37 ± 4.72   Final old_old: 40.43 ± 5.00   Final old_new: 24.94 ± 4.14   Final new_new: 11.82 ± 8.10
Hits@30:
  Final val: 64.13 ± 2.14   Final Test: 44.41 ± 2.58   Final old_old: 47.69 ± 2.89   Final old_new: 31.31 ± 2.28   Final new_new: 13.18 ± 6.10
Hits@50:
  Final val: 72.48 ± 0.98   Final Test: 54.62 ± 2.59   Final old_old: 58.06 ± 3.04   Final old_new: 41.06 ± 1.56   Final new_new: 17.73 ± 5.43
AUC:
  Final val: 96.69 ± 0.23   Final Test: 96.00 ± 0.23   Final old_old: 96.64 ± 0.22   Final old_new: 93.44 ± 0.51   Final new_new: 89.57 ± 2.07
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 37.53 ± 3.85   Final Test: 26.41 ± 7.17   Final old_old: 28.60 ± 7.74   Final old_new: 17.53 ± 5.21   Final new_new: 7.73 ± 4.43
Hits@20:
  Final val: 49.98 ± 4.30   Final Test: 34.87 ± 4.75   Final old_old: 37.71 ± 5.41   Final old_new: 23.48 ± 2.31   Final new_new: 8.64 ± 2.96
Hits@30:
  Final val: 59.47 ± 3.35   Final Test: 44.53 ± 4.24   Final old_old: 47.57 ± 4.29   Final old_new: 32.39 ± 5.22   Final new_new: 15.00 ± 10.11
Hits@50:
  Final val: 70.24 ± 1.88   Final Test: 53.96 ± 3.15   Final old_old: 57.40 ± 3.57   Final old_new: 40.28 ± 2.31   Final new_new: 20.00 ± 6.89
AUC:
  Final val: 96.49 ± 0.25   Final Test: 95.90 ± 0.17   Final old_old: 96.54 ± 0.20   Final old_new: 93.32 ± 0.33   Final new_new: 89.85 ± 1.06
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.03 ± 5.10   Final Test: 31.46 ± 6.67   Final old_old: 34.68 ± 7.29   Final old_new: 18.39 ± 4.26   Final new_new: 4.09 ± 2.96
Hits@20:
  Final val: 63.47 ± 4.36   Final Test: 43.28 ± 6.69   Final old_old: 47.25 ± 7.13   Final old_new: 27.28 ± 5.25   Final new_new: 8.18 ± 3.80
Hits@30:
  Final val: 69.38 ± 3.56   Final Test: 51.19 ± 4.59   Final old_old: 55.48 ± 5.11   Final old_new: 34.06 ± 2.66   Final new_new: 10.00 ± 3.45
Hits@50:
  Final val: 77.27 ± 2.35   Final Test: 62.39 ± 4.53   Final old_old: 66.81 ± 4.82   Final old_new: 45.11 ± 3.53   Final new_new: 13.18 ± 6.31
AUC:
  Final val: 97.42 ± 0.14   Final Test: 96.83 ± 0.19   Final old_old: 97.50 ± 0.22   Final old_new: 94.19 ± 0.33   Final new_new: 89.75 ± 2.08
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.68 ± 5.40   Final Test: 32.31 ± 7.08   Final old_old: 35.65 ± 7.59   Final old_new: 18.74 ± 5.16   Final new_new: 4.09 ± 2.96
Hits@20:
  Final val: 63.41 ± 4.32   Final Test: 42.49 ± 8.08   Final old_old: 46.40 ± 8.50   Final old_new: 26.75 ± 6.51   Final new_new: 7.73 ± 3.80
Hits@30:
  Final val: 70.21 ± 2.85   Final Test: 53.00 ± 3.73   Final old_old: 57.23 ± 3.71   Final old_new: 36.25 ± 3.93   Final new_new: 10.45 ± 5.70
Hits@50:
  Final val: 77.34 ± 1.69   Final Test: 60.42 ± 3.34   Final old_old: 64.87 ± 3.56   Final old_new: 43.00 ± 2.73   Final new_new: 12.27 ± 3.05
AUC:
  Final val: 97.40 ± 0.12   Final Test: 96.81 ± 0.14   Final old_old: 97.44 ± 0.17   Final old_new: 94.31 ± 0.36   Final new_new: 90.45 ± 1.19
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 40.95 ± 5.60   Final Test: 31.15 ± 5.06   Final old_old: 33.83 ± 5.58   Final old_new: 20.33 ± 3.12   Final new_new: 7.27 ± 2.49
Hits@20:
  Final val: 54.17 ± 4.31   Final Test: 37.21 ± 4.83   Final old_old: 40.17 ± 5.20   Final old_new: 25.31 ± 3.45   Final new_new: 10.00 ± 4.71
Hits@30:
  Final val: 63.17 ± 1.95   Final Test: 45.57 ± 5.10   Final old_old: 48.99 ± 5.23   Final old_new: 31.86 ± 4.96   Final new_new: 14.09 ± 7.94
Hits@50:
  Final val: 72.30 ± 1.13   Final Test: 55.36 ± 2.65   Final old_old: 59.03 ± 2.95   Final old_new: 40.86 ± 2.25   Final new_new: 16.82 ± 6.55
AUC:
  Final val: 96.68 ± 0.17   Final Test: 95.95 ± 0.13   Final old_old: 96.58 ± 0.15   Final old_new: 93.45 ± 0.38   Final new_new: 89.92 ± 1.35
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.05 ± 3.27   Final Test: 33.27 ± 3.68   Final old_old: 36.79 ± 3.59   Final old_new: 18.99 ± 4.27   Final new_new: 3.18 ± 2.03
Hits@20:
  Final val: 62.77 ± 2.10   Final Test: 42.13 ± 3.22   Final old_old: 45.99 ± 3.33   Final old_new: 26.75 ± 2.86   Final new_new: 5.00 ± 2.96
Hits@30:
  Final val: 68.36 ± 1.06   Final Test: 50.02 ± 4.13   Final old_old: 54.21 ± 4.29   Final old_new: 33.32 ± 3.53   Final new_new: 9.55 ± 4.66
Hits@50:
  Final val: 76.86 ± 1.19   Final Test: 61.35 ± 4.37   Final old_old: 65.61 ± 4.47   Final old_new: 44.56 ± 4.17   Final new_new: 17.27 ± 3.45
AUC:
  Final val: 97.36 ± 0.10   Final Test: 96.77 ± 0.22   Final old_old: 97.41 ± 0.20   Final old_new: 94.26 ± 0.51   Final new_new: 90.34 ± 1.32
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.50 ± 3.08   Final Test: 32.56 ± 7.51   Final old_old: 35.43 ± 8.22   Final old_new: 20.88 ± 4.94   Final new_new: 9.09 ± 2.27
Hits@20:
  Final val: 57.06 ± 2.05   Final Test: 39.42 ± 3.65   Final old_old: 42.71 ± 4.00   Final old_new: 26.17 ± 2.22   Final new_new: 10.45 ± 2.59
Hits@30:
  Final val: 64.00 ± 2.44   Final Test: 48.26 ± 3.20   Final old_old: 51.85 ± 3.37   Final old_new: 33.88 ± 2.86   Final new_new: 15.45 ± 2.49
Hits@50:
  Final val: 72.26 ± 1.65   Final Test: 56.72 ± 3.67   Final old_old: 60.49 ± 3.96   Final old_new: 41.79 ± 2.63   Final new_new: 18.18 ± 5.79
AUC:
  Final val: 96.67 ± 0.29   Final Test: 95.96 ± 0.22   Final old_old: 96.61 ± 0.28   Final old_new: 93.41 ± 0.34   Final new_new: 88.94 ± 0.87
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.46 ± 4.07   Final Test: 32.58 ± 4.24   Final old_old: 35.75 ± 4.54   Final old_new: 19.70 ± 3.24   Final new_new: 5.91 ± 3.05
Hits@20:
  Final val: 63.95 ± 2.64   Final Test: 48.09 ± 3.76   Final old_old: 52.06 ± 4.04   Final old_new: 32.22 ± 2.78   Final new_new: 10.00 ± 3.45
Hits@30:
  Final val: 70.52 ± 2.18   Final Test: 53.15 ± 5.05   Final old_old: 57.33 ± 5.39   Final old_new: 36.55 ± 4.29   Final new_new: 11.36 ± 4.25
Hits@50:
  Final val: 77.42 ± 1.26   Final Test: 60.76 ± 4.33   Final old_old: 65.13 ± 4.40   Final old_new: 43.55 ± 4.38   Final new_new: 15.00 ± 5.93
AUC:
  Final val: 97.44 ± 0.16   Final Test: 96.86 ± 0.16   Final old_old: 97.52 ± 0.19   Final old_new: 94.27 ± 0.50   Final new_new: 90.33 ± 1.85
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.99 ± 3.43   Final Test: 32.35 ± 2.34   Final old_old: 35.69 ± 2.45   Final old_new: 18.84 ± 2.15   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 63.27 ± 4.23   Final Test: 42.39 ± 7.70   Final old_old: 46.31 ± 7.98   Final old_new: 26.57 ± 6.62   Final new_new: 8.18 ± 5.93
Hits@30:
  Final val: 69.85 ± 2.44   Final Test: 51.83 ± 5.19   Final old_old: 56.10 ± 5.22   Final old_new: 34.71 ± 5.34   Final new_new: 11.36 ± 4.55
Hits@50:
  Final val: 77.17 ± 2.19   Final Test: 62.64 ± 3.67   Final old_old: 66.74 ± 3.75   Final old_new: 46.55 ± 3.57   Final new_new: 17.73 ± 3.37
AUC:
  Final val: 97.41 ± 0.12   Final Test: 96.87 ± 0.18   Final old_old: 97.52 ± 0.20   Final old_new: 94.30 ± 0.42   Final new_new: 90.12 ± 1.76
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.96 ± 3.06   Final Test: 34.33 ± 4.05   Final old_old: 37.31 ± 4.29   Final old_new: 22.19 ± 3.11   Final new_new: 10.45 ± 3.45
Hits@20:
  Final val: 57.09 ± 2.64   Final Test: 42.37 ± 2.73   Final old_old: 45.49 ± 2.82   Final old_new: 29.90 ± 2.78   Final new_new: 11.82 ± 4.93
Hits@30:
  Final val: 63.87 ± 2.08   Final Test: 47.47 ± 4.85   Final old_old: 50.87 ± 5.14   Final old_new: 34.06 ± 3.86   Final new_new: 12.27 ± 3.45
Hits@50:
  Final val: 72.48 ± 0.48   Final Test: 58.65 ± 2.69   Final old_old: 62.36 ± 2.71   Final old_new: 44.13 ± 2.97   Final new_new: 18.18 ± 1.61
AUC:
  Final val: 96.66 ± 0.25   Final Test: 96.03 ± 0.38   Final old_old: 96.66 ± 0.36   Final old_new: 93.50 ± 0.50   Final new_new: 89.88 ± 1.67
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.44 ± 4.95   Final Test: 34.85 ± 3.44   Final old_old: 38.02 ± 3.70   Final old_new: 22.09 ± 2.88   Final new_new: 5.45 ± 1.24
Hits@20:
  Final val: 60.75 ± 5.54   Final Test: 41.04 ± 8.04   Final old_old: 44.36 ± 8.73   Final old_new: 27.86 ± 5.35   Final new_new: 7.73 ± 1.24
Hits@30:
  Final val: 68.29 ± 3.70   Final Test: 48.50 ± 5.61   Final old_old: 52.44 ± 5.82   Final old_new: 32.80 ± 4.91   Final new_new: 10.45 ± 4.71
Hits@50:
  Final val: 76.26 ± 2.12   Final Test: 59.17 ± 5.41   Final old_old: 63.27 ± 5.38   Final old_new: 43.00 ± 5.70   Final new_new: 16.82 ± 4.71
AUC:
  Final val: 97.31 ± 0.15   Final Test: 96.76 ± 0.14   Final old_old: 97.38 ± 0.14   Final old_new: 94.28 ± 0.37   Final new_new: 90.44 ± 1.20
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.88 ± 3.81   Final Test: 29.09 ± 2.58   Final old_old: 31.72 ± 2.83   Final old_new: 18.36 ± 2.01   Final new_new: 7.73 ± 2.03
Hits@20:
  Final val: 56.73 ± 1.68   Final Test: 38.84 ± 6.36   Final old_old: 41.94 ± 6.63   Final old_new: 26.27 ± 5.26   Final new_new: 12.73 ± 6.93
Hits@30:
  Final val: 63.63 ± 0.82   Final Test: 45.46 ± 5.62   Final old_old: 48.84 ± 5.91   Final old_new: 31.94 ± 4.70   Final new_new: 13.18 ± 5.18
Hits@50:
  Final val: 71.70 ± 1.12   Final Test: 57.58 ± 2.90   Final old_old: 61.26 ± 3.14   Final old_new: 43.02 ± 2.12   Final new_new: 20.00 ± 3.37
AUC:
  Final val: 96.72 ± 0.22   Final Test: 96.08 ± 0.28   Final old_old: 96.71 ± 0.24   Final old_new: 93.55 ± 0.54   Final new_new: 89.74 ± 1.41
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.06 ± 1.21   Final Test: 33.59 ± 2.81   Final old_old: 36.99 ± 3.07   Final old_new: 19.85 ± 1.93   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 61.91 ± 2.05   Final Test: 42.99 ± 6.84   Final old_old: 46.77 ± 7.28   Final old_new: 27.83 ± 5.03   Final new_new: 7.73 ± 6.14
Hits@30:
  Final val: 68.94 ± 1.23   Final Test: 50.44 ± 4.21   Final old_old: 54.55 ± 4.36   Final old_new: 34.16 ± 3.61   Final new_new: 9.09 ± 3.59
Hits@50:
  Final val: 76.58 ± 1.38   Final Test: 60.17 ± 4.10   Final old_old: 64.46 ± 4.23   Final old_new: 43.22 ± 3.66   Final new_new: 15.91 ± 4.25
AUC:
  Final val: 97.40 ± 0.07   Final Test: 96.84 ± 0.14   Final old_old: 97.48 ± 0.17   Final old_new: 94.32 ± 0.25   Final new_new: 90.28 ± 1.11
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.59 ± 3.60   Final Test: 30.24 ± 7.41   Final old_old: 32.90 ± 7.76   Final old_new: 19.55 ± 6.10   Final new_new: 5.91 ± 3.45
Hits@20:
  Final val: 56.32 ± 1.43   Final Test: 40.52 ± 8.43   Final old_old: 43.72 ± 8.64   Final old_new: 27.81 ± 7.76   Final new_new: 9.09 ± 4.82
Hits@30:
  Final val: 65.39 ± 1.44   Final Test: 48.94 ± 2.32   Final old_old: 52.60 ± 2.47   Final old_new: 34.53 ± 2.08   Final new_new: 10.00 ± 3.80
Hits@50:
  Final val: 74.33 ± 0.89   Final Test: 59.17 ± 3.54   Final old_old: 63.15 ± 3.73   Final old_new: 43.55 ± 2.91   Final new_new: 16.36 ± 4.37
AUC:
  Final val: 96.82 ± 0.23   Final Test: 96.24 ± 0.26   Final old_old: 96.91 ± 0.28   Final old_new: 93.63 ± 0.23   Final new_new: 88.82 ± 0.84
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.82 ± 6.58   Final Test: 32.56 ± 6.69   Final old_old: 35.98 ± 7.24   Final old_new: 18.69 ± 4.71   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 62.98 ± 3.85   Final Test: 42.48 ± 5.73   Final old_old: 46.39 ± 6.36   Final old_new: 26.75 ± 3.16   Final new_new: 7.27 ± 2.49
Hits@30:
  Final val: 70.58 ± 2.91   Final Test: 51.80 ± 3.92   Final old_old: 56.13 ± 4.34   Final old_new: 34.56 ± 2.75   Final new_new: 9.55 ± 3.73
Hits@50:
  Final val: 77.01 ± 2.32   Final Test: 61.57 ± 3.30   Final old_old: 65.82 ± 3.80   Final old_new: 44.86 ± 1.38   Final new_new: 15.45 ± 2.96
AUC:
  Final val: 97.42 ± 0.15   Final Test: 96.82 ± 0.13   Final old_old: 97.47 ± 0.21   Final old_new: 94.26 ± 0.26   Final new_new: 90.08 ± 1.29
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.82 ± 2.47   Final Test: 29.38 ± 2.88   Final old_old: 32.00 ± 3.23   Final old_new: 18.77 ± 1.55   Final new_new: 7.73 ± 5.23
Hits@20:
  Final val: 53.40 ± 2.32   Final Test: 39.55 ± 3.13   Final old_old: 42.68 ± 3.31   Final old_new: 26.93 ± 2.75   Final new_new: 11.36 ± 5.57
Hits@30:
  Final val: 61.96 ± 2.16   Final Test: 46.24 ± 3.87   Final old_old: 49.68 ± 4.05   Final old_new: 32.57 ± 3.34   Final new_new: 12.27 ± 4.13
Hits@50:
  Final val: 70.72 ± 1.81   Final Test: 55.19 ± 4.06   Final old_old: 59.00 ± 4.46   Final old_new: 40.18 ± 2.70   Final new_new: 14.55 ± 4.43
AUC:
  Final val: 96.68 ± 0.26   Final Test: 96.07 ± 0.23   Final old_old: 96.71 ± 0.26   Final old_new: 93.49 ± 0.17   Final new_new: 90.31 ± 0.91
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.10 ± 4.60   Final Test: 27.98 ± 5.72   Final old_old: 30.46 ± 6.15   Final old_new: 17.91 ± 4.43   Final new_new: 7.27 ± 2.96
Hits@20:
  Final val: 53.38 ± 3.20   Final Test: 37.22 ± 3.38   Final old_old: 40.07 ± 3.60   Final old_new: 25.82 ± 2.71   Final new_new: 9.55 ± 3.37
Hits@30:
  Final val: 62.24 ± 2.89   Final Test: 43.34 ± 5.32   Final old_old: 46.72 ± 5.79   Final old_new: 29.92 ± 3.77   Final new_new: 9.09 ± 5.33
Hits@50:
  Final val: 71.46 ± 2.20   Final Test: 55.65 ± 2.79   Final old_old: 59.47 ± 2.89   Final old_new: 40.63 ± 2.68   Final new_new: 15.00 ± 3.45
AUC:
  Final val: 96.58 ± 0.31   Final Test: 95.98 ± 0.24   Final old_old: 96.60 ± 0.24   Final old_new: 93.47 ± 0.31   Final new_new: 90.34 ± 1.10
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.49 ± 5.22   Final Test: 27.18 ± 5.12   Final old_old: 29.61 ± 5.32   Final old_new: 17.33 ± 4.52   Final new_new: 5.91 ± 5.47
Hits@20:
  Final val: 56.55 ± 4.83   Final Test: 41.11 ± 4.52   Final old_old: 44.44 ± 4.76   Final old_new: 27.81 ± 3.83   Final new_new: 9.09 ± 4.55
Hits@30:
  Final val: 64.64 ± 3.96   Final Test: 48.33 ± 2.67   Final old_old: 51.76 ± 2.86   Final old_new: 34.84 ± 2.61   Final new_new: 11.36 ± 5.08
Hits@50:
  Final val: 72.01 ± 2.50   Final Test: 57.55 ± 3.48   Final old_old: 61.27 ± 3.73   Final old_new: 43.00 ± 3.18   Final new_new: 16.36 ± 5.88
AUC:
  Final val: 96.65 ± 0.24   Final Test: 96.10 ± 0.23   Final old_old: 96.73 ± 0.25   Final old_new: 93.65 ± 0.31   Final new_new: 89.49 ± 1.68
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.18 ± 3.05   Final Test: 31.37 ± 2.97   Final old_old: 34.46 ± 3.34   Final old_new: 18.92 ± 1.86   Final new_new: 3.64 ± 2.59
Hits@20:
  Final val: 63.39 ± 1.92   Final Test: 43.96 ± 4.79   Final old_old: 47.73 ± 5.00   Final old_new: 28.99 ± 4.29   Final new_new: 6.36 ± 1.02
Hits@30:
  Final val: 70.40 ± 1.89   Final Test: 51.37 ± 3.79   Final old_old: 55.31 ± 4.06   Final old_new: 35.84 ± 3.12   Final new_new: 9.09 ± 2.78
Hits@50:
  Final val: 77.46 ± 1.75   Final Test: 62.78 ± 4.02   Final old_old: 66.96 ± 4.08   Final old_new: 46.50 ± 4.07   Final new_new: 15.00 ± 4.13
AUC:
  Final val: 97.49 ± 0.07   Final Test: 97.07 ± 0.10   Final old_old: 97.59 ± 0.12   Final old_new: 95.06 ± 0.20   Final new_new: 91.06 ± 1.30
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.00 ± 5.09   Final Test: 31.85 ± 5.08   Final old_old: 35.11 ± 5.49   Final old_new: 18.66 ± 3.97   Final new_new: 3.64 ± 2.03
Hits@20:
  Final val: 63.35 ± 3.07   Final Test: 41.89 ± 4.95   Final old_old: 45.56 ± 5.42   Final old_new: 27.30 ± 3.19   Final new_new: 5.00 ± 1.90
Hits@30:
  Final val: 69.58 ± 3.01   Final Test: 50.22 ± 2.27   Final old_old: 54.47 ± 2.67   Final old_new: 33.32 ± 0.87   Final new_new: 8.18 ± 2.59
Hits@50:
  Final val: 76.56 ± 1.95   Final Test: 61.52 ± 2.02   Final old_old: 65.83 ± 2.17   Final old_new: 44.66 ± 2.16   Final new_new: 13.64 ± 3.21
AUC:
  Final val: 97.36 ± 0.13   Final Test: 96.81 ± 0.16   Final old_old: 97.46 ± 0.17   Final old_new: 94.20 ± 0.45   Final new_new: 90.41 ± 1.18
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.57 ± 3.47   Final Test: 30.25 ± 3.93   Final old_old: 32.76 ± 4.22   Final old_new: 20.23 ± 2.90   Final new_new: 5.45 ± 3.45
Hits@20:
  Final val: 58.82 ± 3.41   Final Test: 43.74 ± 1.70   Final old_old: 46.97 ± 1.84   Final old_new: 30.98 ± 1.58   Final new_new: 9.55 ± 4.37
Hits@30:
  Final val: 66.34 ± 2.54   Final Test: 48.08 ± 4.87   Final old_old: 51.59 ± 5.13   Final old_new: 34.21 ± 4.08   Final new_new: 11.36 ± 6.63
Hits@50:
  Final val: 74.16 ± 2.23   Final Test: 58.71 ± 2.73   Final old_old: 62.47 ± 2.81   Final old_new: 43.88 ± 2.51   Final new_new: 19.55 ± 3.45
AUC:
  Final val: 96.75 ± 0.21   Final Test: 96.20 ± 0.26   Final old_old: 96.79 ± 0.24   Final old_new: 93.86 ± 0.35   Final new_new: 90.79 ± 0.95
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.90 ± 7.02   Final Test: 24.60 ± 3.74   Final old_old: 26.92 ± 4.03   Final old_new: 15.21 ± 2.83   Final new_new: 5.00 ± 1.90
Hits@20:
  Final val: 54.67 ± 3.86   Final Test: 36.45 ± 4.77   Final old_old: 39.47 ± 4.98   Final old_new: 24.26 ± 4.34   Final new_new: 10.00 ± 3.80
Hits@30:
  Final val: 62.91 ± 2.41   Final Test: 44.28 ± 2.94   Final old_old: 47.53 ± 3.11   Final old_new: 31.31 ± 2.57   Final new_new: 13.64 ± 7.01
Hits@50:
  Final val: 71.26 ± 1.54   Final Test: 56.32 ± 3.72   Final old_old: 59.96 ± 3.68   Final old_new: 41.99 ± 4.22   Final new_new: 17.73 ± 7.77
AUC:
  Final val: 96.66 ± 0.23   Final Test: 96.00 ± 0.23   Final old_old: 96.68 ± 0.26   Final old_new: 93.29 ± 0.40   Final new_new: 89.15 ± 1.62
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.99 ± 5.12   Final Test: 30.98 ± 5.92   Final old_old: 34.08 ± 6.27   Final old_new: 18.44 ± 4.88   Final new_new: 3.64 ± 2.03
Hits@20:
  Final val: 61.97 ± 3.58   Final Test: 44.14 ± 5.08   Final old_old: 47.95 ± 5.73   Final old_new: 28.97 ± 2.69   Final new_new: 6.36 ± 2.49
Hits@30:
  Final val: 68.39 ± 3.43   Final Test: 50.39 ± 4.27   Final old_old: 54.44 ± 4.52   Final old_new: 34.26 ± 3.57   Final new_new: 10.91 ± 2.96
Hits@50:
  Final val: 76.21 ± 1.68   Final Test: 60.02 ± 3.43   Final old_old: 64.41 ± 3.39   Final old_new: 42.75 ± 3.91   Final new_new: 13.18 ± 3.73
AUC:
  Final val: 97.32 ± 0.13   Final Test: 96.77 ± 0.14   Final old_old: 97.43 ± 0.14   Final old_new: 94.14 ± 0.27   Final new_new: 90.17 ± 0.50
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.94 ± 3.86   Final Test: 29.01 ± 4.48   Final old_old: 32.07 ± 4.88   Final old_new: 16.62 ± 2.95   Final new_new: 2.27 ± 1.61
Hits@20:
  Final val: 62.16 ± 4.48   Final Test: 41.82 ± 6.30   Final old_old: 45.54 ± 6.63   Final old_new: 26.90 ± 5.23   Final new_new: 7.27 ± 1.90
Hits@30:
  Final val: 68.81 ± 3.07   Final Test: 50.45 ± 5.19   Final old_old: 54.43 ± 5.36   Final old_new: 34.69 ± 4.68   Final new_new: 9.55 ± 2.96
Hits@50:
  Final val: 76.61 ± 0.57   Final Test: 59.61 ± 2.34   Final old_old: 63.86 ± 2.41   Final old_new: 42.97 ± 2.06   Final new_new: 12.27 ± 5.23
AUC:
  Final val: 97.36 ± 0.04   Final Test: 96.76 ± 0.10   Final old_old: 97.41 ± 0.10   Final old_new: 94.19 ± 0.26   Final new_new: 90.61 ± 0.87
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.28 ± 3.63   Final Test: 28.50 ± 5.92   Final old_old: 30.99 ± 6.38   Final old_new: 18.36 ± 4.36   Final new_new: 7.73 ± 2.03
Hits@20:
  Final val: 57.13 ± 4.08   Final Test: 39.84 ± 3.41   Final old_old: 43.07 ± 3.64   Final old_new: 26.83 ± 2.79   Final new_new: 10.45 ± 4.43
Hits@30:
  Final val: 65.19 ± 3.61   Final Test: 48.14 ± 1.57   Final old_old: 51.74 ± 1.82   Final old_new: 33.85 ± 1.58   Final new_new: 12.27 ± 2.03
Hits@50:
  Final val: 74.08 ± 2.72   Final Test: 59.52 ± 2.84   Final old_old: 63.39 ± 2.80   Final old_new: 44.23 ± 3.56   Final new_new: 19.09 ± 6.14
AUC:
  Final val: 96.87 ± 0.22   Final Test: 96.34 ± 0.25   Final old_old: 96.99 ± 0.20   Final old_new: 93.79 ± 0.62   Final new_new: 89.06 ± 0.92
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.59 ± 4.92   Final Test: 33.13 ± 6.63   Final old_old: 36.50 ± 7.13   Final old_new: 19.50 ± 4.81   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 63.77 ± 2.96   Final Test: 44.32 ± 5.01   Final old_old: 48.23 ± 5.29   Final old_new: 28.64 ± 4.21   Final new_new: 8.18 ± 4.13
Hits@30:
  Final val: 70.54 ± 1.98   Final Test: 52.83 ± 2.80   Final old_old: 57.11 ± 3.22   Final old_new: 35.79 ± 0.99   Final new_new: 10.91 ± 3.37
Hits@50:
  Final val: 77.55 ± 1.55   Final Test: 61.64 ± 2.86   Final old_old: 65.83 ± 3.14   Final old_new: 45.16 ± 1.90   Final new_new: 16.82 ± 5.93
AUC:
  Final val: 97.44 ± 0.11   Final Test: 96.84 ± 0.13   Final old_old: 97.50 ± 0.21   Final old_new: 94.21 ± 0.30   Final new_new: 90.40 ± 0.99
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.26 ± 3.55   Final Test: 32.65 ± 2.06   Final old_old: 35.78 ± 2.24   Final old_new: 20.08 ± 1.62   Final new_new: 4.09 ± 1.02
Hits@20:
  Final val: 63.86 ± 2.37   Final Test: 44.61 ± 1.55   Final old_old: 48.38 ± 1.64   Final old_new: 29.62 ± 2.02   Final new_new: 7.27 ± 2.96
Hits@30:
  Final val: 69.93 ± 1.61   Final Test: 55.41 ± 1.76   Final old_old: 59.53 ± 1.96   Final old_new: 39.32 ± 1.30   Final new_new: 9.55 ± 4.07
Hits@50:
  Final val: 77.24 ± 1.51   Final Test: 61.87 ± 1.94   Final old_old: 66.14 ± 2.10   Final old_new: 45.34 ± 1.30   Final new_new: 11.82 ± 3.37
AUC:
  Final val: 97.44 ± 0.10   Final Test: 97.04 ± 0.06   Final old_old: 97.56 ± 0.08   Final old_new: 95.01 ± 0.20   Final new_new: 91.31 ± 1.13
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.06 ± 7.39   Final Test: 26.04 ± 2.31   Final old_old: 28.46 ± 2.51   Final old_new: 16.17 ± 1.98   Final new_new: 6.82 ± 4.55
Hits@20:
  Final val: 57.72 ± 4.58   Final Test: 37.54 ± 5.72   Final old_old: 40.37 ± 6.01   Final old_new: 26.32 ± 4.78   Final new_new: 9.09 ± 3.59
Hits@30:
  Final val: 63.31 ± 3.03   Final Test: 46.96 ± 2.28   Final old_old: 50.09 ± 2.43   Final old_new: 34.61 ± 2.11   Final new_new: 14.55 ± 4.13
Hits@50:
  Final val: 71.28 ± 1.52   Final Test: 57.14 ± 3.60   Final old_old: 60.45 ± 3.90   Final old_new: 44.23 ± 2.62   Final new_new: 20.00 ± 4.07
AUC:
  Final val: 96.42 ± 0.15   Final Test: 95.89 ± 0.23   Final old_old: 96.43 ± 0.26   Final old_new: 93.74 ± 0.28   Final new_new: 90.79 ± 1.38
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.58 ± 4.24   Final Test: 29.14 ± 5.34   Final old_old: 32.18 ± 5.67   Final old_new: 16.78 ± 4.14   Final new_new: 3.64 ± 2.03
Hits@20:
  Final val: 62.05 ± 3.13   Final Test: 41.67 ± 3.05   Final old_old: 45.29 ± 3.29   Final old_new: 27.30 ± 2.38   Final new_new: 5.00 ± 2.96
Hits@30:
  Final val: 69.26 ± 1.36   Final Test: 50.08 ± 1.95   Final old_old: 53.94 ± 2.25   Final old_new: 34.99 ± 1.26   Final new_new: 7.27 ± 2.96
Hits@50:
  Final val: 76.95 ± 1.55   Final Test: 60.03 ± 2.69   Final old_old: 64.22 ± 2.77   Final old_new: 43.75 ± 2.68   Final new_new: 11.36 ± 3.21
AUC:
  Final val: 97.54 ± 0.06   Final Test: 97.07 ± 0.11   Final old_old: 97.58 ± 0.14   Final old_new: 95.08 ± 0.11   Final new_new: 91.82 ± 1.08
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.64 ± 3.24   Final Test: 26.97 ± 4.00   Final old_old: 29.28 ± 4.37   Final old_new: 17.68 ± 2.71   Final new_new: 5.91 ± 3.05
Hits@20:
  Final val: 54.04 ± 3.68   Final Test: 35.52 ± 2.48   Final old_old: 38.47 ± 2.59   Final old_new: 23.65 ± 2.23   Final new_new: 8.18 ± 3.05
Hits@30:
  Final val: 61.36 ± 2.39   Final Test: 43.00 ± 2.67   Final old_old: 46.08 ± 2.70   Final old_new: 30.88 ± 3.28   Final new_new: 10.00 ± 6.74
Hits@50:
  Final val: 71.14 ± 1.09   Final Test: 54.42 ± 3.64   Final old_old: 58.04 ± 3.99   Final old_new: 40.23 ± 2.60   Final new_new: 15.00 ± 6.74
AUC:
  Final val: 96.65 ± 0.30   Final Test: 95.95 ± 0.30   Final old_old: 96.58 ± 0.29   Final old_new: 93.42 ± 0.43   Final new_new: 90.26 ± 1.53
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.72 ± 5.82   Final Test: 33.56 ± 8.80   Final old_old: 36.81 ± 9.33   Final old_new: 20.50 ± 6.77   Final new_new: 4.09 ± 3.37
Hits@20:
  Final val: 61.93 ± 6.21   Final Test: 41.91 ± 7.19   Final old_old: 45.63 ± 7.71   Final old_new: 26.93 ± 5.19   Final new_new: 8.18 ± 3.45
Hits@30:
  Final val: 69.39 ± 4.37   Final Test: 51.41 ± 5.16   Final old_old: 55.44 ± 5.50   Final old_new: 35.59 ± 3.92   Final new_new: 7.73 ± 3.45
Hits@50:
  Final val: 76.91 ± 2.75   Final Test: 59.26 ± 6.85   Final old_old: 63.58 ± 7.19   Final old_new: 42.19 ± 5.62   Final new_new: 14.09 ± 4.93
AUC:
  Final val: 97.39 ± 0.26   Final Test: 96.80 ± 0.24   Final old_old: 97.45 ± 0.30   Final old_new: 94.23 ± 0.30   Final new_new: 90.04 ± 1.04
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.83 ± 7.53   Final Test: 32.52 ± 7.09   Final old_old: 35.25 ± 7.59   Final old_new: 21.49 ± 5.25   Final new_new: 8.64 ± 3.73
Hits@20:
  Final val: 55.57 ± 5.21   Final Test: 39.66 ± 6.33   Final old_old: 42.79 ± 6.78   Final old_new: 27.00 ± 4.65   Final new_new: 12.27 ± 3.45
Hits@30:
  Final val: 62.47 ± 3.72   Final Test: 47.00 ± 6.05   Final old_old: 50.34 ± 6.28   Final old_new: 33.63 ± 5.15   Final new_new: 15.91 ± 3.59
Hits@50:
  Final val: 69.62 ± 3.38   Final Test: 54.44 ± 5.25   Final old_old: 57.71 ± 5.21   Final old_new: 41.51 ± 5.51   Final new_new: 20.91 ± 4.37
AUC:
  Final val: 96.43 ± 0.19   Final Test: 95.91 ± 0.18   Final old_old: 96.44 ± 0.17   Final old_new: 93.79 ± 0.31   Final new_new: 91.26 ± 0.65
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.52 ± 3.44   Final Test: 30.75 ± 2.95   Final old_old: 33.36 ± 3.09   Final old_new: 20.28 ± 2.81   Final new_new: 6.36 ± 3.73
Hits@20:
  Final val: 59.15 ± 3.54   Final Test: 38.84 ± 3.55   Final old_old: 42.00 ± 3.81   Final old_new: 26.20 ± 2.61   Final new_new: 8.64 ± 4.07
Hits@30:
  Final val: 66.95 ± 2.34   Final Test: 48.91 ± 2.00   Final old_old: 52.52 ± 2.19   Final old_new: 34.58 ± 1.69   Final new_new: 12.73 ± 5.23
Hits@50:
  Final val: 74.55 ± 1.09   Final Test: 57.15 ± 1.59   Final old_old: 60.94 ± 1.66   Final old_new: 42.12 ± 1.31   Final new_new: 19.55 ± 4.13
AUC:
  Final val: 96.81 ± 0.25   Final Test: 96.23 ± 0.19   Final old_old: 96.81 ± 0.22   Final old_new: 93.94 ± 0.28   Final new_new: 90.15 ± 1.68
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.54 ± 3.54   Final Test: 29.74 ± 7.84   Final old_old: 32.10 ± 8.37   Final old_new: 20.25 ± 6.02   Final new_new: 8.18 ± 2.03
Hits@20:
  Final val: 56.01 ± 3.44   Final Test: 40.25 ± 7.11   Final old_old: 43.22 ± 7.34   Final old_new: 28.46 ± 6.52   Final new_new: 10.45 ± 3.45
Hits@30:
  Final val: 61.31 ± 2.88   Final Test: 46.30 ± 5.99   Final old_old: 49.40 ± 6.25   Final old_new: 34.03 ± 5.13   Final new_new: 14.55 ± 4.71
Hits@50:
  Final val: 69.23 ± 1.65   Final Test: 54.11 ± 7.29   Final old_old: 57.36 ± 7.48   Final old_new: 41.31 ± 6.88   Final new_new: 20.00 ± 4.37
AUC:
  Final val: 96.42 ± 0.17   Final Test: 95.92 ± 0.15   Final old_old: 96.41 ± 0.16   Final old_new: 93.97 ± 0.04   Final new_new: 90.54 ± 1.78
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.35 ± 4.87   Final Test: 30.67 ± 6.70   Final old_old: 33.55 ± 7.01   Final old_new: 18.92 ± 5.54   Final new_new: 7.73 ± 5.23
Hits@20:
  Final val: 60.51 ± 2.96   Final Test: 41.42 ± 4.87   Final old_old: 44.88 ± 5.12   Final old_new: 27.58 ± 3.96   Final new_new: 9.09 ± 5.57
Hits@30:
  Final val: 67.08 ± 2.47   Final Test: 50.39 ± 2.09   Final old_old: 54.09 ± 2.07   Final old_new: 35.82 ± 2.69   Final new_new: 11.82 ± 4.37
Hits@50:
  Final val: 74.59 ± 1.25   Final Test: 61.21 ± 2.14   Final old_old: 65.06 ± 2.18   Final old_new: 45.97 ± 2.33   Final new_new: 21.82 ± 4.43
AUC:
  Final val: 96.91 ± 0.19   Final Test: 96.45 ± 0.19   Final old_old: 97.05 ± 0.20   Final old_new: 94.07 ± 0.35   Final new_new: 90.33 ± 1.55
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.30 ± 2.51   Final Test: 32.79 ± 5.39   Final old_old: 35.95 ± 5.64   Final old_new: 20.18 ± 4.89   Final new_new: 2.27 ± 0.00
Hits@20:
  Final val: 62.12 ± 3.50   Final Test: 44.45 ± 3.43   Final old_old: 48.04 ± 3.52   Final old_new: 30.25 ± 4.00   Final new_new: 8.18 ± 2.03
Hits@30:
  Final val: 68.07 ± 1.71   Final Test: 52.16 ± 3.06   Final old_old: 55.88 ± 3.07   Final old_new: 37.68 ± 3.30   Final new_new: 9.09 ± 3.21
Hits@50:
  Final val: 74.58 ± 1.46   Final Test: 60.36 ± 1.84   Final old_old: 64.11 ± 1.77   Final old_new: 45.74 ± 2.26   Final new_new: 17.27 ± 4.13
AUC:
  Final val: 97.08 ± 0.11   Final Test: 96.64 ± 0.11   Final old_old: 97.09 ± 0.11   Final old_new: 94.82 ± 0.27   Final new_new: 92.40 ± 0.60
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.73 ± 3.66   Final Test: 33.31 ± 1.90   Final old_old: 36.48 ± 2.13   Final old_new: 20.65 ± 1.53   Final new_new: 2.73 ± 1.90
Hits@20:
  Final val: 62.34 ± 3.58   Final Test: 45.08 ± 4.10   Final old_old: 48.62 ± 4.16   Final old_new: 31.23 ± 4.16   Final new_new: 5.91 ± 3.45
Hits@30:
  Final val: 70.31 ± 2.59   Final Test: 52.09 ± 2.54   Final old_old: 55.93 ± 2.65   Final old_new: 37.23 ± 2.11   Final new_new: 6.82 ± 2.78
Hits@50:
  Final val: 77.80 ± 1.85   Final Test: 61.76 ± 1.45   Final old_old: 65.85 ± 1.76   Final old_new: 45.89 ± 1.06   Final new_new: 14.09 ± 2.96
AUC:
  Final val: 97.43 ± 0.03   Final Test: 97.01 ± 0.09   Final old_old: 97.50 ± 0.11   Final old_new: 95.06 ± 0.22   Final new_new: 92.11 ± 0.79
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.28 ± 3.78   Final Test: 34.70 ± 4.53   Final old_old: 38.10 ± 4.57   Final old_new: 21.03 ± 4.66   Final new_new: 3.18 ± 1.24
Hits@20:
  Final val: 61.46 ± 2.94   Final Test: 43.13 ± 4.79   Final old_old: 46.83 ± 5.10   Final old_new: 28.49 ± 3.86   Final new_new: 5.45 ± 2.59
Hits@30:
  Final val: 68.17 ± 1.71   Final Test: 50.81 ± 3.85   Final old_old: 54.89 ± 3.95   Final old_new: 34.76 ± 3.84   Final new_new: 7.73 ± 4.13
Hits@50:
  Final val: 76.61 ± 1.32   Final Test: 61.57 ± 2.12   Final old_old: 66.12 ± 2.18   Final old_new: 43.60 ± 1.96   Final new_new: 14.09 ± 6.31
AUC:
  Final val: 97.45 ± 0.17   Final Test: 96.85 ± 0.09   Final old_old: 97.53 ± 0.18   Final old_new: 94.12 ± 0.52   Final new_new: 90.52 ± 1.74
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.45 ± 4.42   Final Test: 29.23 ± 5.18   Final old_old: 32.03 ± 5.22   Final old_new: 18.04 ± 5.35   Final new_new: 3.18 ± 1.24
Hits@20:
  Final val: 61.44 ± 2.65   Final Test: 40.97 ± 4.73   Final old_old: 44.30 ± 4.88   Final old_new: 27.91 ± 4.51   Final new_new: 5.00 ± 1.90
Hits@30:
  Final val: 66.49 ± 1.83   Final Test: 50.60 ± 1.68   Final old_old: 54.25 ± 1.61   Final old_new: 36.40 ± 2.74   Final new_new: 9.09 ± 1.61
Hits@50:
  Final val: 73.94 ± 1.69   Final Test: 58.00 ± 2.67   Final old_old: 61.74 ± 2.72   Final old_new: 43.38 ± 2.86   Final new_new: 15.91 ± 3.59
AUC:
  Final val: 97.07 ± 0.07   Final Test: 96.65 ± 0.09   Final old_old: 97.08 ± 0.08   Final old_new: 94.96 ± 0.20   Final new_new: 91.74 ± 1.26
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.95 ± 5.74   Final Test: 31.39 ± 5.60   Final old_old: 34.17 ± 6.12   Final old_new: 20.15 ± 3.49   Final new_new: 7.73 ± 5.23
Hits@20:
  Final val: 59.69 ± 3.06   Final Test: 45.17 ± 3.21   Final old_old: 48.69 ± 3.34   Final old_new: 31.06 ± 2.80   Final new_new: 11.82 ± 5.66
Hits@30:
  Final val: 65.61 ± 1.63   Final Test: 49.48 ± 1.94   Final old_old: 53.19 ± 1.90   Final old_new: 34.74 ± 2.40   Final new_new: 13.18 ± 5.18
Hits@50:
  Final val: 73.81 ± 1.26   Final Test: 58.10 ± 1.82   Final old_old: 62.02 ± 1.97   Final old_new: 42.57 ± 1.66   Final new_new: 18.18 ± 5.57
AUC:
  Final val: 96.73 ± 0.18   Final Test: 96.19 ± 0.09   Final old_old: 96.80 ± 0.17   Final old_new: 93.77 ± 0.30   Final new_new: 90.35 ± 1.14
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.29 ± 6.19   Final Test: 30.84 ± 4.19   Final old_old: 33.85 ± 4.45   Final old_new: 18.77 ± 3.35   Final new_new: 2.73 ± 1.90
Hits@20:
  Final val: 64.07 ± 5.24   Final Test: 41.59 ± 3.57   Final old_old: 45.18 ± 3.76   Final old_new: 27.30 ± 3.30   Final new_new: 5.91 ± 3.45
Hits@30:
  Final val: 70.07 ± 3.57   Final Test: 51.16 ± 1.74   Final old_old: 55.11 ± 1.67   Final old_new: 35.54 ± 2.43   Final new_new: 10.00 ± 3.80
Hits@50:
  Final val: 77.90 ± 1.26   Final Test: 61.18 ± 2.43   Final old_old: 65.33 ± 2.39   Final old_new: 44.81 ± 2.76   Final new_new: 18.18 ± 3.21
AUC:
  Final val: 97.63 ± 0.09   Final Test: 97.10 ± 0.08   Final old_old: 97.60 ± 0.09   Final old_new: 95.17 ± 0.20   Final new_new: 91.57 ± 0.70
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.99 ± 4.84   Final Test: 33.64 ± 6.51   Final old_old: 37.04 ± 7.04   Final old_new: 19.82 ± 4.42   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 62.14 ± 2.57   Final Test: 41.57 ± 5.64   Final old_old: 45.19 ± 5.90   Final old_new: 26.95 ± 4.88   Final new_new: 9.55 ± 5.43
Hits@30:
  Final val: 68.18 ± 2.15   Final Test: 52.22 ± 4.00   Final old_old: 56.03 ± 4.28   Final old_new: 37.20 ± 3.41   Final new_new: 12.27 ± 3.80
Hits@50:
  Final val: 74.41 ± 2.06   Final Test: 60.36 ± 2.59   Final old_old: 64.27 ± 2.87   Final old_new: 45.01 ± 1.67   Final new_new: 17.73 ± 4.93
AUC:
  Final val: 97.10 ± 0.09   Final Test: 96.70 ± 0.15   Final old_old: 97.15 ± 0.13   Final old_new: 94.92 ± 0.28   Final new_new: 91.90 ± 0.81
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.97 ± 5.32   Final Test: 32.12 ± 2.14   Final old_old: 34.77 ± 2.53   Final old_new: 21.44 ± 1.60   Final new_new: 8.18 ± 4.98
Hits@20:
  Final val: 59.32 ± 2.37   Final Test: 42.92 ± 2.60   Final old_old: 46.26 ± 2.82   Final old_new: 29.47 ± 2.20   Final new_new: 12.27 ± 4.98
Hits@30:
  Final val: 67.38 ± 1.46   Final Test: 50.47 ± 2.06   Final old_old: 53.99 ± 2.22   Final old_new: 36.45 ± 1.80   Final new_new: 15.45 ± 5.88
Hits@50:
  Final val: 74.52 ± 1.06   Final Test: 60.19 ± 2.10   Final old_old: 63.88 ± 2.17   Final old_new: 45.67 ± 2.32   Final new_new: 20.91 ± 6.89
AUC:
  Final val: 96.80 ± 0.25   Final Test: 96.33 ± 0.26   Final old_old: 96.90 ± 0.24   Final old_new: 94.02 ± 0.33   Final new_new: 91.07 ± 2.10
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.66 ± 1.73   Final Test: 28.34 ± 4.14   Final old_old: 30.79 ± 4.38   Final old_new: 18.51 ± 3.47   Final new_new: 5.45 ± 3.05
Hits@20:
  Final val: 54.75 ± 1.25   Final Test: 37.31 ± 5.14   Final old_old: 40.38 ± 5.43   Final old_new: 25.09 ± 4.10   Final new_new: 7.27 ± 5.18
Hits@30:
  Final val: 62.21 ± 1.42   Final Test: 44.16 ± 2.32   Final old_old: 47.54 ± 2.39   Final old_new: 30.68 ± 2.43   Final new_new: 11.36 ± 4.25
Hits@50:
  Final val: 71.36 ± 1.27   Final Test: 53.53 ± 2.56   Final old_old: 57.02 ± 2.38   Final old_new: 39.77 ± 3.54   Final new_new: 16.36 ± 6.51
AUC:
  Final val: 96.75 ± 0.24   Final Test: 96.04 ± 0.24   Final old_old: 96.73 ± 0.26   Final old_new: 93.35 ± 0.16   Final new_new: 88.95 ± 1.35
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.11 ± 4.16   Final Test: 30.61 ± 4.74   Final old_old: 33.18 ± 4.98   Final old_new: 20.33 ± 3.92   Final new_new: 6.82 ± 3.21
Hits@20:
  Final val: 56.05 ± 2.44   Final Test: 39.19 ± 2.18   Final old_old: 42.24 ± 2.43   Final old_new: 26.93 ± 1.30   Final new_new: 11.36 ± 3.94
Hits@30:
  Final val: 61.85 ± 1.51   Final Test: 46.54 ± 1.69   Final old_old: 49.92 ± 1.80   Final old_new: 33.00 ± 1.71   Final new_new: 15.00 ± 6.35
Hits@50:
  Final val: 68.68 ± 1.19   Final Test: 53.15 ± 3.43   Final old_old: 56.50 ± 3.34   Final old_new: 39.92 ± 4.15   Final new_new: 18.64 ± 1.90
AUC:
  Final val: 96.39 ± 0.21   Final Test: 95.84 ± 0.21   Final old_old: 96.33 ± 0.19   Final old_new: 93.90 ± 0.26   Final new_new: 90.77 ± 1.53
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 54.04 ± 3.11   Final Test: 31.58 ± 2.57   Final old_old: 34.39 ± 2.81   Final old_new: 20.35 ± 1.89   Final new_new: 4.55 ± 4.82
Hits@20:
  Final val: 65.41 ± 0.68   Final Test: 40.40 ± 4.36   Final old_old: 43.67 ± 4.70   Final old_new: 27.25 ± 2.91   Final new_new: 10.45 ± 6.14
Hits@30:
  Final val: 70.93 ± 0.82   Final Test: 51.32 ± 2.31   Final old_old: 55.03 ± 2.35   Final old_new: 36.47 ± 2.28   Final new_new: 16.36 ± 5.18
Hits@50:
  Final val: 77.35 ± 0.62   Final Test: 60.07 ± 1.97   Final old_old: 63.96 ± 2.08   Final old_new: 44.71 ± 1.75   Final new_new: 20.00 ± 4.93
AUC:
  Final val: 97.17 ± 0.14   Final Test: 96.56 ± 0.08   Final old_old: 97.09 ± 0.11   Final old_new: 94.45 ± 0.14   Final new_new: 91.01 ± 0.98
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.46 ± 7.29   Final Test: 28.72 ± 7.24   Final old_old: 31.29 ± 7.47   Final old_new: 18.29 ± 6.61   Final new_new: 7.73 ± 3.80
Hits@20:
  Final val: 56.43 ± 4.73   Final Test: 39.00 ± 4.33   Final old_old: 42.07 ± 4.51   Final old_new: 26.85 ± 3.88   Final new_new: 8.18 ± 3.45
Hits@30:
  Final val: 62.59 ± 3.75   Final Test: 45.82 ± 3.63   Final old_old: 49.08 ± 3.85   Final old_new: 32.95 ± 3.08   Final new_new: 11.36 ± 2.78
Hits@50:
  Final val: 69.97 ± 1.76   Final Test: 55.31 ± 2.02   Final old_old: 58.69 ± 2.24   Final old_new: 42.22 ± 2.11   Final new_new: 15.91 ± 6.82
AUC:
  Final val: 96.41 ± 0.21   Final Test: 95.93 ± 0.20   Final old_old: 96.45 ± 0.19   Final old_new: 93.89 ± 0.27   Final new_new: 90.04 ± 0.97
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.36 ± 5.59   Final Test: 35.77 ± 8.17   Final old_old: 39.15 ± 8.60   Final old_new: 22.29 ± 6.96   Final new_new: 3.18 ± 2.59
Hits@20:
  Final val: 64.73 ± 3.34   Final Test: 44.72 ± 7.38   Final old_old: 48.41 ± 7.54   Final old_new: 30.08 ± 7.45   Final new_new: 8.18 ± 5.23
Hits@30:
  Final val: 71.04 ± 2.08   Final Test: 54.70 ± 4.52   Final old_old: 58.65 ± 4.79   Final old_new: 39.17 ± 3.88   Final new_new: 11.82 ± 4.37
Hits@50:
  Final val: 77.74 ± 1.59   Final Test: 63.99 ± 2.49   Final old_old: 67.98 ± 2.67   Final old_new: 48.51 ± 2.19   Final new_new: 18.18 ± 3.59
AUC:
  Final val: 97.43 ± 0.13   Final Test: 97.04 ± 0.15   Final old_old: 97.55 ± 0.16   Final old_new: 95.07 ± 0.18   Final new_new: 90.82 ± 0.78
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.69 ± 2.06   Final Test: 34.62 ± 4.80   Final old_old: 38.05 ± 5.11   Final old_new: 20.76 ± 3.74   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 66.08 ± 1.59   Final Test: 45.79 ± 2.23   Final old_old: 49.62 ± 2.42   Final old_new: 30.65 ± 1.99   Final new_new: 5.91 ± 2.03
Hits@30:
  Final val: 72.67 ± 1.34   Final Test: 53.42 ± 3.73   Final old_old: 57.53 ± 4.10   Final old_new: 37.30 ± 2.46   Final new_new: 8.64 ± 2.96
Hits@50:
  Final val: 78.83 ± 0.78   Final Test: 64.11 ± 2.31   Final old_old: 68.34 ± 2.44   Final old_new: 47.66 ± 2.64   Final new_new: 15.45 ± 5.43
AUC:
  Final val: 97.50 ± 0.07   Final Test: 97.05 ± 0.08   Final old_old: 97.52 ± 0.08   Final old_new: 95.24 ± 0.20   Final new_new: 91.93 ± 0.75
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.01 ± 1.71   Final Test: 26.19 ± 4.01   Final old_old: 28.67 ± 4.23   Final old_new: 16.07 ± 3.38   Final new_new: 6.36 ± 1.90
Hits@20:
  Final val: 56.96 ± 1.36   Final Test: 32.99 ± 5.10   Final old_old: 35.78 ± 5.50   Final old_new: 21.76 ± 3.82   Final new_new: 8.18 ± 3.80
Hits@30:
  Final val: 63.19 ± 1.94   Final Test: 44.54 ± 3.90   Final old_old: 47.99 ± 4.33   Final old_new: 30.71 ± 2.40   Final new_new: 11.82 ± 4.93
Hits@50:
  Final val: 70.86 ± 1.27   Final Test: 53.31 ± 3.04   Final old_old: 56.87 ± 3.23   Final old_new: 39.14 ± 3.00   Final new_new: 17.73 ± 5.18
AUC:
  Final val: 96.56 ± 0.13   Final Test: 95.90 ± 0.16   Final old_old: 96.47 ± 0.14   Final old_new: 93.67 ± 0.27   Final new_new: 90.39 ± 1.35
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.30 ± 3.37   Final Test: 34.25 ± 2.89   Final old_old: 37.60 ± 3.08   Final old_new: 20.76 ± 2.40   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 63.17 ± 3.71   Final Test: 45.28 ± 2.91   Final old_old: 48.96 ± 3.25   Final old_new: 30.78 ± 1.87   Final new_new: 6.82 ± 0.00
Hits@30:
  Final val: 70.06 ± 2.46   Final Test: 56.06 ± 2.14   Final old_old: 60.07 ± 2.36   Final old_new: 40.33 ± 1.83   Final new_new: 12.73 ± 5.93
Hits@50:
  Final val: 78.19 ± 1.28   Final Test: 64.82 ± 2.10   Final old_old: 68.83 ± 2.13   Final old_new: 49.17 ± 1.87   Final new_new: 20.00 ± 6.70
AUC:
  Final val: 97.60 ± 0.10   Final Test: 97.32 ± 0.08   Final old_old: 97.75 ± 0.09   Final old_new: 95.64 ± 0.10   Final new_new: 92.40 ± 0.63
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.30 ± 2.38   Final Test: 33.72 ± 2.80   Final old_old: 37.02 ± 2.99   Final old_new: 20.38 ± 2.11   Final new_new: 5.00 ± 2.49
Hits@20:
  Final val: 60.21 ± 1.64   Final Test: 44.93 ± 4.19   Final old_old: 48.56 ± 4.20   Final old_new: 30.55 ± 4.35   Final new_new: 8.64 ± 4.07
Hits@30:
  Final val: 67.26 ± 1.24   Final Test: 51.60 ± 2.39   Final old_old: 55.34 ± 2.38   Final old_new: 36.88 ± 2.81   Final new_new: 12.27 ± 3.80
Hits@50:
  Final val: 74.09 ± 1.39   Final Test: 59.49 ± 1.88   Final old_old: 63.15 ± 1.94   Final old_new: 45.24 ± 2.06   Final new_new: 17.73 ± 7.08
AUC:
  Final val: 96.99 ± 0.05   Final Test: 96.66 ± 0.06   Final old_old: 97.11 ± 0.04   Final old_new: 94.87 ± 0.22   Final new_new: 92.46 ± 1.03
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.33 ± 1.10   Final Test: 28.61 ± 3.97   Final old_old: 31.10 ± 4.20   Final old_new: 18.69 ± 3.04   Final new_new: 4.09 ± 2.49
Hits@20:
  Final val: 61.93 ± 1.79   Final Test: 42.47 ± 2.39   Final old_old: 45.81 ± 2.56   Final old_new: 29.35 ± 1.93   Final new_new: 6.82 ± 2.27
Hits@30:
  Final val: 67.15 ± 1.58   Final Test: 49.36 ± 3.57   Final old_old: 53.23 ± 3.71   Final old_new: 33.95 ± 3.14   Final new_new: 11.36 ± 4.25
Hits@50:
  Final val: 74.56 ± 0.95   Final Test: 58.70 ± 2.66   Final old_old: 62.43 ± 2.54   Final old_new: 44.11 ± 3.14   Final new_new: 17.27 ± 6.93
AUC:
  Final val: 96.96 ± 0.14   Final Test: 96.43 ± 0.17   Final old_old: 96.95 ± 0.11   Final old_new: 94.45 ± 0.46   Final new_new: 90.18 ± 1.10
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.56 ± 3.50   Final Test: 32.02 ± 3.36   Final old_old: 35.37 ± 3.45   Final old_new: 18.34 ± 3.12   Final new_new: 5.00 ± 2.49
Hits@20:
  Final val: 65.25 ± 1.60   Final Test: 42.75 ± 1.60   Final old_old: 46.69 ± 1.62   Final old_new: 26.95 ± 1.69   Final new_new: 6.36 ± 2.49
Hits@30:
  Final val: 71.58 ± 1.55   Final Test: 52.12 ± 1.20   Final old_old: 56.48 ± 1.10   Final old_new: 34.81 ± 2.07   Final new_new: 8.64 ± 1.02
Hits@50:
  Final val: 77.86 ± 1.60   Final Test: 60.76 ± 2.43   Final old_old: 65.08 ± 2.81   Final old_new: 43.83 ± 1.00   Final new_new: 13.64 ± 5.79
AUC:
  Final val: 97.55 ± 0.11   Final Test: 96.97 ± 0.10   Final old_old: 97.60 ± 0.11   Final old_new: 94.51 ± 0.18   Final new_new: 90.55 ± 0.60
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.39 ± 2.92   Final Test: 33.07 ± 4.02   Final old_old: 36.32 ± 4.34   Final old_new: 19.97 ± 3.29   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 61.94 ± 1.80   Final Test: 45.52 ± 4.40   Final old_old: 49.02 ± 4.66   Final old_new: 31.84 ± 3.49   Final new_new: 7.27 ± 1.90
Hits@30:
  Final val: 67.72 ± 1.71   Final Test: 52.80 ± 3.88   Final old_old: 56.61 ± 4.07   Final old_new: 37.86 ± 3.07   Final new_new: 10.91 ± 5.66
Hits@50:
  Final val: 73.97 ± 1.02   Final Test: 61.58 ± 1.73   Final old_old: 65.20 ± 1.77   Final old_new: 47.63 ± 1.91   Final new_new: 16.82 ± 4.98
AUC:
  Final val: 96.99 ± 0.06   Final Test: 96.60 ± 0.10   Final old_old: 97.04 ± 0.09   Final old_new: 94.81 ± 0.14   Final new_new: 92.52 ± 0.91
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.78 ± 6.82   Final Test: 31.94 ± 11.84   Final old_old: 34.60 ± 12.80   Final old_new: 21.36 ± 8.04   Final new_new: 5.45 ± 3.05
Hits@20:
  Final val: 58.91 ± 5.78   Final Test: 40.69 ± 5.97   Final old_old: 43.94 ± 6.41   Final old_new: 27.86 ± 4.46   Final new_new: 6.82 ± 2.78
Hits@30:
  Final val: 65.49 ± 4.11   Final Test: 47.65 ± 7.07   Final old_old: 51.17 ± 7.29   Final old_new: 33.80 ± 6.50   Final new_new: 10.00 ± 2.59
Hits@50:
  Final val: 74.03 ± 2.57   Final Test: 58.77 ± 2.54   Final old_old: 62.50 ± 2.73   Final old_new: 44.28 ± 2.21   Final new_new: 15.00 ± 2.03
AUC:
  Final val: 96.79 ± 0.21   Final Test: 96.20 ± 0.19   Final old_old: 96.81 ± 0.23   Final old_new: 93.74 ± 0.18   Final new_new: 90.74 ± 0.61
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.42 ± 3.47   Final Test: 34.11 ± 6.32   Final old_old: 36.93 ± 6.79   Final old_new: 22.80 ± 4.56   Final new_new: 8.64 ± 5.66
Hits@20:
  Final val: 62.39 ± 1.77   Final Test: 44.24 ± 6.16   Final old_old: 47.65 ± 6.48   Final old_new: 30.60 ± 4.90   Final new_new: 11.82 ± 6.89
Hits@30:
  Final val: 68.59 ± 1.49   Final Test: 53.84 ± 3.56   Final old_old: 57.71 ± 3.87   Final old_new: 38.29 ± 2.46   Final new_new: 19.09 ± 4.71
Hits@50:
  Final val: 76.56 ± 0.42   Final Test: 62.77 ± 2.14   Final old_old: 66.81 ± 2.25   Final old_new: 46.62 ± 1.93   Final new_new: 24.09 ± 5.23
AUC:
  Final val: 97.19 ± 0.20   Final Test: 96.66 ± 0.11   Final old_old: 97.20 ± 0.07   Final old_new: 94.53 ± 0.33   Final new_new: 90.30 ± 0.91
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.15 ± 3.95   Final Test: 29.47 ± 8.85   Final old_old: 32.39 ± 9.52   Final old_new: 17.71 ± 6.23   Final new_new: 3.18 ± 2.59
Hits@20:
  Final val: 60.16 ± 2.63   Final Test: 40.67 ± 2.91   Final old_old: 44.19 ± 3.19   Final old_new: 26.57 ± 1.89   Final new_new: 8.18 ± 2.59
Hits@30:
  Final val: 66.73 ± 1.84   Final Test: 47.66 ± 2.39   Final old_old: 51.44 ± 2.48   Final old_new: 32.59 ± 2.44   Final new_new: 10.91 ± 4.93
Hits@50:
  Final val: 74.54 ± 1.50   Final Test: 57.64 ± 2.41   Final old_old: 61.59 ± 2.54   Final old_new: 42.14 ± 2.31   Final new_new: 15.00 ± 3.45
AUC:
  Final val: 97.18 ± 0.09   Final Test: 96.63 ± 0.14   Final old_old: 97.18 ± 0.10   Final old_new: 94.49 ± 0.39   Final new_new: 90.70 ± 1.25
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 54.27 ± 1.38   Final Test: 33.75 ± 6.20   Final old_old: 37.03 ± 6.46   Final old_new: 20.63 ± 5.29   Final new_new: 2.73 ± 2.96
Hits@20:
  Final val: 65.73 ± 1.44   Final Test: 49.28 ± 7.72   Final old_old: 53.43 ± 8.00   Final old_new: 32.82 ± 6.81   Final new_new: 6.82 ± 4.55
Hits@30:
  Final val: 72.08 ± 1.95   Final Test: 56.63 ± 3.83   Final old_old: 60.90 ± 3.65   Final old_new: 39.85 ± 4.85   Final new_new: 11.36 ± 4.55
Hits@50:
  Final val: 78.79 ± 1.43   Final Test: 64.88 ± 2.48   Final old_old: 68.99 ± 2.29   Final old_new: 48.92 ± 3.58   Final new_new: 17.73 ± 1.90
AUC:
  Final val: 97.40 ± 0.06   Final Test: 97.01 ± 0.08   Final old_old: 97.39 ± 0.05   Final old_new: 95.49 ± 0.28   Final new_new: 92.68 ± 0.82
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.25 ± 3.80   Final Test: 29.65 ± 3.13   Final old_old: 32.10 ± 3.26   Final old_new: 19.87 ± 3.13   Final new_new: 6.36 ± 3.37
Hits@20:
  Final val: 55.40 ± 2.40   Final Test: 38.42 ± 4.61   Final old_old: 41.28 ± 4.89   Final old_new: 27.18 ± 4.03   Final new_new: 8.18 ± 2.59
Hits@30:
  Final val: 61.73 ± 1.47   Final Test: 47.93 ± 3.33   Final old_old: 51.16 ± 3.44   Final old_new: 35.37 ± 3.63   Final new_new: 10.91 ± 4.93
Hits@50:
  Final val: 69.03 ± 0.97   Final Test: 55.35 ± 2.83   Final old_old: 58.64 ± 2.65   Final old_new: 42.54 ± 3.82   Final new_new: 18.18 ± 5.79
AUC:
  Final val: 96.25 ± 0.18   Final Test: 95.85 ± 0.12   Final old_old: 96.33 ± 0.12   Final old_new: 93.94 ± 0.19   Final new_new: 91.10 ± 1.28
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.99 ± 5.40   Final Test: 26.99 ± 3.97   Final old_old: 29.35 ± 4.38   Final old_new: 17.43 ± 2.72   Final new_new: 6.36 ± 1.90
Hits@20:
  Final val: 54.59 ± 5.76   Final Test: 38.67 ± 5.29   Final old_old: 41.80 ± 5.89   Final old_new: 26.07 ± 3.35   Final new_new: 10.00 ± 4.43
Hits@30:
  Final val: 62.09 ± 4.64   Final Test: 47.81 ± 2.88   Final old_old: 51.30 ± 3.36   Final old_new: 33.78 ± 1.77   Final new_new: 16.36 ± 5.18
Hits@50:
  Final val: 71.51 ± 3.21   Final Test: 56.38 ± 2.36   Final old_old: 60.09 ± 2.80   Final old_new: 41.59 ± 1.74   Final new_new: 20.45 ± 2.78
AUC:
  Final val: 96.73 ± 0.36   Final Test: 96.09 ± 0.28   Final old_old: 96.78 ± 0.27   Final old_new: 93.32 ± 0.53   Final new_new: 90.03 ± 1.53
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 54.50 ± 2.00   Final Test: 35.97 ± 4.78   Final old_old: 39.42 ± 5.04   Final old_new: 22.24 ± 3.88   Final new_new: 2.73 ± 1.90
Hits@20:
  Final val: 66.44 ± 1.87   Final Test: 49.49 ± 2.54   Final old_old: 53.64 ± 2.67   Final old_new: 33.02 ± 2.38   Final new_new: 7.27 ± 1.90
Hits@30:
  Final val: 72.91 ± 0.74   Final Test: 56.29 ± 3.68   Final old_old: 60.61 ± 3.87   Final old_new: 39.19 ± 3.20   Final new_new: 11.36 ± 3.21
Hits@50:
  Final val: 80.07 ± 1.49   Final Test: 66.47 ± 2.38   Final old_old: 70.87 ± 2.41   Final old_new: 49.22 ± 2.19   Final new_new: 18.18 ± 5.33
AUC:
  Final val: 97.66 ± 0.05   Final Test: 97.26 ± 0.02   Final old_old: 97.72 ± 0.03   Final old_new: 95.49 ± 0.09   Final new_new: 92.41 ± 0.62
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.54 ± 3.47   Final Test: 33.47 ± 7.23   Final old_old: 36.60 ± 7.85   Final old_new: 20.96 ± 4.93   Final new_new: 3.64 ± 2.03
Hits@20:
  Final val: 65.24 ± 1.86   Final Test: 46.10 ± 3.35   Final old_old: 49.94 ± 3.71   Final old_new: 30.91 ± 1.85   Final new_new: 6.82 ± 3.94
Hits@30:
  Final val: 71.38 ± 1.37   Final Test: 52.60 ± 2.61   Final old_old: 56.69 ± 2.95   Final old_new: 36.50 ± 1.13   Final new_new: 9.55 ± 4.37
Hits@50:
  Final val: 78.08 ± 1.18   Final Test: 63.28 ± 1.17   Final old_old: 67.56 ± 1.40   Final old_new: 46.47 ± 0.93   Final new_new: 16.82 ± 4.13
AUC:
  Final val: 97.51 ± 0.09   Final Test: 97.05 ± 0.12   Final old_old: 97.54 ± 0.10   Final old_new: 95.13 ± 0.27   Final new_new: 91.69 ± 0.79
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.40 ± 4.55   Final Test: 27.91 ± 5.44   Final old_old: 30.34 ± 5.74   Final old_new: 18.19 ± 4.34   Final new_new: 5.00 ± 2.96
Hits@20:
  Final val: 56.05 ± 3.48   Final Test: 40.11 ± 5.12   Final old_old: 42.86 ± 5.29   Final old_new: 29.37 ± 4.77   Final new_new: 9.55 ± 4.66
Hits@30:
  Final val: 62.62 ± 1.90   Final Test: 47.12 ± 5.79   Final old_old: 50.18 ± 6.12   Final old_new: 35.09 ± 4.65   Final new_new: 15.00 ± 2.03
Hits@50:
  Final val: 69.17 ± 1.44   Final Test: 55.66 ± 3.45   Final old_old: 58.86 ± 3.62   Final old_new: 43.15 ± 2.93   Final new_new: 20.45 ± 7.54
AUC:
  Final val: 96.37 ± 0.19   Final Test: 95.93 ± 0.11   Final old_old: 96.40 ± 0.10   Final old_new: 94.04 ± 0.31   Final new_new: 91.31 ± 0.29
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.44 ± 3.67   Final Test: 33.05 ± 6.18   Final old_old: 35.94 ± 6.60   Final old_new: 21.41 ± 4.58   Final new_new: 7.73 ± 2.59
Hits@20:
  Final val: 61.44 ± 2.57   Final Test: 46.60 ± 4.11   Final old_old: 50.12 ± 4.34   Final old_new: 32.52 ± 3.73   Final new_new: 13.18 ± 5.66
Hits@30:
  Final val: 67.46 ± 1.64   Final Test: 51.78 ± 2.32   Final old_old: 55.39 ± 2.51   Final old_new: 37.41 ± 1.99   Final new_new: 16.36 ± 2.96
Hits@50:
  Final val: 74.20 ± 0.92   Final Test: 59.53 ± 2.30   Final old_old: 63.16 ± 2.14   Final old_new: 45.11 ± 3.51   Final new_new: 23.18 ± 2.96
AUC:
  Final val: 96.85 ± 0.15   Final Test: 96.45 ± 0.07   Final old_old: 96.92 ± 0.08   Final old_new: 94.61 ± 0.09   Final new_new: 90.91 ± 0.96
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.33 ± 5.51   Final Test: 27.17 ± 4.94   Final old_old: 29.55 ± 5.39   Final old_new: 17.56 ± 3.11   Final new_new: 5.91 ± 3.05
Hits@20:
  Final val: 57.66 ± 3.73   Final Test: 35.55 ± 2.70   Final old_old: 38.48 ± 2.87   Final old_new: 23.73 ± 2.08   Final new_new: 9.55 ± 4.93
Hits@30:
  Final val: 63.51 ± 2.33   Final Test: 44.90 ± 3.89   Final old_old: 48.31 ± 4.05   Final old_new: 31.23 ± 3.28   Final new_new: 13.18 ± 4.93
Hits@50:
  Final val: 71.83 ± 1.49   Final Test: 53.71 ± 2.15   Final old_old: 57.61 ± 2.22   Final old_new: 38.14 ± 2.24   Final new_new: 15.91 ± 4.25
AUC:
  Final val: 96.59 ± 0.27   Final Test: 95.83 ± 0.19   Final old_old: 96.53 ± 0.15   Final old_new: 93.06 ± 0.51   Final new_new: 88.72 ± 1.00
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.85 ± 2.01   Final Test: 29.60 ± 3.73   Final old_old: 32.59 ± 3.88   Final old_new: 17.53 ± 3.44   Final new_new: 2.73 ± 1.02
Hits@20:
  Final val: 60.32 ± 1.53   Final Test: 42.87 ± 2.24   Final old_old: 46.53 ± 2.54   Final old_new: 28.26 ± 1.11   Final new_new: 7.27 ± 2.49
Hits@30:
  Final val: 68.45 ± 0.66   Final Test: 49.77 ± 1.56   Final old_old: 53.79 ± 1.78   Final old_new: 33.83 ± 1.29   Final new_new: 8.64 ± 2.96
Hits@50:
  Final val: 77.05 ± 1.26   Final Test: 61.00 ± 1.64   Final old_old: 65.38 ± 1.54   Final old_new: 43.70 ± 2.28   Final new_new: 15.45 ± 6.70
AUC:
  Final val: 97.45 ± 0.13   Final Test: 96.87 ± 0.19   Final old_old: 97.50 ± 0.17   Final old_new: 94.34 ± 0.29   Final new_new: 90.90 ± 0.95
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.01 ± 1.40   Final Test: 31.20 ± 5.66   Final old_old: 34.28 ± 6.05   Final old_new: 18.74 ± 4.24   Final new_new: 5.00 ± 2.49
Hits@20:
  Final val: 61.74 ± 1.37   Final Test: 41.24 ± 4.65   Final old_old: 44.80 ± 4.89   Final old_new: 27.08 ± 3.98   Final new_new: 6.36 ± 2.96
Hits@30:
  Final val: 66.89 ± 1.93   Final Test: 52.03 ± 2.19   Final old_old: 55.84 ± 2.13   Final old_new: 37.15 ± 2.65   Final new_new: 10.00 ± 2.59
Hits@50:
  Final val: 73.60 ± 0.79   Final Test: 60.63 ± 1.93   Final old_old: 64.39 ± 1.71   Final old_new: 46.10 ± 3.03   Final new_new: 16.36 ± 3.37
AUC:
  Final val: 96.94 ± 0.06   Final Test: 96.52 ± 0.08   Final old_old: 96.98 ± 0.06   Final old_new: 94.71 ± 0.29   Final new_new: 92.30 ± 0.77
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.63 ± 3.26   Final Test: 34.01 ± 5.31   Final old_old: 37.53 ± 5.48   Final old_new: 19.87 ± 5.13   Final new_new: 1.36 ± 1.24
Hits@20:
  Final val: 64.39 ± 1.86   Final Test: 45.31 ± 2.82   Final old_old: 49.20 ± 3.07   Final old_new: 29.92 ± 2.41   Final new_new: 5.00 ± 3.73
Hits@30:
  Final val: 69.90 ± 1.69   Final Test: 55.42 ± 4.07   Final old_old: 59.28 ± 4.23   Final old_new: 40.50 ± 3.52   Final new_new: 10.00 ± 5.47
Hits@50:
  Final val: 77.33 ± 1.15   Final Test: 63.94 ± 3.39   Final old_old: 67.78 ± 3.58   Final old_new: 49.22 ± 2.95   Final new_new: 16.36 ± 6.51
AUC:
  Final val: 97.29 ± 0.08   Final Test: 96.95 ± 0.05   Final old_old: 97.34 ± 0.08   Final old_new: 95.44 ± 0.12   Final new_new: 93.05 ± 0.59
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 55.11 ± 3.83   Final Test: 30.30 ± 3.45   Final old_old: 32.97 ± 3.89   Final old_new: 19.65 ± 2.05   Final new_new: 5.00 ± 2.96
Hits@20:
  Final val: 65.70 ± 1.23   Final Test: 46.00 ± 4.36   Final old_old: 49.46 ± 4.53   Final old_new: 32.19 ± 3.83   Final new_new: 12.73 ± 3.45
Hits@30:
  Final val: 70.87 ± 1.62   Final Test: 50.61 ± 4.59   Final old_old: 54.29 ± 4.97   Final old_new: 36.07 ± 3.28   Final new_new: 12.27 ± 2.59
Hits@50:
  Final val: 76.79 ± 0.96   Final Test: 62.37 ± 2.74   Final old_old: 66.27 ± 2.65   Final old_new: 46.93 ± 3.13   Final new_new: 22.73 ± 8.35
AUC:
  Final val: 97.22 ± 0.08   Final Test: 96.77 ± 0.05   Final old_old: 97.27 ± 0.05   Final old_new: 94.78 ± 0.19   Final new_new: 91.57 ± 0.92
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.78 ± 3.58   Final Test: 34.27 ± 3.77   Final old_old: 37.49 ± 3.85   Final old_new: 21.34 ± 3.66   Final new_new: 5.00 ± 1.90
Hits@20:
  Final val: 58.96 ± 2.50   Final Test: 43.95 ± 2.66   Final old_old: 47.50 ± 2.57   Final old_new: 30.00 ± 3.49   Final new_new: 6.36 ± 2.49
Hits@30:
  Final val: 65.30 ± 1.62   Final Test: 51.20 ± 1.69   Final old_old: 54.91 ± 1.67   Final old_new: 36.85 ± 2.48   Final new_new: 7.73 ± 2.03
Hits@50:
  Final val: 72.55 ± 0.86   Final Test: 59.97 ± 2.42   Final old_old: 63.66 ± 2.39   Final old_new: 45.82 ± 2.85   Final new_new: 13.64 ± 3.21
AUC:
  Final val: 96.88 ± 0.02   Final Test: 96.51 ± 0.10   Final old_old: 96.95 ± 0.09   Final old_new: 94.76 ± 0.14   Final new_new: 91.83 ± 0.43
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.54 ± 5.27   Final Test: 30.48 ± 5.86   Final old_old: 32.95 ± 6.32   Final old_new: 20.45 ± 4.46   Final new_new: 9.09 ± 5.08
Hits@20:
  Final val: 57.95 ± 1.34   Final Test: 36.31 ± 6.52   Final old_old: 39.19 ± 6.89   Final old_new: 24.71 ± 5.12   Final new_new: 10.45 ± 4.71
Hits@30:
  Final val: 65.05 ± 1.91   Final Test: 46.57 ± 4.17   Final old_old: 50.00 ± 4.23   Final old_new: 32.87 ± 4.03   Final new_new: 13.64 ± 7.01
Hits@50:
  Final val: 74.38 ± 1.60   Final Test: 59.13 ± 2.44   Final old_old: 62.81 ± 2.56   Final old_new: 44.56 ± 3.00   Final new_new: 21.82 ± 7.64
AUC:
  Final val: 96.81 ± 0.25   Final Test: 96.40 ± 0.23   Final old_old: 96.96 ± 0.20   Final old_new: 94.15 ± 0.40   Final new_new: 91.03 ± 1.29
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.58 ± 6.37   Final Test: 30.26 ± 6.17   Final old_old: 33.42 ± 6.67   Final old_new: 17.46 ± 4.24   Final new_new: 3.18 ± 2.59
Hits@20:
  Final val: 60.53 ± 6.44   Final Test: 42.72 ± 6.80   Final old_old: 46.60 ± 7.34   Final old_new: 27.13 ± 4.76   Final new_new: 6.82 ± 2.78
Hits@30:
  Final val: 67.27 ± 5.59   Final Test: 51.07 ± 4.53   Final old_old: 55.25 ± 4.77   Final old_new: 34.58 ± 3.63   Final new_new: 7.27 ± 2.49
Hits@50:
  Final val: 74.97 ± 2.57   Final Test: 59.03 ± 2.81   Final old_old: 63.18 ± 2.91   Final old_new: 42.82 ± 2.84   Final new_new: 11.82 ± 2.96
AUC:
  Final val: 97.26 ± 0.19   Final Test: 96.75 ± 0.16   Final old_old: 97.34 ± 0.23   Final old_new: 94.36 ± 0.34   Final new_new: 91.24 ± 0.91
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.86 ± 6.88   Final Test: 28.69 ± 5.37   Final old_old: 31.05 ± 5.71   Final old_new: 19.37 ± 4.29   Final new_new: 3.64 ± 2.03
Hits@20:
  Final val: 58.84 ± 4.76   Final Test: 45.84 ± 3.27   Final old_old: 49.16 ± 3.52   Final old_new: 32.77 ± 2.78   Final new_new: 10.45 ± 2.03
Hits@30:
  Final val: 65.76 ± 3.29   Final Test: 53.64 ± 2.24   Final old_old: 57.23 ± 2.32   Final old_new: 39.57 ± 2.22   Final new_new: 14.55 ± 3.45
Hits@50:
  Final val: 73.60 ± 1.84   Final Test: 59.64 ± 2.39   Final old_old: 63.40 ± 2.46   Final old_new: 45.09 ± 2.30   Final new_new: 15.45 ± 4.37
AUC:
  Final val: 96.85 ± 0.17   Final Test: 96.39 ± 0.08   Final old_old: 96.84 ± 0.08   Final old_new: 94.61 ± 0.20   Final new_new: 91.44 ± 1.50
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 37.13 ± 4.17   Final Test: 22.38 ± 9.18   Final old_old: 24.54 ± 10.03   Final old_new: 13.55 ± 5.88   Final new_new: 5.45 ± 3.80
Hits@20:
  Final val: 50.03 ± 2.76   Final Test: 34.60 ± 5.68   Final old_old: 37.66 ± 6.02   Final old_new: 22.22 ± 4.57   Final new_new: 8.64 ± 4.93
Hits@30:
  Final val: 59.10 ± 3.46   Final Test: 44.38 ± 3.25   Final old_old: 47.94 ± 3.57   Final old_new: 30.15 ± 3.28   Final new_new: 10.45 ± 2.59
Hits@50:
  Final val: 70.22 ± 2.19   Final Test: 53.92 ± 4.20   Final old_old: 57.77 ± 4.75   Final old_new: 38.69 ± 2.00   Final new_new: 13.64 ± 4.25
AUC:
  Final val: 96.92 ± 0.22   Final Test: 96.14 ± 0.15   Final old_old: 96.91 ± 0.22   Final old_new: 93.10 ± 0.37   Final new_new: 88.79 ± 1.23
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.26 ± 4.65   Final Test: 32.07 ± 5.64   Final old_old: 35.15 ± 6.08   Final old_new: 19.77 ± 4.15   Final new_new: 2.73 ± 2.96
Hits@20:
  Final val: 64.44 ± 2.78   Final Test: 46.47 ± 3.56   Final old_old: 50.41 ± 3.94   Final old_new: 30.71 ± 2.30   Final new_new: 8.64 ± 2.96
Hits@30:
  Final val: 70.68 ± 1.94   Final Test: 53.80 ± 1.53   Final old_old: 58.01 ± 1.67   Final old_new: 37.08 ± 1.70   Final new_new: 11.82 ± 4.37
Hits@50:
  Final val: 78.57 ± 0.98   Final Test: 64.22 ± 1.69   Final old_old: 68.26 ± 1.42   Final old_new: 48.39 ± 3.01   Final new_new: 20.00 ± 4.07
AUC:
  Final val: 97.66 ± 0.08   Final Test: 97.25 ± 0.05   Final old_old: 97.68 ± 0.05   Final old_new: 95.59 ± 0.11   Final new_new: 92.10 ± 0.65
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 36.87 ± 4.03   Final Test: 21.32 ± 3.51   Final old_old: 23.13 ± 3.75   Final old_new: 13.93 ± 2.59   Final new_new: 6.82 ± 3.21
Hits@20:
  Final val: 44.17 ± 4.80   Final Test: 29.00 ± 3.95   Final old_old: 31.00 ± 4.29   Final old_new: 21.03 ± 2.81   Final new_new: 10.00 ± 3.80
Hits@30:
  Final val: 49.60 ± 3.52   Final Test: 34.64 ± 3.99   Final old_old: 36.89 ± 4.28   Final old_new: 25.67 ± 2.81   Final new_new: 12.73 ± 6.14
Hits@50:
  Final val: 57.43 ± 2.66   Final Test: 44.56 ± 4.50   Final old_old: 46.91 ± 4.31   Final old_new: 35.29 ± 5.48   Final new_new: 20.45 ± 8.35
AUC:
  Final val: 94.37 ± 0.11   Final Test: 94.28 ± 0.18   Final old_old: 94.59 ± 0.22   Final old_new: 93.05 ± 0.25   Final new_new: 91.04 ± 0.19
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 35.25 ± 3.25   Final Test: 20.85 ± 2.42   Final old_old: 22.49 ± 2.44   Final old_new: 14.11 ± 2.35   Final new_new: 8.64 ± 3.37
Hits@20:
  Final val: 43.07 ± 1.72   Final Test: 28.78 ± 5.09   Final old_old: 30.74 ± 5.26   Final old_new: 20.71 ± 4.57   Final new_new: 14.55 ± 5.93
Hits@30:
  Final val: 50.08 ± 2.62   Final Test: 35.25 ± 3.46   Final old_old: 37.65 ± 3.69   Final old_new: 25.42 ± 3.14   Final new_new: 16.36 ± 6.89
Hits@50:
  Final val: 57.84 ± 2.13   Final Test: 45.00 ± 3.30   Final old_old: 47.40 ± 2.90   Final old_new: 35.52 ± 5.30   Final new_new: 20.45 ± 6.63
AUC:
  Final val: 94.41 ± 0.11   Final Test: 94.07 ± 0.14   Final old_old: 94.41 ± 0.11   Final old_new: 92.78 ± 0.50   Final new_new: 90.42 ± 1.42
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.71 ± 4.97   Final Test: 32.95 ± 6.84   Final old_old: 36.13 ± 7.15   Final old_new: 20.10 ± 5.72   Final new_new: 5.45 ± 4.98
Hits@20:
  Final val: 64.53 ± 2.95   Final Test: 42.40 ± 2.27   Final old_old: 46.03 ± 2.21   Final old_new: 27.88 ± 2.82   Final new_new: 7.73 ± 2.59
Hits@30:
  Final val: 70.79 ± 1.79   Final Test: 51.18 ± 2.12   Final old_old: 55.32 ± 1.95   Final old_new: 34.81 ± 2.92   Final new_new: 7.73 ± 3.80
Hits@50:
  Final val: 77.67 ± 1.38   Final Test: 61.10 ± 2.66   Final old_old: 65.20 ± 2.64   Final old_new: 45.06 ± 3.17   Final new_new: 15.45 ± 2.49
AUC:
  Final val: 97.49 ± 0.08   Final Test: 97.01 ± 0.07   Final old_old: 97.45 ± 0.07   Final old_new: 95.25 ± 0.09   Final new_new: 92.68 ± 0.40
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 39.10 ± 1.97   Final Test: 23.25 ± 3.40   Final old_old: 25.10 ± 3.44   Final old_new: 15.77 ± 3.47   Final new_new: 7.27 ± 2.96
Hits@20:
  Final val: 47.68 ± 2.36   Final Test: 32.12 ± 6.38   Final old_old: 34.27 ± 6.37   Final old_new: 23.50 ± 6.72   Final new_new: 12.27 ± 3.80
Hits@30:
  Final val: 52.86 ± 1.63   Final Test: 40.13 ± 5.86   Final old_old: 42.67 ± 6.03   Final old_new: 30.10 ± 5.54   Final new_new: 13.64 ± 4.25
Hits@50:
  Final val: 60.28 ± 1.18   Final Test: 50.53 ± 2.34   Final old_old: 53.19 ± 2.15   Final old_new: 40.25 ± 3.72   Final new_new: 19.55 ± 4.71
AUC:
  Final val: 94.90 ± 0.09   Final Test: 94.60 ± 0.26   Final old_old: 95.00 ± 0.24   Final old_new: 93.01 ± 0.36   Final new_new: 90.52 ± 0.90
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.97 ± 6.18   Final Test: 30.82 ± 3.92   Final old_old: 33.34 ± 4.34   Final old_new: 20.63 ± 2.50   Final new_new: 9.09 ± 5.79
Hits@20:
  Final val: 60.72 ± 7.17   Final Test: 42.24 ± 4.05   Final old_old: 45.29 ± 4.31   Final old_new: 30.05 ± 3.30   Final new_new: 13.18 ± 2.96
Hits@30:
  Final val: 67.40 ± 5.61   Final Test: 50.80 ± 5.46   Final old_old: 54.16 ± 5.79   Final old_new: 37.53 ± 4.46   Final new_new: 15.91 ± 5.08
Hits@50:
  Final val: 75.18 ± 2.90   Final Test: 57.05 ± 4.28   Final old_old: 60.70 ± 4.64   Final old_new: 42.57 ± 2.99   Final new_new: 20.91 ± 4.93
AUC:
  Final val: 96.93 ± 0.18   Final Test: 96.24 ± 0.24   Final old_old: 96.88 ± 0.25   Final old_new: 93.79 ± 0.22   Final new_new: 88.11 ± 1.20
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 54.46 ± 2.86   Final Test: 35.51 ± 2.27   Final old_old: 38.97 ± 2.51   Final old_new: 21.71 ± 1.69   Final new_new: 2.27 ± 1.61
Hits@20:
  Final val: 64.95 ± 2.34   Final Test: 47.93 ± 2.27   Final old_old: 51.76 ± 2.40   Final old_new: 32.72 ± 1.78   Final new_new: 9.09 ± 3.59
Hits@30:
  Final val: 71.26 ± 1.36   Final Test: 54.13 ± 2.87   Final old_old: 58.14 ± 2.92   Final old_new: 38.41 ± 2.99   Final new_new: 10.45 ± 3.45
Hits@50:
  Final val: 77.43 ± 0.78   Final Test: 64.01 ± 3.76   Final old_old: 67.92 ± 4.02   Final old_new: 48.97 ± 2.84   Final new_new: 16.36 ± 3.73
AUC:
  Final val: 97.26 ± 0.07   Final Test: 96.86 ± 0.10   Final old_old: 97.27 ± 0.07   Final old_new: 95.29 ± 0.24   Final new_new: 91.91 ± 0.74
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 39.21 ± 1.72   Final Test: 26.81 ± 3.34   Final old_old: 28.90 ± 3.44   Final old_new: 18.34 ± 3.02   Final new_new: 9.55 ± 2.96
Hits@20:
  Final val: 47.61 ± 2.73   Final Test: 32.75 ± 3.62   Final old_old: 35.03 ± 3.79   Final old_new: 23.68 ± 3.12   Final new_new: 10.00 ± 3.80
Hits@30:
  Final val: 52.62 ± 2.14   Final Test: 39.46 ± 2.43   Final old_old: 41.97 ± 2.45   Final old_new: 29.47 ± 2.44   Final new_new: 15.00 ± 2.59
Hits@50:
  Final val: 61.29 ± 1.17   Final Test: 47.86 ± 2.21   Final old_old: 50.38 ± 2.54   Final old_new: 37.96 ± 1.53   Final new_new: 20.91 ± 3.73
AUC:
  Final val: 95.38 ± 0.08   Final Test: 95.16 ± 0.10   Final old_old: 95.49 ± 0.11   Final old_new: 93.92 ± 0.17   Final new_new: 91.26 ± 0.62
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.71 ± 3.98   Final Test: 37.77 ± 7.28   Final old_old: 41.82 ± 7.97   Final old_new: 21.39 ± 4.45   Final new_new: 2.27 ± 2.78
Hits@20:
  Final val: 63.60 ± 2.31   Final Test: 48.00 ± 7.13   Final old_old: 52.72 ± 7.46   Final old_new: 29.14 ± 5.90   Final new_new: 2.73 ± 3.73
Hits@30:
  Final val: 69.70 ± 1.47   Final Test: 56.48 ± 5.62   Final old_old: 61.69 ± 5.78   Final old_new: 35.77 ± 5.26   Final new_new: 5.00 ± 2.96
Hits@50:
  Final val: 77.13 ± 0.59   Final Test: 63.89 ± 2.69   Final old_old: 69.11 ± 2.73   Final old_new: 43.25 ± 2.78   Final new_new: 10.00 ± 3.45
AUC:
  Final val: 97.58 ± 0.03   Final Test: 96.96 ± 0.11   Final old_old: 97.68 ± 0.09   Final old_new: 94.10 ± 0.29   Final new_new: 89.91 ± 0.39
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 38.86 ± 2.20   Final Test: 26.42 ± 3.88   Final old_old: 28.55 ± 3.98   Final old_new: 17.68 ± 3.81   Final new_new: 10.00 ± 2.03
Hits@20:
  Final val: 46.65 ± 1.94   Final Test: 32.61 ± 1.67   Final old_old: 34.81 ± 1.62   Final old_new: 23.78 ± 2.49   Final new_new: 12.27 ± 2.03
Hits@30:
  Final val: 52.74 ± 2.56   Final Test: 37.12 ± 3.69   Final old_old: 39.57 ± 4.00   Final old_new: 27.36 ± 2.49   Final new_new: 13.18 ± 1.90
Hits@50:
  Final val: 60.55 ± 1.03   Final Test: 46.88 ± 3.11   Final old_old: 49.36 ± 3.09   Final old_new: 37.33 ± 3.44   Final new_new: 16.82 ± 1.24
AUC:
  Final val: 95.36 ± 0.10   Final Test: 95.13 ± 0.18   Final old_old: 95.43 ± 0.16   Final old_new: 93.90 ± 0.26   Final new_new: 92.24 ± 1.01
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.74 ± 5.51   Final Test: 34.84 ± 5.56   Final old_old: 37.86 ± 6.09   Final old_new: 22.67 ± 3.87   Final new_new: 7.27 ± 3.37
Hits@20:
  Final val: 59.70 ± 2.96   Final Test: 44.01 ± 5.94   Final old_old: 47.41 ± 6.26   Final old_new: 30.48 ± 4.81   Final new_new: 10.45 ± 4.71
Hits@30:
  Final val: 65.74 ± 2.56   Final Test: 48.71 ± 4.31   Final old_old: 52.28 ± 4.35   Final old_new: 34.56 ± 4.47   Final new_new: 12.27 ± 4.43
Hits@50:
  Final val: 72.57 ± 1.50   Final Test: 56.90 ± 4.24   Final old_old: 60.60 ± 4.12   Final old_new: 42.29 ± 4.85   Final new_new: 19.09 ± 5.93
AUC:
  Final val: 96.60 ± 0.24   Final Test: 96.18 ± 0.22   Final old_old: 96.71 ± 0.18   Final old_new: 94.10 ± 0.43   Final new_new: 90.89 ± 1.14
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.51 ± 2.69   Final Test: 33.29 ± 5.13   Final old_old: 35.94 ± 5.37   Final old_new: 22.67 ± 4.48   Final new_new: 9.09 ± 4.82
Hits@20:
  Final val: 50.78 ± 2.37   Final Test: 41.10 ± 4.24   Final old_old: 43.89 ± 4.53   Final old_new: 29.92 ± 3.43   Final new_new: 15.00 ± 2.59
Hits@30:
  Final val: 56.97 ± 2.05   Final Test: 45.52 ± 2.04   Final old_old: 48.41 ± 2.30   Final old_new: 34.13 ± 1.09   Final new_new: 15.45 ± 1.90
Hits@50:
  Final val: 65.85 ± 2.19   Final Test: 54.25 ± 2.48   Final old_old: 56.97 ± 2.48   Final old_new: 43.70 ± 2.48   Final new_new: 23.18 ± 5.66
AUC:
  Final val: 95.85 ± 0.15   Final Test: 95.75 ± 0.15   Final old_old: 96.10 ± 0.14   Final old_new: 94.36 ± 0.27   Final new_new: 92.44 ± 1.23
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 55.07 ± 4.43   Final Test: 35.52 ± 3.60   Final old_old: 39.01 ± 3.85   Final old_new: 21.46 ± 3.15   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 65.55 ± 3.46   Final Test: 45.89 ± 5.98   Final old_old: 49.72 ± 6.61   Final old_new: 30.78 ± 3.62   Final new_new: 6.36 ± 4.07
Hits@30:
  Final val: 71.79 ± 2.19   Final Test: 54.81 ± 3.97   Final old_old: 58.94 ± 4.37   Final old_new: 38.64 ± 2.83   Final new_new: 9.09 ± 5.08
Hits@50:
  Final val: 78.44 ± 1.81   Final Test: 62.86 ± 3.59   Final old_old: 66.99 ± 3.69   Final old_new: 46.80 ± 3.71   Final new_new: 15.91 ± 6.63
AUC:
  Final val: 97.72 ± 0.07   Final Test: 97.19 ± 0.05   Final old_old: 97.73 ± 0.08   Final old_new: 95.09 ± 0.14   Final new_new: 91.27 ± 1.40
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.48 ± 4.26   Final Test: 29.30 ± 3.41   Final old_old: 31.95 ± 3.77   Final old_new: 18.59 ± 2.17   Final new_new: 6.36 ± 3.73
Hits@20:
  Final val: 59.27 ± 2.54   Final Test: 43.26 ± 5.01   Final old_old: 46.57 ± 5.47   Final old_new: 30.08 ± 3.46   Final new_new: 10.45 ± 2.59
Hits@30:
  Final val: 66.51 ± 2.09   Final Test: 47.98 ± 4.91   Final old_old: 51.39 ± 5.04   Final old_new: 34.41 ± 4.84   Final new_new: 13.64 ± 5.33
Hits@50:
  Final val: 73.93 ± 0.85   Final Test: 58.54 ± 2.98   Final old_old: 62.11 ± 3.02   Final old_new: 44.51 ± 3.26   Final new_new: 20.00 ± 7.43
AUC:
  Final val: 96.67 ± 0.12   Final Test: 96.08 ± 0.26   Final old_old: 96.65 ± 0.27   Final old_new: 93.80 ± 0.42   Final new_new: 90.10 ± 1.29
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 34.51 ± 4.63   Final Test: 19.16 ± 3.88   Final old_old: 20.83 ± 4.00   Final old_new: 12.22 ± 3.45   Final new_new: 7.73 ± 3.45
Hits@20:
  Final val: 42.64 ± 3.43   Final Test: 27.46 ± 3.86   Final old_old: 29.47 ± 3.93   Final old_new: 19.35 ± 3.70   Final new_new: 9.55 ± 3.37
Hits@30:
  Final val: 47.85 ± 2.57   Final Test: 32.49 ± 2.52   Final old_old: 34.82 ± 2.76   Final old_new: 23.17 ± 1.92   Final new_new: 10.45 ± 3.45
Hits@50:
  Final val: 55.66 ± 2.10   Final Test: 39.99 ± 2.25   Final old_old: 42.22 ± 2.39   Final old_new: 31.13 ± 2.47   Final new_new: 17.73 ± 5.18
AUC:
  Final val: 94.27 ± 0.09   Final Test: 93.93 ± 0.20   Final old_old: 94.23 ± 0.16   Final old_new: 92.77 ± 0.45   Final new_new: 90.72 ± 0.45
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.87 ± 7.12   Final Test: 33.29 ± 1.76   Final old_old: 36.38 ± 1.88   Final old_new: 20.83 ± 1.54   Final new_new: 5.91 ± 2.03
Hits@20:
  Final val: 63.35 ± 3.85   Final Test: 44.58 ± 4.53   Final old_old: 48.09 ± 4.88   Final old_new: 30.58 ± 3.24   Final new_new: 10.91 ± 1.90
Hits@30:
  Final val: 69.74 ± 2.10   Final Test: 53.19 ± 2.82   Final old_old: 56.98 ± 2.71   Final old_new: 38.44 ± 3.43   Final new_new: 10.00 ± 4.43
Hits@50:
  Final val: 75.87 ± 1.21   Final Test: 63.51 ± 0.96   Final old_old: 67.14 ± 1.18   Final old_new: 49.70 ± 1.19   Final new_new: 16.36 ± 1.90
AUC:
  Final val: 97.14 ± 0.10   Final Test: 96.85 ± 0.11   Final old_old: 97.28 ± 0.10   Final old_new: 95.20 ± 0.15   Final new_new: 91.77 ± 0.89
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 36.14 ± 1.66   Final Test: 20.95 ± 2.29   Final old_old: 22.72 ± 2.26   Final old_new: 13.73 ± 2.50   Final new_new: 7.27 ± 2.96
Hits@20:
  Final val: 43.24 ± 1.01   Final Test: 26.87 ± 2.74   Final old_old: 28.92 ± 2.90   Final old_new: 18.56 ± 2.27   Final new_new: 9.55 ± 1.02
Hits@30:
  Final val: 48.33 ± 1.48   Final Test: 34.37 ± 1.77   Final old_old: 36.64 ± 1.81   Final old_new: 25.26 ± 1.80   Final new_new: 13.18 ± 2.49
Hits@50:
  Final val: 56.35 ± 1.21   Final Test: 41.37 ± 1.56   Final old_old: 43.86 ± 1.68   Final old_new: 31.54 ± 1.60   Final new_new: 15.00 ± 2.03
AUC:
  Final val: 94.09 ± 0.07   Final Test: 93.89 ± 0.17   Final old_old: 94.17 ± 0.10   Final old_new: 92.83 ± 0.49   Final new_new: 90.79 ± 1.07
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.68 ± 4.29   Final Test: 23.27 ± 8.14   Final old_old: 25.58 ± 8.78   Final old_new: 13.88 ± 5.60   Final new_new: 4.55 ± 2.27
Hits@20:
  Final val: 54.59 ± 3.04   Final Test: 36.40 ± 9.57   Final old_old: 39.64 ± 10.35   Final old_new: 23.30 ± 6.28   Final new_new: 8.18 ± 6.55
Hits@30:
  Final val: 62.24 ± 2.37   Final Test: 46.57 ± 5.79   Final old_old: 50.45 ± 6.38   Final old_new: 30.91 ± 3.66   Final new_new: 12.73 ± 1.24
Hits@50:
  Final val: 71.71 ± 1.88   Final Test: 55.46 ± 2.67   Final old_old: 59.72 ± 3.19   Final old_new: 38.29 ± 1.10   Final new_new: 17.73 ± 4.93
AUC:
  Final val: 97.00 ± 0.20   Final Test: 96.28 ± 0.19   Final old_old: 97.00 ± 0.23   Final old_new: 93.37 ± 0.23   Final new_new: 89.27 ± 1.61
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.91 ± 7.46   Final Test: 32.88 ± 5.24   Final old_old: 35.66 ± 5.48   Final old_new: 21.66 ± 4.34   Final new_new: 8.18 ± 2.59
Hits@20:
  Final val: 60.19 ± 6.39   Final Test: 43.28 ± 4.15   Final old_old: 46.54 ± 4.29   Final old_new: 30.28 ± 3.95   Final new_new: 11.82 ± 1.90
Hits@30:
  Final val: 67.48 ± 2.85   Final Test: 51.09 ± 4.39   Final old_old: 54.60 ± 4.53   Final old_new: 37.18 ± 4.37   Final new_new: 15.45 ± 4.07
Hits@50:
  Final val: 76.10 ± 0.99   Final Test: 60.34 ± 2.58   Final old_old: 64.20 ± 2.40   Final old_new: 45.16 ± 3.72   Final new_new: 19.09 ± 6.14
AUC:
  Final val: 96.95 ± 0.13   Final Test: 96.28 ± 0.16   Final old_old: 96.92 ± 0.15   Final old_new: 93.77 ± 0.25   Final new_new: 88.66 ± 1.95
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 38.64 ± 1.98   Final Test: 22.34 ± 3.59   Final old_old: 24.26 ± 3.76   Final old_new: 14.69 ± 3.30   Final new_new: 4.09 ± 2.96
Hits@20:
  Final val: 49.54 ± 1.72   Final Test: 30.56 ± 2.74   Final old_old: 32.90 ± 2.75   Final old_new: 21.34 ± 3.22   Final new_new: 5.91 ± 3.45
Hits@30:
  Final val: 55.43 ± 2.06   Final Test: 37.64 ± 1.51   Final old_old: 40.23 ± 1.57   Final old_new: 27.43 ± 1.68   Final new_new: 10.45 ± 3.80
Hits@50:
  Final val: 64.63 ± 1.31   Final Test: 48.04 ± 3.87   Final old_old: 51.00 ± 4.04   Final old_new: 36.25 ± 3.49   Final new_new: 19.55 ± 4.98
AUC:
  Final val: 95.74 ± 0.14   Final Test: 95.08 ± 0.22   Final old_old: 95.56 ± 0.21   Final old_new: 93.21 ± 0.39   Final new_new: 88.94 ± 1.40
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.82 ± 5.70   Final Test: 33.20 ± 5.89   Final old_old: 36.59 ± 6.37   Final old_new: 19.57 ± 4.09   Final new_new: 2.27 ± 1.61
Hits@20:
  Final val: 64.85 ± 2.92   Final Test: 42.22 ± 6.62   Final old_old: 45.99 ± 7.21   Final old_new: 27.28 ± 4.53   Final new_new: 4.09 ± 4.37
Hits@30:
  Final val: 71.36 ± 1.98   Final Test: 50.96 ± 5.46   Final old_old: 54.80 ± 5.76   Final old_new: 35.87 ± 4.63   Final new_new: 10.45 ± 3.45
Hits@50:
  Final val: 78.02 ± 1.63   Final Test: 60.74 ± 3.34   Final old_old: 64.79 ± 3.45   Final old_new: 44.86 ± 3.26   Final new_new: 16.82 ± 4.71
AUC:
  Final val: 97.31 ± 0.15   Final Test: 96.80 ± 0.15   Final old_old: 97.32 ± 0.17   Final old_new: 94.72 ± 0.18   Final new_new: 91.91 ± 0.68
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 37.11 ± 1.57   Final Test: 23.67 ± 3.87   Final old_old: 25.48 ± 4.00   Final old_new: 16.30 ± 3.47   Final new_new: 8.64 ± 2.49
Hits@20:
  Final val: 45.02 ± 1.31   Final Test: 29.30 ± 3.17   Final old_old: 31.28 ± 3.29   Final old_new: 21.39 ± 2.62   Final new_new: 10.45 ± 6.14
Hits@30:
  Final val: 50.65 ± 0.73   Final Test: 35.79 ± 1.39   Final old_old: 38.01 ± 1.36   Final old_new: 27.08 ± 1.82   Final new_new: 11.82 ± 4.37
Hits@50:
  Final val: 58.84 ± 1.62   Final Test: 45.35 ± 1.48   Final old_old: 47.78 ± 1.54   Final old_new: 35.77 ± 1.44   Final new_new: 20.45 ± 2.27
AUC:
  Final val: 95.20 ± 0.08   Final Test: 94.96 ± 0.07   Final old_old: 95.23 ± 0.10   Final old_new: 93.85 ± 0.20   Final new_new: 92.93 ± 0.53
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 36.56 ± 2.01   Final Test: 27.22 ± 1.82   Final old_old: 29.19 ± 1.99   Final old_new: 19.35 ± 1.75   Final new_new: 8.64 ± 2.96
Hits@20:
  Final val: 43.83 ± 1.06   Final Test: 31.34 ± 3.46   Final old_old: 33.34 ± 3.43   Final old_new: 23.38 ± 3.86   Final new_new: 11.82 ± 2.96
Hits@30:
  Final val: 49.69 ± 0.65   Final Test: 35.70 ± 3.66   Final old_old: 37.80 ± 3.57   Final old_new: 27.41 ± 4.56   Final new_new: 14.09 ± 4.37
Hits@50:
  Final val: 57.57 ± 1.43   Final Test: 44.06 ± 3.18   Final old_old: 46.25 ± 2.90   Final old_new: 35.42 ± 4.58   Final new_new: 21.82 ± 9.45
AUC:
  Final val: 95.14 ± 0.10   Final Test: 94.91 ± 0.09   Final old_old: 95.19 ± 0.09   Final old_new: 93.80 ± 0.08   Final new_new: 92.39 ± 0.85
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.11 ± 4.77   Final Test: 34.19 ± 3.65   Final old_old: 38.11 ± 3.65   Final old_new: 18.16 ± 3.90   Final new_new: 2.73 ± 1.90
Hits@20:
  Final val: 59.87 ± 3.12   Final Test: 44.04 ± 2.84   Final old_old: 48.43 ± 3.48   Final old_new: 26.30 ± 0.42   Final new_new: 5.45 ± 2.03
Hits@30:
  Final val: 67.38 ± 2.30   Final Test: 50.04 ± 4.18   Final old_old: 54.70 ± 4.45   Final old_new: 31.36 ± 3.55   Final new_new: 6.82 ± 1.61
Hits@50:
  Final val: 75.96 ± 1.52   Final Test: 62.00 ± 2.74   Final old_old: 66.95 ± 3.01   Final old_new: 42.32 ± 1.71   Final new_new: 12.73 ± 3.45
AUC:
  Final val: 97.60 ± 0.08   Final Test: 96.99 ± 0.11   Final old_old: 97.71 ± 0.14   Final old_new: 94.12 ± 0.10   Final new_new: 89.79 ± 0.95
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 55.37 ± 4.54   Final Test: 35.48 ± 5.65   Final old_old: 39.03 ± 6.09   Final old_new: 21.18 ± 4.02   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 67.23 ± 1.76   Final Test: 45.94 ± 3.71   Final old_old: 49.92 ± 3.87   Final old_new: 30.15 ± 3.41   Final new_new: 5.45 ± 2.03
Hits@30:
  Final val: 72.33 ± 1.76   Final Test: 54.04 ± 5.15   Final old_old: 58.22 ± 5.22   Final old_new: 37.63 ± 5.34   Final new_new: 8.18 ± 4.13
Hits@50:
  Final val: 79.00 ± 0.72   Final Test: 63.33 ± 3.32   Final old_old: 67.57 ± 2.99   Final old_new: 46.88 ± 4.90   Final new_new: 13.64 ± 5.33
AUC:
  Final val: 97.73 ± 0.10   Final Test: 97.21 ± 0.09   Final old_old: 97.74 ± 0.10   Final old_new: 95.13 ± 0.28   Final new_new: 91.28 ± 1.46
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.40 ± 2.55   Final Test: 31.80 ± 5.52   Final old_old: 34.42 ± 5.87   Final old_new: 21.31 ± 4.40   Final new_new: 7.73 ± 5.47
Hits@20:
  Final val: 56.78 ± 2.62   Final Test: 40.58 ± 2.94   Final old_old: 43.57 ± 3.04   Final old_new: 28.56 ± 2.82   Final new_new: 13.18 ± 2.96
Hits@30:
  Final val: 65.34 ± 2.10   Final Test: 44.73 ± 2.99   Final old_old: 47.85 ± 3.18   Final old_new: 32.29 ± 2.87   Final new_new: 14.09 ± 4.37
Hits@50:
  Final val: 73.83 ± 1.43   Final Test: 58.43 ± 1.23   Final old_old: 61.98 ± 1.28   Final old_new: 44.36 ± 1.65   Final new_new: 22.73 ± 4.82
AUC:
  Final val: 96.63 ± 0.14   Final Test: 96.14 ± 0.21   Final old_old: 96.71 ± 0.22   Final old_new: 93.88 ± 0.27   Final new_new: 90.21 ± 1.78
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.43 ± 5.93   Final Test: 30.65 ± 3.27   Final old_old: 33.25 ± 3.36   Final old_new: 20.18 ± 3.03   Final new_new: 7.27 ± 4.07
Hits@20:
  Final val: 58.59 ± 2.53   Final Test: 42.32 ± 3.98   Final old_old: 45.68 ± 4.29   Final old_new: 28.97 ± 2.78   Final new_new: 8.64 ± 4.66
Hits@30:
  Final val: 64.95 ± 1.35   Final Test: 48.54 ± 3.54   Final old_old: 52.28 ± 3.82   Final old_new: 33.60 ± 2.72   Final new_new: 11.82 ± 4.37
Hits@50:
  Final val: 72.11 ± 0.62   Final Test: 58.73 ± 3.27   Final old_old: 62.53 ± 3.26   Final old_new: 43.85 ± 3.50   Final new_new: 16.36 ± 4.07
AUC:
  Final val: 96.49 ± 0.08   Final Test: 96.20 ± 0.20   Final old_old: 96.73 ± 0.18   Final old_new: 94.12 ± 0.29   Final new_new: 90.00 ± 1.45
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.54 ± 4.10   Final Test: 29.16 ± 6.02   Final old_old: 31.96 ± 6.42   Final old_new: 17.86 ± 4.71   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 54.15 ± 2.11   Final Test: 36.52 ± 6.24   Final old_old: 39.59 ± 6.76   Final old_new: 24.16 ± 4.19   Final new_new: 8.64 ± 2.96
Hits@30:
  Final val: 61.38 ± 2.17   Final Test: 44.84 ± 2.63   Final old_old: 48.14 ± 2.94   Final old_new: 31.66 ± 1.33   Final new_new: 12.73 ± 3.45
Hits@50:
  Final val: 69.31 ± 1.49   Final Test: 53.18 ± 2.14   Final old_old: 56.43 ± 2.09   Final old_new: 40.48 ± 2.65   Final new_new: 17.73 ± 3.37
AUC:
  Final val: 96.49 ± 0.05   Final Test: 96.09 ± 0.05   Final old_old: 96.53 ± 0.06   Final old_new: 94.34 ± 0.24   Final new_new: 91.83 ± 1.21
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 31.57 ± 3.11   Final Test: 19.36 ± 2.06   Final old_old: 20.78 ± 2.13   Final old_new: 13.50 ± 2.11   Final new_new: 9.55 ± 5.43
Hits@20:
  Final val: 39.46 ± 2.11   Final Test: 26.63 ± 4.28   Final old_old: 28.37 ± 4.47   Final old_new: 19.45 ± 3.69   Final new_new: 14.09 ± 6.70
Hits@30:
  Final val: 44.91 ± 0.93   Final Test: 30.79 ± 2.26   Final old_old: 32.57 ± 2.31   Final old_new: 23.58 ± 2.41   Final new_new: 15.91 ± 4.25
Hits@50:
  Final val: 52.58 ± 1.20   Final Test: 39.43 ± 2.44   Final old_old: 41.25 ± 2.60   Final old_new: 32.24 ± 1.79   Final new_new: 20.91 ± 6.89
AUC:
  Final val: 94.02 ± 0.12   Final Test: 93.86 ± 0.11   Final old_old: 94.07 ± 0.11   Final old_new: 93.02 ± 0.24   Final new_new: 91.46 ± 0.49
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 33.01 ± 1.55   Final Test: 20.88 ± 3.58   Final old_old: 22.49 ± 3.77   Final old_new: 14.33 ± 2.99   Final new_new: 6.82 ± 3.21
Hits@20:
  Final val: 40.81 ± 1.35   Final Test: 27.84 ± 4.25   Final old_old: 29.45 ± 4.59   Final old_new: 21.44 ± 2.88   Final new_new: 11.36 ± 6.63
Hits@30:
  Final val: 46.31 ± 1.51   Final Test: 33.70 ± 2.09   Final old_old: 35.69 ± 2.16   Final old_new: 25.84 ± 2.19   Final new_new: 13.18 ± 7.43
Hits@50:
  Final val: 53.79 ± 1.39   Final Test: 41.08 ± 2.97   Final old_old: 43.11 ± 3.19   Final old_new: 33.15 ± 2.47   Final new_new: 18.64 ± 4.37
AUC:
  Final val: 94.00 ± 0.12   Final Test: 93.85 ± 0.07   Final old_old: 94.13 ± 0.07   Final old_new: 92.75 ± 0.27   Final new_new: 90.47 ± 1.54
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.85 ± 3.06   Final Test: 31.43 ± 3.28   Final old_old: 34.04 ± 3.39   Final old_new: 20.83 ± 2.90   Final new_new: 10.00 ± 3.45
Hits@20:
  Final val: 59.62 ± 1.57   Final Test: 40.90 ± 1.74   Final old_old: 43.95 ± 1.71   Final old_new: 28.72 ± 2.02   Final new_new: 12.27 ± 3.45
Hits@30:
  Final val: 66.92 ± 1.06   Final Test: 46.13 ± 4.84   Final old_old: 49.37 ± 5.28   Final old_new: 33.38 ± 3.55   Final new_new: 10.91 ± 4.93
Hits@50:
  Final val: 75.37 ± 0.44   Final Test: 58.10 ± 1.76   Final old_old: 61.71 ± 1.72   Final old_new: 43.88 ± 2.03   Final new_new: 19.09 ± 4.71
AUC:
  Final val: 96.94 ± 0.20   Final Test: 96.24 ± 0.24   Final old_old: 96.86 ± 0.23   Final old_new: 93.81 ± 0.43   Final new_new: 88.97 ± 1.33
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.10 ± 4.39   Final Test: 30.76 ± 6.45   Final old_old: 33.88 ± 6.75   Final old_new: 18.19 ± 5.63   Final new_new: 2.73 ± 4.07
Hits@20:
  Final val: 62.89 ± 2.95   Final Test: 43.93 ± 7.11   Final old_old: 47.70 ± 7.25   Final old_new: 28.99 ± 7.02   Final new_new: 5.91 ± 3.45
Hits@30:
  Final val: 69.71 ± 1.99   Final Test: 53.64 ± 6.23   Final old_old: 57.54 ± 6.45   Final old_new: 38.31 ± 5.78   Final new_new: 11.36 ± 2.27
Hits@50:
  Final val: 76.25 ± 1.09   Final Test: 61.31 ± 3.57   Final old_old: 65.26 ± 3.91   Final old_new: 45.82 ± 2.49   Final new_new: 18.64 ± 2.96
AUC:
  Final val: 97.30 ± 0.11   Final Test: 96.82 ± 0.14   Final old_old: 97.32 ± 0.13   Final old_new: 94.86 ± 0.25   Final new_new: 91.63 ± 1.18
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 34.68 ± 1.56   Final Test: 22.08 ± 3.72   Final old_old: 23.87 ± 3.92   Final old_new: 14.81 ± 3.07   Final new_new: 6.82 ± 3.59
Hits@20:
  Final val: 42.07 ± 1.02   Final Test: 30.49 ± 2.83   Final old_old: 32.36 ± 2.87   Final old_new: 23.05 ± 3.05   Final new_new: 12.27 ± 4.13
Hits@30:
  Final val: 47.27 ± 1.42   Final Test: 33.33 ± 2.06   Final old_old: 35.31 ± 2.13   Final old_new: 25.49 ± 2.34   Final new_new: 13.64 ± 4.55
Hits@50:
  Final val: 55.23 ± 1.71   Final Test: 42.31 ± 1.95   Final old_old: 44.25 ± 2.22   Final old_new: 34.79 ± 1.21   Final new_new: 19.09 ± 5.23
AUC:
  Final val: 95.03 ± 0.10   Final Test: 94.86 ± 0.12   Final old_old: 95.13 ± 0.14   Final old_new: 93.78 ± 0.14   Final new_new: 92.05 ± 0.42
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.86 ± 4.90   Final Test: 32.67 ± 3.98   Final old_old: 35.70 ± 4.43   Final old_new: 20.65 ± 2.61   Final new_new: 2.73 ± 3.73
Hits@20:
  Final val: 62.95 ± 0.51   Final Test: 44.71 ± 5.28   Final old_old: 48.36 ± 5.28   Final old_new: 30.35 ± 5.45   Final new_new: 6.36 ± 4.07
Hits@30:
  Final val: 69.42 ± 0.91   Final Test: 54.16 ± 3.67   Final old_old: 58.03 ± 3.69   Final old_new: 39.04 ± 3.72   Final new_new: 10.91 ± 4.37
Hits@50:
  Final val: 76.26 ± 0.79   Final Test: 62.56 ± 2.29   Final old_old: 66.33 ± 2.12   Final old_new: 48.06 ± 3.17   Final new_new: 15.91 ± 2.78
AUC:
  Final val: 97.08 ± 0.10   Final Test: 96.83 ± 0.16   Final old_old: 97.25 ± 0.15   Final old_new: 95.25 ± 0.27   Final new_new: 91.29 ± 0.37
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 34.41 ± 1.57   Final Test: 23.20 ± 2.34   Final old_old: 25.00 ± 2.49   Final old_new: 15.92 ± 1.79   Final new_new: 8.18 ± 2.59
Hits@20:
  Final val: 40.63 ± 1.06   Final Test: 30.08 ± 2.51   Final old_old: 31.95 ± 2.59   Final old_new: 22.54 ± 2.08   Final new_new: 13.64 ± 6.22
Hits@30:
  Final val: 45.83 ± 0.99   Final Test: 37.03 ± 1.07   Final old_old: 38.85 ± 1.02   Final old_new: 29.80 ± 1.73   Final new_new: 18.64 ± 5.43
Hits@50:
  Final val: 54.34 ± 0.51   Final Test: 44.37 ± 1.80   Final old_old: 46.18 ± 1.78   Final old_new: 37.38 ± 2.06   Final new_new: 22.73 ± 3.21
AUC:
  Final val: 94.92 ± 0.12   Final Test: 94.87 ± 0.09   Final old_old: 95.13 ± 0.09   Final old_new: 93.86 ± 0.17   Final new_new: 92.33 ± 0.42
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.16 ± 4.37   Final Test: 28.41 ± 3.23   Final old_old: 31.03 ± 3.39   Final old_new: 17.81 ± 2.62   Final new_new: 5.91 ± 5.47
Hits@20:
  Final val: 57.52 ± 2.10   Final Test: 40.44 ± 2.85   Final old_old: 43.62 ± 3.01   Final old_new: 27.86 ± 2.46   Final new_new: 7.73 ± 4.71
Hits@30:
  Final val: 64.34 ± 1.14   Final Test: 43.71 ± 2.82   Final old_old: 46.99 ± 2.86   Final old_new: 30.65 ± 2.91   Final new_new: 11.82 ± 1.90
Hits@50:
  Final val: 71.93 ± 0.90   Final Test: 57.09 ± 3.22   Final old_old: 60.69 ± 3.14   Final old_new: 42.95 ± 3.80   Final new_new: 18.18 ± 3.59
AUC:
  Final val: 96.53 ± 0.19   Final Test: 95.95 ± 0.15   Final old_old: 96.54 ± 0.16   Final old_new: 93.64 ± 0.41   Final new_new: 89.19 ± 1.58
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 53.87 ± 3.39   Final Test: 35.41 ± 4.04   Final old_old: 38.82 ± 4.33   Final old_new: 21.74 ± 3.13   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 65.53 ± 1.68   Final Test: 45.80 ± 2.71   Final old_old: 49.43 ± 2.96   Final old_new: 31.54 ± 2.02   Final new_new: 6.82 ± 2.27
Hits@30:
  Final val: 71.49 ± 1.30   Final Test: 55.25 ± 3.13   Final old_old: 59.43 ± 3.24   Final old_new: 38.82 ± 2.84   Final new_new: 10.00 ± 5.23
Hits@50:
  Final val: 78.64 ± 1.13   Final Test: 62.69 ± 1.57   Final old_old: 67.04 ± 1.59   Final old_new: 45.69 ± 1.77   Final new_new: 14.09 ± 3.73
AUC:
  Final val: 97.69 ± 0.08   Final Test: 97.12 ± 0.11   Final old_old: 97.67 ± 0.11   Final old_new: 94.95 ± 0.23   Final new_new: 91.40 ± 1.56
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.99 ± 9.11   Final Test: 30.02 ± 6.17   Final old_old: 32.96 ± 6.75   Final old_new: 18.06 ± 4.06   Final new_new: 6.36 ± 2.96
Hits@20:
  Final val: 56.99 ± 6.44   Final Test: 39.91 ± 5.35   Final old_old: 43.26 ± 5.89   Final old_new: 26.50 ± 3.25   Final new_new: 7.73 ± 2.03
Hits@30:
  Final val: 63.92 ± 5.04   Final Test: 46.41 ± 5.42   Final old_old: 50.44 ± 6.01   Final old_new: 30.18 ± 3.33   Final new_new: 10.00 ± 3.45
Hits@50:
  Final val: 72.70 ± 3.61   Final Test: 56.94 ± 5.21   Final old_old: 61.28 ± 5.57   Final old_new: 39.55 ± 4.38   Final new_new: 16.36 ± 4.93
AUC:
  Final val: 97.05 ± 0.18   Final Test: 96.22 ± 0.14   Final old_old: 97.04 ± 0.16   Final old_new: 92.93 ± 0.55   Final new_new: 88.77 ± 0.83
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.38 ± 3.65   Final Test: 23.73 ± 2.94   Final old_old: 25.86 ± 2.97   Final old_new: 15.16 ± 3.12   Final new_new: 4.55 ± 3.59
Hits@20:
  Final val: 50.40 ± 1.88   Final Test: 35.40 ± 4.30   Final old_old: 38.22 ± 4.53   Final old_new: 23.98 ± 3.80   Final new_new: 10.45 ± 3.45
Hits@30:
  Final val: 55.84 ± 1.90   Final Test: 41.22 ± 2.65   Final old_old: 44.23 ± 2.94   Final old_new: 29.27 ± 1.72   Final new_new: 10.91 ± 2.96
Hits@50:
  Final val: 63.94 ± 1.33   Final Test: 49.35 ± 2.39   Final old_old: 52.46 ± 2.68   Final old_new: 37.18 ± 2.21   Final new_new: 15.45 ± 6.31
AUC:
  Final val: 95.90 ± 0.23   Final Test: 95.44 ± 0.28   Final old_old: 95.90 ± 0.31   Final old_new: 93.60 ± 0.24   Final new_new: 90.87 ± 0.49
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.65 ± 6.17   Final Test: 33.05 ± 6.59   Final old_old: 35.65 ± 6.80   Final old_new: 22.64 ± 5.83   Final new_new: 8.64 ± 5.88
Hits@20:
  Final val: 61.07 ± 3.20   Final Test: 42.55 ± 3.45   Final old_old: 45.70 ± 3.48   Final old_new: 29.90 ± 3.43   Final new_new: 13.18 ± 7.08
Hits@30:
  Final val: 67.40 ± 1.84   Final Test: 50.47 ± 4.48   Final old_old: 53.92 ± 4.49   Final old_new: 36.80 ± 4.72   Final new_new: 15.00 ± 5.93
Hits@50:
  Final val: 74.79 ± 0.82   Final Test: 58.70 ± 3.24   Final old_old: 62.25 ± 3.34   Final old_new: 44.69 ± 3.27   Final new_new: 21.82 ± 1.24
AUC:
  Final val: 96.66 ± 0.13   Final Test: 96.19 ± 0.25   Final old_old: 96.71 ± 0.27   Final old_new: 94.11 ± 0.28   Final new_new: 90.73 ± 0.98
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.89 ± 2.60   Final Test: 33.97 ± 1.60   Final old_old: 36.78 ± 1.60   Final old_new: 22.67 ± 2.29   Final new_new: 7.73 ± 3.05
Hits@20:
  Final val: 59.71 ± 1.88   Final Test: 42.68 ± 2.47   Final old_old: 45.96 ± 2.71   Final old_new: 29.60 ± 1.69   Final new_new: 10.45 ± 4.71
Hits@30:
  Final val: 65.32 ± 1.86   Final Test: 48.22 ± 4.13   Final old_old: 51.69 ± 4.36   Final old_new: 34.63 ± 3.58   Final new_new: 9.55 ± 2.49
Hits@50:
  Final val: 72.64 ± 1.29   Final Test: 57.14 ± 2.58   Final old_old: 60.85 ± 2.58   Final old_new: 42.64 ± 3.00   Final new_new: 15.45 ± 4.66
AUC:
  Final val: 96.81 ± 0.07   Final Test: 96.35 ± 0.19   Final old_old: 96.83 ± 0.16   Final old_new: 94.47 ± 0.30   Final new_new: 90.64 ± 0.95
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=0.001, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 40.20 ± 3.74   Final Test: 27.83 ± 7.43   Final old_old: 30.06 ± 7.96   Final old_new: 18.77 ± 5.44   Final new_new: 9.09 ± 4.25
Hits@20:
  Final val: 52.23 ± 3.88   Final Test: 35.24 ± 10.00   Final old_old: 37.91 ± 10.55   Final old_new: 24.56 ± 7.92   Final new_new: 10.00 ± 7.11
Hits@30:
  Final val: 58.88 ± 2.61   Final Test: 43.55 ± 1.98   Final old_old: 46.58 ± 1.92   Final old_new: 31.59 ± 2.55   Final new_new: 12.27 ± 5.70
Hits@50:
  Final val: 67.33 ± 2.14   Final Test: 52.11 ± 2.54   Final old_old: 55.33 ± 2.53   Final old_new: 39.62 ± 3.03   Final new_new: 14.09 ± 7.08
AUC:
  Final val: 96.16 ± 0.34   Final Test: 95.62 ± 0.35   Final old_old: 96.21 ± 0.31   Final old_new: 93.24 ± 0.56   Final new_new: 89.98 ± 1.54
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.84 ± 6.78   Final Test: 32.48 ± 5.29   Final old_old: 35.17 ± 5.66   Final old_new: 21.61 ± 3.94   Final new_new: 9.09 ± 5.33
Hits@20:
  Final val: 61.91 ± 3.36   Final Test: 42.91 ± 2.72   Final old_old: 46.03 ± 2.90   Final old_new: 30.40 ± 2.45   Final new_new: 13.64 ± 7.54
Hits@30:
  Final val: 68.25 ± 1.11   Final Test: 51.91 ± 4.63   Final old_old: 55.46 ± 4.72   Final old_new: 37.86 ± 4.69   Final new_new: 15.00 ± 4.13
Hits@50:
  Final val: 75.54 ± 0.78   Final Test: 62.05 ± 3.82   Final old_old: 65.74 ± 3.84   Final old_new: 47.61 ± 4.24   Final new_new: 21.82 ± 5.93
AUC:
  Final val: 96.93 ± 0.19   Final Test: 96.47 ± 0.26   Final old_old: 97.05 ± 0.25   Final old_new: 94.14 ± 0.52   Final new_new: 91.01 ± 0.99
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.88 ± 1.42   Final Test: 36.77 ± 1.55   Final old_old: 39.97 ± 1.44   Final old_new: 24.06 ± 2.46   Final new_new: 4.55 ± 0.00
Hits@20:
  Final val: 62.69 ± 2.03   Final Test: 43.85 ± 4.34   Final old_old: 47.40 ± 4.52   Final old_new: 29.95 ± 3.69   Final new_new: 5.45 ± 2.59
Hits@30:
  Final val: 68.64 ± 1.19   Final Test: 50.90 ± 2.84   Final old_old: 54.70 ± 2.99   Final old_new: 36.05 ± 2.51   Final new_new: 8.64 ± 2.49
Hits@50:
  Final val: 75.32 ± 1.45   Final Test: 58.74 ± 2.22   Final old_old: 62.53 ± 2.16   Final old_new: 43.93 ± 2.80   Final new_new: 16.36 ± 1.90
AUC:
  Final val: 97.25 ± 0.08   Final Test: 96.88 ± 0.06   Final old_old: 97.38 ± 0.06   Final old_new: 94.90 ± 0.31   Final new_new: 91.32 ± 0.77
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.97 ± 2.22   Final Test: 34.29 ± 4.68   Final old_old: 37.60 ± 4.85   Final old_new: 21.03 ± 4.31   Final new_new: 2.73 ± 2.96
Hits@20:
  Final val: 62.84 ± 1.90   Final Test: 44.40 ± 5.58   Final old_old: 48.12 ± 5.97   Final old_new: 29.57 ± 3.87   Final new_new: 7.73 ± 6.55
Hits@30:
  Final val: 70.43 ± 2.12   Final Test: 53.40 ± 3.25   Final old_old: 57.42 ± 3.44   Final old_new: 37.46 ± 2.85   Final new_new: 12.73 ± 4.13
Hits@50:
  Final val: 76.69 ± 0.82   Final Test: 63.50 ± 0.24   Final old_old: 67.39 ± 0.34   Final old_new: 48.21 ± 0.83   Final new_new: 21.82 ± 2.03
AUC:
  Final val: 97.32 ± 0.11   Final Test: 96.92 ± 0.14   Final old_old: 97.42 ± 0.14   Final old_new: 94.95 ± 0.26   Final new_new: 92.28 ± 0.63
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.51 ± 3.90   Final Test: 30.12 ± 8.70   Final old_old: 33.10 ± 9.34   Final old_new: 17.96 ± 6.22   Final new_new: 6.36 ± 2.96
Hits@20:
  Final val: 56.58 ± 1.16   Final Test: 37.84 ± 6.04   Final old_old: 40.95 ± 6.31   Final old_new: 25.49 ± 5.10   Final new_new: 6.36 ± 2.96
Hits@30:
  Final val: 62.05 ± 0.64   Final Test: 46.03 ± 5.06   Final old_old: 49.52 ± 5.33   Final old_new: 32.34 ± 4.03   Final new_new: 8.18 ± 4.71
Hits@50:
  Final val: 69.00 ± 0.64   Final Test: 55.76 ± 2.39   Final old_old: 59.04 ± 2.38   Final old_new: 43.10 ± 2.73   Final new_new: 16.82 ± 2.59
AUC:
  Final val: 96.47 ± 0.11   Final Test: 96.04 ± 0.13   Final old_old: 96.46 ± 0.12   Final old_new: 94.39 ± 0.18   Final new_new: 91.60 ± 0.83
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.81 ± 4.92   Final Test: 34.15 ± 3.90   Final old_old: 38.44 ± 4.25   Final old_new: 16.55 ± 2.67   Final new_new: 1.82 ± 1.02
Hits@20:
  Final val: 62.16 ± 2.56   Final Test: 46.54 ± 3.68   Final old_old: 51.38 ± 3.85   Final old_new: 27.05 ± 3.42   Final new_new: 3.18 ± 1.24
Hits@30:
  Final val: 68.47 ± 1.79   Final Test: 54.89 ± 1.92   Final old_old: 59.97 ± 1.92   Final old_new: 34.53 ± 2.16   Final new_new: 6.82 ± 2.27
Hits@50:
  Final val: 77.00 ± 1.55   Final Test: 62.18 ± 3.08   Final old_old: 67.46 ± 3.02   Final old_new: 41.21 ± 3.75   Final new_new: 9.09 ± 2.27
AUC:
  Final val: 97.66 ± 0.10   Final Test: 97.02 ± 0.14   Final old_old: 97.79 ± 0.12   Final old_new: 93.88 ± 0.37   Final new_new: 90.06 ± 1.15
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.07 ± 3.34   Final Test: 33.63 ± 5.80   Final old_old: 36.74 ± 6.14   Final old_new: 21.23 ± 4.77   Final new_new: 3.18 ± 1.24
Hits@20:
  Final val: 63.15 ± 2.33   Final Test: 45.51 ± 1.92   Final old_old: 49.16 ± 2.07   Final old_new: 31.16 ± 1.62   Final new_new: 6.36 ± 2.96
Hits@30:
  Final val: 69.86 ± 1.29   Final Test: 54.83 ± 2.56   Final old_old: 58.92 ± 2.64   Final old_new: 38.79 ± 2.54   Final new_new: 10.45 ± 3.45
Hits@50:
  Final val: 76.71 ± 1.03   Final Test: 65.38 ± 1.21   Final old_old: 69.35 ± 1.40   Final old_new: 49.87 ± 2.04   Final new_new: 21.36 ± 4.13
AUC:
  Final val: 97.27 ± 0.05   Final Test: 96.96 ± 0.09   Final old_old: 97.34 ± 0.12   Final old_new: 95.48 ± 0.15   Final new_new: 91.89 ± 0.68
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.13 ± 4.31   Final Test: 32.84 ± 5.68   Final old_old: 36.00 ± 5.99   Final old_new: 20.15 ± 4.63   Final new_new: 3.18 ± 1.24
Hits@20:
  Final val: 64.11 ± 4.27   Final Test: 42.33 ± 2.40   Final old_old: 45.96 ± 2.60   Final old_new: 27.96 ± 1.65   Final new_new: 5.45 ± 2.03
Hits@30:
  Final val: 71.65 ± 2.49   Final Test: 53.23 ± 1.38   Final old_old: 57.31 ± 1.25   Final old_new: 37.08 ± 2.12   Final new_new: 11.36 ± 3.59
Hits@50:
  Final val: 79.67 ± 0.99   Final Test: 63.77 ± 1.69   Final old_old: 67.92 ± 1.83   Final old_new: 47.71 ± 1.97   Final new_new: 15.00 ± 3.45
AUC:
  Final val: 97.69 ± 0.07   Final Test: 97.19 ± 0.09   Final old_old: 97.68 ± 0.08   Final old_new: 95.28 ± 0.19   Final new_new: 92.15 ± 0.38
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.71 ± 2.94   Final Test: 29.85 ± 5.68   Final old_old: 33.10 ± 6.13   Final old_new: 16.55 ± 4.08   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 57.93 ± 1.46   Final Test: 39.05 ± 3.92   Final old_old: 42.77 ± 3.97   Final old_new: 24.03 ± 3.97   Final new_new: 5.91 ± 2.03
Hits@30:
  Final val: 63.83 ± 1.24   Final Test: 47.09 ± 0.51   Final old_old: 51.03 ± 0.49   Final old_new: 31.36 ± 2.23   Final new_new: 9.09 ± 3.21
Hits@50:
  Final val: 71.16 ± 0.94   Final Test: 57.75 ± 2.34   Final old_old: 61.75 ± 2.45   Final old_new: 41.96 ± 1.96   Final new_new: 15.91 ± 4.55
AUC:
  Final val: 96.73 ± 0.09   Final Test: 96.26 ± 0.10   Final old_old: 96.82 ± 0.10   Final old_new: 94.05 ± 0.21   Final new_new: 90.79 ± 0.83
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.03 ± 2.49   Final Test: 31.51 ± 6.01   Final old_old: 34.16 ± 6.36   Final old_new: 20.96 ± 4.72   Final new_new: 5.91 ± 4.13
Hits@20:
  Final val: 57.20 ± 1.11   Final Test: 40.52 ± 4.27   Final old_old: 43.56 ± 4.16   Final old_new: 28.54 ± 4.72   Final new_new: 8.64 ± 7.43
Hits@30:
  Final val: 64.70 ± 0.85   Final Test: 48.35 ± 3.61   Final old_old: 51.71 ± 3.64   Final old_new: 35.16 ± 3.51   Final new_new: 11.82 ± 7.08
Hits@50:
  Final val: 73.15 ± 1.30   Final Test: 58.60 ± 2.16   Final old_old: 62.38 ± 2.18   Final old_new: 43.70 ± 2.96   Final new_new: 18.64 ± 7.08
AUC:
  Final val: 96.73 ± 0.25   Final Test: 96.25 ± 0.25   Final old_old: 96.78 ± 0.24   Final old_new: 94.12 ± 0.27   Final new_new: 91.18 ± 1.90
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 40.38 ± 2.29   Final Test: 28.45 ± 4.02   Final old_old: 30.97 ± 4.18   Final old_new: 18.41 ± 3.60   Final new_new: 3.64 ± 3.45
Hits@20:
  Final val: 50.33 ± 2.12   Final Test: 37.65 ± 3.71   Final old_old: 40.52 ± 3.94   Final old_new: 26.25 ± 3.29   Final new_new: 9.09 ± 2.78
Hits@30:
  Final val: 56.04 ± 1.81   Final Test: 46.36 ± 5.07   Final old_old: 49.32 ± 5.27   Final old_new: 35.01 ± 4.48   Final new_new: 10.00 ± 1.24
Hits@50:
  Final val: 63.81 ± 1.15   Final Test: 50.37 ± 4.04   Final old_old: 53.13 ± 4.31   Final old_new: 39.87 ± 3.26   Final new_new: 15.00 ± 4.13
AUC:
  Final val: 95.92 ± 0.20   Final Test: 95.51 ± 0.29   Final old_old: 95.93 ± 0.32   Final old_new: 93.83 ± 0.33   Final new_new: 90.95 ± 1.00
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.88 ± 4.32   Final Test: 33.52 ± 2.43   Final old_old: 36.40 ± 2.53   Final old_new: 21.89 ± 2.32   Final new_new: 8.18 ± 3.80
Hits@20:
  Final val: 60.54 ± 2.54   Final Test: 41.68 ± 6.28   Final old_old: 44.85 ± 6.64   Final old_new: 29.02 ± 4.92   Final new_new: 11.82 ± 4.93
Hits@30:
  Final val: 67.38 ± 1.77   Final Test: 50.55 ± 2.23   Final old_old: 54.02 ± 2.33   Final old_new: 36.83 ± 1.82   Final new_new: 15.45 ± 3.73
Hits@50:
  Final val: 75.74 ± 1.16   Final Test: 59.65 ± 1.13   Final old_old: 63.43 ± 0.96   Final old_new: 44.58 ± 2.11   Final new_new: 22.73 ± 4.25
AUC:
  Final val: 97.00 ± 0.20   Final Test: 96.50 ± 0.31   Final old_old: 97.10 ± 0.30   Final old_new: 94.15 ± 0.32   Final new_new: 89.65 ± 1.47
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.47 ± 3.26   Final Test: 24.75 ± 7.52   Final old_old: 27.01 ± 8.15   Final old_new: 15.57 ± 4.96   Final new_new: 5.91 ± 3.45
Hits@20:
  Final val: 52.65 ± 2.45   Final Test: 35.21 ± 6.10   Final old_old: 38.07 ± 6.47   Final old_new: 23.65 ± 4.80   Final new_new: 10.00 ± 3.05
Hits@30:
  Final val: 59.87 ± 1.68   Final Test: 42.48 ± 2.69   Final old_old: 45.97 ± 2.98   Final old_new: 28.24 ± 1.54   Final new_new: 14.55 ± 4.71
Hits@50:
  Final val: 69.26 ± 2.45   Final Test: 52.81 ± 2.24   Final old_old: 56.52 ± 2.61   Final old_new: 37.86 ± 1.09   Final new_new: 20.00 ± 3.73
AUC:
  Final val: 96.63 ± 0.27   Final Test: 95.91 ± 0.28   Final old_old: 96.60 ± 0.30   Final old_new: 93.14 ± 0.30   Final new_new: 89.51 ± 1.34
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.08 ± 2.28   Final Test: 26.15 ± 6.74   Final old_old: 28.64 ± 7.12   Final old_new: 16.10 ± 5.47   Final new_new: 4.55 ± 2.27
Hits@20:
  Final val: 54.29 ± 2.94   Final Test: 36.85 ± 4.14   Final old_old: 40.00 ± 4.26   Final old_new: 24.21 ± 3.96   Final new_new: 7.73 ± 4.13
Hits@30:
  Final val: 61.01 ± 1.49   Final Test: 41.05 ± 4.16   Final old_old: 44.24 ± 4.59   Final old_new: 28.41 ± 2.81   Final new_new: 9.09 ± 1.61
Hits@50:
  Final val: 70.47 ± 1.61   Final Test: 53.08 ± 2.71   Final old_old: 56.88 ± 2.68   Final old_new: 38.01 ± 3.18   Final new_new: 14.09 ± 5.43
AUC:
  Final val: 96.39 ± 0.22   Final Test: 95.70 ± 0.31   Final old_old: 96.38 ± 0.34   Final old_new: 92.95 ± 0.42   Final new_new: 89.61 ± 2.30
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.70 ± 5.89   Final Test: 27.09 ± 8.11   Final old_old: 29.17 ± 8.59   Final old_new: 18.92 ± 6.50   Final new_new: 5.00 ± 1.90
Hits@20:
  Final val: 57.64 ± 4.03   Final Test: 41.63 ± 1.49   Final old_old: 44.50 ± 1.55   Final old_new: 30.15 ± 1.79   Final new_new: 14.55 ± 3.45
Hits@30:
  Final val: 64.15 ± 2.54   Final Test: 48.54 ± 2.70   Final old_old: 51.64 ± 2.93   Final old_new: 36.27 ± 2.03   Final new_new: 17.27 ± 3.80
Hits@50:
  Final val: 72.44 ± 1.75   Final Test: 58.15 ± 2.35   Final old_old: 61.40 ± 2.41   Final old_new: 45.39 ± 2.67   Final new_new: 22.73 ± 5.57
AUC:
  Final val: 96.54 ± 0.21   Final Test: 96.07 ± 0.20   Final old_old: 96.62 ± 0.18   Final old_new: 93.95 ± 0.48   Final new_new: 88.95 ± 1.79
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.33 ± 6.16   Final Test: 33.78 ± 6.37   Final old_old: 36.92 ± 6.58   Final old_new: 21.21 ± 5.82   Final new_new: 4.55 ± 2.27
Hits@20:
  Final val: 62.83 ± 3.00   Final Test: 44.44 ± 6.57   Final old_old: 47.85 ± 6.94   Final old_new: 31.08 ± 5.02   Final new_new: 7.27 ± 5.18
Hits@30:
  Final val: 68.76 ± 2.47   Final Test: 54.34 ± 2.88   Final old_old: 58.25 ± 3.03   Final old_new: 39.09 ± 2.33   Final new_new: 10.91 ± 2.96
Hits@50:
  Final val: 76.05 ± 1.16   Final Test: 60.46 ± 2.99   Final old_old: 64.39 ± 3.09   Final old_new: 45.09 ± 2.66   Final new_new: 16.36 ± 5.88
AUC:
  Final val: 97.30 ± 0.07   Final Test: 96.91 ± 0.13   Final old_old: 97.39 ± 0.13   Final old_new: 95.03 ± 0.23   Final new_new: 91.65 ± 1.01
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.08 ± 2.54   Final Test: 29.40 ± 3.52   Final old_old: 31.97 ± 3.81   Final old_new: 18.92 ± 2.61   Final new_new: 8.64 ± 4.93
Hits@20:
  Final val: 52.28 ± 1.76   Final Test: 33.83 ± 3.29   Final old_old: 36.75 ± 3.59   Final old_new: 21.96 ± 1.99   Final new_new: 9.55 ± 4.93
Hits@30:
  Final val: 59.35 ± 1.25   Final Test: 41.26 ± 3.98   Final old_old: 44.27 ± 4.19   Final old_new: 29.27 ± 2.95   Final new_new: 11.82 ± 6.31
Hits@50:
  Final val: 67.79 ± 1.46   Final Test: 50.26 ± 3.59   Final old_old: 53.50 ± 3.67   Final old_new: 37.53 ± 4.03   Final new_new: 15.45 ± 4.93
AUC:
  Final val: 96.21 ± 0.28   Final Test: 95.56 ± 0.31   Final old_old: 96.16 ± 0.29   Final old_new: 93.13 ± 0.48   Final new_new: 90.14 ± 1.08
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.41 ± 1.57   Final Test: 29.27 ± 4.66   Final old_old: 32.13 ± 4.95   Final old_new: 17.63 ± 3.67   Final new_new: 5.00 ± 1.90
Hits@20:
  Final val: 56.78 ± 1.56   Final Test: 41.93 ± 6.22   Final old_old: 45.20 ± 6.62   Final old_new: 28.92 ± 4.89   Final new_new: 9.55 ± 2.49
Hits@30:
  Final val: 62.63 ± 1.05   Final Test: 44.14 ± 4.65   Final old_old: 47.43 ± 4.93   Final old_new: 31.18 ± 3.82   Final new_new: 9.55 ± 1.90
Hits@50:
  Final val: 69.28 ± 1.27   Final Test: 53.80 ± 4.14   Final old_old: 57.17 ± 4.26   Final old_new: 40.71 ± 3.87   Final new_new: 15.45 ± 3.73
AUC:
  Final val: 96.51 ± 0.10   Final Test: 96.01 ± 0.10   Final old_old: 96.46 ± 0.10   Final old_new: 94.21 ± 0.27   Final new_new: 91.80 ± 0.74
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.45 ± 5.43   Final Test: 35.03 ± 3.87   Final old_old: 38.42 ± 4.09   Final old_new: 21.31 ± 3.29   Final new_new: 5.45 ± 2.59
Hits@20:
  Final val: 64.78 ± 3.09   Final Test: 46.30 ± 3.19   Final old_old: 50.06 ± 3.24   Final old_new: 31.46 ± 3.24   Final new_new: 7.27 ± 4.07
Hits@30:
  Final val: 71.80 ± 1.96   Final Test: 54.06 ± 4.17   Final old_old: 58.10 ± 4.31   Final old_new: 38.24 ± 3.72   Final new_new: 10.45 ± 5.47
Hits@50:
  Final val: 78.86 ± 1.18   Final Test: 64.07 ± 1.91   Final old_old: 68.26 ± 1.93   Final old_new: 47.71 ± 2.03   Final new_new: 16.82 ± 3.05
AUC:
  Final val: 97.72 ± 0.08   Final Test: 97.25 ± 0.07   Final old_old: 97.74 ± 0.06   Final old_new: 95.33 ± 0.18   Final new_new: 91.67 ± 0.71
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.34 ± 3.61   Final Test: 27.75 ± 6.78   Final old_old: 30.78 ± 7.45   Final old_new: 15.47 ± 4.32   Final new_new: 1.82 ± 1.02
Hits@20:
  Final val: 57.63 ± 3.12   Final Test: 37.53 ± 4.81   Final old_old: 41.04 ± 5.53   Final old_new: 23.58 ± 2.53   Final new_new: 2.73 ± 1.90
Hits@30:
  Final val: 64.31 ± 2.87   Final Test: 46.47 ± 6.45   Final old_old: 50.13 ± 6.68   Final old_new: 32.12 ± 5.80   Final new_new: 6.82 ± 2.78
Hits@50:
  Final val: 72.09 ± 3.10   Final Test: 57.38 ± 4.15   Final old_old: 61.55 ± 4.49   Final old_new: 41.08 ± 3.18   Final new_new: 10.91 ± 4.37
AUC:
  Final val: 96.96 ± 0.22   Final Test: 96.39 ± 0.24   Final old_old: 97.01 ± 0.27   Final old_new: 93.96 ± 0.52   Final new_new: 89.98 ± 1.98
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.76 ± 3.50   Final Test: 35.03 ± 5.25   Final old_old: 38.04 ± 5.68   Final old_new: 23.10 ± 4.15   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 64.10 ± 1.96   Final Test: 44.92 ± 4.69   Final old_old: 48.12 ± 4.88   Final old_new: 32.42 ± 4.25   Final new_new: 9.09 ± 4.55
Hits@30:
  Final val: 69.97 ± 1.50   Final Test: 51.37 ± 3.14   Final old_old: 54.83 ± 3.28   Final old_new: 37.93 ± 3.19   Final new_new: 11.36 ± 3.94
Hits@50:
  Final val: 77.20 ± 1.26   Final Test: 61.86 ± 2.10   Final old_old: 65.47 ± 2.11   Final old_new: 47.88 ± 2.68   Final new_new: 19.55 ± 3.80
AUC:
  Final val: 97.46 ± 0.06   Final Test: 97.03 ± 0.16   Final old_old: 97.52 ± 0.11   Final old_new: 95.11 ± 0.33   Final new_new: 91.20 ± 1.36
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.76 ± 3.17   Final Test: 34.29 ± 1.50   Final old_old: 37.30 ± 1.58   Final old_new: 22.24 ± 1.32   Final new_new: 6.36 ± 3.37
Hits@20:
  Final val: 61.36 ± 0.60   Final Test: 46.42 ± 3.14   Final old_old: 49.92 ± 3.31   Final old_new: 32.54 ± 2.64   Final new_new: 10.91 ± 2.49
Hits@30:
  Final val: 66.81 ± 0.48   Final Test: 52.53 ± 2.34   Final old_old: 56.28 ± 2.25   Final old_new: 37.78 ± 2.88   Final new_new: 12.27 ± 5.70
Hits@50:
  Final val: 72.98 ± 0.48   Final Test: 60.35 ± 0.93   Final old_old: 63.93 ± 1.14   Final old_new: 46.50 ± 1.17   Final new_new: 17.73 ± 6.31
AUC:
  Final val: 96.65 ± 0.20   Final Test: 96.21 ± 0.26   Final old_old: 96.79 ± 0.24   Final old_new: 93.95 ± 0.48   Final new_new: 89.79 ± 1.50
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.85 ± 1.87   Final Test: 33.56 ± 4.64   Final old_old: 36.70 ± 4.99   Final old_new: 20.88 ± 3.65   Final new_new: 5.45 ± 2.03
Hits@20:
  Final val: 57.15 ± 2.30   Final Test: 38.44 ± 4.11   Final old_old: 41.70 ± 4.34   Final old_new: 25.37 ± 3.82   Final new_new: 8.18 ± 3.45
Hits@30:
  Final val: 63.71 ± 1.50   Final Test: 47.48 ± 3.21   Final old_old: 51.03 ± 3.18   Final old_new: 33.38 ± 3.69   Final new_new: 12.27 ± 3.80
Hits@50:
  Final val: 71.18 ± 1.31   Final Test: 56.47 ± 3.34   Final old_old: 60.23 ± 3.66   Final old_new: 41.76 ± 2.45   Final new_new: 15.45 ± 6.70
AUC:
  Final val: 96.76 ± 0.09   Final Test: 96.19 ± 0.08   Final old_old: 96.74 ± 0.08   Final old_new: 94.00 ± 0.25   Final new_new: 90.70 ± 0.69
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.55 ± 3.57   Final Test: 35.46 ± 3.30   Final old_old: 39.03 ± 3.91   Final old_new: 20.98 ± 1.03   Final new_new: 5.00 ± 1.90
Hits@20:
  Final val: 58.63 ± 3.66   Final Test: 41.70 ± 2.69   Final old_old: 45.67 ± 3.07   Final old_new: 25.69 ± 1.48   Final new_new: 6.36 ± 1.02
Hits@30:
  Final val: 65.55 ± 2.89   Final Test: 50.91 ± 3.85   Final old_old: 55.27 ± 4.25   Final old_new: 33.45 ± 2.51   Final new_new: 9.55 ± 1.90
Hits@50:
  Final val: 74.72 ± 2.33   Final Test: 59.17 ± 3.07   Final old_old: 63.72 ± 3.41   Final old_new: 41.08 ± 2.12   Final new_new: 14.09 ± 3.37
AUC:
  Final val: 97.31 ± 0.19   Final Test: 96.61 ± 0.23   Final old_old: 97.41 ± 0.22   Final old_new: 93.40 ± 0.39   Final new_new: 89.02 ± 1.68
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 37.76 ± 3.05   Final Test: 25.08 ± 5.71   Final old_old: 27.27 ± 6.05   Final old_new: 16.30 ± 4.49   Final new_new: 5.00 ± 1.02
Hits@20:
  Final val: 47.43 ± 2.71   Final Test: 30.79 ± 6.58   Final old_old: 33.22 ± 7.04   Final old_new: 21.08 ± 4.94   Final new_new: 7.27 ± 1.02
Hits@30:
  Final val: 55.02 ± 1.85   Final Test: 39.44 ± 6.26   Final old_old: 42.24 ± 6.48   Final old_new: 28.31 ± 5.49   Final new_new: 11.82 ± 3.73
Hits@50:
  Final val: 64.12 ± 1.08   Final Test: 50.77 ± 2.47   Final old_old: 53.82 ± 2.57   Final old_new: 38.99 ± 2.39   Final new_new: 14.55 ± 2.03
AUC:
  Final val: 95.92 ± 0.17   Final Test: 95.50 ± 0.26   Final old_old: 95.95 ± 0.28   Final old_new: 93.70 ± 0.31   Final new_new: 91.15 ± 0.75
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.73 ± 5.35   Final Test: 31.88 ± 3.52   Final old_old: 34.65 ± 3.80   Final old_new: 20.73 ± 2.71   Final new_new: 6.82 ± 2.78
Hits@20:
  Final val: 63.88 ± 3.35   Final Test: 45.05 ± 3.96   Final old_old: 48.48 ± 4.37   Final old_new: 31.31 ± 2.71   Final new_new: 12.73 ± 3.45
Hits@30:
  Final val: 69.32 ± 1.81   Final Test: 49.21 ± 3.21   Final old_old: 52.82 ± 3.32   Final old_new: 34.71 ± 3.09   Final new_new: 15.91 ± 2.27
Hits@50:
  Final val: 76.51 ± 1.08   Final Test: 59.32 ± 2.52   Final old_old: 63.23 ± 2.65   Final old_new: 43.60 ± 2.17   Final new_new: 24.09 ± 3.05
AUC:
  Final val: 97.06 ± 0.18   Final Test: 96.52 ± 0.23   Final old_old: 97.13 ± 0.18   Final old_new: 94.06 ± 0.51   Final new_new: 90.71 ± 2.36
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.43 ± 2.31   Final Test: 36.57 ± 7.50   Final old_old: 40.04 ± 7.85   Final old_new: 22.70 ± 6.53   Final new_new: 3.18 ± 1.24
Hits@20:
  Final val: 63.66 ± 1.71   Final Test: 46.16 ± 3.40   Final old_old: 50.06 ± 3.53   Final old_new: 30.65 ± 3.10   Final new_new: 7.73 ± 4.98
Hits@30:
  Final val: 70.61 ± 0.88   Final Test: 55.21 ± 2.95   Final old_old: 59.28 ± 3.01   Final old_new: 39.24 ± 3.11   Final new_new: 11.36 ± 4.55
Hits@50:
  Final val: 77.15 ± 1.07   Final Test: 63.85 ± 0.34   Final old_old: 67.95 ± 0.51   Final old_new: 47.86 ± 1.27   Final new_new: 17.73 ± 3.73
AUC:
  Final val: 97.32 ± 0.06   Final Test: 96.94 ± 0.12   Final old_old: 97.40 ± 0.11   Final old_new: 95.14 ± 0.17   Final new_new: 91.85 ± 0.75
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 40.83 ± 1.66   Final Test: 26.72 ± 2.38   Final old_old: 29.33 ± 2.59   Final old_new: 16.15 ± 1.61   Final new_new: 4.55 ± 2.78
Hits@20:
  Final val: 52.60 ± 2.50   Final Test: 34.66 ± 3.93   Final old_old: 37.70 ± 4.21   Final old_new: 22.32 ± 2.89   Final new_new: 10.00 ± 7.11
Hits@30:
  Final val: 59.95 ± 2.33   Final Test: 43.32 ± 3.39   Final old_old: 46.76 ± 3.56   Final old_new: 29.57 ± 2.74   Final new_new: 10.91 ± 6.31
Hits@50:
  Final val: 69.35 ± 2.25   Final Test: 54.36 ± 2.42   Final old_old: 58.16 ± 2.55   Final old_new: 39.47 ± 2.13   Final new_new: 12.73 ± 5.23
AUC:
  Final val: 96.37 ± 0.25   Final Test: 95.69 ± 0.35   Final old_old: 96.35 ± 0.39   Final old_new: 93.00 ± 0.34   Final new_new: 89.59 ± 1.11
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.52 ± 2.93   Final Test: 29.44 ± 5.93   Final old_old: 31.85 ± 6.35   Final old_new: 19.92 ± 4.37   Final new_new: 5.00 ± 2.49
Hits@20:
  Final val: 58.04 ± 3.76   Final Test: 39.88 ± 5.32   Final old_old: 42.85 ± 5.57   Final old_new: 28.16 ± 4.69   Final new_new: 8.64 ± 5.88
Hits@30:
  Final val: 65.34 ± 2.31   Final Test: 50.37 ± 4.53   Final old_old: 53.76 ± 4.62   Final old_new: 37.15 ± 4.53   Final new_new: 11.82 ± 5.18
Hits@50:
  Final val: 73.10 ± 1.16   Final Test: 59.80 ± 2.71   Final old_old: 63.28 ± 2.67   Final old_new: 46.37 ± 3.42   Final new_new: 18.18 ± 3.59
AUC:
  Final val: 96.59 ± 0.17   Final Test: 96.12 ± 0.09   Final old_old: 96.64 ± 0.05   Final old_new: 94.06 ± 0.30   Final new_new: 90.53 ± 1.62
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.58 ± 6.48   Final Test: 27.50 ± 5.58   Final old_old: 30.23 ± 5.87   Final old_new: 16.20 ± 4.59   Final new_new: 8.64 ± 2.49
Hits@20:
  Final val: 51.97 ± 4.96   Final Test: 34.61 ± 5.07   Final old_old: 37.64 ± 5.51   Final old_new: 22.09 ± 3.36   Final new_new: 12.73 ± 2.59
Hits@30:
  Final val: 58.89 ± 4.18   Final Test: 43.74 ± 6.37   Final old_old: 47.23 ± 6.75   Final old_new: 29.57 ± 5.05   Final new_new: 15.00 ± 3.45
Hits@50:
  Final val: 69.07 ± 3.67   Final Test: 52.11 ± 3.17   Final old_old: 55.91 ± 3.36   Final old_new: 36.83 ± 2.77   Final new_new: 17.27 ± 1.24
AUC:
  Final val: 96.52 ± 0.39   Final Test: 95.78 ± 0.43   Final old_old: 96.49 ± 0.44   Final old_new: 92.87 ± 0.50   Final new_new: 89.68 ± 0.92
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.37 ± 2.54   Final Test: 30.84 ± 2.06   Final old_old: 33.33 ± 2.32   Final old_new: 20.83 ± 1.17   Final new_new: 7.73 ± 2.59
Hits@20:
  Final val: 52.03 ± 2.37   Final Test: 37.30 ± 5.68   Final old_old: 40.10 ± 5.80   Final old_new: 26.12 ± 5.54   Final new_new: 10.45 ± 5.23
Hits@30:
  Final val: 59.59 ± 1.81   Final Test: 43.81 ± 3.76   Final old_old: 46.69 ± 3.93   Final old_new: 32.49 ± 3.36   Final new_new: 12.27 ± 7.47
Hits@50:
  Final val: 67.70 ± 0.63   Final Test: 53.74 ± 3.27   Final old_old: 57.07 ± 3.51   Final old_new: 40.63 ± 2.58   Final new_new: 19.09 ± 9.85
AUC:
  Final val: 96.11 ± 0.27   Final Test: 95.57 ± 0.31   Final old_old: 96.19 ± 0.30   Final old_new: 93.07 ± 0.46   Final new_new: 89.93 ± 1.00
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.52 ± 1.31   Final Test: 28.69 ± 3.37   Final old_old: 31.67 ± 3.84   Final old_new: 16.52 ± 1.38   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 57.94 ± 1.37   Final Test: 40.18 ± 3.99   Final old_old: 43.61 ± 4.31   Final old_new: 26.45 ± 2.88   Final new_new: 7.73 ± 3.45
Hits@30:
  Final val: 62.93 ± 0.56   Final Test: 47.11 ± 5.75   Final old_old: 50.58 ± 6.02   Final old_new: 33.50 ± 4.98   Final new_new: 9.55 ± 3.73
Hits@50:
  Final val: 69.28 ± 0.78   Final Test: 55.82 ± 3.12   Final old_old: 59.20 ± 3.32   Final old_new: 42.64 ± 3.00   Final new_new: 17.73 ± 2.49
AUC:
  Final val: 96.46 ± 0.10   Final Test: 96.00 ± 0.11   Final old_old: 96.44 ± 0.12   Final old_new: 94.28 ± 0.20   Final new_new: 91.54 ± 0.72
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 53.09 ± 4.47   Final Test: 33.57 ± 7.58   Final old_old: 36.88 ± 8.13   Final old_new: 20.28 ± 5.68   Final new_new: 3.18 ± 1.24
Hits@20:
  Final val: 65.72 ± 3.08   Final Test: 44.70 ± 5.69   Final old_old: 48.60 ± 6.14   Final old_new: 29.22 ± 4.09   Final new_new: 5.91 ± 1.24
Hits@30:
  Final val: 72.63 ± 2.16   Final Test: 54.28 ± 6.04   Final old_old: 58.46 ± 6.44   Final old_new: 37.76 ± 4.63   Final new_new: 10.45 ± 1.24
Hits@50:
  Final val: 79.58 ± 1.25   Final Test: 63.43 ± 4.92   Final old_old: 67.40 ± 5.07   Final old_new: 47.88 ± 4.46   Final new_new: 19.09 ± 4.98
AUC:
  Final val: 97.72 ± 0.07   Final Test: 97.28 ± 0.12   Final old_old: 97.76 ± 0.11   Final old_new: 95.36 ± 0.20   Final new_new: 92.32 ± 0.69
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.19 ± 5.95   Final Test: 34.20 ± 3.59   Final old_old: 37.04 ± 3.74   Final old_new: 22.82 ± 3.46   Final new_new: 6.82 ± 3.59
Hits@20:
  Final val: 59.45 ± 2.91   Final Test: 43.71 ± 3.70   Final old_old: 47.07 ± 3.96   Final old_new: 30.48 ± 3.06   Final new_new: 7.73 ± 2.59
Hits@30:
  Final val: 65.84 ± 1.97   Final Test: 51.04 ± 3.12   Final old_old: 54.57 ± 3.30   Final old_new: 37.13 ± 2.51   Final new_new: 13.64 ± 2.78
Hits@50:
  Final val: 73.15 ± 1.03   Final Test: 60.60 ± 2.29   Final old_old: 64.22 ± 2.19   Final old_new: 46.45 ± 3.03   Final new_new: 20.45 ± 8.04
AUC:
  Final val: 96.72 ± 0.29   Final Test: 96.39 ± 0.29   Final old_old: 96.89 ± 0.26   Final old_new: 94.45 ± 0.43   Final new_new: 91.02 ± 1.57
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.02 ± 5.71   Final Test: 31.94 ± 4.17   Final old_old: 34.81 ± 4.48   Final old_new: 20.71 ± 3.10   Final new_new: 0.91 ± 1.24
Hits@20:
  Final val: 64.39 ± 3.42   Final Test: 46.84 ± 1.83   Final old_old: 50.26 ± 2.13   Final old_new: 33.40 ± 0.71   Final new_new: 10.45 ± 5.93
Hits@30:
  Final val: 70.90 ± 2.66   Final Test: 54.53 ± 1.40   Final old_old: 58.25 ± 1.51   Final old_new: 40.00 ± 1.64   Final new_new: 13.18 ± 4.37
Hits@50:
  Final val: 77.53 ± 1.11   Final Test: 63.09 ± 2.30   Final old_old: 66.74 ± 2.48   Final old_new: 48.94 ± 1.62   Final new_new: 20.00 ± 4.93
AUC:
  Final val: 97.49 ± 0.07   Final Test: 97.08 ± 0.05   Final old_old: 97.57 ± 0.03   Final old_new: 95.14 ± 0.29   Final new_new: 91.81 ± 0.78
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.04 ± 2.17   Final Test: 29.02 ± 4.65   Final old_old: 32.06 ± 4.97   Final old_new: 16.78 ± 3.51   Final new_new: 1.82 ± 1.90
Hits@20:
  Final val: 58.48 ± 2.76   Final Test: 40.72 ± 4.10   Final old_old: 44.45 ± 4.24   Final old_new: 25.89 ± 3.83   Final new_new: 3.64 ± 2.59
Hits@30:
  Final val: 65.32 ± 3.37   Final Test: 50.12 ± 4.86   Final old_old: 54.03 ± 5.17   Final old_new: 34.81 ± 3.70   Final new_new: 6.36 ± 2.96
Hits@50:
  Final val: 73.12 ± 2.34   Final Test: 58.54 ± 2.55   Final old_old: 62.74 ± 2.79   Final old_new: 42.22 ± 1.73   Final new_new: 9.55 ± 6.31
AUC:
  Final val: 97.03 ± 0.21   Final Test: 96.44 ± 0.11   Final old_old: 97.09 ± 0.13   Final old_new: 93.90 ± 0.44   Final new_new: 89.60 ± 1.98
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 39.65 ± 1.64   Final Test: 23.41 ± 4.71   Final old_old: 25.65 ± 4.77   Final old_new: 14.26 ± 4.66   Final new_new: 5.91 ± 3.45
Hits@20:
  Final val: 48.12 ± 2.00   Final Test: 34.27 ± 2.46   Final old_old: 36.91 ± 2.83   Final old_new: 23.70 ± 0.99   Final new_new: 9.55 ± 3.37
Hits@30:
  Final val: 53.93 ± 1.23   Final Test: 42.18 ± 5.35   Final old_old: 45.09 ± 5.61   Final old_new: 30.65 ± 4.36   Final new_new: 12.27 ± 3.45
Hits@50:
  Final val: 61.75 ± 1.02   Final Test: 51.15 ± 2.60   Final old_old: 54.07 ± 2.49   Final old_new: 39.82 ± 3.57   Final new_new: 16.82 ± 2.59
AUC:
  Final val: 95.51 ± 0.18   Final Test: 95.15 ± 0.11   Final old_old: 95.56 ± 0.08   Final old_new: 93.53 ± 0.27   Final new_new: 90.51 ± 1.66
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.33 ± 2.80   Final Test: 30.63 ± 7.64   Final old_old: 33.83 ± 8.17   Final old_new: 17.61 ± 5.50   Final new_new: 4.55 ± 4.55
Hits@20:
  Final val: 59.08 ± 1.33   Final Test: 39.66 ± 4.74   Final old_old: 43.31 ± 5.00   Final old_new: 24.99 ± 3.76   Final new_new: 5.91 ± 4.13
Hits@30:
  Final val: 64.73 ± 0.79   Final Test: 47.09 ± 3.17   Final old_old: 51.04 ± 3.39   Final old_new: 31.36 ± 2.57   Final new_new: 8.64 ± 4.93
Hits@50:
  Final val: 71.19 ± 0.79   Final Test: 55.50 ± 1.21   Final old_old: 59.38 ± 1.32   Final old_new: 40.38 ± 1.48   Final new_new: 12.27 ± 5.70
AUC:
  Final val: 96.71 ± 0.10   Final Test: 96.26 ± 0.10   Final old_old: 96.81 ± 0.08   Final old_new: 94.06 ± 0.28   Final new_new: 91.00 ± 1.09
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.38 ± 9.61   Final Test: 29.38 ± 2.10   Final old_old: 31.95 ± 2.37   Final old_new: 19.12 ± 1.16   Final new_new: 5.00 ± 2.96
Hits@20:
  Final val: 63.40 ± 5.59   Final Test: 41.53 ± 4.20   Final old_old: 44.82 ± 4.58   Final old_new: 28.46 ± 2.66   Final new_new: 9.09 ± 5.79
Hits@30:
  Final val: 69.68 ± 3.02   Final Test: 52.01 ± 4.45   Final old_old: 55.79 ± 4.58   Final old_new: 36.88 ± 4.19   Final new_new: 16.36 ± 4.07
Hits@50:
  Final val: 76.42 ± 1.38   Final Test: 61.94 ± 1.93   Final old_old: 65.98 ± 1.88   Final old_new: 45.84 ± 2.42   Final new_new: 21.82 ± 3.45
AUC:
  Final val: 97.20 ± 0.06   Final Test: 96.74 ± 0.10   Final old_old: 97.31 ± 0.07   Final old_new: 94.47 ± 0.24   Final new_new: 90.72 ± 1.31
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.71 ± 3.70   Final Test: 30.44 ± 6.15   Final old_old: 33.70 ± 6.64   Final old_new: 17.25 ± 4.55   Final new_new: 2.73 ± 1.90
Hits@20:
  Final val: 61.37 ± 3.24   Final Test: 41.78 ± 5.28   Final old_old: 45.76 ± 5.95   Final old_new: 25.82 ± 2.81   Final new_new: 5.00 ± 1.90
Hits@30:
  Final val: 67.75 ± 3.31   Final Test: 49.82 ± 4.61   Final old_old: 54.35 ± 5.13   Final old_new: 31.69 ± 2.73   Final new_new: 7.27 ± 2.49
Hits@50:
  Final val: 75.46 ± 2.57   Final Test: 59.69 ± 4.76   Final old_old: 64.49 ± 5.18   Final old_new: 40.63 ± 3.10   Final new_new: 11.82 ± 4.37
AUC:
  Final val: 97.32 ± 0.24   Final Test: 96.60 ± 0.22   Final old_old: 97.41 ± 0.27   Final old_new: 93.39 ± 0.24   Final new_new: 88.44 ± 2.02
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.71 ± 4.48   Final Test: 33.03 ± 3.93   Final old_old: 36.05 ± 4.13   Final old_new: 21.01 ± 3.38   Final new_new: 2.73 ± 1.90
Hits@20:
  Final val: 65.19 ± 2.14   Final Test: 46.33 ± 4.06   Final old_old: 50.10 ± 4.22   Final old_new: 31.41 ± 3.57   Final new_new: 7.73 ± 2.59
Hits@30:
  Final val: 70.84 ± 2.38   Final Test: 54.62 ± 2.92   Final old_old: 58.35 ± 3.00   Final old_new: 40.03 ± 2.79   Final new_new: 12.73 ± 4.43
Hits@50:
  Final val: 76.95 ± 2.03   Final Test: 62.83 ± 3.66   Final old_old: 66.66 ± 3.87   Final old_new: 47.86 ± 2.97   Final new_new: 20.45 ± 2.78
AUC:
  Final val: 97.29 ± 0.11   Final Test: 96.90 ± 0.11   Final old_old: 97.34 ± 0.14   Final old_new: 95.14 ± 0.19   Final new_new: 92.57 ± 0.78
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.46 ± 2.26   Final Test: 30.70 ± 7.27   Final old_old: 33.13 ± 7.60   Final old_new: 20.88 ± 6.22   Final new_new: 9.55 ± 2.49
Hits@20:
  Final val: 60.42 ± 3.08   Final Test: 43.97 ± 8.49   Final old_old: 47.01 ± 8.77   Final old_new: 31.96 ± 7.75   Final new_new: 12.27 ± 1.24
Hits@30:
  Final val: 66.53 ± 1.95   Final Test: 52.58 ± 4.18   Final old_old: 55.85 ± 4.18   Final old_new: 39.72 ± 4.68   Final new_new: 17.27 ± 3.05
Hits@50:
  Final val: 73.84 ± 1.52   Final Test: 60.45 ± 2.46   Final old_old: 63.70 ± 2.43   Final old_new: 47.91 ± 3.13   Final new_new: 21.36 ± 4.43
AUC:
  Final val: 96.64 ± 0.17   Final Test: 96.21 ± 0.15   Final old_old: 96.77 ± 0.17   Final old_new: 94.05 ± 0.34   Final new_new: 89.16 ± 1.52
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.92 ± 2.81   Final Test: 27.69 ± 3.97   Final old_old: 30.50 ± 4.10   Final old_new: 16.37 ± 3.62   Final new_new: 2.27 ± 2.78
Hits@20:
  Final val: 55.14 ± 2.17   Final Test: 37.59 ± 4.63   Final old_old: 40.75 ± 4.76   Final old_new: 25.01 ± 4.31   Final new_new: 5.91 ± 2.59
Hits@30:
  Final val: 59.97 ± 1.55   Final Test: 45.37 ± 3.21   Final old_old: 48.58 ± 3.08   Final old_new: 32.82 ± 4.06   Final new_new: 9.55 ± 4.93
Hits@50:
  Final val: 66.73 ± 0.74   Final Test: 55.59 ± 1.84   Final old_old: 58.46 ± 1.94   Final old_new: 44.51 ± 1.90   Final new_new: 20.45 ± 6.82
AUC:
  Final val: 96.17 ± 0.08   Final Test: 95.84 ± 0.05   Final old_old: 96.22 ± 0.03   Final old_new: 94.32 ± 0.34   Final new_new: 91.71 ± 0.79
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.56 ± 3.63   Final Test: 35.70 ± 5.78   Final old_old: 39.06 ± 6.13   Final old_new: 22.37 ± 4.66   Final new_new: 2.27 ± 0.00
Hits@20:
  Final val: 65.71 ± 2.34   Final Test: 48.05 ± 2.97   Final old_old: 51.96 ± 3.08   Final old_new: 32.64 ± 2.67   Final new_new: 7.27 ± 3.37
Hits@30:
  Final val: 71.84 ± 1.61   Final Test: 56.38 ± 2.94   Final old_old: 60.45 ± 3.10   Final old_new: 40.35 ± 2.33   Final new_new: 13.64 ± 4.55
Hits@50:
  Final val: 78.09 ± 0.89   Final Test: 67.30 ± 1.35   Final old_old: 71.27 ± 1.52   Final old_new: 51.61 ± 0.70   Final new_new: 26.36 ± 6.14
AUC:
  Final val: 97.66 ± 0.09   Final Test: 97.40 ± 0.10   Final old_old: 97.81 ± 0.11   Final old_new: 95.77 ± 0.18   Final new_new: 92.86 ± 0.51
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.44 ± 3.96   Final Test: 27.27 ± 4.07   Final old_old: 29.75 ± 4.37   Final old_new: 17.25 ± 3.05   Final new_new: 5.45 ± 3.45
Hits@20:
  Final val: 53.16 ± 2.91   Final Test: 32.34 ± 3.79   Final old_old: 35.19 ± 4.11   Final old_new: 20.88 ± 2.65   Final new_new: 6.82 ± 3.21
Hits@30:
  Final val: 60.82 ± 2.24   Final Test: 43.03 ± 1.76   Final old_old: 46.36 ± 1.99   Final old_new: 29.77 ± 1.34   Final new_new: 10.45 ± 6.93
Hits@50:
  Final val: 69.55 ± 2.47   Final Test: 49.70 ± 5.25   Final old_old: 53.34 ± 5.35   Final old_new: 35.39 ± 5.23   Final new_new: 10.45 ± 4.71
AUC:
  Final val: 96.43 ± 0.31   Final Test: 95.66 ± 0.28   Final old_old: 96.34 ± 0.31   Final old_new: 92.99 ± 0.31   Final new_new: 88.79 ± 1.75
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.02 ± 2.80   Final Test: 24.50 ± 4.92   Final old_old: 26.73 ± 5.33   Final old_new: 15.39 ± 3.62   Final new_new: 6.36 ± 4.07
Hits@20:
  Final val: 50.51 ± 0.97   Final Test: 34.37 ± 4.88   Final old_old: 37.07 ± 5.10   Final old_new: 23.50 ± 4.01   Final new_new: 10.00 ± 4.71
Hits@30:
  Final val: 56.09 ± 1.28   Final Test: 42.90 ± 7.11   Final old_old: 45.88 ± 7.35   Final old_new: 31.13 ± 6.36   Final new_new: 12.27 ± 6.14
Hits@50:
  Final val: 63.89 ± 1.19   Final Test: 47.68 ± 2.43   Final old_old: 50.73 ± 2.63   Final old_new: 35.74 ± 1.87   Final new_new: 14.09 ± 5.18
AUC:
  Final val: 95.79 ± 0.23   Final Test: 95.30 ± 0.22   Final old_old: 95.76 ± 0.25   Final old_new: 93.52 ± 0.38   Final new_new: 90.51 ± 1.30
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 39.75 ± 4.99   Final Test: 25.03 ± 4.39   Final old_old: 27.28 ± 4.69   Final old_new: 15.84 ± 3.29   Final new_new: 6.82 ± 0.00
Hits@20:
  Final val: 51.04 ± 4.27   Final Test: 32.98 ± 6.29   Final old_old: 35.94 ± 6.83   Final old_new: 20.91 ± 4.01   Final new_new: 9.09 ± 4.25
Hits@30:
  Final val: 58.92 ± 3.69   Final Test: 39.41 ± 5.15   Final old_old: 42.64 ± 5.43   Final old_new: 26.30 ± 4.04   Final new_new: 12.73 ± 3.45
Hits@50:
  Final val: 68.57 ± 4.19   Final Test: 50.03 ± 4.18   Final old_old: 53.76 ± 4.53   Final old_new: 35.11 ± 2.81   Final new_new: 14.55 ± 2.59
AUC:
  Final val: 96.54 ± 0.41   Final Test: 95.71 ± 0.36   Final old_old: 96.47 ± 0.37   Final old_new: 92.63 ± 0.39   Final new_new: 89.27 ± 0.88
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 53.42 ± 2.78   Final Test: 33.47 ± 6.92   Final old_old: 36.30 ± 7.29   Final old_new: 22.09 ± 5.56   Final new_new: 7.73 ± 2.59
Hits@20:
  Final val: 61.48 ± 1.57   Final Test: 44.77 ± 2.43   Final old_old: 48.31 ± 2.42   Final old_new: 30.73 ± 2.69   Final new_new: 8.64 ± 1.90
Hits@30:
  Final val: 66.76 ± 1.46   Final Test: 52.06 ± 1.33   Final old_old: 55.84 ± 1.28   Final old_new: 37.18 ± 1.62   Final new_new: 11.36 ± 2.27
Hits@50:
  Final val: 73.49 ± 0.76   Final Test: 58.81 ± 0.76   Final old_old: 62.38 ± 0.72   Final old_new: 45.01 ± 1.98   Final new_new: 16.36 ± 2.96
AUC:
  Final val: 96.86 ± 0.13   Final Test: 96.37 ± 0.12   Final old_old: 96.87 ± 0.13   Final old_new: 94.39 ± 0.31   Final new_new: 91.06 ± 1.35
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.08 ± 5.57   Final Test: 29.60 ± 3.87   Final old_old: 32.22 ± 4.10   Final old_new: 19.04 ± 3.20   Final new_new: 5.91 ± 1.24
Hits@20:
  Final val: 62.64 ± 4.41   Final Test: 44.70 ± 4.42   Final old_old: 48.10 ± 4.70   Final old_new: 30.96 ± 3.54   Final new_new: 15.00 ± 2.59
Hits@30:
  Final val: 69.07 ± 2.97   Final Test: 52.13 ± 2.16   Final old_old: 55.90 ± 2.24   Final old_new: 37.03 ± 2.12   Final new_new: 16.36 ± 2.49
Hits@50:
  Final val: 76.44 ± 1.11   Final Test: 60.03 ± 1.66   Final old_old: 63.94 ± 1.81   Final old_new: 44.26 ± 1.19   Final new_new: 25.45 ± 1.90
AUC:
  Final val: 97.22 ± 0.10   Final Test: 96.68 ± 0.08   Final old_old: 97.22 ± 0.05   Final old_new: 94.51 ± 0.24   Final new_new: 91.73 ± 0.49
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 39.96 ± 2.93   Final Test: 29.48 ± 2.95   Final old_old: 31.79 ± 3.37   Final old_new: 20.18 ± 1.90   Final new_new: 9.09 ± 4.82
Hits@20:
  Final val: 49.78 ± 2.07   Final Test: 38.75 ± 3.91   Final old_old: 41.51 ± 4.18   Final old_new: 27.83 ± 3.26   Final new_new: 9.55 ± 2.49
Hits@30:
  Final val: 54.86 ± 1.28   Final Test: 44.18 ± 3.11   Final old_old: 47.09 ± 3.43   Final old_new: 32.85 ± 2.06   Final new_new: 11.36 ± 5.33
Hits@50:
  Final val: 62.09 ± 1.00   Final Test: 49.86 ± 4.64   Final old_old: 52.75 ± 4.77   Final old_new: 38.69 ± 4.45   Final new_new: 15.91 ± 4.25
AUC:
  Final val: 95.57 ± 0.29   Final Test: 95.23 ± 0.21   Final old_old: 95.65 ± 0.18   Final old_new: 93.54 ± 0.44   Final new_new: 90.83 ± 0.73
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.08 ± 3.91   Final Test: 31.65 ± 7.25   Final old_old: 34.79 ± 8.05   Final old_new: 19.04 ± 4.20   Final new_new: 2.73 ± 1.02
Hits@20:
  Final val: 57.66 ± 1.84   Final Test: 39.43 ± 6.11   Final old_old: 42.84 ± 6.77   Final old_new: 25.94 ± 3.89   Final new_new: 4.09 ± 1.90
Hits@30:
  Final val: 63.92 ± 3.07   Final Test: 48.04 ± 7.25   Final old_old: 51.83 ± 7.97   Final old_new: 33.00 ± 4.53   Final new_new: 10.00 ± 5.47
Hits@50:
  Final val: 72.18 ± 2.85   Final Test: 57.81 ± 5.34   Final old_old: 61.80 ± 5.97   Final old_new: 42.14 ± 3.21   Final new_new: 14.09 ± 4.93
AUC:
  Final val: 96.91 ± 0.26   Final Test: 96.44 ± 0.24   Final old_old: 97.04 ± 0.28   Final old_new: 94.09 ± 0.25   Final new_new: 90.20 ± 1.99
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.14 ± 3.02   Final Test: 32.18 ± 5.85   Final old_old: 34.91 ± 6.23   Final old_new: 21.26 ± 4.26   Final new_new: 6.36 ± 4.37
Hits@20:
  Final val: 63.99 ± 2.16   Final Test: 45.38 ± 4.30   Final old_old: 48.86 ± 4.59   Final old_new: 31.44 ± 3.14   Final new_new: 12.27 ± 6.55
Hits@30:
  Final val: 71.02 ± 1.64   Final Test: 54.51 ± 3.33   Final old_old: 58.21 ± 3.60   Final old_new: 39.82 ± 2.43   Final new_new: 16.82 ± 6.14
Hits@50:
  Final val: 77.71 ± 0.95   Final Test: 62.55 ± 3.77   Final old_old: 66.36 ± 3.95   Final old_new: 47.51 ± 3.30   Final new_new: 22.73 ± 4.25
AUC:
  Final val: 97.50 ± 0.14   Final Test: 97.07 ± 0.18   Final old_old: 97.56 ± 0.17   Final old_new: 95.11 ± 0.23   Final new_new: 91.84 ± 0.93
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.80 ± 1.41   Final Test: 32.07 ± 3.80   Final old_old: 35.39 ± 4.01   Final old_new: 18.59 ± 3.21   Final new_new: 4.55 ± 2.27
Hits@20:
  Final val: 57.56 ± 1.48   Final Test: 41.22 ± 1.85   Final old_old: 44.78 ± 2.02   Final old_new: 27.00 ± 1.43   Final new_new: 7.27 ± 2.96
Hits@30:
  Final val: 62.16 ± 1.16   Final Test: 47.43 ± 1.37   Final old_old: 50.99 ± 1.59   Final old_new: 33.38 ± 1.33   Final new_new: 10.45 ± 3.45
Hits@50:
  Final val: 68.25 ± 1.37   Final Test: 55.71 ± 2.11   Final old_old: 59.01 ± 2.19   Final old_new: 42.85 ± 2.13   Final new_new: 18.18 ± 4.25
AUC:
  Final val: 96.39 ± 0.11   Final Test: 95.96 ± 0.14   Final old_old: 96.40 ± 0.13   Final old_new: 94.31 ± 0.28   Final new_new: 90.39 ± 0.79
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.59 ± 2.33   Final Test: 26.56 ± 5.89   Final old_old: 29.29 ± 6.32   Final old_new: 15.47 ± 4.34   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 57.14 ± 3.12   Final Test: 40.79 ± 4.29   Final old_old: 44.34 ± 4.66   Final old_new: 26.60 ± 3.02   Final new_new: 7.73 ± 3.05
Hits@30:
  Final val: 64.91 ± 1.99   Final Test: 48.97 ± 4.78   Final old_old: 53.01 ± 5.13   Final old_new: 32.92 ± 3.47   Final new_new: 8.18 ± 2.59
Hits@50:
  Final val: 74.06 ± 0.95   Final Test: 60.13 ± 3.17   Final old_old: 64.58 ± 3.43   Final old_new: 42.52 ± 2.46   Final new_new: 14.55 ± 2.03
AUC:
  Final val: 97.20 ± 0.15   Final Test: 96.59 ± 0.20   Final old_old: 97.34 ± 0.21   Final old_new: 93.59 ± 0.21   Final new_new: 89.39 ± 1.64
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.98 ± 5.26   Final Test: 29.96 ± 4.30   Final old_old: 33.18 ± 4.77   Final old_new: 16.88 ± 2.57   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 65.27 ± 2.70   Final Test: 42.97 ± 4.73   Final old_old: 46.94 ± 5.23   Final old_new: 27.05 ± 2.84   Final new_new: 6.36 ± 5.18
Hits@30:
  Final val: 71.33 ± 1.47   Final Test: 54.01 ± 3.53   Final old_old: 57.95 ± 3.74   Final old_new: 38.66 ± 3.03   Final new_new: 8.64 ± 3.73
Hits@50:
  Final val: 77.85 ± 1.31   Final Test: 63.65 ± 1.47   Final old_old: 67.54 ± 1.59   Final old_new: 48.51 ± 1.81   Final new_new: 18.64 ± 1.90
AUC:
  Final val: 97.31 ± 0.06   Final Test: 96.93 ± 0.07   Final old_old: 97.32 ± 0.06   Final old_new: 95.39 ± 0.13   Final new_new: 92.46 ± 0.68
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 40.08 ± 4.59   Final Test: 29.86 ± 3.28   Final old_old: 32.27 ± 3.66   Final old_new: 20.18 ± 1.79   Final new_new: 7.73 ± 2.59
Hits@20:
  Final val: 52.15 ± 1.48   Final Test: 37.14 ± 2.10   Final old_old: 40.02 ± 2.14   Final old_new: 25.62 ± 2.31   Final new_new: 10.00 ± 4.13
Hits@30:
  Final val: 58.92 ± 1.21   Final Test: 43.46 ± 3.75   Final old_old: 46.46 ± 4.02   Final old_new: 31.59 ± 3.27   Final new_new: 12.73 ± 2.59
Hits@50:
  Final val: 66.63 ± 1.59   Final Test: 52.30 ± 4.29   Final old_old: 55.35 ± 4.21   Final old_new: 40.48 ± 4.94   Final new_new: 15.91 ± 3.21
AUC:
  Final val: 96.10 ± 0.23   Final Test: 95.50 ± 0.21   Final old_old: 96.04 ± 0.26   Final old_new: 93.36 ± 0.39   Final new_new: 90.30 ± 1.64
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 54.91 ± 4.36   Final Test: 34.49 ± 4.34   Final old_old: 37.95 ± 4.82   Final old_new: 20.65 ± 2.34   Final new_new: 1.82 ± 1.90
Hits@20:
  Final val: 66.61 ± 3.62   Final Test: 49.84 ± 4.87   Final old_old: 53.97 ± 5.09   Final old_new: 33.53 ± 4.24   Final new_new: 7.27 ± 4.37
Hits@30:
  Final val: 72.80 ± 1.84   Final Test: 57.65 ± 3.53   Final old_old: 61.88 ± 3.42   Final old_new: 41.06 ± 4.17   Final new_new: 12.27 ± 4.43
Hits@50:
  Final val: 79.48 ± 1.78   Final Test: 65.73 ± 2.32   Final old_old: 69.97 ± 2.42   Final old_new: 49.09 ± 2.54   Final new_new: 19.55 ± 3.45
AUC:
  Final val: 97.71 ± 0.06   Final Test: 97.40 ± 0.09   Final old_old: 97.83 ± 0.07   Final old_new: 95.73 ± 0.20   Final new_new: 92.13 ± 0.98
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.45 ± 1.44   Final Test: 27.34 ± 2.07   Final old_old: 30.36 ± 2.14   Final old_new: 15.01 ± 1.88   Final new_new: 3.18 ± 1.24
Hits@20:
  Final val: 54.31 ± 1.35   Final Test: 37.88 ± 3.71   Final old_old: 41.25 ± 3.92   Final old_new: 24.56 ± 2.99   Final new_new: 3.18 ± 1.24
Hits@30:
  Final val: 59.44 ± 1.53   Final Test: 45.91 ± 3.67   Final old_old: 49.23 ± 3.62   Final old_new: 33.00 ± 3.86   Final new_new: 7.73 ± 4.43
Hits@50:
  Final val: 66.38 ± 1.32   Final Test: 53.81 ± 1.67   Final old_old: 56.96 ± 1.58   Final old_new: 41.71 ± 2.39   Final new_new: 15.00 ± 3.45
AUC:
  Final val: 96.11 ± 0.13   Final Test: 95.82 ± 0.07   Final old_old: 96.21 ± 0.07   Final old_new: 94.27 ± 0.14   Final new_new: 91.71 ± 0.55
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 40.81 ± 3.61   Final Test: 26.83 ± 4.84   Final old_old: 29.23 ± 5.02   Final old_new: 17.13 ± 4.35   Final new_new: 6.36 ± 2.49
Hits@20:
  Final val: 49.69 ± 3.52   Final Test: 37.00 ± 4.06   Final old_old: 39.98 ± 4.39   Final old_new: 25.06 ± 3.02   Final new_new: 9.09 ± 2.78
Hits@30:
  Final val: 56.75 ± 1.56   Final Test: 40.83 ± 4.19   Final old_old: 43.80 ± 4.37   Final old_new: 28.82 ± 3.65   Final new_new: 14.55 ± 3.80
Hits@50:
  Final val: 64.27 ± 0.89   Final Test: 50.66 ± 2.43   Final old_old: 53.94 ± 2.28   Final old_new: 37.68 ± 3.59   Final new_new: 17.27 ± 3.05
AUC:
  Final val: 95.80 ± 0.24   Final Test: 95.39 ± 0.17   Final old_old: 95.85 ± 0.17   Final old_new: 93.56 ± 0.24   Final new_new: 90.64 ± 1.32
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.73 ± 3.47   Final Test: 34.69 ± 3.82   Final old_old: 37.36 ± 3.86   Final old_new: 23.98 ± 3.96   Final new_new: 10.00 ± 2.59
Hits@20:
  Final val: 57.89 ± 2.41   Final Test: 44.83 ± 3.66   Final old_old: 48.03 ± 3.67   Final old_new: 32.14 ± 3.70   Final new_new: 12.73 ± 4.71
Hits@30:
  Final val: 65.09 ± 2.20   Final Test: 52.67 ± 2.02   Final old_old: 56.09 ± 2.16   Final old_new: 39.19 ± 1.60   Final new_new: 17.27 ± 4.71
Hits@50:
  Final val: 72.55 ± 1.20   Final Test: 60.03 ± 3.28   Final old_old: 63.37 ± 3.12   Final old_new: 46.93 ± 4.19   Final new_new: 23.18 ± 5.88
AUC:
  Final val: 96.78 ± 0.14   Final Test: 96.41 ± 0.17   Final old_old: 96.88 ± 0.18   Final old_new: 94.56 ± 0.14   Final new_new: 91.86 ± 1.21
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.75 ± 4.77   Final Test: 31.77 ± 5.28   Final old_old: 34.37 ± 5.64   Final old_new: 21.39 ± 4.22   Final new_new: 6.82 ± 2.78
Hits@20:
  Final val: 60.83 ± 3.61   Final Test: 45.34 ± 4.84   Final old_old: 48.43 ± 4.88   Final old_new: 33.07 ± 4.97   Final new_new: 14.55 ± 4.98
Hits@30:
  Final val: 67.52 ± 2.71   Final Test: 51.15 ± 4.92   Final old_old: 54.53 ± 4.83   Final old_new: 37.68 ± 5.38   Final new_new: 18.64 ± 6.10
Hits@50:
  Final val: 74.36 ± 2.03   Final Test: 59.71 ± 3.34   Final old_old: 63.11 ± 3.46   Final old_new: 46.35 ± 3.29   Final new_new: 23.18 ± 3.73
AUC:
  Final val: 96.59 ± 0.13   Final Test: 96.14 ± 0.20   Final old_old: 96.73 ± 0.14   Final old_new: 93.86 ± 0.46   Final new_new: 88.84 ± 1.28
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.20 ± 1.62   Final Test: 28.91 ± 3.65   Final old_old: 31.59 ± 3.97   Final old_new: 18.04 ± 2.48   Final new_new: 5.91 ± 2.03
Hits@20:
  Final val: 53.69 ± 2.22   Final Test: 39.07 ± 1.22   Final old_old: 42.41 ± 1.30   Final old_new: 25.77 ± 1.63   Final new_new: 6.36 ± 1.90
Hits@30:
  Final val: 61.66 ± 2.41   Final Test: 44.73 ± 3.41   Final old_old: 48.18 ± 3.68   Final old_new: 30.91 ± 2.97   Final new_new: 11.82 ± 1.90
Hits@50:
  Final val: 70.65 ± 2.63   Final Test: 55.65 ± 2.55   Final old_old: 59.44 ± 2.54   Final old_new: 40.96 ± 2.86   Final new_new: 11.36 ± 5.57
AUC:
  Final val: 96.66 ± 0.27   Final Test: 96.06 ± 0.32   Final old_old: 96.73 ± 0.32   Final old_new: 93.35 ± 0.42   Final new_new: 89.91 ± 0.70
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.34 ± 4.22   Final Test: 35.50 ± 3.92   Final old_old: 38.34 ± 4.03   Final old_new: 24.13 ± 3.54   Final new_new: 9.55 ± 2.96
Hits@20:
  Final val: 62.20 ± 3.08   Final Test: 46.19 ± 4.06   Final old_old: 49.66 ± 4.22   Final old_new: 32.44 ± 3.97   Final new_new: 10.91 ± 3.37
Hits@30:
  Final val: 69.48 ± 1.86   Final Test: 51.87 ± 2.32   Final old_old: 55.57 ± 2.38   Final old_new: 37.20 ± 2.65   Final new_new: 14.09 ± 2.96
Hits@50:
  Final val: 76.24 ± 1.02   Final Test: 61.47 ± 2.87   Final old_old: 65.49 ± 3.01   Final old_new: 45.64 ± 2.78   Final new_new: 18.64 ± 4.93
AUC:
  Final val: 97.28 ± 0.07   Final Test: 96.73 ± 0.08   Final old_old: 97.25 ± 0.08   Final old_new: 94.63 ± 0.18   Final new_new: 91.52 ± 1.70
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.91 ± 1.87   Final Test: 32.67 ± 2.79   Final old_old: 35.98 ± 2.69   Final old_new: 19.29 ± 3.32   Final new_new: 4.55 ± 2.78
Hits@20:
  Final val: 55.90 ± 1.41   Final Test: 41.00 ± 2.42   Final old_old: 44.55 ± 2.46   Final old_new: 26.88 ± 2.72   Final new_new: 5.91 ± 3.05
Hits@30:
  Final val: 62.37 ± 1.18   Final Test: 45.79 ± 2.01   Final old_old: 49.40 ± 2.07   Final old_new: 31.61 ± 2.00   Final new_new: 6.82 ± 3.59
Hits@50:
  Final val: 69.99 ± 1.15   Final Test: 56.72 ± 2.64   Final old_old: 60.28 ± 2.57   Final old_new: 43.12 ± 2.91   Final new_new: 11.36 ± 4.55
AUC:
  Final val: 96.63 ± 0.10   Final Test: 96.20 ± 0.10   Final old_old: 96.70 ± 0.08   Final old_new: 94.26 ± 0.32   Final new_new: 90.97 ± 0.74
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 40.27 ± 3.41   Final Test: 26.34 ± 3.31   Final old_old: 28.42 ± 3.48   Final old_new: 17.98 ± 3.24   Final new_new: 6.82 ± 2.27
Hits@20:
  Final val: 48.82 ± 3.16   Final Test: 34.30 ± 4.27   Final old_old: 36.78 ± 4.18   Final old_new: 24.48 ± 5.12   Final new_new: 9.55 ± 1.90
Hits@30:
  Final val: 54.30 ± 1.70   Final Test: 39.81 ± 3.69   Final old_old: 42.49 ± 3.46   Final old_new: 29.24 ± 4.81   Final new_new: 11.36 ± 3.94
Hits@50:
  Final val: 62.08 ± 1.13   Final Test: 50.22 ± 3.50   Final old_old: 53.04 ± 3.03   Final old_new: 39.32 ± 5.57   Final new_new: 16.82 ± 6.14
AUC:
  Final val: 95.53 ± 0.20   Final Test: 95.23 ± 0.35   Final old_old: 95.63 ± 0.33   Final old_new: 93.71 ± 0.46   Final new_new: 90.55 ± 0.69
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 53.90 ± 3.55   Final Test: 36.84 ± 6.05   Final old_old: 40.26 ± 6.26   Final old_new: 23.17 ± 5.31   Final new_new: 4.55 ± 2.78
Hits@20:
  Final val: 64.44 ± 1.39   Final Test: 46.25 ± 3.09   Final old_old: 49.96 ± 3.05   Final old_new: 31.51 ± 3.93   Final new_new: 9.09 ± 2.27
Hits@30:
  Final val: 70.16 ± 1.07   Final Test: 54.55 ± 3.54   Final old_old: 58.43 ± 3.69   Final old_new: 39.19 ± 3.19   Final new_new: 15.00 ± 3.45
Hits@50:
  Final val: 76.88 ± 0.86   Final Test: 62.11 ± 1.06   Final old_old: 66.02 ± 1.03   Final old_new: 46.80 ± 1.57   Final new_new: 19.09 ± 2.59
AUC:
  Final val: 97.25 ± 0.10   Final Test: 96.89 ± 0.05   Final old_old: 97.32 ± 0.05   Final old_new: 95.23 ± 0.18   Final new_new: 92.40 ± 0.91
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.71 ± 1.93   Final Test: 30.81 ± 2.59   Final old_old: 33.94 ± 2.75   Final old_new: 18.14 ± 2.37   Final new_new: 4.55 ± 2.27
Hits@20:
  Final val: 54.66 ± 2.24   Final Test: 40.20 ± 2.61   Final old_old: 43.66 ± 2.45   Final old_new: 26.40 ± 3.53   Final new_new: 6.36 ± 2.49
Hits@30:
  Final val: 61.06 ± 2.00   Final Test: 46.61 ± 1.43   Final old_old: 50.10 ± 1.54   Final old_new: 32.90 ± 1.24   Final new_new: 9.09 ± 1.61
Hits@50:
  Final val: 67.73 ± 1.36   Final Test: 54.12 ± 2.36   Final old_old: 57.34 ± 2.35   Final old_new: 41.74 ± 2.99   Final new_new: 15.00 ± 2.59
AUC:
  Final val: 96.33 ± 0.16   Final Test: 95.97 ± 0.16   Final old_old: 96.39 ± 0.14   Final old_new: 94.33 ± 0.29   Final new_new: 91.48 ± 0.72
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.77 ± 4.74   Final Test: 37.74 ± 5.57   Final old_old: 40.85 ± 6.04   Final old_new: 25.39 ± 3.80   Final new_new: 6.36 ± 3.73
Hits@20:
  Final val: 64.54 ± 2.88   Final Test: 47.06 ± 6.21   Final old_old: 50.59 ± 6.55   Final old_new: 33.12 ± 5.01   Final new_new: 10.91 ± 3.73
Hits@30:
  Final val: 71.74 ± 0.93   Final Test: 53.75 ± 3.43   Final old_old: 57.53 ± 3.70   Final old_new: 38.87 ± 2.49   Final new_new: 13.64 ± 5.33
Hits@50:
  Final val: 78.16 ± 0.64   Final Test: 64.72 ± 2.05   Final old_old: 68.49 ± 1.93   Final old_new: 50.13 ± 2.79   Final new_new: 20.45 ± 3.59
AUC:
  Final val: 97.56 ± 0.07   Final Test: 97.11 ± 0.10   Final old_old: 97.60 ± 0.10   Final old_new: 95.24 ± 0.15   Final new_new: 91.46 ± 0.92
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.68 ± 4.62   Final Test: 39.58 ± 4.30   Final old_old: 43.22 ± 4.64   Final old_new: 25.16 ± 3.16   Final new_new: 3.18 ± 3.05
Hits@20:
  Final val: 65.42 ± 1.39   Final Test: 49.51 ± 3.69   Final old_old: 53.61 ± 3.79   Final old_new: 33.35 ± 3.36   Final new_new: 6.36 ± 4.93
Hits@30:
  Final val: 71.75 ± 0.59   Final Test: 58.87 ± 2.07   Final old_old: 63.21 ± 2.23   Final old_new: 41.94 ± 1.67   Final new_new: 9.55 ± 3.37
Hits@50:
  Final val: 79.21 ± 1.32   Final Test: 65.71 ± 1.17   Final old_old: 70.02 ± 1.24   Final old_new: 48.89 ± 2.20   Final new_new: 17.73 ± 5.18
AUC:
  Final val: 97.72 ± 0.06   Final Test: 97.33 ± 0.10   Final old_old: 97.76 ± 0.09   Final old_new: 95.65 ± 0.15   Final new_new: 92.48 ± 0.50
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.60 ± 2.68   Final Test: 30.33 ± 4.70   Final old_old: 32.93 ± 5.04   Final old_new: 19.92 ± 3.35   Final new_new: 6.36 ± 4.07
Hits@20:
  Final val: 57.38 ± 1.31   Final Test: 44.28 ± 1.58   Final old_old: 47.73 ± 1.65   Final old_new: 30.45 ± 1.72   Final new_new: 12.27 ± 2.59
Hits@30:
  Final val: 64.12 ± 2.08   Final Test: 50.03 ± 1.58   Final old_old: 53.59 ± 1.39   Final old_new: 35.92 ± 2.50   Final new_new: 14.09 ± 3.73
Hits@50:
  Final val: 71.84 ± 1.22   Final Test: 58.76 ± 2.26   Final old_old: 62.10 ± 2.33   Final old_new: 45.74 ± 1.84   Final new_new: 20.91 ± 6.10
AUC:
  Final val: 96.75 ± 0.10   Final Test: 96.44 ± 0.08   Final old_old: 96.92 ± 0.07   Final old_new: 94.58 ± 0.24   Final new_new: 91.19 ± 1.77
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.14 ± 2.53   Final Test: 28.12 ± 3.00   Final old_old: 31.14 ± 3.15   Final old_new: 15.82 ± 2.46   Final new_new: 3.18 ± 2.59
Hits@20:
  Final val: 54.84 ± 1.06   Final Test: 37.05 ± 2.15   Final old_old: 40.30 ± 2.37   Final old_new: 24.08 ± 1.34   Final new_new: 5.91 ± 2.59
Hits@30:
  Final val: 59.73 ± 0.95   Final Test: 45.57 ± 1.89   Final old_old: 48.90 ± 2.01   Final old_new: 32.44 ± 1.92   Final new_new: 10.00 ± 4.13
Hits@50:
  Final val: 66.43 ± 0.37   Final Test: 54.01 ± 2.45   Final old_old: 57.19 ± 2.61   Final old_new: 41.79 ± 1.92   Final new_new: 14.55 ± 2.03
AUC:
  Final val: 96.09 ± 0.18   Final Test: 95.76 ± 0.15   Final old_old: 96.15 ± 0.14   Final old_new: 94.26 ± 0.25   Final new_new: 91.60 ± 0.62
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.69 ± 5.15   Final Test: 32.19 ± 4.62   Final old_old: 35.57 ± 5.39   Final old_new: 18.54 ± 2.75   Final new_new: 2.27 ± 1.61
Hits@20:
  Final val: 59.00 ± 2.12   Final Test: 41.69 ± 3.24   Final old_old: 45.63 ± 3.75   Final old_new: 25.94 ± 2.36   Final new_new: 4.55 ± 2.27
Hits@30:
  Final val: 66.65 ± 2.63   Final Test: 47.62 ± 3.98   Final old_old: 51.90 ± 4.50   Final old_new: 30.58 ± 2.48   Final new_new: 6.36 ± 4.37
Hits@50:
  Final val: 74.98 ± 3.08   Final Test: 61.45 ± 3.17   Final old_old: 66.09 ± 3.53   Final old_new: 43.10 ± 2.23   Final new_new: 13.64 ± 3.59
AUC:
  Final val: 97.35 ± 0.24   Final Test: 96.67 ± 0.20   Final old_old: 97.44 ± 0.22   Final old_new: 93.62 ± 0.55   Final new_new: 88.28 ± 1.16
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 39.34 ± 3.99   Final Test: 27.27 ± 2.19   Final old_old: 29.76 ± 2.27   Final old_new: 17.20 ± 2.33   Final new_new: 5.00 ± 4.07
Hits@20:
  Final val: 51.52 ± 1.93   Final Test: 35.03 ± 2.31   Final old_old: 37.92 ± 2.49   Final old_new: 23.58 ± 2.41   Final new_new: 5.45 ± 4.13
Hits@30:
  Final val: 59.03 ± 1.42   Final Test: 39.22 ± 3.05   Final old_old: 42.27 ± 3.28   Final old_new: 27.15 ± 2.60   Final new_new: 8.18 ± 3.80
Hits@50:
  Final val: 67.30 ± 1.59   Final Test: 50.88 ± 3.34   Final old_old: 54.21 ± 3.61   Final old_new: 37.83 ± 2.94   Final new_new: 14.09 ± 5.18
AUC:
  Final val: 96.14 ± 0.29   Final Test: 95.54 ± 0.32   Final old_old: 96.15 ± 0.32   Final old_new: 93.13 ± 0.34   Final new_new: 88.86 ± 1.65
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.68 ± 3.70   Final Test: 26.31 ± 7.43   Final old_old: 28.51 ± 7.99   Final old_new: 17.43 ± 5.37   Final new_new: 6.82 ± 2.27
Hits@20:
  Final val: 50.77 ± 1.72   Final Test: 38.70 ± 3.62   Final old_old: 41.57 ± 3.85   Final old_new: 27.23 ± 3.15   Final new_new: 11.36 ± 2.27
Hits@30:
  Final val: 55.82 ± 0.56   Final Test: 41.86 ± 2.62   Final old_old: 44.94 ± 2.82   Final old_new: 29.55 ± 2.01   Final new_new: 12.27 ± 4.43
Hits@50:
  Final val: 63.58 ± 0.61   Final Test: 50.02 ± 1.84   Final old_old: 53.16 ± 1.83   Final old_new: 37.66 ± 2.07   Final new_new: 16.36 ± 2.96
AUC:
  Final val: 95.74 ± 0.20   Final Test: 95.30 ± 0.14   Final old_old: 95.74 ± 0.17   Final old_new: 93.55 ± 0.13   Final new_new: 90.74 ± 0.92
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.68 ± 6.23   Final Test: 30.76 ± 2.60   Final old_old: 33.24 ± 2.75   Final old_new: 20.88 ± 2.32   Final new_new: 6.36 ± 1.90
Hits@20:
  Final val: 58.79 ± 2.95   Final Test: 42.56 ± 6.22   Final old_old: 45.66 ± 6.67   Final old_new: 30.18 ± 4.66   Final new_new: 12.73 ± 5.23
Hits@30:
  Final val: 66.22 ± 1.87   Final Test: 51.05 ± 4.82   Final old_old: 54.60 ± 5.37   Final old_new: 37.00 ± 2.70   Final new_new: 14.55 ± 4.98
Hits@50:
  Final val: 74.04 ± 1.42   Final Test: 58.43 ± 5.08   Final old_old: 62.11 ± 5.42   Final old_new: 43.80 ± 4.03   Final new_new: 22.27 ± 7.61
AUC:
  Final val: 97.04 ± 0.16   Final Test: 96.53 ± 0.35   Final old_old: 97.08 ± 0.31   Final old_new: 94.41 ± 0.49   Final new_new: 89.67 ± 1.97
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.28 ± 3.60   Final Test: 36.03 ± 3.96   Final old_old: 39.31 ± 4.57   Final old_new: 22.92 ± 1.88   Final new_new: 4.55 ± 2.78
Hits@20:
  Final val: 63.86 ± 1.54   Final Test: 48.27 ± 2.94   Final old_old: 52.11 ± 3.15   Final old_new: 33.17 ± 2.58   Final new_new: 7.73 ± 4.43
Hits@30:
  Final val: 69.21 ± 0.94   Final Test: 55.84 ± 2.66   Final old_old: 59.75 ± 2.99   Final old_new: 40.55 ± 1.65   Final new_new: 12.73 ± 5.23
Hits@50:
  Final val: 76.14 ± 0.37   Final Test: 64.73 ± 2.66   Final old_old: 68.53 ± 2.73   Final old_new: 49.95 ± 2.67   Final new_new: 21.82 ± 5.23
AUC:
  Final val: 97.19 ± 0.11   Final Test: 96.77 ± 0.13   Final old_old: 97.18 ± 0.12   Final old_new: 95.19 ± 0.20   Final new_new: 91.94 ± 0.59
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 35.70 ± 2.95   Final Test: 24.82 ± 1.32   Final old_old: 27.11 ± 1.24   Final old_new: 15.37 ± 1.78   Final new_new: 8.64 ± 2.49
Hits@20:
  Final val: 47.75 ± 2.13   Final Test: 32.73 ± 2.81   Final old_old: 35.41 ± 3.04   Final old_new: 21.86 ± 2.39   Final new_new: 10.45 ± 1.24
Hits@30:
  Final val: 56.56 ± 1.46   Final Test: 38.70 ± 3.98   Final old_old: 41.63 ± 3.94   Final old_new: 26.78 ± 4.34   Final new_new: 14.55 ± 5.23
Hits@50:
  Final val: 67.93 ± 2.11   Final Test: 51.39 ± 2.61   Final old_old: 54.95 ± 2.67   Final old_new: 37.15 ± 3.08   Final new_new: 17.27 ± 4.43
AUC:
  Final val: 96.53 ± 0.22   Final Test: 95.80 ± 0.38   Final old_old: 96.53 ± 0.37   Final old_new: 92.88 ± 0.46   Final new_new: 89.30 ± 0.78
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.98 ± 1.60   Final Test: 33.03 ± 6.32   Final old_old: 35.60 ± 6.41   Final old_new: 22.92 ± 6.13   Final new_new: 5.91 ± 4.71
Hits@20:
  Final val: 62.80 ± 1.30   Final Test: 44.43 ± 2.81   Final old_old: 47.58 ± 2.77   Final old_new: 32.02 ± 3.20   Final new_new: 11.36 ± 4.55
Hits@30:
  Final val: 68.62 ± 1.48   Final Test: 50.19 ± 2.19   Final old_old: 53.63 ± 2.28   Final old_new: 36.75 ± 2.22   Final new_new: 11.82 ± 5.18
Hits@50:
  Final val: 75.12 ± 1.52   Final Test: 58.28 ± 4.21   Final old_old: 61.78 ± 4.03   Final old_new: 44.91 ± 5.30   Final new_new: 13.64 ± 3.59
AUC:
  Final val: 96.68 ± 0.15   Final Test: 96.24 ± 0.13   Final old_old: 96.78 ± 0.10   Final old_new: 94.15 ± 0.45   Final new_new: 89.61 ± 0.80
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.86 ± 3.72   Final Test: 28.31 ± 3.09   Final old_old: 31.33 ± 3.26   Final old_new: 16.05 ± 2.63   Final new_new: 2.73 ± 1.90
Hits@20:
  Final val: 56.14 ± 1.99   Final Test: 41.61 ± 4.66   Final old_old: 45.17 ± 5.10   Final old_new: 27.48 ± 2.94   Final new_new: 6.36 ± 3.73
Hits@30:
  Final val: 63.26 ± 1.26   Final Test: 47.79 ± 4.00   Final old_old: 51.63 ± 4.23   Final old_new: 32.67 ± 3.35   Final new_new: 7.27 ± 1.90
Hits@50:
  Final val: 71.21 ± 1.37   Final Test: 56.90 ± 3.54   Final old_old: 60.85 ± 3.72   Final old_new: 41.51 ± 3.11   Final new_new: 12.27 ± 4.98
AUC:
  Final val: 96.77 ± 0.10   Final Test: 96.26 ± 0.07   Final old_old: 96.82 ± 0.11   Final old_new: 94.04 ± 0.26   Final new_new: 90.05 ± 0.81
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 35.46 ± 2.37   Final Test: 20.98 ± 4.07   Final old_old: 22.69 ± 4.52   Final old_new: 13.90 ± 2.81   Final new_new: 8.64 ± 2.49
Hits@20:
  Final val: 41.84 ± 2.83   Final Test: 29.46 ± 3.43   Final old_old: 31.68 ± 3.91   Final old_new: 20.48 ± 1.72   Final new_new: 10.00 ± 3.45
Hits@30:
  Final val: 47.22 ± 1.65   Final Test: 33.37 ± 2.43   Final old_old: 35.77 ± 2.50   Final old_new: 23.65 ± 2.88   Final new_new: 13.18 ± 4.37
Hits@50:
  Final val: 54.51 ± 1.33   Final Test: 42.28 ± 2.23   Final old_old: 44.71 ± 2.29   Final old_new: 32.67 ± 2.36   Final new_new: 17.73 ± 7.08
AUC:
  Final val: 94.12 ± 0.15   Final Test: 93.85 ± 0.25   Final old_old: 94.09 ± 0.25   Final old_new: 92.93 ± 0.35   Final new_new: 91.33 ± 0.99
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.79 ± 4.17   Final Test: 29.36 ± 3.76   Final old_old: 32.21 ± 3.98   Final old_new: 17.96 ± 3.36   Final new_new: 2.27 ± 2.78
Hits@20:
  Final val: 60.86 ± 2.58   Final Test: 45.24 ± 2.71   Final old_old: 48.90 ± 2.75   Final old_new: 30.83 ± 2.85   Final new_new: 7.27 ± 1.02
Hits@30:
  Final val: 68.82 ± 1.90   Final Test: 54.10 ± 4.16   Final old_old: 58.00 ± 4.44   Final old_new: 38.77 ± 3.13   Final new_new: 12.27 ± 5.47
Hits@50:
  Final val: 78.39 ± 0.89   Final Test: 62.62 ± 3.54   Final old_old: 66.69 ± 3.54   Final old_new: 46.65 ± 4.07   Final new_new: 19.09 ± 7.29
AUC:
  Final val: 97.67 ± 0.08   Final Test: 97.24 ± 0.09   Final old_old: 97.74 ± 0.08   Final old_new: 95.27 ± 0.27   Final new_new: 92.27 ± 1.27
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.72 ± 3.25   Final Test: 28.12 ± 2.67   Final old_old: 30.88 ± 2.85   Final old_new: 16.93 ± 2.45   Final new_new: 4.55 ± 3.21
Hits@20:
  Final val: 55.82 ± 2.78   Final Test: 38.54 ± 3.37   Final old_old: 41.83 ± 3.55   Final old_new: 25.44 ± 3.01   Final new_new: 6.36 ± 1.90
Hits@30:
  Final val: 61.34 ± 1.87   Final Test: 48.46 ± 3.84   Final old_old: 51.92 ± 4.01   Final old_new: 34.96 ± 3.45   Final new_new: 10.00 ± 2.59
Hits@50:
  Final val: 67.95 ± 1.15   Final Test: 54.19 ± 2.09   Final old_old: 57.53 ± 2.26   Final old_new: 41.28 ± 1.83   Final new_new: 14.55 ± 3.80
AUC:
  Final val: 96.38 ± 0.13   Final Test: 95.94 ± 0.15   Final old_old: 96.35 ± 0.12   Final old_new: 94.31 ± 0.33   Final new_new: 91.41 ± 1.27
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.88 ± 4.80   Final Test: 28.19 ± 5.26   Final old_old: 30.44 ± 5.92   Final old_new: 19.29 ± 2.63   Final new_new: 5.00 ± 1.90
Hits@20:
  Final val: 56.23 ± 3.00   Final Test: 39.43 ± 4.51   Final old_old: 42.10 ± 5.00   Final old_new: 29.04 ± 3.05   Final new_new: 8.64 ± 3.73
Hits@30:
  Final val: 61.20 ± 2.21   Final Test: 45.83 ± 3.13   Final old_old: 48.78 ± 3.04   Final old_new: 34.33 ± 3.86   Final new_new: 12.27 ± 4.43
Hits@50:
  Final val: 68.32 ± 1.19   Final Test: 56.30 ± 1.26   Final old_old: 59.43 ± 1.32   Final old_new: 44.28 ± 1.55   Final new_new: 17.73 ± 2.96
AUC:
  Final val: 96.09 ± 0.21   Final Test: 95.68 ± 0.28   Final old_old: 96.13 ± 0.30   Final old_new: 93.93 ± 0.26   Final new_new: 91.10 ± 0.59
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.30 ± 2.77   Final Test: 32.54 ± 6.44   Final old_old: 35.83 ± 6.76   Final old_new: 19.24 ± 5.51   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 57.63 ± 3.69   Final Test: 42.19 ± 5.79   Final old_old: 46.05 ± 6.08   Final old_new: 26.70 ± 4.86   Final new_new: 6.82 ± 3.94
Hits@30:
  Final val: 66.56 ± 2.11   Final Test: 50.63 ± 6.63   Final old_old: 54.67 ± 6.93   Final old_new: 34.66 ± 5.74   Final new_new: 8.18 ± 3.05
Hits@50:
  Final val: 75.34 ± 2.37   Final Test: 61.44 ± 6.09   Final old_old: 65.81 ± 6.34   Final old_new: 44.31 ± 5.20   Final new_new: 13.64 ± 7.54
AUC:
  Final val: 97.37 ± 0.23   Final Test: 96.71 ± 0.17   Final old_old: 97.44 ± 0.21   Final old_new: 93.82 ± 0.35   Final new_new: 89.82 ± 1.62
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 37.40 ± 2.79   Final Test: 26.42 ± 1.74   Final old_old: 28.58 ± 1.89   Final old_new: 17.63 ± 1.28   Final new_new: 8.64 ± 2.96
Hits@20:
  Final val: 45.43 ± 2.64   Final Test: 30.29 ± 1.92   Final old_old: 32.41 ± 1.93   Final old_new: 21.74 ± 2.03   Final new_new: 11.82 ± 3.37
Hits@30:
  Final val: 50.34 ± 2.05   Final Test: 35.84 ± 2.67   Final old_old: 38.18 ± 2.64   Final old_new: 26.47 ± 3.05   Final new_new: 13.64 ± 5.33
Hits@50:
  Final val: 58.77 ± 1.53   Final Test: 44.46 ± 1.00   Final old_old: 46.85 ± 1.28   Final old_new: 35.09 ± 1.36   Final new_new: 19.09 ± 4.13
AUC:
  Final val: 95.03 ± 0.17   Final Test: 94.77 ± 0.15   Final old_old: 95.01 ± 0.12   Final old_new: 93.79 ± 0.35   Final new_new: 92.27 ± 0.90
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 37.78 ± 4.19   Final Test: 26.55 ± 3.58   Final old_old: 29.00 ± 3.79   Final old_new: 16.62 ± 3.10   Final new_new: 5.91 ± 3.45
Hits@20:
  Final val: 50.03 ± 2.77   Final Test: 30.42 ± 2.88   Final old_old: 33.05 ± 2.99   Final old_new: 19.85 ± 2.66   Final new_new: 6.36 ± 2.96
Hits@30:
  Final val: 57.01 ± 3.12   Final Test: 40.05 ± 2.44   Final old_old: 43.35 ± 2.65   Final old_new: 26.78 ± 2.24   Final new_new: 10.00 ± 5.47
Hits@50:
  Final val: 66.91 ± 1.81   Final Test: 49.29 ± 4.47   Final old_old: 52.74 ± 4.67   Final old_new: 35.54 ± 3.88   Final new_new: 15.45 ± 4.93
AUC:
  Final val: 96.16 ± 0.23   Final Test: 95.59 ± 0.25   Final old_old: 96.27 ± 0.22   Final old_new: 92.89 ± 0.60   Final new_new: 89.21 ± 1.36
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 51.06 ± 3.97   Final Test: 33.16 ± 5.42   Final old_old: 36.04 ± 5.53   Final old_new: 21.71 ± 5.14   Final new_new: 4.55 ± 6.63
Hits@20:
  Final val: 65.71 ± 2.22   Final Test: 44.86 ± 2.99   Final old_old: 48.38 ± 2.97   Final old_new: 31.01 ± 3.48   Final new_new: 7.27 ± 4.93
Hits@30:
  Final val: 72.11 ± 1.14   Final Test: 53.44 ± 1.49   Final old_old: 57.20 ± 1.64   Final old_new: 38.66 ± 1.42   Final new_new: 12.73 ± 2.03
Hits@50:
  Final val: 78.29 ± 1.26   Final Test: 63.06 ± 1.64   Final old_old: 66.75 ± 1.29   Final old_new: 48.72 ± 2.95   Final new_new: 20.00 ± 6.70
AUC:
  Final val: 97.57 ± 0.09   Final Test: 97.13 ± 0.08   Final old_old: 97.59 ± 0.07   Final old_new: 95.34 ± 0.13   Final new_new: 92.37 ± 1.10
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.56 ± 3.53   Final Test: 28.96 ± 8.21   Final old_old: 31.37 ± 8.71   Final old_new: 19.37 ± 6.20   Final new_new: 5.00 ± 4.37
Hits@20:
  Final val: 60.86 ± 1.86   Final Test: 40.70 ± 4.71   Final old_old: 43.68 ± 4.89   Final old_new: 28.92 ± 4.21   Final new_new: 9.55 ± 3.73
Hits@30:
  Final val: 66.87 ± 0.94   Final Test: 49.70 ± 5.56   Final old_old: 53.21 ± 5.83   Final old_new: 35.77 ± 4.78   Final new_new: 13.64 ± 4.55
Hits@50:
  Final val: 74.98 ± 0.77   Final Test: 61.72 ± 2.88   Final old_old: 65.37 ± 2.79   Final old_new: 47.36 ± 3.19   Final new_new: 22.73 ± 9.09
AUC:
  Final val: 97.02 ± 0.25   Final Test: 96.52 ± 0.20   Final old_old: 97.05 ± 0.18   Final old_new: 94.46 ± 0.34   Final new_new: 90.07 ± 1.28
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.93 ± 2.90   Final Test: 29.84 ± 4.89   Final old_old: 32.80 ± 5.02   Final old_new: 17.96 ± 4.67   Final new_new: 2.73 ± 1.02
Hits@20:
  Final val: 54.97 ± 3.24   Final Test: 46.06 ± 4.51   Final old_old: 49.22 ± 4.53   Final old_new: 33.93 ± 4.81   Final new_new: 7.27 ± 1.90
Hits@30:
  Final val: 62.75 ± 2.25   Final Test: 49.21 ± 3.51   Final old_old: 52.59 ± 3.65   Final old_new: 35.99 ± 3.62   Final new_new: 11.82 ± 4.93
Hits@50:
  Final val: 71.82 ± 0.94   Final Test: 60.13 ± 2.74   Final old_old: 63.58 ± 2.73   Final old_new: 46.85 ± 2.64   Final new_new: 17.73 ± 7.08
AUC:
  Final val: 96.70 ± 0.08   Final Test: 96.36 ± 0.09   Final old_old: 96.78 ± 0.11   Final old_new: 94.66 ± 0.20   Final new_new: 92.26 ± 0.50
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 35.57 ± 0.83   Final Test: 23.32 ± 3.26   Final old_old: 25.33 ± 3.41   Final old_new: 15.09 ± 2.94   Final new_new: 7.73 ± 3.80
Hits@20:
  Final val: 42.89 ± 1.26   Final Test: 29.54 ± 2.65   Final old_old: 31.81 ± 2.75   Final old_new: 20.28 ± 2.44   Final new_new: 10.91 ± 3.73
Hits@30:
  Final val: 48.25 ± 1.30   Final Test: 34.28 ± 2.28   Final old_old: 36.81 ± 2.57   Final old_new: 24.01 ± 1.63   Final new_new: 13.18 ± 4.37
Hits@50:
  Final val: 55.45 ± 0.87   Final Test: 42.97 ± 1.55   Final old_old: 45.57 ± 1.71   Final old_new: 32.75 ± 2.60   Final new_new: 15.45 ± 4.37
AUC:
  Final val: 93.92 ± 0.13   Final Test: 93.82 ± 0.15   Final old_old: 94.08 ± 0.17   Final old_new: 92.80 ± 0.15   Final new_new: 91.10 ± 0.92
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.90 ± 4.48   Final Test: 31.03 ± 3.99   Final old_old: 33.92 ± 4.02   Final old_new: 19.52 ± 4.20   Final new_new: 2.73 ± 1.90
Hits@20:
  Final val: 59.81 ± 1.90   Final Test: 44.48 ± 3.10   Final old_old: 48.07 ± 3.06   Final old_new: 30.25 ± 3.63   Final new_new: 8.64 ± 4.66
Hits@30:
  Final val: 68.00 ± 2.03   Final Test: 54.85 ± 1.46   Final old_old: 58.71 ± 1.44   Final old_new: 39.77 ± 2.76   Final new_new: 12.27 ± 5.23
Hits@50:
  Final val: 77.32 ± 1.09   Final Test: 62.42 ± 1.32   Final old_old: 66.28 ± 1.42   Final old_new: 47.25 ± 1.76   Final new_new: 20.91 ± 6.31
AUC:
  Final val: 97.63 ± 0.01   Final Test: 97.23 ± 0.07   Final old_old: 97.74 ± 0.08   Final old_new: 95.24 ± 0.18   Final new_new: 91.93 ± 1.55
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 32.75 ± 2.43   Final Test: 17.95 ± 2.76   Final old_old: 19.34 ± 3.01   Final old_new: 12.27 ± 2.20   Final new_new: 7.27 ± 1.90
Hits@20:
  Final val: 39.63 ± 2.03   Final Test: 27.36 ± 1.38   Final old_old: 29.02 ± 1.21   Final old_new: 20.78 ± 2.18   Final new_new: 10.91 ± 5.88
Hits@30:
  Final val: 45.48 ± 1.59   Final Test: 33.09 ± 3.18   Final old_old: 35.01 ± 3.46   Final old_new: 25.42 ± 2.22   Final new_new: 14.09 ± 3.37
Hits@50:
  Final val: 53.12 ± 0.99   Final Test: 39.48 ± 1.89   Final old_old: 41.53 ± 1.99   Final old_new: 31.44 ± 2.00   Final new_new: 17.27 ± 5.47
AUC:
  Final val: 94.02 ± 0.12   Final Test: 93.79 ± 0.25   Final old_old: 94.01 ± 0.22   Final old_new: 92.93 ± 0.41   Final new_new: 91.16 ± 1.14
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.05 ± 4.23   Final Test: 28.63 ± 6.70   Final old_old: 30.65 ± 7.18   Final old_new: 20.58 ± 5.15   Final new_new: 8.64 ± 2.49
Hits@20:
  Final val: 54.57 ± 2.97   Final Test: 40.19 ± 6.37   Final old_old: 42.78 ± 6.56   Final old_new: 29.87 ± 6.18   Final new_new: 15.00 ± 2.59
Hits@30:
  Final val: 60.81 ± 2.03   Final Test: 47.12 ± 2.99   Final old_old: 49.77 ± 3.19   Final old_new: 36.80 ± 2.31   Final new_new: 16.82 ± 4.71
Hits@50:
  Final val: 68.48 ± 1.42   Final Test: 58.42 ± 1.85   Final old_old: 61.30 ± 1.88   Final old_new: 47.33 ± 1.98   Final new_new: 23.18 ± 1.90
AUC:
  Final val: 96.19 ± 0.22   Final Test: 95.78 ± 0.18   Final old_old: 96.23 ± 0.18   Final old_new: 93.98 ± 0.21   Final new_new: 91.32 ± 0.84
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.04 ± 4.63   Final Test: 32.12 ± 3.07   Final old_old: 34.59 ± 3.38   Final old_new: 22.29 ± 2.05   Final new_new: 8.18 ± 3.45
Hits@20:
  Final val: 59.03 ± 3.95   Final Test: 42.97 ± 7.59   Final old_old: 45.81 ± 8.05   Final old_new: 31.74 ± 6.05   Final new_new: 13.18 ± 4.37
Hits@30:
  Final val: 66.50 ± 2.97   Final Test: 51.28 ± 4.46   Final old_old: 54.53 ± 4.48   Final old_new: 38.49 ± 4.54   Final new_new: 17.27 ± 3.45
Hits@50:
  Final val: 73.92 ± 1.93   Final Test: 59.63 ± 3.83   Final old_old: 63.04 ± 3.84   Final old_new: 46.37 ± 3.89   Final new_new: 20.45 ± 4.82
AUC:
  Final val: 96.67 ± 0.15   Final Test: 96.20 ± 0.20   Final old_old: 96.74 ± 0.18   Final old_new: 94.11 ± 0.32   Final new_new: 90.11 ± 1.37
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.36 ± 3.88   Final Test: 25.54 ± 4.87   Final old_old: 27.94 ± 5.09   Final old_new: 15.84 ± 4.78   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 52.68 ± 2.39   Final Test: 35.66 ± 2.16   Final old_old: 38.63 ± 2.02   Final old_new: 23.63 ± 3.29   Final new_new: 10.45 ± 2.59
Hits@30:
  Final val: 59.97 ± 1.45   Final Test: 44.45 ± 2.47   Final old_old: 47.84 ± 2.44   Final old_new: 30.86 ± 3.20   Final new_new: 13.18 ± 4.93
Hits@50:
  Final val: 69.63 ± 1.12   Final Test: 53.47 ± 1.97   Final old_old: 57.11 ± 2.12   Final old_new: 39.04 ± 1.86   Final new_new: 16.82 ± 6.35
AUC:
  Final val: 96.57 ± 0.31   Final Test: 95.79 ± 0.37   Final old_old: 96.54 ± 0.30   Final old_new: 92.77 ± 0.80   Final new_new: 89.44 ± 1.25
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.62 ± 2.77   Final Test: 29.71 ± 3.48   Final old_old: 32.66 ± 3.59   Final old_new: 17.88 ± 3.41   Final new_new: 2.73 ± 1.02
Hits@20:
  Final val: 57.52 ± 2.26   Final Test: 42.81 ± 1.76   Final old_old: 46.37 ± 1.70   Final old_new: 28.69 ± 2.46   Final new_new: 6.36 ± 2.49
Hits@30:
  Final val: 64.68 ± 1.05   Final Test: 49.06 ± 3.32   Final old_old: 52.72 ± 3.33   Final old_new: 34.61 ± 3.44   Final new_new: 10.91 ± 6.89
Hits@50:
  Final val: 73.04 ± 0.98   Final Test: 58.45 ± 3.08   Final old_old: 62.33 ± 3.16   Final old_new: 43.25 ± 2.79   Final new_new: 15.91 ± 7.01
AUC:
  Final val: 96.95 ± 0.11   Final Test: 96.42 ± 0.14   Final old_old: 97.03 ± 0.14   Final old_new: 94.03 ± 0.29   Final new_new: 89.83 ± 1.35
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 35.55 ± 2.41   Final Test: 24.27 ± 1.60   Final old_old: 26.17 ± 1.91   Final old_new: 16.50 ± 1.46   Final new_new: 8.64 ± 3.37
Hits@20:
  Final val: 44.07 ± 1.66   Final Test: 29.57 ± 1.95   Final old_old: 31.63 ± 1.92   Final old_new: 21.23 ± 2.80   Final new_new: 12.27 ± 2.59
Hits@30:
  Final val: 49.62 ± 1.27   Final Test: 36.51 ± 3.52   Final old_old: 38.67 ± 3.44   Final old_new: 27.86 ± 4.28   Final new_new: 16.36 ± 5.66
Hits@50:
  Final val: 57.87 ± 0.87   Final Test: 46.46 ± 1.27   Final old_old: 48.78 ± 1.22   Final old_new: 37.36 ± 1.99   Final new_new: 21.36 ± 3.05
AUC:
  Final val: 94.89 ± 0.08   Final Test: 94.64 ± 0.13   Final old_old: 94.87 ± 0.11   Final old_new: 93.73 ± 0.28   Final new_new: 92.04 ± 0.90
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 35.48 ± 2.00   Final Test: 24.29 ± 2.58   Final old_old: 26.20 ± 2.73   Final old_new: 16.57 ± 2.23   Final new_new: 7.27 ± 3.37
Hits@20:
  Final val: 42.24 ± 0.61   Final Test: 29.66 ± 2.36   Final old_old: 31.65 ± 2.59   Final old_new: 21.76 ± 2.41   Final new_new: 10.00 ± 4.43
Hits@30:
  Final val: 47.50 ± 1.57   Final Test: 34.88 ± 2.55   Final old_old: 36.89 ± 2.43   Final old_new: 26.90 ± 3.73   Final new_new: 14.55 ± 4.71
Hits@50:
  Final val: 54.87 ± 1.39   Final Test: 44.54 ± 1.84   Final old_old: 46.44 ± 1.70   Final old_new: 37.23 ± 3.06   Final new_new: 20.91 ± 5.66
AUC:
  Final val: 94.76 ± 0.11   Final Test: 94.55 ± 0.19   Final old_old: 94.80 ± 0.18   Final old_new: 93.58 ± 0.28   Final new_new: 92.29 ± 0.54
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.05 ± 4.27   Final Test: 33.74 ± 6.77   Final old_old: 36.82 ± 7.08   Final old_new: 21.39 ± 5.71   Final new_new: 5.45 ± 3.80
Hits@20:
  Final val: 55.32 ± 4.15   Final Test: 44.20 ± 4.37   Final old_old: 47.47 ± 4.31   Final old_new: 31.41 ± 4.85   Final new_new: 8.18 ± 6.55
Hits@30:
  Final val: 63.05 ± 2.28   Final Test: 50.06 ± 2.22   Final old_old: 53.51 ± 2.09   Final old_new: 36.57 ± 3.63   Final new_new: 11.36 ± 6.63
Hits@50:
  Final val: 71.38 ± 1.23   Final Test: 58.41 ± 2.84   Final old_old: 62.04 ± 2.51   Final old_new: 44.28 ± 4.43   Final new_new: 16.82 ± 9.72
AUC:
  Final val: 96.67 ± 0.13   Final Test: 96.35 ± 0.14   Final old_old: 96.76 ± 0.15   Final old_new: 94.68 ± 0.16   Final new_new: 92.70 ± 1.08
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.60 ± 6.28   Final Test: 34.11 ± 1.75   Final old_old: 37.12 ± 1.91   Final old_new: 21.99 ± 1.29   Final new_new: 7.27 ± 5.43
Hits@20:
  Final val: 57.79 ± 5.12   Final Test: 45.58 ± 4.59   Final old_old: 48.87 ± 4.81   Final old_new: 32.57 ± 3.88   Final new_new: 11.82 ± 4.37
Hits@30:
  Final val: 65.31 ± 3.83   Final Test: 49.75 ± 6.54   Final old_old: 53.41 ± 6.79   Final old_new: 35.14 ± 5.73   Final new_new: 14.55 ± 2.03
Hits@50:
  Final val: 73.54 ± 2.21   Final Test: 60.97 ± 3.59   Final old_old: 64.66 ± 3.63   Final old_new: 46.47 ± 3.68   Final new_new: 21.36 ± 4.43
AUC:
  Final val: 97.01 ± 0.13   Final Test: 96.59 ± 0.26   Final old_old: 97.12 ± 0.26   Final old_new: 94.52 ± 0.41   Final new_new: 91.38 ± 1.09
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.99 ± 2.37   Final Test: 33.32 ± 3.50   Final old_old: 36.40 ± 3.88   Final old_new: 21.11 ± 2.31   Final new_new: 2.73 ± 2.96
Hits@20:
  Final val: 66.25 ± 0.71   Final Test: 47.05 ± 3.82   Final old_old: 50.70 ± 3.99   Final old_new: 32.75 ± 3.29   Final new_new: 6.82 ± 4.55
Hits@30:
  Final val: 71.74 ± 0.49   Final Test: 54.94 ± 3.17   Final old_old: 58.81 ± 3.25   Final old_new: 39.80 ± 3.21   Final new_new: 12.27 ± 6.14
Hits@50:
  Final val: 78.34 ± 0.74   Final Test: 64.29 ± 3.28   Final old_old: 68.17 ± 3.16   Final old_new: 49.22 ± 3.92   Final new_new: 19.55 ± 6.55
AUC:
  Final val: 97.56 ± 0.07   Final Test: 97.10 ± 0.09   Final old_old: 97.54 ± 0.06   Final old_new: 95.38 ± 0.27   Final new_new: 91.79 ± 0.93
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 34.37 ± 2.13   Final Test: 20.62 ± 3.86   Final old_old: 22.27 ± 4.01   Final old_new: 13.90 ± 3.37   Final new_new: 6.82 ± 2.27
Hits@20:
  Final val: 41.22 ± 2.01   Final Test: 28.51 ± 3.44   Final old_old: 30.54 ± 3.62   Final old_new: 20.35 ± 2.93   Final new_new: 10.00 ± 2.59
Hits@30:
  Final val: 46.01 ± 2.21   Final Test: 33.53 ± 3.84   Final old_old: 35.72 ± 3.93   Final old_new: 24.81 ± 3.74   Final new_new: 12.27 ± 1.24
Hits@50:
  Final val: 52.79 ± 1.35   Final Test: 40.03 ± 3.41   Final old_old: 42.27 ± 3.27   Final old_new: 31.16 ± 4.30   Final new_new: 17.27 ± 3.45
AUC:
  Final val: 93.80 ± 0.23   Final Test: 93.62 ± 0.28   Final old_old: 93.86 ± 0.27   Final old_new: 92.70 ± 0.45   Final new_new: 90.26 ± 1.28
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.07 ± 3.30   Final Test: 29.56 ± 4.66   Final old_old: 32.76 ± 4.97   Final old_new: 16.40 ± 3.71   Final new_new: 5.00 ± 2.96
Hits@20:
  Final val: 58.93 ± 3.60   Final Test: 41.23 ± 2.56   Final old_old: 45.24 ± 2.89   Final old_new: 25.11 ± 1.34   Final new_new: 5.00 ± 2.96
Hits@30:
  Final val: 66.37 ± 2.89   Final Test: 47.39 ± 2.72   Final old_old: 51.73 ± 3.06   Final old_new: 30.00 ± 1.69   Final new_new: 7.27 ± 3.37
Hits@50:
  Final val: 75.03 ± 2.40   Final Test: 57.85 ± 2.64   Final old_old: 62.48 ± 2.93   Final old_new: 39.40 ± 1.61   Final new_new: 13.18 ± 3.37
AUC:
  Final val: 97.39 ± 0.18   Final Test: 96.68 ± 0.12   Final old_old: 97.45 ± 0.19   Final old_new: 93.58 ± 0.28   Final new_new: 89.27 ± 1.29
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 38.70 ± 0.86   Final Test: 26.30 ± 2.57   Final old_old: 28.63 ± 2.83   Final old_new: 16.80 ± 1.74   Final new_new: 7.27 ± 1.90
Hits@20:
  Final val: 46.95 ± 0.50   Final Test: 34.13 ± 3.94   Final old_old: 36.76 ± 3.92   Final old_new: 23.50 ± 4.20   Final new_new: 10.91 ± 4.93
Hits@30:
  Final val: 51.41 ± 1.51   Final Test: 39.31 ± 2.12   Final old_old: 42.13 ± 2.17   Final old_new: 27.91 ± 2.38   Final new_new: 15.00 ± 5.70
Hits@50:
  Final val: 58.31 ± 0.67   Final Test: 47.98 ± 2.62   Final old_old: 50.91 ± 2.70   Final old_new: 36.30 ± 2.75   Final new_new: 20.00 ± 4.66
AUC:
  Final val: 94.46 ± 0.17   Final Test: 94.23 ± 0.10   Final old_old: 94.56 ± 0.08   Final old_new: 92.95 ± 0.27   Final new_new: 90.82 ± 0.33
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 32.03 ± 1.43   Final Test: 18.62 ± 1.82   Final old_old: 19.94 ± 1.89   Final old_new: 13.10 ± 1.83   Final new_new: 10.00 ± 3.80
Hits@20:
  Final val: 38.20 ± 1.81   Final Test: 25.38 ± 2.10   Final old_old: 27.15 ± 2.43   Final old_new: 18.16 ± 0.85   Final new_new: 11.36 ± 3.59
Hits@30:
  Final val: 42.82 ± 1.90   Final Test: 32.70 ± 3.07   Final old_old: 34.55 ± 3.10   Final old_new: 25.29 ± 3.04   Final new_new: 15.45 ± 4.37
Hits@50:
  Final val: 50.71 ± 1.18   Final Test: 39.18 ± 0.80   Final old_old: 41.26 ± 0.84   Final old_new: 30.91 ± 1.34   Final new_new: 18.64 ± 4.07
AUC:
  Final val: 93.72 ± 0.09   Final Test: 93.60 ± 0.16   Final old_old: 93.76 ± 0.16   Final old_new: 92.98 ± 0.29   Final new_new: 91.48 ± 0.91
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.30 ± 4.25   Final Test: 30.16 ± 3.49   Final old_old: 32.34 ± 3.77   Final old_new: 21.51 ± 2.56   Final new_new: 8.64 ± 3.73
Hits@20:
  Final val: 53.03 ± 4.01   Final Test: 41.55 ± 4.14   Final old_old: 44.12 ± 4.11   Final old_new: 31.44 ± 4.48   Final new_new: 14.09 ± 1.02
Hits@30:
  Final val: 59.81 ± 3.88   Final Test: 49.94 ± 4.36   Final old_old: 52.78 ± 4.39   Final old_new: 38.82 ± 4.47   Final new_new: 19.09 ± 5.93
Hits@50:
  Final val: 67.43 ± 2.39   Final Test: 55.30 ± 4.28   Final old_old: 58.14 ± 4.01   Final old_new: 44.23 ± 5.61   Final new_new: 23.18 ± 5.43
AUC:
  Final val: 96.13 ± 0.25   Final Test: 95.76 ± 0.31   Final old_old: 96.18 ± 0.34   Final old_new: 94.13 ± 0.25   Final new_new: 91.54 ± 1.05
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.32 ± 5.87   Final Test: 32.19 ± 4.52   Final old_old: 35.31 ± 4.85   Final old_new: 19.70 ± 3.68   Final new_new: 2.73 ± 1.90
Hits@20:
  Final val: 60.50 ± 1.94   Final Test: 46.71 ± 2.50   Final old_old: 50.45 ± 2.74   Final old_new: 31.96 ± 1.72   Final new_new: 7.27 ± 4.37
Hits@30:
  Final val: 68.38 ± 1.75   Final Test: 51.35 ± 1.89   Final old_old: 55.29 ± 2.07   Final old_new: 35.82 ± 1.20   Final new_new: 10.00 ± 4.71
Hits@50:
  Final val: 77.87 ± 0.58   Final Test: 63.38 ± 2.39   Final old_old: 67.48 ± 2.28   Final old_new: 47.23 ± 3.48   Final new_new: 19.55 ± 7.97
AUC:
  Final val: 97.63 ± 0.06   Final Test: 97.24 ± 0.09   Final old_old: 97.73 ± 0.09   Final old_new: 95.32 ± 0.29   Final new_new: 91.74 ± 1.52
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.28 ± 4.96   Final Test: 33.34 ± 5.17   Final old_old: 36.08 ± 5.77   Final old_new: 22.47 ± 2.93   Final new_new: 5.45 ± 2.03
Hits@20:
  Final val: 58.73 ± 3.05   Final Test: 42.67 ± 3.80   Final old_old: 45.84 ± 4.10   Final old_new: 30.28 ± 3.08   Final new_new: 7.73 ± 2.59
Hits@30:
  Final val: 65.80 ± 2.07   Final Test: 49.93 ± 2.41   Final old_old: 53.52 ± 2.43   Final old_new: 35.89 ± 2.78   Final new_new: 10.45 ± 5.47
Hits@50:
  Final val: 73.92 ± 0.84   Final Test: 60.02 ± 1.21   Final old_old: 63.75 ± 1.12   Final old_new: 45.44 ± 1.94   Final new_new: 18.18 ± 4.25
AUC:
  Final val: 96.69 ± 0.17   Final Test: 96.37 ± 0.18   Final old_old: 96.85 ± 0.17   Final old_new: 94.53 ± 0.34   Final new_new: 90.73 ± 1.34
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 35.98 ± 0.84   Final Test: 26.19 ± 2.73   Final old_old: 28.08 ± 2.74   Final old_new: 18.54 ± 2.93   Final new_new: 10.00 ± 3.05
Hits@20:
  Final val: 43.73 ± 1.47   Final Test: 29.86 ± 1.32   Final old_old: 31.96 ± 1.33   Final old_new: 21.46 ± 1.60   Final new_new: 10.45 ± 2.59
Hits@30:
  Final val: 48.30 ± 1.49   Final Test: 35.64 ± 0.87   Final old_old: 37.70 ± 0.93   Final old_new: 27.48 ± 1.05   Final new_new: 15.00 ± 4.13
Hits@50:
  Final val: 55.58 ± 1.25   Final Test: 43.70 ± 2.07   Final old_old: 45.76 ± 2.07   Final old_new: 35.69 ± 2.21   Final new_new: 20.00 ± 4.37
AUC:
  Final val: 94.64 ± 0.04   Final Test: 94.48 ± 0.17   Final old_old: 94.69 ± 0.16   Final old_new: 93.68 ± 0.28   Final new_new: 91.96 ± 0.78
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 40.78 ± 2.65   Final Test: 31.47 ± 3.60   Final old_old: 34.17 ± 3.64   Final old_new: 20.58 ± 4.32   Final new_new: 7.73 ± 4.13
Hits@20:
  Final val: 47.73 ± 1.18   Final Test: 38.02 ± 2.81   Final old_old: 40.63 ± 2.92   Final old_new: 27.61 ± 2.54   Final new_new: 12.27 ± 5.23
Hits@30:
  Final val: 53.15 ± 1.32   Final Test: 41.00 ± 3.87   Final old_old: 43.64 ± 4.41   Final old_new: 30.68 ± 1.96   Final new_new: 11.82 ± 5.18
Hits@50:
  Final val: 61.62 ± 0.81   Final Test: 50.48 ± 3.13   Final old_old: 53.00 ± 3.31   Final old_new: 40.60 ± 2.77   Final new_new: 22.27 ± 4.07
AUC:
  Final val: 95.36 ± 0.12   Final Test: 95.19 ± 0.17   Final old_old: 95.47 ± 0.14   Final old_new: 94.05 ± 0.30   Final new_new: 92.20 ± 0.67
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 33.37 ± 0.92   Final Test: 21.01 ± 3.56   Final old_old: 22.68 ± 3.63   Final old_new: 14.33 ± 3.66   Final new_new: 4.55 ± 2.27
Hits@20:
  Final val: 39.71 ± 0.43   Final Test: 28.69 ± 3.02   Final old_old: 30.39 ± 3.03   Final old_new: 21.89 ± 3.33   Final new_new: 12.73 ± 4.71
Hits@30:
  Final val: 44.57 ± 0.90   Final Test: 32.68 ± 2.10   Final old_old: 34.33 ± 2.09   Final old_new: 26.25 ± 2.25   Final new_new: 13.64 ± 3.59
Hits@50:
  Final val: 52.52 ± 0.65   Final Test: 41.13 ± 2.02   Final old_old: 42.85 ± 2.00   Final old_new: 34.51 ± 2.25   Final new_new: 20.00 ± 2.49
AUC:
  Final val: 94.56 ± 0.14   Final Test: 94.35 ± 0.10   Final old_old: 94.55 ± 0.14   Final old_new: 93.54 ± 0.19   Final new_new: 92.38 ± 0.23
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 36.53 ± 5.02   Final Test: 23.44 ± 2.64   Final old_old: 25.83 ± 2.98   Final old_new: 13.73 ± 1.65   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 48.30 ± 4.68   Final Test: 35.03 ± 5.38   Final old_old: 38.20 ± 5.61   Final old_new: 22.12 ± 4.65   Final new_new: 8.64 ± 4.07
Hits@30:
  Final val: 57.84 ± 2.88   Final Test: 42.40 ± 1.88   Final old_old: 45.92 ± 2.10   Final old_new: 28.31 ± 1.22   Final new_new: 9.55 ± 2.96
Hits@50:
  Final val: 68.42 ± 1.45   Final Test: 51.05 ± 4.30   Final old_old: 54.88 ± 4.60   Final old_new: 35.67 ± 3.23   Final new_new: 15.91 ± 5.08
AUC:
  Final val: 96.73 ± 0.09   Final Test: 95.93 ± 0.18   Final old_old: 96.71 ± 0.15   Final old_new: 92.79 ± 0.57   Final new_new: 88.81 ± 1.74
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.45 ± 2.35   Final Test: 29.79 ± 6.99   Final old_old: 32.93 ± 7.61   Final old_new: 17.08 ± 4.72   Final new_new: 3.18 ± 2.59
Hits@20:
  Final val: 57.61 ± 1.47   Final Test: 45.28 ± 4.34   Final old_old: 49.00 ± 4.68   Final old_new: 30.48 ± 3.33   Final new_new: 8.64 ± 2.96
Hits@30:
  Final val: 65.10 ± 1.57   Final Test: 52.06 ± 3.66   Final old_old: 55.94 ± 3.90   Final old_new: 36.73 ± 3.15   Final new_new: 12.27 ± 4.71
Hits@50:
  Final val: 72.51 ± 1.02   Final Test: 59.81 ± 2.78   Final old_old: 63.58 ± 2.98   Final old_new: 45.14 ± 2.44   Final new_new: 16.82 ± 4.71
AUC:
  Final val: 96.69 ± 0.11   Final Test: 96.33 ± 0.13   Final old_old: 96.74 ± 0.13   Final old_new: 94.65 ± 0.17   Final new_new: 92.31 ± 0.72
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.50 ± 6.09   Final Test: 29.17 ± 3.72   Final old_old: 31.63 ± 3.99   Final old_new: 19.32 ± 3.24   Final new_new: 6.36 ± 4.93
Hits@20:
  Final val: 60.30 ± 4.15   Final Test: 38.76 ± 3.91   Final old_old: 41.86 ± 3.98   Final old_new: 26.40 ± 4.09   Final new_new: 8.64 ± 3.37
Hits@30:
  Final val: 67.91 ± 1.62   Final Test: 48.00 ± 5.62   Final old_old: 51.57 ± 5.80   Final old_new: 33.78 ± 5.56   Final new_new: 12.73 ± 4.43
Hits@50:
  Final val: 75.85 ± 0.69   Final Test: 61.03 ± 2.83   Final old_old: 64.88 ± 2.73   Final old_new: 45.92 ± 3.63   Final new_new: 19.09 ± 5.23
AUC:
  Final val: 97.12 ± 0.13   Final Test: 96.45 ± 0.22   Final old_old: 96.99 ± 0.21   Final old_new: 94.26 ± 0.20   Final new_new: 91.82 ± 1.17
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 22.68 ± 2.40   Final Test: 17.09 ± 2.08   Final old_old: 18.15 ± 2.12   Final old_new: 12.90 ± 2.23   Final new_new: 6.82 ± 5.33
Hits@20:
  Final val: 31.06 ± 1.21   Final Test: 22.97 ± 2.22   Final old_old: 24.00 ± 2.37   Final old_new: 18.87 ± 2.04   Final new_new: 13.64 ± 3.21
Hits@30:
  Final val: 35.71 ± 0.92   Final Test: 27.85 ± 2.31   Final old_old: 29.06 ± 2.26   Final old_new: 22.92 ± 2.80   Final new_new: 18.64 ± 5.18
Hits@50:
  Final val: 43.91 ± 0.52   Final Test: 30.25 ± 3.52   Final old_old: 31.41 ± 3.38   Final old_new: 25.64 ± 4.39   Final new_new: 18.64 ± 6.10
AUC:
  Final val: 92.95 ± 0.17   Final Test: 92.50 ± 0.17   Final old_old: 92.63 ± 0.17   Final old_new: 92.11 ± 0.17   Final new_new: 89.31 ± 1.46
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 56.11 ± 2.38   Final Test: 32.60 ± 3.23   Final old_old: 35.89 ± 3.46   Final old_new: 19.42 ± 2.46   Final new_new: 2.27 ± 3.21
Hits@20:
  Final val: 66.53 ± 1.67   Final Test: 44.50 ± 2.97   Final old_old: 48.34 ± 2.89   Final old_new: 29.19 ± 3.63   Final new_new: 6.82 ± 4.25
Hits@30:
  Final val: 72.03 ± 1.02   Final Test: 55.42 ± 3.53   Final old_old: 59.35 ± 3.39   Final old_new: 39.92 ± 4.28   Final new_new: 14.09 ± 3.73
Hits@50:
  Final val: 78.49 ± 0.57   Final Test: 64.50 ± 1.40   Final old_old: 68.51 ± 1.30   Final old_new: 48.89 ± 2.82   Final new_new: 18.64 ± 1.90
AUC:
  Final val: 97.40 ± 0.10   Final Test: 97.01 ± 0.09   Final old_old: 97.42 ± 0.08   Final old_new: 95.44 ± 0.15   Final new_new: 91.53 ± 0.69
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 32.39 ± 1.16   Final Test: 19.33 ± 2.94   Final old_old: 20.91 ± 3.09   Final old_new: 12.80 ± 2.32   Final new_new: 8.64 ± 2.96
Hits@20:
  Final val: 38.93 ± 0.92   Final Test: 26.68 ± 3.18   Final old_old: 28.53 ± 3.17   Final old_new: 19.09 ± 3.37   Final new_new: 11.82 ± 4.37
Hits@30:
  Final val: 43.65 ± 1.80   Final Test: 31.50 ± 3.78   Final old_old: 33.54 ± 3.82   Final old_new: 23.17 ± 3.58   Final new_new: 14.55 ± 5.93
Hits@50:
  Final val: 51.34 ± 0.77   Final Test: 40.88 ± 1.27   Final old_old: 42.99 ± 1.17   Final old_new: 32.57 ± 1.95   Final new_new: 18.64 ± 2.96
AUC:
  Final val: 93.73 ± 0.11   Final Test: 93.59 ± 0.24   Final old_old: 93.77 ± 0.21   Final old_new: 92.90 ± 0.43   Final new_new: 91.31 ± 1.09
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 39.99 ± 2.86   Final Test: 18.85 ± 2.70   Final old_old: 20.72 ± 2.81   Final old_new: 11.31 ± 2.39   Final new_new: 2.27 ± 1.61
Hits@20:
  Final val: 49.45 ± 0.82   Final Test: 28.35 ± 3.03   Final old_old: 30.89 ± 3.32   Final old_new: 18.06 ± 2.35   Final new_new: 6.36 ± 2.96
Hits@30:
  Final val: 54.79 ± 0.96   Final Test: 34.29 ± 1.72   Final old_old: 37.13 ± 1.65   Final old_new: 22.80 ± 3.04   Final new_new: 10.00 ± 2.59
Hits@50:
  Final val: 61.66 ± 0.83   Final Test: 44.74 ± 2.95   Final old_old: 48.17 ± 3.36   Final old_new: 31.11 ± 1.48   Final new_new: 10.91 ± 4.37
AUC:
  Final val: 95.25 ± 0.09   Final Test: 94.76 ± 0.17   Final old_old: 95.24 ± 0.21   Final old_new: 92.87 ± 0.28   Final new_new: 90.24 ± 0.95
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 26.28 ± 1.01   Final Test: 17.85 ± 2.37   Final old_old: 18.89 ± 2.46   Final old_new: 13.75 ± 2.04   Final new_new: 6.82 ± 4.55
Hits@20:
  Final val: 33.19 ± 0.62   Final Test: 25.60 ± 2.86   Final old_old: 26.52 ± 2.77   Final old_new: 21.86 ± 3.46   Final new_new: 17.73 ± 4.37
Hits@30:
  Final val: 38.37 ± 1.21   Final Test: 28.76 ± 3.49   Final old_old: 29.91 ± 3.35   Final old_new: 24.13 ± 4.49   Final new_new: 18.18 ± 6.63
Hits@50:
  Final val: 46.77 ± 0.41   Final Test: 35.55 ± 2.38   Final old_old: 36.53 ± 2.46   Final old_new: 31.66 ± 2.26   Final new_new: 25.00 ± 2.78
AUC:
  Final val: 93.65 ± 0.13   Final Test: 93.33 ± 0.13   Final old_old: 93.41 ± 0.14   Final old_new: 93.13 ± 0.15   Final new_new: 90.69 ± 0.60
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.94 ± 4.54   Final Test: 25.43 ± 7.35   Final old_old: 27.46 ± 7.73   Final old_new: 17.25 ± 6.00   Final new_new: 6.36 ± 4.07
Hits@20:
  Final val: 56.73 ± 2.90   Final Test: 41.42 ± 3.87   Final old_old: 44.11 ± 3.89   Final old_new: 30.88 ± 3.93   Final new_new: 11.82 ± 4.37
Hits@30:
  Final val: 62.11 ± 2.04   Final Test: 47.20 ± 1.98   Final old_old: 50.08 ± 1.89   Final old_new: 36.02 ± 2.46   Final new_new: 14.09 ± 6.51
Hits@50:
  Final val: 68.80 ± 0.90   Final Test: 57.17 ± 1.31   Final old_old: 60.26 ± 1.33   Final old_new: 45.26 ± 1.56   Final new_new: 19.09 ± 5.70
AUC:
  Final val: 96.19 ± 0.23   Final Test: 95.86 ± 0.25   Final old_old: 96.26 ± 0.23   Final old_new: 94.27 ± 0.33   Final new_new: 91.68 ± 1.40
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.20 ± 4.35   Final Test: 28.12 ± 5.33   Final old_old: 31.34 ± 5.72   Final old_new: 14.99 ± 4.26   Final new_new: 1.36 ± 2.03
Hits@20:
  Final val: 58.61 ± 1.87   Final Test: 44.41 ± 5.09   Final old_old: 49.03 ± 5.35   Final old_new: 25.82 ± 4.44   Final new_new: 2.27 ± 1.61
Hits@30:
  Final val: 67.22 ± 1.39   Final Test: 53.66 ± 4.45   Final old_old: 58.46 ± 4.61   Final old_new: 34.58 ± 4.20   Final new_new: 5.91 ± 4.43
Hits@50:
  Final val: 75.57 ± 0.76   Final Test: 63.76 ± 2.00   Final old_old: 68.86 ± 1.94   Final old_new: 43.75 ± 2.66   Final new_new: 9.09 ± 6.63
AUC:
  Final val: 97.48 ± 0.05   Final Test: 96.94 ± 0.14   Final old_old: 97.64 ± 0.09   Final old_new: 94.13 ± 0.37   Final new_new: 90.63 ± 0.64
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.32 ± 6.68   Final Test: 33.73 ± 4.55   Final old_old: 36.80 ± 4.94   Final old_new: 21.44 ± 2.98   Final new_new: 4.09 ± 1.90
Hits@20:
  Final val: 60.02 ± 3.64   Final Test: 43.78 ± 4.41   Final old_old: 47.30 ± 4.90   Final old_new: 29.87 ± 2.53   Final new_new: 7.73 ± 3.45
Hits@30:
  Final val: 68.98 ± 1.66   Final Test: 54.81 ± 2.41   Final old_old: 58.93 ± 2.65   Final old_new: 38.46 ± 1.83   Final new_new: 13.18 ± 1.90
Hits@50:
  Final val: 78.03 ± 1.69   Final Test: 62.76 ± 1.27   Final old_old: 67.18 ± 1.31   Final old_new: 45.37 ± 1.37   Final new_new: 16.36 ± 1.90
AUC:
  Final val: 97.66 ± 0.12   Final Test: 97.24 ± 0.09   Final old_old: 97.75 ± 0.09   Final old_new: 95.25 ± 0.21   Final new_new: 91.81 ± 0.89
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 33.08 ± 2.11   Final Test: 22.60 ± 0.95   Final old_old: 24.39 ± 0.89   Final old_new: 15.37 ± 1.34   Final new_new: 6.36 ± 4.37
Hits@20:
  Final val: 39.58 ± 1.00   Final Test: 28.96 ± 1.59   Final old_old: 30.83 ± 1.65   Final old_new: 21.61 ± 1.61   Final new_new: 9.55 ± 4.37
Hits@30:
  Final val: 44.15 ± 0.88   Final Test: 32.39 ± 0.33   Final old_old: 34.24 ± 0.32   Final old_new: 25.11 ± 0.94   Final new_new: 13.18 ± 1.90
Hits@50:
  Final val: 51.94 ± 1.47   Final Test: 42.35 ± 1.77   Final old_old: 44.25 ± 1.48   Final old_new: 35.04 ± 3.12   Final new_new: 18.64 ± 4.37
AUC:
  Final val: 94.52 ± 0.07   Final Test: 94.30 ± 0.14   Final old_old: 94.51 ± 0.18   Final old_new: 93.47 ± 0.11   Final new_new: 92.07 ± 0.60
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.44 ± 5.21   Final Test: 35.96 ± 5.00   Final old_old: 38.69 ± 5.22   Final old_new: 25.21 ± 4.32   Final new_new: 7.27 ± 4.07
Hits@20:
  Final val: 60.49 ± 3.93   Final Test: 44.37 ± 2.43   Final old_old: 47.60 ± 2.40   Final old_new: 31.54 ± 2.51   Final new_new: 12.27 ± 5.47
Hits@30:
  Final val: 66.86 ± 1.99   Final Test: 51.93 ± 3.10   Final old_old: 55.54 ± 3.03   Final old_new: 37.63 ± 3.61   Final new_new: 15.91 ± 7.87
Hits@50:
  Final val: 73.72 ± 1.22   Final Test: 60.85 ± 1.90   Final old_old: 64.41 ± 1.72   Final old_new: 46.95 ± 2.96   Final new_new: 21.36 ± 4.43
AUC:
  Final val: 96.67 ± 0.09   Final Test: 96.32 ± 0.13   Final old_old: 96.77 ± 0.10   Final old_new: 94.58 ± 0.33   Final new_new: 90.67 ± 1.09
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 23.68 ± 2.95   Final Test: 15.81 ± 3.40   Final old_old: 16.68 ± 3.54   Final old_new: 12.44 ± 3.22   Final new_new: 5.91 ± 3.45
Hits@20:
  Final val: 30.95 ± 1.65   Final Test: 22.33 ± 2.34   Final old_old: 23.47 ± 2.47   Final old_new: 17.71 ± 2.43   Final new_new: 13.18 ± 2.96
Hits@30:
  Final val: 35.83 ± 1.36   Final Test: 26.68 ± 3.32   Final old_old: 27.72 ± 3.39   Final old_new: 22.37 ± 3.27   Final new_new: 20.45 ± 4.82
Hits@50:
  Final val: 44.23 ± 0.78   Final Test: 33.65 ± 2.43   Final old_old: 35.02 ± 2.65   Final old_new: 27.98 ± 1.60   Final new_new: 24.55 ± 3.37
AUC:
  Final val: 92.95 ± 0.15   Final Test: 92.49 ± 0.21   Final old_old: 92.63 ± 0.20   Final old_new: 92.08 ± 0.30   Final new_new: 89.03 ± 1.22
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.46 ± 1.95   Final Test: 27.33 ± 1.71   Final old_old: 29.98 ± 1.66   Final old_new: 16.45 ± 1.92   Final new_new: 7.73 ± 3.45
Hits@20:
  Final val: 50.81 ± 1.42   Final Test: 35.27 ± 2.61   Final old_old: 38.22 ± 2.55   Final old_new: 23.27 ± 3.03   Final new_new: 10.45 ± 3.05
Hits@30:
  Final val: 57.07 ± 1.48   Final Test: 40.60 ± 4.53   Final old_old: 43.60 ± 4.83   Final old_new: 28.56 ± 3.53   Final new_new: 12.27 ± 2.59
Hits@50:
  Final val: 65.16 ± 1.35   Final Test: 50.99 ± 2.59   Final old_old: 54.28 ± 2.71   Final old_new: 38.09 ± 2.60   Final new_new: 15.00 ± 3.80
AUC:
  Final val: 95.99 ± 0.10   Final Test: 95.59 ± 0.13   Final old_old: 95.96 ± 0.10   Final old_new: 94.16 ± 0.31   Final new_new: 91.21 ± 0.92
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 26.37 ± 1.19   Final Test: 19.26 ± 3.64   Final old_old: 20.42 ± 3.86   Final old_new: 14.66 ± 2.72   Final new_new: 7.27 ± 5.18
Hits@20:
  Final val: 32.62 ± 0.81   Final Test: 25.19 ± 2.79   Final old_old: 26.25 ± 2.63   Final old_new: 20.93 ± 3.57   Final new_new: 15.45 ± 4.37
Hits@30:
  Final val: 37.86 ± 0.92   Final Test: 29.65 ± 1.12   Final old_old: 30.80 ± 0.95   Final old_new: 25.16 ± 1.82   Final new_new: 16.82 ± 5.23
Hits@50:
  Final val: 46.49 ± 0.91   Final Test: 34.04 ± 3.41   Final old_old: 34.77 ± 3.33   Final old_new: 31.16 ± 3.80   Final new_new: 25.91 ± 5.23
AUC:
  Final val: 93.61 ± 0.13   Final Test: 93.30 ± 0.19   Final old_old: 93.41 ± 0.19   Final old_new: 92.99 ± 0.25   Final new_new: 90.34 ± 0.82
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.91 ± 7.00   Final Test: 30.77 ± 4.62   Final old_old: 33.14 ± 4.85   Final old_new: 21.44 ± 4.23   Final new_new: 6.36 ± 4.93
Hits@20:
  Final val: 56.65 ± 6.22   Final Test: 37.17 ± 4.62   Final old_old: 40.28 ± 5.09   Final old_new: 24.66 ± 3.07   Final new_new: 8.64 ± 4.37
Hits@30:
  Final val: 65.24 ± 5.26   Final Test: 46.93 ± 4.30   Final old_old: 50.62 ± 4.66   Final old_new: 32.12 ± 2.93   Final new_new: 12.73 ± 2.59
Hits@50:
  Final val: 74.04 ± 3.28   Final Test: 59.76 ± 2.74   Final old_old: 63.57 ± 2.84   Final old_new: 44.61 ± 2.46   Final new_new: 21.82 ± 3.80
AUC:
  Final val: 97.05 ± 0.19   Final Test: 96.49 ± 0.16   Final old_old: 97.00 ± 0.16   Final old_new: 94.45 ± 0.33   Final new_new: 91.55 ± 2.32
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 37.47 ± 1.51   Final Test: 17.59 ± 3.71   Final old_old: 19.47 ± 4.15   Final old_new: 9.87 ± 2.02   Final new_new: 3.64 ± 2.03
Hits@20:
  Final val: 43.40 ± 1.30   Final Test: 26.67 ± 5.23   Final old_old: 29.04 ± 5.57   Final old_new: 17.00 ± 3.92   Final new_new: 7.27 ± 2.96
Hits@30:
  Final val: 47.23 ± 1.45   Final Test: 32.28 ± 4.80   Final old_old: 34.60 ± 4.79   Final old_new: 23.02 ± 5.09   Final new_new: 10.00 ± 3.80
Hits@50:
  Final val: 54.73 ± 1.42   Final Test: 36.89 ± 1.23   Final old_old: 39.10 ± 1.36   Final old_new: 28.21 ± 1.56   Final new_new: 12.27 ± 1.24
AUC:
  Final val: 94.29 ± 0.29   Final Test: 93.67 ± 0.23   Final old_old: 94.01 ± 0.25   Final old_new: 92.42 ± 0.36   Final new_new: 88.36 ± 0.96
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 53.83 ± 2.54   Final Test: 29.47 ± 3.45   Final old_old: 32.31 ± 3.65   Final old_new: 18.11 ± 2.89   Final new_new: 2.27 ± 3.21
Hits@20:
  Final val: 64.24 ± 3.22   Final Test: 46.77 ± 4.72   Final old_old: 50.45 ± 4.85   Final old_new: 32.29 ± 4.47   Final new_new: 7.73 ± 4.71
Hits@30:
  Final val: 70.17 ± 1.80   Final Test: 56.09 ± 5.02   Final old_old: 59.87 ± 5.12   Final old_new: 41.39 ± 4.58   Final new_new: 12.27 ± 8.74
Hits@50:
  Final val: 76.84 ± 1.12   Final Test: 64.84 ± 1.18   Final old_old: 68.71 ± 1.19   Final old_new: 49.87 ± 1.14   Final new_new: 19.09 ± 6.14
AUC:
  Final val: 97.27 ± 0.08   Final Test: 96.96 ± 0.12   Final old_old: 97.33 ± 0.12   Final old_new: 95.54 ± 0.17   Final new_new: 91.80 ± 0.29
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.38 ± 5.97   Final Test: 33.72 ± 4.99   Final old_old: 37.00 ± 5.08   Final old_new: 20.55 ± 4.97   Final new_new: 3.64 ± 4.71
Hits@20:
  Final val: 59.41 ± 4.42   Final Test: 47.06 ± 1.07   Final old_old: 50.83 ± 1.42   Final old_new: 32.19 ± 2.63   Final new_new: 8.18 ± 3.80
Hits@30:
  Final val: 65.95 ± 3.50   Final Test: 52.58 ± 1.88   Final old_old: 56.39 ± 2.16   Final old_new: 37.66 ± 1.94   Final new_new: 11.36 ± 5.33
Hits@50:
  Final val: 72.84 ± 2.50   Final Test: 59.78 ± 2.85   Final old_old: 63.51 ± 2.88   Final old_new: 45.16 ± 3.57   Final new_new: 19.09 ± 9.59
AUC:
  Final val: 96.68 ± 0.16   Final Test: 96.34 ± 0.14   Final old_old: 96.79 ± 0.16   Final old_new: 94.53 ± 0.28   Final new_new: 92.42 ± 0.13
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.11 ± 7.45   Final Test: 27.71 ± 5.51   Final old_old: 30.35 ± 5.86   Final old_new: 17.03 ± 4.06   Final new_new: 5.00 ± 5.18
Hits@20:
  Final val: 53.18 ± 6.72   Final Test: 37.20 ± 5.07   Final old_old: 40.57 ± 5.35   Final old_new: 23.48 ± 4.07   Final new_new: 9.09 ± 4.82
Hits@30:
  Final val: 62.30 ± 4.40   Final Test: 46.06 ± 4.56   Final old_old: 50.06 ± 4.95   Final old_new: 29.77 ± 3.62   Final new_new: 13.18 ± 5.66
Hits@50:
  Final val: 71.77 ± 2.60   Final Test: 54.15 ± 5.21   Final old_old: 58.50 ± 5.61   Final old_new: 36.55 ± 3.65   Final new_new: 16.82 ± 7.81
AUC:
  Final val: 96.96 ± 0.21   Final Test: 96.18 ± 0.30   Final old_old: 96.98 ± 0.28   Final old_new: 92.94 ± 0.50   Final new_new: 89.88 ± 1.93
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 23.76 ± 2.31   Final Test: 14.84 ± 4.13   Final old_old: 15.55 ± 4.31   Final old_new: 12.12 ± 3.80   Final new_new: 5.45 ± 3.45
Hits@20:
  Final val: 30.74 ± 1.28   Final Test: 21.96 ± 3.96   Final old_old: 22.92 ± 4.04   Final old_new: 18.16 ± 3.89   Final new_new: 12.73 ± 3.05
Hits@30:
  Final val: 36.47 ± 1.10   Final Test: 26.18 ± 1.73   Final old_old: 27.26 ± 1.88   Final old_new: 21.74 ± 1.60   Final new_new: 18.64 ± 6.51
Hits@50:
  Final val: 43.87 ± 0.70   Final Test: 32.58 ± 3.76   Final old_old: 33.90 ± 3.87   Final old_new: 27.18 ± 3.79   Final new_new: 21.82 ± 3.45
AUC:
  Final val: 92.99 ± 0.12   Final Test: 92.59 ± 0.16   Final old_old: 92.73 ± 0.14   Final old_new: 92.18 ± 0.27   Final new_new: 89.10 ± 1.49
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 40.63 ± 2.65   Final Test: 25.44 ± 8.64   Final old_old: 27.77 ± 9.12   Final old_new: 15.99 ± 6.95   Final new_new: 5.91 ± 4.43
Hits@20:
  Final val: 51.57 ± 2.88   Final Test: 35.42 ± 4.01   Final old_old: 38.41 ± 4.29   Final old_new: 23.32 ± 3.02   Final new_new: 9.55 ± 4.37
Hits@30:
  Final val: 58.64 ± 1.59   Final Test: 41.61 ± 4.19   Final old_old: 44.88 ± 4.39   Final old_new: 28.49 ± 3.64   Final new_new: 11.36 ± 3.21
Hits@50:
  Final val: 67.65 ± 1.04   Final Test: 52.01 ± 4.37   Final old_old: 55.55 ± 4.29   Final old_new: 38.04 ± 5.00   Final new_new: 15.00 ± 6.93
AUC:
  Final val: 96.06 ± 0.18   Final Test: 95.55 ± 0.24   Final old_old: 96.20 ± 0.24   Final old_new: 93.06 ± 0.53   Final new_new: 87.56 ± 1.62
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 39.26 ± 1.40   Final Test: 22.83 ± 3.67   Final old_old: 25.13 ± 3.90   Final old_new: 13.45 ± 2.82   Final new_new: 3.64 ± 2.59
Hits@20:
  Final val: 44.83 ± 0.82   Final Test: 30.50 ± 1.64   Final old_old: 32.67 ± 1.74   Final old_new: 21.79 ± 1.65   Final new_new: 10.91 ± 2.96
Hits@30:
  Final val: 49.46 ± 0.54   Final Test: 38.09 ± 2.28   Final old_old: 40.43 ± 2.64   Final old_new: 28.82 ± 0.91   Final new_new: 14.55 ± 2.03
Hits@50:
  Final val: 56.53 ± 0.55   Final Test: 43.13 ± 2.18   Final old_old: 45.13 ± 2.05   Final old_new: 35.14 ± 3.05   Final new_new: 24.55 ± 1.90
AUC:
  Final val: 94.75 ± 0.22   Final Test: 94.35 ± 0.16   Final old_old: 94.65 ± 0.16   Final old_new: 93.20 ± 0.23   Final new_new: 90.83 ± 1.22
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.13 ± 7.74   Final Test: 30.69 ± 4.39   Final old_old: 33.63 ± 4.67   Final old_new: 18.97 ± 3.37   Final new_new: 2.27 ± 1.61
Hits@20:
  Final val: 60.74 ± 3.93   Final Test: 43.92 ± 5.48   Final old_old: 47.36 ± 5.87   Final old_new: 30.38 ± 4.20   Final new_new: 7.73 ± 2.59
Hits@30:
  Final val: 69.23 ± 2.25   Final Test: 53.88 ± 2.56   Final old_old: 57.85 ± 2.70   Final old_new: 38.39 ± 2.00   Final new_new: 9.09 ± 2.78
Hits@50:
  Final val: 77.77 ± 1.65   Final Test: 60.80 ± 4.53   Final old_old: 64.73 ± 4.79   Final old_new: 45.42 ± 3.93   Final new_new: 18.18 ± 8.50
AUC:
  Final val: 97.66 ± 0.11   Final Test: 97.15 ± 0.04   Final old_old: 97.64 ± 0.05   Final old_new: 95.18 ± 0.13   Final new_new: 91.97 ± 1.02
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.66 ± 5.34   Final Test: 32.13 ± 3.74   Final old_old: 34.72 ± 3.79   Final old_new: 21.84 ± 3.78   Final new_new: 5.91 ± 4.13
Hits@20:
  Final val: 58.80 ± 2.89   Final Test: 42.67 ± 3.78   Final old_old: 45.54 ± 3.70   Final old_new: 31.31 ± 4.47   Final new_new: 12.73 ± 3.45
Hits@30:
  Final val: 65.35 ± 1.96   Final Test: 50.47 ± 4.96   Final old_old: 53.90 ± 5.11   Final old_new: 36.93 ± 4.38   Final new_new: 15.00 ± 5.70
Hits@50:
  Final val: 72.99 ± 0.81   Final Test: 55.60 ± 2.15   Final old_old: 59.28 ± 2.12   Final old_new: 41.01 ± 2.68   Final new_new: 19.09 ± 4.13
AUC:
  Final val: 96.86 ± 0.16   Final Test: 96.46 ± 0.12   Final old_old: 96.90 ± 0.12   Final old_new: 94.75 ± 0.24   Final new_new: 91.37 ± 0.95
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.45 ± 5.87   Final Test: 24.42 ± 5.33   Final old_old: 26.53 ± 5.67   Final old_new: 15.94 ± 4.10   Final new_new: 5.00 ± 2.96
Hits@20:
  Final val: 52.87 ± 4.75   Final Test: 39.77 ± 3.34   Final old_old: 42.69 ± 3.53   Final old_new: 28.11 ± 3.09   Final new_new: 10.91 ± 5.43
Hits@30:
  Final val: 58.59 ± 3.33   Final Test: 46.81 ± 4.13   Final old_old: 49.79 ± 4.27   Final old_new: 35.09 ± 4.02   Final new_new: 15.45 ± 4.07
Hits@50:
  Final val: 66.85 ± 1.31   Final Test: 54.95 ± 1.86   Final old_old: 58.13 ± 2.00   Final old_new: 42.52 ± 1.86   Final new_new: 19.55 ± 3.45
AUC:
  Final val: 96.18 ± 0.19   Final Test: 95.84 ± 0.26   Final old_old: 96.26 ± 0.20   Final old_new: 94.23 ± 0.53   Final new_new: 90.84 ± 1.81
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 26.62 ± 1.13   Final Test: 19.29 ± 2.00   Final old_old: 20.36 ± 1.89   Final old_new: 14.99 ± 2.47   Final new_new: 9.55 ± 4.37
Hits@20:
  Final val: 32.99 ± 0.61   Final Test: 26.02 ± 2.82   Final old_old: 27.09 ± 2.60   Final old_new: 21.79 ± 3.99   Final new_new: 14.55 ± 6.74
Hits@30:
  Final val: 38.32 ± 1.18   Final Test: 29.04 ± 2.57   Final old_old: 30.10 ± 2.54   Final old_new: 24.61 ± 2.70   Final new_new: 21.82 ± 6.55
Hits@50:
  Final val: 46.54 ± 1.04   Final Test: 34.59 ± 1.45   Final old_old: 35.60 ± 1.58   Final old_new: 30.53 ± 1.57   Final new_new: 25.45 ± 1.90
AUC:
  Final val: 93.62 ± 0.13   Final Test: 93.27 ± 0.23   Final old_old: 93.37 ± 0.24   Final old_new: 92.98 ± 0.19   Final new_new: 90.42 ± 0.56
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.44 ± 6.60   Final Test: 30.15 ± 4.35   Final old_old: 33.87 ± 4.53   Final old_new: 14.94 ± 4.09   Final new_new: 1.36 ± 3.05
Hits@20:
  Final val: 58.03 ± 4.37   Final Test: 37.21 ± 1.13   Final old_old: 41.40 ± 1.48   Final old_new: 20.10 ± 1.88   Final new_new: 4.09 ± 6.89
Hits@30:
  Final val: 66.04 ± 2.35   Final Test: 49.13 ± 3.07   Final old_old: 53.87 ± 3.63   Final old_new: 30.08 ± 1.95   Final new_new: 6.36 ± 6.10
Hits@50:
  Final val: 74.99 ± 1.60   Final Test: 60.26 ± 1.94   Final old_old: 65.30 ± 2.34   Final old_new: 40.05 ± 1.56   Final new_new: 13.64 ± 5.33
AUC:
  Final val: 97.52 ± 0.13   Final Test: 96.81 ± 0.13   Final old_old: 97.58 ± 0.13   Final old_new: 93.74 ± 0.30   Final new_new: 89.22 ± 1.49
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 35.66 ± 2.39   Final Test: 16.21 ± 4.15   Final old_old: 18.22 ± 4.59   Final old_new: 7.93 ± 2.58   Final new_new: 1.82 ± 2.49
Hits@20:
  Final val: 41.96 ± 1.81   Final Test: 24.63 ± 5.95   Final old_old: 26.80 ± 6.31   Final old_new: 15.82 ± 4.62   Final new_new: 5.91 ± 3.45
Hits@30:
  Final val: 46.23 ± 2.17   Final Test: 28.47 ± 3.24   Final old_old: 30.70 ± 3.21   Final old_new: 19.45 ± 3.68   Final new_new: 9.09 ± 2.78
Hits@50:
  Final val: 54.23 ± 1.64   Final Test: 36.10 ± 3.49   Final old_old: 38.30 ± 3.63   Final old_new: 27.25 ± 3.11   Final new_new: 16.36 ± 3.73
AUC:
  Final val: 94.26 ± 0.22   Final Test: 93.67 ± 0.20   Final old_old: 94.00 ± 0.22   Final old_new: 92.43 ± 0.37   Final new_new: 88.97 ± 1.09
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.09 ± 2.98   Final Test: 31.83 ± 5.91   Final old_old: 34.67 ± 6.05   Final old_new: 20.45 ± 5.47   Final new_new: 4.55 ± 3.59
Hits@20:
  Final val: 56.12 ± 2.33   Final Test: 40.53 ± 4.41   Final old_old: 43.77 ± 4.52   Final old_new: 27.73 ± 4.13   Final new_new: 7.27 ± 3.37
Hits@30:
  Final val: 62.02 ± 1.94   Final Test: 45.19 ± 4.15   Final old_old: 48.54 ± 4.19   Final old_new: 31.99 ± 4.14   Final new_new: 9.09 ± 4.25
Hits@50:
  Final val: 69.75 ± 1.46   Final Test: 57.00 ± 1.78   Final old_old: 60.48 ± 1.65   Final old_new: 43.68 ± 2.88   Final new_new: 13.18 ± 3.37
AUC:
  Final val: 96.63 ± 0.15   Final Test: 96.32 ± 0.12   Final old_old: 96.81 ± 0.09   Final old_new: 94.38 ± 0.31   Final new_new: 90.64 ± 0.58
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 25.07 ± 2.75   Final Test: 15.43 ± 0.83   Final old_old: 16.39 ± 0.61   Final old_new: 11.69 ± 1.79   Final new_new: 4.55 ± 3.59
Hits@20:
  Final val: 31.47 ± 2.43   Final Test: 21.50 ± 3.31   Final old_old: 22.41 ± 3.64   Final old_new: 17.93 ± 2.13   Final new_new: 11.82 ± 3.73
Hits@30:
  Final val: 36.53 ± 1.73   Final Test: 25.75 ± 2.54   Final old_old: 26.96 ± 2.48   Final old_new: 21.01 ± 2.92   Final new_new: 12.73 ± 4.71
Hits@50:
  Final val: 44.80 ± 1.45   Final Test: 32.58 ± 2.75   Final old_old: 33.94 ± 2.80   Final old_new: 27.15 ± 2.88   Final new_new: 20.00 ± 6.31
AUC:
  Final val: 92.91 ± 0.13   Final Test: 92.53 ± 0.16   Final old_old: 92.65 ± 0.18   Final old_new: 92.17 ± 0.11   Final new_new: 89.29 ± 0.50
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.52 ± 2.29   Final Test: 32.32 ± 2.38   Final old_old: 35.27 ± 2.59   Final old_new: 20.53 ± 1.74   Final new_new: 4.09 ± 3.37
Hits@20:
  Final val: 56.12 ± 2.05   Final Test: 42.83 ± 4.84   Final old_old: 46.20 ± 4.95   Final old_new: 29.55 ± 4.49   Final new_new: 7.27 ± 5.66
Hits@30:
  Final val: 64.54 ± 1.20   Final Test: 49.95 ± 3.25   Final old_old: 53.57 ± 3.34   Final old_new: 35.74 ± 2.89   Final new_new: 10.91 ± 8.41
Hits@50:
  Final val: 71.93 ± 0.62   Final Test: 58.66 ± 2.07   Final old_old: 62.12 ± 2.00   Final old_new: 45.16 ± 2.52   Final new_new: 20.45 ± 5.79
AUC:
  Final val: 96.68 ± 0.10   Final Test: 96.38 ± 0.11   Final old_old: 96.80 ± 0.14   Final old_new: 94.70 ± 0.18   Final new_new: 92.13 ± 0.88
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.45 ± 5.60   Final Test: 27.70 ± 3.59   Final old_old: 30.28 ± 3.97   Final old_new: 17.30 ± 2.20   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 60.49 ± 3.19   Final Test: 38.72 ± 4.36   Final old_old: 41.85 ± 4.70   Final old_new: 26.25 ± 3.28   Final new_new: 7.73 ± 1.24
Hits@30:
  Final val: 66.68 ± 2.13   Final Test: 47.26 ± 5.41   Final old_old: 50.98 ± 5.63   Final old_new: 32.34 ± 4.88   Final new_new: 12.73 ± 4.13
Hits@50:
  Final val: 74.73 ± 1.66   Final Test: 56.44 ± 4.84   Final old_old: 60.36 ± 5.06   Final old_new: 40.96 ± 3.97   Final new_new: 16.36 ± 5.43
AUC:
  Final val: 97.04 ± 0.18   Final Test: 96.45 ± 0.27   Final old_old: 97.00 ± 0.25   Final old_new: 94.30 ± 0.43   Final new_new: 90.74 ± 1.10
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.14 ± 2.74   Final Test: 32.83 ± 4.23   Final old_old: 35.79 ± 4.41   Final old_new: 21.18 ± 3.51   Final new_new: 1.82 ± 4.07
Hits@20:
  Final val: 63.05 ± 1.68   Final Test: 45.30 ± 3.14   Final old_old: 49.04 ± 3.28   Final old_new: 30.43 ± 2.61   Final new_new: 8.64 ± 5.18
Hits@30:
  Final val: 69.87 ± 2.27   Final Test: 54.29 ± 2.98   Final old_old: 58.24 ± 2.97   Final old_new: 38.69 ± 3.02   Final new_new: 14.09 ± 5.18
Hits@50:
  Final val: 76.90 ± 1.57   Final Test: 64.95 ± 1.09   Final old_old: 68.86 ± 1.15   Final old_new: 49.75 ± 1.26   Final new_new: 20.00 ± 3.37
AUC:
  Final val: 97.30 ± 0.05   Final Test: 97.06 ± 0.10   Final old_old: 97.42 ± 0.07   Final old_new: 95.68 ± 0.20   Final new_new: 92.51 ± 1.08
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 33.94 ± 3.98   Final Test: 22.65 ± 4.40   Final old_old: 24.88 ± 4.81   Final old_new: 13.53 ± 3.00   Final new_new: 5.91 ± 2.59
Hits@20:
  Final val: 49.30 ± 4.34   Final Test: 33.46 ± 2.27   Final old_old: 36.57 ± 2.42   Final old_new: 20.91 ± 1.86   Final new_new: 6.82 ± 6.22
Hits@30:
  Final val: 58.98 ± 3.80   Final Test: 43.10 ± 3.18   Final old_old: 46.90 ± 3.55   Final old_new: 27.78 ± 2.02   Final new_new: 9.09 ± 3.21
Hits@50:
  Final val: 69.99 ± 2.22   Final Test: 54.73 ± 3.51   Final old_old: 59.03 ± 3.69   Final old_new: 37.43 ± 3.18   Final new_new: 15.91 ± 5.79
AUC:
  Final val: 96.94 ± 0.22   Final Test: 96.23 ± 0.25   Final old_old: 96.98 ± 0.18   Final old_new: 93.17 ± 0.60   Final new_new: 89.44 ± 1.34
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 26.12 ± 1.02   Final Test: 19.29 ± 2.64   Final old_old: 20.18 ± 2.77   Final old_new: 15.77 ± 2.19   Final new_new: 10.00 ± 2.59
Hits@20:
  Final val: 32.96 ± 1.72   Final Test: 25.60 ± 1.12   Final old_old: 26.57 ± 1.15   Final old_new: 21.64 ± 1.19   Final new_new: 17.27 ± 3.05
Hits@30:
  Final val: 38.18 ± 1.76   Final Test: 28.29 ± 1.74   Final old_old: 29.14 ± 1.84   Final old_new: 24.86 ± 1.29   Final new_new: 20.91 ± 4.07
Hits@50:
  Final val: 45.53 ± 1.47   Final Test: 35.13 ± 1.43   Final old_old: 36.15 ± 1.51   Final old_new: 31.06 ± 1.36   Final new_new: 25.45 ± 3.37
AUC:
  Final val: 93.59 ± 0.14   Final Test: 93.17 ± 0.21   Final old_old: 93.26 ± 0.24   Final old_new: 92.91 ± 0.17   Final new_new: 90.73 ± 0.65
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 39.14 ± 0.40   Final Test: 24.02 ± 2.40   Final old_old: 26.13 ± 2.51   Final old_new: 15.49 ± 1.98   Final new_new: 5.91 ± 2.59
Hits@20:
  Final val: 45.55 ± 0.94   Final Test: 30.03 ± 4.11   Final old_old: 32.25 ± 4.21   Final old_new: 21.06 ± 3.60   Final new_new: 10.91 ± 7.94
Hits@30:
  Final val: 50.45 ± 0.72   Final Test: 38.58 ± 2.65   Final old_old: 40.76 ± 2.93   Final old_new: 29.82 ± 1.66   Final new_new: 18.64 ± 3.73
Hits@50:
  Final val: 56.93 ± 1.24   Final Test: 43.87 ± 1.75   Final old_old: 45.90 ± 1.85   Final old_new: 35.82 ± 1.57   Final new_new: 23.64 ± 2.03
AUC:
  Final val: 94.74 ± 0.18   Final Test: 94.29 ± 0.18   Final old_old: 94.61 ± 0.18   Final old_new: 93.06 ± 0.26   Final new_new: 90.28 ± 0.91
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.28 ± 4.16   Final Test: 33.44 ± 3.06   Final old_old: 36.42 ± 3.18   Final old_new: 21.54 ± 2.59   Final new_new: 5.00 ± 4.07
Hits@20:
  Final val: 61.90 ± 2.57   Final Test: 43.78 ± 3.13   Final old_old: 47.30 ± 3.47   Final old_new: 29.90 ± 1.92   Final new_new: 6.82 ± 3.59
Hits@30:
  Final val: 69.32 ± 2.25   Final Test: 54.44 ± 1.86   Final old_old: 58.55 ± 1.91   Final old_new: 38.16 ± 2.11   Final new_new: 12.27 ± 4.98
Hits@50:
  Final val: 78.09 ± 2.24   Final Test: 62.32 ± 3.78   Final old_old: 66.37 ± 3.92   Final old_new: 46.22 ± 3.33   Final new_new: 21.82 ± 6.74
AUC:
  Final val: 97.68 ± 0.13   Final Test: 97.17 ± 0.11   Final old_old: 97.66 ± 0.09   Final old_new: 95.23 ± 0.30   Final new_new: 92.02 ± 1.02
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 39.02 ± 3.11   Final Test: 19.60 ± 3.18   Final old_old: 21.70 ± 3.41   Final old_new: 11.06 ± 2.39   Final new_new: 2.27 ± 2.78
Hits@20:
  Final val: 46.37 ± 2.83   Final Test: 30.32 ± 4.80   Final old_old: 33.04 ± 5.02   Final old_new: 19.35 ± 3.96   Final new_new: 5.91 ± 6.74
Hits@30:
  Final val: 51.75 ± 1.88   Final Test: 37.32 ± 4.23   Final old_old: 40.14 ± 4.32   Final old_new: 26.10 ± 4.15   Final new_new: 10.00 ± 7.11
Hits@50:
  Final val: 59.62 ± 1.79   Final Test: 43.90 ± 3.37   Final old_old: 46.74 ± 3.37   Final old_new: 32.64 ± 3.72   Final new_new: 14.55 ± 5.93
AUC:
  Final val: 94.86 ± 0.44   Final Test: 94.33 ± 0.47   Final old_old: 94.85 ± 0.53   Final old_new: 92.33 ± 0.26   Final new_new: 87.74 ± 1.99
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 25.44 ± 2.35   Final Test: 14.63 ± 2.37   Final old_old: 15.51 ± 2.46   Final old_new: 11.13 ± 2.12   Final new_new: 5.91 ± 3.05
Hits@20:
  Final val: 30.93 ± 1.81   Final Test: 23.00 ± 2.44   Final old_old: 23.95 ± 2.20   Final old_new: 19.14 ± 3.41   Final new_new: 15.00 ± 6.74
Hits@30:
  Final val: 36.59 ± 1.07   Final Test: 26.02 ± 3.25   Final old_old: 27.31 ± 3.10   Final old_new: 20.81 ± 3.97   Final new_new: 15.00 ± 4.71
Hits@50:
  Final val: 44.04 ± 0.80   Final Test: 33.37 ± 3.18   Final old_old: 34.66 ± 3.04   Final old_new: 28.14 ± 3.71   Final new_new: 22.73 ± 8.35
AUC:
  Final val: 92.98 ± 0.16   Final Test: 92.61 ± 0.12   Final old_old: 92.73 ± 0.13   Final old_new: 92.26 ± 0.24   Final new_new: 89.30 ± 0.62
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.66 ± 5.48   Final Test: 28.96 ± 5.41   Final old_old: 31.49 ± 6.00   Final old_new: 18.66 ± 3.08   Final new_new: 7.73 ± 3.45
Hits@20:
  Final val: 56.10 ± 4.19   Final Test: 41.11 ± 1.73   Final old_old: 44.16 ± 1.96   Final old_new: 28.89 ± 2.32   Final new_new: 12.73 ± 4.71
Hits@30:
  Final val: 61.22 ± 3.19   Final Test: 47.23 ± 5.61   Final old_old: 50.38 ± 5.76   Final old_new: 34.76 ± 5.28   Final new_new: 15.00 ± 5.70
Hits@50:
  Final val: 68.25 ± 1.80   Final Test: 55.31 ± 4.26   Final old_old: 58.51 ± 4.10   Final old_new: 42.87 ± 5.03   Final new_new: 18.18 ± 8.50
AUC:
  Final val: 96.16 ± 0.18   Final Test: 95.86 ± 0.22   Final old_old: 96.29 ± 0.24   Final old_new: 94.20 ± 0.09   Final new_new: 90.72 ± 1.10
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 35.87 ± 2.53   Final Test: 18.23 ± 3.02   Final old_old: 20.05 ± 3.35   Final old_new: 10.81 ± 1.91   Final new_new: 3.64 ± 2.59
Hits@20:
  Final val: 42.12 ± 1.80   Final Test: 26.22 ± 3.87   Final old_old: 28.43 ± 3.91   Final old_new: 17.23 ± 3.72   Final new_new: 8.18 ± 4.13
Hits@30:
  Final val: 47.74 ± 1.37   Final Test: 32.06 ± 4.34   Final old_old: 34.12 ± 4.31   Final old_new: 23.78 ± 5.04   Final new_new: 13.18 ± 4.37
Hits@50:
  Final val: 54.73 ± 1.40   Final Test: 36.93 ± 2.69   Final old_old: 39.09 ± 2.85   Final old_new: 28.34 ± 2.20   Final new_new: 15.45 ± 2.49
AUC:
  Final val: 94.25 ± 0.21   Final Test: 93.66 ± 0.27   Final old_old: 94.00 ± 0.28   Final old_new: 92.43 ± 0.34   Final new_new: 88.55 ± 0.93
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.18 ± 5.23   Final Test: 33.92 ± 5.56   Final old_old: 37.74 ± 5.84   Final old_new: 18.39 ± 4.76   Final new_new: 2.73 ± 2.96
Hits@20:
  Final val: 56.44 ± 3.77   Final Test: 42.59 ± 3.05   Final old_old: 46.91 ± 3.43   Final old_new: 25.19 ± 2.05   Final new_new: 4.09 ± 2.96
Hits@30:
  Final val: 65.53 ± 1.88   Final Test: 49.59 ± 6.06   Final old_old: 54.41 ± 6.70   Final old_new: 30.25 ± 3.73   Final new_new: 5.00 ± 2.49
Hits@50:
  Final val: 75.41 ± 0.72   Final Test: 61.67 ± 1.32   Final old_old: 66.67 ± 1.41   Final old_new: 41.79 ± 1.55   Final new_new: 11.82 ± 6.31
AUC:
  Final val: 97.56 ± 0.07   Final Test: 96.84 ± 0.17   Final old_old: 97.62 ± 0.10   Final old_new: 93.77 ± 0.48   Final new_new: 88.83 ± 0.40
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.76 ± 5.23   Final Test: 27.13 ± 8.35   Final old_old: 28.92 ± 9.13   Final old_new: 20.20 ± 5.27   Final new_new: 6.36 ± 3.37
Hits@20:
  Final val: 53.01 ± 4.29   Final Test: 37.09 ± 6.73   Final old_old: 39.35 ± 7.42   Final old_new: 28.24 ± 3.98   Final new_new: 12.27 ± 3.45
Hits@30:
  Final val: 59.19 ± 3.67   Final Test: 44.77 ± 7.78   Final old_old: 47.18 ± 8.36   Final old_new: 35.44 ± 5.56   Final new_new: 16.36 ± 2.96
Hits@50:
  Final val: 66.97 ± 2.23   Final Test: 55.41 ± 3.40   Final old_old: 57.73 ± 3.57   Final old_new: 46.80 ± 2.84   Final new_new: 21.36 ± 4.71
AUC:
  Final val: 95.91 ± 0.12   Final Test: 95.54 ± 0.17   Final old_old: 95.91 ± 0.19   Final old_new: 94.15 ± 0.37   Final new_new: 90.79 ± 1.10
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 26.12 ± 1.31   Final Test: 18.71 ± 1.03   Final old_old: 19.70 ± 1.14   Final old_new: 14.81 ± 0.88   Final new_new: 7.73 ± 3.45
Hits@20:
  Final val: 31.92 ± 0.95   Final Test: 23.80 ± 1.90   Final old_old: 24.89 ± 2.04   Final old_new: 19.32 ± 1.52   Final new_new: 15.00 ± 4.98
Hits@30:
  Final val: 37.44 ± 0.88   Final Test: 30.09 ± 1.68   Final old_old: 31.18 ± 1.73   Final old_new: 25.67 ± 1.77   Final new_new: 20.91 ± 4.37
Hits@50:
  Final val: 45.24 ± 0.73   Final Test: 34.29 ± 1.00   Final old_old: 35.37 ± 1.04   Final old_new: 29.87 ± 1.71   Final new_new: 25.91 ± 2.03
AUC:
  Final val: 93.53 ± 0.09   Final Test: 93.20 ± 0.11   Final old_old: 93.29 ± 0.11   Final old_new: 92.89 ± 0.21   Final new_new: 91.11 ± 0.46
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.14 ± 6.63   Final Test: 30.58 ± 5.10   Final old_old: 33.57 ± 5.43   Final old_new: 18.61 ± 4.08   Final new_new: 3.18 ± 1.24
Hits@20:
  Final val: 55.71 ± 4.38   Final Test: 44.38 ± 3.55   Final old_old: 47.77 ± 3.85   Final old_new: 30.96 ± 2.39   Final new_new: 10.00 ± 3.45
Hits@30:
  Final val: 63.15 ± 3.17   Final Test: 53.28 ± 1.74   Final old_old: 56.84 ± 1.69   Final old_new: 39.35 ± 2.19   Final new_new: 13.64 ± 4.82
Hits@50:
  Final val: 71.09 ± 1.60   Final Test: 60.60 ± 1.30   Final old_old: 64.00 ± 1.55   Final old_new: 47.63 ± 0.86   Final new_new: 17.73 ± 4.37
AUC:
  Final val: 96.72 ± 0.13   Final Test: 96.35 ± 0.10   Final old_old: 96.78 ± 0.10   Final old_new: 94.70 ± 0.23   Final new_new: 91.27 ± 1.54
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 50.58 ± 4.08   Final Test: 32.15 ± 4.58   Final old_old: 34.99 ± 5.12   Final old_new: 20.58 ± 2.38   Final new_new: 8.64 ± 4.37
Hits@20:
  Final val: 63.12 ± 2.19   Final Test: 43.63 ± 4.49   Final old_old: 47.07 ± 4.62   Final old_new: 29.80 ± 4.13   Final new_new: 12.73 ± 4.43
Hits@30:
  Final val: 69.07 ± 1.42   Final Test: 51.68 ± 8.48   Final old_old: 55.37 ± 8.59   Final old_new: 36.90 ± 8.25   Final new_new: 16.36 ± 6.70
Hits@50:
  Final val: 76.83 ± 1.62   Final Test: 62.95 ± 5.25   Final old_old: 66.74 ± 5.20   Final old_new: 47.98 ± 5.66   Final new_new: 23.64 ± 3.80
AUC:
  Final val: 97.20 ± 0.15   Final Test: 96.71 ± 0.05   Final old_old: 97.23 ± 0.09   Final old_new: 94.69 ± 0.12   Final new_new: 91.17 ± 1.53
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 39.88 ± 0.82   Final Test: 22.22 ± 1.34   Final old_old: 24.31 ± 1.54   Final old_new: 13.75 ± 0.60   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 45.50 ± 0.86   Final Test: 30.10 ± 3.37   Final old_old: 32.29 ± 3.51   Final old_new: 21.41 ± 3.12   Final new_new: 8.18 ± 4.71
Hits@30:
  Final val: 49.85 ± 0.88   Final Test: 35.31 ± 1.82   Final old_old: 37.46 ± 2.15   Final old_new: 26.70 ± 1.50   Final new_new: 15.00 ± 5.47
Hits@50:
  Final val: 56.32 ± 1.36   Final Test: 43.15 ± 2.14   Final old_old: 45.27 ± 1.96   Final old_new: 34.69 ± 3.53   Final new_new: 22.27 ± 8.26
AUC:
  Final val: 94.74 ± 0.19   Final Test: 94.38 ± 0.16   Final old_old: 94.69 ± 0.16   Final old_new: 93.17 ± 0.26   Final new_new: 90.57 ± 0.83
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.96 ± 1.01   Final Test: 22.68 ± 2.46   Final old_old: 25.37 ± 2.53   Final old_new: 11.64 ± 2.31   Final new_new: 2.27 ± 2.78
Hits@20:
  Final val: 51.24 ± 2.17   Final Test: 31.52 ± 2.09   Final old_old: 34.56 ± 2.35   Final old_new: 19.29 ± 1.39   Final new_new: 4.09 ± 3.73
Hits@30:
  Final val: 56.14 ± 1.32   Final Test: 37.44 ± 1.75   Final old_old: 40.32 ± 1.70   Final old_new: 26.05 ± 2.34   Final new_new: 7.73 ± 4.43
Hits@50:
  Final val: 64.13 ± 1.05   Final Test: 46.25 ± 1.98   Final old_old: 48.92 ± 2.37   Final old_new: 35.99 ± 1.47   Final new_new: 13.18 ± 6.51
AUC:
  Final val: 95.93 ± 0.24   Final Test: 95.31 ± 0.14   Final old_old: 95.81 ± 0.15   Final old_new: 93.39 ± 0.26   Final new_new: 89.52 ± 0.77
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 24.34 ± 2.13   Final Test: 16.03 ± 2.37   Final old_old: 17.01 ± 2.54   Final old_new: 12.17 ± 2.11   Final new_new: 5.91 ± 3.05
Hits@20:
  Final val: 31.00 ± 2.59   Final Test: 22.18 ± 2.16   Final old_old: 23.11 ± 2.21   Final old_new: 18.41 ± 2.24   Final new_new: 14.55 ± 4.43
Hits@30:
  Final val: 35.73 ± 2.26   Final Test: 25.86 ± 2.20   Final old_old: 26.86 ± 2.35   Final old_new: 21.84 ± 1.98   Final new_new: 16.36 ± 3.73
Hits@50:
  Final val: 43.95 ± 1.74   Final Test: 32.27 ± 1.69   Final old_old: 33.40 ± 1.77   Final old_new: 27.81 ± 2.14   Final new_new: 20.45 ± 2.78
AUC:
  Final val: 92.97 ± 0.18   Final Test: 92.56 ± 0.18   Final old_old: 92.67 ± 0.18   Final old_new: 92.23 ± 0.26   Final new_new: 89.39 ± 0.45
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 34.48 ± 1.15   Final Test: 20.19 ± 4.04   Final old_old: 22.27 ± 4.33   Final old_new: 11.81 ± 3.02   Final new_new: 1.82 ± 1.90
Hits@20:
  Final val: 40.84 ± 0.58   Final Test: 25.82 ± 5.12   Final old_old: 28.01 ± 5.29   Final old_new: 16.98 ± 4.63   Final new_new: 6.82 ± 1.61
Hits@30:
  Final val: 44.98 ± 0.42   Final Test: 31.46 ± 4.80   Final old_old: 33.66 ± 5.15   Final old_new: 22.59 ± 3.94   Final new_new: 11.82 ± 3.37
Hits@50:
  Final val: 53.26 ± 0.77   Final Test: 38.45 ± 3.05   Final old_old: 40.72 ± 3.36   Final old_new: 29.40 ± 2.18   Final new_new: 16.82 ± 3.45
AUC:
  Final val: 94.18 ± 0.27   Final Test: 93.61 ± 0.26   Final old_old: 93.92 ± 0.32   Final old_new: 92.45 ± 0.19   Final new_new: 89.36 ± 0.70
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 25.65 ± 0.83   Final Test: 19.03 ± 2.87   Final old_old: 20.04 ± 3.07   Final old_new: 15.01 ± 2.23   Final new_new: 8.64 ± 2.49
Hits@20:
  Final val: 32.39 ± 0.62   Final Test: 23.31 ± 3.00   Final old_old: 24.34 ± 3.36   Final old_new: 19.14 ± 1.57   Final new_new: 14.55 ± 3.80
Hits@30:
  Final val: 37.55 ± 1.01   Final Test: 28.68 ± 1.33   Final old_old: 29.72 ± 1.69   Final old_new: 24.38 ± 1.13   Final new_new: 21.36 ± 2.03
Hits@50:
  Final val: 45.53 ± 1.06   Final Test: 33.82 ± 0.71   Final old_old: 34.98 ± 0.75   Final old_new: 29.07 ± 1.48   Final new_new: 24.55 ± 4.66
AUC:
  Final val: 93.43 ± 0.15   Final Test: 93.08 ± 0.12   Final old_old: 93.15 ± 0.13   Final old_new: 92.89 ± 0.10   Final new_new: 90.80 ± 0.67
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.33 ± 5.35   Final Test: 32.20 ± 4.57   Final old_old: 35.25 ± 4.77   Final old_new: 19.92 ± 3.91   Final new_new: 4.09 ± 3.37
Hits@20:
  Final val: 62.76 ± 4.38   Final Test: 44.69 ± 4.37   Final old_old: 48.47 ± 4.65   Final old_new: 29.67 ± 3.73   Final new_new: 7.27 ± 2.49
Hits@30:
  Final val: 70.74 ± 2.71   Final Test: 51.93 ± 4.80   Final old_old: 55.89 ± 5.24   Final old_new: 36.25 ± 3.44   Final new_new: 12.27 ± 1.24
Hits@50:
  Final val: 79.72 ± 1.56   Final Test: 64.05 ± 2.20   Final old_old: 68.22 ± 2.30   Final old_new: 47.61 ± 2.12   Final new_new: 20.45 ± 4.25
AUC:
  Final val: 97.76 ± 0.06   Final Test: 97.31 ± 0.05   Final old_old: 97.78 ± 0.04   Final old_new: 95.45 ± 0.14   Final new_new: 92.00 ± 0.78
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.50 ± 8.23   Final Test: 28.28 ± 3.40   Final old_old: 30.51 ± 3.52   Final old_new: 19.27 ± 3.37   Final new_new: 8.18 ± 4.13
Hits@20:
  Final val: 54.52 ± 4.39   Final Test: 37.28 ± 4.33   Final old_old: 39.95 ± 4.62   Final old_new: 26.57 ± 3.46   Final new_new: 12.73 ± 6.55
Hits@30:
  Final val: 60.85 ± 2.70   Final Test: 46.15 ± 5.12   Final old_old: 49.13 ± 5.25   Final old_new: 34.33 ± 4.83   Final new_new: 15.91 ± 4.82
Hits@50:
  Final val: 68.56 ± 0.85   Final Test: 55.46 ± 5.08   Final old_old: 58.56 ± 5.14   Final old_new: 43.32 ± 5.28   Final new_new: 21.82 ± 5.93
AUC:
  Final val: 96.37 ± 0.16   Final Test: 95.89 ± 0.19   Final old_old: 96.31 ± 0.17   Final old_new: 94.21 ± 0.31   Final new_new: 91.65 ± 0.82
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.60 ± 8.85   Final Test: 25.48 ± 7.93   Final old_old: 28.13 ± 8.65   Final old_new: 14.61 ± 5.21   Final new_new: 5.45 ± 2.59
Hits@20:
  Final val: 52.36 ± 8.02   Final Test: 39.26 ± 9.82   Final old_old: 42.96 ± 10.74   Final old_new: 24.21 ± 6.43   Final new_new: 9.55 ± 3.73
Hits@30:
  Final val: 59.31 ± 5.98   Final Test: 46.70 ± 7.70   Final old_old: 50.48 ± 8.48   Final old_new: 31.34 ± 5.55   Final new_new: 15.00 ± 6.14
Hits@50:
  Final val: 68.61 ± 5.43   Final Test: 54.28 ± 6.13   Final old_old: 58.61 ± 6.83   Final old_new: 36.73 ± 3.81   Final new_new: 17.73 ± 2.96
AUC:
  Final val: 96.59 ± 0.47   Final Test: 95.65 ± 0.42   Final old_old: 96.54 ± 0.55   Final old_new: 92.04 ± 0.46   Final new_new: 88.18 ± 1.14
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=0.001, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.93 ± 2.30   Final Test: 26.10 ± 4.18   Final old_old: 28.31 ± 4.43   Final old_new: 17.41 ± 3.54   Final new_new: 2.73 ± 2.49
Hits@20:
  Final val: 54.37 ± 1.84   Final Test: 39.74 ± 7.37   Final old_old: 42.57 ± 7.83   Final old_new: 28.59 ± 5.69   Final new_new: 10.00 ± 4.71
Hits@30:
  Final val: 62.62 ± 2.37   Final Test: 52.76 ± 2.94   Final old_old: 55.97 ± 3.16   Final old_new: 40.23 ± 2.37   Final new_new: 17.27 ± 6.14
Hits@50:
  Final val: 72.98 ± 1.27   Final Test: 61.47 ± 1.91   Final old_old: 64.60 ± 1.96   Final old_new: 49.27 ± 1.88   Final new_new: 25.91 ± 4.13
AUC:
  Final val: 97.02 ± 0.10   Final Test: 96.81 ± 0.23   Final old_old: 97.24 ± 0.18   Final old_new: 95.19 ± 0.43   Final new_new: 91.56 ± 0.81
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 39.13 ± 2.16   Final Test: 16.85 ± 6.10   Final old_old: 18.82 ± 6.60   Final old_new: 8.79 ± 4.15   Final new_new: 1.36 ± 2.03
Hits@20:
  Final val: 46.90 ± 2.59   Final Test: 28.49 ± 2.40   Final old_old: 31.04 ± 2.88   Final old_new: 18.24 ± 1.34   Final new_new: 5.00 ± 4.37
Hits@30:
  Final val: 52.90 ± 1.86   Final Test: 36.75 ± 2.89   Final old_old: 39.42 ± 3.12   Final old_new: 26.25 ± 2.85   Final new_new: 8.18 ± 7.29
Hits@50:
  Final val: 60.14 ± 2.28   Final Test: 43.84 ± 4.38   Final old_old: 46.55 ± 4.68   Final old_new: 33.32 ± 3.75   Final new_new: 12.73 ± 6.55
AUC:
  Final val: 94.95 ± 0.47   Final Test: 94.37 ± 0.52   Final old_old: 94.84 ± 0.59   Final old_new: 92.55 ± 0.33   Final new_new: 88.34 ± 1.82
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 23.07 ± 2.02   Final Test: 11.75 ± 1.80   Final old_old: 12.63 ± 1.78   Final old_new: 8.04 ± 1.85   Final new_new: 7.27 ± 7.61
Hits@20:
  Final val: 29.92 ± 1.22   Final Test: 19.29 ± 1.93   Final old_old: 20.28 ± 2.00   Final old_new: 15.16 ± 1.98   Final new_new: 12.73 ± 7.29
Hits@30:
  Final val: 34.48 ± 1.16   Final Test: 22.77 ± 2.35   Final old_old: 23.85 ± 2.37   Final old_new: 18.31 ± 2.27   Final new_new: 15.45 ± 5.43
Hits@50:
  Final val: 41.97 ± 1.29   Final Test: 29.63 ± 2.62   Final old_old: 30.74 ± 2.45   Final old_new: 24.94 ± 3.79   Final new_new: 23.64 ± 6.93
AUC:
  Final val: 92.44 ± 0.34   Final Test: 92.09 ± 0.25   Final old_old: 92.18 ± 0.18   Final old_new: 91.82 ± 0.57   Final new_new: 89.74 ± 0.66
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.75 ± 3.63   Final Test: 26.74 ± 4.53   Final old_old: 29.12 ± 4.77   Final old_new: 17.25 ± 3.69   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 59.21 ± 3.86   Final Test: 43.25 ± 3.63   Final old_old: 46.43 ± 3.90   Final old_new: 30.71 ± 2.85   Final new_new: 9.55 ± 1.02
Hits@30:
  Final val: 66.69 ± 3.07   Final Test: 51.69 ± 3.27   Final old_old: 55.39 ± 3.33   Final old_new: 37.15 ± 3.26   Final new_new: 12.27 ± 3.45
Hits@50:
  Final val: 75.39 ± 1.78   Final Test: 60.57 ± 0.64   Final old_old: 64.41 ± 0.51   Final old_new: 45.24 ± 1.31   Final new_new: 24.09 ± 6.55
AUC:
  Final val: 97.18 ± 0.15   Final Test: 96.66 ± 0.09   Final old_old: 97.23 ± 0.08   Final old_new: 94.35 ± 0.21   Final new_new: 91.44 ± 1.64
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 38.62 ± 0.43   Final Test: 23.04 ± 2.83   Final old_old: 25.18 ± 3.14   Final old_new: 14.43 ± 2.10   Final new_new: 4.09 ± 2.96
Hits@20:
  Final val: 44.70 ± 0.63   Final Test: 30.02 ± 2.98   Final old_old: 32.10 ± 3.48   Final old_new: 21.71 ± 1.14   Final new_new: 10.00 ± 2.59
Hits@30:
  Final val: 48.84 ± 0.59   Final Test: 33.49 ± 3.05   Final old_old: 35.77 ± 3.31   Final old_new: 24.38 ± 2.04   Final new_new: 11.36 ± 2.27
Hits@50:
  Final val: 55.82 ± 0.65   Final Test: 41.72 ± 1.65   Final old_old: 43.89 ± 1.91   Final old_new: 33.07 ± 1.06   Final new_new: 20.45 ± 4.25
AUC:
  Final val: 94.56 ± 0.19   Final Test: 94.19 ± 0.16   Final old_old: 94.48 ± 0.13   Final old_new: 93.07 ± 0.29   Final new_new: 90.55 ± 0.68
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.78 ± 3.77   Final Test: 28.95 ± 2.26   Final old_old: 31.85 ± 2.21   Final old_new: 17.23 ± 2.60   Final new_new: 3.64 ± 3.45
Hits@20:
  Final val: 58.35 ± 2.96   Final Test: 43.12 ± 1.69   Final old_old: 46.66 ± 1.64   Final old_new: 28.94 ± 2.26   Final new_new: 9.55 ± 3.73
Hits@30:
  Final val: 65.31 ± 1.64   Final Test: 49.84 ± 0.45   Final old_old: 53.52 ± 0.41   Final old_new: 35.29 ± 1.65   Final new_new: 11.36 ± 2.27
Hits@50:
  Final val: 72.94 ± 0.36   Final Test: 59.77 ± 3.48   Final old_old: 63.33 ± 3.64   Final old_new: 45.72 ± 2.84   Final new_new: 21.82 ± 6.14
AUC:
  Final val: 96.79 ± 0.06   Final Test: 96.45 ± 0.09   Final old_old: 96.86 ± 0.08   Final old_new: 94.80 ± 0.25   Final new_new: 92.45 ± 0.94
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 40.64 ± 4.84   Final Test: 23.04 ± 3.84   Final old_old: 25.84 ± 4.43   Final old_new: 11.59 ± 1.66   Final new_new: 0.91 ± 2.03
Hits@20:
  Final val: 52.42 ± 6.09   Final Test: 37.18 ± 5.17   Final old_old: 41.10 ± 5.76   Final old_new: 21.31 ± 3.64   Final new_new: 3.64 ± 4.71
Hits@30:
  Final val: 61.15 ± 3.90   Final Test: 43.46 ± 3.25   Final old_old: 47.92 ± 3.59   Final old_new: 25.49 ± 2.68   Final new_new: 3.18 ± 4.71
Hits@50:
  Final val: 70.95 ± 2.35   Final Test: 56.41 ± 3.69   Final old_old: 61.25 ± 4.06   Final old_new: 37.13 ± 2.35   Final new_new: 9.55 ± 4.93
AUC:
  Final val: 97.23 ± 0.19   Final Test: 96.35 ± 0.22   Final old_old: 97.25 ± 0.22   Final old_new: 92.75 ± 0.29   Final new_new: 87.87 ± 1.07
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 23.78 ± 1.57   Final Test: 16.79 ± 1.54   Final old_old: 17.58 ± 1.56   Final old_new: 13.50 ± 1.62   Final new_new: 11.36 ± 2.27
Hits@20:
  Final val: 29.20 ± 1.68   Final Test: 22.60 ± 0.70   Final old_old: 23.39 ± 0.75   Final old_new: 19.24 ± 1.22   Final new_new: 18.64 ± 2.49
Hits@30:
  Final val: 33.19 ± 1.74   Final Test: 25.85 ± 1.82   Final old_old: 26.56 ± 1.79   Final old_new: 22.82 ± 2.09   Final new_new: 23.18 ± 3.73
Hits@50:
  Final val: 40.93 ± 2.43   Final Test: 31.08 ± 1.96   Final old_old: 31.92 ± 2.10   Final old_new: 27.43 ± 1.48   Final new_new: 27.73 ± 2.96
AUC:
  Final val: 92.91 ± 0.17   Final Test: 92.57 ± 0.24   Final old_old: 92.62 ± 0.25   Final old_new: 92.46 ± 0.23   Final new_new: 90.68 ± 0.72
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.08 ± 1.76   Final Test: 22.16 ± 4.37   Final old_old: 24.68 ± 4.61   Final old_new: 11.91 ± 3.54   Final new_new: 1.36 ± 1.24
Hits@20:
  Final val: 50.55 ± 1.72   Final Test: 30.47 ± 3.44   Final old_old: 33.33 ± 3.32   Final old_new: 19.02 ± 4.19   Final new_new: 3.64 ± 3.05
Hits@30:
  Final val: 56.29 ± 1.49   Final Test: 37.01 ± 2.10   Final old_old: 39.96 ± 2.29   Final old_new: 25.24 ± 1.67   Final new_new: 8.18 ± 3.45
Hits@50:
  Final val: 64.22 ± 1.10   Final Test: 47.57 ± 3.19   Final old_old: 50.23 ± 3.21   Final old_new: 37.33 ± 3.42   Final new_new: 15.00 ± 2.59
AUC:
  Final val: 95.91 ± 0.19   Final Test: 95.39 ± 0.13   Final old_old: 95.86 ± 0.16   Final old_new: 93.56 ± 0.15   Final new_new: 89.73 ± 1.10
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 35.02 ± 1.13   Final Test: 17.82 ± 5.56   Final old_old: 19.68 ± 5.97   Final old_new: 10.20 ± 3.84   Final new_new: 3.64 ± 4.13
Hits@20:
  Final val: 41.86 ± 1.06   Final Test: 24.80 ± 4.54   Final old_old: 26.88 ± 4.90   Final old_new: 16.47 ± 3.52   Final new_new: 5.00 ± 1.90
Hits@30:
  Final val: 46.70 ± 1.22   Final Test: 30.78 ± 2.01   Final old_old: 33.14 ± 2.07   Final old_new: 21.44 ± 2.19   Final new_new: 6.36 ± 4.37
Hits@50:
  Final val: 53.23 ± 1.25   Final Test: 39.25 ± 2.50   Final old_old: 41.31 ± 2.87   Final old_new: 31.06 ± 2.30   Final new_new: 18.18 ± 5.79
AUC:
  Final val: 94.20 ± 0.22   Final Test: 93.51 ± 0.19   Final old_old: 93.81 ± 0.25   Final old_new: 92.41 ± 0.28   Final new_new: 88.87 ± 1.37
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 52.16 ± 8.04   Final Test: 35.10 ± 6.33   Final old_old: 38.37 ± 6.98   Final old_new: 22.04 ± 4.07   Final new_new: 3.64 ± 2.59
Hits@20:
  Final val: 64.53 ± 3.48   Final Test: 49.34 ± 2.60   Final old_old: 53.07 ± 3.19   Final old_new: 34.71 ± 1.93   Final new_new: 9.09 ± 3.59
Hits@30:
  Final val: 70.78 ± 2.21   Final Test: 54.03 ± 1.77   Final old_old: 58.14 ± 2.01   Final old_new: 37.81 ± 1.75   Final new_new: 11.36 ± 5.57
Hits@50:
  Final val: 78.12 ± 1.36   Final Test: 63.57 ± 2.02   Final old_old: 67.78 ± 2.21   Final old_new: 46.93 ± 1.76   Final new_new: 20.45 ± 5.08
AUC:
  Final val: 97.73 ± 0.11   Final Test: 97.29 ± 0.13   Final old_old: 97.74 ± 0.13   Final old_new: 95.52 ± 0.15   Final new_new: 92.59 ± 0.94
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.05 ± 6.79   Final Test: 29.93 ± 6.04   Final old_old: 32.23 ± 6.32   Final old_new: 20.50 ± 5.18   Final new_new: 11.36 ± 2.78
Hits@20:
  Final val: 53.95 ± 2.86   Final Test: 40.80 ± 3.11   Final old_old: 43.44 ± 3.33   Final old_new: 30.35 ± 2.40   Final new_new: 13.64 ± 0.00
Hits@30:
  Final val: 61.21 ± 1.30   Final Test: 46.60 ± 2.83   Final old_old: 49.54 ± 2.76   Final old_new: 35.09 ± 3.56   Final new_new: 15.00 ± 4.98
Hits@50:
  Final val: 68.67 ± 0.94   Final Test: 56.02 ± 2.57   Final old_old: 58.89 ± 2.65   Final old_new: 44.86 ± 2.91   Final new_new: 23.18 ± 3.73
AUC:
  Final val: 96.35 ± 0.08   Final Test: 96.02 ± 0.11   Final old_old: 96.45 ± 0.09   Final old_new: 94.31 ± 0.20   Final new_new: 91.82 ± 0.66
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 21.75 ± 2.08   Final Test: 13.69 ± 1.41   Final old_old: 14.39 ± 1.21   Final old_new: 10.86 ± 2.40   Final new_new: 7.27 ± 2.96
Hits@20:
  Final val: 28.34 ± 1.26   Final Test: 19.86 ± 0.91   Final old_old: 20.70 ± 0.78   Final old_new: 16.35 ± 2.12   Final new_new: 14.09 ± 2.96
Hits@30:
  Final val: 33.33 ± 1.20   Final Test: 22.03 ± 1.49   Final old_old: 22.83 ± 1.42   Final old_new: 18.56 ± 2.39   Final new_new: 18.64 ± 1.90
Hits@50:
  Final val: 40.45 ± 1.06   Final Test: 31.51 ± 1.64   Final old_old: 32.50 ± 1.83   Final old_new: 27.48 ± 2.00   Final new_new: 24.09 ± 3.45
AUC:
  Final val: 92.38 ± 0.37   Final Test: 91.98 ± 0.26   Final old_old: 92.06 ± 0.25   Final old_new: 91.76 ± 0.36   Final new_new: 89.69 ± 0.65
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.82 ± 4.34   Final Test: 23.78 ± 2.34   Final old_old: 25.35 ± 2.46   Final old_new: 17.71 ± 2.19   Final new_new: 5.45 ± 2.03
Hits@20:
  Final val: 56.59 ± 2.10   Final Test: 37.07 ± 5.01   Final old_old: 39.38 ± 4.99   Final old_new: 27.96 ± 5.24   Final new_new: 13.18 ± 3.37
Hits@30:
  Final val: 62.11 ± 1.72   Final Test: 41.94 ± 4.99   Final old_old: 44.44 ± 5.05   Final old_new: 32.09 ± 5.09   Final new_new: 15.45 ± 4.93
Hits@50:
  Final val: 68.89 ± 1.26   Final Test: 53.06 ± 2.47   Final old_old: 55.71 ± 2.42   Final old_new: 42.75 ± 2.85   Final new_new: 22.27 ± 3.73
AUC:
  Final val: 96.03 ± 0.22   Final Test: 95.52 ± 0.20   Final old_old: 95.95 ± 0.19   Final old_new: 93.91 ± 0.28   Final new_new: 89.78 ± 1.08
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 37.03 ± 2.34   Final Test: 18.36 ± 4.65   Final old_old: 20.53 ± 5.00   Final old_new: 9.52 ± 3.26   Final new_new: 0.91 ± 1.24
Hits@20:
  Final val: 44.89 ± 2.85   Final Test: 27.55 ± 5.20   Final old_old: 30.01 ± 5.39   Final old_new: 17.68 ± 4.65   Final new_new: 4.55 ± 3.21
Hits@30:
  Final val: 50.07 ± 2.30   Final Test: 33.90 ± 4.54   Final old_old: 36.55 ± 4.70   Final old_new: 23.27 ± 4.06   Final new_new: 9.09 ± 3.59
Hits@50:
  Final val: 58.37 ± 1.23   Final Test: 43.35 ± 2.02   Final old_old: 46.14 ± 2.01   Final old_new: 32.37 ± 2.14   Final new_new: 14.09 ± 4.93
AUC:
  Final val: 94.91 ± 0.46   Final Test: 94.39 ± 0.44   Final old_old: 94.92 ± 0.51   Final old_new: 92.39 ± 0.23   Final new_new: 87.84 ± 1.38
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 23.27 ± 1.44   Final Test: 15.77 ± 1.46   Final old_old: 16.48 ± 1.47   Final old_new: 12.95 ± 1.58   Final new_new: 8.64 ± 4.07
Hits@20:
  Final val: 29.02 ± 1.73   Final Test: 21.52 ± 1.41   Final old_old: 22.22 ± 1.57   Final old_new: 18.49 ± 1.25   Final new_new: 19.55 ± 3.45
Hits@30:
  Final val: 33.29 ± 1.71   Final Test: 25.52 ± 1.81   Final old_old: 26.25 ± 2.00   Final old_new: 22.47 ± 1.36   Final new_new: 21.36 ± 2.03
Hits@50:
  Final val: 41.15 ± 1.48   Final Test: 31.03 ± 2.04   Final old_old: 31.87 ± 1.95   Final old_new: 27.43 ± 2.48   Final new_new: 27.27 ± 4.25
AUC:
  Final val: 92.90 ± 0.06   Final Test: 92.44 ± 0.12   Final old_old: 92.48 ± 0.12   Final old_new: 92.35 ± 0.16   Final new_new: 90.76 ± 0.86
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 38.00 ± 1.84   Final Test: 23.53 ± 2.89   Final old_old: 25.63 ± 2.97   Final old_new: 15.06 ± 2.75   Final new_new: 4.55 ± 2.27
Hits@20:
  Final val: 44.78 ± 0.70   Final Test: 27.23 ± 3.06   Final old_old: 29.33 ± 3.15   Final old_new: 18.69 ± 3.02   Final new_new: 9.55 ± 5.88
Hits@30:
  Final val: 49.16 ± 0.81   Final Test: 34.67 ± 1.45   Final old_old: 36.80 ± 1.59   Final old_new: 26.25 ± 1.52   Final new_new: 12.27 ± 3.80
Hits@50:
  Final val: 55.64 ± 0.74   Final Test: 41.49 ± 2.50   Final old_old: 43.47 ± 2.62   Final old_new: 33.78 ± 2.85   Final new_new: 19.09 ± 5.47
AUC:
  Final val: 94.55 ± 0.24   Final Test: 94.05 ± 0.17   Final old_old: 94.32 ± 0.19   Final old_new: 93.05 ± 0.28   Final new_new: 90.45 ± 0.64
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 47.49 ± 8.47   Final Test: 32.12 ± 3.46   Final old_old: 34.61 ± 3.72   Final old_new: 22.37 ± 2.60   Final new_new: 5.45 ± 2.03
Hits@20:
  Final val: 60.19 ± 3.54   Final Test: 44.34 ± 3.62   Final old_old: 47.55 ± 3.63   Final old_new: 31.61 ± 4.13   Final new_new: 11.36 ± 4.82
Hits@30:
  Final val: 67.43 ± 1.36   Final Test: 50.23 ± 4.85   Final old_old: 53.73 ± 4.80   Final old_new: 36.47 ± 5.35   Final new_new: 12.27 ± 6.55
Hits@50:
  Final val: 75.80 ± 1.01   Final Test: 61.93 ± 2.14   Final old_old: 65.78 ± 1.89   Final old_new: 46.52 ± 3.47   Final new_new: 25.00 ± 4.55
AUC:
  Final val: 97.16 ± 0.06   Final Test: 96.79 ± 0.08   Final old_old: 97.30 ± 0.10   Final old_new: 94.77 ± 0.16   Final new_new: 91.04 ± 0.85
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GCN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 46.14 ± 5.86   Final Test: 34.88 ± 3.33   Final old_old: 38.09 ± 3.58   Final old_new: 21.91 ± 2.70   Final new_new: 6.36 ± 3.73
Hits@20:
  Final val: 59.74 ± 4.05   Final Test: 45.75 ± 2.71   Final old_old: 49.14 ± 3.04   Final old_new: 32.14 ± 1.88   Final new_new: 14.09 ± 4.93
Hits@30:
  Final val: 66.20 ± 2.01   Final Test: 51.03 ± 3.35   Final old_old: 54.63 ± 3.34   Final old_new: 36.62 ± 3.53   Final new_new: 16.36 ± 4.66
Hits@50:
  Final val: 73.14 ± 2.07   Final Test: 58.85 ± 2.52   Final old_old: 62.45 ± 2.31   Final old_new: 44.48 ± 3.48   Final new_new: 24.09 ± 9.03
AUC:
  Final val: 96.79 ± 0.08   Final Test: 96.50 ± 0.04   Final old_old: 96.88 ± 0.08   Final old_new: 94.98 ± 0.15   Final new_new: 92.30 ± 0.52
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.54 ± 9.21   Final Test: 24.92 ± 6.61   Final old_old: 27.40 ± 7.20   Final old_new: 14.71 ± 4.34   Final new_new: 5.91 ± 3.80
Hits@20:
  Final val: 52.14 ± 6.91   Final Test: 37.22 ± 6.20   Final old_old: 40.53 ± 6.94   Final old_new: 23.73 ± 3.44   Final new_new: 10.45 ± 2.59
Hits@30:
  Final val: 58.94 ± 6.08   Final Test: 43.24 ± 5.89   Final old_old: 47.03 ± 6.48   Final old_new: 27.83 ± 3.53   Final new_new: 11.36 ± 3.59
Hits@50:
  Final val: 68.24 ± 4.47   Final Test: 53.95 ± 3.92   Final old_old: 58.14 ± 4.42   Final old_new: 36.95 ± 2.75   Final new_new: 18.64 ± 5.66
AUC:
  Final val: 96.54 ± 0.41   Final Test: 95.59 ± 0.40   Final old_old: 96.48 ± 0.45   Final old_new: 92.02 ± 0.22   Final new_new: 87.12 ± 0.96
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=0.001, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 21.29 ± 1.83   Final Test: 13.53 ± 0.95   Final old_old: 14.21 ± 0.88   Final old_new: 10.76 ± 1.45   Final new_new: 8.18 ± 3.05
Hits@20:
  Final val: 27.80 ± 1.44   Final Test: 18.03 ± 1.40   Final old_old: 18.68 ± 1.40   Final old_new: 15.54 ± 1.75   Final new_new: 10.45 ± 5.23
Hits@30:
  Final val: 32.51 ± 1.57   Final Test: 22.68 ± 2.15   Final old_old: 23.58 ± 2.06   Final old_new: 18.97 ± 2.73   Final new_new: 15.91 ± 3.59
Hits@50:
  Final val: 40.04 ± 0.65   Final Test: 29.93 ± 1.10   Final old_old: 30.86 ± 0.90   Final old_new: 26.15 ± 2.24   Final new_new: 22.73 ± 3.59
AUC:
  Final val: 92.25 ± 0.28   Final Test: 91.89 ± 0.31   Final old_old: 91.96 ± 0.29   Final old_new: 91.70 ± 0.40   Final new_new: 89.23 ± 0.36
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_CN_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.15 ± 3.08   Final Test: 27.34 ± 3.55   Final old_old: 29.69 ± 3.90   Final old_new: 18.06 ± 2.60   Final new_new: 3.18 ± 2.03
Hits@20:
  Final val: 55.18 ± 2.60   Final Test: 40.27 ± 6.85   Final old_old: 43.21 ± 7.29   Final old_new: 28.66 ± 5.39   Final new_new: 9.09 ± 1.61
Hits@30:
  Final val: 63.04 ± 2.69   Final Test: 51.04 ± 4.28   Final old_old: 54.42 ± 4.39   Final old_new: 37.78 ± 4.23   Final new_new: 14.09 ± 5.88
Hits@50:
  Final val: 72.73 ± 2.43   Final Test: 60.62 ± 3.69   Final old_old: 63.78 ± 3.96   Final old_new: 48.31 ± 3.29   Final new_new: 25.00 ± 5.57
AUC:
  Final val: 97.02 ± 0.20   Final Test: 96.77 ± 0.22   Final old_old: 97.23 ± 0.24   Final old_new: 95.01 ± 0.33   Final new_new: 90.65 ± 0.88
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 35.75 ± 1.84   Final Test: 17.75 ± 4.37   Final old_old: 19.42 ± 4.63   Final old_new: 10.96 ± 3.45   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 42.03 ± 1.25   Final Test: 26.11 ± 5.18   Final old_old: 28.28 ± 5.44   Final old_new: 17.36 ± 4.27   Final new_new: 6.82 ± 3.59
Hits@30:
  Final val: 46.87 ± 0.74   Final Test: 30.25 ± 4.88   Final old_old: 32.44 ± 4.97   Final old_new: 21.56 ± 4.93   Final new_new: 8.64 ± 1.02
Hits@50:
  Final val: 54.22 ± 0.82   Final Test: 37.87 ± 1.78   Final old_old: 39.97 ± 1.80   Final old_new: 29.60 ± 2.01   Final new_new: 15.45 ± 4.07
AUC:
  Final val: 94.16 ± 0.23   Final Test: 93.53 ± 0.33   Final old_old: 93.80 ± 0.36   Final old_new: 92.53 ± 0.25   Final new_new: 89.50 ± 0.70
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.48 ± 3.58   Final Test: 28.75 ± 4.49   Final old_old: 30.95 ± 4.97   Final old_new: 20.00 ± 2.79   Final new_new: 7.27 ± 2.96
Hits@20:
  Final val: 55.23 ± 2.78   Final Test: 40.77 ± 5.80   Final old_old: 43.43 ± 6.10   Final old_new: 30.13 ± 4.93   Final new_new: 15.45 ± 3.37
Hits@30:
  Final val: 61.50 ± 1.57   Final Test: 47.55 ± 5.13   Final old_old: 50.32 ± 5.01   Final old_new: 36.70 ± 5.88   Final new_new: 17.27 ± 6.74
Hits@50:
  Final val: 69.51 ± 0.39   Final Test: 56.51 ± 2.75   Final old_old: 59.28 ± 2.65   Final old_new: 45.72 ± 3.49   Final new_new: 25.00 ± 6.63
AUC:
  Final val: 96.33 ± 0.08   Final Test: 95.98 ± 0.11   Final old_old: 96.40 ± 0.08   Final old_new: 94.32 ± 0.32   Final new_new: 91.47 ± 1.06
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_GAT_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 21.25 ± 1.56   Final Test: 15.37 ± 2.99   Final old_old: 15.88 ± 2.83   Final old_new: 13.35 ± 3.85   Final new_new: 10.45 ± 6.14
Hits@20:
  Final val: 27.91 ± 0.97   Final Test: 20.06 ± 1.77   Final old_old: 20.75 ± 1.73   Final old_new: 17.30 ± 2.08   Final new_new: 13.64 ± 7.36
Hits@30:
  Final val: 32.89 ± 1.06   Final Test: 24.90 ± 1.93   Final old_old: 25.45 ± 2.06   Final old_new: 22.49 ± 1.51   Final new_new: 23.18 ± 5.18
Hits@50:
  Final val: 40.95 ± 0.91   Final Test: 30.02 ± 0.91   Final old_old: 30.64 ± 0.95   Final old_new: 27.41 ± 1.57   Final new_new: 26.82 ± 2.49
AUC:
  Final val: 92.80 ± 0.06   Final Test: 92.36 ± 0.17   Final old_old: 92.39 ± 0.19   Final old_new: 92.33 ± 0.07   Final new_new: 90.80 ± 0.91
All runs:
Hits@10:
  Final val: 42.64 ± 0.65   Final Test: 22.55 ± 2.00   Final old_old: 25.20 ± 2.28   Final old_new: 11.76 ± 1.47   Final new_new: 0.91 ± 1.24
Hits@20:
  Final val: 51.52 ± 1.33   Final Test: 29.70 ± 3.76   Final old_old: 32.69 ± 3.78   Final old_new: 17.73 ± 3.91   Final new_new: 2.27 ± 1.61
Hits@30:
  Final val: 57.52 ± 1.53   Final Test: 37.28 ± 1.03   Final old_old: 40.24 ± 0.95   Final old_new: 25.59 ± 1.41   Final new_new: 6.82 ± 4.55
Hits@50:
  Final val: 65.65 ± 1.87   Final Test: 47.26 ± 2.12   Final old_old: 50.26 ± 2.20   Final old_new: 35.59 ± 2.42   Final new_new: 12.27 ± 4.43
AUC:
  Final val: 95.90 ± 0.18   Final Test: 95.28 ± 0.07   Final old_old: 95.77 ± 0.06   Final old_new: 93.40 ± 0.18   Final new_new: 89.18 ± 1.44
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 49.55 ± 4.40   Final Test: 37.21 ± 3.75   Final old_old: 40.73 ± 4.02   Final old_new: 23.22 ± 2.80   Final new_new: 2.27 ± 2.78
Hits@20:
  Final val: 62.68 ± 4.96   Final Test: 48.15 ± 5.97   Final old_old: 52.03 ± 6.38   Final old_new: 33.05 ± 4.59   Final new_new: 4.09 ± 2.96
Hits@30:
  Final val: 71.27 ± 1.70   Final Test: 54.06 ± 4.65   Final old_old: 58.11 ± 5.05   Final old_new: 38.26 ± 3.13   Final new_new: 8.18 ± 2.59
Hits@50:
  Final val: 78.98 ± 1.35   Final Test: 63.68 ± 2.05   Final old_old: 67.92 ± 2.16   Final old_new: 47.36 ± 2.25   Final new_new: 12.27 ± 1.24
AUC:
  Final val: 97.72 ± 0.08   Final Test: 97.29 ± 0.10   Final old_old: 97.75 ± 0.10   Final old_new: 95.52 ± 0.27   Final new_new: 91.89 ± 0.60
All runs:
Hits@10:
  Final val: 42.38 ± 2.16   Final Test: 28.81 ± 6.07   Final old_old: 32.31 ± 6.73   Final old_new: 14.43 ± 3.59   Final new_new: 1.82 ± 1.90
Hits@20:
  Final val: 54.53 ± 2.49   Final Test: 37.90 ± 3.58   Final old_old: 42.08 ± 4.01   Final old_new: 20.88 ± 1.78   Final new_new: 4.09 ± 2.49
Hits@30:
  Final val: 61.88 ± 2.72   Final Test: 46.50 ± 3.86   Final old_old: 51.18 ± 4.04   Final old_new: 27.53 ± 3.16   Final new_new: 5.91 ± 4.13
Hits@50:
  Final val: 72.71 ± 2.53   Final Test: 58.59 ± 1.39   Final old_old: 63.68 ± 1.54   Final old_new: 38.31 ± 1.52   Final new_new: 8.18 ± 4.71
AUC:
  Final val: 97.25 ± 0.20   Final Test: 96.48 ± 0.24   Final old_old: 97.37 ± 0.21   Final old_new: 92.89 ± 0.35   Final new_new: 88.57 ± 0.96
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.85 ± 3.84   Final Test: 30.95 ± 3.64   Final old_old: 33.81 ± 3.99   Final old_new: 19.62 ± 2.35   Final new_new: 2.27 ± 1.61
Hits@20:
  Final val: 57.54 ± 2.80   Final Test: 45.70 ± 3.27   Final old_old: 48.81 ± 3.19   Final old_new: 33.43 ± 4.11   Final new_new: 12.73 ± 8.29
Hits@30:
  Final val: 64.96 ± 2.02   Final Test: 52.39 ± 3.61   Final old_old: 55.80 ± 3.85   Final old_new: 39.04 ± 3.35   Final new_new: 14.55 ± 4.13
Hits@50:
  Final val: 72.82 ± 1.54   Final Test: 60.12 ± 3.09   Final old_old: 63.47 ± 3.28   Final old_new: 47.05 ± 3.02   Final new_new: 22.27 ± 1.90
AUC:
  Final val: 96.85 ± 0.07   Final Test: 96.49 ± 0.08   Final old_old: 96.87 ± 0.10   Final old_new: 94.99 ± 0.12   Final new_new: 92.64 ± 0.69
All runs:
Hits@10:
  Final val: 38.50 ± 1.40   Final Test: 22.67 ± 2.28   Final old_old: 24.67 ± 2.27   Final old_new: 14.69 ± 2.81   Final new_new: 3.64 ± 4.43
Hits@20:
  Final val: 44.77 ± 1.48   Final Test: 27.75 ± 2.69   Final old_old: 29.76 ± 2.89   Final old_new: 19.72 ± 2.64   Final new_new: 9.09 ± 2.27
Hits@30:
  Final val: 48.84 ± 0.90   Final Test: 34.02 ± 3.18   Final old_old: 36.24 ± 3.11   Final old_new: 25.16 ± 4.38   Final new_new: 12.73 ± 2.59
Hits@50:
  Final val: 56.05 ± 0.58   Final Test: 41.58 ± 0.92   Final old_old: 43.87 ± 1.10   Final old_new: 32.49 ± 0.99   Final new_new: 18.18 ± 1.61
AUC:
  Final val: 94.48 ± 0.27   Final Test: 94.01 ± 0.35   Final old_old: 94.28 ± 0.37   Final old_new: 93.03 ± 0.29   Final new_new: 90.17 ± 1.24
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 38.30 ± 2.71   Final Test: 17.48 ± 4.86   Final old_old: 19.40 ± 5.25   Final old_new: 9.70 ± 3.48   Final new_new: 0.91 ± 1.24
Hits@20:
  Final val: 45.49 ± 2.44   Final Test: 27.75 ± 1.79   Final old_old: 30.33 ± 1.83   Final old_new: 17.43 ± 1.97   Final new_new: 3.64 ± 2.03
Hits@30:
  Final val: 51.05 ± 2.49   Final Test: 34.51 ± 1.95   Final old_old: 37.09 ± 2.09   Final old_new: 24.33 ± 1.88   Final new_new: 7.27 ± 2.96
Hits@50:
  Final val: 58.43 ± 1.39   Final Test: 45.32 ± 2.60   Final old_old: 48.14 ± 2.59   Final old_new: 34.43 ± 2.73   Final new_new: 11.82 ± 3.73
AUC:
  Final val: 94.84 ± 0.24   Final Test: 94.18 ± 0.27   Final old_old: 94.69 ± 0.37   Final old_new: 92.29 ± 0.17   Final new_new: 87.13 ± 2.25
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.87 ± 3.84   Final Test: 26.20 ± 3.72   Final old_old: 27.94 ± 4.07   Final old_new: 19.45 ± 2.28   Final new_new: 5.91 ± 4.71
Hits@20:
  Final val: 50.72 ± 4.48   Final Test: 34.57 ± 5.65   Final old_old: 36.59 ± 6.05   Final old_new: 26.90 ± 4.32   Final new_new: 8.18 ± 3.45
Hits@30:
  Final val: 57.20 ± 3.81   Final Test: 41.39 ± 2.83   Final old_old: 43.52 ± 3.10   Final old_new: 33.30 ± 1.93   Final new_new: 13.64 ± 2.78
Hits@50:
  Final val: 65.18 ± 2.53   Final Test: 52.18 ± 1.72   Final old_old: 54.67 ± 1.87   Final old_new: 42.80 ± 1.56   Final new_new: 18.64 ± 4.07
AUC:
  Final val: 95.86 ± 0.11   Final Test: 95.41 ± 0.10   Final old_old: 95.76 ± 0.12   Final old_new: 94.13 ± 0.10   Final new_new: 89.89 ± 0.76
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 31.78 ± 1.95   Final Test: 16.89 ± 3.58   Final old_old: 18.37 ± 3.78   Final old_new: 10.91 ± 2.97   Final new_new: 3.64 ± 1.24
Hits@20:
  Final val: 37.36 ± 1.76   Final Test: 23.06 ± 1.07   Final old_old: 24.79 ± 1.17   Final old_new: 16.07 ± 0.66   Final new_new: 8.18 ± 2.03
Hits@30:
  Final val: 42.09 ± 1.21   Final Test: 27.99 ± 3.32   Final old_old: 29.61 ± 3.46   Final old_new: 21.51 ± 2.76   Final new_new: 12.27 ± 6.35
Hits@50:
  Final val: 48.55 ± 1.16   Final Test: 32.17 ± 3.89   Final old_old: 33.80 ± 3.85   Final old_new: 25.79 ± 4.27   Final new_new: 14.09 ± 3.37
AUC:
  Final val: 93.04 ± 0.16   Final Test: 92.50 ± 0.24   Final old_old: 92.70 ± 0.30   Final old_new: 91.82 ± 0.20   Final new_new: 89.12 ± 1.03
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 30.76 ± 2.72   Final Test: 23.84 ± 8.12   Final old_old: 26.11 ± 8.93   Final old_new: 14.58 ± 5.05   Final new_new: 5.45 ± 3.05
Hits@20:
  Final val: 42.62 ± 5.01   Final Test: 30.62 ± 6.43   Final old_old: 33.30 ± 7.10   Final old_new: 19.75 ± 3.86   Final new_new: 7.27 ± 3.37
Hits@30:
  Final val: 51.57 ± 5.80   Final Test: 39.88 ± 6.08   Final old_old: 43.16 ± 6.57   Final old_new: 26.65 ± 4.56   Final new_new: 10.91 ± 4.07
Hits@50:
  Final val: 64.06 ± 5.79   Final Test: 48.22 ± 5.61   Final old_old: 51.91 ± 6.13   Final old_new: 33.35 ± 4.15   Final new_new: 15.00 ± 1.24
AUC:
  Final val: 96.35 ± 0.47   Final Test: 95.37 ± 0.46   Final old_old: 96.23 ± 0.51   Final old_new: 91.85 ± 0.30   Final new_new: 88.13 ± 0.38
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=0.001, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.45 ± 1.89   Final Test: 23.78 ± 0.99   Final old_old: 26.38 ± 1.14   Final old_new: 13.25 ± 1.41   Final new_new: 0.91 ± 1.24
Hits@20:
  Final val: 49.06 ± 2.54   Final Test: 32.42 ± 1.57   Final old_old: 35.20 ± 1.68   Final old_new: 21.36 ± 1.62   Final new_new: 4.55 ± 2.27
Hits@30:
  Final val: 56.24 ± 0.97   Final Test: 39.40 ± 1.26   Final old_old: 42.24 ± 1.08   Final old_new: 28.36 ± 2.54   Final new_new: 6.82 ± 2.78
Hits@50:
  Final val: 63.12 ± 1.61   Final Test: 46.13 ± 2.21   Final old_old: 48.94 ± 2.20   Final old_new: 35.31 ± 2.37   Final new_new: 11.36 ± 5.79
AUC:
  Final val: 95.56 ± 0.15   Final Test: 95.17 ± 0.20   Final old_old: 95.63 ± 0.16   Final old_new: 93.41 ± 0.36   Final new_new: 89.56 ± 0.83
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 31.65 ± 1.18   Final Test: 19.86 ± 2.98   Final old_old: 21.42 ± 3.08   Final old_new: 13.45 ± 2.70   Final new_new: 7.73 ± 2.03
Hits@20:
  Final val: 37.03 ± 2.36   Final Test: 26.79 ± 1.67   Final old_old: 28.42 ± 1.42   Final old_new: 20.08 ± 2.87   Final new_new: 14.55 ± 4.71
Hits@30:
  Final val: 41.51 ± 1.21   Final Test: 32.52 ± 0.79   Final old_old: 34.19 ± 0.61   Final old_new: 25.54 ± 1.88   Final new_new: 21.82 ± 6.35
Hits@50:
  Final val: 48.44 ± 0.77   Final Test: 37.58 ± 3.15   Final old_old: 38.99 ± 3.25   Final old_new: 31.66 ± 3.13   Final new_new: 29.09 ± 5.18
AUC:
  Final val: 93.63 ± 0.21   Final Test: 93.24 ± 0.22   Final old_old: 93.41 ± 0.26   Final old_new: 92.58 ± 0.16   Final new_new: 90.70 ± 0.91
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.31 ± 2.27   Final Test: 29.20 ± 7.49   Final old_old: 31.45 ± 7.89   Final old_new: 20.53 ± 6.11   Final new_new: 1.82 ± 1.90
Hits@20:
  Final val: 54.75 ± 3.39   Final Test: 40.09 ± 2.86   Final old_old: 42.89 ± 2.69   Final old_new: 29.12 ± 3.81   Final new_new: 9.55 ± 6.70
Hits@30:
  Final val: 62.14 ± 4.38   Final Test: 50.10 ± 4.38   Final old_old: 53.11 ± 4.66   Final old_new: 38.36 ± 4.26   Final new_new: 15.91 ± 5.33
Hits@50:
  Final val: 71.49 ± 2.91   Final Test: 62.88 ± 3.01   Final old_old: 66.01 ± 2.81   Final old_new: 50.55 ± 4.01   Final new_new: 29.55 ± 9.23
AUC:
  Final val: 97.01 ± 0.14   Final Test: 96.79 ± 0.10   Final old_old: 97.20 ± 0.14   Final old_new: 95.23 ± 0.25   Final new_new: 91.32 ± 0.58
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 31.15 ± 1.78   Final Test: 15.99 ± 2.68   Final old_old: 17.29 ± 2.83   Final old_new: 10.71 ± 2.28   Final new_new: 5.45 ± 1.24
Hits@20:
  Final val: 37.09 ± 1.52   Final Test: 22.27 ± 3.39   Final old_old: 23.76 ± 3.55   Final old_new: 16.30 ± 3.29   Final new_new: 8.64 ± 5.88
Hits@30:
  Final val: 41.13 ± 1.05   Final Test: 26.91 ± 3.26   Final old_old: 28.76 ± 3.24   Final old_new: 19.42 ± 3.64   Final new_new: 11.36 ± 5.79
Hits@50:
  Final val: 48.19 ± 0.94   Final Test: 35.31 ± 2.19   Final old_old: 37.01 ± 2.23   Final old_new: 28.39 ± 2.76   Final new_new: 21.36 ± 3.45
AUC:
  Final val: 92.83 ± 0.39   Final Test: 92.37 ± 0.29   Final old_old: 92.56 ± 0.31   Final old_new: 91.63 ± 0.41   Final new_new: 89.82 ± 0.35
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.34 ± 3.11   Final Test: 28.90 ± 3.24   Final old_old: 32.54 ± 3.56   Final old_new: 13.98 ± 2.01   Final new_new: 1.36 ± 2.03
Hits@20:
  Final val: 56.10 ± 2.83   Final Test: 36.75 ± 4.05   Final old_old: 40.93 ± 4.48   Final old_new: 19.72 ± 2.78   Final new_new: 2.27 ± 2.78
Hits@30:
  Final val: 62.80 ± 2.21   Final Test: 44.92 ± 4.74   Final old_old: 49.54 ± 4.97   Final old_new: 26.42 ± 4.05   Final new_new: 2.27 ± 2.78
Hits@50:
  Final val: 72.60 ± 1.04   Final Test: 58.54 ± 2.83   Final old_old: 63.60 ± 2.84   Final old_new: 38.39 ± 2.99   Final new_new: 9.09 ± 0.00
AUC:
  Final val: 97.28 ± 0.15   Final Test: 96.41 ± 0.17   Final old_old: 97.32 ± 0.15   Final old_new: 92.70 ± 0.25   Final new_new: 88.64 ± 0.98
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 38.39 ± 2.43   Final Test: 17.89 ± 4.15   Final old_old: 19.96 ± 4.62   Final old_new: 9.50 ± 2.40   Final new_new: 0.91 ± 1.24
Hits@20:
  Final val: 46.21 ± 2.15   Final Test: 28.96 ± 3.03   Final old_old: 31.60 ± 3.20   Final old_new: 18.39 ± 2.88   Final new_new: 4.55 ± 2.27
Hits@30:
  Final val: 52.02 ± 1.25   Final Test: 34.75 ± 2.44   Final old_old: 37.68 ± 2.54   Final old_new: 23.10 ± 2.36   Final new_new: 5.91 ± 2.59
Hits@50:
  Final val: 58.80 ± 1.14   Final Test: 45.11 ± 3.43   Final old_old: 48.07 ± 3.51   Final old_new: 33.45 ± 3.82   Final new_new: 14.55 ± 7.29
AUC:
  Final val: 94.94 ± 0.21   Final Test: 94.22 ± 0.25   Final old_old: 94.74 ± 0.31   Final old_new: 92.24 ± 0.19   Final new_new: 87.71 ± 1.47
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 30.17 ± 1.70   Final Test: 21.42 ± 3.17   Final old_old: 22.53 ± 3.11   Final old_new: 16.85 ± 3.53   Final new_new: 13.18 ± 6.31
Hits@20:
  Final val: 36.18 ± 0.95   Final Test: 26.58 ± 0.75   Final old_old: 28.03 ± 1.04   Final old_new: 20.58 ± 1.11   Final new_new: 16.82 ± 3.45
Hits@30:
  Final val: 40.81 ± 0.86   Final Test: 29.90 ± 2.86   Final old_old: 31.32 ± 3.15   Final old_new: 24.13 ± 2.10   Final new_new: 18.64 ± 4.07
Hits@50:
  Final val: 48.38 ± 0.98   Final Test: 37.00 ± 1.77   Final old_old: 38.56 ± 1.95   Final old_new: 30.50 ± 1.80   Final new_new: 26.36 ± 2.59
AUC:
  Final val: 93.54 ± 0.09   Final Test: 93.17 ± 0.10   Final old_old: 93.33 ± 0.10   Final old_new: 92.62 ± 0.13   Final new_new: 90.72 ± 0.43
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 39.85 ± 2.68   Final Test: 24.44 ± 1.63   Final old_old: 26.97 ± 2.06   Final old_new: 14.18 ± 1.84   Final new_new: 2.73 ± 1.90
Hits@20:
  Final val: 48.50 ± 2.65   Final Test: 29.10 ± 1.09   Final old_old: 31.82 ± 0.87   Final old_new: 18.16 ± 2.44   Final new_new: 4.09 ± 3.37
Hits@30:
  Final val: 54.04 ± 2.77   Final Test: 36.53 ± 2.19   Final old_old: 39.35 ± 2.06   Final old_new: 25.44 ± 3.20   Final new_new: 5.91 ± 6.14
Hits@50:
  Final val: 62.05 ± 2.63   Final Test: 45.34 ± 2.56   Final old_old: 48.02 ± 2.51   Final old_new: 35.06 ± 3.16   Final new_new: 12.27 ± 3.45
AUC:
  Final val: 95.52 ± 0.30   Final Test: 95.06 ± 0.27   Final old_old: 95.51 ± 0.29   Final old_new: 93.34 ± 0.34   Final new_new: 89.55 ± 1.16
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 29.51 ± 1.99   Final Test: 14.79 ± 2.70   Final old_old: 16.05 ± 2.94   Final old_new: 9.65 ± 1.85   Final new_new: 4.55 ± 4.25
Hits@20:
  Final val: 35.43 ± 1.92   Final Test: 21.67 ± 2.23   Final old_old: 23.07 ± 2.37   Final old_new: 16.10 ± 2.33   Final new_new: 8.18 ± 3.05
Hits@30:
  Final val: 39.94 ± 2.47   Final Test: 25.99 ± 4.75   Final old_old: 27.57 ± 4.73   Final old_new: 19.52 ± 5.32   Final new_new: 13.64 ± 4.82
Hits@50:
  Final val: 46.67 ± 2.20   Final Test: 33.08 ± 2.74   Final old_old: 34.57 ± 2.92   Final old_new: 27.08 ± 2.02   Final new_new: 19.55 ± 5.70
AUC:
  Final val: 92.85 ± 0.35   Final Test: 92.43 ± 0.24   Final old_old: 92.58 ± 0.24   Final old_new: 91.88 ± 0.33   Final new_new: 90.73 ± 0.77
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_AA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.71 ± 2.72   Final Test: 23.87 ± 4.84   Final old_old: 25.51 ± 5.18   Final old_new: 17.46 ± 3.87   Final new_new: 5.45 ± 2.03
Hits@20:
  Final val: 54.95 ± 2.81   Final Test: 37.81 ± 2.87   Final old_old: 40.30 ± 3.12   Final old_new: 28.14 ± 2.62   Final new_new: 9.55 ± 2.49
Hits@30:
  Final val: 61.16 ± 1.96   Final Test: 45.42 ± 3.30   Final old_old: 48.13 ± 3.35   Final old_new: 34.96 ± 3.45   Final new_new: 12.73 ± 5.47
Hits@50:
  Final val: 68.81 ± 1.35   Final Test: 53.31 ± 1.35   Final old_old: 56.16 ± 0.98   Final old_new: 42.17 ± 2.94   Final new_new: 20.91 ± 5.18
AUC:
  Final val: 95.99 ± 0.18   Final Test: 95.49 ± 0.18   Final old_old: 95.91 ± 0.16   Final old_new: 93.90 ± 0.28   Final new_new: 89.71 ± 1.23
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 38.78 ± 6.10   Final Test: 22.12 ± 6.72   Final old_old: 24.49 ± 7.34   Final old_new: 12.39 ± 4.41   Final new_new: 4.09 ± 2.49
Hits@20:
  Final val: 50.31 ± 4.27   Final Test: 33.41 ± 5.28   Final old_old: 36.51 ± 5.45   Final old_new: 20.93 ± 5.19   Final new_new: 5.45 ± 3.45
Hits@30:
  Final val: 57.26 ± 3.09   Final Test: 41.58 ± 4.94   Final old_old: 45.22 ± 5.25   Final old_new: 26.98 ± 4.16   Final new_new: 8.18 ± 2.59
Hits@50:
  Final val: 67.17 ± 2.59   Final Test: 52.30 ± 3.52   Final old_old: 56.25 ± 3.53   Final old_new: 36.68 ± 4.35   Final new_new: 10.91 ± 3.37
AUC:
  Final val: 96.48 ± 0.35   Final Test: 95.59 ± 0.42   Final old_old: 96.42 ± 0.40   Final old_new: 92.37 ± 0.63   Final new_new: 86.51 ± 2.19
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=1, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 29.65 ± 0.65   Final Test: 20.55 ± 2.10   Final old_old: 21.95 ± 2.28   Final old_new: 14.91 ± 1.54   Final new_new: 7.73 ± 2.59
Hits@20:
  Final val: 36.11 ± 1.10   Final Test: 24.64 ± 2.12   Final old_old: 26.08 ± 2.33   Final old_new: 18.82 ± 1.46   Final new_new: 12.27 ± 5.23
Hits@30:
  Final val: 40.13 ± 1.10   Final Test: 27.49 ± 1.29   Final old_old: 28.88 ± 1.19   Final old_new: 21.71 ± 1.68   Final new_new: 17.73 ± 6.51
Hits@50:
  Final val: 47.61 ± 1.00   Final Test: 34.74 ± 1.69   Final old_old: 36.09 ± 1.60   Final old_new: 29.24 ± 2.79   Final new_new: 24.09 ± 4.13
AUC:
  Final val: 93.36 ± 0.11   Final Test: 92.96 ± 0.09   Final old_old: 93.07 ± 0.10   Final old_new: 92.56 ± 0.24   Final new_new: 90.91 ± 0.49
All runs:
Hits@10:
  Final val: 38.16 ± 0.72   Final Test: 20.97 ± 3.56   Final old_old: 23.22 ± 3.94   Final old_new: 11.89 ± 2.19   Final new_new: 1.36 ± 1.24
Hits@20:
  Final val: 46.14 ± 1.66   Final Test: 28.44 ± 4.36   Final old_old: 30.97 ± 4.44   Final old_new: 18.29 ± 4.29   Final new_new: 5.00 ± 4.37
Hits@30:
  Final val: 51.84 ± 1.04   Final Test: 35.58 ± 2.69   Final old_old: 38.29 ± 2.68   Final old_new: 24.91 ± 3.35   Final new_new: 6.82 ± 5.33
Hits@50:
  Final val: 60.04 ± 0.91   Final Test: 43.63 ± 2.06   Final old_old: 46.39 ± 2.08   Final old_new: 33.00 ± 2.60   Final new_new: 10.45 ± 6.74
AUC:
  Final val: 94.93 ± 0.39   Final Test: 94.26 ± 0.33   Final old_old: 94.78 ± 0.44   Final old_new: 92.24 ± 0.10   Final new_new: 87.59 ± 1.71
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.02 ± 3.44   Final Test: 28.99 ± 2.34   Final old_old: 32.42 ± 2.45   Final old_new: 15.04 ± 2.48   Final new_new: 0.45 ± 1.02
Hits@20:
  Final val: 54.34 ± 3.56   Final Test: 39.35 ± 3.71   Final old_old: 43.65 ± 4.21   Final old_new: 21.91 ± 2.95   Final new_new: 3.18 ± 2.03
Hits@30:
  Final val: 62.65 ± 2.68   Final Test: 46.75 ± 4.17   Final old_old: 51.65 ± 4.33   Final old_new: 27.05 ± 3.98   Final new_new: 2.27 ± 2.78
Hits@50:
  Final val: 72.83 ± 2.03   Final Test: 57.57 ± 1.75   Final old_old: 62.68 ± 1.98   Final old_new: 37.23 ± 1.31   Final new_new: 6.82 ± 4.25
AUC:
  Final val: 97.28 ± 0.16   Final Test: 96.44 ± 0.18   Final old_old: 97.33 ± 0.17   Final old_new: 92.85 ± 0.27   Final new_new: 88.95 ± 1.08
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.80 ± 3.42   Final Test: 29.21 ± 4.23   Final old_old: 31.51 ± 4.56   Final old_new: 20.28 ± 3.53   Final new_new: 2.73 ± 2.96
Hits@20:
  Final val: 56.30 ± 3.80   Final Test: 42.92 ± 1.85   Final old_old: 46.05 ± 2.27   Final old_new: 30.86 ± 2.51   Final new_new: 5.45 ± 3.45
Hits@30:
  Final val: 63.83 ± 3.74   Final Test: 51.70 ± 5.17   Final old_old: 55.16 ± 5.90   Final old_new: 38.29 ± 2.71   Final new_new: 11.36 ± 7.71
Hits@50:
  Final val: 72.99 ± 3.64   Final Test: 62.25 ± 3.49   Final old_old: 65.97 ± 3.93   Final old_new: 47.68 ± 3.41   Final new_new: 21.82 ± 10.49
AUC:
  Final val: 97.08 ± 0.22   Final Test: 96.91 ± 0.20   Final old_old: 97.39 ± 0.26   Final old_new: 95.04 ± 0.11   Final new_new: 91.30 ± 0.75
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.47 ± 1.68   Final Test: 26.33 ± 4.09   Final old_old: 29.31 ± 3.98   Final old_new: 14.13 ± 4.83   Final new_new: 2.73 ± 2.96
Hits@20:
  Final val: 51.84 ± 3.43   Final Test: 33.76 ± 1.54   Final old_old: 36.92 ± 1.48   Final old_new: 21.06 ± 2.36   Final new_new: 5.00 ± 2.96
Hits@30:
  Final val: 57.95 ± 2.23   Final Test: 40.23 ± 3.36   Final old_old: 43.51 ± 3.46   Final old_new: 27.13 ± 3.58   Final new_new: 8.64 ± 4.93
Hits@50:
  Final val: 64.88 ± 2.12   Final Test: 48.09 ± 3.81   Final old_old: 51.13 ± 4.06   Final old_new: 36.27 ± 3.78   Final new_new: 13.64 ± 4.55
AUC:
  Final val: 95.80 ± 0.17   Final Test: 95.28 ± 0.04   Final old_old: 95.74 ± 0.09   Final old_new: 93.57 ± 0.22   Final new_new: 88.90 ± 1.78
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 35.94 ± 0.61   Final Test: 23.64 ± 3.64   Final old_old: 25.87 ± 3.80   Final old_new: 14.56 ± 2.92   Final new_new: 5.45 ± 4.43
Hits@20:
  Final val: 42.98 ± 1.56   Final Test: 25.41 ± 2.49   Final old_old: 27.47 ± 2.56   Final old_new: 17.10 ± 2.46   Final new_new: 7.27 ± 1.90
Hits@30:
  Final val: 47.59 ± 1.50   Final Test: 33.55 ± 3.08   Final old_old: 35.89 ± 3.18   Final old_new: 24.13 ± 2.66   Final new_new: 12.27 ± 4.13
Hits@50:
  Final val: 53.42 ± 1.24   Final Test: 38.34 ± 2.80   Final old_old: 40.18 ± 2.80   Final old_new: 31.08 ± 2.74   Final new_new: 19.09 ± 6.14
AUC:
  Final val: 93.97 ± 0.17   Final Test: 93.64 ± 0.17   Final old_old: 93.95 ± 0.18   Final old_new: 92.47 ± 0.32   Final new_new: 89.39 ± 1.09
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 39.71 ± 7.35   Final Test: 24.05 ± 5.39   Final old_old: 26.59 ± 6.03   Final old_new: 13.58 ± 2.92   Final new_new: 5.45 ± 2.03
Hits@20:
  Final val: 50.95 ± 5.12   Final Test: 30.84 ± 7.24   Final old_old: 33.76 ± 7.80   Final old_new: 18.89 ± 5.05   Final new_new: 7.73 ± 2.59
Hits@30:
  Final val: 58.90 ± 3.99   Final Test: 41.53 ± 3.69   Final old_old: 45.18 ± 3.73   Final old_new: 26.80 ± 3.61   Final new_new: 9.55 ± 4.66
Hits@50:
  Final val: 68.48 ± 3.76   Final Test: 54.21 ± 1.94   Final old_old: 58.49 ± 2.15   Final old_new: 37.10 ± 2.14   Final new_new: 13.64 ± 4.25
AUC:
  Final val: 96.52 ± 0.36   Final Test: 95.50 ± 0.33   Final old_old: 96.41 ± 0.33   Final old_new: 91.86 ± 0.65   Final new_new: 87.20 ± 1.46
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.70 ± 4.15   Final Test: 24.64 ± 6.19   Final old_old: 26.47 ± 6.73   Final old_new: 17.53 ± 4.13   Final new_new: 3.64 ± 2.59
Hits@20:
  Final val: 53.65 ± 1.47   Final Test: 37.34 ± 3.52   Final old_old: 39.64 ± 3.71   Final old_new: 28.44 ± 3.22   Final new_new: 9.55 ± 4.37
Hits@30:
  Final val: 60.12 ± 0.90   Final Test: 45.52 ± 2.70   Final old_old: 48.17 ± 2.71   Final old_new: 35.26 ± 2.76   Final new_new: 14.09 ± 5.66
Hits@50:
  Final val: 67.45 ± 0.55   Final Test: 53.12 ± 2.16   Final old_old: 55.78 ± 2.11   Final old_new: 42.85 ± 2.79   Final new_new: 21.36 ± 3.45
AUC:
  Final val: 95.95 ± 0.16   Final Test: 95.48 ± 0.08   Final old_old: 95.84 ± 0.11   Final old_new: 94.12 ± 0.38   Final new_new: 89.92 ± 1.52
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=1, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 36.92 ± 0.61   Final Test: 24.57 ± 2.47   Final old_old: 26.88 ± 2.47   Final old_new: 15.11 ± 2.44   Final new_new: 6.82 ± 6.63
Hits@20:
  Final val: 43.28 ± 0.75   Final Test: 31.56 ± 1.70   Final old_old: 33.99 ± 1.68   Final old_new: 21.71 ± 1.87   Final new_new: 10.91 ± 5.18
Hits@30:
  Final val: 47.64 ± 0.94   Final Test: 35.36 ± 2.31   Final old_old: 37.80 ± 2.39   Final old_new: 25.64 ± 2.17   Final new_new: 11.36 ± 6.63
Hits@50:
  Final val: 54.39 ± 0.47   Final Test: 41.83 ± 2.77   Final old_old: 44.26 ± 2.70   Final old_new: 32.29 ± 3.25   Final new_new: 15.91 ± 7.54
AUC:
  Final val: 94.34 ± 0.19   Final Test: 93.94 ± 0.14   Final old_old: 94.21 ± 0.13   Final old_new: 92.98 ± 0.26   Final new_new: 89.89 ± 0.71
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 38.78 ± 5.98   Final Test: 21.24 ± 7.01   Final old_old: 23.72 ± 7.72   Final old_new: 11.03 ± 4.23   Final new_new: 2.27 ± 0.00
Hits@20:
  Final val: 50.58 ± 4.71   Final Test: 31.08 ± 4.72   Final old_old: 34.45 ± 5.22   Final old_new: 17.38 ± 2.86   Final new_new: 3.18 ± 2.03
Hits@30:
  Final val: 60.06 ± 3.20   Final Test: 42.00 ± 3.36   Final old_old: 46.02 ± 3.68   Final old_new: 25.74 ± 2.01   Final new_new: 6.82 ± 4.25
Hits@50:
  Final val: 71.56 ± 2.88   Final Test: 53.47 ± 3.87   Final old_old: 58.02 ± 4.22   Final old_new: 35.26 ± 2.75   Final new_new: 10.00 ± 5.23
AUC:
  Final val: 97.20 ± 0.23   Final Test: 96.26 ± 0.17   Final old_old: 97.16 ± 0.14   Final old_new: 92.63 ± 0.38   Final new_new: 88.76 ± 0.73
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 37.05 ± 1.87   Final Test: 19.76 ± 2.26   Final old_old: 21.95 ± 2.46   Final old_new: 10.71 ± 1.74   Final new_new: 4.55 ± 1.61
Hits@20:
  Final val: 45.11 ± 2.30   Final Test: 30.09 ± 2.10   Final old_old: 32.61 ± 2.22   Final old_new: 19.87 ± 1.86   Final new_new: 8.64 ± 2.96
Hits@30:
  Final val: 49.93 ± 1.54   Final Test: 33.89 ± 0.59   Final old_old: 36.31 ± 0.65   Final old_new: 24.38 ± 1.11   Final new_new: 8.18 ± 3.05
Hits@50:
  Final val: 57.07 ± 0.68   Final Test: 41.71 ± 1.16   Final old_old: 43.86 ± 1.22   Final old_new: 33.40 ± 1.29   Final new_new: 15.45 ± 4.07
AUC:
  Final val: 94.31 ± 0.33   Final Test: 93.69 ± 0.23   Final old_old: 94.06 ± 0.22   Final old_new: 92.32 ± 0.28   Final new_new: 88.17 ± 1.07
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 44.15 ± 4.36   Final Test: 27.78 ± 4.34   Final old_old: 29.99 ± 4.72   Final old_new: 19.17 ± 2.87   Final new_new: 2.73 ± 1.90
Hits@20:
  Final val: 55.55 ± 3.85   Final Test: 39.04 ± 3.73   Final old_old: 41.96 ± 3.99   Final old_new: 27.56 ± 2.82   Final new_new: 7.73 ± 5.93
Hits@30:
  Final val: 63.92 ± 3.96   Final Test: 51.25 ± 5.04   Final old_old: 54.53 ± 5.35   Final old_new: 38.39 ± 4.23   Final new_new: 15.45 ± 4.66
Hits@50:
  Final val: 72.14 ± 3.02   Final Test: 61.93 ± 3.56   Final old_old: 65.24 ± 3.79   Final old_new: 49.12 ± 3.00   Final new_new: 22.73 ± 5.08
AUC:
  Final val: 97.02 ± 0.17   Final Test: 96.77 ± 0.17   Final old_old: 97.19 ± 0.21   Final old_new: 95.17 ± 0.17   Final new_new: 90.59 ± 0.97
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 40.07 ± 2.25   Final Test: 22.55 ± 2.62   Final old_old: 24.90 ± 2.86   Final old_new: 13.07 ± 2.34   Final new_new: 1.82 ± 1.02
Hits@20:
  Final val: 46.50 ± 2.17   Final Test: 28.96 ± 1.36   Final old_old: 31.52 ± 1.70   Final old_new: 18.74 ± 1.54   Final new_new: 3.64 ± 1.24
Hits@30:
  Final val: 50.88 ± 2.29   Final Test: 34.39 ± 2.11   Final old_old: 37.04 ± 2.53   Final old_new: 23.95 ± 1.12   Final new_new: 6.36 ± 3.73
Hits@50:
  Final val: 57.68 ± 1.48   Final Test: 39.73 ± 2.42   Final old_old: 41.72 ± 2.58   Final old_new: 32.12 ± 2.18   Final new_new: 15.00 ± 6.35
AUC:
  Final val: 94.71 ± 0.22   Final Test: 94.18 ± 0.31   Final old_old: 94.50 ± 0.34   Final old_new: 92.98 ± 0.19   Final new_new: 89.20 ± 0.53
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 38.58 ± 5.71   Final Test: 24.83 ± 5.47   Final old_old: 27.26 ± 6.01   Final old_new: 14.79 ± 3.42   Final new_new: 6.82 ± 4.25
Hits@20:
  Final val: 49.87 ± 5.33   Final Test: 34.50 ± 5.88   Final old_old: 37.53 ± 6.51   Final old_new: 22.22 ± 3.35   Final new_new: 8.18 ± 3.05
Hits@30:
  Final val: 56.02 ± 4.26   Final Test: 43.81 ± 5.63   Final old_old: 47.43 ± 6.15   Final old_new: 29.17 ± 3.79   Final new_new: 12.73 ± 5.23
Hits@50:
  Final val: 66.29 ± 3.66   Final Test: 53.15 ± 7.07   Final old_old: 57.08 ± 7.87   Final old_new: 37.28 ± 3.81   Final new_new: 18.64 ± 7.08
AUC:
  Final val: 96.45 ± 0.34   Final Test: 95.63 ± 0.45   Final old_old: 96.42 ± 0.42   Final old_new: 92.48 ± 0.60   Final new_new: 88.53 ± 2.71
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 36.99 ± 1.68   Final Test: 20.23 ± 3.25   Final old_old: 22.38 ± 3.39   Final old_new: 11.49 ± 2.83   Final new_new: 2.73 ± 4.07
Hits@20:
  Final val: 45.68 ± 1.21   Final Test: 27.35 ± 2.92   Final old_old: 30.01 ± 3.14   Final old_new: 16.57 ± 2.22   Final new_new: 4.55 ± 2.27
Hits@30:
  Final val: 51.65 ± 1.28   Final Test: 32.73 ± 3.27   Final old_old: 35.67 ± 3.41   Final old_new: 20.88 ± 3.18   Final new_new: 5.91 ± 3.05
Hits@50:
  Final val: 58.43 ± 1.22   Final Test: 43.18 ± 1.24   Final old_old: 46.31 ± 1.36   Final old_new: 31.03 ± 1.87   Final new_new: 7.27 ± 5.43
AUC:
  Final val: 94.68 ± 0.29   Final Test: 94.00 ± 0.22   Final old_old: 94.50 ± 0.28   Final old_new: 92.09 ± 0.15   Final new_new: 87.53 ± 1.30
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_RA_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.97 ± 5.45   Final Test: 28.17 ± 3.34   Final old_old: 31.57 ± 3.89   Final old_new: 14.28 ± 1.19   Final new_new: 0.91 ± 1.24
Hits@20:
  Final val: 55.39 ± 5.16   Final Test: 40.28 ± 5.98   Final old_old: 44.72 ± 6.61   Final old_new: 22.24 ± 3.37   Final new_new: 2.73 ± 2.49
Hits@30:
  Final val: 63.89 ± 3.57   Final Test: 50.38 ± 4.82   Final old_old: 55.46 ± 5.26   Final old_new: 29.97 ± 3.17   Final new_new: 3.64 ± 3.45
Hits@50:
  Final val: 73.33 ± 1.88   Final Test: 59.59 ± 4.29   Final old_old: 64.78 ± 4.66   Final old_new: 38.89 ± 3.04   Final new_new: 9.55 ± 5.88
AUC:
  Final val: 97.22 ± 0.14   Final Test: 96.43 ± 0.16   Final old_old: 97.36 ± 0.17   Final old_new: 92.74 ± 0.34   Final new_new: 87.82 ± 1.61
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.79 ± 2.82   Final Test: 23.75 ± 2.61   Final old_old: 25.57 ± 2.58   Final old_new: 16.62 ± 2.87   Final new_new: 3.18 ± 3.05
Hits@20:
  Final val: 54.57 ± 1.50   Final Test: 37.38 ± 3.37   Final old_old: 39.92 ± 3.26   Final old_new: 27.63 ± 4.07   Final new_new: 5.91 ± 4.71
Hits@30:
  Final val: 60.63 ± 0.89   Final Test: 46.34 ± 2.38   Final old_old: 49.23 ± 2.39   Final old_new: 35.21 ± 2.68   Final new_new: 10.45 ± 3.80
Hits@50:
  Final val: 67.78 ± 1.11   Final Test: 54.49 ± 2.46   Final old_old: 57.44 ± 2.45   Final old_new: 43.22 ± 2.63   Final new_new: 17.27 ± 6.35
AUC:
  Final val: 96.00 ± 0.18   Final Test: 95.45 ± 0.12   Final old_old: 95.85 ± 0.18   Final old_new: 93.95 ± 0.38   Final new_new: 89.80 ± 1.00
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=1, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.35 ± 0.82   Final Test: 24.64 ± 1.18   Final old_old: 27.08 ± 1.26   Final old_new: 14.84 ± 1.02   Final new_new: 2.73 ± 2.96
Hits@20:
  Final val: 49.21 ± 1.41   Final Test: 29.92 ± 1.13   Final old_old: 32.62 ± 1.43   Final old_new: 19.07 ± 1.41   Final new_new: 4.55 ± 3.21
Hits@30:
  Final val: 54.44 ± 0.96   Final Test: 36.69 ± 2.52   Final old_old: 39.64 ± 2.40   Final old_new: 24.96 ± 3.10   Final new_new: 7.27 ± 4.93
Hits@50:
  Final val: 61.52 ± 0.96   Final Test: 44.03 ± 1.97   Final old_old: 46.86 ± 2.01   Final old_new: 33.02 ± 2.09   Final new_new: 10.91 ± 2.49
AUC:
  Final val: 95.36 ± 0.14   Final Test: 94.89 ± 0.06   Final old_old: 95.38 ± 0.07   Final old_new: 93.04 ± 0.20   Final new_new: 88.03 ± 1.61
All runs:
Hits@10:
  Final val: 39.97 ± 4.19   Final Test: 23.67 ± 4.82   Final old_old: 26.00 ± 5.24   Final old_new: 14.23 ± 3.23   Final new_new: 3.18 ± 2.03
Hits@20:
  Final val: 51.58 ± 4.31   Final Test: 33.69 ± 6.57   Final old_old: 36.77 ± 7.01   Final old_new: 21.28 ± 4.97   Final new_new: 6.36 ± 2.96
Hits@30:
  Final val: 58.36 ± 3.47   Final Test: 41.11 ± 6.26   Final old_old: 44.81 ± 6.66   Final old_new: 26.20 ± 5.15   Final new_new: 8.18 ± 2.59
Hits@50:
  Final val: 68.97 ± 2.79   Final Test: 52.29 ± 3.93   Final old_old: 56.50 ± 4.27   Final old_new: 35.47 ± 3.20   Final new_new: 11.82 ± 3.73
AUC:
  Final val: 96.72 ± 0.32   Final Test: 95.83 ± 0.29   Final old_old: 96.71 ± 0.32   Final old_new: 92.26 ± 0.64   Final new_new: 88.65 ± 1.30
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 48.20 ± 2.55   Final Test: 30.93 ± 7.68   Final old_old: 33.62 ± 7.98   Final old_new: 20.33 ± 6.58   Final new_new: 3.18 ± 4.98
Hits@20:
  Final val: 59.42 ± 1.99   Final Test: 43.76 ± 4.72   Final old_old: 47.01 ± 4.94   Final old_new: 30.91 ± 4.04   Final new_new: 10.00 ± 5.23
Hits@30:
  Final val: 66.44 ± 1.54   Final Test: 54.01 ± 3.69   Final old_old: 57.56 ± 3.31   Final old_new: 39.97 ± 5.31   Final new_new: 17.27 ± 8.74
Hits@50:
  Final val: 74.28 ± 1.16   Final Test: 62.92 ± 3.26   Final old_old: 66.34 ± 3.30   Final old_new: 49.62 ± 3.47   Final new_new: 23.18 ± 8.10
AUC:
  Final val: 97.16 ± 0.09   Final Test: 96.92 ± 0.12   Final old_old: 97.38 ± 0.12   Final old_new: 95.16 ± 0.20   Final new_new: 91.01 ± 1.00
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.91 ± 4.50   Final Test: 28.25 ± 3.34   Final old_old: 31.92 ± 3.76   Final old_new: 13.10 ± 1.78   Final new_new: 1.36 ± 2.03
Hits@20:
  Final val: 56.10 ± 2.47   Final Test: 37.24 ± 6.37   Final old_old: 41.41 ± 6.67   Final old_new: 20.28 ± 5.34   Final new_new: 3.18 ± 3.80
Hits@30:
  Final val: 64.08 ± 0.70   Final Test: 48.97 ± 3.94   Final old_old: 53.86 ± 4.01   Final old_new: 29.24 ± 4.08   Final new_new: 5.91 ± 3.05
Hits@50:
  Final val: 73.58 ± 0.64   Final Test: 60.30 ± 1.72   Final old_old: 65.45 ± 1.64   Final old_new: 39.77 ± 2.40   Final new_new: 10.45 ± 3.45
AUC:
  Final val: 97.42 ± 0.05   Final Test: 96.61 ± 0.10   Final old_old: 97.46 ± 0.07   Final old_new: 93.22 ± 0.31   Final new_new: 88.91 ± 1.13
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.17 ± 3.90   Final Test: 31.47 ± 8.58   Final old_old: 33.43 ± 8.98   Final old_new: 24.08 ± 7.28   Final new_new: 5.45 ± 4.43
Hits@20:
  Final val: 52.64 ± 2.96   Final Test: 40.56 ± 7.11   Final old_old: 42.87 ± 7.63   Final old_new: 31.64 ± 5.23   Final new_new: 12.27 ± 4.98
Hits@30:
  Final val: 59.53 ± 1.94   Final Test: 49.18 ± 4.28   Final old_old: 51.75 ± 4.50   Final old_new: 39.24 ± 3.86   Final new_new: 18.18 ± 4.55
Hits@50:
  Final val: 67.61 ± 1.44   Final Test: 54.94 ± 4.41   Final old_old: 57.55 ± 4.83   Final old_new: 44.94 ± 2.92   Final new_new: 22.73 ± 6.43
AUC:
  Final val: 96.25 ± 0.21   Final Test: 95.91 ± 0.22   Final old_old: 96.33 ± 0.27   Final old_new: 94.30 ± 0.26   Final new_new: 90.05 ± 0.85
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=10, margin=0.05, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 37.38 ± 10.55   Final Test: 22.04 ± 9.11   Final old_old: 24.32 ± 9.70   Final old_new: 12.64 ± 6.66   Final new_new: 5.91 ± 5.93
Hits@20:
  Final val: 47.77 ± 9.39   Final Test: 33.38 ± 9.90   Final old_old: 36.50 ± 10.80   Final old_new: 20.63 ± 6.29   Final new_new: 8.18 ± 4.43
Hits@30:
  Final val: 55.47 ± 8.60   Final Test: 42.82 ± 10.05   Final old_old: 46.60 ± 11.15   Final old_new: 27.58 ± 5.54   Final new_new: 9.55 ± 4.07
Hits@50:
  Final val: 67.06 ± 5.31   Final Test: 52.37 ± 7.19   Final old_old: 56.48 ± 7.80   Final old_new: 35.74 ± 5.06   Final new_new: 16.36 ± 4.37
AUC:
  Final val: 96.73 ± 0.34   Final Test: 95.91 ± 0.51   Final old_old: 96.74 ± 0.51   Final old_new: 92.49 ± 0.51   Final new_new: 89.51 ± 2.30
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.97 ± 5.04   Final Test: 28.86 ± 4.79   Final old_old: 32.49 ± 5.26   Final old_new: 13.93 ± 2.84   Final new_new: 1.82 ± 1.90
Hits@20:
  Final val: 55.36 ± 2.30   Final Test: 41.40 ± 2.49   Final old_old: 45.89 ± 2.64   Final old_new: 23.07 ± 2.06   Final new_new: 5.00 ± 2.96
Hits@30:
  Final val: 63.08 ± 2.04   Final Test: 48.10 ± 5.63   Final old_old: 52.99 ± 5.92   Final old_new: 28.31 ± 4.71   Final new_new: 5.91 ± 3.80
Hits@50:
  Final val: 73.78 ± 0.98   Final Test: 58.33 ± 3.37   Final old_old: 63.46 ± 3.41   Final old_new: 37.83 ± 3.47   Final new_new: 9.55 ± 4.07
AUC:
  Final val: 97.42 ± 0.12   Final Test: 96.60 ± 0.12   Final old_old: 97.45 ± 0.07   Final old_new: 93.18 ± 0.27   Final new_new: 89.74 ± 1.75
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.85 ± 3.56   Final Test: 28.63 ± 4.04   Final old_old: 31.05 ± 4.57   Final old_new: 19.09 ± 1.94   Final new_new: 2.27 ± 2.27
Hits@20:
  Final val: 55.81 ± 2.31   Final Test: 40.26 ± 4.78   Final old_old: 43.26 ± 5.10   Final old_new: 28.61 ± 3.81   Final new_new: 5.00 ± 1.90
Hits@30:
  Final val: 64.13 ± 2.32   Final Test: 51.54 ± 1.38   Final old_old: 54.84 ± 1.30   Final old_new: 38.74 ± 2.34   Final new_new: 13.18 ± 4.93
Hits@50:
  Final val: 73.08 ± 1.71   Final Test: 61.80 ± 2.96   Final old_old: 65.28 ± 3.02   Final old_new: 48.16 ± 3.47   Final new_new: 23.18 ± 4.07
AUC:
  Final val: 97.03 ± 0.05   Final Test: 96.82 ± 0.09   Final old_old: 97.20 ± 0.12   Final old_new: 95.39 ± 0.08   Final new_new: 91.59 ± 1.06
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 42.44 ± 11.71   Final Test: 27.81 ± 8.00   Final old_old: 30.88 ± 9.05   Final old_new: 15.19 ± 3.85   Final new_new: 5.00 ± 1.02
Hits@20:
  Final val: 53.61 ± 11.60   Final Test: 37.57 ± 9.75   Final old_old: 41.04 ± 10.85   Final old_new: 23.53 ± 5.57   Final new_new: 8.18 ± 4.13
Hits@30:
  Final val: 60.97 ± 9.57   Final Test: 44.78 ± 9.45   Final old_old: 48.61 ± 10.29   Final old_new: 29.29 ± 6.32   Final new_new: 11.36 ± 4.82
Hits@50:
  Final val: 70.60 ± 7.07   Final Test: 56.18 ± 7.31   Final old_old: 60.37 ± 8.07   Final old_new: 39.37 ± 5.11   Final new_new: 17.73 ± 8.71
AUC:
  Final val: 96.91 ± 0.48   Final Test: 96.05 ± 0.44   Final old_old: 96.86 ± 0.45   Final old_new: 92.84 ± 0.55   Final new_new: 88.05 ± 1.45
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_SAGE_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 45.94 ± 2.95   Final Test: 28.40 ± 4.15   Final old_old: 30.49 ± 4.43   Final old_new: 20.28 ± 3.21   Final new_new: 4.55 ± 2.27
Hits@20:
  Final val: 55.03 ± 2.30   Final Test: 39.91 ± 2.55   Final old_old: 42.52 ± 2.87   Final old_new: 29.72 ± 1.63   Final new_new: 10.91 ± 5.66
Hits@30:
  Final val: 60.99 ± 1.64   Final Test: 44.97 ± 3.51   Final old_old: 47.90 ± 3.73   Final old_new: 33.50 ± 2.83   Final new_new: 13.18 ± 3.37
Hits@50:
  Final val: 68.12 ± 1.05   Final Test: 56.02 ± 3.24   Final old_old: 58.81 ± 3.49   Final old_new: 45.26 ± 2.46   Final new_new: 21.82 ± 6.55
AUC:
  Final val: 96.33 ± 0.14   Final Test: 95.94 ± 0.14   Final old_old: 96.34 ± 0.13   Final old_new: 94.44 ± 0.18   Final new_new: 90.80 ± 1.47
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=10, margin=0.1, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 40.03 ± 6.97   Final Test: 27.97 ± 6.88   Final old_old: 31.41 ± 7.48   Final old_new: 13.88 ± 4.62   Final new_new: 1.82 ± 2.49
Hits@20:
  Final val: 53.22 ± 6.31   Final Test: 37.99 ± 7.12   Final old_old: 42.01 ± 7.98   Final old_new: 21.71 ± 3.87   Final new_new: 3.64 ± 2.03
Hits@30:
  Final val: 62.33 ± 4.14   Final Test: 49.86 ± 6.09   Final old_old: 54.65 ± 6.74   Final old_new: 30.68 ± 3.56   Final new_new: 5.00 ± 3.73
Hits@50:
  Final val: 73.28 ± 2.48   Final Test: 58.49 ± 4.15   Final old_old: 63.54 ± 4.56   Final old_new: 38.36 ± 2.53   Final new_new: 9.09 ± 4.55
AUC:
  Final val: 97.40 ± 0.19   Final Test: 96.62 ± 0.18   Final old_old: 97.45 ± 0.17   Final old_new: 93.25 ± 0.33   Final new_new: 89.89 ± 1.91
All runs:
Hits@10:
  Final val: 45.48 ± 3.18   Final Test: 25.14 ± 2.11   Final old_old: 27.43 ± 2.29   Final old_new: 16.07 ± 1.87   Final new_new: 1.82 ± 1.90
Hits@20:
  Final val: 57.59 ± 2.98   Final Test: 38.40 ± 3.04   Final old_old: 41.41 ± 3.19   Final old_new: 26.57 ± 2.62   Final new_new: 6.36 ± 3.73
Hits@30:
  Final val: 64.74 ± 2.02   Final Test: 49.49 ± 2.33   Final old_old: 52.85 ± 2.40   Final old_new: 36.40 ± 2.88   Final new_new: 10.91 ± 6.70
Hits@50:
  Final val: 72.50 ± 1.42   Final Test: 61.42 ± 3.41   Final old_old: 64.65 ± 3.58   Final old_new: 48.97 ± 3.13   Final new_new: 22.73 ± 5.57
AUC:
  Final val: 96.94 ± 0.03   Final Test: 96.65 ± 0.05   Final old_old: 97.04 ± 0.07   Final old_new: 95.18 ± 0.20   Final new_new: 90.98 ± 0.91
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 43.47 ± 3.56   Final Test: 30.61 ± 3.44   Final old_old: 32.79 ± 3.51   Final old_new: 22.12 ± 3.45   Final new_new: 5.45 ± 3.80
Hits@20:
  Final val: 54.24 ± 1.77   Final Test: 38.63 ± 2.90   Final old_old: 41.30 ± 3.21   Final old_new: 28.24 ± 1.88   Final new_new: 8.18 ± 3.45
Hits@30:
  Final val: 60.32 ± 1.13   Final Test: 47.75 ± 2.42   Final old_old: 50.48 ± 2.49   Final old_new: 37.25 ± 2.26   Final new_new: 14.55 ± 4.13
Hits@50:
  Final val: 68.62 ± 0.52   Final Test: 54.94 ± 0.80   Final old_old: 57.80 ± 0.99   Final old_new: 44.08 ± 1.58   Final new_new: 17.73 ± 1.02
AUC:
  Final val: 96.38 ± 0.07   Final Test: 96.02 ± 0.08   Final old_old: 96.39 ± 0.06   Final old_new: 94.67 ± 0.24   Final new_new: 90.72 ± 1.33
Namespace(distill_pred_path='/home/qinzongyue/HeaRT/benchmarking/exist_setting_small/saved/pubmed_capped_shortest_path_pred_production.pth', device=0, log_steps=100, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.3, link_batch_size=65536, node_batch_size=22473, lr=0.01, epochs=20000, eval_steps=5, runs=5, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=1, KD_RM=0, KD_LM=0, LLP_D=10, LLP_R=10, margin=0.2, l2=0, rw_step=3, ns_rate=1, hops=2, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
All runs:
Hits@10:
  Final val: 41.34 ± 3.29   Final Test: 27.88 ± 1.62   Final old_old: 30.19 ± 1.67   Final old_new: 18.79 ± 1.67   Final new_new: 2.73 ± 2.96
Hits@20:
  Final val: 52.21 ± 1.75   Final Test: 41.19 ± 5.20   Final old_old: 44.02 ± 5.25   Final old_new: 30.00 ± 5.19   Final new_new: 12.27 ± 5.23
Hits@30:
  Final val: 60.79 ± 1.94   Final Test: 48.54 ± 2.44   Final old_old: 51.61 ± 2.43   Final old_new: 36.50 ± 3.06   Final new_new: 15.45 ± 5.18
Hits@50:
  Final val: 70.30 ± 2.26   Final Test: 57.98 ± 1.71   Final old_old: 60.99 ± 1.65   Final old_new: 46.32 ± 2.07   Final new_new: 22.73 ± 5.33
AUC:
  Final val: 96.92 ± 0.11   Final Test: 96.68 ± 0.08   Final old_old: 97.04 ± 0.10   Final old_new: 95.36 ± 0.08   Final new_new: 91.61 ± 1.07
Namespace(heuristic=False, device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=65536, lr=0.01, epochs=20000, eval_steps=5, runs=10, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=0.1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.2, rw_step=3, ns_rate=5, hops=3, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(heuristic=False, device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=65536, lr=0.01, epochs=20000, eval_steps=5, runs=10, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=0.1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.2, rw_step=3, ns_rate=5, hops=3, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(heuristic=False, device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=65536, lr=0.01, epochs=20000, eval_steps=5, runs=10, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=0.1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.2, rw_step=3, ns_rate=5, hops=3, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(heuristic=False, device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=65536, lr=0.01, epochs=20000, eval_steps=5, runs=10, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=0.1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.2, rw_step=3, ns_rate=5, hops=3, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(heuristic=False, device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=65536, lr=0.01, epochs=20000, eval_steps=5, runs=10, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=0.1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.2, rw_step=3, ns_rate=5, hops=3, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(heuristic=False, device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=65536, lr=0.01, epochs=20000, eval_steps=5, runs=10, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=0.1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.2, rw_step=3, ns_rate=5, hops=3, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(heuristic=False, device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=65536, lr=0.01, epochs=20000, eval_steps=5, runs=10, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=0.1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.2, rw_step=3, ns_rate=5, hops=3, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(heuristic=False, device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=65536, lr=0.01, epochs=20000, eval_steps=5, runs=10, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=0.1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.2, rw_step=3, ns_rate=5, hops=3, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
Namespace(heuristic=False, device=0, log_steps=1, encoder='sage', num_layers=2, hidden_channels=256, dropout=0.5, link_batch_size=65536, node_batch_size=65536, lr=0.01, epochs=20000, eval_steps=5, runs=10, dataset_dir='../data', datasets='pubmed', predictor='mlp', patience=100, metric='Hits@20', use_valedges_as_input=False, True_label=0.1, KD_RM=0, KD_LM=0, LLP_D=1, LLP_R=1, margin=0.2, rw_step=3, ns_rate=5, hops=3, ps_method='nb', transductive='production', minibatch=False)
LLP (Relational Distillation)
