# Data loading ----
suppressMessages(library(tidyverse))
library(meta)
library(here)
source(here('paper_scripts', 'utils', 'utils.R'))
library(gt)

experiments <- map(c(7777, 7778, 7779, 7780, 7781), load_experiments, percentiles = TRUE)
names(experiments) <- c('s7777', 's7778', 's7779', 's7780', 's7781')

experiments <- map(experiments,
                   function(x) x %>%
                     select(-metric, -mfs_scaled) %>%
                     filter((mfs_metaod & !mfs_catch22) | (!mfs_metaod & mfs_catch22)))

# 2. Recommenders comparison ----

experiments <- map(experiments,
                   function(x) x %>% filter(mfs_catch22 & !mfs_metaod))

## Stratified case
compare_two_recommenders <- function(x, y, scenario){
  map_dbl(experiments, function(dat){
    internal_res <- dat %>% filter(strategy == scenario) %>% pull(result)
    internal_res <- internal_res[[1]]
    if(!(x %in% names(internal_res)) | !(y %in% names(internal_res))) return(999)
    wilcox.test(x = internal_res[[x]], y = internal_res[[y]], paired = T)$p.value
  }) %>% as.numeric
}

tests_scenario1 <- expand.grid(rec_a = c('orthus', 'regression', 'global_best', 'metaod', 'nearest_neighbor'),
                     rec_b = c('orthus', 'regression', 'global_best', 'metaod', 'nearest_neighbor'),
                     stringsAsFactors = F) %>%
  filter(rec_a != rec_b) %>%
  mutate(combination = map2_chr(rec_a, rec_b, function(x,y) paste(sort(c(x, y)), collapse = '-'))) %>%
  distinct(combination) %>%
  mutate(rec_a = map_chr(combination, function(x) strsplit(x, '-')[[1]][1]),
         rec_b = map_chr(combination, function(x) strsplit(x, '-')[[1]][2])) %>%
  select(-combination) %>%
  mutate(p_values = map2(rec_a, rec_b, compare_two_recommenders, scenario = 'stratified'))

tests_scenario2 <- expand.grid(rec_a = c('orthus', 'regression', 'global_best', 'metaod', 'nearest_neighbor'),
                               rec_b = c('orthus', 'regression', 'global_best', 'metaod', 'nearest_neighbor'),
                               stringsAsFactors = F) %>%
  filter(rec_a != rec_b) %>%
  mutate(combination = map2_chr(rec_a, rec_b, function(x,y) paste(sort(c(x, y)), collapse = '-'))) %>%
  distinct(combination) %>%
  mutate(rec_a = map_chr(combination, function(x) strsplit(x, '-')[[1]][1]),
         rec_b = map_chr(combination, function(x) strsplit(x, '-')[[1]][2])) %>%
  select(-combination) %>%
  mutate(p_values = map2(rec_a, rec_b, compare_two_recommenders, scenario = 'grouped'))

tests <- left_join(x = tests_scenario1 %>% unnest(cols = p_values) %>% mutate(seed = rep(1:5, 10)),
                   y = tests_scenario2 %>% unnest(cols = p_values) %>% mutate(seed = rep(1:5, 10)),
                   by = c('rec_a', 'rec_b', 'seed'))

tests <- tests %>%
  filter(p_values.x <= 1,
         p_values.y <= 1) %>%
  arrange(rec_a, rec_b) %>%
  select('Recommender 1' = 'rec_a',
         'Recommender 2' = 'rec_b',
         'Seed' = 'seed',
         'Scenario I' = 'p_values.x',
         'Scenario II' = 'p_values.y')

tests$Seed <- as.character(tests$Seed)
tests$`Recommender 1` <- gsub(pattern = '_', replacement = ' ', x = tests$`Recommender 1`)
tests$`Recommender 2` <- gsub(pattern = '_', replacement = ' ', x = tests$`Recommender 2`)
tests$`Recommender 1` <- stringr:::str_to_title(tests$`Recommender 1`)
tests$`Recommender 2` <- stringr:::str_to_title(tests$`Recommender 2`)

tests %>% gt() %>% fmt_number(decimals = 3) %>% as_latex()

