summarise(X = sum(choice == "0"),
Y = sum(choice == "1"))
sim_gpt4_counts['condition'] = "GPT-4"
sim_gpt4_counts['group'] = "After"
mean(sim_gpt4_counts$X)
mean(sim_gpt_counts$X)
############################ Gemini data ##########################
gemini_data = read.csv('sim_data/gemini/gemini_1.5.csv')
gemini_data$X = NULL
colnames(gemini_data) = c("subjID", "trial", "choice", "outcome", "probsX", "probsY")
gemini_data_df <- as.data.table(gemini_data)
gemini_counts <- gemini_data_df %>%
group_by(subjID) %>%
summarise(X = sum(choice == "0"),
Y = sum(choice == "1"))
gemini_counts['condition'] = "Gemini_1.5"
gemini_counts['group'] = "Before"
mean(gemini_counts$X)
mean(gpt_counts$X)
mean(sim_gpt_counts$X)
############################ Gemini_vs_ADV ###################################
sim_gemini_list <- list()
for (i in 0:49){
data = read.csv(paste0("evaluate_model/RNN_adv_sim_400000_eps_0.01_lr_0.001/gemini_1.5/events_", i,".csv"))
sim_gemini_list <- append(sim_gemini_list, list(data))
}
sim_gemini <- do.call(rbind, sim_gemini_list)
sim_gemini['subjID'] = rep(0:49, each = 100)
sim_gemini$subjID <- paste(sim_gemini$subjID, "gemini_1.5", sep="-")
# colnames(df) = c("subjID", "trial", "choice", "outcome", "probsX", "probsY")
sim_gemini['choice'] =  factor(sim_gemini$real.model.action,
levels = c("[[1. 0.]]", "[[0. 1.]]"),
labels = c("0", "1"))
sim_gemini <- sim_gemini %>%
mutate(outcome = case_when(choice==1 & r2==0 ~ 0,
choice==1 & r2==1 ~ 1,
choice==0 & r1==0 ~ 0,
choice==0 & r1==1 ~ 1))
sim_gemini_df <- as.data.table(sim_gemini)
sim_gemini_counts <- sim_gemini_df %>%
group_by(subjID) %>%
summarise(X = sum(choice == "0"),
Y = sum(choice == "1"))
sim_gemini_counts['condition'] = "Gemini_1.5"
sim_gemini_counts['group'] = "After"
mean(sim_gemini_counts$X)
library(plyr)
library(ggplot2)
library(wesanderson)
rm(list = ls())
read_mturk_res = function(data_path){
# dirs = list.dirs(data_path, full.names = TRUE, recursive = FALSE)
dirs = list.files(data_path, full.names = TRUE, recursive = FALSE)
require(ramify)
output = list()
index = 1
for (d in dirs){
f = paste0(d, '/data/output.csv')
df = read.csv(f)
df$repay = as.numeric(gsub("\\[|\\]", "", df$repay))
df$adv.action = as.numeric(gsub("\\[|\\]", "",df$adv.action))
df$id = d
df$trial = c(1:10)
df$trustee_earn =  3 * df$investment - df$repay
df$repay_precent =  100 * df$repay / (df$investment * 3 + 0.0000001)
df$learner.discr = floor(clip(df$investment - 0.0001, 0, 1000) / 4)
df$learner.discr = floor(pmax(0, pmin(df$investment - 0.0001, 1000)) / 4)
df$investor_earn = (20 - df$investment) + df$repay
output[[index]] = df
index = index + 1
}
require(data.table)
all_s = as.data.table(as.data.frame(rbindlist(output)))
all_s[, prev_repay_precent := shift(repay_precent, 1), by=.(id)]
all_s$prev_repay_precent_disc = cut(all_s$prev_repay_precent, 5, method="length", na.omit=FALSE)
all_s$investement.discr = floor(clip(all_s$investment - 0.0001, 0, 1000) / 4) + 1
all_s
}
read_sim_gpt = function(data_path){
# dirs = list.dirs(data_path, full.names = TRUE, recursive = FALSE)
files <- list.files(path = data_path, pattern = "\\.csv$", full.names = TRUE)
require(ramify)
output = list()
index = 1
for (f in files){
df = read.csv(f)
df$repay = as.numeric(gsub("\\[|\\]", "", df$repay))
df$id = f
df$trial = c(1:10)
df$trustee_earn =  3 * df$investment - df$repay
df$repay_percent =  100 * df$repay / (df$investment * 3 + 0.0000001)
df$learner.discr = floor(clip(df$investment - 0.0001, 0, 1000) / 4)
df$learner.discr = floor(pmax(0, pmin(df$investment - 0.0001, 1000)) / 4)
df$investor_earn = (20 - df$investment) + df$repay
output[[index]] = df
index = index + 1
}
require(data.table)
all_s = as.data.table(as.data.frame(rbindlist(output)))
all_s[, prev_repay_percent := shift(repay_percent, 1), by=.(id)]
break_points = c(-0.1, 20, 40, 60, 80, 100)
all_s$prev_repay_percent_disc = cut(all_s$prev_repay_percent, breaks=break_points, method="length", na.omit=FALSE)
all_s$investement.discr = floor(clip(all_s$investment - 0.0001, 0, 1000) / 4) + 1
all_s
}
read_sim_data_real = function(data_path, sim_count, adv_action_count=5){
dd = list()
indx = 1
for (i in c(0:sim_count)){
# for human
data = read.csv(paste0(data_path, '/events_', i,".csv"))
data$sbj = i
data$trial = (data$X + 1 )
# data$percent_return = (as.integer(gsub("\\[|\\]", "", data$adv.action)) * (100 / (adv_action_count - 1)))
data$repay = as.integer(gsub("\\[|\\]", "", data$learner.reward))
data$investment = as.integer(gsub("\\[|\\]", "", data$real_model.action.cont))
data$learner.discr = NA
data$learner.discr[data$learner.action == '[[[0 0 0 0 1]]]'] = 5
data$learner.discr[data$learner.action == '[[[0 0 0 1 0]]]'] = 4
data$learner.discr[data$learner.action == '[[[0 0 1 0 0]]]'] = 3
data$learner.discr[data$learner.action == '[[[0 1 0 0 0]]]'] = 2
data$learner.discr[data$learner.action == '[[[1 0 0 0 0]]]'] = 1
dd[[indx]] = data
indx = indx + 1
}
require(data.table)
all_d = as.data.frame(rbindlist(dd))
# all_d$repay_precent = all_d$percent_return
# all_d$investment = as.integer(gsub("\\[|\\]", "", all_d$real_model.action.cont))
all_d$repay_percent =  100 * all_d$repay / (all_d$investment * 3 + 0.0000001)
all_d$id = all_d$sbj
all_d = as.data.table(all_d)
all_d[, prev_repay_percent := shift(repay_percent, 1), by=.(id)]
all_d$prev_repay_percent_disc = cut(all_d$prev_repay_percent, 5, method="length", na.omit=FALSE)
all_d$investement.discr = floor(clip(all_d$investment - 0.0001, 0, 1000) / 4) + 1
all_d$trustee_earn =  3 * all_d$investment - all_d$repay
all_d$investor_earn = (20 - all_d$investment) + all_d$repay
all_d = as.data.frame(all_d)
}
read_sim_data_ler = function(data_path, sim_count, adv_action_count=5){
dd = list()
indx = 1
for (i in c(0:sim_count)){
# for human
data = read.csv(paste0(data_path, '/events_', i,".csv"))
data$sbj = i
data$trial = (data$X + 1 )
# data$percent_return = (as.integer(gsub("\\[|\\]", "", data$adv.action)) * (100 / (adv_action_count - 1)))
data$repay = as.integer(gsub("\\[|\\]", "", data$learner.reward))
data$investment = as.integer(gsub("\\[|\\]", "", data$learner.action.cont))
data$learner.discr = NA
data$learner.discr[data$learner.action == '[[[0 0 0 0 1]]]'] = 5
data$learner.discr[data$learner.action == '[[[0 0 0 1 0]]]'] = 4
data$learner.discr[data$learner.action == '[[[0 0 1 0 0]]]'] = 3
data$learner.discr[data$learner.action == '[[[0 1 0 0 0]]]'] = 2
data$learner.discr[data$learner.action == '[[[1 0 0 0 0]]]'] = 1
dd[[indx]] = data
indx = indx + 1
}
require(data.table)
all_d = as.data.frame(rbindlist(dd))
all_d$repay_percent =  100 * all_d$repay / (all_d$investment * 3 + 0.0000001)
all_d$id = all_d$sbj
all_d = as.data.table(all_d)
all_d[, prev_repay_percent := shift(repay_percent, 1), by=.(id)]
all_d$prev_repay_percent_disc = cut(all_d$prev_repay_percent, 5, method="length", na.omit=FALSE)
all_d$investement.discr = floor(clip(all_d$investment - 0.0001, 0, 1000) / 4) + 1
all_d$trustee_earn =  3 * all_d$investment - all_d$repay
all_d$investor_earn = (20 - all_d$investment) + all_d$repay
all_d = as.data.frame(all_d)
}
plot_trustee_earn = function(data, x_order){
data_r = ddply(subset(data, T), c("id", "group", "cat"), function(x){data.frame(trustee_earn=sum(x$trustee_earn))})
dark2_colors <- c("#1b9e77", "#d95f02", "#7570b3", "#e6ab02")
ggplot(subset(data_r, T), aes(y = trustee_earn, x =group, fill=cat)) +
#  stat_summary(fun.y = "mean", geom = "bar", position = position_dodge(), fill=col) +
geom_bar(stat = "summary", fun.y = "mean") +
stat_summary(fun.data = mean_cl_normal, geom="linerange", colour="black",
position=position_dodge(.9),  fun.args = list(mult = 1), size=0.2) +
ylab("trustee earning") +
xlab("") +
theme_bw() +
scale_fill_manual(values=dark2_colors,
limits=c("Human","GPT-3.5", "GPT-4", "Gemini"),
labels = c("Human", "GPT-3.5", "GPT-4", "Gemini")) +
# scale_fill_manual(values = wes_palette("GrandBudapest1", n = 3))+
scale_x_discrete(limits = x_order) +
coord_cartesian(ylim=c(0, 380))+
# blk_theme_grid_hor(legend_position ="none", margins = c(1,1,1,1), rotate_x = F) +
guides(fill = guide_legend(keywidth = 0.5, keyheight = 3.0))+
theme(legend.position="none")+
theme(axis.title.y = element_text(size = 18), # Increase Y-axis label size and make it bold
axis.text.x = element_text(size = 14), # Increase X-axis tick text size
axis.text.y = element_text(size = 14))+ # Increase legend title size
theme(axis.text.x = element_text(angle = 45, vjust = 1, hjust=1))
}
plot_investor_earn = function(data, x_order){
data_r = ddply(subset(data, T), c("id", "group", "cat"), function(x){data.frame(investor_earn=sum(x$investor_earn))})
dark2_colors <- c("#1b9e77", "#d95f02", "#7570b3", "#e6ab02")
ggplot(subset(data_r, T), aes(y = investor_earn, x =group, fill=cat)) +
#  stat_summary(fun.y = "mean", geom = "bar", position = position_dodge(), fill=col) +
geom_bar(stat = "summary", fun.y = "mean") +
stat_summary(fun.data = mean_cl_normal, geom="linerange", colour="black",
position=position_dodge(.9),  fun.args = list(mult = 1), size=0.2) +
ylab("investor earning") +
xlab("") +
theme_bw() +
scale_fill_manual(values=dark2_colors,
limits=c("Human","GPT-3.5", "GPT-4", "Gemini"),
labels = c("Human", "GPT-3.5", "GPT-4", "Gemini")) +
scale_x_discrete(limits = x_order) +
coord_cartesian(ylim=c(0, 380))+
# blk_theme_grid_hor(legend_position ="none", margins = c(1,1,1,1), rotate_x = F) +
guides(fill = guide_legend(keywidth = 0.5, keyheight = 3.0))+
theme(legend.position="none") +
theme(axis.title.y = element_text(size = 18), # Increase Y-axis label size and make it bold
axis.text.x = element_text(size = 14), # Increase X-axis tick text size
axis.text.y = element_text(size = 14)) + # Increase legend title size
theme(axis.text.x = element_text(angle = 45, vjust = 1, hjust=1))
}
plot_fairness = function(data, x_order){
data_r = ddply(subset(data, T), c("id", "group", "cat"), function(x){data.frame(diffs=sum(x$trustee_earn - x$investor_earn))})
dark2_colors <- c("#1b9e77", "#d95f02", "#7570b3", "#e6ab02")
ggplot(subset(data_r, T), aes(y = diffs, x =group, fill=cat)) +
#  stat_summary(fun.y = "mean", geom = "bar", position = position_dodge(), fill=col) +
geom_bar(stat = "summary", fun.y = "mean") +
stat_summary(fun.data = mean_cl_normal, geom="linerange", colour="black",
position=position_dodge(.9),  fun.args = list(mult = 1), size=0.2) +
ylab("trustee earning-investor earning") +
xlab("") +
theme_bw() +
scale_fill_manual(values=dark2_colors,
limits=c("Human","GPT-3.5", "GPT-4", "Gemini"),
labels = c("Human", "GPT-3.5", "GPT-4", "Gemini")) +
scale_x_discrete(limits = x_order) +
# blk_theme_grid_hor(legend_position ="none", margins = c(1,1,1,1), rotate_x = F) +
guides(fill = guide_legend(keywidth = 0.5, keyheight = 3.0))+
theme(legend.position="none")+
theme(axis.title.y = element_text(size = 18), # Increase Y-axis label size and make it bold
axis.text.x = element_text(size = 14), # Increase X-axis tick text size
axis.text.y = element_text(size = 14)) + # Increase legend title size
theme(axis.text.x = element_text(angle = 45, vjust = 1, hjust=1))
}
subj_rnd = read_mturk_res('human_data/RND/')
subj_rnd$group = "Human vs RND"
setwd("~/Documents/Projects/adv_LLM_NIPS/MRTT")
subj_rnd = read_mturk_res('human_data/RND/')
subj_rnd$group = "Human vs RND"
subj_rnd$cat = "Human"
subj_fair = read_mturk_res('human_data/FAIR/')
subj_fair$group = "Human vs FAIR"
subj_fair$cat = "Human"
subj_max = read_mturk_res('human_data/MAX/')
subj_max$group = "Human vs MAX"
subj_max$cat = "Human"
gpt_rnd = read_sim_gpt("sim_data/gpt/gpt-3.5-turbo")
gpt_rnd$group = "GPT-3.5 vs RND"
gpt_rnd$cat = "GPT-3.5"
gpt_adv_fair = read_sim_data_real("evaluate_model/fair_max/RL_nc_dqn_buf_400000_eps_0.01_lr_0.001", 49)
gpt_adv_fair$group = "GPT-3.5 vs FAIR"
gpt_adv_fair$cat = "GPT-3.5"
# gpt_adv_sim_fair = read_sim_data_ler("evaluate_model/fair_max_sim/RL_nc_dqn_buf_400000_eps_0.1_lr_0.0001", 199)
# gpt_adv_sim_fair$group = "GPT-3.5 vs FAIR"
# gpt_adv_sim_fair$cat = "GPT-3.5"
gpt_adv_max = read_sim_data_real("evaluate_model/earn_max/RL_nc_dqn_buf_400000_eps_0.01_lr_0.001", 49)
gpt_adv_max$group = "GPT-3.5 vs MAX"
gpt_adv_max$cat = "GPT-3.5"
gpt4_rnd = read_sim_gpt("sim_data/gpt/gpt-4-turbo")
gpt4_rnd$group = "GPT-4 vs RND"
gpt4_rnd$cat = "GPT-4"
gpt4_adv_fair = read_sim_data_real("evaluate_model/fair_max4/RL_nc_dqn_buf_400000_eps_0.01_lr_0.001", 49)
gpt4_adv_fair$group = "GPT-4 vs FAIR"
gpt4_adv_fair$cat = "GPT-4"
# gpt4_adv_sim_fair = read_sim_data_ler("evaluate_model/fair_max4_sim/RL_nc_dqn_buf_400000_eps_0.2_lr_1e-05", 199)
# gpt4_adv_sim_fair$group = "GPT-4 vs FAIR"
# gpt4_adv_sim_fair$cat = "GPT-4"
gpt4_adv_max = read_sim_data_real("evaluate_model/earn_max4/RL_nc_dqn_buf_400000_eps_0.01_lr_0.001", 49)
gpt4_adv_max$group = "GPT-4 vs MAX"
gpt4_adv_max$cat = "GPT-4"
gemini_rnd = read_sim_gpt("sim_data/gemini/gemini_1.5")
gemini_rnd$group = "Gemini vs RND"
gemini_rnd$cat = "Gemini"
gemini_adv_max = read_sim_data_real("evaluate_model/earn_max_gemini/RL_nc_dqn_buf_400000_eps_0.01_lr_0.001", 49)
gemini_adv_max$group = "Gemini vs MAX"
gemini_adv_max$cat = "Gemini"
# gemini_adv_sim_max = read_sim_data_ler("evaluate_model/earn_max_gemini_sim/RL_nc_dqn_buf_400000_eps_0.01_lr_0.001", 199)
# gemini_adv_sim_max$group = "Gemini vs MAX"
# gemini_adv_sim_max$cat = "Gemini"
gemini_adv_fair = read_sim_data_real("evaluate_model/fair_max_gemini/RL_nc_dqn_buf_400000_eps_0.01_lr_0.001", 49)
gemini_adv_fair$group = "Gemini vs FAIR"
gemini_adv_fair$cat = "Gemini"
alld_rnd = rbind(subj_rnd[,c("id", "trustee_earn", "group", "cat")],
subj_max[,c("id", "trustee_earn", "group", "cat")],
subj_fair[,c("id", "trustee_earn", "group", "cat")],
gpt_rnd[,c("id", "trustee_earn", "group", "cat")],
gpt_adv_max[,c("id", "trustee_earn", "group", "cat")],
gpt_adv_fair[,c("id", "trustee_earn", "group", "cat")],
gpt4_rnd[,c("id", "trustee_earn", "group", "cat")],
gpt4_adv_max[,c("id", "trustee_earn", "group", "cat")],
gpt4_adv_fair[,c("id", "trustee_earn", "group", "cat")],
gemini_rnd[,c("id", "trustee_earn", "group", "cat")],
gemini_adv_max[,c("id", "trustee_earn", "group", "cat")],
gemini_adv_fair[,c("id", "trustee_earn", "group", "cat")])
plot_trustee_earn(alld_rnd, c(subj_rnd$group[1],
subj_fair$group[1],
subj_max$group[1],
gpt_rnd$group[1],
gpt_adv_fair$group[1],
gpt_adv_max$group[1],
gpt4_rnd$group[1],
gpt4_adv_fair$group[1],
gpt4_adv_max$group[1],
gemini_rnd$group[1],
gemini_adv_fair$group[1],
gemini_adv_max$group[1]))
View(alld_rnd)
data = alld_rnd
data_r = ddply(subset(data, T), c("id", "group", "cat"), function(x){data.frame(trustee_earn=sum(x$trustee_earn))})
View(data_r)
df_mean <- data_r %>%
group_by(group) %>%
summarize(mean_value = mean(trustee_earn))
View(df_mean)
df_mean <- data_r %>%
filter(group == "Human vs FAIR") %>%
summarize(mean_value = mean(trustee_earn))
View(df_mean)
df_mean <- data_r %>%
filter(group == "Human vs MAX") %>%
summarize(mean_value = mean(trustee_earn))
df_mean <- data_r %>%
filter(group == "Gemini vs MAX") %>%
summarize(mean_value = mean(trustee_earn))
df_mean <- data_r %>%
filter(group == "GPT-3.5 vs MAX") %>%
summarize(mean_value = mean(trustee_earn))
df_mean <- data_r %>%
filter(group == "Gemini vs MAX") %>%
summarize(mean_value = mean(trustee_earn))
alld_rnd = rbind(subj_rnd[,c("id", "investor_earn", "group", "cat")],
subj_max[,c("id", "investor_earn", "group", "cat")],
subj_fair[,c("id", "investor_earn", "group", "cat")],
gpt_rnd[,c("id", "investor_earn", "group", "cat")],
gpt_adv_fair[,c("id", "investor_earn", "group", "cat")],
gpt_adv_max[,c("id", "investor_earn", "group", "cat")],
gpt4_rnd[,c("id", "investor_earn", "group", "cat")],
gpt4_adv_fair[,c("id", "investor_earn", "group", "cat")],
gpt4_adv_max[,c("id", "investor_earn", "group", "cat")],
gemini_rnd[,c("id", "investor_earn", "group", "cat")],
gemini_adv_max[,c("id", "investor_earn", "group", "cat")],
gemini_adv_fair[,c("id", "investor_earn", "group", "cat")])
data = alld_rnd
data_r = ddply(subset(data, T), c("id", "group", "cat"), function(x){data.frame(investor_earn=sum(x$investor_earn))})
df_mean <- data_r %>%
filter(group == "Gemini vs MAX") %>%
summarize(mean_value = mean(trustee_earn))
df_mean <- data_r %>%
filter(group == "Gemini vs MAX") %>%
summarize(mean_value = mean(investor_earn))
library(ggplot2)
source('R/utils/helper.R')
source("R/utils/summary_utils.R")
############################ GPT-3.5 data ##########################
gpt_data = read.csv('sim_data/gpt/gpt-3.5-turbo.csv')
setwd("~/Documents/Projects/adv_LLM_NIPS/bandit")
source('R/utils/helper.R')
source("R/utils/summary_utils.R")
############################ GPT-3.5 data ##########################
gpt_data = read.csv('sim_data/gpt/gpt-3.5-turbo.csv')
gpt_data$X = NULL
colnames(gpt_data) = c("subjID", "trial", "choice", "outcome", "probsX", "probsY")
gpt_data_df <- as.data.table(gpt_data)
gpt_counts <- gpt_data_df %>%
group_by(subjID) %>%
summarise(X = sum(choice == "0"),
Y = sum(choice == "1"))
gpt_counts['condition'] = "GPT-3.5"
gpt_counts['group'] = "Before"
mean(gpt_counts$X)
############################ GPT-3.5_vs_ADV ###################################
sim_gpt_list <- list()
for (i in 1:49){
data = read.csv(paste0("evaluate_model/RNN_adv_sim_400000_eps_0.01_lr_0.001/gpt-3.5-turbo/events_", i,".csv"))
sim_gpt_list <- append(sim_gpt_list, list(data))
}
sim_gpt <- do.call(rbind, sim_gpt_list)
sim_gpt['subjID'] = rep(1:49, each = 100)
sim_gpt$subjID <- paste(sim_gpt$subjID, "gpt-3.5", sep="-")
# colnames(df) = c("subjID", "trial", "choice", "outcome", "probsX", "probsY")
sim_gpt['choice'] =  factor(sim_gpt$real.model.action,
levels = c("[[1. 0.]]", "[[0. 1.]]"),
labels = c("0", "1"))
sim_gpt <- sim_gpt %>%
mutate(outcome = case_when(choice==1 & r2==0 ~ 0,
choice==1 & r2==1 ~ 1,
choice==0 & r1==0 ~ 0,
choice==0 & r1==1 ~ 1))
sim_gpt_df <- as.data.table(sim_gpt)
sim_gpt_counts <- sim_gpt_df %>%
group_by(subjID) %>%
summarise(X = sum(choice == "0"),
Y = sum(choice == "1"))
sim_gpt_counts['condition'] = "GPT-3.5"
sim_gpt_counts['group'] = "After"
mean(sim_gpt_counts$X)
#############################  human data ###########################
human_dynamic = read.csv('human_data/merged_dynamic.csv') ## 97 subjects
human_static = read.csv("human_data/merged_static.csv") ## 387 subjects
human_combined = rbind(human_dynamic, human_static)
human_combined$X = NULL
colnames(human_combined) = c("choice", "outcome", "subjID", "block", "schedule_type")
human_counts <- human_combined %>%
group_by(subjID) %>%
summarise(X = sum(choice == "0"),
Y = sum(choice == "1"))
human_counts['condition'] = "Human"
human_counts['group'] = "Before"
############################### human vs. ADV ######################
dirs = list.files("human_data/human_adv_data/", full.names = TRUE, recursive = FALSE)
require(ramify)
sim_human_list = list()
for (d in dirs){
data = read.csv(d)
sim_human_list <- append(sim_human_list, list(data))
}
sim_human <- do.call(rbind, sim_human_list)
sim_human['subjID'] = rep(1:157, each = 100)
sim_human_table <- as.data.table(sim_human)
sim_human_counts <- sim_human_table %>%
group_by(subjID) %>%
summarise(X = sum(is_biased_choice == " true"),
Y = sum(is_biased_choice == " false"))
sim_human_counts['condition'] = "Human"
sim_human_counts['group'] = "After"
############################ GPT-4 data ##########################
gpt4_data = read.csv('sim_data/gpt/gpt-4-turbo.csv')
gpt4_data$X = NULL
colnames(gpt4_data) = c("subjID", "trial", "choice", "outcome", "probsX", "probsY")
gpt4_data_df <- as.data.table(gpt4_data)
gpt4_counts <- gpt4_data_df %>%
group_by(subjID) %>%
summarise(X = sum(choice == "0"),
Y = sum(choice == "1"))
gpt4_counts['condition'] = "GPT-4"
gpt4_counts['group'] = "Before"
mean(gpt4_counts$X)
############################ GPT-4_vs_ADV ###################################
sim_gpt4_list <- list()
for (i in 0:49){
data = read.csv(paste0("evaluate_model/RNN_adv_sim_400000_eps_0.01_lr_0.001/gpt-4-turbo/events_", i,".csv"))
sim_gpt4_list <- append(sim_gpt4_list, list(data))
}
sim_gpt4 <- do.call(rbind, sim_gpt4_list)
sim_gpt4['subjID'] = rep(0:49, each = 100)
sim_gpt4$subjID <- paste(sim_gpt4$subjID, "gpt-4", sep="-")
# colnames(df) = c("subjID", "trial", "choice", "outcome", "probsX", "probsY")
sim_gpt4['choice'] =  factor(sim_gpt4$real.model.action,
levels = c("[[1. 0.]]", "[[0. 1.]]"),
labels = c("0", "1"))
sim_gpt4 <- sim_gpt4 %>%
mutate(outcome = case_when(choice==1 & r2==0 ~ 0,
choice==1 & r2==1 ~ 1,
choice==0 & r1==0 ~ 0,
choice==0 & r1==1 ~ 1))
sim_gpt4_df <- as.data.table(sim_gpt4)
sim_gpt4_counts <- sim_gpt4_df %>%
group_by(subjID) %>%
summarise(X = sum(choice == "0"),
Y = sum(choice == "1"))
sim_gpt4_counts['condition'] = "GPT-4"
sim_gpt4_counts['group'] = "After"
mean(sim_gpt4_counts$X)
############################ Gemini data ##########################
gemini_data = read.csv('sim_data/gemini/gemini_1.5.csv')
gemini_data$X = NULL
colnames(gemini_data) = c("subjID", "trial", "choice", "outcome", "probsX", "probsY")
gemini_data_df <- as.data.table(gemini_data)
gemini_counts <- gemini_data_df %>%
group_by(subjID) %>%
summarise(X = sum(choice == "0"),
Y = sum(choice == "1"))
gemini_counts['condition'] = "Gemini_1.5"
gemini_counts['group'] = "Before"
mean(gemini_counts$X)
############################ Gemini_vs_ADV ###################################
sim_gemini_list <- list()
for (i in 0:49){
data = read.csv(paste0("evaluate_model/RNN_adv_sim_400000_eps_0.01_lr_0.001/gemini_1.5/events_", i,".csv"))
sim_gemini_list <- append(sim_gemini_list, list(data))
}
sim_gemini <- do.call(rbind, sim_gemini_list)
sim_gemini['subjID'] = rep(0:49, each = 100)
sim_gemini$subjID <- paste(sim_gemini$subjID, "gemini_1.5", sep="-")
# colnames(df) = c("subjID", "trial", "choice", "outcome", "probsX", "probsY")
sim_gemini['choice'] =  factor(sim_gemini$real.model.action,
levels = c("[[1. 0.]]", "[[0. 1.]]"),
labels = c("0", "1"))
sim_gemini <- sim_gemini %>%
mutate(outcome = case_when(choice==1 & r2==0 ~ 0,
choice==1 & r2==1 ~ 1,
choice==0 & r1==0 ~ 0,
choice==0 & r1==1 ~ 1))
sim_gemini_df <- as.data.table(sim_gemini)
sim_gemini_counts <- sim_gemini_df %>%
group_by(subjID) %>%
summarise(X = sum(choice == "0"),
Y = sum(choice == "1"))
sim_gemini_counts['condition'] = "Gemini_1.5"
sim_gemini_counts['group'] = "After"
mean(gpt_counts$X)
View(gpt4_counts)
View(gpt_data)
