---
title: "Simulation"
author: "Tian Qin"
date: "2024-01-30"
output: html_document
---

## Simulation data

### Simulation

```{r,warning=FALSE}

library(RiemannLebesgueForest)
library(randomForest)
library(MASS)
library(caret)
set.seed(42)


X = data.frame(replicate(100,(runif(1500,min=0, max=1))))  #U[0,1]^100


fx <- function(x){
  
  10* prod( exp(-2*x[1:5]^2) )+sum(x[6:35])
  
  
}
ysig <- apply(X,1,fx)


yNoise = rnorm(1500,sd=1.3)
y = ysig + yNoise

train_index <- sample(1:nrow(X),size = 1000)
xtrain <- X[train_index,]
ytrain <- y[train_index]
xtest <- X[-train_index,]
ytest <- y[-train_index]
```



## Number of trees 

```{r}

# Initial setting: S=5, T=5,B=5,depth=3,step=0.5,size=20,alpha=0.7

ntrees <- 1:500

mse_rlf <- c()
mse_rf  <- c()
for(i in ntrees){
  
   rlf <- RLForest(ytrain~.,data=xtrain,ntree=i,ntreesub=10,replace=FALSE)


          mse_rlf[i] <- mean((predict(rlf,newdata = xtest)-ytest)^2)


    
    
   rf <- randomForest(ytrain~.,data=xtrain,ntree=i)
   mse_rf[i]<- mean((predict(rf,newdata =xtest)-ytest)^2)

  
}


plot(mse_rlf,type='l',col='red')
points(mse_rf,type='l',col='blue')
  #cvbpbtrf[i] <- sum(diag(rft))/sum(rft)
#plot(test_accuracies,type='l')


```


```{r}
#test_acc_nt <- data.frame(ntrees=1:500,test_MSE_RLF=mse_rlf[1:500],test_MSE_RF=mse_rf[1:500])

 f1<-ggplot(test_acc_nt,aes(x=ntrees))+
     geom_line(aes(y = test_MSE_RLF,colour="RLF"),linetype = "solid", size=2) + 
   geom_line(aes(y = test_MSE_RF,colour="RF"),linetype = "solid", size=2) + 
          scale_colour_manual("", 
                      values = c("RLF"="orange", "RF"="blue")) +
   xlab('Number of trees')+
  ylab('Mean squared error') +
   theme(text = element_text(family = "serif"))+
theme(panel.grid.major = element_blank(), panel.grid.minor = element_blank(),
panel.background = element_blank(), axis.line = element_line(colour = "black"),panel.border  = element_rect(color = "black",fill = NA,size = 1))+
 theme(axis.text.x = element_text(vjust = 0.5, hjust=1,size=40),axis.ticks.length=unit(.5, "cm"), legend.position=c(0.9,0.25),legend.key = element_rect(colour = NA, fill = NA),axis.title.x=element_text(size = 40))+
 theme(axis.text.y = element_text(vjust = 0.5, hjust=1,size=40),axis.ticks.length=unit(.5, "cm"), legend.position=c(0.9,0.25),legend.key = element_rect(colour = NA, fill = NA),axis.title.y=element_text(size = 30))+
    theme(legend.position =c(0.8,0.78))+
   theme(axis.text=element_text(size=50), legend.text = element_text(size=35))+
theme(legend.key.width = unit(3,"cm"))

f1





```


## Number of local trees 

```{r}
set.seed(42)
# Initial setting: S=5, T=5,B=5,depth=3,step=0.5,size=20,alpha=0.7

nsubtrees <- 1:50

mse_rlf <- c()
mse_rf  <- c()
   rf <- randomForest(ytrain~.,data=xtrain,ntree=100)
   mse_rf[1]<- mean((predict(rf,newdata =xtest)-ytest)^2)
for(i in nsubtrees){
  
   rlf <- RLForest(ytrain~.,data=xtrain,ntree=100,ntreesub=i,replace=FALSE)


          mse_rlf[i] <- mean((predict(rlf,newdata = xtest)-ytest)^2)



  
}


plot(mse_rlf,type='l',col='red')
abline(h=mse_rf,type='l',col='blue')
  #cvbpbtrf[i] <- sum(diag(rft))/sum(rft)
#plot(test_accuracies,type='l')


```


```{r}
test_acc_nst <- data.frame(nsubtrees=nsubtrees,test_MSE_RLF=mse_rlf,test_MSE_RF=rep(mse_rf,length(nsubtrees)))

 f2<-ggplot(test_acc_nst,aes(x=nsubtrees))+
     geom_line(aes(y = test_MSE_RLF,colour="RLF"),linetype = "solid", size=2) + 
   geom_line(aes(y = test_MSE_RF,colour="RF"),linetype = "solid", size=2) + 
          scale_colour_manual("", 
                      values = c("RLF"="orange", "RF"="blue")) +
   xlab('Number of local trees')+
  ylab('Mean squared error') +
   theme(text = element_text(family = "serif"))+
   scale_y_continuous(limits=c(3,5.0))+
theme(panel.grid.major = element_blank(), panel.grid.minor = element_blank(),
panel.background = element_blank(), axis.line = element_line(colour = "black"),panel.border  = element_rect(color = "black",fill = NA,size = 1))+
 theme(axis.text.x = element_text(vjust = 0.5, hjust=1,size=40),axis.ticks.length=unit(0.5, "cm"),legend.position=c(0.9,0.25),legend.key = element_rect(colour = NA, fill = NA),axis.title.x=element_text(size = 40))+
 theme(axis.text.y = element_text(vjust = 0.5, hjust=1,size=40),axis.ticks.length=unit(0.5, "cm"), legend.position=c(0.9,0.25),legend.key = element_rect(colour = NA, fill = NA),axis.title.y=element_text(size = 30))+
    theme(legend.position =c(0.8,0.78))+
   theme(axis.text=element_text(size=50), legend.text = element_text(size=35))+
theme(legend.key.width = unit(3,"cm"))

f2






```



## Noisy variables

```{r}

set.seed(42)
# Initial setting: S=5, T=5,B=5,depth=3,step=0.5,size=20,alpha=0.7

nv <- 35:100

mse_rlf_ed <- c()
mse_rf_ed  <- c()

for(i in nv){
  
   rlf <- RLForest(ytrain~.,data=xtrain[,1:i],ntree=100,ntreesub=10,replace=FALSE)


          mse_rlf_ed[i-34] <- mean((predict(rlf,newdata = xtest[,1:i])-ytest)^2)


   rf <- randomForest(ytrain~.,data=xtrain[,1:i],ntree=100)
   mse_rf_ed[i-34]<- mean((predict(rf,newdata =xtest[,1:i])-ytest)^2)
  
}


plot(mse_rlf_ed,type='l',col='red')
points(mse_rf_ed,type='l',col='blue')

  #cvbpbtrf[i] <- sum(diag(rft))/sum(rft)
#plot(test_accuracies_B,type='l')


```


```{r}
test_acc_nv <- data.frame(nvs=nv-35,test_MSE_RLF=mse_rlf_ed,test_MSE_RF=mse_rf_ed)

 f3<-ggplot(test_acc_nv,aes(x=nvs))+
     geom_line(aes(y = test_MSE_RLF,colour="RLF"),linetype = "solid", size=2) + 
   geom_line(aes(y = test_MSE_RF,colour="RF"),linetype = "solid", size=2) + 
          scale_colour_manual("", 
                      values = c("RLF"="orange", "RF"="blue")) +
   xlab('Number of noisy variables')+
  ylab('Mean squared error') +
   theme(text = element_text(family = "serif"))+
   scale_y_continuous(limits=c(3,5.0))+
theme(panel.grid.major = element_blank(), panel.grid.minor = element_blank(),
panel.background = element_blank(), axis.line = element_line(colour = "black"),panel.border  = element_rect(color = "black",fill = NA,size = 1))+
 theme(axis.text.x = element_text(vjust = 0.5, hjust=1,size=40),axis.ticks.length=unit(0.5, "cm"),legend.position=c(0.9,0.25),legend.key = element_rect(colour = NA, fill = NA),axis.title.x=element_text(size = 40))+
 theme(axis.text.y = element_text(vjust = 0.5, hjust=1,size=40),axis.ticks.length=unit(0.5, "cm"), legend.position=c(0.9,0.25),legend.key = element_rect(colour = NA, fill = NA),axis.title.y=element_text(size = 30))+
    theme(legend.position =c(0.8,0.78))+
   theme(axis.text=element_text(size=50), legend.text = element_text(size=35))+
theme(legend.key.width = unit(3,"cm"))

f3





```


## alpha

```{r}

set.seed(42)
# Initial setting: S=5, T=5,B=5,depth=3,step=0.5,size=20,alpha=0.7

alphas <- seq(0.4,1,0.01)

mse_rlf_a <- c()
mse_rf_a <- c()
   rf <- randomForest(ytrain~.,data=xtrain,ntree=100)
   mse_rf_a[1]<- mean((predict(rf,newdata =xtest)-ytest)^2)
for(i in 1:length(alphas)){
  
   rlf <- RLForest(ytrain~.,data=xtrain,ntree=100,ntreesub=10,replace=FALSE,sampsize=ceiling(alphas[i]*nrow(xtrain)))


          mse_rlf_a[i] <- mean((predict(rlf,newdata = xtest)-ytest)^2)



  
}


plot(mse_rlf_a,type='l',col='red')
points(mse_rf_a,type='l',col='blue')



```


```{r}
test_acc_a <- data.frame(alphas=alphas,test_MSE_RLF=mse_rlf_a,test_MSE_RF= rep(mse_rf_a,length(alphas)))

 f4<-ggplot(test_acc_a,aes(x=alphas))+
     geom_line(aes(y = test_MSE_RLF,colour="RLF"),linetype = "solid", size=2) + 
   geom_line(aes(y = test_MSE_RF,colour="RF"),linetype = "solid", size=2) + 
          scale_colour_manual("", 
                      values = c("RLF"="orange", "RF"="blue")) +
   xlab('Subagging ratio (RLF)')+
  ylab('Mean squared error') +
   theme(text = element_text(family = "serif"))+
   scale_y_continuous(limits=c(3,5.0))+
theme(panel.grid.major = element_blank(), panel.grid.minor = element_blank(),
panel.background = element_blank(), axis.line = element_line(colour = "black"),panel.border  = element_rect(color = "black",fill = NA,size = 1))+
 theme(axis.text.x = element_text(vjust = 0.5, hjust=1,size=40),axis.ticks.length=unit(0.5, "cm"),legend.position=c(0.9,0.25),legend.key = element_rect(colour = NA, fill = NA),axis.title.x=element_text(size = 40))+
 theme(axis.text.y = element_text(vjust = 0.5, hjust=1,size=40),axis.ticks.length=unit(0.5, "cm"), legend.position=c(0.9,0.25),legend.key = element_rect(colour = NA, fill = NA),axis.title.y=element_text(size = 30))+
    theme(legend.position =c(0.8,0.78))+
   theme(axis.text=element_text(size=50), legend.text = element_text(size=35))+
theme(legend.key.width = unit(3,"cm"))

f4




```



## Lp

```{r}

set.seed(42)
# Initial setting: S=5, T=5,B=5,depth=3,step=0.5,size=20,alpha=0.7

Lps <- seq(0.1,1,0.05)

mse_rlf_p <- c()
mse_rf_p <- c()
   rf <- randomForest(ytrain~.,data=xtrain,ntree=100)
   mse_rf_p[1]<- mean((predict(rf,newdata =xtest)-ytest)^2)
for(i in 1:length(Lps)){
  
   rlf <- RLForest(ytrain~.,data=xtrain,ntree=100,ntreesub=10,replace=FALSE,Lp=Lps[i])


          mse_rlf_p[i] <- mean((predict(rlf,newdata = xtest)-ytest)^2)



  
}


plot(mse_rlf_p,type='l',col='red')
points(mse_rf_p,type='l',col='blue')



```


```{r}
library(ggplot2)
library(latex2exp)
test_acc_p <- data.frame(Lps=Lps,test_MSE_RLF=mse_rlf_p,test_MSE_RF= rep(mse_rf_p,length(Lps)))

 f5<-ggplot(test_acc_p,aes(x=Lps))+
     geom_line(aes(y = test_MSE_RLF,colour="RLF"),linetype = "solid", size=2) + 
   geom_line(aes(y = test_MSE_RF,colour="RF"),linetype = "solid", size=2) + 
          scale_colour_manual("", 
                      values = c("RLF"="orange", "RF"="blue")) +
   xlab(label=unname(TeX("Control probability $\\tilde{p}$")))+
  ylab('Mean squared error') +
   theme(text = element_text(family = "serif"))+
   scale_y_continuous(limits=c(3,5.0))+
theme(panel.grid.major = element_blank(), panel.grid.minor = element_blank(),
panel.background = element_blank(), axis.line = element_line(colour = "black"),panel.border  = element_rect(color = "black",fill = NA,size = 1))+
 theme(axis.text.x = element_text(vjust = 0.5, hjust=1,size=40),axis.ticks.length=unit(0.5, "cm"),legend.position=c(0.9,0.25),legend.key = element_rect(colour = NA, fill = NA),axis.title.x=element_text(size = 40))+
 theme(axis.text.y = element_text(vjust = 0.5, hjust=1,size=40),axis.ticks.length=unit(0.5, "cm"), legend.position=c(0.9,0.25),legend.key = element_rect(colour = NA, fill = NA),axis.title.y=element_text(size = 30))+
    theme(legend.position =c(0.8,0.78))+
   theme(axis.text=element_text(size=50), legend.text = element_text(size=35))+
theme(legend.key.width = unit(3,"cm"))

f5




```





## Example 2: Mixture model 

```{r}
library(plotly)
xSeq = seq(-2, 2, 0.01)

set.seed(42)
n1  = 3000
x = rnorm(n1)

p = sample(c(1,2),size=n1,prob=c(1/2,1/2),replace=TRUE)
y <- c()
for(i in 1:n1){
  error <- rnorm(1)
  if(p[i]==1){
    y[i] <- 5*x[i]+error
  }else if(p[i]==2){
    y[i] <- 10+5*x[i] +error
  }
    

}


data <- data.frame(intercept=1,x=x,y=y)

#train_index <- sample(1:nrow(data),size = 2500)

train_index <- createDataPartition(data$y, p =0.8, 
                                  list = FALSE, 
                                  times = 1)
xtrain <- data[train_index,1:2]
ytrain <- data[train_index,3]

validation_index<- createDataPartition(ytrain,
                                           p = .75,
                                           list = FALSE,
                                           times = 1)
xtrain_train <- xtrain[validation_index,1:2]
ytrain_train <- ytrain[validation_index]
xtrain_valid <- xtrain[-validation_index,1:2]
ytrain_valid <- ytrain[-validation_index]
xtest <- data[-train_index,1:2]
ytest <- data[-train_index,3]
result <- data.frame(x=xtest,sampley=ytest,xeq= seq(-2, 2, length=600))
ordered_result <- result[order(result[,2]),]
fx1 <- function(x){
  5*x
}

fx2 <- function(x){
  5*x+10
}

 f9<-ggplot(result,aes(x=xeq))+
      xlab('x')+
      stat_function(fun=fx1,colour = "blue",linetype = "solid", size=2)+
      stat_function(fun=fx2,colour = "blue",linetype = "solid", size=2)+
   scale_color_manual(values=c('samply'="orange","rf"="red","rlf"="green"))+
      geom_point(aes(x=x.x,y = sampley,color='samply'),size=2) + 
     theme(legend.position="none")+
    ylab('y')+
   theme(panel.grid.major = element_blank(), panel.grid.minor = element_blank(),
panel.background = element_blank(), axis.line = element_line(colour = "black"),panel.border  = element_rect(color = "black",fill = NA,size = 1))+
   theme(axis.text=element_text(size=50), legend.text = element_text(size=35))+
    theme(axis.text.x = element_text(vjust = 0.5, hjust=1,size=40),axis.ticks.length=unit(0.5, "cm"),axis.title.x=element_text(size = 40))+
 theme(axis.text.y = element_text(vjust = 0.5, hjust=1,size=40),axis.ticks.length=unit(0.5, "cm"),axis.title.y=element_text(size = 30))
f9
```


```{r}
set.seed(42)
hyper_grid_rf <- expand.grid(
  sample.fraction = c(.5, .63, .8),
  min.node.size = c( 5, 10,15), 
  n.trees = c(50,100,150,200),
  mse=NA
)
# execute full cartesian grid search
for(i in seq_len(nrow(hyper_grid_rf))) {
  # fit model for ith hyperparameter combination
  fit <- randomForest(
    formula         = ytrain_train ~ ., 
    data            = xtrain_train, 
    ntree       = hyper_grid_rf$n.trees[i],
    sampsize            =ceiling(hyper_grid_rf$sample.fraction[i]*nrow(xtrain_train))  ,
    nodesize    = hyper_grid_rf$min.node.size[i],
    verbose         = FALSE
  )
  # export OOB error 
  hyper_grid_rf$mse[i] <- mean((predict(fit,newdata =xtrain_valid)-ytrain_valid)^2)
}
hyper_grid_rf %>%
  arrange(mse) %>%
  head(10)

ordered_hyper_grid_rf <- hyper_grid_rf[order(hyper_grid_rf$mse),]

```


```{r}
hyper_grid_rlf <- expand.grid(
  Lp=c(0.2,0.4,0.6,0.8),
  ntreesub=c(10,20,50),
  mse=NA
)
# execute full cartesian grid search
for(i in seq_len(nrow(hyper_grid_rlf))) {
  # fit model for ith hyperparameter combination
  fit <- RLForest(
    formula         = ytrain_train ~ ., 
    data            = xtrain_train, 
    ntree       = 100,
    replace         = FALSE,
    ntreesub=hyper_grid_rlf$ntreesub[i],
    Lp = hyper_grid_rlf$Lp[i],

    verbose         = FALSE
  )
  # export OOB error 
  hyper_grid_rlf$mse[i] <- mean((predict(fit,newdata =xtrain_valid)-ytrain_valid)^2)
}
hyper_grid_rlf %>%
  arrange(mse) %>%
  head(10)
ordered_hyper_grid_rlf <- hyper_grid_rlf[order(hyper_grid_rlf$mse),]


```



```{r}




   rf <- randomForest(
    formula         = ytrain_train ~ ., 
    data            = xtrain_train, 
    ntree       = ordered_hyper_grid_rf$n.trees[1],
     sampsize            =ceiling(ordered_hyper_grid_rf$sample.fraction[1]*nrow(xtrain_train))  ,
    nodesize    = ordered_hyper_grid_rf$min.node.size[1],
    verbose         = FALSE
  )
   mean((predict(rf,newdata =xtest)-ytest)^2)
   
  #  
      rlf <-RLForest(
    formula         = ytrain_train ~ .,
    data            = xtrain_train,
    ntree=100,
        ntreesub=ordered_hyper_grid_rlf$ntreesub[1],
    Lp = hyper_grid_rlf$Lp[1],
    replace=FALSE,
    verbose         = FALSE
  ) #RLForest(ytrain_train~.,data=xtrain_train,ntree=100,ntreesub=10,replace=FALSE,Lp=Lps[i])


      mean((predict(rlf,newdata = xtest)-ytest)^2)




```

```{r}
##random forest
#rf <- randomForest(ytrain~.,data=xtrain,ntree=100)
pred_rf <- predict(rf,newdata =xtest)
mean((pred_rf-ytest)^2)
#elf <- RLForest(ytrain~.,data=as.data.frame(xtrain),ntree=100,ntreesub=10,replace=FALSE,data_driven=TRUE)
pred_rlf <- predict(rlf,newdata =xtest)
mean((pred_rlf-ytest)^2)
result <- data.frame(x=xtest, y_pred_rf=pred_rf,y_pred_rlf=pred_rlf,sampley=ytest)
ordered_result <- result[order(result$x.x),]


 test_acc_sin <- ordered_result#data.frame(Lps=Lps,test_MSE_RLF=mse_rlf_p,test_MSE_RF= rep(mse_rf_p,length(Lps)))

 f8<-ggplot(test_acc_sin,aes(x=x.x))+
      stat_function(fun=fx1,colour = "blue",linetype = "solid", size=2)+
      stat_function(fun=fx2,colour = "blue",linetype = "solid", size=2)+
      geom_point(aes(y = sampley,color='samply'),size=1) + 
     geom_line(aes(y = y_pred_rf,color="rf"), size=1) + 
        geom_line(aes(y = y_pred_rlf,color="rlf"), size=1) +
   scale_color_manual(values=c('samply'="orange","rf"="red","rlf"="green"))+
     theme(legend.position="none")+
   xlab('x')+
    ylab('y')+
   theme(panel.grid.major = element_blank(), panel.grid.minor = element_blank(),
panel.background = element_blank(), axis.line = element_line(colour = "black"),panel.border  = element_rect(color = "black",fill = NA,size = 1))+
   theme(axis.text=element_text(size=50), legend.text = element_text(size=35))+
    theme(axis.text.x = element_text(vjust = 0.5, hjust=1,size=40),axis.ticks.length=unit(0.5, "cm"),axis.title.x=element_text(size = 40))+
 theme(axis.text.y = element_text(vjust = 0.5, hjust=1,size=40),axis.ticks.length=unit(0.5, "cm"),axis.title.y=element_text(size = 30))
f8

```


```{r}
library(ggplot2)
library(latex2exp)
test_acc_px3 <- data.frame(Lps=Lps,test_MSE_RLF=mse_rlf_p,test_MSE_RF= rep(mse_rf_p,length(Lps)))

 f10<-ggplot(test_acc_px3,aes(x=Lps))+
     geom_line(aes(y = test_MSE_RLF,colour="RLF"),linetype = "solid", size=2) + 
   geom_line(aes(y = test_MSE_RF,colour="RF"),linetype = "solid", size=2) + 
          scale_colour_manual("", 
                      values = c("RLF"="orange", "RF"="blue")) +
   xlab(label=unname(TeX("$\\tilde{p}$")))+
  ylab('Mean squared error') +
   theme(text = element_text(family = "serif"))+
   scale_y_continuous(limits=c(28,45))+
theme(panel.grid.major = element_blank(), panel.grid.minor = element_blank(),
panel.background = element_blank(), axis.line = element_line(colour = "black"),panel.border  = element_rect(color = "black",fill = NA,size = 1))+
 theme(axis.text.x = element_text(vjust = 0.5, hjust=1,size=40),axis.ticks.length=unit(0.5, "cm"),legend.position=c(0.9,0.25),legend.key = element_rect(colour = NA, fill = NA),axis.title.x=element_text(size = 40))+
 theme(axis.text.y = element_text(vjust = 0.5, hjust=1,size=40),axis.ticks.length=unit(0.5, "cm"), legend.position=c(0.9,0.25),legend.key = element_rect(colour = NA, fill = NA),axis.title.y=element_text(size = 30))+
    theme(legend.position =c(0.8,0.78))+
   theme(axis.text=element_text(size=50), legend.text = element_text(size=35))+
theme(legend.key.width = unit(3,"cm"))

f10


```


## Test curve of RLF and RF for example 2

```{r}

set.seed(42)
# Initial setting: S=5, T=5,B=5,depth=3,step=0.5,size=20,alpha=0.7

Lps <- seq(0.1,1,0.05)

mse_rlf_p <- c()
mse_rf_p <- c()
   rf <- randomForest(ytrain~.,data=xtrain,ntree=100)
   mse_rf_p[1]<- mean((predict(rf,newdata =xtest)-ytest)^2)
for(i in 1:length(Lps)){
  
   rlf <- RLForest(ytrain~.,data=xtrain,ntree=100,ntreesub=10,replace=FALSE,Lp=Lps[i])


          mse_rlf_p[i] <- mean((predict(rlf,newdata = xtest)-ytest)^2)



  
}


plot(mse_rlf_p,type='l',col='red')
points(mse_rf_p,type='l',col='blue')



```


```{r}
library(ggplot2)
library(latex2exp)
test_acc_p <- data.frame(Lps=Lps,test_MSE_RLF=mse_rlf_p,test_MSE_RF= rep(mse_rf_p,length(Lps)))

 f5<-ggplot(test_acc_p,aes(x=Lps))+
     geom_line(aes(y = test_MSE_RLF,colour="RLF"),linetype = "solid", size=2) + 
   geom_line(aes(y = test_MSE_RF,colour="RF"),linetype = "solid", size=2) + 
          scale_colour_manual("", 
                      values = c("RLF"="orange", "RF"="blue")) +
   xlab(label=unname(TeX("Control probability $\\tilde{p}$")))+
  ylab('Mean squared error') +
   theme(text = element_text(family = "serif"))+
   scale_y_continuous(limits=c(27,50))+
theme(panel.grid.major = element_blank(), panel.grid.minor = element_blank(),
panel.background = element_blank(), axis.line = element_line(colour = "black"),panel.border  = element_rect(color = "black",fill = NA,size = 1))+
 theme(axis.text.x = element_text(vjust = 0.5, hjust=1,size=40),axis.ticks.length=unit(0.5, "cm"),legend.position=c(0.9,0.25),legend.key = element_rect(colour = NA, fill = NA),axis.title.x=element_text(size = 40))+
 theme(axis.text.y = element_text(vjust = 0.5, hjust=1,size=40),axis.ticks.length=unit(0.5, "cm"), legend.position=c(0.9,0.25),legend.key = element_rect(colour = NA, fill = NA),axis.title.y=element_text(size = 30))+
    theme(legend.position =c(0.8,0.78))+
   theme(axis.text=element_text(size=50), legend.text = element_text(size=35))+
theme(legend.key.width = unit(3,"cm"))

f5




```




### Sin



## One dimension case （sin16X)





```{r}


library(plotly)
xSeq = seq(0, 1, 0.01)

set.seed(0217)

n <- 3000


X = data.frame(x=runif(n,min=0, max=1))#data.frame(x=runif(n,min=0, max=1)) #data.frame(replicate(100,(runif(1500,min=0, max=1))))  #U[0,1]^100


fx <- function(x){

  sin(16*x)#+cos(16*x)
 # 2*exp(-2*x)
}
ysig <- apply(X,1,fx)


yNoise = rnorm(n,sd=1)
y = ysig + yNoise


data <- data.frame(intercept=1,x=X$x,y=y)

#train_index <- sample(1:nrow(data),size = 2500)

train_index <- createDataPartition(data$y, p =0.8, 
                                  list = FALSE, 
                                  times = 1)
xtrain <- data[train_index,1:2]
ytrain <- data[train_index,3]

validation_index<- createDataPartition(ytrain,
                                           p = .75,
                                           list = FALSE,
                                           times = 1)
xtrain_train <- xtrain[validation_index,1:2]
ytrain_train <- ytrain[validation_index]
xtrain_valid <- xtrain[-validation_index,1:2]
ytrain_valid <- ytrain[-validation_index]
xtest <- data[-train_index,1:2]
ytest <- data[-train_index,3]
result <- data.frame(x=xtest,sampley=ytest,xeq= seq(0, 1, length=600))
#ordered_result <- result[order(result[,2]),]







```






```{r}
 f10<-ggplot(result,aes(x=xeq))+
      xlab('x')+
      stat_function(fun=fx,colour = "blue",linetype = "solid", size=2)+
   scale_color_manual(values=c('samply'="orange","rf"="red","rlf"="green"))+
      geom_point(aes(x=x.x,y = sampley,color='samply'),size=2) + 
     theme(legend.position="none")+
    ylab('y')+
   theme(panel.grid.major = element_blank(), panel.grid.minor = element_blank(),
panel.background = element_blank(), axis.line = element_line(colour = "black"),panel.border  = element_rect(color = "black",fill = NA,size = 1))+
   theme(axis.text=element_text(size=50), legend.text = element_text(size=35))+
    theme(axis.text.x = element_text(vjust = 0.5, hjust=1,size=40),axis.ticks.length=unit(0.5, "cm"),axis.title.x=element_text(size = 40))+
 theme(axis.text.y = element_text(vjust = 0.5, hjust=1,size=40),axis.ticks.length=unit(0.5, "cm"),axis.title.y=element_text(size = 30))
f10


```


```{r}


hyper_grid_rf <- expand.grid(
   sample.fraction = c(.5, .63, .8),
  min.node.size = c( 5, 10,15), 
  n.trees = c(50,100,150,200),
  mse=NA
)
# execute full cartesian grid search
for(i in seq_len(nrow(hyper_grid_rf))) {
  # fit model for ith hyperparameter combination
  fit <- randomForest(
    formula         = ytrain_train ~ ., 
    data            = xtrain_train, 
    ntree       = hyper_grid_rf$n.trees[i],
    sampsize             = ceiling(hyper_grid_rf$sample.fraction[i]*nrow(xtrain_train)) ,
    nodesize    = hyper_grid_rf$min.node.size[i],
    verbose         = FALSE
  )
  # export OOB error 
  hyper_grid_rf$mse[i] <- mean((predict(fit,newdata =xtrain_valid)-ytrain_valid)^2)
}
hyper_grid_rf %>%
  arrange(mse) %>%
  head(10)


ordered_hyper_grid_rf <- hyper_grid_rf[order(hyper_grid_rf$mse),]



```




```{r}
hyper_grid_rlf <- expand.grid(
  Lp=c(0.2,0.4,0.6,0.8),
  ntreesub=c(10,20,50),
  mse=NA
)
# execute full cartesian grid search
for(i in seq_len(nrow(hyper_grid_rlf))) {
  # fit model for ith hyperparameter combination
  fit <- RLForest(
    formula         = ytrain_train ~ ., 
    data            = xtrain_train, 
    ntree       = 100,
    replace         = FALSE,
    ntreesub=hyper_grid_rlf$ntreesub[i],
    Lp = hyper_grid_rlf$Lp[i],

    verbose         = FALSE
  )
  # export OOB error 
  hyper_grid_rlf$mse[i] <- mean((predict(fit,newdata =xtrain_valid)-ytrain_valid)^2)
}
hyper_grid_rlf %>%
  arrange(mse) %>%
  head(10)

ordered_hyper_grid_rlf <- hyper_grid_rlf[order(hyper_grid_rlf$mse),]

```


```{r}


   rf <- randomForest(
    formula         = ytrain_train ~ ., 
    data            = xtrain_train, 
    ntree       = ordered_hyper_grid_rf$n.trees[1],
     sampsize            =ceiling(ordered_hyper_grid_rf$sample.fraction[1]*nrow(xtrain_train))  ,
    nodesize    = ordered_hyper_grid_rf$min.node.size[1],
    verbose         = FALSE
  )
   mean((predict(rf,newdata =xtest)-ytest)^2)
   
  #  
      rlf <-RLForest(
    formula         = ytrain_train ~ .,
    data            = xtrain_train,
    ntree=100,
        ntreesub=ordered_hyper_grid_rlf$ntreesub[1],
    Lp = ordered_hyper_grid_rlf$Lp[1],
    replace=FALSE,
    verbose         = FALSE
  ) #RLForest(ytrain_train~.,data=xtrain_train,ntree=100,ntreesub=10,replace=FALSE,Lp=Lps[i])


    mean((predict(rlf,newdata =xtest)-ytest)^2)



```



```{r}
##random forest
#rf <- randomForest(ytrain~.,data=xtrain,ntree=100)
pred_rf <- predict(rf,newdata =xtest)
mean((pred_rf-ytest)^2)
#elf <- RLForest(ytrain~.,data=as.data.frame(xtrain),ntree=100,ntreesub=10,replace=FALSE,data_driven=TRUE)
pred_rlf <- predict(rlf,newdata =xtest)
mean((pred_rlf-ytest)^2)
result <- data.frame(x=xtest, y_pred_rf=pred_rf,y_pred_rlf=pred_rlf,sampley=ytest)
ordered_result <- result[order(result$x.x),]


 test_acc_sin <- ordered_result#data.frame(Lps=Lps,test_MSE_RLF=mse_rlf_p,test_MSE_RF= rep(mse_rf_p,length(Lps)))
# 
#  f6<-ggplot(test_acc_sin,aes(x=x))+
#      geom_line(aes(y = y_pred_rf,colour="rf"),linetype = "solid", size=1) + 
#         geom_line(aes(y = y_pred_rlf,colour="rlf"),linetype = "solid", size=1) + 
# stat_function(fun=fx,colour = "blue",linetype = "solid", size=2)+
#      scale_color_manual(values=c("rf"="red","rlf"="green"))+
#      theme(legend.position="none")+
#     ylab('y')+
#    theme(panel.grid.major = element_blank(), panel.grid.minor = element_blank(),
# panel.background = element_blank(), axis.line = element_line(colour = "black"),panel.border  = element_rect(color = "black",fill = NA,size = 1))+
#    theme(axis.text=element_text(size=50), legend.text = element_text(size=35))+
#     theme(axis.text.x = element_text(vjust = 0.5, hjust=1,size=40),axis.ticks.length=unit(0.5, "cm"),axis.title.x=element_text(size = 40))+
#  theme(axis.text.y = element_text(vjust = 0.5, hjust=1,size=40),axis.ticks.length=unit(0.5, "cm"),axis.title.y=element_text(size = 30))
# f6

 f7<-ggplot(test_acc_sin,aes(x=x.x))+
      geom_point(aes(y = sampley,color='samply'),size=1) + 
     geom_line(aes(y = y_pred_rf,color="rf"), size=1) + 
        geom_line(aes(y = y_pred_rlf,color="rlf"), size=1) +
   stat_function(fun=fx,colour = "blue",linetype = "solid", size=2)+
   scale_color_manual(values=c('samply'="orange","rf"="red","rlf"="green"))+
     theme(legend.position="none")+
   xlab('x')+
    ylab('y')+
   theme(panel.grid.major = element_blank(), panel.grid.minor = element_blank(),
panel.background = element_blank(), axis.line = element_line(colour = "black"),panel.border  = element_rect(color = "black",fill = NA,size = 1))+
   theme(axis.text=element_text(size=50), legend.text = element_text(size=35))+
    theme(axis.text.x = element_text(vjust = 0.5, hjust=1,size=40),axis.ticks.length=unit(0.5, "cm"),axis.title.x=element_text(size = 40))+
 theme(axis.text.y = element_text(vjust = 0.5, hjust=1,size=40),axis.ticks.length=unit(0.5, "cm"),axis.title.y=element_text(size = 30))
f7

```


## Test curve of RLF and RF for example 2

```{r}

set.seed(42)
# Initial setting: S=5, T=5,B=5,depth=3,step=0.5,size=20,alpha=0.7

Lps <- seq(0.1,1,0.05)

mse_rlf_p <- c()
mse_rf_p <- c()
   rf <- randomForest(ytrain~.,data=xtrain,ntree=100)
   mse_rf_p[1]<- mean((predict(rf,newdata =xtest)-ytest)^2)
for(i in 1:length(Lps)){
  
   rlf <- RLForest(ytrain~.,data=xtrain,ntree=100,ntreesub=10,replace=FALSE,Lp=Lps[i])


          mse_rlf_p[i] <- mean((predict(rlf,newdata = xtest)-ytest)^2)



  
}


plot(mse_rlf_p,type='l',col='red')
points(mse_rf_p,type='l',col='blue')



```


```{r}
library(ggplot2)
library(latex2exp)
test_acc_p <- data.frame(Lps=Lps,test_MSE_RLF=mse_rlf_p,test_MSE_RF= rep(mse_rf_p,length(Lps)))

 f12<-ggplot(test_acc_p,aes(x=Lps))+
     geom_line(aes(y = test_MSE_RLF,colour="RLF"),linetype = "solid", size=2) + 
   geom_line(aes(y = test_MSE_RF,colour="RF"),linetype = "solid", size=2) + 
          scale_colour_manual("", 
                      values = c("RLF"="orange", "RF"="blue")) +
   xlab(label=unname(TeX("Control probability $\\tilde{p}$")))+
  ylab('Mean squared error') +
   theme(text = element_text(family = "serif"))+
   scale_y_continuous(limits=c(1,2))+
theme(panel.grid.major = element_blank(), panel.grid.minor = element_blank(),
panel.background = element_blank(), axis.line = element_line(colour = "black"),panel.border  = element_rect(color = "black",fill = NA,size = 1))+
 theme(axis.text.x = element_text(vjust = 0.5, hjust=1,size=40),axis.ticks.length=unit(0.5, "cm"),legend.position=c(0.9,0.25),legend.key = element_rect(colour = NA, fill = NA),axis.title.x=element_text(size = 40))+
 theme(axis.text.y = element_text(vjust = 0.5, hjust=1,size=40),axis.ticks.length=unit(0.5, "cm"), legend.position=c(0.9,0.25),legend.key = element_rect(colour = NA, fill = NA),axis.title.y=element_text(size = 30))+
    theme(legend.position =c(0.75,0.78))+
   theme(axis.text=element_text(size=50), legend.text = element_text(size=35))+
theme(legend.key.width = unit(3,"cm"))

f12




```
