#!/usr/bin/env python
# coding: utf-8

# In[1]:


from google.colab import drive
drive.mount('/content/gdrive')


# In[2]:


pip install PIMS


# In[4]:


pip install fastcluster


# In[5]:


import torch
import numpy as np
from torchvision import datasets
import torchvision.transforms as transforms
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
import matplotlib.pyplot as plt
import torchvision
import torch
from torchvision import transforms

plt.rcParams['axes.facecolor'] = 'white'
plt.rcParams['figure.figsize'] = 9, 6


# In[6]:


get_ipython().run_line_magic('matplotlib', 'inline')

# importing relevant libraries
import numpy as np
import pandas as pd
import scipy as sp
import matplotlib.pyplot as plt
import sklearn
from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier
from sklearn.preprocessing import OneHotEncoder, StandardScaler
from sklearn.model_selection import cross_val_predict, StratifiedKFold
from sklearn.metrics import roc_auc_score, average_precision_score, precision_recall_curve, auc#plot_precision_recall_curve
from sklearn.datasets import make_classification
from sklearn.preprocessing import MinMaxScaler
from tqdm import tqdm
from umap import UMAP
from pynndescent import NNDescent
from fastcluster import single
from scipy.cluster.hierarchy import cut_tree, fcluster, dendrogram
from scipy.spatial.distance import squareform
from sklearn.tree import DecisionTreeClassifier, ExtraTreeClassifier
from pims import ImageSequence
from PIL import Image
from scipy.spatial.distance import hamming
import warnings
warnings.filterwarnings('ignore')

# turning off automatic plot showing, and setting style
plt.style.use('bmh')


# In[ ]:


if torch.cuda.is_available():
    device = torch.device('cuda')
else:
    device = torch.device('cpu')

print(device)


# In[ ]:


# define the NN architecture
class ConvAutoencoder(nn.Module):
    def __init__(self):
        super(ConvAutoencoder, self).__init__()
        ## encoder layers ##
        # conv layer (depth from 1 --> 16), 3x3 kernels
        self.conv1 = nn.Conv2d(1, 16, 3, padding=1)
        # conv layer (depth from 16 --> 4), 3x3 kernels
        self.conv2 = nn.Conv2d(16, 4, 3, padding=1)
        # pooling layer to reduce x-y dims by two; kernel and stride of 2
        self.pool = nn.MaxPool2d(2, 2)

        ## decoder layers ##
        ## a kernel of 2 and a stride of 2 will increase the spatial dims by 2
        self.t_conv1 = nn.ConvTranspose2d(4, 16, 2, stride=2)
        self.t_conv2 = nn.ConvTranspose2d(16, 1, 2, stride=2)


    def forward(self, x):
        ## encode ##
        # add hidden layers with relu activation function
        # and maxpooling after
        x = F.relu(self.conv1(x))
        x = self.pool(x)
        # add second hidden layer
        x = F.relu(self.conv2(x))
        x = self.pool(x)  # compressed representation

        ## decode ##
        # add transpose conv layers, with relu activation function
        x = F.relu(self.t_conv1(x))
        # output layer (with sigmoid for scaling from 0 to 1)
        x = F.sigmoid(self.t_conv2(x))

        return x

    def forward_encoder(self, x):
        x = F.relu(self.conv1(x))
        x = self.pool(x)
        # add second hidden layer
        x = F.relu(self.conv2(x))
        x = self.pool(x)  # compressed representation

        return x


# Training function
def train(model, train_loader, criterion, optimizer, num_epochs=10, online = False):
    model.train()
    for epoch in range(num_epochs):
        running_loss = 0.0
        if not online:
            for images in train_loader:
                noisy_images = images #+ torch.randn(images.size()) * 0.1  # Adding Gaussian noise
                noisy_images = torchvision.transforms.Grayscale(num_output_channels=1)(noisy_images)
                optimizer.zero_grad()
                outputs = model(noisy_images)
                loss = criterion(outputs, images)
                loss.backward()
                optimizer.step()
                running_loss += loss.item()
        else:
            for images, _ in train_loader:
                noisy_images = images #+ torch.randn(images.size()) * 0.1  # Adding Gaussian noise
                noisy_images = torchvision.transforms.Grayscale(num_output_channels=1)(noisy_images)
                optimizer.zero_grad()
                outputs = model(noisy_images)
                loss = criterion(outputs, images)
                loss.backward()
                optimizer.step()
                running_loss += loss.item()
        print(f"Epoch [{epoch + 1}/{num_epochs}], Loss: {running_loss / len(train_loader)}")

# Data preprocessing
transform = transforms.Compose([
    transforms.ToTensor(),
])

# Initialize the model, criterion, and optimizer
model = ConvAutoencoder()
criterion = nn.MSELoss()
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)


# In[ ]:


# Load CIFAR-10 data
train_dataset = torchvision.datasets.MNIST(root=".MNIST/train", train=True, transform=transform, download=True)
train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=64, shuffle=True, num_workers=4)

# Training the model
train(model, train_loader, criterion, optimizer, num_epochs=30, online=True)


# In[ ]:


train_dataset = torchvision.datasets.MNIST(root=".MNIST/train", train=True, transform=transform, download=True)
train_loader = torch.utils.data.DataLoader(train_dataset, batch_size = 60000, shuffle=True)

print(type(train_dataset))

for batch in train_loader:
    img, labels = batch
    #img = img.reshape(-1, 28*28)

    # Generating output
    out = model.forward(img)
    latent_train = model.forward_encoder(img)

X_train = img.detach().numpy()
latent_train = latent_train.detach().numpy()
y = labels.detach().numpy()
print(y)
print(X_train.shape)
print(latent_train.shape)
print(y.shape)


# In[ ]:


X_train = X_train.reshape(-1,28*28)
latent_train = latent_train.reshape(-1,4*7*7)
print(X_train.shape)
print(latent_train.shape)
print(np.mean(np.cov(X_train.T)))
print(np.mean(np.cov(latent_train.T)))


# In[ ]:


# y = y[:2400]
# latent_train = latent_train[:2400,:]


# In[ ]:


# np.random.shuffle(y)
# print(y)


# # Tree Model

# In[ ]:


et = ExtraTreesClassifier(n_estimators=500, min_samples_leaf=100,
                          max_features='sqrt', bootstrap=True, class_weight='balanced', n_jobs=-1)

# et = RandomForestClassifier(n_estimators=500, min_samples_leaf=100,
#                           max_features='sqrt', bootstrap=True, class_weight='balanced', n_jobs=-1)

# validation instance
skf = StratifiedKFold(n_splits=5, shuffle=True)

# getting the model validation predictions
preds = cross_val_predict(et, latent_train, y, cv=skf, method='predict_proba')

# evaluating the model
print('Area under the ROC Curve:', roc_auc_score(y, preds, multi_class='ovo'))


# In[ ]:


et.fit(latent_train,y)


# In[ ]:


leaves_train = et.apply(latent_train)
print(leaves_train.shape)
print(leaves_train)

distances_train = np.zeros((500,500))

for i in range(500):
    for j in range(500):
        distances_train[i,j] = hamming(leaves_train[i,:], leaves_train[j,:])

score_train = sum(distances_train)/499

print(np.mean(score_train))
print(np.cov(score_train))


# In[ ]:


# model_Conv_AE_OOD_MNIST_train_30 = 'classifier.pt'
# path = "/content/gdrive/My Drive/{model_Conv_AE_OOD_MNIST_train_30}"
# torch.save(model.state_dict(), path)


# In[ ]:


# model_Conv_AE_OOD_MNIST_train_30 = 'classifier.pt'
# path = "/content/gdrive/My Drive/{model_Conv_AE_OOD_MNIST_train_30}"
# model.load_state_dict(torch.load(path))


# # Testing on ID Data

# In[ ]:


#num_epoch = 30


# In[ ]:


# test_dataset_MNIST = torchvision.datasets.MNIST(root=".MNIST/test", train=False, transform=transform, download=True)
# test_loader_MNIST = torch.utils.data.DataLoader(test_dataset_MNIST, batch_size = 64, shuffle=True)

# # model = ConvAutoencoder()
# # criterion = nn.MSELoss()
# # optimizer = torch.optim.Adam(model.parameters(), lr=0.001)

# train(model, test_loader_MNIST, criterion, optimizer, num_epochs=num_epoch, online=True)


# In[ ]:


test_dataset_MNIST = torchvision.datasets.MNIST(root=".MNIST/test", train=False, transform=transform, download=True)
test_loader_MNIST = torch.utils.data.DataLoader(test_dataset_MNIST, batch_size = 10000, shuffle=True)

for batch in test_loader_MNIST:
    img, _ = batch
    #img = torchvision.transforms.Grayscale(num_output_channels=1)(img)
    #img = img.reshape(-1, 28*28)

    # Generating output
    out = model.forward(img)
    latent_test = model.forward_encoder(img)


latent_test = latent_test.detach().numpy()
X_test = img.detach().numpy()
print(X_test.shape)
print(latent_test.shape)
X_test = X_test.reshape(-1,1*28*28)
latent_test = latent_test.reshape(-1,4*7*7)
print(X_test.shape)
print(latent_test.shape)


# In[ ]:


latent_test_in = latent_test

leaves_test_in = et.apply(latent_test_in)
print(leaves_test_in.shape)
print(leaves_test_in)

distances_test_in = np.zeros((500,500))

for i in range(500):
    for j in range(500):
        distances_test_in[i,j] = hamming(leaves_test_in[i,:], leaves_test_in[j,:])

score_test_in = sum(distances_test_in)/499
print(np.mean(score_test_in))
print(np.cov(score_test_in))


# # Testing on OOD Data

# ## FashionMNIST

# In[ ]:


# model_Conv_AE_OOD_MNIST_train_30 = 'classifier.pt'
# path = "/content/gdrive/My Drive/{model_Conv_AE_OOD_MNIST_train_30}"
# model.load_state_dict(torch.load(path))


# In[ ]:


df_test_out = pd.read_csv('/content/gdrive/MyDrive/TOOD/datasets/fashion-mnist_test.csv')
data_test_out = np.array(df_test_out)

X_test_out = data_test_out[:, 1:785]/255
X_test_out = X_test_out.reshape((-1,1,28,28))
print(X_test_out.shape)

# X_test_out_tensor = torch.tensor(X_test_out, dtype=torch.float32)
# test_loader_Fashion = torch.utils.data.DataLoader(X_test_out_tensor, batch_size=64, shuffle=True)

# # model = ConvAutoencoder()
# # criterion = nn.MSELoss()
# # optimizer = torch.optim.Adam(model.parameters(), lr=0.001)

# train(model, test_loader_Fashion, criterion, optimizer, num_epochs=num_epoch)


# In[ ]:


X_test_out_tensor = torch.tensor(X_test_out, dtype=torch.float32)
test_loader_Fashion = torch.utils.data.DataLoader(X_test_out_tensor, batch_size=10000, shuffle=True)

for batch in test_loader_Fashion:

    img = batch
    #img = img.reshape(-1,1,28,28)

    # Generating output
    out = model.forward(img)
    latent_test_out = model.forward_encoder(img)

latent_test_out_Fashion = latent_test_out.detach().numpy()
#latent_test_out = latent_test_out[0,:,:]
print(latent_test_out_Fashion.shape)
latent_test_out_Fashion = latent_test_out_Fashion.reshape(-1,4*7*7)
print(latent_test_out_Fashion.shape)


# In[ ]:


leaves_test_out_Fashion = et.apply(latent_test_out_Fashion)

print(leaves_test_out_Fashion.shape)
print(leaves_test_out_Fashion)

distances_test_out_Fashion = np.zeros((500,500))

for i in range(500):
    for j in range(500):
        distances_test_out_Fashion[i,j] = hamming(leaves_test_out_Fashion[i,:], leaves_test_out_Fashion[j,:])

score_test_out_Fashion = sum(distances_test_out_Fashion)/499

print(np.mean(score_test_out_Fashion))
print(np.cov(score_test_out_Fashion))


# ## BreastMNIST

# In[ ]:


# model_Conv_AE_OOD_MNIST_train_30 = 'classifier.pt'
# path = "/content/gdrive/My Drive/{model_Conv_AE_OOD_MNIST_train_30}"
# model.load_state_dict(torch.load(path))


# In[ ]:


df = np.load('/content/gdrive/MyDrive/TOOD/datasets/breastmnist.npz')
#X_test_out = df['test_images']/255
[size, _, _] = df['train_images'].shape
X_test_out = df['train_images'].reshape((size, 784))/255

X_test_out = X_test_out.reshape((-1,1,28,28))

# X_test_out_tensor = torch.tensor(X_test_out, dtype=torch.float32)
# test_loader_ChestMNIST = torch.utils.data.DataLoader(X_test_out_tensor, batch_size=64, shuffle=True)

# # model = ConvAutoencoder()
# # criterion = nn.MSELoss()
# # optimizer = torch.optim.Adam(model.parameters(), lr=0.001)

# train(model, test_loader_ChestMNIST, criterion, optimizer, num_epochs=num_epoch)


# In[ ]:


[size, _, _] = df['train_images'].shape
print(size)

X_test_out_tensor = torch.tensor(X_test_out, dtype=torch.float32)
test_loader_BreastMNIST = torch.utils.data.DataLoader(X_test_out_tensor, batch_size=size, shuffle=True)

for batch in test_loader_BreastMNIST:

    img = batch
    #img = img.reshape(-1,1,28,28)

    # Generating output
    out = model.forward(img)
    latent_test_out = model.forward_encoder(img)

latent_test_out_Breast = latent_test_out.detach().numpy()
#latent_test_out = latent_test_out[0,:,:]
print(latent_test_out_Breast.shape)
latent_test_out_Breast = latent_test_out_Breast.reshape(-1,4*7*7)
print(latent_test_out_Breast.shape)


# In[ ]:


leaves_test_out_Breast = et.apply(latent_test_out_Breast)

print(leaves_test_out_Breast.shape)
print(leaves_test_out_Breast)

distances_test_out_Breast = np.zeros((500,500))

for i in range(500):
    for j in range(500):
        distances_test_out_Breast[i,j] = hamming(leaves_test_out_Breast[i,:], leaves_test_out_Breast[j,:])

score_test_out_Breast = sum(distances_test_out_Breast)/499

print(np.mean(score_test_out_Breast))
print(np.cov(score_test_out_Breast))


# In[ ]:





# ## ChestMNIST

# In[ ]:


# model_Conv_AE_OOD_MNIST_train_30 = 'classifier.pt'
# path = "/content/gdrive/My Drive/{model_Conv_AE_OOD_MNIST_train_30}"
# model.load_state_dict(torch.load(path))


# In[ ]:


df = np.load('/content/gdrive/MyDrive/TOOD/datasets/chestmnist.npz')
#X_test_out = df['test_images']/255
[size, _, _] = df['test_images'].shape
X_test_out = df['test_images'].reshape((size, 784))/255

X_test_out = X_test_out.reshape((-1,1,28,28))

# X_test_out_tensor = torch.tensor(X_test_out, dtype=torch.float32)
# test_loader_ChestMNIST = torch.utils.data.DataLoader(X_test_out_tensor, batch_size=64, shuffle=True)

# # model = ConvAutoencoder()
# # criterion = nn.MSELoss()
# # optimizer = torch.optim.Adam(model.parameters(), lr=0.001)

# train(model, test_loader_ChestMNIST, criterion, optimizer, num_epochs=num_epoch)


# In[ ]:


[size, _, _] = df['test_images'].shape
print(size)

X_test_out_tensor = torch.tensor(X_test_out, dtype=torch.float32)
test_loader_ChestMNIST = torch.utils.data.DataLoader(X_test_out_tensor, batch_size=size, shuffle=True)

for batch in test_loader_ChestMNIST:

    img = batch
    #img = img.reshape(-1,1,28,28)

    # Generating output
    out = model.forward(img)
    latent_test_out = model.forward_encoder(img)

latent_test_out_Chest = latent_test_out.detach().numpy()
#latent_test_out = latent_test_out[0,:,:]
print(latent_test_out_Chest.shape)
latent_test_out_Chest = latent_test_out_Chest.reshape(-1,4*7*7)
print(latent_test_out_Chest.shape)


# In[ ]:


leaves_test_out_Chest = et.apply(latent_test_out_Chest)

print(leaves_test_out_Chest.shape)
print(leaves_test_out_Chest)

distances_test_out_Chest = np.zeros((500,500))

for i in range(500):
    for j in range(500):
        distances_test_out_Chest[i,j] = hamming(leaves_test_out_Chest[i,:], leaves_test_out_Chest[j,:])

score_test_out_Chest = sum(distances_test_out_Chest)/499

print(np.mean(score_test_out_Chest))
print(np.cov(score_test_out_Chest))


# ## OctMNIST

# In[ ]:


# model_Conv_AE_OOD_MNIST_train_30 = 'classifier.pt'
# path = "/content/gdrive/My Drive/{model_Conv_AE_OOD_MNIST_train_30}"
# model.load_state_dict(torch.load(path))


# In[ ]:


df = np.load('/content/gdrive/MyDrive/TOOD/datasets/octmnist.npz')
#X_test_out = df['test_images']/255
[size, _, _] = df['test_images'].shape
X_test_out = df['test_images'].reshape((size, 784))/255

X_test_out = X_test_out.reshape((-1,1,28,28))

# X_test_out_tensor = torch.tensor(X_test_out, dtype=torch.float32)
# test_loader_ChestMNIST = torch.utils.data.DataLoader(X_test_out_tensor, batch_size=64, shuffle=True)

# # model = ConvAutoencoder()
# # criterion = nn.MSELoss()
# # optimizer = torch.optim.Adam(model.parameters(), lr=0.001)

# train(model, test_loader_ChestMNIST, criterion, optimizer, num_epochs=num_epoch)


# In[ ]:


[size, _, _] = df['test_images'].shape
print(size)

X_test_out_tensor = torch.tensor(X_test_out, dtype=torch.float32)
test_loader_OctMNIST = torch.utils.data.DataLoader(X_test_out_tensor, batch_size=size, shuffle=True)

for batch in test_loader_OctMNIST:

    img = batch
    #img = img.reshape(-1,1,28,28)

    # Generating output
    out = model.forward(img)
    latent_test_out = model.forward_encoder(img)

latent_test_out_Oct = latent_test_out.detach().numpy()
#latent_test_out = latent_test_out[0,:,:]
print(latent_test_out_Oct.shape)
latent_test_out_Oct = latent_test_out_Oct.reshape(-1,4*7*7)
print(latent_test_out_Oct.shape)


# In[ ]:


leaves_test_out_Oct = et.apply(latent_test_out_Oct)

print(leaves_test_out_Oct.shape)
print(leaves_test_out_Oct)

distances_test_out_Oct = np.zeros((500,500))

for i in range(500):
    for j in range(500):
        distances_test_out_Oct[i,j] = hamming(leaves_test_out_Oct[i,:], leaves_test_out_Oct[j,:])

score_test_out_Oct = sum(distances_test_out_Oct)/499

print(np.mean(score_test_out_Oct))
print(np.cov(score_test_out_Oct))


# ## PneumMNIST

# In[ ]:


# model_Conv_AE_OOD_MNIST_train_30 = 'classifier.pt'
# path = "/content/gdrive/My Drive/{model_Conv_AE_OOD_MNIST_train_30}"
# model.load_state_dict(torch.load(path))


# In[ ]:


df = np.load('/content/gdrive/MyDrive/TOOD/datasets/pneumoniamnist.npz')
#X_test_out = df['train_images']/255
[size, _, _] = df['train_images'].shape
X_test_out = df['train_images'].reshape((size, 784))/255

X_test_out = X_test_out.reshape((-1,1,28,28))

# X_test_out_tensor = torch.tensor(X_test_out, dtype=torch.float32)
# test_loader_Pneum = torch.utils.data.DataLoader(X_test_out_tensor, batch_size=64, shuffle=True)

# # model = ConvAutoencoder()
# # criterion = nn.MSELoss()
# # optimizer = torch.optim.Adam(model.parameters(), lr=0.001)

# train(model, test_loader_Pneum, criterion, optimizer, num_epochs=num_epoch)


# In[ ]:


[size, _, _] = df['train_images'].shape
print(size)

X_test_out_tensor = torch.tensor(X_test_out, dtype=torch.float32)
test_loader_Pneum = torch.utils.data.DataLoader(X_test_out_tensor, batch_size=size, shuffle=True)

for batch in test_loader_Pneum:

    img = batch
    #img = img.reshape(-1,1,28,28)

    # Generating output
    out = model.forward(img)
    latent_test_out = model.forward_encoder(img)

latent_test_out_Pneum = latent_test_out.detach().numpy()
#latent_test_out = latent_test_out[0,:,:]
print(latent_test_out_Pneum.shape)
latent_test_out_Pneum = latent_test_out_Pneum.reshape(-1,4*7*7)
print(latent_test_out_Pneum.shape)


# In[ ]:


leaves_test_out_Pneum = et.apply(latent_test_out_Pneum)

print(leaves_test_out_Pneum.shape)
print(leaves_test_out_Pneum)

distances_test_out_Pneum = np.zeros((500,500))

for i in range(500):
    for j in range(500):
        distances_test_out_Pneum[i,j] = hamming(leaves_test_out_Pneum[i,:], leaves_test_out_Pneum[j,:])

score_test_out_Pneum = sum(distances_test_out_Pneum)/499

print(np.mean(score_test_out_Pneum))
print(np.cov(score_test_out_Pneum))


# ## OrganaMNIST

# In[ ]:


# model_Conv_AE_OOD_MNIST_train_30 = 'classifier.pt'
# path = "/content/gdrive/My Drive/{model_Conv_AE_OOD_MNIST_train_30}"
# model.load_state_dict(torch.load(path))


# In[ ]:


df = np.load('/content/gdrive/MyDrive/TOOD/datasets/organamnist.npz')
#X_test_out = df['train_images']/255
[size, _, _] = df['test_images'].shape
X_test_out = df['test_images'].reshape((size, 784))/255

X_test_out = X_test_out.reshape((-1,1,28,28))

# X_test_out_tensor = torch.tensor(X_test_out, dtype=torch.float32)
# test_loader_Organc = torch.utils.data.DataLoader(X_test_out_tensor, batch_size=64, shuffle=True)

# # model = ConvAutoencoder()
# # criterion = nn.MSELoss()
# # optimizer = torch.optim.Adam(model.parameters(), lr=0.001)

# train(model, test_loader_Organc, criterion, optimizer, num_epochs=num_epoch)


# In[ ]:


[size, _, _] = df['test_images'].shape
print(size)

X_test_out_tensor = torch.tensor(X_test_out, dtype=torch.float32)
test_loader_Organa = torch.utils.data.DataLoader(X_test_out_tensor, batch_size=size, shuffle=True)

for batch in test_loader_Organa:

    img = batch
    #img = img.reshape(-1,1,28,28)

    # Generating output
    out = model.forward(img)
    latent_test_out = model.forward_encoder(img)

latent_test_out_Organa = latent_test_out.detach().numpy()
#latent_test_out = latent_test_out[0,:,:]
print(latent_test_out_Organa.shape)
latent_test_out_Organa = latent_test_out_Organa.reshape(-1,4*7*7)
print(latent_test_out_Organa.shape)


# In[ ]:


leaves_test_out_Organa = et.apply(latent_test_out_Organa)

print(leaves_test_out_Organa.shape)
print(leaves_test_out_Organa)

distances_test_out_Organa = np.zeros((500,500))

for i in range(500):
    for j in range(500):
        distances_test_out_Organa[i,j] = hamming(leaves_test_out_Organa[i,:], leaves_test_out_Organa[j,:])

score_test_out_Organa = sum(distances_test_out_Organa)/499

print(np.mean(score_test_out_Organa))
print(np.cov(score_test_out_Organa))


# ## OrgancMNIST

# In[ ]:


# model_Conv_AE_OOD_MNIST_train_30 = 'classifier.pt'
# path = "/content/gdrive/My Drive/{model_Conv_AE_OOD_MNIST_train_30}"
# model.load_state_dict(torch.load(path))


# In[ ]:


df = np.load('/content/gdrive/MyDrive/TOOD/datasets/organcmnist.npz')
#X_test_out = df['train_images']/255
[size, _, _] = df['test_images'].shape
X_test_out = df['test_images'].reshape((size, 784))/255

X_test_out = X_test_out.reshape((-1,1,28,28))

# X_test_out_tensor = torch.tensor(X_test_out, dtype=torch.float32)
# test_loader_Organc = torch.utils.data.DataLoader(X_test_out_tensor, batch_size=64, shuffle=True)

# # model = ConvAutoencoder()
# # criterion = nn.MSELoss()
# # optimizer = torch.optim.Adam(model.parameters(), lr=0.001)

# train(model, test_loader_Organc, criterion, optimizer, num_epochs=num_epoch)


# In[ ]:


[size, _, _] = df['test_images'].shape
print(size)

X_test_out_tensor = torch.tensor(X_test_out, dtype=torch.float32)
test_loader_Organc = torch.utils.data.DataLoader(X_test_out_tensor, batch_size=size, shuffle=True)

for batch in test_loader_Organc:

    img = batch
    #img = img.reshape(-1,1,28,28)

    # Generating output
    out = model.forward(img)
    latent_test_out = model.forward_encoder(img)

latent_test_out_Organc = latent_test_out.detach().numpy()
#latent_test_out = latent_test_out[0,:,:]
print(latent_test_out_Organc.shape)
latent_test_out_Organc = latent_test_out_Organc.reshape(-1,4*7*7)
print(latent_test_out_Organc.shape)


# In[ ]:


leaves_test_out_Organc = et.apply(latent_test_out_Organc)

print(leaves_test_out_Organc.shape)
print(leaves_test_out_Organc)

distances_test_out_Organc = np.zeros((500,500))

for i in range(500):
    for j in range(500):
        distances_test_out_Organc[i,j] = hamming(leaves_test_out_Organc[i,:], leaves_test_out_Organc[j,:])

score_test_out_Organc = sum(distances_test_out_Organc)/499

print(np.mean(score_test_out_Organc))
print(np.cov(score_test_out_Organc))


# ## OrgansMNIST

# In[ ]:


# model_Conv_AE_OOD_MNIST_train_30 = 'classifier.pt'
# path = "/content/gdrive/My Drive/{model_Conv_AE_OOD_MNIST_train_30}"
# model.load_state_dict(torch.load(path))


# In[ ]:


df = np.load('/content/gdrive/MyDrive/TOOD/datasets/organsmnist.npz')
#X_test_out = df['train_images']/255
[size, _, _] = df['test_images'].shape
X_test_out = df['test_images'].reshape((size, 784))/255

X_test_out = X_test_out.reshape((-1,1,28,28))

# X_test_out_tensor = torch.tensor(X_test_out, dtype=torch.float32)
# test_loader_Organc = torch.utils.data.DataLoader(X_test_out_tensor, batch_size=64, shuffle=True)

# # model = ConvAutoencoder()
# # criterion = nn.MSELoss()
# # optimizer = torch.optim.Adam(model.parameters(), lr=0.001)

# train(model, test_loader_Organc, criterion, optimizer, num_epochs=num_epoch)


# In[ ]:


[size, _, _] = df['test_images'].shape
print(size)

X_test_out_tensor = torch.tensor(X_test_out, dtype=torch.float32)
test_loader_Organs = torch.utils.data.DataLoader(X_test_out_tensor, batch_size=size, shuffle=True)

for batch in test_loader_Organs:

    img = batch
    #img = img.reshape(-1,1,28,28)

    # Generating output
    out = model.forward(img)
    latent_test_out = model.forward_encoder(img)

latent_test_out_Organs = latent_test_out.detach().numpy()
#latent_test_out = latent_test_out[0,:,:]
print(latent_test_out_Organs.shape)
latent_test_out_Organs = latent_test_out_Organs.reshape(-1,4*7*7)
print(latent_test_out_Organs.shape)


# In[ ]:


leaves_test_out_Organs = et.apply(latent_test_out_Organs)

print(leaves_test_out_Organs.shape)
print(leaves_test_out_Organs)

distances_test_out_Organs = np.zeros((500,500))

for i in range(500):
    for j in range(500):
        distances_test_out_Organs[i,j] = hamming(leaves_test_out_Organs[i,:], leaves_test_out_Organs[j,:])

score_test_out_Organs = sum(distances_test_out_Organs)/499

print(np.mean(score_test_out_Organs))
print(np.cov(score_test_out_Organs))


# ## TissueMNIST

# In[ ]:


# model_Conv_AE_OOD_MNIST_train_30 = 'classifier.pt'
# path = "/content/gdrive/My Drive/{model_Conv_AE_OOD_MNIST_train_30}"
# model.load_state_dict(torch.load(path))


# In[ ]:


df = np.load('/content/gdrive/MyDrive/TOOD/datasets/tissuemnist.npz')
#X_test_out = df['test_images']/255
[size, _, _] = df['test_images'].shape
X_test_out = df['test_images'].reshape((size, 784))/255

X_test_out = X_test_out.reshape((-1,1,28,28))

# X_test_out_tensor = torch.tensor(X_test_out, dtype=torch.float32)
# test_loader_Tissue = torch.utils.data.DataLoader(X_test_out_tensor, batch_size=64, shuffle=True)

# # model = ConvAutoencoder()
# # criterion = nn.MSELoss()
# # optimizer = torch.optim.Adam(model.parameters(), lr=0.001)

# train(model, test_loader_Tissue, criterion, optimizer, num_epochs=num_epoch)


# In[ ]:


[size, _, _] = df['test_images'].shape
print(size)

X_test_out_tensor = torch.tensor(X_test_out, dtype=torch.float32)
test_loader_Tissue = torch.utils.data.DataLoader(X_test_out_tensor, batch_size=size, shuffle=True)

for batch in test_loader_Tissue:

    img = batch
    #img = img.reshape(-1,1,28,28)

    # Generating output
    out = model.forward(img)
    latent_test_out = model.forward_encoder(img)

latent_test_out_Tissue = latent_test_out.detach().numpy()
#latent_test_out = latent_test_out[0,:,:]
print(latent_test_out_Tissue.shape)
latent_test_out_Tissue = latent_test_out_Tissue.reshape(-1,4*7*7)
print(latent_test_out_Tissue.shape)


# In[ ]:


leaves_test_out_Tissue = et.apply(latent_test_out_Tissue)

print(leaves_test_out_Tissue.shape)
print(leaves_test_out_Tissue)

distances_test_out_Tissue = np.zeros((500,500))

for i in range(500):
    for j in range(500):
        distances_test_out_Tissue[i,j] = hamming(leaves_test_out_Tissue[i,:], leaves_test_out_Tissue[j,:])

score_test_out_Tissue = sum(distances_test_out_Tissue)/499

print(np.mean(score_test_out_Tissue))
print(np.cov(score_test_out_Tissue))


# ## KMNIST

# In[ ]:


# model_Conv_AE_OOD_MNIST_train_30 = 'classifier.pt'
# path = "/content/gdrive/My Drive/{model_Conv_AE_OOD_MNIST_train_30}"
# model.load_state_dict(torch.load(path))


# In[ ]:


# test_dataset_KMNIST = torchvision.datasets.KMNIST(root="./data", train=False, transform=transform, download=True)
# test_loader_KMNIST = torch.utils.data.DataLoader(test_dataset_KMNIST, batch_size = 64, shuffle=True)

# # model = ConvAutoencoder()
# # criterion = nn.MSELoss()
# # optimizer = torch.optim.Adam(model.parameters(), lr=0.001)

# train(model, test_loader_KMNIST, criterion, optimizer, num_epochs=num_epoch, online=True)


# In[ ]:


transform = transforms.Compose(
    [transforms.Resize((28, 28)),  # Resize images to match the size of MNIST
     transforms.ToTensor()])

test_dataset_KMNIST = torchvision.datasets.KMNIST(root='./data', train=False, download=True, transform=transform)
test_loader_KMNIST = torch.utils.data.DataLoader(test_dataset_KMNIST, batch_size=10000, shuffle=True)

for batch in test_loader_KMNIST:
    img, _ = batch
    #img = torchvision.transforms.Grayscale(num_output_channels=1)(img)

    # Generating output
    out = model.forward(img)
    latent_test_out = model.forward_encoder(img)


latent_test_out_KMNIST = latent_test_out.detach().numpy()
latent_test_out_KMNIST = latent_test_out_KMNIST.reshape(-1,4*7*7)
print(latent_test_out_KMNIST.shape)


# In[ ]:


leaves_test_out_KMNIST = et.apply(latent_test_out_KMNIST)

print(leaves_test_out_KMNIST.shape)
print(leaves_test_out_KMNIST)

distances_test_out_KMNIST = np.zeros((500,500))

for i in range(500):
    for j in range(500):
        distances_test_out_KMNIST[i,j] = hamming(leaves_test_out_KMNIST[i,:], leaves_test_out_KMNIST[j,:])

score_test_out_KMNIST = sum(distances_test_out_KMNIST)/499

print(np.mean(score_test_out_KMNIST))
print(np.cov(score_test_out_KMNIST))


# ## QMNIST

# In[ ]:


# model_Conv_AE_OOD_MNIST_train_30 = 'classifier.pt'
# path = "/content/gdrive/My Drive/{model_Conv_AE_OOD_MNIST_train_30}"
# model.load_state_dict(torch.load(path))


# In[ ]:


# test_dataset_QMNIST = torchvision.datasets.QMNIST(root="./data", train=False, transform=transform, download=True)
# test_loader_QMNIST = torch.utils.data.DataLoader(test_dataset_QMNIST, batch_size = 64, shuffle=True)

# # model = ConvAutoencoder()
# # criterion = nn.MSELoss()
# # optimizer = torch.optim.Adam(model.parameters(), lr=0.001)

# train(model, test_loader_QMNIST, criterion, optimizer, num_epochs=num_epoch, online=True)


# In[ ]:


transform = transforms.Compose(
    [transforms.Resize((28, 28)),  # Resize images to match the size of MNIST
     transforms.ToTensor()])

test_dataset_QMNIST = torchvision.datasets.QMNIST(root='./data', train=False, download=True, transform=transform)
test_loader_QMNIST = torch.utils.data.DataLoader(test_dataset_QMNIST, batch_size=10000, shuffle=True)

for batch in test_loader_QMNIST:
    img, _ = batch
    #img = torchvision.transforms.Grayscale(num_output_channels=1)(img)

    # Generating output
    out = model.forward(img)
    latent_test_out = model.forward_encoder(img)


latent_test_out_QMNIST = latent_test_out.detach().numpy()
latent_test_out_QMNIST = latent_test_out_QMNIST.reshape(-1,4*7*7)
print(latent_test_out_QMNIST.shape)


# In[ ]:


leaves_test_out_QMNIST = et.apply(latent_test_out_QMNIST)

print(leaves_test_out_QMNIST.shape)
print(leaves_test_out_QMNIST)

distances_test_out_QMNIST = np.zeros((500,500))

for i in range(500):
    for j in range(500):
        distances_test_out_QMNIST[i,j] = hamming(leaves_test_out_QMNIST[i,:], leaves_test_out_QMNIST[j,:])

score_test_out_QMNIST = sum(distances_test_out_QMNIST)/499

print(np.mean(score_test_out_QMNIST))
print(np.cov(score_test_out_QMNIST))


# ## Omniglot

# In[ ]:


# model_Conv_AE_OOD_MNIST_train_30 = 'classifier.pt'
# path = "/content/gdrive/My Drive/{model_Conv_AE_OOD_MNIST_train_30}"
# model.load_state_dict(torch.load(path))


# In[ ]:


# test_dataset_Omniglot = torchvision.datasets.Omniglot(root='./data', download=True, transform=transform)
# test_loader_Omniglot = torch.utils.data.DataLoader(test_dataset_QMNIST, batch_size = 64, shuffle=True)

# # model = ConvAutoencoder()
# # criterion = nn.MSELoss()
# # optimizer = torch.optim.Adam(model.parameters(), lr=0.001)

# train(model, test_loader_Omniglot, criterion, optimizer, num_epochs=num_epoch, online=True)


# In[ ]:


transform = transforms.Compose(
    [transforms.Resize((28, 28)),  # Resize images to match the size of MNIST
     transforms.ToTensor()])

test_dataset_Omniglot = torchvision.datasets.Omniglot(root='./data', download=True, transform=transform)

# Creating Dataloaders from the
# training and testing dataset
test_loader_Omniglot = torch.utils.data.DataLoader(test_dataset_Omniglot, batch_size=32460, shuffle=True)

for batch in test_loader_Omniglot:
    img, _ = batch
    #img = img.reshape(-1,1,28,28)

    # Generating output
    out = model.forward(img)
    latent_test_out = model.forward_encoder(img)


latent_test_out_Omniglot = latent_test_out.detach().numpy()
latent_test_out_Omniglot = latent_test_out_Omniglot.reshape(-1,4*7*7)
print(latent_test_out_Omniglot.shape)


# In[ ]:


leaves_test_out_Omniglot = et.apply(latent_test_out_Omniglot)

print(leaves_test_out_Omniglot.shape)
print(leaves_test_out_Omniglot)

distances_test_out_Omniglot = np.zeros((500,500))

for i in range(500):
    for j in range(500):
        distances_test_out_Omniglot[i,j] = hamming(leaves_test_out_Omniglot[i,:], leaves_test_out_Omniglot[j,:])

score_test_out_Omniglot = sum(distances_test_out_Omniglot)/499

print(np.mean(score_test_out_Omniglot))
print(np.cov(score_test_out_Omniglot))


# ## Cifar10bw

# In[ ]:


# model_Conv_AE_OOD_MNIST_train_30 = 'classifier.pt'
# path = "/content/gdrive/My Drive/{model_Conv_AE_OOD_MNIST_train_30}"
# model.load_state_dict(torch.load(path))


# In[ ]:


# test_dataset_Cifar10bw = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=transform)
# test_loader_Cifar10bw = torch.utils.data.DataLoader(test_dataset_Cifar10bw, batch_size = 64, shuffle=True)

# # model = ConvAutoencoder()
# # criterion = nn.MSELoss()
# # optimizer = torch.optim.Adam(model.parameters(), lr=0.001)

# train(model, test_loader_Cifar10bw, criterion, optimizer, num_epochs=num_epoch, online=True)


# In[ ]:


transform = transforms.Compose(
    [transforms.Resize((28, 28)),  # Resize images to match the size of MNIST
     transforms.ToTensor()])

test_dataset_Cifar10bw = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=transform)

# Creating Dataloaders from the
# training and testing dataset
test_loader_Cifar10bw = torch.utils.data.DataLoader(test_dataset_Cifar10bw, batch_size=10000, shuffle=True)

for batch in test_loader_Cifar10bw:
    img, _ = batch
    img = torchvision.transforms.Grayscale(num_output_channels=1)(img)
    #img = img.reshape(-1,1,28,28)

    # Generating output
    out = model.forward(img)
    latent_test_out = model.forward_encoder(img)


latent_test_out_Cifar10bw = latent_test_out.detach().numpy()
latent_test_out_Cifar10bw = latent_test_out_Cifar10bw.reshape(-1,4*7*7)
print(latent_test_out_Cifar10bw.shape)


# In[ ]:


leaves_test_out_Cifar10bw = et.apply(latent_test_out_Cifar10bw)

print(leaves_test_out_Cifar10bw.shape)
print(leaves_test_out_Cifar10bw)

distances_test_out_Cifar10bw = np.zeros((500,500))

for i in range(500):
    for j in range(500):
        distances_test_out_Cifar10bw[i,j] = hamming(leaves_test_out_Cifar10bw[i,:], leaves_test_out_Cifar10bw[j,:])

score_test_out_Cifar10bw = sum(distances_test_out_Cifar10bw)/499

print(np.mean(score_test_out_Cifar10bw))
print(np.cov(score_test_out_Cifar10bw))


# ## NotMNIST

# In[ ]:


# model_Conv_AE_OOD_MNIST_train_30 = 'classifier.pt'
# path = "/content/gdrive/My Drive/{model_Conv_AE_OOD_MNIST_train_30}"
# model.load_state_dict(torch.load(path))


# In[ ]:


from pims import ImageSequence
from PIL import Image

images_A = ImageSequence('/content/gdrive/MyDrive/TOOD/datasets/NotMNIST_small/A/*.png')
images_B = ImageSequence('/content/gdrive/MyDrive/TOOD/datasets/NotMNIST_small/B/*.png')
images_C = ImageSequence('/content/gdrive/MyDrive/TOOD/datasets/NotMNIST_small/C/*.png')
images_D = ImageSequence('/content/gdrive/MyDrive/TOOD/datasets/NotMNIST_small/D/*.png')
images_E = ImageSequence('/content/gdrive/MyDrive/TOOD/datasets/NotMNIST_small/E/*.png')
images_F = ImageSequence('/content/gdrive/MyDrive/TOOD/datasets/NotMNIST_small/F/*.png')
images_G = ImageSequence('/content/gdrive/MyDrive/TOOD/datasets/NotMNIST_small/G/*.png')
images_H = ImageSequence('/content/gdrive/MyDrive/TOOD/datasets/NotMNIST_small/H/*.png')
images_I = ImageSequence('/content/gdrive/MyDrive/TOOD/datasets/NotMNIST_small/I/*.png')
images_J = ImageSequence('/content/gdrive/MyDrive/TOOD/datasets/NotMNIST_small/J/*.png')

X_data_A = np.zeros((50, 28**2))
X_data_B = np.zeros((50, 28**2))
X_data_C = np.zeros((50, 28**2))
X_data_D = np.zeros((50, 28**2))
X_data_E = np.zeros((50, 28**2))
X_data_F = np.zeros((50, 28**2))
X_data_G = np.zeros((50, 28**2))
X_data_H = np.zeros((50, 28**2))
X_data_I = np.zeros((50, 28**2))
X_data_J = np.zeros((50, 28**2))

for i in range(50):
    X_data_A[i,:] = np.reshape(np.array(images_A[i]), (1,28**2))/255
    X_data_B[i,:] = np.reshape(np.array(images_B[i]), (1,28**2))/255
    X_data_C[i,:] = np.reshape(np.array(images_C[i]), (1,28**2))/255
    X_data_D[i,:] = np.reshape(np.array(images_D[i]), (1,28**2))/255
    X_data_E[i,:] = np.reshape(np.array(images_E[i]), (1,28**2))/255
    X_data_F[i,:] = np.reshape(np.array(images_F[i]), (1,28**2))/255
    X_data_G[i,:] = np.reshape(np.array(images_G[i]), (1,28**2))/255
    X_data_H[i,:] = np.reshape(np.array(images_H[i]), (1,28**2))/255
    X_data_I[i,:] = np.reshape(np.array(images_I[i]), (1,28**2))/255
    X_data_J[i,:] = np.reshape(np.array(images_J[i]), (1,28**2))/255


X_test_out = np.vstack([X_data_A, X_data_B, X_data_C, X_data_D, X_data_E, X_data_F, X_data_G, X_data_H, X_data_I, X_data_J])
X_test_out = X_test_out.reshape((-1,1,28,28))
print(X_test_out.shape)


# In[ ]:


# X_test_out_tensor = torch.tensor(X_test_out, dtype=torch.float32)
# test_loader_NotMNIST = torch.utils.data.DataLoader(X_test_out_tensor, batch_size=64, shuffle=True)

# # model = ConvAutoencoder()
# # criterion = nn.MSELoss()
# # optimizer = torch.optim.Adam(model.parameters(), lr=0.001)

# train(model, test_loader_NotMNIST, criterion, optimizer, num_epochs=num_epoch)


# In[ ]:


test_loader_NotMNIST = torch.utils.data.DataLoader(X_test_out_tensor, batch_size=10000, shuffle=True)

for batch in test_loader_NotMNIST:

    img = batch
    #img = img.reshape(-1,1,28,28)

    # Generating output
    out = model.forward(img)
    latent_test_out = model.forward_encoder(img)

latent_test_out_NotMNIST = latent_test_out.detach().numpy()
latent_test_out_NotMNIST = latent_test_out_NotMNIST.reshape(-1,4*7*7)
print(latent_test_out_NotMNIST.shape)


# In[ ]:


leaves_test_out_NotMNIST = et.apply(latent_test_out_NotMNIST)

print(leaves_test_out_NotMNIST.shape)
print(leaves_test_out_NotMNIST)

distances_test_out_NotMNIST = np.zeros((500,500))

for i in range(500):
    for j in range(500):
        distances_test_out_NotMNIST[i,j] = hamming(leaves_test_out_NotMNIST[i,:], leaves_test_out_NotMNIST[j,:])

score_test_out_NotMNIST = sum(distances_test_out_NotMNIST)/499

print(np.mean(score_test_out_NotMNIST))
print(np.cov(score_test_out_NotMNIST))


# In[ ]:





# ## Gaussian

# In[ ]:


# model_Conv_AE_OOD_MNIST_train_30 = 'classifier.pt'
# path = "/content/gdrive/My Drive/{model_Conv_AE_OOD_MNIST_train_30}"
# model.load_state_dict(torch.load(path))


# In[ ]:


# Gaussian random noise
mean_test_out = np.random.uniform(low=0,high=1,size=784)
cov_test_out = np.random.rand(784,784)
cov_test_out = np.matmul(cov_test_out, cov_test_out.T)
X_test_out = np.random.multivariate_normal(mean_test_out, cov_test_out, 10000)
X_test_out = MinMaxScaler().fit_transform(X_test_out)
print(X_test_out.shape)
X_test_out = X_test_out.reshape((-1,1,28,28))
print(X_test_out.shape)


# In[ ]:


# X_test_out_tensor = torch.tensor(X_test_out, dtype=torch.float32)
# train_loader_Gaussian = torch.utils.data.DataLoader(X_test_out_tensor, batch_size=64, shuffle=True)

# # model = ConvAutoencoder()
# # criterion = nn.MSELoss()
# # optimizer = torch.optim.Adam(model.parameters(), lr=0.001)

# train(model, train_loader_Gaussian, criterion, optimizer, num_epochs=num_epoch)


# In[ ]:


X_test_out_tensor = torch.tensor(X_test_out, dtype=torch.float32)
test_loader = torch.utils.data.DataLoader(X_test_out_tensor, batch_size=10000)

for batch in test_loader:
    img = batch
    img = img.reshape(-1,1,28,28)
    # Generating output
    #out = model.forward(img)
    latent_test_out = model.forward_encoder(img)


latent_test_out_Gaussian = latent_test_out.detach().numpy()
print(latent_test_out_Gaussian.shape)
latent_test_out_Gaussian = latent_test_out_Gaussian.reshape(-1,4*7*7)
print(latent_test_out_Gaussian.shape)


# In[ ]:


leaves_test_out_Gaussian = et.apply(latent_test_out_Gaussian)

print(leaves_test_out_Gaussian.shape)
print(leaves_test_out_Gaussian)

distances_test_out_Gaussian = np.zeros((500,500))

for i in range(500):
    for j in range(500):
        distances_test_out_Gaussian[i,j] = hamming(leaves_test_out_Gaussian[i,:], leaves_test_out_Gaussian[j,:])

score_test_out_Gaussian = sum(distances_test_out_Gaussian)/499

print(np.mean(score_test_out_Gaussian))
print(np.cov(score_test_out_Gaussian))


# ## Uniform

# In[ ]:


# model_Conv_AE_OOD_MNIST_train_30 = 'classifier.pt'
# path = "/content/gdrive/My Drive/{model_Conv_AE_OOD_MNIST_train_30}"
# model.load_state_dict(torch.load(path))


# In[ ]:


# Uniform random noise
x_min = [0]*784
x_max = [1]*784
X_test_out = np.random.uniform(low=x_min, high=x_max, size=(10000,784))
X_test_out = MinMaxScaler().fit_transform(X_test_out)
print(X_test_out.shape)
X_test_out = X_test_out.reshape((-1,1,28,28))
print(X_test_out.shape)


# In[ ]:


# X_test_out_tensor = torch.tensor(X_test_out, dtype=torch.float32)
# train_loader_Uniform = torch.utils.data.DataLoader(X_test_out_tensor, batch_size=64, shuffle=True)

# # model = ConvAutoencoder()
# # criterion = nn.MSELoss()
# # optimizer = torch.optim.Adam(model.parameters(), lr=0.001)

# train(model, train_loader_Uniform, criterion, optimizer, num_epochs=num_epoch)


# In[ ]:


X_test_out_tensor = torch.tensor(X_test_out, dtype=torch.float32)
test_loader = torch.utils.data.DataLoader(X_test_out_tensor, batch_size=10000)

for batch in test_loader:
    img = batch
    img = img.reshape(-1,1,28,28)
    # Generating output
    #out = model.forward(img)
    latent_test_out = model.forward_encoder(img)


latent_test_out_Uniform = latent_test_out.detach().numpy()
print(latent_test_out_Uniform.shape)
latent_test_out_Uniform = latent_test_out_Uniform.reshape(-1,4*7*7)
print(latent_test_out_Uniform.shape)


# In[ ]:


leaves_test_out_Uniform = et.apply(latent_test_out_Uniform)

print(leaves_test_out_Uniform.shape)
print(leaves_test_out_Uniform)

distances_test_out_Uniform = np.zeros((500,500))

for i in range(500):
    for j in range(500):
        distances_test_out_Uniform[i,j] = hamming(leaves_test_out_Uniform[i,:], leaves_test_out_Uniform[j,:])

score_test_out_Uniform = sum(distances_test_out_Uniform)/499

print(np.mean(score_test_out_Uniform))
print(np.cov(score_test_out_Uniform))


# In[ ]:





# In[ ]:





# # Results

# In[ ]:


score_pred_Breast = np.concatenate([score_test_in, score_test_out_Breast])
score_pred_Chest = np.concatenate([score_test_in, score_test_out_Chest])
score_pred_Oct = np.concatenate([score_test_in, score_test_out_Oct])
score_pred_Organa = np.concatenate([score_test_in, score_test_out_Organa])
score_pred_Organc = np.concatenate([score_test_in, score_test_out_Organc])
score_pred_Organs = np.concatenate([score_test_in, score_test_out_Organs])
score_pred_Pneum = np.concatenate([score_test_in, score_test_out_Pneum])
score_pred_Tissue = np.concatenate([score_test_in, score_test_out_Tissue])
score_pred_KMNIST = np.concatenate([score_test_in, score_test_out_KMNIST])
score_pred_QMNIST = np.concatenate([score_test_in, score_test_out_QMNIST])
score_pred_Omniglot = np.concatenate([score_test_in, score_test_out_Omniglot])
score_pred_Cifar10bw = np.concatenate([score_test_in, score_test_out_Cifar10bw])
score_pred_NotMNIST = np.concatenate([score_test_in, score_test_out_NotMNIST])
score_pred_Gaussian = np.concatenate([score_test_in, score_test_out_Gaussian])
score_pred_Uniform = np.concatenate([score_test_in, score_test_out_Uniform])
score_true = np.concatenate([np.ones(500), np.zeros(500)])

print(roc_auc_score(score_true, score_pred_Breast))
print(roc_auc_score(score_true, score_pred_Chest))
print(roc_auc_score(score_true, score_pred_Oct))
print(roc_auc_score(score_true, score_pred_Organa))
print(roc_auc_score(score_true, score_pred_Organc))
print(roc_auc_score(score_true, score_pred_Organs))
print(roc_auc_score(score_true, score_pred_Pneum))
print(roc_auc_score(score_true, score_pred_Tissue))
print(roc_auc_score(score_true, score_pred_KMNIST))
print(roc_auc_score(score_true, score_pred_QMNIST))
print(roc_auc_score(score_true, score_pred_Omniglot))
print(roc_auc_score(score_true, score_pred_Cifar10bw))
print(roc_auc_score(score_true, score_pred_NotMNIST))
print(roc_auc_score(score_true, score_pred_Gaussian))
print(roc_auc_score(score_true, score_pred_Uniform))


# In[ ]:


precision_Breast, recall_Breast, thresholds_Breast = precision_recall_curve(score_true, score_pred_Breast)
precision_Chest, recall_Chest, thresholds_Chest = precision_recall_curve(score_true, score_pred_Chest)
precision_Oct, recall_Oct, thresholds_Oct = precision_recall_curve(score_true, score_pred_Oct)
precision_Organa, recall_Organa, thresholds_Organa = precision_recall_curve(score_true, score_pred_Organa)
precision_Organc, recall_Organc, thresholds_Organc = precision_recall_curve(score_true, score_pred_Organc)
precision_Organs, recall_Organs, thresholds_Organs = precision_recall_curve(score_true, score_pred_Organs)
precision_Pneum, recall_Pneum, thresholds_Pneum = precision_recall_curve(score_true, score_pred_Pneum)
precision_Tissue, recall_Tissue, thresholds_Tissue = precision_recall_curve(score_true, score_pred_Tissue)
precision_KMNIST, recall_KMNIST, thresholds_KMNIST = precision_recall_curve(score_true, score_pred_KMNIST)
precision_QMNIST, recall_QMNIST, thresholds_QMNIST = precision_recall_curve(score_true, score_pred_QMNIST)
precision_Omniglot, recall_Omniglot, thresholds_Omniglot = precision_recall_curve(score_true, score_pred_Omniglot)
precision_Cifar10bw, recall_Cifar10bw, threshold_Cifar10bw = precision_recall_curve(score_true, score_pred_Cifar10bw)
precision_NotMNIST, recall_NotMNIST, thresholds_NotMNIST = precision_recall_curve(score_true, score_pred_NotMNIST)
precision_Gaussian, recall_Gaussian, thresholds_Gaussian = precision_recall_curve(score_true, score_pred_Gaussian)
precision_Uniform, recall_Uniform, thresholds_Uniform = precision_recall_curve(score_true, score_pred_Uniform)

auc_precision_recall_Breast = auc(recall_Breast, precision_Breast)
auc_precision_recall_Chest = auc(recall_Chest, precision_Chest)
auc_precision_recall_Oct = auc(recall_Oct, precision_Oct)
auc_precision_recall_Organa = auc(recall_Organa, precision_Organa)
auc_precision_recall_Organc = auc(recall_Organc, precision_Organc)
auc_precision_recall_Organs = auc(recall_Organs, precision_Organs)
auc_precision_recall_Pneum = auc(recall_Pneum, precision_Pneum)
auc_precision_recall_Tissue = auc(recall_Tissue, precision_Tissue)
auc_precision_recall_KMNIST = auc(recall_KMNIST, precision_KMNIST)
auc_precision_recall_QMNIST = auc(recall_QMNIST, precision_QMNIST)
auc_precision_recall_Omniglot = auc(recall_Omniglot, precision_Omniglot)
auc_precision_recall_Cifar10bw = auc(recall_Cifar10bw, precision_Cifar10bw)
auc_precision_recall_NotMNIST = auc(recall_NotMNIST, precision_NotMNIST)
auc_precision_recall_Gaussian = auc(recall_Gaussian, precision_Gaussian)
auc_precision_recall_Uniform = auc(recall_Uniform, precision_Uniform)

print(auc_precision_recall_Breast)
print(auc_precision_recall_Chest)
print(auc_precision_recall_Oct)
print(auc_precision_recall_Organa)
print(auc_precision_recall_Organc)
print(auc_precision_recall_Organs)
print(auc_precision_recall_Pneum)
print(auc_precision_recall_Tissue)
print(auc_precision_recall_KMNIST)
print(auc_precision_recall_QMNIST)
print(auc_precision_recall_Omniglot)
print(auc_precision_recall_Cifar10bw)
print(auc_precision_recall_NotMNIST)
print(auc_precision_recall_Gaussian)
print(auc_precision_recall_Uniform)


# In[ ]:


def compute_fpr95(y_true, y_pred_probs):
    fpr, tpr, thresholds = sklearn.metrics.roc_curve(y_true, y_pred_probs)
    idx = np.abs(tpr - 0.95).argmin()
    fpr95 = fpr[idx]
    return fpr95

# Example usage
# Assuming you have y_true (true labels) and y_pred_probs (predicted probabilities)
fpr95_score_Breast = compute_fpr95(score_true, score_pred_Breast)
fpr95_score_Chest = compute_fpr95(score_true, score_pred_Chest)
fpr95_score_Oct = compute_fpr95(score_true, score_pred_Oct)
fpr95_score_Organa = compute_fpr95(score_true, score_pred_Organa)
fpr95_score_Organc = compute_fpr95(score_true, score_pred_Organc)
fpr95_score_Organs = compute_fpr95(score_true, score_pred_Organs)
fpr95_score_Pneum = compute_fpr95(score_true, score_pred_Pneum)
fpr95_score_Tissue = compute_fpr95(score_true, score_pred_Tissue)
fpr95_score_KMNIST = compute_fpr95(score_true, score_pred_KMNIST)
fpr95_score_QMNIST = compute_fpr95(score_true, score_pred_QMNIST)
fpr95_score_Omniglot = compute_fpr95(score_true, score_pred_Omniglot)
fpr95_score_NotMNIST = compute_fpr95(score_true, score_pred_NotMNIST)
fpr95_score_Cifar10bw = compute_fpr95(score_true, score_pred_Cifar10bw)
fpr95_score_Gaussian = compute_fpr95(score_true, score_pred_Gaussian)
fpr95_score_Uniform = compute_fpr95(score_true, score_pred_Uniform)

print(fpr95_score_Breast)
print(fpr95_score_Chest)
print(fpr95_score_Oct)
print(fpr95_score_Organa)
print(fpr95_score_Organc)
print(fpr95_score_Organs)
print(fpr95_score_Pneum)
print(fpr95_score_Tissue)
print(fpr95_score_KMNIST)
print(fpr95_score_QMNIST)
print(fpr95_score_Omniglot)
print(fpr95_score_NotMNIST)
print(fpr95_score_Cifar10bw)
print(fpr95_score_Uniform)
print(fpr95_score_Gaussian)


# In[ ]:





# In[ ]:


print(np.mean(score_test_in), np.cov(score_test_in))
print(np.mean(score_test_out_Breast), np.cov(score_test_out_Breast))
print(np.mean(score_test_out_Chest), np.cov(score_test_out_Chest))
print(np.mean(score_test_out_Oct), np.cov(score_test_out_Oct))
print(np.mean(score_test_out_Organa), np.cov(score_test_out_Organa))
print(np.mean(score_test_out_Organc), np.cov(score_test_out_Organc))
print(np.mean(score_test_out_Organs), np.cov(score_test_out_Organs))
print(np.mean(score_test_out_Pneum), np.cov(score_test_out_Pneum))
print(np.mean(score_test_out_Tissue), np.cov(score_test_out_Tissue))
print(np.mean(score_test_out_KMNIST), np.cov(score_test_out_KMNIST))
print(np.mean(score_test_out_QMNIST), np.cov(score_test_out_QMNIST))
print(np.mean(score_test_out_Omniglot), np.cov(score_test_out_Omniglot))
print(np.mean(score_test_out_Cifar10bw), np.cov(score_test_out_Cifar10bw))
print(np.mean(score_test_out_NotMNIST), np.cov(score_test_out_NotMNIST))
print(np.mean(score_test_out_Gaussian), np.cov(score_test_out_Gaussian))
print(np.mean(score_test_out_Uniform), np.cov(score_test_out_Uniform))


# In[ ]:





# In[ ]:





# In[ ]:




