from torch.utils.data import Dataset
import transformers
from dataclasses import dataclass, field
from typing import Dict, Optional, Sequence, List
import json
import torch
import os
import pandas as pd
from tqdm import tqdm
import random
import logging
from llava.constants import IGNORE_INDEX, PROTEIN_TOKEN_INDEX, DEFAULT_PROTEIN_TOKEN, DEFAULT_PROT_START_TOKEN, DEFAULT_PROT_END_TOKEN
from llava import conversation as conversation_lib
from llava.model import LlavaMPTForCausalLM
from llava.model.language_model.llava_esm import *
from llava.mm_utils import tokenizer_image_token, tokenizer_protein_token
from llava.utils.bio_args import *
import copy
import obonet
import re
import pathlib
import pickle

local_rank = None

def rank0_print(*args):
    if local_rank == 0:
        print(*args)


def maybe_zero_3(param, ignore_status=False, name=None):
    from deepspeed import zero
    from deepspeed.runtime.zero.partition_parameters import ZeroParamStatus
    if hasattr(param, "ds_id"):
        if param.ds_status == ZeroParamStatus.NOT_AVAILABLE:
            if not ignore_status:
                logging.warning(f"{name}: param.ds_status != ZeroParamStatus.NOT_AVAILABLE: {param.ds_status}")
        with zero.GatheredParameters([param]):
            param = param.data.detach().cpu().clone()
    else:
        param = param.detach().cpu().clone()
    return param


# Borrowed from peft.utils.get_peft_model_state_dict
def get_peft_state_maybe_zero_3(named_params, bias):
    if bias == "none":
        to_return = {k: t for k, t in named_params if "lora_" in k}
    elif bias == "all":
        to_return = {k: t for k, t in named_params if "lora_" in k or "bias" in k}
    elif bias == "lora_only":
        to_return = {}
        maybe_lora_bias = {}
        lora_bias_names = set()
        for k, t in named_params:
            if "lora_" in k:
                to_return[k] = t
                bias_name = k.split("lora_")[0] + "bias"
                lora_bias_names.add(bias_name)
            elif "bias" in k:
                maybe_lora_bias[k] = t
        for k, t in maybe_lora_bias:
            if bias_name in lora_bias_names:
                to_return[bias_name] = t
    else:
        raise NotImplementedError
    to_return = {k: maybe_zero_3(v, ignore_status=True) for k, v in to_return.items()}
    return to_return


def get_peft_state_non_lora_maybe_zero_3(named_params, require_grad_only=True):
    to_return = {k: t for k, t in named_params if "lora_" not in k}
    if require_grad_only:
        to_return = {k: t for k, t in to_return.items() if t.requires_grad}
    to_return = {k: maybe_zero_3(v, ignore_status=True).cpu() for k, v in to_return.items()}
    return to_return


def get_mm_adapter_state_maybe_zero_3(named_params, keys_to_match):
    to_return = {k: t for k, t in named_params if any(key_match in k for key_match in keys_to_match)}
    to_return = {k: maybe_zero_3(v, ignore_status=True).cpu() for k, v in to_return.items()}
    return to_return


def find_all_linear_names(model):
    cls = torch.nn.Linear
    lora_module_names = set()
    multimodal_keywords = ['mm_projector', 'protein_tower', 'protein_resampler']
    for name, module in model.named_modules():
        if any(mm_keyword in name for mm_keyword in multimodal_keywords):
            continue
        if isinstance(module, cls):
            names = name.split('.')
            lora_module_names.add(names[0] if len(names) == 1 else names[-1])

    if 'lm_head' in lora_module_names: # needed for 16-bit
        lora_module_names.remove('lm_head')
    return list(lora_module_names)


def safe_save_model_for_hf_trainer(trainer: transformers.Trainer,
                                   output_dir: str):
    """Collects the state dict and dump to disk."""

    if getattr(trainer.args, "tune_mm_mlp_adapter", False):
        # Only save Adapter
        keys_to_match = ['mm_projector', 'smiles_projector']
        if getattr(trainer.args, "use_im_start_end", False):
            keys_to_match.extend(['embed_tokens', 'embed_in'])

        weight_to_save = get_mm_adapter_state_maybe_zero_3(trainer.model.named_parameters(), keys_to_match)
        trainer.model.config.save_pretrained(output_dir)

        current_folder = output_dir.split('/')[-1]
        parent_folder = os.path.dirname(output_dir)
        if trainer.args.local_rank == 0 or trainer.args.local_rank == -1:
            if current_folder.startswith('checkpoint-'):
                mm_projector_folder = os.path.join(parent_folder, "mm_projector")
                os.makedirs(mm_projector_folder, exist_ok=True)
                torch.save(weight_to_save, os.path.join(mm_projector_folder, f'{current_folder}.bin'))
            else:
                torch.save(weight_to_save, os.path.join(output_dir, f'mm_projector.bin'))
        return

    if trainer.deepspeed:
        torch.cuda.synchronize()
        trainer.save_model(output_dir)
        return

    state_dict = trainer.model.state_dict()
    if trainer.args.should_save:
        cpu_state_dict = {
            key: value.cpu()
            for key, value in state_dict.items()
        }
        del state_dict
        trainer._save(output_dir, state_dict=cpu_state_dict)  # noqa


def smart_tokenizer_and_embedding_resize(
    special_tokens_dict: Dict,
    tokenizer: transformers.PreTrainedTokenizer,
    model: transformers.PreTrainedModel,
):
    """Resize tokenizer and embedding.

    Note: This is the unoptimized version that may make your embedding size not be divisible by 64.
    """
    num_new_tokens = tokenizer.add_special_tokens(special_tokens_dict)
    model.resize_token_embeddings(len(tokenizer))

    if num_new_tokens > 0:
        input_embeddings = model.get_input_embeddings().weight.data
        output_embeddings = model.get_output_embeddings().weight.data

        input_embeddings_avg = input_embeddings[:-num_new_tokens].mean(
            dim=0, keepdim=True)
        output_embeddings_avg = output_embeddings[:-num_new_tokens].mean(
            dim=0, keepdim=True)

        input_embeddings[-num_new_tokens:] = input_embeddings_avg
        output_embeddings[-num_new_tokens:] = output_embeddings_avg


def _tokenize_fn(strings: Sequence[str],
                 tokenizer: transformers.PreTrainedTokenizer) -> Dict:
    """Tokenize a list of strings."""
    tokenized_list = [
        tokenizer(
            text,
            return_tensors="pt",
            padding="longest",
            max_length=tokenizer.model_max_length,
            truncation=True,
        ) for text in strings
    ]
    input_ids = labels = [
        tokenized.input_ids[0] for tokenized in tokenized_list
    ]
    input_ids_lens = labels_lens = [
        tokenized.input_ids.ne(tokenizer.pad_token_id).sum().item()
        for tokenized in tokenized_list
    ]
    return dict(
        input_ids=input_ids,
        labels=labels,
        input_ids_lens=input_ids_lens,
        labels_lens=labels_lens,
    )


def _mask_targets(target, tokenized_lens, speakers):
    # cur_idx = 0
    cur_idx = tokenized_lens[0]
    tokenized_lens = tokenized_lens[1:]
    target[:cur_idx] = IGNORE_INDEX
    for tokenized_len, speaker in zip(tokenized_lens, speakers):
        if speaker == "human":
            target[cur_idx+2:cur_idx + tokenized_len] = IGNORE_INDEX
        cur_idx += tokenized_len


def _add_speaker_and_signal(header, source, get_conversation=True):
    """Add speaker and start/end signal on each round."""
    BEGIN_SIGNAL = "### "
    END_SIGNAL = "\n"
    conversation = header
    for sentence in source:
        from_str = sentence["from"]
        if from_str.lower() == "human":
            from_str = conversation_lib.default_conversation.roles[0]
        elif from_str.lower() == "gpt":
            from_str = conversation_lib.default_conversation.roles[1]
        else:
            from_str = 'unknown'
        sentence["value"] = (BEGIN_SIGNAL + from_str + ": " +
                             sentence["value"] + END_SIGNAL)
        if get_conversation:
            conversation += sentence["value"]
    conversation += BEGIN_SIGNAL
    return conversation


def preprocess_multimodal(
    sources: Sequence[str],
    data_args: DataArguments
) -> Dict:
    is_multimodal = data_args.is_multimodal
    if not is_multimodal:
        return sources

    for source in sources:
        for sentence in source:
            if DEFAULT_PROTEIN_TOKEN in sentence['value']:
                sentence['value'] = sentence['value'].replace(DEFAULT_PROTEIN_TOKEN, '').strip()
                sentence['value'] = DEFAULT_PROTEIN_TOKEN + '\n' + sentence['value']
                sentence['value'] = sentence['value'].strip()
                if "mmtag" in conversation_lib.default_conversation.version:
                    sentence['value'] = sentence['value'].replace(DEFAULT_PROTEIN_TOKEN, '<Image>' + DEFAULT_PROTEIN_TOKEN + '</Image>')
            replace_token = DEFAULT_PROTEIN_TOKEN
            if data_args.mm_use_prot_start_end:
                replace_token = DEFAULT_PROT_START_TOKEN + replace_token + DEFAULT_PROT_END_TOKEN
            sentence["value"] = sentence["value"].replace(DEFAULT_PROTEIN_TOKEN, replace_token)

    return sources


def preprocess_llama_2(
    sources,
    tokenizer: transformers.PreTrainedTokenizer,
    has_protein: bool = False
) -> Dict:
    conv = conversation_lib.default_conversation.copy()
    roles = {"human": conv.roles[0], "biological gpt": conv.roles[1]}

    # Apply prompt templates
    conversations = []
    for i, source in enumerate(sources):
        if roles[source[0]["from"]] != conv.roles[0]:
            # Skip the first one if it is not from human
            source = source[1:]

        conv.messages = []
        for j, sentence in enumerate(source):
            role = roles[sentence["from"]]
            assert role == conv.roles[j % 2], f"{i}"
            conv.append_message(role, sentence["value"])
        conversations.append(conv.get_prompt())

    # Tokenize conversations
    # print(conversations)

    if has_protein:
        input_ids = torch.stack([tokenizer_protein_token(prompt, tokenizer, return_tensors='pt') for prompt in conversations], dim=0)
    else:
        input_ids = tokenizer(
            conversations,
            return_tensors="pt",
            padding="longest",
            max_length=tokenizer.model_max_length,
            truncation=True,
        ).input_ids

    targets = input_ids.clone()

    assert conv.sep_style == conversation_lib.SeparatorStyle.LLAMA_2

    # Mask targets
    sep = "[/INST] "
    for conversation, target in zip(conversations, targets):
        total_len = int(target.ne(tokenizer.pad_token_id).sum())

        rounds = conversation.split(conv.sep2)
        cur_len = 1
        target[:cur_len] = IGNORE_INDEX
        for i, rou in enumerate(rounds):
            if rou == "":
                break

            parts = rou.split(sep)
            if len(parts) != 2:
                break
            parts[0] += sep

            if has_protein:
                round_len = len(tokenizer_protein_token(rou, tokenizer))
                instruction_len = len(tokenizer_protein_token(parts[0], tokenizer)) - 2
            else:
                round_len = len(tokenizer(rou).input_ids)
                instruction_len = len(tokenizer(parts[0]).input_ids) - 2

            target[cur_len : cur_len + instruction_len] = IGNORE_INDEX

            cur_len += round_len
        target[cur_len:] = IGNORE_INDEX

        if cur_len < tokenizer.model_max_length:
            if cur_len != total_len:
                target[:] = IGNORE_INDEX
                print(
                    f"WARNING: tokenization mismatch: {cur_len} vs. {total_len}."
                    f" (ignored)"
                )

    return dict(
        input_ids=input_ids,
        labels=targets,
    )


def preprocess_v1(
    sources,
    tokenizer: transformers.PreTrainedTokenizer,
    has_protein: bool = False
) -> Dict:
    conv = conversation_lib.default_conversation.copy()
    roles = {"human": conv.roles[0], "biological gpt": conv.roles[1]}
    
    # Apply prompt templates
    conversations = []
    for i, source in enumerate(sources):
        if roles[source[0]["from"]] != conv.roles[0]:
            # Skip the first one if it is not from human
            source = source[1:]

        conv.messages = []
        for j, sentence in enumerate(source):
            role = roles[sentence["from"]]
            assert role == conv.roles[j % 2], f"{i}"
            conv.append_message(role, sentence["value"])
        conversations.append(conv.get_prompt())

    # Tokenize conversations
    if has_protein:
        input_ids = torch.stack([tokenizer_protein_token(prompt, tokenizer, return_tensors='pt') for prompt in conversations], dim=0)
    else:
        input_ids = tokenizer(
            conversations,
            return_tensors="pt",
            padding="longest",
            max_length=tokenizer.model_max_length,
            truncation=True,
        ).input_ids

    targets = input_ids.clone()

    assert conv.sep_style == conversation_lib.SeparatorStyle.TWO

    # Mask targets
    sep = conv.sep + conv.roles[1] + ": "
    for conversation, target in zip(conversations, targets):
        total_len = int(target.ne(tokenizer.pad_token_id).sum())

        rounds = conversation.split(conv.sep2)
        cur_len = 1
        target[:cur_len] = IGNORE_INDEX
        for i, rou in enumerate(rounds):
            if rou == "":
                break

            parts = rou.split(sep)
            if len(parts) != 2:
                break
            parts[0] += sep

            if has_protein:
                round_len = len(tokenizer_protein_token(rou, tokenizer))
                instruction_len = len(tokenizer_protein_token(parts[0], tokenizer)) - 2
            else:
                round_len = len(tokenizer(rou).input_ids)
                instruction_len = len(tokenizer(parts[0]).input_ids) - 2

            target[cur_len : cur_len + instruction_len] = IGNORE_INDEX

            cur_len += round_len
        target[cur_len:] = IGNORE_INDEX

        if cur_len < tokenizer.model_max_length:
            if cur_len != total_len:
                target[:] = IGNORE_INDEX
                print(
                    f"WARNING: tokenization mismatch: {cur_len} vs. {total_len}."
                    f" (ignored)"
                )

    return dict(
        input_ids=input_ids,
        labels=targets,
    )


def preprocess_mpt(
    sources,
    tokenizer: transformers.PreTrainedTokenizer,
) -> Dict:
    conv = conversation_lib.default_conversation.copy()
    roles = {"human": conv.roles[0], "gpt": conv.roles[1]}

    # Apply prompt templates
    conversations = []
    for i, source in enumerate(sources):
        if roles[source[0]["from"]] != conv.roles[0]:
            # Skip the first one if it is not from human
            source = source[1:]

        conv.messages = []
        for j, sentence in enumerate(source):
            role = roles[sentence["from"]]
            assert role == conv.roles[j % 2], f"{i}"
            conv.append_message(role, sentence["value"])
        conversations.append(conv.get_prompt())

    # Tokenize conversations
    input_ids = torch.stack([tokenizer_protein_token(prompt, tokenizer, return_tensors='pt') for prompt in conversations], dim=0)
    targets = input_ids.clone()
    assert conv.sep_style == conversation_lib.SeparatorStyle.MPT

    # Mask targets
    sep = conv.sep + conv.roles[1]
    for conversation, target in zip(conversations, targets):
        total_len = int(target.ne(tokenizer.pad_token_id).sum())

        rounds = conversation.split(conv.sep)
        re_rounds = [conv.sep.join(rounds[:3])] # system + user + gpt
        for conv_idx in range(3, len(rounds), 2):
            re_rounds.append(conv.sep.join(rounds[conv_idx:conv_idx+2]))    # user + gpt
        cur_len = 0
        target[:cur_len] = IGNORE_INDEX
        for i, rou in enumerate(re_rounds):
            if rou == "":
                break

            parts = rou.split(sep)
            if len(parts) != 2:
                break
            parts[0] += sep
            round_len = len(tokenizer_protein_token(rou, tokenizer)) + len(tokenizer_protein_token(conv.sep, tokenizer))
            instruction_len = len(tokenizer_protein_token(parts[0], tokenizer))
            target[cur_len : cur_len + instruction_len] = IGNORE_INDEX

            cur_len += round_len
        target[cur_len:] = IGNORE_INDEX

        if cur_len < tokenizer.model_max_length:
            if cur_len != total_len:
                target[:] = IGNORE_INDEX
                print(
                    f"WARNING: tokenization mismatch: {cur_len} vs. {total_len}."
                    f" (ignored)"
                )

    return dict(
        input_ids=input_ids,
        labels=targets,
    )


def preprocess_plain(
    sources: Sequence[str],
    tokenizer: transformers.PreTrainedTokenizer,
) -> Dict:
    # add end signal and concatenate together
    conversations = []
    for source in sources:
        assert len(source) == 2
        # print(source)
        # print(source[0]['value'], DEFAULT_PROTEIN_TOKEN in source[0]['value'])
        assert DEFAULT_PROTEIN_TOKEN in source[0]['value']
        source[0]['value'] = DEFAULT_PROTEIN_TOKEN
        conversation = source[0]['value'] + source[1]['value'] + conversation_lib.default_conversation.sep
        conversations.append(conversation)
    # tokenize conversations
    input_ids = [tokenizer_protein_token(prompt, tokenizer, return_tensors='pt') for prompt in conversations]
    targets = copy.deepcopy(input_ids)
    for target, source in zip(targets, sources):
        tokenized_len = len(tokenizer_protein_token(source[0]['value'], tokenizer))
        target[:tokenized_len] = IGNORE_INDEX

    return dict(input_ids=input_ids, labels=targets)


def preprocess(
    sources: Sequence[str],
    tokenizer: transformers.PreTrainedTokenizer,
    has_protein: bool = False
) -> Dict:
    """
    Given a list of sources, each is a conversation list. This transform:
    1. Add signal '### ' at the beginning each sentence, with end signal '\n';
    2. Concatenate conversations together;
    3. Tokenize the concatenated conversation;
    4. Make a deepcopy as the target. Mask human words with IGNORE_INDEX.
    """
    if conversation_lib.default_conversation.sep_style == conversation_lib.SeparatorStyle.PLAIN:
        return preprocess_plain(sources, tokenizer)
    if conversation_lib.default_conversation.sep_style == conversation_lib.SeparatorStyle.LLAMA_2:
        return preprocess_llama_2(sources, tokenizer, has_protein=has_protein)
    if conversation_lib.default_conversation.version.startswith("v1"):
        return preprocess_v1(sources, tokenizer, has_protein=has_protein)
    if conversation_lib.default_conversation.version == "mpt":
        return preprocess_mpt(sources, tokenizer)
    # add end signal and concatenate together
    conversations = []
    for source in sources:
        header = f"{conversation_lib.default_conversation.system}\n\n"
        conversation = _add_speaker_and_signal(header, source)
        conversations.append(conversation)
    # tokenize conversations
    def get_tokenize_len(prompts):
        return [len(tokenizer_protein_token(prompt, tokenizer)) for prompt in prompts]

    if has_protein:
        input_ids = [tokenizer_protein_token(prompt, tokenizer, return_tensors='pt') for prompt in conversations]
    else:
        conversations_tokenized = _tokenize_fn(conversations, tokenizer)
        input_ids = conversations_tokenized["input_ids"]

    targets = copy.deepcopy(input_ids)
    for target, source in zip(targets, sources):
        if has_protein:
            tokenized_lens = get_tokenize_len([header] + [s["value"] for s in source])
        else:
            tokenized_lens = _tokenize_fn([header] + [s["value"] for s in source], tokenizer)["input_ids_lens"]
        speakers = [sentence["from"] for sentence in source]
        _mask_targets(target, tokenized_lens, speakers)

    return dict(input_ids=input_ids, labels=targets)

@dataclass
class DataCollatorForSupervisedDataset(object):
    """Collate examples for supervised fine-tuning."""

    tokenizer: transformers.PreTrainedTokenizer

    def __call__(self, instances: Sequence[Dict]) -> Dict[str, torch.Tensor]:
        input_ids, labels = tuple([instance[key] for instance in instances]
                                  for key in ("input_ids", "labels"))
        input_ids = torch.nn.utils.rnn.pad_sequence(
            input_ids,
            batch_first=True,
            padding_value=self.tokenizer.pad_token_id)
        labels = torch.nn.utils.rnn.pad_sequence(labels,
                                                 batch_first=True,
                                                 padding_value=IGNORE_INDEX)
        input_ids = input_ids[:, :self.tokenizer.model_max_length]
        labels = labels[:, :self.tokenizer.model_max_length]
        batch = dict(
            input_ids=input_ids,
            labels=labels,
            attention_mask=input_ids.ne(self.tokenizer.pad_token_id),
        )

        if 'protein' in instances[0]:
            proteins = [instance['protein'] for instance in instances]
            batch['images'] = proteins
            # if all(x is not None and x.shape == images[0].shape for x in images):
            #     batch['images'] = torch.stack(images)
            # else:
            #     batch['images'] = images
        if 't_labels' in instances[0]:
            t_labels = [instance['t_labels'] for instance in instances]
            batch['t_labels'] = t_labels
        if 'smiles' in instances[0]:
            smiles = [instance['smiles'] for instance in instances]
            batch['smiles'] = smiles
        if 'spc_inds' in instances[0]:
            spc_inds = [instance['spc_inds'] for instance in instances]
            batch['spc_inds'] = spc_inds
        
        return batch

class LazySupervisedDataset(Dataset):
    """Dataset for supervised fine-tuning."""

    def __init__(self, prot_data_path, 
                 tokenizer: transformers.PreTrainedTokenizer,
                 data_args: DataArguments, 
                 max_labels = 5,
                 is_eval = False):
        super(LazySupervisedDataset, self).__init__()
        protein_data = pd.read_pickle(prot_data_path)
        self.go_graph = obonet.read_obo(data_args.go_term_graph)
        self.max_labels = max_labels
        self.is_eval = is_eval
        self.protein_data = []
        keys = []
        for key in protein_data:
            keys.append(key)
        # print(dict_data)
        self.caption_template = [
            "List the GO term funtion names of the protein seperate with ','.\n<protein>",
        ]
        
        self.predict_label_fn = lambda input_data: ",".join(self.label_sample(input_data['annotations']))
        # print(protein_data.head(5), keys)
        
        for i in tqdm(protein_data.index[:1000]):
            single_frame = {}
            for key in keys:
                single_frame[key] = protein_data[key][i]
            single_frame['text'] = random.choice(self.caption_template)
            self.protein_data.append(single_frame)
                # self.protein_data[key].append(protein_data[key][i])
        # self.protein_data = pd.read_pickle(data_args.protein_pkl)
        
        self.tokenizer = tokenizer
 
        self.data_args = data_args
        self.max_protein_length = self.data_args.max_protein_length
        
    def label_sample(self, items):
        sample_len = min(len(items), self.max_labels)
        if self.is_eval:
            return [ii for ii in items[:sample_len]]
        else:
            return random.sample(items, sample_len)

    def __len__(self):
        return len(self.protein_data)

    @property
    def lengths(self):
        length_list = []
        for sample in self.protein_data:
            protein_tokens = self.max_protein_length
            prompt_length = len(sample['text'].split())
            label_length = len(self.predict_label_fn(sample).split())
            length_list.append(protein_tokens+prompt_length+label_length)
        return length_list

    @property
    def modality_lengths(self):
        length_list = []
        for sample in self.protein_data:
            protein_tokens = self.max_protein_length
            prompt_length = len(sample['text'].split())
            label_length = len(self.predict_label_fn(sample).split())
            cur_len = prompt_length+label_length
            
            length_list.append(cur_len)
        return length_list



    def __getitem__(self, i) -> Dict[str, torch.Tensor]:
        data_frame = self.protein_data[i]
        # if isinstance(i, int):
        #     sources = [sources]
        data_dict = {}
        # data_dict['text'] = 
        
        # predict_label_fn = lambda input_data: " ".join(input_data['annotations'])
        
        sources = [
            {"from": "human", "value": data_frame['text']},
            {"from": "gpt", "value": self.predict_label_fn(data_frame)}
        ]

        data_dict = preprocess(
            [sources],
            self.tokenizer,
            has_protein=('sequences' in data_frame))
        for key in data_dict:
            data_dict[key] = data_dict[key][0]
            
        data_dict['protein'] = data_frame['sequences'][:self.max_protein_length]
        # print(data_dict)
        
        # print(sources[1]['value'])
        # print(len(data_dict['protein']), len(data_dict['input_ids']), len(data_dict['labels']))
        return data_dict

def get_mmseq_file(str):
    file_pp = re.compile(r"/(.*?)\.csv")
    path_pp = re.compile(r"(.*)/")
    
    mmseq_path = pathlib.Path(path_pp.findall(str)[0])
    return mmseq_path.joinpath(f"{file_pp.findall(str)[0]}_query.m8")

def load_mmseq_file(f_path, is_eval = False):
    pd_data = pd.read_csv(f_path, sep='\t', header=None)
    mmseq_mapping = {}
    for i in tqdm(range(len(pd_data))):
        data_frame = pd_data.iloc[i]
        if data_frame[0] == data_frame[1]:
            continue
        if data_frame[0] not in mmseq_mapping:
            mmseq_mapping[data_frame[0]] = {'ref': data_frame[1], 'score': data_frame[2]}
        if mmseq_mapping[data_frame[0]]['score'] < data_frame[2]:
                mmseq_mapping[data_frame[0]] = {'ref': data_frame[1], 'score': data_frame[2]}
        # elif is_eval:
        #     if mmseq_mapping[data_frame[0]]['score'] < data_frame[2]:
        #         mmseq_mapping[data_frame[0]] = {'ref': data_frame[1], 'score': data_frame[2]}
        # else:
        #     if mmseq_mapping[data_frame[0]]['score'] > data_frame[2]:
        #         mmseq_mapping[data_frame[0]] = {'ref': data_frame[1], 'score': data_frame[2]}

    return mmseq_mapping

def get_database_file(str):
    path_pp = re.compile(r"(.*)/")
    database_path = pathlib.Path(path_pp.findall(str)[0])
    return database_path.joinpath(f"split100.csv")

def load_database(d_file):
    pd_data = pd.read_csv(d_file, sep='\t')
    
    db_mapping = {}
    for i in tqdm(range(len(pd_data))):
        data_frame = pd_data.iloc[i]
        db_mapping[data_frame['Entry']] = [data_frame['Sequence'], data_frame['EC number']]
    return db_mapping

class ECNumberDataset(Dataset):
    """Dataset for supervised fine-tuning."""

    def __init__(self, ec_num_path, 
                 tokenizer: transformers.PreTrainedTokenizer,
                 data_args, 
                 max_labels = 5,
                 is_eval = False,
                 retrieval_mmseq = False,
                 retrieval_smiles = False):
        super(ECNumberDataset, self).__init__()
        protein_data = pd.read_csv(ec_num_path, sep='\t')
        self.max_labels = max_labels
        self.is_eval = is_eval
        self.enzclass_file = data_args.enzclass_file
        self.with_comments = data_args.with_comments
        self.smiles_path = pathlib.Path(data_args.smiles_path)
        
        self.protein_data = []
        keys = []
        for key in protein_data:
            keys.append(key)
        # print(dict_data)
        if retrieval_mmseq and not retrieval_smiles:
            self.caption_template = [
                "Protein: <protein>\nCandidate protein: <protein>\nList all enzyme commission numbers that are associated with the protein, this is protein is similar with the candidate protein.\n"
            ]
            self.spc_token_inds = [0, 0]
        elif retrieval_mmseq and retrieval_smiles:
            
            with open(data_args.ecnumber_rheadid_file, 'r') as fin:
                self.ec_number_rhead_id_mapping = json.load(fin)
            self.caption_template = [
                "Protein: <protein>\nCandidate protein: <protein>\nOne of the generated product: <protein>\nList all enzyme commission numbers that are associated with the protein, this is protein is similar with the candidate protein.\n"
            ]
            self.spc_token_inds = [0, 0, 1]
        elif retrieval_smiles:
            with open(data_args.ecnumber_rheadid_file, 'r') as fin:
                self.ec_number_rhead_id_mapping = json.load(fin)
            self.caption_template = [
                "Protein: <protein>\nOne of the generated product: <protein>\nList all enzyme commission numbers that are associated with the protein, this is protein is similar with the candidate protein.\n"
            ]
            self.spc_token_inds = [0, 1]
        else:
            self.caption_template = [
                "Protein: <protein>\nList all enzyme commission numbers that are associated with the protein\n"
                # "Which EC numbers are accociated with:\n<protein>"
            ]
            self.spc_token_inds = [0]
        # with open(data_args.query_template, 'r') as fin:
        #     cap_template = json.load(fin)
        # for i in range(len(cap_template)):
        #     temp = cap_template[f'{i+1}']
        #     self.caption_template.append(
        #         f"Protein: <protein>\n{temp['Request']}\n"
        #     )
        def re_code(ec):
            # return ec
            ec_list = ec.split('.')
            return ec_list[0] + "Z" + ec_list[1] + 'Z' + ec_list[2] + 'Z' + ec_list[3]
        self.re_code = re_code

        def naive_fn(input_data):
            ecs = input_data['EC number'].split(';')
            for i in range(len(ecs)):
                # ec_list = ecs[i].split('.')
                ecs[i] = self.re_code(ecs[i])
            
            return f"The corresponding enzyme commission numbers {'are' if len(ecs) > 1 else 'is'} {';'.join(ecs)}</s>"
            
        # self.predict_label_fn = lambda input_data: f"The corresponding enzyme commission numbers {'are' if ';' in input_data['EC number'] else 'is'} {input_data['EC number']}</s>"
        self.predict_label_fn = naive_fn
        if self.with_comments:
            with open(data_args.comments_file, 'r') as fin:
                self.ec_dest = json.load(fin)
            def label_fn(input_data):
                if self.is_eval:
                    return input_data['EC number']
                prot_ec = input_data['EC number'].split(';')
                prot_label = [f"EC number {self.re_code(ii)}: {self.ec_dest[ii]['Comment'].strip()}" for ii in prot_ec]
                return ";\n".join(prot_label) + ";\n"
            self.predict_label_fn = label_fn
        # print(protein_data.head(5), keys)
        
        for i in tqdm(protein_data.index):
            single_frame = {}
            for key in keys:
                single_frame[key] = protein_data[key][i]
            single_frame['text'] = random.choice(self.caption_template)
            self.protein_data.append(single_frame)
                # self.protein_data[key].append(protein_data[key][i])
        # self.protein_data = pd.read_pickle(data_args.protein_pkl)
        # self.protein_data = self.protein_data[:2000]
        self.retrieval_mmseq = retrieval_mmseq
        self.retrieval_smiles = retrieval_smiles
        # if retrieval_mmseq:
        mmseq_mapping_cache = get_mmseq_file(ec_num_path).__str__()+'.cache'
        db_mapping_cache = get_database_file(ec_num_path).__str__() + '.cache'
        if not self.is_eval:
            if pathlib.Path(mmseq_mapping_cache).exists():
                with open(mmseq_mapping_cache, 'rb') as fin:
                    self.mmseq_mapping = pickle.load(fin)
            else:
                self.mmseq_mapping = load_mmseq_file(get_mmseq_file(ec_num_path))
                with open(mmseq_mapping_cache, 'wb') as fout:
                    pickle.dump(self.mmseq_mapping, fout)
        else:
            self.mmseq_mapping = load_mmseq_file(get_mmseq_file(ec_num_path))
        if not self.is_eval:
            if pathlib.Path(db_mapping_cache).exists():
                with open(db_mapping_cache, 'rb') as fin:
                    self.db_mapping = pickle.load(fin)
            else:
                self.db_mapping = load_database(get_database_file(ec_num_path))
                with open(db_mapping_cache, 'wb') as fout:
                    pickle.dump(self.db_mapping, fout)
        else:
            self.db_mapping = load_database(get_database_file(ec_num_path))
            # self.db_mapping = load_database(get_database_file(ec_num_path))
            
        self.tokenizer = tokenizer
 
        self.data_args = data_args
        self.max_protein_length = self.data_args.max_protein_length


    def __len__(self):
        return len(self.protein_data)

    @property
    def lengths(self):
        length_list = []
        for sample in self.protein_data:
            protein_tokens = self.max_protein_length
            prompt_length = len(sample['text'].split())
            label_length = len(self.predict_label_fn(sample).split())
            length_list.append(protein_tokens+prompt_length+label_length)
        return length_list

    @property
    def modality_lengths(self):
        length_list = []
        for sample in self.protein_data:
            protein_tokens = self.max_protein_length
            prompt_length = len(sample['text'].split())
            label_length = len(self.predict_label_fn(sample).split())
            cur_len = prompt_length+label_length
            
            length_list.append(cur_len)
        return length_list



    def __getitem__(self, i) -> Dict[str, torch.Tensor]:
        data_frame = self.protein_data[i]
        # if isinstance(i, int):
        #     sources = [sources]
        data_dict = {}
        # data_dict['text'] = 
        
        # predict_label_fn = lambda input_data: " ".join(input_data['annotations'])
        
        sources = [
            {"from": "human", "value": data_frame['text']},
            {"from": "biological gpt", "value": self.predict_label_fn(data_frame)}
        ]
        # print(sources)
        data_dict = preprocess(
            [sources],
            self.tokenizer,
            has_protein=('Sequence' in data_frame))
        for key in data_dict:
            data_dict[key] = data_dict[key][0]
        
        if self.retrieval_mmseq:
            if data_frame['Entry'] in self.mmseq_mapping:
                data_dict['protein'] = [data_frame['Sequence'][:self.max_protein_length], 
                                        self.db_mapping[
                                                        self.mmseq_mapping[
                                                                            data_frame['Entry']
                                                                        ]['ref']
                                                    ][0][: self.max_protein_length]
                                    ]
            else:
                # print(f"The special protein: {data_frame['Entry']}")
                data_dict['protein'] = [data_frame['Sequence'][:self.max_protein_length], data_frame['Sequence'][:self.max_protein_length]]
            
        else:
            data_dict['protein'] = data_frame['Sequence'][:self.max_protein_length]
        if self.retrieval_smiles:
            ecn = random.choice(data_frame['EC number'].split(';'))
            if not self.is_eval:
                if ecn in self.ec_number_rhead_id_mapping:
                    smiles_id = self.ec_number_rhead_id_mapping[ecn][0]
                else:
                    ecn3 = ecn[:ecn.rfind('.')]
                    if ecn3 in self.ec_number_rhead_id_mapping:
                        smiles_id = self.ec_number_rhead_id_mapping[ecn3][0]
                    else:
                        smiles_id = None
                if smiles_id is not None:
                    with open(self.smiles_path.joinpath(f"{smiles_id}.pkl"), 'rb') as fin:
                        smiles_feature = pickle.load(fin)['generate'][0]
                    data_dict['smiles'] = smiles_feature[:, :self.max_protein_length]
                else:
                    data_dict['smiles'] = torch.zeros(1, self.max_protein_length, 768)
            else:
                if data_frame['Entry'] in self.mmseq_mapping:
                    ref_ecn = self.db_mapping[self.mmseq_mapping[data_frame['Entry']]['ref']][1]
                    if ref_ecn in self.ec_number_rhead_id_mapping:
                        smiles_id = self.ec_number_rhead_id_mapping[ref_ecn][0]
                    else:
                        ref_ecn3 = ref_ecn[:ref_ecn.rfind('.')]
                        if ref_ecn3 in self.ec_number_rhead_id_mapping:
                            smiles_id = self.ec_number_rhead_id_mapping[ref_ecn3][0]
                        else:
                            smiles_id = None
                        # smiles_id = random.choice(self.ec_number_rhead_id_mapping[ref_ecn3])
                    if smiles_id is not None:
                        with open(self.smiles_path.joinpath(f"{smiles_id}.pkl"), 'rb') as fin:
                            smiles_feature = pickle.load(fin)['generate'][0]
                        data_dict['smiles'] = smiles_feature[:, :self.max_protein_length]
                    else:
                        data_dict['smiles'] = torch.zeros(1, self.max_protein_length, 768)
                    # data_dict['smiles'] = torch.nn.functional.pad(data_dict['smiles'], (0, 0, 0, self.max_protein_length - data_dict['smiles'].shape[1]))
                else:
                    data_dict['smiles'] = torch.zeros(1, self.max_protein_length, 768)
            if self.max_protein_length - data_dict['smiles'].shape[1] > 0:
                data_dict['smiles'] = torch.nn.functional.pad(data_dict['smiles'], (0, 0, 0, self.max_protein_length - data_dict['smiles'].shape[1]))
        data_dict['spc_inds'] = self.spc_token_inds
        # if self.is_eval:
        #     data_dict['t_labels'] = self.predict_label_fn(data_frame)
        # rank0_print(data_dict)
        
        # print(sources[1]['value'])
        # print(len(data_dict['protein']), len(data_dict['input_ids']), len(data_dict['labels']))
        return data_dict
    
    
