from transformers import AutoTokenizer

def tokenize_string(input_string):
    # 初始化tokenizer
    tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2.5-Coder-7B")

    # 对输入字符串进行tokenization
    tokens = tokenizer.tokenize(input_string)
    
    # 打印token
    print(f"Tokens: {tokens}")

    # 获取token ids
    token_ids = tokenizer.convert_tokens_to_ids(tokens)
    
    # 打印token ids
    print(f"Token IDs: {token_ids}")

if __name__ == "__main__":
    input_string = "       b, l, u = map(int, input().split())\n        exams.append([b, l, u])\n\n"
    tokenize_string(input_string)
