import json
import pandas as pd
import tiktoken

res = pd.read_json(path_or_buf="more_sophisticated_monitored_and_tampering.jsonl", lines=True)
# print(res)
token_number = 0
enc = tiktoken.get_encoding("cl100k_base")
flag = 0
for text_list in res["messages"]:
    for text_dict in text_list:
        txt = text_dict["content"]
        encoded = enc.encode(str(txt))
        token_number += len(encoded)

print(f"total number of token is: {token_number}")
