import string




path = '/nfs/data/$user/projects/crash/diffusion/diffusers/examples/textual_inversion/attack_concepts/prompts-negative-raw.txt'

# Define a string containing all punctuations
punctuations = string.punctuation
digits = string.digits

# Create a translation table to remove the punctuations
remove_punc = str.maketrans('', '', punctuations)
remove_digit = str.maketrans('', '', digits)

# Remove the punctuations from the sample string
# result = sample_string.translate(translator)

visited = {}
with open(path, 'r') as f:
    lines = f.readlines()

    neg_prompts = []
    for line in lines:
        if len(line) <= 1: continue

        for word in line.split(','):
            word = word.strip()
            if len(word) == 0 or ' ' in word:  ## remove phrases
                continue
            
            word = word.translate(remove_punc)

            if word in visited:
                continue

            neg_prompts.append(word)
            visited[word] = True

save_path = path.replace('-raw', '-filtered')
with open(save_path, 'w') as f:
    f.write('\n'.join(neg_prompts))
