import os
from tqdm import tqdm
from image_synthesis.data.utils.tsv_file import TSVFile, CompositeTSVFile, tsv_writer

indir = '/mnt2/datasets/tophost-art/'
dataset_names = ['dreamstime']
# dataset_names = ['alamy', 'featurepics', 'freepik', 'istockphoto']

for ds in dataset_names:
    print('verify %s===>' % ds)
    image_tsv = TSVFile(os.path.join(indir, 'filtered_%s_image_tsv_combined_001.tsv' % ds))
    text_tsv = TSVFile(os.path.join(indir, 'filtered_%s_text_tsv_combined_001.tsv' % ds))

    assert len(image_tsv) == len(text_tsv), 'length should match'

    for idx in tqdm(range(len(image_tsv))):
        text_data = text_tsv[idx]
        img_data = image_tsv[idx]
        assert text_data[0]==img_data[0], 'filename not match, please check'

import torch

torch.nn.LayerNorm    