

def diversity(lines):
    d1 = 0.0
    d2 = 0.0
    uni_set = set()
    uni_num = 0
    bi_set = set()
    bi_num = 0
    for line in lines:
        flist = line.split(' ')
        for x in flist:
            uni_set.add(x)
            uni_num += 1
        for i in range(len(flist) - 1):
            bi_set.add(flist[i] + ' ' + flist[i+1])
            bi_num += 1
    d1 += len(uni_set) / uni_num
    d2 += len(bi_set) / bi_num
    print('diversity:', d1, d2)
    print('distinct sentences', len(set(lines))/float(len(lines)))
    