| import pandas as pd | |
| from transformers import AutoTokenizer | |
| tokenizer = AutoTokenizer.from_pretrained("facebook/esm2_t33_650M_UR50D") | |
| for i in range(6, 50): | |
| df = pd.read_csv(f'/home/tc415/discrete-diffusion-guidance/samples/step_128/{i}.csv') | |
| seqs = df['sequence'].tolist() | |
| outliers = [] | |
| for seq in seqs: | |
| tok = tokenizer(seq)['input_ids'] | |
| if len(tok) - 2 != i: | |
| outliers.append(seq) | |
| print(f"{i}.csv: {outliers}") | |