File size: 5,653 Bytes
fc508dc afac2a3 fc508dc afac2a3 9378c69 fc508dc 9378c69 fc508dc 9378c69 fc508dc 9378c69 fc508dc |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 |
# coding=utf-8
# Copyright 2022 Meta and The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tokenization classes for ESM."""
import os
from huggingface_hub import hf_hub_download
from typing import List, Optional
#from transformers.models.esm.tokenization_esm import PreTrainedTokenizer
from transformers import EsmTokenizer, PreTrainedTokenizer
VOCAB_FILES_NAMES = {"vocab_file": "vocab.txt"}
def load_vocab_file(vocab_file):
with open(vocab_file, "r") as f:
lines = f.read().splitlines()
return [l.strip() for l in lines]
class IsoformerTokenizer(PreTrainedTokenizer):
"""
Constructs Isoformer tokenizer.
"""
def __init__(
self,
**kwargs
):
# Get the model ID from kwargs
model_id = kwargs.get("name_or_path", None) # This will be "InstaDeepAI/isoformer"
# Use hf_hub_download to get the local path to each vocabulary file.
# This function intelligently uses the local cache if the file is already downloaded.
if model_id:
try:
dna_vocab_path = hf_hub_download(repo_id=model_id, filename="dna_vocab_list.txt")
rna_vocab_path = hf_hub_download(repo_id=model_id, filename="rna_vocab_list.txt")
protein_vocab_path = hf_hub_download(repo_id=model_id, filename="protein_vocab_list.txt")
except Exception as e:
# Fallback in case hf_hub_download fails (e.g., if model_id was a local path not a Hub ID)
# This fallback might not be perfect for all edge cases, but covers the common local loading.
print(f"Warning: Failed to resolve model files via hf_hub_download. Attempting local fallback. Error: {e}")
dna_vocab_path = os.path.join(model_id, "dna_vocab_list.txt")
rna_vocab_path = os.path.join(model_id, "rna_vocab_list.txt")
protein_vocab_path = os.path.join(model_id, "protein_vocab_list.txt")
else:
# Fallback if model_id is not found (unlikely for AutoTokenizer.from_pretrained)
print("Warning: Could not determine model_id from kwargs. Falling back to relative paths.")
dna_vocab_path = "dna_vocab_list.txt"
rna_vocab_path = "rna_vocab_list.txt"
protein_vocab_path = "protein_vocab_list.txt"
dna_hf_tokenizer = EsmTokenizer(dna_vocab_path, model_max_length=196608)
dna_hf_tokenizer.eos_token = None # Stops the tokenizer adding an EOS/SEP token at the end
dna_hf_tokenizer.init_kwargs["eos_token"] = None # Ensures it doesn't come back when reloading
dna_hf_tokenizer.bos_token = None # Stops the tokenizer adding an BOS/SEP token at the end
dna_hf_tokenizer.init_kwargs["bos_token"] = None # Ensures it doesn't come back when reloading
rna_hf_tokenizer = EsmTokenizer(rna_vocab_path, model_max_length=1024)
rna_hf_tokenizer.eos_token = None # Stops the tokenizer adding an EOS/SEP token at the end
rna_hf_tokenizer.init_kwargs["eos_token"] = None # Ensures it doesn't come back when reloading
protein_hf_tokenizer = EsmTokenizer(protein_vocab_path, model_max_length=1024)
# protein_hf_tokenizer.eos_token = None # Stops the tokenizer adding an EOS/SEP token at the end
# protein_hf_tokenizer.init_kwargs["eos_token"] = None # Ensures it doesn't come back when reloading
self.dna_tokenizer = dna_hf_tokenizer
self.rna_tokenizer = rna_hf_tokenizer
self.protein_tokenizer = protein_hf_tokenizer
self.dna_tokens = open(dna_vocab_path, "r").read() .split("\n")
self.rna_tokens = open(rna_vocab_path, "r").read() .split("\n")
self.protein_tokens = open(protein_vocab_path, "r").read() .split("\n")
super().__init__(**kwargs)
def __call__(self, dna_input, rna_input, protein_input):
dna_output = self.dna_tokenizer(dna_input)
rna_output = self.rna_tokenizer(rna_input, max_length=1024, padding="max_length")
protein_output = self.protein_tokenizer(protein_input, max_length=1024, padding="max_length")
return dna_output, rna_output, protein_output
def _add_tokens(self, *args, **kwargs):
pass # Override this with an empty method to stop errors
def save_vocabulary(self, save_directory, filename_prefix):
vocab_file_dna = os.path.join(save_directory, (filename_prefix + "-" if filename_prefix else "") + "dna_vocab_list.txt")
vocab_file_rna = os.path.join(save_directory, (filename_prefix + "-" if filename_prefix else "") + "rna_vocab_list.txt")
vocab_file_protein = os.path.join(save_directory, (filename_prefix + "-" if filename_prefix else "") + "protein_vocab_list.txt")
with open(vocab_file_dna, "w") as f:
f.write("\n".join(self.dna_tokens))
with open(vocab_file_rna, "w") as f:
f.write("\n".join(self.rna_tokens))
with open(vocab_file_protein, "w") as f:
f.write("\n".join(self.protein_tokens))
return (vocab_file_dna,vocab_file_rna,vocab_file_protein, ) |