. Example:
texts = ["this is my test text", "this is another test text"] * 100
# load a pre-trained tokenizer
tokenizer = BertTokenizerFast.from_pretrained(
"bert-base-uncased",
add_special_tokens=True,
max_length=input_dimensions,
pad_to_max_length=True,
return_tensors="pt",
)
encoded_text = tokenizer(texts, padding=True, truncation=True, return_tensors="pt")
. Example:
texts = ["this is my test text", "this is another test text"] * 100