cerebras.modelzoo.data_preparation.nlp.t5.preprocess_t5_dataset.tokenize_text

cerebras.modelzoo.data_preparation.nlp.t5.preprocess_t5_dataset.tokenize_text(text, sp_model)

previous

cerebras.modelzoo.data_preparation.nlp.t5.preprocess_t5_dataset.tokenize_dataset

next

cerebras.modelzoo.data_preparation.nlp.t5.test_convert_t5_dataset