cerebras.modelzoo.data_preparation.data_preprocessing.utils.check_fim_special_tokens

cerebras.modelzoo.data_preparation.data_preprocessing.utils.check_fim_special_tokens(params, tokenizer)

previous

cerebras.modelzoo.data_preparation.data_preprocessing.utils.append_eos_to_multiple_semantic_regions

next

cerebras.modelzoo.data_preparation.data_preprocessing.utils.chunk