cerebras.modelzoo.data_preparation.nlp.hdf5_preprocessing.utils.validate_tokens

cerebras.modelzoo.data_preparation.nlp.hdf5_preprocessing.utils.validate_tokens(tokens, min_len=2)

previous

cerebras.modelzoo.data_preparation.nlp.hdf5_preprocessing.utils.update_params

next

cerebras.modelzoo.data_preparation.nlp.hdf5_preprocessing.utils.verify_saved_hdf5_files