cerebras.modelzoo.data_preparation.nlp.hdf5_preprocessing.utils.handle_jsonl

cerebras.modelzoo.data_preparation.nlp.hdf5_preprocessing.utils.handle_jsonl(jsonl_reader, get_meta, autojoin_paragraphs, para_joiner, key=None)

previous

cerebras.modelzoo.data_preparation.nlp.hdf5_preprocessing.utils.handle_bos_token_default

next

cerebras.modelzoo.data_preparation.nlp.hdf5_preprocessing.utils.has_valid_extension