cerebras.modelzoo.data_preparation.nlp.slimpajama.dedup.generate_duplicate_pairs.split_files

cerebras.modelzoo.data_preparation.nlp.slimpajama.dedup.generate_duplicate_pairs.split_files(input_dir, n_proc)

previous

cerebras.modelzoo.data_preparation.nlp.slimpajama.dedup.generate_duplicate_pairs.lsh

next

cerebras.modelzoo.data_preparation.nlp.slimpajama.dedup.generate_duplicates_dict