Source code for cerebras.modelzoo.data_preparation.nlp.extractive_summarisation_utils

# Copyright 2022 Cerebras Systems.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import csv
import glob
import json
import logging
import os
from collections import defaultdict, namedtuple

from cerebras.modelzoo.data_preparation.nlp.bert.bertsum_data_processor import (
    BertData,
    RougeBasedLabelsFormatter,
)

logging.basicConfig(level=logging.INFO)


BertInputFeatures = namedtuple(
    "BertInputFeatures", ["input_token_ids", "labels", "segment_ids", "cls_ids"]
)


[docs]class BertCSVFormatter: def __init__(self, params): """ Converts input into bert format, sets extractive summarization targets based on the rouge score between references and input sentences. :param params: dict params: BertData configuration parameters. """ self.bert_data = BertData(params) self.labels_formatter = RougeBasedLabelsFormatter() self.max_sequence_length = params.max_sequence_length self.max_cls_tokens = params.max_cls_tokens self.input_path = os.path.abspath(params.input_path) self.output_path = os.path.abspath(params.output_path) if not os.path.exists(self.output_path): os.makedirs(self.output_path) def _json_to_csv(self, json_input_file, csv_output_file, meta_data): with open(json_input_file, "r") as fin, open( csv_output_file, "w", newline="" ) as fout: csv_writer = csv.DictWriter( fout, fieldnames=BertInputFeatures._fields, quoting=csv.QUOTE_MINIMAL, ) csv_writer.writeheader() logging.info( f"Converting {json_input_file} to CSV and saving in {csv_output_file}" ) for i, data in enumerate(json.load(fin)): source, target = data["src"], data["tgt"] # Get sentences which are present in the summarization. oracle_ids = self.labels_formatter.process(source, target, 3) # Convert input into bert tf format. bert_data = self.bert_data.process(source, target, oracle_ids) if not bert_data: logging.info( f"Skipping index: {i} in {json_input_file}. Source or " f"target field is empty." ) continue input_tokens, labels, segment_ids, cls_ids, _, _ = bert_data bert_features = BertInputFeatures( input_tokens, labels, segment_ids, cls_ids ) csv_writer.writerow(bert_features._asdict()) meta_data[os.path.basename(csv_output_file)] += 1 def process(self): logging.info( f"Preparing to convert to bert format {self.input_path} to " f"{self.output_path}." ) for corpus_type in ["valid", "test", "train"]: output_path = os.path.join(self.output_path, corpus_type) if not os.path.exists(output_path): os.makedirs(output_path) input_files = glob.iglob( os.path.join(self.input_path, f"{corpus_type}-*.json") ) meta_data = defaultdict(int) for input_file in input_files: output_file = os.path.join( output_path, os.path.basename(input_file).replace("json", "csv"), ) self._json_to_csv(input_file, output_file, meta_data) logging.info( f"Converted simplified JSON to CSV for {corpus_type} set. " f"Writing metadata file." ) meta_file = os.path.join(output_path, "meta.dat") with open(meta_file, "w") as fout: for output_file, num_lines in meta_data.items(): fout.write(f"{output_file} {num_lines}\n") logging.info( f"Done converting to CSV, files saved to {self.output_path}." )