diff --git a/seqio/loggers.py b/seqio/loggers.py index 34a82730..2dc8f265 100644 --- a/seqio/loggers.py +++ b/seqio/loggers.py @@ -480,9 +480,11 @@ def __call__( ) field_names = ["target"] + inference_types + num_examples = 0 for example_index, (inp, *results) in enumerate(examples_with_results): if inp is None: continue + num_examples += 1 # tfds.as_numpy does not convert ragged tensors for k in inp: if isinstance(inp[k], tf.RaggedTensor): @@ -504,10 +506,14 @@ def __call__( json_str = json.dumps(json_dict, cls=self._json_encoder_cls) f.write(json_str + "\n") write_time = time.time() - write_tick + if write_time > 0: + examples_per_sec = num_examples / write_time + else: + examples_per_sec = float("inf") logging.info( - "Writing completed in %02f seconds (%02f examples/sec).", + "Writing completed in %.2f seconds (%.2f examples/sec).", write_time, - len(inferences) / write_time, + examples_per_sec, ) diff --git a/seqio/loggers_test.py b/seqio/loggers_test.py index a71b66f9..a9ac3c3f 100644 --- a/seqio/loggers_test.py +++ b/seqio/loggers_test.py @@ -22,6 +22,7 @@ import json import os from typing import Optional +from unittest import mock import numpy as np from seqio import loggers @@ -217,10 +218,10 @@ def test_log_generic(self): class JSONLoggerTest(tf.test.TestCase): - def _get_task_dataset_for_write_to_file_tests(self): + def _get_task_dataset_for_write_to_file_tests(self, num_examples: int = 2): x = [ - {"inputs_pretokenized": "i0", "targets_pretokenized": "t0"}, - {"inputs_pretokenized": "i1", "targets_pretokenized": "t1"}, + {"inputs_pretokenized": f"i{i}", "targets_pretokenized": f"t{i}"} + for i in range(num_examples) ] output_types = { "inputs_pretokenized": tf.string, @@ -740,6 +741,57 @@ def test_logging_metrics_only(self): tf.io.gfile.exists(os.path.join(tmp_dir, "test-000042.jsonl")) ) + def test_no_crash_when_write_time_is_zero(self): + inferences = {"prediction": ["pred0", "pred1"], "score": [0.2, 0.3]} + targets = ["target0", "target1"] + tmp_dir = self.create_tempdir().full_path + task_dataset = self._get_task_dataset_for_write_to_file_tests() + + logger = loggers.JSONLogger(tmp_dir) + with mock.patch("seqio.loggers.time.time", return_value=100.0): + logger( + task_name="test", + step=42, + metrics={"accuracy": metrics_lib.Scalar(100)}, + dataset=task_dataset, + inferences=inferences, + targets=targets, + ) + + # The inference file is still written without a ZeroDivisionError when the + # write duration rounds to zero. + self.assertTrue( + tf.io.gfile.exists(os.path.join(tmp_dir, "test-000042.jsonl")) + ) + + def test_logs_examples_per_second(self): + inferences = { + "prediction": ["pred0", "pred1", "pred2"], + "score": [0.2, 0.3, 0.4], + } + targets = ["target0", "target1", "target2"] + tmp_dir = self.create_tempdir().full_path + task_dataset = self._get_task_dataset_for_write_to_file_tests(3) + + logger = loggers.JSONLogger(tmp_dir) + with mock.patch( + "seqio.loggers.time.time", side_effect=[100.0, 101.0] + ), self.assertLogs(level="INFO") as log: + logger( + task_name="test", + step=42, + metrics={"accuracy": metrics_lib.Scalar(100)}, + dataset=task_dataset, + inferences=inferences, + targets=targets, + ) + + # The rate is computed from the number of examples written (3), not the + # number of inference types (2). + self.assertTrue( + any("3.00 examples/sec" in line for line in log.output) + ) + class TensorAndNumpyEncoderLoggerTest(tf.test.TestCase):