Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions seqio/loggers.py
Original file line number Diff line number Diff line change
Expand Up @@ -480,9 +480,11 @@ def __call__(
)
field_names = ["target"] + inference_types

num_examples = 0
for example_index, (inp, *results) in enumerate(examples_with_results):
if inp is None:
continue
num_examples += 1
# tfds.as_numpy does not convert ragged tensors
for k in inp:
if isinstance(inp[k], tf.RaggedTensor):
Expand All @@ -504,10 +506,14 @@ def __call__(
json_str = json.dumps(json_dict, cls=self._json_encoder_cls)
f.write(json_str + "\n")
write_time = time.time() - write_tick
if write_time > 0:
examples_per_sec = num_examples / write_time
else:
examples_per_sec = float("inf")
logging.info(
"Writing completed in %02f seconds (%02f examples/sec).",
"Writing completed in %.2f seconds (%.2f examples/sec).",
write_time,
len(inferences) / write_time,
examples_per_sec,
)


58 changes: 55 additions & 3 deletions seqio/loggers_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import json
import os
from typing import Optional
from unittest import mock

import numpy as np
from seqio import loggers
Expand Down Expand Up @@ -217,10 +218,10 @@ def test_log_generic(self):

class JSONLoggerTest(tf.test.TestCase):

def _get_task_dataset_for_write_to_file_tests(self):
def _get_task_dataset_for_write_to_file_tests(self, num_examples: int = 2):
x = [
{"inputs_pretokenized": "i0", "targets_pretokenized": "t0"},
{"inputs_pretokenized": "i1", "targets_pretokenized": "t1"},
{"inputs_pretokenized": f"i{i}", "targets_pretokenized": f"t{i}"}
for i in range(num_examples)
]
output_types = {
"inputs_pretokenized": tf.string,
Expand Down Expand Up @@ -740,6 +741,57 @@ def test_logging_metrics_only(self):
tf.io.gfile.exists(os.path.join(tmp_dir, "test-000042.jsonl"))
)

def test_no_crash_when_write_time_is_zero(self):
inferences = {"prediction": ["pred0", "pred1"], "score": [0.2, 0.3]}
targets = ["target0", "target1"]
tmp_dir = self.create_tempdir().full_path
task_dataset = self._get_task_dataset_for_write_to_file_tests()

logger = loggers.JSONLogger(tmp_dir)
with mock.patch("seqio.loggers.time.time", return_value=100.0):
logger(
task_name="test",
step=42,
metrics={"accuracy": metrics_lib.Scalar(100)},
dataset=task_dataset,
inferences=inferences,
targets=targets,
)

# The inference file is still written without a ZeroDivisionError when the
# write duration rounds to zero.
self.assertTrue(
tf.io.gfile.exists(os.path.join(tmp_dir, "test-000042.jsonl"))
)

def test_logs_examples_per_second(self):
inferences = {
"prediction": ["pred0", "pred1", "pred2"],
"score": [0.2, 0.3, 0.4],
}
targets = ["target0", "target1", "target2"]
tmp_dir = self.create_tempdir().full_path
task_dataset = self._get_task_dataset_for_write_to_file_tests(3)

logger = loggers.JSONLogger(tmp_dir)
with mock.patch(
"seqio.loggers.time.time", side_effect=[100.0, 101.0]
), self.assertLogs(level="INFO") as log:
logger(
task_name="test",
step=42,
metrics={"accuracy": metrics_lib.Scalar(100)},
dataset=task_dataset,
inferences=inferences,
targets=targets,
)

# The rate is computed from the number of examples written (3), not the
# number of inference types (2).
self.assertTrue(
any("3.00 examples/sec" in line for line in log.output)
)


class TensorAndNumpyEncoderLoggerTest(tf.test.TestCase):

Expand Down
Loading