Source code for data_juicer.format.json_formatter
from .formatter import FORMATTERS, LocalFormatter
[docs]@FORMATTERS.register_module()
class JsonFormatter(LocalFormatter):
"""
The class is used to load and format json-type files.
Default suffixes is `['.json', '.jsonl', '.jsonl.zst']`
"""
SUFFIXES = ['.json', '.jsonl', '.jsonl.zst']
[docs] def __init__(self, dataset_path, suffixes=None, **kwargs):
"""
Initialization method.
:param dataset_path: a dataset file or a dataset directory
:param suffixes: files with specified suffixes to be processed
:param kwargs: extra args
"""
super().__init__(
dataset_path=dataset_path,
suffixes=suffixes if suffixes else self.SUFFIXES,
type='json',
**kwargs,
)