Source code for data_juicer.core.executor

import os
from time import time

from loguru import logger

from data_juicer.config import init_configs
from data_juicer.core.data import Dataset
from data_juicer.format.load import load_formatter
from data_juicer.format.mixture_formatter import MixtureFormatter
from data_juicer.ops import OPERATORS, load_ops
from data_juicer.utils import cache_utils
from data_juicer.utils.ckpt_utils import CheckpointManager

from ..ops.selector.frequency_specified_field_selector import \
    FrequencySpecifiedFieldSelector
from ..ops.selector.topk_specified_field_selector import \
    TopkSpecifiedFieldSelector
from .exporter import Exporter
from .tracer import Tracer


[docs]class Executor: """ This Executor class is used to process a specific dataset. It will load the dataset and unify the format, then apply all the ops in the config file in order and generate a processed dataset. """
[docs] def __init__(self, cfg=None): """ Initialization method. :param cfg: optional config dict. """ self.cfg = init_configs() if cfg is None else cfg self.work_dir = self.cfg.work_dir self.tracer = None self.ckpt_manager = None # only enable it when using cache if self.cfg.use_cache: logger.info(f'Using cache compression method: ' f'[{self.cfg.cache_compress}]') cache_utils.CACHE_COMPRESS = self.cfg.cache_compress # setup formatter logger.info('Setting up data formatter...') self.formatter = load_formatter( dataset_path=self.cfg.dataset_path, generated_dataset_config=self.cfg.generated_dataset_config, text_keys=self.cfg.text_keys, suffixes=self.cfg.suffixes, add_suffix=self.cfg.add_suffix) # whether to use checkpoint mechanism. If it's true, Executor will # check if there are existing checkpoints first and try to load the # checkpoints. If the checkpoints are loaded successfully, ops that # have been processed will be skipped. if self.cfg.use_checkpoint: logger.info('Preparing checkpoint manager...') self.ckpt_dir = os.path.join(self.work_dir, 'ckpt') self.ckpt_manager = CheckpointManager(self.ckpt_dir, self.cfg.process, self.cfg.np) if self.ckpt_manager.ckpt_available: logger.info('Found existed dataset checkpoint.') self.cfg.process = self.ckpt_manager.get_left_process_list() # prepare exporter and check export path suffix logger.info('Preparing exporter...') self.exporter = Exporter( self.cfg.export_path, self.cfg.export_shard_size, self.cfg.export_in_parallel, self.cfg.np, keep_stats_in_res_ds=self.cfg.keep_stats_in_res_ds, keep_hashes_in_res_ds=self.cfg.keep_hashes_in_res_ds) # setup tracer self.open_tracer = self.cfg.open_tracer if self.open_tracer: logger.info('Preparing tracer...') self.tracer = Tracer(self.work_dir, show_num=self.cfg.trace_num) self.op_list_to_trace = self.cfg.op_list_to_trace if len(self.cfg.op_list_to_trace) == 0: logger.info('Trace for all ops.') self.op_list_to_trace = set(OPERATORS.modules.keys())
[docs] def sample_data(self, dataset_to_sample: Dataset = None, load_data_np=None, sample_ratio: float = 1.0, sample_algo: str = 'uniform', **kwargs): """ Sample a subset from the given dataset. :param dataset_to_sample: Dataset to sample from. If None, will use the formatter linked by the executor. Default is None. :param load_data_np: number of workers when loading the dataset. :param sample_ratio: The ratio of the sample size to the original dataset size. Default is 1.0 (no sampling). :param sample_algo: Sampling algorithm to use. Options are "uniform", "frequency_specified_field_selector", or "topk_specified_field_selector". Default is "uniform". :return: A sampled Dataset. """ # Determine the dataset to sample from if dataset_to_sample is not None: dataset = dataset_to_sample elif self.cfg.use_checkpoint and self.ckpt_manager.ckpt_available: logger.info('Loading dataset from checkpoint...') dataset = self.ckpt_manager.load_ckpt() elif hasattr(self, 'formatter'): logger.info('Loading dataset from data formatter...') if load_data_np is None: load_data_np = self.cfg.np dataset = self.formatter.load_dataset(load_data_np, self.cfg) else: raise ValueError('No dataset available to sample from.') # Perform sampling based on the specified algorithm if sample_algo == 'uniform': return MixtureFormatter.random_sample(dataset, sample_ratio) elif sample_algo == 'frequency_specified_field_selector': dj_op = FrequencySpecifiedFieldSelector(**kwargs) return dj_op.process(dataset) elif sample_algo == 'topk_specified_field_selector': dj_op = TopkSpecifiedFieldSelector(**kwargs) return dj_op.process(dataset) else: raise ValueError(f'Unsupported sample_algo: {sample_algo}')
[docs] def run(self, load_data_np=None): """ Running the dataset process pipeline. :param load_data_np: number of workers when loading the dataset. :return: processed dataset. """ # 1. format data if self.cfg.use_checkpoint and self.ckpt_manager.ckpt_available: logger.info('Loading dataset from checkpoint...') dataset = self.ckpt_manager.load_ckpt() else: logger.info('Loading dataset from data formatter...') if load_data_np is None: load_data_np = self.cfg.np dataset = self.formatter.load_dataset(load_data_np, self.cfg) # 2. extract processes logger.info('Preparing process operators...') ops = load_ops(self.cfg.process, self.cfg.op_fusion) # 3. data process # - If tracer is open, trace each op after it's processed # - If checkpoint is open, clean the cache files after each process logger.info('Processing data...') tstart = time() dataset = dataset.process(ops, work_dir=self.work_dir, exporter=self.exporter, checkpointer=self.ckpt_manager, tracer=self.tracer) tend = time() logger.info(f'All OPs are done in {tend - tstart:.3f}s.') # 4. data export logger.info('Exporting dataset to disk...') self.exporter.export(dataset) # compress the last dataset after exporting if self.cfg.use_cache and self.cfg.cache_compress: from data_juicer.utils.compress import compress compress(dataset) return dataset