trinity.buffer.reader.file_reader module

trinity.buffer.reader.file_reader module#

Filed based buffer reader.

class trinity.buffer.reader.file_reader.DummyProgressBar[源代码]#

基类：object

__init__()[源代码]#

update(num: int)[源代码]#

close()[源代码]#

class trinity.buffer.reader.file_reader.BaseFileReader[源代码]#

基类：BufferReader

async read_async(batch_size: int | None = None, **kwargs)[源代码]#: Read from buffer asynchronously.

class trinity.buffer.reader.file_reader.FileReader(config: StorageConfig)[源代码]#

基类：BaseFileReader

Provide a unified interface for Experience and Task file readers.

__init__(config: StorageConfig)[源代码]#

read(batch_size: int | None = None, **kwargs) → List[源代码]#: Read from buffer.

read_with_indices(indices: List[int]) → List[源代码]#: Read tasks with indices.

async read_with_indices_async(indices: List[int]) → List[源代码]#: Read tasks with indices asynchronously.

state_dict()[源代码]#: Return the state of the reader as a dict. :returns: A dict containing the reader state. At minimum, it should contain

the current_index field.

load_state_dict(state_dict)[源代码]#

class trinity.buffer.reader.file_reader.ExperienceFileReader(config: StorageConfig)[源代码]#

基类：BaseFileReader

Reader for SFT / DPO file data.

__init__(config: StorageConfig)[源代码]#

read(batch_size: int | None = None, **kwargs) → List[源代码]#: Read from buffer.

state_dict()[源代码]#: Return the state of the reader as a dict. :returns: A dict containing the reader state. At minimum, it should contain

the current_index field.

load_state_dict(state_dict)[源代码]#

class trinity.buffer.reader.file_reader.TaskFileReader(config: StorageConfig)[源代码]#

基类：BaseFileReader

A Reader for task file data.

__init__(config: StorageConfig)[源代码]#

read(batch_size: int | None = None, **kwargs) → List[源代码]#: Read from buffer.

read_with_indices(indices: List[int]) → List[源代码]#: Read tasks with indices.

async read_with_indices_async(indices: List[int]) → List[源代码]#: Read tasks with indices asynchronously.

state_dict()[源代码]#: Return the state of the reader as a dict. :returns: A dict containing the reader state. At minimum, it should contain

the current_index field.

load_state_dict(state_dict)[源代码]#