trinity.buffer.reader.file_reader module

trinity.buffer.reader.file_reader module#

Filed based buffer reader.

class trinity.buffer.reader.file_reader.DummyProgressBar[source]#

Bases: object

__init__()[source]#

update(num: int)[source]#

close()[source]#

class trinity.buffer.reader.file_reader.BaseFileReader[source]#

Bases: BufferReader

async read_async(batch_size: int | None = None)[source]#: Read from buffer asynchronously.

class trinity.buffer.reader.file_reader.FileReader(config: StorageConfig)[source]#

Bases: BaseFileReader

Provide a unified interface for Experience and Task file readers.

__init__(config: StorageConfig)[source]#

read(batch_size: int | None = None) → List[source]#: Read from buffer.

read_with_indices(indices: List[int]) → List[source]#: Read tasks with indices.

async read_with_indices_async(indices: List[int]) → List[source]#: Read tasks with indices asynchronously.

state_dict()[source]#: Return the state of the reader as a dict. :returns: A dict containing the reader state. At minimum, it should contain

the current_index field.

load_state_dict(state_dict)[source]#

class trinity.buffer.reader.file_reader.ExperienceFileReader(config: StorageConfig)[source]#

Bases: BaseFileReader

Reader for SFT / DPO file data.

__init__(config: StorageConfig)[source]#

read(batch_size: int | None = None) → List[source]#: Read from buffer.

state_dict()[source]#: Return the state of the reader as a dict. :returns: A dict containing the reader state. At minimum, it should contain

the current_index field.

load_state_dict(state_dict)[source]#

class trinity.buffer.reader.file_reader.TaskFileReader(config: StorageConfig)[source]#

Bases: BaseFileReader

A Reader for task file data.

__init__(config: StorageConfig)[source]#

read(batch_size: int | None = None) → List[source]#: Read from buffer.

read_with_indices(indices: List[int]) → List[source]#: Read tasks with indices.

async read_with_indices_async(indices: List[int]) → List[source]#: Read tasks with indices asynchronously.

state_dict()[source]#: Return the state of the reader as a dict. :returns: A dict containing the reader state. At minimum, it should contain

the current_index field.

load_state_dict(state_dict)[source]#