trinity.buffer.reader.file_reader module
Filed based buffer reader.
-
class trinity.buffer.reader.file_reader.DummyProgressBar[source]
Bases: object
-
__init__()[source]
-
update(num: int)[source]
-
close()[source]
-
class trinity.buffer.reader.file_reader.BaseFileReader[source]
Bases: BufferReader
-
async read_async(batch_size: int | None = None)[source]
Read from buffer asynchronously.
-
class trinity.buffer.reader.file_reader.ExperienceFileReader(config: StorageConfig)[source]
Bases: BaseFileReader
Reader for SFT / DPO file data.
-
__init__(config: StorageConfig)[source]
-
read(batch_size: int | None = None) → List[source]
Read from buffer.
-
class trinity.buffer.reader.file_reader.TaskFileReader(config: StorageConfig)[source]
Bases: BaseFileReader
A Reader for task file data.
-
__init__(config: StorageConfig)[source]
-
read(batch_size: int | None = None) → List[source]
Read from buffer.
-
read_with_indices(indices: List[int]) → List[source]
Read tasks with indices.
-
async read_with_indices_async(indices: List[int]) → List[source]
Read tasks with indices asynchronously.
