Go to the documentation of this file.
8 #ifndef DMLC_RECORDIO_H_
9 #define DMLC_RECORDIO_H_
13 #include "./logging.h"
45 static const uint32_t
kMagic = 0xced7230a;
52 inline static uint32_t
EncodeLRec(uint32_t cflag, uint32_t length) {
53 return (cflag << 29U) | length;
61 return (rec >> 29U) & 7U;
69 return rec & ((1U << 29U) - 1U);
76 : stream_(stream), seek_stream_(dynamic_cast<
SeekStream*>(stream)),
78 CHECK(
sizeof(uint32_t) == 4) <<
"uint32_t needs to be 4 bytes";
98 return except_counter_;
103 CHECK(seek_stream_ != NULL) <<
"The input stream is not seekable";
104 return seek_stream_->
Tell();
113 size_t except_counter_;
126 : stream_(stream), seek_stream_(dynamic_cast<
SeekStream*>(stream)),
127 end_of_stream_(false) {
128 CHECK(
sizeof(uint32_t) == 4) <<
"uint32_t needs to be 4 bytes";
139 CHECK(seek_stream_ != NULL) <<
"The input stream is not seekable";
140 seek_stream_->
Seek(pos);
145 CHECK(seek_stream_ != NULL) <<
"The input stream is not seekable";
146 return seek_stream_->
Tell();
175 unsigned part_index = 0,
176 unsigned num_parts = 1);
192 char *pbegin_, *pend_;
196 #endif // DMLC_RECORDIO_H_
static const uint32_t kMagic
magic number of recordio note: (kMagic >> 29U) & 7 > 3 this ensures lrec will not be kMagic
Definition: recordio.h:45
size_t except_counter(void) const
Definition: recordio.h:97
size_t Tell(void)
tell the current position of the input stream
Definition: recordio.h:102
RecordIOReader(Stream *stream)
constructor
Definition: recordio.h:125
reader of binary recordio to reads in record from stream
Definition: recordio.h:119
namespace for dmlc
Definition: array_view.h:12
virtual size_t Tell(void)=0
tell the position of the stream
reader of binary recordio from Blob returned by InputSplit This class divides the blob into several i...
Definition: recordio.h:166
static uint32_t EncodeLRec(uint32_t cflag, uint32_t length)
encode the lrecord
Definition: recordio.h:52
void WriteRecord(const void *buf, size_t size)
write record to the stream
RecordIOWriter(Stream *stream)
constructor
Definition: recordio.h:75
interface of i/o stream that support seek
Definition: io.h:109
size_t Tell(void)
tell the current position of the input stream
Definition: recordio.h:144
defines serializable interface of dmlc
static uint32_t DecodeLength(uint32_t rec)
decode the length part of lrecord
Definition: recordio.h:68
void WriteRecord(const std::string &data)
write record to the stream
Definition: recordio.h:90
static uint32_t DecodeFlag(uint32_t rec)
decode the flag part of lrecord
Definition: recordio.h:60
writer of binary recordio binary format for recordio recordio format: magic lrecord data pad
Definition: recordio.h:38
bool NextRecord(std::string *out_rec)
read next complete record from stream
void Seek(size_t pos)
seek to certain position of the input stream
Definition: recordio.h:138
bool NextRecord(InputSplit::Blob *out_rec)
read next complete record from stream the blob contains the memory content NOTE: this function is not...
interface of stream I/O for serialization
Definition: io.h:30
RecordIOChunkReader(InputSplit::Blob chunk, unsigned part_index=0, unsigned num_parts=1)
constructor
virtual void Seek(size_t pos)=0
seek to certain position of the file