15 #include "./logging.h" 20 typedef unsigned __int64 uint64_t;
38 virtual size_t Read(
void *ptr,
size_t size) = 0;
44 virtual void Write(
const void *ptr,
size_t size) = 0;
58 const char*
const flag,
59 bool allow_null =
false);
74 inline void Write(
const T &data);
88 inline bool Read(T *out_data);
96 inline void WriteArray(
const T* data,
size_t num_elems);
105 inline bool ReadArray(T* data,
size_t num_elems);
114 virtual void Seek(
size_t pos) = 0;
116 virtual size_t Tell(
void) = 0;
127 static SeekStream *CreateForRead(
const char *uri,
128 bool allow_null =
false);
140 virtual void Load(
Stream *fi) = 0;
145 virtual void Save(
Stream *fo)
const = 0;
174 virtual size_t GetTotalSize(
void) = 0;
176 virtual void BeforeFirst(
void) = 0;
190 virtual bool NextRecord(
Blob *out_rec) = 0;
210 virtual bool NextChunk(
Blob *out_chunk) = 0;
231 return NextChunk(out_chunk);
242 virtual void ResetPartition(
unsigned part_index,
unsigned num_parts) = 0;
294 const char *index_uri,
298 const bool shuffle =
false,
300 const size_t batch_size = 256,
301 const bool recurse_directories =
false);
304 #ifndef _LIBCPP_SGX_NO_IOSTREAMS 318 class ostream :
public std::basic_ostream<char> {
326 size_t buffer_size = (1 << 10))
327 :
std::basic_ostream<char>(NULL), buf_(buffer_size) {
328 this->set_stream(stream);
339 buf_.set_stream(stream);
345 return buf_.bytes_out();
350 class OutBuf :
public std::streambuf {
352 explicit OutBuf(
size_t buffer_size)
353 : stream_(NULL), buffer_(buffer_size), bytes_out_(0) {
354 if (buffer_size == 0) buffer_.resize(2);
357 inline void set_stream(
Stream *stream);
359 inline size_t bytes_out()
const {
return bytes_out_; }
364 std::vector<char> buffer_;
368 inline int_type sync(
void);
370 inline int_type overflow(
int c);
389 class istream :
public std::basic_istream<char> {
397 size_t buffer_size = (1 << 10))
398 :
std::basic_istream<char>(NULL), buf_(buffer_size) {
399 this->set_stream(stream);
407 buf_.set_stream(stream);
412 return buf_.bytes_read();
417 class InBuf :
public std::streambuf {
419 explicit InBuf(
size_t buffer_size)
420 : stream_(NULL), bytes_read_(0),
421 buffer_(buffer_size) {
422 if (buffer_size == 0) buffer_.resize(2);
425 inline void set_stream(
Stream *stream);
427 inline size_t bytes_read(
void)
const {
436 std::vector<char> buffer_;
438 inline int_type underflow();
461 for (
size_t i = 0; i < num_elems; ++i) {
462 this->Write<T>(data[i]);
468 for (
size_t i = 0; i < num_elems; ++i) {
469 if (!this->Read<T>(data + i))
return false;
474 #ifndef _LIBCPP_SGX_NO_IOSTREAMS 476 inline void ostream::OutBuf::set_stream(
Stream *stream) {
477 if (stream_ != NULL) this->pubsync();
478 this->stream_ = stream;
479 this->setp(&buffer_[0], &buffer_[0] + buffer_.size() - 1);
481 inline int ostream::OutBuf::sync(
void) {
482 if (stream_ == NULL)
return -1;
483 std::ptrdiff_t n = pptr() - pbase();
484 stream_->Write(pbase(), n);
485 this->pbump(-static_cast<int>(n));
489 inline int ostream::OutBuf::overflow(
int c) {
491 std::ptrdiff_t n = pptr() - pbase();
492 this->pbump(-static_cast<int>(n));
494 stream_->Write(pbase(), n);
497 stream_->Write(pbase(), n + 1);
504 inline void istream::InBuf::set_stream(
Stream *stream) {
506 this->setg(&buffer_[0], &buffer_[0], &buffer_[0]);
508 inline int istream::InBuf::underflow() {
509 char *bhead = &buffer_[0];
510 if (this->gptr() == this->egptr()) {
511 size_t sz = stream_->Read(bhead, buffer_.size());
512 this->setg(bhead, bhead, bhead + sz);
515 if (this->gptr() == this->egptr()) {
516 return traits_type::eof();
518 return traits_type::to_int_type(*gptr());
539 explicit URI(
const char *uri) {
540 const char *p = std::strstr(uri,
"://");
544 protocol = std::string(uri, p - uri + 3);
546 p = std::strchr(uri,
'/');
548 host = uri; name =
'/';
550 host = std::string(uri, p - uri);
556 inline std::string
str(
void)
const {
557 return protocol + host + name;
605 virtual void ListDirectory(
const URI &path, std::vector<FileInfo> *out_list) = 0;
611 virtual void ListDirectoryRecursive(
const URI &path,
612 std::vector<FileInfo> *out_list);
621 const char*
const flag,
622 bool allow_null =
false) = 0;
630 bool allow_null =
false) = 0;
FileType type
the type of the file
Definition: io.h:576
virtual ~SeekStream(void)
Definition: io.h:112
URI(void)
enable default constructor
Definition: io.h:535
virtual ~FileSystem()
virtual destructor
Definition: io.h:593
virtual ~istream() DMLC_NO_EXCEPTION
Definition: io.h:401
interface for serializable objects
Definition: io.h:132
#define DMLC_THROW_EXCEPTION
Definition: base.h:224
virtual ~Serializable()
virtual destructor
Definition: io.h:135
URI(const char *uri)
construct from URI string
Definition: io.h:539
std::string str(void) const
string representation
Definition: io.h:556
FileInfo()
default constructor
Definition: io.h:578
#define DMLC_NO_EXCEPTION
Definition: base.h:225
void set_stream(Stream *stream)
set internal stream to be stream, reset states
Definition: io.h:406
serializer template class that helps serialization. This file do not need to be directly used by most...
static Stream * Create(const char *uri, const char *const flag, bool allow_null=false)
generic factory function create an stream, the stream will close the underlying files upon deletion ...
Definition: optional.h:241
virtual ~Stream(void)
virtual destructor
Definition: io.h:46
common data structure for URI
Definition: io.h:525
ostream(Stream *stream, size_t buffer_size=(1<< 10))
construct std::ostream type
Definition: io.h:325
interface of stream I/O for serialization
Definition: io.h:30
file system system interface
Definition: io.h:582
static void Write(Stream *strm, const T &data)
write data to stream
Definition: serializer.h:265
use to store file information
Definition: io.h:570
void WriteArray(const T *data, size_t num_elems)
Endian aware write array of data.
Definition: io.h:460
FileType
type of file
Definition: io.h:562
size_t size
the size of the file
Definition: io.h:574
std::string protocol
protocol
Definition: io.h:527
URI path
full path to the file
Definition: io.h:572
namespace for dmlc
Definition: array_view.h:12
a std::ostream class that can can wrap Stream objects, can use ostream with that output to underlying...
Definition: io.h:318
bool ReadArray(T *data, size_t num_elems)
Endian aware read array of data.
Definition: io.h:467
virtual size_t Read(void *ptr, size_t size)=0
reads data from a stream
virtual ~ostream() DMLC_NO_EXCEPTION
Definition: io.h:331
interface of i/o stream that support seek
Definition: io.h:109
virtual void Write(const void *ptr, size_t size)=0
writes data to a stream
the file is directory
Definition: io.h:566
size_t bytes_read(void) const
Definition: io.h:411
std::string host
host name, namenode for HDFS, bucket name for s3
Definition: io.h:531
std::string name
name of the path
Definition: io.h:533
the file is file
Definition: io.h:564
void set_stream(Stream *stream)
set internal stream to be stream, reset states
Definition: io.h:338
a std::istream class that can can wrap Stream objects, can use istream with that output to underlying...
Definition: io.h:389
istream(Stream *stream, size_t buffer_size=(1<< 10))
construct std::ostream type
Definition: io.h:396
static bool Read(Stream *strm, T *data)
read data to stream
Definition: serializer.h:283
size_t bytes_written(void) const
Definition: io.h:344