mxnet
io.h
Go to the documentation of this file.
1 /*
2  * Licensed to the Apache Software Foundation (ASF) under one
3  * or more contributor license agreements. See the NOTICE file
4  * distributed with this work for additional information
5  * regarding copyright ownership. The ASF licenses this file
6  * to you under the Apache License, Version 2.0 (the
7  * "License"); you may not use this file except in compliance
8  * with the License. You may obtain a copy of the License at
9  *
10  * http://www.apache.org/licenses/LICENSE-2.0
11  *
12  * Unless required by applicable law or agreed to in writing,
13  * software distributed under the License is distributed on an
14  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15  * KIND, either express or implied. See the License for the
16  * specific language governing permissions and limitations
17  * under the License.
18  */
19 
24 #ifndef MXNET_IO_H_
25 #define MXNET_IO_H_
26 
27 #include <dmlc/data.h>
28 #include <dmlc/registry.h>
29 #include <vector>
30 #include <string>
31 #include <utility>
32 #include <queue>
33 #include "./base.h"
34 #include "./ndarray.h"
35 
36 namespace mxnet {
41 template<typename DType>
42 class IIterator : public dmlc::DataIter<DType> {
43  public:
48  virtual void Init(const std::vector<std::pair<std::string, std::string> >& kwargs) = 0;
50  virtual void BeforeFirst(void) = 0;
52  virtual bool Next(void) = 0;
54  virtual const DType &Value(void) const = 0;
56  virtual ~IIterator(void) {}
58  std::vector<std::string> data_names;
60  inline void SetDataName(const std::string data_name) {
61  data_names.push_back(data_name);
62  }
63 }; // class IIterator
64 
66 struct DataInst {
68  unsigned index;
70  std::vector<TBlob> data;
72  std::string extra_data;
73 }; // struct DataInst
74 
78 struct DataBatch {
80  std::vector<NDArray> data;
82  std::vector<uint64_t> index;
84  std::string extra_data;
87 }; // struct DataBatch
88 
90 typedef std::function<IIterator<DataBatch> *()> DataIteratorFactory;
95  : public dmlc::FunctionRegEntryBase<DataIteratorReg,
96  DataIteratorFactory> {
97 };
98 //--------------------------------------------------------------
99 // The following part are API Registration of Iterators
100 //--------------------------------------------------------------
113 #define MXNET_REGISTER_IO_ITER(name) \
114  DMLC_REGISTRY_REGISTER(::mxnet::DataIteratorReg, DataIteratorReg, name)
115 } // namespace mxnet
116 #endif // MXNET_IO_H_
std::vector< std::string > data_names
store the name of each data, it could be used for making NDArrays
Definition: io.h:58
void SetDataName(const std::string data_name)
set data name to each attribute of data
Definition: io.h:60
namespace of mxnet
Definition: base.h:126
std::string extra_data
extra data to be fed to the network
Definition: io.h:72
int num_batch_padd
num of example padded to batch
Definition: io.h:86
std::vector< uint64_t > index
index of image data
Definition: io.h:82
unsigned index
unique id for instance
Definition: io.h:68
virtual const DType & Value(void) const =0
get current data
std::string extra_data
extra data to be fed to the network
Definition: io.h:84
iterator type
Definition: io.h:42
Registry entry for DataIterator factory functions.
Definition: io.h:94
virtual void Init(const std::vector< std::pair< std::string, std::string > > &kwargs)=0
set the parameters and init iter
virtual ~IIterator(void)
constructor
Definition: io.h:56
virtual bool Next(void)=0
move to next item
DataBatch of NDArray, returned by Iterator.
Definition: io.h:78
a single data instance
Definition: io.h:66
std::vector< TBlob > data
content of data
Definition: io.h:70
virtual void BeforeFirst(void)=0
reset the iterator
std::vector< NDArray > data
content of dense data, if this DataBatch is dense
Definition: io.h:80
std::function< IIterator< DataBatch > *()> DataIteratorFactory
typedef the factory function of data iterator
Definition: io.h:90