data.h

Go to the documentation of this file.

#ifndef DMLC_DATA_H_
#define DMLC_DATA_H_

#include <string>
#include <vector>
#include <map>
#include "./base.h"
#include "./io.h"
#include "./logging.h"
#include "./registry.h"

// To help C Preprocessor with processing c++ templated types
#define __DMLC_COMMA ,

namespace dmlc {
typedef float real_t;

typedef unsigned index_t;

// This file describes common data structure that can be used
// for large-scale machine learning, this may not be a complete list
// But we will keep the most common and useful ones, and keep adding new ones
template<typename DType>
class DataIter {
 public:
  virtual ~DataIter(void) DMLC_THROW_EXCEPTION {}
  virtual void BeforeFirst(void) = 0;
  virtual bool Next(void) = 0;
  virtual const DType &Value(void) const = 0;
};

template<typename IndexType, typename DType = real_t>
class Row {
 public:
  const DType *label;
  const real_t *weight;
  const uint64_t *qid;
  size_t length;
  const IndexType *field;
  const IndexType *index;
  const DType *value;
  inline IndexType get_field(size_t i) const {
    return field[i];
  }
  inline IndexType get_index(size_t i) const {
    return index[i];
  }
  inline DType get_value(size_t i) const {
    return value == NULL ? DType(1.0f) : value[i];
  }
  inline DType get_label() const {
    return *label;
  }
  inline real_t get_weight() const {
    return weight == NULL ? 1.0f : *weight;
  }
  inline uint64_t get_qid() const {
    return qid == NULL ? 0 : *qid;
  }
  template<typename V>
  inline V SDot(const V *weight, size_t size) const {
    V sum = static_cast<V>(0);
    if (value == NULL) {
      for (size_t i = 0; i < length; ++i) {
        CHECK(index[i] < size) << "feature index exceed bound";
        sum += weight[index[i]];
      }
    } else {
      for (size_t i = 0; i < length; ++i) {
        CHECK(index[i] < size) << "feature index exceed bound";
        sum += weight[index[i]] * value[i];
      }
    }
    return sum;
  }
};

template<typename IndexType, typename DType = real_t>
struct RowBlock {
  size_t size;
  const size_t *offset;
  const DType *label;
  const real_t *weight;
  const uint64_t *qid;
  const IndexType *field;
  const IndexType *index;
  const DType *value;
  inline Row<IndexType, DType> operator[](size_t rowid) const;
  inline size_t MemCostBytes(void) const {
    size_t cost = size * (sizeof(size_t) + sizeof(DType));
    if (weight != NULL) cost += size * sizeof(real_t);
    if (qid != NULL) cost += size * sizeof(size_t);
    size_t ndata = offset[size] - offset[0];
    if (field != NULL) cost += ndata * sizeof(IndexType);
    if (index != NULL) cost += ndata * sizeof(IndexType);
    if (value != NULL) cost += ndata * sizeof(DType);
    return cost;
  }
  inline RowBlock Slice(size_t begin, size_t end) const {
    CHECK(begin <= end && end <= size);
    RowBlock ret;
    ret.size = end - begin;
    ret.label = label + begin;
    if (weight != NULL) {
      ret.weight = weight + begin;
    } else {
      ret.weight = NULL;
    }
    if (qid != NULL) {
      ret.qid = qid + begin;
    } else {
      ret.qid = NULL;
    }
    ret.offset = offset + begin;
    ret.field = field;
    ret.index = index;
    ret.value = value;
    return ret;
  }
};

template<typename IndexType, typename DType = real_t>
class RowBlockIter : public DataIter<RowBlock<IndexType, DType> > {
 public:
  static RowBlockIter<IndexType, DType> *
  Create(const char *uri,
         unsigned part_index,
         unsigned num_parts,
         const char *type);
  virtual size_t NumCol() const = 0;
};

template <typename IndexType, typename DType = real_t>
class Parser : public DataIter<RowBlock<IndexType, DType> > {
 public:
  static Parser<IndexType, DType> *
  Create(const char *uri_,
         unsigned part_index,
         unsigned num_parts,
         const char *type);
  virtual size_t BytesRead(void) const = 0;
  typedef Parser<IndexType, DType>* (*Factory)
      (const std::string& path,
       const std::map<std::string, std::string>& args,
       unsigned part_index,
       unsigned num_parts);
};

template<typename IndexType, typename DType = real_t>
struct ParserFactoryReg
    : public FunctionRegEntryBase<ParserFactoryReg<IndexType, DType>,
                                  typename Parser<IndexType, DType>::Factory> {};

#define DMLC_REGISTER_DATA_PARSER(IndexType, DataType, TypeName, FactoryFunction) \
  DMLC_REGISTRY_REGISTER(ParserFactoryReg<IndexType __DMLC_COMMA DataType>,           \
                         ParserFactoryReg ## _ ## IndexType ## _ ## DataType, TypeName)  \
  .set_body(FactoryFunction)


// implementation of operator[]
template<typename IndexType, typename DType>
inline Row<IndexType, DType>
RowBlock<IndexType, DType>::operator[](size_t rowid) const {
  CHECK(rowid < size);
  Row<IndexType, DType> inst;
  inst.label = label + rowid;
  if (weight != NULL) {
    inst.weight = weight + rowid;
  } else {
    inst.weight = NULL;
  }
  if (qid != NULL) {
    inst.qid = qid + rowid;
  } else {
    inst.qid = NULL;
  }
  inst.length = offset[rowid + 1] - offset[rowid];
  if (field != NULL) {
    inst.field = field + offset[rowid];
  } else {
    inst.field = NULL;
  }
  inst.index = index + offset[rowid];
  if (value == NULL) {
    inst.value = NULL;
  } else {
    inst.value = value + offset[rowid];
  }
  return inst;
}

}  // namespace dmlc
#endif  // DMLC_DATA_H_