Skip to content

Commit

Permalink
added design of zero-copy methods
Browse files Browse the repository at this point in the history
  • Loading branch information
@gsomix
gsomix committed Aug 16, 2012
1 parent 3ce4fd2 commit d3f9d75
Show file tree
Hide file tree
Showing 5 changed files with 254 additions and 37 deletions.
6 changes: 3 additions & 3 deletions ...features_dense_buffer_protocol_modular.py → ...dular/features_dense_protocols_modular.py
View file
Expand Up @@ -9,7 +9,7 @@

parameter_list = [[data]]

def features_dense_real_modular(in_data=data):
def features_dense_protocols_modular(in_data=data):
m_real=array(in_data, dtype=float64, order='F')
f_real=RealFeatures(m_real)

Expand Down Expand Up @@ -78,5 +78,5 @@ def features_dense_real_modular(in_data=data):
return f_real[:,0]

if __name__=='__main__':
print('dense_real')
features_dense_real_modular(*parameter_list[0])
print('dense_protocols')
features_dense_protocols_modular(*parameter_list[0])
39 changes: 39 additions & 0 deletions examples/undocumented/python_modular/features_dense_zero_copy_modular.py
View file
@@ -0,0 +1,39 @@
import numpy
from shogun.Features import RealFeatures
from numpy import array, float64, int64

# create dense matrice
data=[[1,2,3],[4,5,6],[7,8,9],[-1,-2,-3]]

parameter_list = [[data]]

def features_dense_zero_copy_modular(in_data=data):
if numpy.__version__ >= '1.5':
feats=numpy.array(in_data, dtype=float64, order='F')

a=RealFeatures()
a.frombuffer(feats, False)

b=numpy.array(a, copy=False)
c=numpy.array(a, copy=True)

d=RealFeatures()
d.frombuffer(a, False)

e=RealFeatures()
e.frombuffer(a, True)

a[:,0]=0
print a[0:4]
print b[0:4]
print c[0:4]
print d[0:4]
print e[0:4]
else:
print "numpy version >= 1.5 is needed"

return feats

if __name__=='__main__':
print('dense_zero_copy')
features_dense_zero_copy_modular(*parameter_list[0])
56 changes: 34 additions & 22 deletions src/interfaces/modular/Features.i
View file
Expand Up @@ -309,62 +309,74 @@ namespace shogun
namespace shogun
{
#ifdef USE_BOOL
PYPROTO_DENSEFEATURES(BoolFeatures, bool, "?\0", NPY_BOOL)
%template(BoolFeatures) CDenseFeatures<bool>;
// something wrong with git
PROTOCOLS_DENSEFEATURES(BoolFeatures, bool, "?\0", NPY_BOOL)
%template(BoolFeatures) CDenseFeatures<bool>;
EXTEND_DENSEFEATURES(BoolFeatures, bool, NPY_BOOL)
#endif

#ifdef USE_CHAR
PYPROTO_DENSEFEATURES(CharFeatures, char, "c\0", NPY_STRING)
%template(CharFeatures) CDenseFeatures<char>;
PROTOCOLS_DENSEFEATURES(CharFeatures, char, "c\0", NPY_STRING)
%template(CharFeatures) CDenseFeatures<char>;
EXTEND_DENSEFEATURES(CharFeatures, char, NPY_STRING)
#endif

#ifdef USE_UINT8
PYPROTO_DENSEFEATURES(ByteFeatures, uint8_t, "B\0", NPY_UINT8)
%template(ByteFeatures) CDenseFeatures<uint8_t>;
PROTOCOLS_DENSEFEATURES(ByteFeatures, uint8_t, "B\0", NPY_UINT8)
%template(ByteFeatures) CDenseFeatures<uint8_t>;
EXTEND_DENSEFEATURES(ByteFeatures, uint8_t, NPY_UINT8)
#endif

#ifdef USE_UINT16
PYPROTO_DENSEFEATURES(WordFeatures, uint16_t, "H\0", NPY_UINT16)
%template(WordFeatures) CDenseFeatures<uint16_t>;
PROTOCOLS_DENSEFEATURES(WordFeatures, uint16_t, "H\0", NPY_UINT16)
%template(WordFeatures) CDenseFeatures<uint16_t>;
EXTEND_DENSEFEATURES(WordFeatures, uint16_t, NPY_UINT16)
#endif

#ifdef USE_INT16
PYPROTO_DENSEFEATURES(ShortFeatures, int16_t, "h\0", NPY_INT16)
%template(ShortFeatures) CDenseFeatures<int16_t>;
PROTOCOLS_DENSEFEATURES(ShortFeatures, int16_t, "h\0", NPY_INT16)
%template(ShortFeatures) CDenseFeatures<int16_t>;
EXTEND_DENSEFEATURES(ShortFeatures, int16_t, NPY_INT16)
#endif

#ifdef USE_INT32
PYPROTO_DENSEFEATURES(IntFeatures, int32_t, "i\0", NPY_INT32)
%template(IntFeatures) CDenseFeatures<int32_t>;
PROTOCOLS_DENSEFEATURES(IntFeatures, int32_t, "i\0", NPY_INT32)
%template(IntFeatures) CDenseFeatures<int32_t>;
EXTEND_DENSEFEATURES(IntFeatures, int32_t, NPY_INT32)
#endif

#ifdef USE_UINT32
PYPROTO_DENSEFEATURES(UIntFeatures, uint32_t, "I\0", NPY_UINT32)
%template(UIntFeatures) CDenseFeatures<uint32_t>;
PROTOCOLS_DENSEFEATURES(UIntFeatures, uint32_t, "I\0", NPY_UINT32)
%template(UIntFeatures) CDenseFeatures<uint32_t>;
EXTEND_DENSEFEATURES(UIntFeatures, uint32_t, NPY_UINT32)
#endif

#ifdef USE_INT64
PYPROTO_DENSEFEATURES(LongIntFeatures, int64_t, "l\0", NPY_INT64)
%template(LongIntFeatures) CDenseFeatures<int64_t>;
PROTOCOLS_DENSEFEATURES(LongIntFeatures, int64_t, "l\0", NPY_INT64)
%template(LongIntFeatures) CDenseFeatures<int64_t>;
EXTEND_DENSEFEATURES(LongIntFeatures, int64_t, NPY_INT64)
#endif

#ifdef USE_UINT64
PYPROTO_DENSEFEATURES(ULongIntFeatures, uint64_t, "L\0", NPY_UINT64)
%template(ULongIntFeatures) CDenseFeatures<uint64_t>;
PROTOCOLS_DENSEFEATURES(ULongIntFeatures, uint64_t, "L\0", NPY_UINT64)
%template(ULongIntFeatures) CDenseFeatures<uint64_t>;
EXTEND_DENSEFEATURES(ULongIntFeatures, uint64_t, NPY_UINT64)
#endif

#ifdef USE_FLOATMAX
%template(LongRealFeatures) CDenseFeatures<floatmax_t>;
%template(LongRealFeatures) CDenseFeatures<floatmax_t>;
#endif

#ifdef USE_FLOAT32
PYPROTO_DENSEFEATURES(ShortRealFeatures, float32_t, "f\0", NPY_FLOAT32)
%template(ShortRealFeatures) CDenseFeatures<float32_t>;
PROTOCOLS_DENSEFEATURES(ShortRealFeatures, float32_t, "f\0", NPY_FLOAT32)
%template(ShortRealFeatures) CDenseFeatures<float32_t>;
EXTEND_DENSEFEATURES(ShortRealFeatures, float32_t, NPY_FLOAT32)
#endif

#ifdef USE_FLOAT64
PYPROTO_DENSEFEATURES(RealFeatures, float64_t, "d\0", NPY_FLOAT64)
PROTOCOLS_DENSEFEATURES(RealFeatures, float64_t, "d\0", NPY_FLOAT64)
%template(RealFeatures) CDenseFeatures<float64_t>;
EXTEND_DENSEFEATURES(RealFeatures, float64_t, NPY_FLOAT64)
#endif
}

Expand Down
161 changes: 158 additions & 3 deletions src/interfaces/python_modular/DenseFeatures_protocols.i
View file
Expand Up @@ -105,7 +105,7 @@ fail:
%enddef // NUMERIC_DENSEFEATURES

/* Python protocols for DenseFeatures */
%define PYPROTO_DENSEFEATURES(class_name, type_name, format_str, typecode)
%define PROTOCOLS_DENSEFEATURES(class_name, type_name, format_str, typecode)

%wrapper
%{
Expand Down Expand Up @@ -147,7 +147,8 @@ static int class_name ## _getbuffer(PyObject *self, Py_buffer *view, int flags)

arg1=reinterpret_cast< CDenseFeatures < type_name >* >(argp1);

info=new buffer_matrix_ ## type_name ## _info;
info=(buffer_matrix_ ## type_name ## _info*) malloc(sizeof(buffer_matrix_ ## type_name ## _info));
new (&info->buf) SGMatrix< type_name >();

info->buf=arg1->get_feature_matrix();
num_feat=arg1->get_num_features();
Expand All @@ -165,6 +166,7 @@ static int class_name ## _getbuffer(PyObject *self, Py_buffer *view, int flags)

info->shape=shape;
info->strides=strides;
info->internal=NULL;

view->ndim=2;

Expand Down Expand Up @@ -203,7 +205,7 @@ static void class_name ## _releasebuffer(PyObject *self, Py_buffer *view)
delete[] temp->strides;

temp->buf=SGMatrix< type_name >();
delete temp;
free(temp);
}
}

Expand Down Expand Up @@ -567,6 +569,67 @@ fail:
return -1;
}

static PyObject* class_name ## _free_feature_matrix(PyObject *self, PyObject *args)
{
PyObject* resultobj=0;
CDenseFeatures< type_name > *arg1=(CDenseFeatures< type_name > *) 0;
void* argp1=0;
int res1=0;

res1=SWIG_ConvertPtr(self, &argp1, SWIG_TypeQuery("shogun::CDenseFeatures<type_name>"), 0 | 0 );
if (!SWIG_IsOK(res1))
{
SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "BoolFeatures_free_feature_matrix" "', argument " "1"" of type '" "shogun::CDenseFeatures< bool > *""'");
}

arg1=reinterpret_cast< CDenseFeatures< type_name > * >(argp1);
{
try
{
Py_buffer* view=NULL;
buffer_matrix_ ## type_name ## _info* temp=NULL;
if (extend_ ## class_name ## _info.count(arg1)>0)
{
view=extend_ ## class_name ## _info[arg1];
temp=(buffer_matrix_ ## type_name ## _info*) view->internal;
view->internal=temp->internal;

PyBuffer_Release(view);
extend_ ## class_name ## _info.erase(arg1);

free(temp);
delete view;
}

(arg1)->free_feature_matrix();
}

catch (Swig::DirectorException &e)
{
SWIG_fail;
}

catch (std::bad_alloc)
{
SWIG_exception(SWIG_MemoryError, const_cast<char*>("Out of memory error.\n"));

SWIG_fail;

}
catch (shogun::ShogunException e)
{
SWIG_exception(SWIG_SystemError, const_cast<char*>(e.get_exception_string()));

SWIG_fail;

}
}
resultobj=SWIG_Py_Void();
return resultobj;
fail:
return NULL;
}

NUMERIC_DENSEFEATURES(class_name, type_name, format_str, add, +)
NUMERIC_DENSEFEATURES(class_name, type_name, format_str, sub, -)
NUMERIC_DENSEFEATURES(class_name, type_name, format_str, mul, *)
Expand All @@ -576,7 +639,20 @@ static long class_name ## _flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_NEWBUFFE

%init
%{
// overload flags slot in DenseFatures proxy class
SwigPyBuiltin__shogun__CDenseFeaturesT_ ## type_name ## _t_type.ht_type.tp_flags = class_name ## _flags;

// overload free_feature_matrix in DenseFeatures
set_method(SwigPyBuiltin__shogun__CDenseFeaturesT_ ## type_name ## _t_methods, "free_feature_matrix", (PyCFunction) class_name ## _free_feature_matrix);

%}

%header
%{

#include <map>
static std::map<CDenseFeatures< type_name >*, Py_buffer*> extend_ ## class_name ## _info;

%}

%feature("python:bf_getbuffer") CDenseFeatures< type_name > #class_name "_getbuffer"
Expand All @@ -595,4 +671,83 @@ SwigPyBuiltin__shogun__CDenseFeaturesT_ ## type_name ## _t_type.ht_type.tp_flags
%feature("python:mp_ass_subscript") CDenseFeatures< type_name > #class_name "_setsubscript"

%enddef /* PYPROTO_DENSEFEATURES */

%define EXTEND_DENSEFEATURES(class_name, type_name, typecode)

%extend CDenseFeatures< type_name >
{

int frombuffer(PyObject* exporter, bool copy)
{
Py_buffer* view=NULL;
buffer_matrix_ ## type_name ## _info* info=NULL;
SGMatrix< type_name > new_feat_matrix;

int res1=0;
int res2=0;

res1=PyObject_CheckBuffer(exporter);
if (!res1)
{
PyErr_SetString(PyExc_BufferError, "this object don't support buffer protocol");
return -1;
}

view=new Py_buffer;
res2=PyObject_GetBuffer(exporter, view, PyBUF_F_CONTIGUOUS | PyBUF_ND | PyBUF_STRIDES | 0);
if (res2!=0 || view->buf==NULL)
{
PyErr_SetString(PyExc_BufferError, "bad buffer");
return -1;
}

// checking that buffer is right
if (view->ndim!=2)
{
PyErr_SetString(PyExc_BufferError, "wrond dimensional");
return -1;
}

if (view->itemsize!=sizeof(type_name))
{
PyErr_SetString(PyExc_BufferError, "wrong type");
return -1;
}

if (view->shape==NULL)
{
PyErr_SetString(PyExc_BufferError, "wrong shape");
return -1;
}

new_feat_matrix=SGMatrix< type_name >((type_name*) view->buf, view->shape[0], view->shape[1], true);

if (copy)
{
$self->set_feature_matrix(new_feat_matrix.clone());
}
else
{
$self->set_feature_matrix(new_feat_matrix);
}

info=(buffer_matrix_ ## type_name ## _info*) malloc(sizeof(buffer_matrix_ ## type_name ## _info));
new (&info->buf) SGMatrix< type_name >();

info->buf=new_feat_matrix;
info->shape=view->shape;
info->strides=view->strides;
info->internal=view->internal;

view->internal=info;

extend_ ## class_name ## _info[$self]=view;

return 0;
}

}

%enddef /* EXTEND_DENSEFEATURES */

#endif /* SWIG_PYTHON */

0 comments on commit d3f9d75

Please sign in to comment.