removed the restriction n_max_matches put on memory allocation

Author: ParticularMiner

setup.py

@@ -29,17 +29,28 @@ def finalize_options(self):
 else:
     extra_compile_args = ['-std=c++0x', '-pthread', '-O3']
 
+array_wrappers_ext = Extension('sparse_dot_topn.array_wrappers',
+                         sources=[
+                                    './sparse_dot_topn/array_wrappers.pyx',
+                                    './sparse_dot_topn/sparse_dot_topn_source.cpp'
+                                ],
+                         extra_compile_args=extra_compile_args,
+                         language='c++')
+
 original_ext = Extension('sparse_dot_topn.sparse_dot_topn',
-                         sources=['./sparse_dot_topn/sparse_dot_topn.pyx',
-                                  './sparse_dot_topn/sparse_dot_topn_source.cpp'],
+                         sources=[
+                                    './sparse_dot_topn/sparse_dot_topn.pyx',
+                                    './sparse_dot_topn/sparse_dot_topn_source.cpp'
+                                ],
                          extra_compile_args=extra_compile_args,
                          language='c++')
 
 threaded_ext = Extension('sparse_dot_topn.sparse_dot_topn_threaded',
                          sources=[
                              './sparse_dot_topn/sparse_dot_topn_threaded.pyx',
                              './sparse_dot_topn/sparse_dot_topn_source.cpp',
-                             './sparse_dot_topn/sparse_dot_topn_parallel.cpp'],
+                             './sparse_dot_topn/sparse_dot_topn_parallel.cpp'
+                            ],
                          extra_compile_args=extra_compile_args,
                          language='c++')
 
@@ -77,5 +88,5 @@ def finalize_options(self):
                       , 'pandas>=0.25.3'
     ],
     cmdclass={'build_ext': my_build_ext},
-    ext_modules=[original_ext, threaded_ext]
+    ext_modules=[array_wrappers_ext, original_ext, threaded_ext]
 )

sparse_dot_topn/__init__.py

@@ -1,2 +1,7 @@
 # flake8: noqa
-from sparse_dot_topn.awesome_cossim_topn import awesome_cossim_topn, awesome_cossim_true_minmax_topn_only
+import sys
+
+if sys.version_info[0] >= 3:
+    from sparse_dot_topn.awesome_cossim_topn import awesome_cossim_topn
+else:
+    from awesome_cossim_topn import awesome_cossim_topn

sparse_dot_topn/array_wrappers.pxd

@@ -0,0 +1,18 @@
+from libcpp.vector cimport vector
+
+# define a Cython array wrapper class to hold a C++ vector of ints, adhering to numpy's buffer protocol:
+cdef class ArrayWrapper_int:
+    cdef int view_count
+    cdef vector[int] vec
+    cdef Py_ssize_t shape[2]
+    cdef Py_ssize_t strides[2]
+
+
+# define a Cython array wrapper class to hold a C++ vector of doubles, adhering to numpy's buffer protocol:
+cdef class ArrayWrapper_double:
+    cdef int view_count
+    cdef vector[double] vec
+    cdef Py_ssize_t shape[2]
+    cdef Py_ssize_t strides[2]
+
+

sparse_dot_topn/array_wrappers.pyx

@@ -0,0 +1,73 @@
+from cpython cimport Py_buffer
+from libcpp.vector cimport vector
+
+# define a Cython array wrapper class to hold a C++ vector of ints, adhering to numpy's buffer protocol:
+cdef class ArrayWrapper_int:
+    # constructor and destructor are fairly unimportant now since
+    # vec will be destroyed automatically.
+
+    def __cinit__(self, vector[int]& data):
+        self.vec.swap(data)
+        self.view_count = 0
+
+    # now implement the buffer protocol for the class
+    # which makes it generally useful to anything that expects an array
+    def __getbuffer__(self, Py_buffer *buffer, int flags):
+        # relevant documentation http://cython.readthedocs.io/en/latest/src/userguide/buffer.html#a-matrix-class
+        cdef Py_ssize_t itemsize = sizeof(self.vec[0])
+
+        self.shape[1] = self.vec.size()
+        self.shape[0] = 1
+        self.strides[1] = <Py_ssize_t>(  <char *>&(self.vec[1]) - <char *>&(self.vec[0]))
+        self.strides[0] = self.vec.size() * self.strides[1]
+        buffer.buf = <char *>&(self.vec[0])
+        buffer.format = 'i'
+        buffer.internal = NULL
+        buffer.itemsize = itemsize
+        buffer.len = self.vec.size() * itemsize   # product(shape) * itemsize
+        buffer.ndim = 2
+        buffer.obj = self
+        buffer.readonly = 0
+        buffer.shape = self.shape
+        buffer.strides = self.strides
+        buffer.suboffsets = NULL
+        self.view_count += 1
+        
+    def __releasebuffer__(self, Py_buffer *buffer):
+        self.view_count -= 1
+
+
+# define a Cython array wrapper class to hold a C++ vector of doubles, adhering to numpy's buffer protocol:
+cdef class ArrayWrapper_double:
+    # constructor and destructor are fairly unimportant now since
+    # vec will be destroyed automatically.
+
+    def __cinit__(self, vector[double]& data):
+        self.vec.swap(data)
+        self.view_count = 0
+
+    # now implement the buffer protocol for the class
+    # which makes it generally useful to anything that expects an array
+    def __getbuffer__(self, Py_buffer *buffer, int flags):
+        # relevant documentation http://cython.readthedocs.io/en/latest/src/userguide/buffer.html#a-matrix-class
+        cdef Py_ssize_t itemsize = sizeof(self.vec[0])
+
+        self.shape[1] = self.vec.size()
+        self.shape[0] = 1
+        self.strides[1] = <Py_ssize_t>(  <char *>&(self.vec[1]) - <char *>&(self.vec[0]))
+        self.strides[0] = self.vec.size() * self.strides[1]
+        buffer.buf = <char *>&(self.vec[0])
+        buffer.format = 'd'
+        buffer.internal = NULL
+        buffer.itemsize = itemsize
+        buffer.len = self.vec.size() * itemsize   # product(shape) * itemsize
+        buffer.ndim = 2
+        buffer.obj = self
+        buffer.readonly = 0
+        buffer.shape = self.shape
+        buffer.strides = self.strides
+        buffer.suboffsets = NULL
+        self.view_count += 1
+        
+    def __releasebuffer__(self, Py_buffer *buffer):
+        self.view_count -= 1