I am writing an extension module for my Python script and cannot figure out, where this heap corruption is coming from.
The use of the extension module should be to create a Numpy array, fill it with some values that are calculated given a few conditions and return the Numpy array to Python.
There is a Python function (in the example below get_data()
), which wraps the extension function and receives the Numpy array from it.
The error occurs, when the Python function returns.
The error does not occur, when I use "OOO"
instead of "NNN"
as format string for Py_BuildValue()
.
Hence, I assume this to be a problem about reference counting. Is this problem familiar to anyone if you look into the example code below?
Started to use WinDbg, but total beginner with that.
Note: Unfortunately cannot share the original code, which makes this a little harder of course. The example code as below did not throw any errors for me, however it uses the same concepts and maybe the problem lies therein.
module.cpp
#define PY_SSIZE_T_CLEAN
#define NPY_NO_DEPRECATED_API NPY_API_VERSION
#include <Python.h>
#include <numpy\arrayobject.h>
// Helper for making some exemplary changes on data array
void applySomeChangesOnData(double* data, double factor, size_t nRows, size_t nColumns) {
for (int m = 0; m < nRows; m++) {
for (int n = 0; n < nColumns; n++) {
data[m*nColumns+n] = factor * (m + n);
}
}
}
PyObject* createArrays(PyObject *self, PyObject *args) {
// Variables needed for parsing inputs
PyObject* tupleShape;
// Parse input
if (!PyArg_ParseTuple(args, "O!", &PyTuple_Type, &tupleShape)) {
// Error when parsing
PyErr_SetString(PyExc_TypeError, "Bad input type(s)");
Py_RETURN_NONE;
}
// Allow only 2D arrays for this example
if ((size_t)PyTuple_Size(tupleShape) != 2) {
PyErr_SetString(PyExc_ValueError, "Array must be 2D");
Py_RETURN_NONE;
}
// Allocate data array
size_t nRows = PyLong_AsLong(PyTuple_GetItem(tupleShape, 0));
size_t nColumns = PyLong_AsLong(PyTuple_GetItem(tupleShape, 1));
size_t nElements = nRows * nColumns;
double* data1 = PyMem_New(double, nElements);
double* data2 = PyMem_New(double, nElements);
double* data3 = PyMem_New(double, nElements);
// Modify data array
applySomeChangesOnData(data1, 1, nRows, nColumns);
applySomeChangesOnData(data2, 10, nRows, nColumns);
applySomeChangesOnData(data3, 100, nRows, nColumns);
// Prepare shape information
size_t nDims = 1;
npy_intp* shape = new npy_intp[nDims];
shape[0] = nElements;
// Create output array and set OWNDATA flag for proper deallocation
PyArrayObject* arr1 = reinterpret_cast<PyArrayObject*>(PyArray_SimpleNewFromData(nDims, shape, NPY_DOUBLE, data1));
PyArrayObject* arr2 = reinterpret_cast<PyArrayObject*>(PyArray_SimpleNewFromData(nDims, shape, NPY_DOUBLE, data2));
PyArrayObject* arr3 = reinterpret_cast<PyArrayObject*>(PyArray_SimpleNewFromData(nDims, shape, NPY_DOUBLE, data3));
if (!arr1 || !arr2 || !arr3) {
PyErr_SetString(PyExc_RuntimeError, "Failed when creating output array");
Py_RETURN_NONE;
}
PyArray_ENABLEFLAGS(arr1, NPY_ARRAY_OWNDATA);
PyArray_ENABLEFLAGS(arr2, NPY_ARRAY_OWNDATA);
PyArray_ENABLEFLAGS(arr3, NPY_ARRAY_OWNDATA);
// Some clean-up
delete[] shape; shape = NULL;
// Return multiple outputs
PyObject* ret = Py_BuildValue("NNN", arr1, arr2, arr3);
return ret;
}
static PyMethodDef extension_methods[] = {
{ "create_arrays", (PyCFunction)createArrays, METH_VARARGS, nullptr },
{ nullptr, nullptr, 0, nullptr },
};
static PyModuleDef extension_module = { PyModuleDef_HEAD_INIT, "extension", "Some docs...", 0, extension_methods };
PyMODINIT_FUNC PyInit_extension() {
import_array();
return PyModule_Create(&extension_module);
}
setup.py
import os
import sys
from setuptools import setup, Extension, find_packages
# Get installation path of Python interpreter
(path_interpreter, _) = os.path.split(sys.executable)
ext_module = Extension('example_package.extension',
sources=['example_package/module.cpp',],
include_dirs=[os.path.join(path_interpreter, 'Lib/site-packages/numpy/core/include')],
extra_compile_args=['/Zi'],
extra_link_args=['/DEBUG'])
setup(
name='example_package',
version='0.1',
ext_modules=[ext_module],
)
debug_extension.py
import numpy as np
from example_package.extension import create_arrays
shape = (4000, 6000)
def get_data():
# Write 5 pairs of arrays into this list
list_pairs = []
for i in range(50):
# Get 3 arrays from extension
(arr1, arr2, arr3) = create_arrays(shape)
# Reshape, transpose and later combine them using np.stack()
arr1 = np.reshape(arr1, shape).transpose()
arr2 = np.reshape(arr2, shape).transpose()
arr3 = np.reshape(arr3, shape).transpose()
list_pairs.append([
np.stack([arr1, arr2], axis=0),
np.stack([arr1, arr3], axis=1),
])
return list_pairs
list_pairs = get_data()
User contributions licensed under CC BY-SA 3.0