Python Ctypes: OSError: exception: access violation reading <location>

I am trying to interface a CPP library with Python using Ctypes.

The cpp code I am trying to interface is available here. All code used here is available here.

The python code that is used to interface is written in gdist.py.

gdist.py

import ctypes
import glob
import os
import sys

import numpy as np
import scipy.sparse


if sys.platform == 'win32':
    libfile = glob.glob('build/*/gdist_c_api.dll')[0]
    libfile = os.path.abspath(libfile)
    lib = ctypes.CDLL(libfile)
elif sys.platform == 'darwin':
    libfile = glob.glob('build/*/gdist*.dylib')[0]
    lib = ctypes.CDLL(libfile)
else:
    libfile = glob.glob('build/*/gdist*.so')[0]
    lib = ctypes.CDLL(libfile)

lib.local_gdist_matrix.argtypes = [
    ctypes.c_uint,
    ctypes.c_uint,
    np.ctypeslib.ndpointer(dtype=np.float64),
    np.ctypeslib.ndpointer(dtype=np.uint32),
    ctypes.POINTER(ctypes.c_uint),
    ctypes.c_double,
]
lib.local_gdist_matrix.restype = ctypes.POINTER(ctypes.c_double)

lib.free_memory.argtypes = [
    ctypes.POINTER(ctypes.c_double),
]
lib.free_memory.restype = None


def local_gdist_matrix(
    vertices,
    triangles,
    max_distance=1e100,
):
    vertices = vertices.ravel()
    triangles = triangles.ravel()
    sparse_matrix_size = ctypes.c_uint(0)
    data = lib.local_gdist_matrix(
        vertices.size,
        triangles.size,
        vertices,
        triangles,
        ctypes.byref(sparse_matrix_size),
        max_distance,
    )
    np_data = np.fromiter(
        data,
        dtype=np.float64,
        count=3 * sparse_matrix_size.value,
    )
    lib.free_memory(data)
    assert np_data.size % 3 == 0
    sizes = np_data.size // 3
    rows = np_data[:sizes]
    columns = np_data[sizes: 2 * sizes]
    np_data = np_data[2 * sizes:]
    return scipy.sparse.csc_matrix(
        (np_data, (rows, columns)), shape=(vertices.size // 3, vertices.size // 3)
    )

C++ code used to interface:

gdist_c_api.h

#include <iostream>
#include <fstream>
#include <inttypes.h>

#include "geodesic_algorithm_exact.h"


#if defined(_WIN32)
#  if defined(DLL_EXPORTS)
#    define DLL_EXPORT_API __declspec(dllexport)
#  else
#    define DLL_EXPORT_API __declspec(dllimport)
#  endif
#else
#  define DLL_EXPORT_API
#endif


double* local_gdist_matrix_impl(
    unsigned number_of_vertices,
    unsigned number_of_triangles,
    double *vertices,
    unsigned *triangles,
    unsigned *sparse_matrix_size,
    double max_distance
);

void free_memory_impl(double *ptr);

extern "C" {
    DLL_EXPORT_API double* local_gdist_matrix(
        unsigned number_of_vertices,
        unsigned number_of_triangles,
        double *vertices,
        unsigned *triangles,
        unsigned *sparse_matrix_size,
        double max_distance
    );

    DLL_EXPORT_API void free_memory(double *ptr);
};

gdist_c_api.cpp

#include "gdist_c_api.h"


double* local_gdist_matrix_impl(
    unsigned number_of_vertices,
    unsigned number_of_triangles,
    double *vertices,
    unsigned *triangles,
    unsigned *sparse_matrix_size,
    double max_distance
) {
    std::vector<double> points (vertices, vertices + number_of_vertices);
    std::vector<unsigned> faces (triangles, triangles + number_of_triangles);
    
    geodesic::Mesh mesh;
    mesh.initialize_mesh_data(points, faces); // create internal mesh data structure including edges
    geodesic::GeodesicAlgorithmExact algorithm(&mesh); // create exact algorithm for the mesh
    std::vector <unsigned> rows_vector, columns_vector;
    std::vector <double> data_vector;

    double distance = 0;

    std::vector<geodesic::SurfacePoint> targets(number_of_vertices), source;

    for (unsigned i = 0; i < number_of_vertices; ++i) {
        targets[i] = geodesic::SurfacePoint(&mesh.vertices()[i]);
    }
    for (unsigned i = 0; i < number_of_vertices / 3; ++i) {
        source.push_back(geodesic::SurfacePoint(&mesh.vertices()[i]));
        algorithm.propagate(source, max_distance, NULL);
        source.pop_back();
        for (unsigned j = 0; j < number_of_vertices / 3; ++j) {
            algorithm.best_source(targets[j], distance);
            if (distance != geodesic::GEODESIC_INF && distance != 0 && distance <= max_distance) {
                rows_vector.push_back(i);
                columns_vector.push_back(j);
                data_vector.push_back(distance);
            }
        }
    }

    double *data;
    data = new double[3 * rows_vector.size()];
    assert (data != NULL); // memory allocation should not fail
    *sparse_matrix_size = rows_vector.size();

    std::copy(rows_vector.begin(), rows_vector.end(), data);
    std::copy(columns_vector.begin(), columns_vector.end(), data + data_vector.size());
    std::copy(data_vector.begin(), data_vector.end(), data + 2 * data_vector.size());

    return data;
}


void free_memory_impl(double *ptr) {
    delete[] ptr;
}


extern "C" {
    double* local_gdist_matrix(
        unsigned number_of_vertices,
        unsigned number_of_triangles,
        double *vertices,
        unsigned *triangles,
        unsigned *sparse_matrix_size,
        double max_distance
    ) {
        return local_gdist_matrix_impl(
            number_of_vertices,
            number_of_triangles,
            vertices,
            triangles,
            sparse_matrix_size,
            max_distance
        );
    }

    void free_memory(double *ptr) {
        free_memory_impl(ptr);
    }
};

Batch script to create DLL file on Windows:

mkdir build\lib.win32
cd build\lib.win32
cl.exe /LD /DDLL_EXPORTS /EHsc /W4 ..\..\geodesic_library\gdist_c_api.cpp

Script to create dylib on macOS:

mkdir build
cd build
mkdir lib.macos
cd lib.macos
clang++ -std=c++11 -shared -fPIC ../../geodesic_library/gdist_c_api.cpp -o gdist_c_api.dylib

Test file: test_gdist.py

import numpy as np

import gdist


def test_equality_with_stable():
    surface_data = 'inner_skull_642'
    expected = np.loadtxt(f'data/{surface_data}/gdist_matrix.txt')
    vertices = np.loadtxt(
        f'data/{surface_data}/vertices.txt',
        dtype=np.float64,
    )
    triangles = np.loadtxt(
        f'data/{surface_data}/triangles.txt',
        dtype=np.uint32,
    )
    actual = gdist.local_gdist_matrix(
        vertices=vertices,
        triangles=triangles,
    )
    actual = actual.toarray()
    np.testing.assert_array_almost_equal(actual, expected)

def test_flat_triangular_mesh():
    data = np.loadtxt("data/flat_triangular_mesh.txt", skiprows=1)
    vertices = data[0:121].astype(np.float64)
    triangles = data[121:].astype(np.uint32)
    distances = gdist.local_gdist_matrix(vertices, triangles)
    epsilon = 1e-6  # the default value used in `assert_array_almost_equal`
    # test if the obtained matrix is symmetric
    assert (abs(distances - distances.T) > epsilon).nnz == 0
    np.testing.assert_array_almost_equal(distances.toarray()[1][0], 0.2)
    # set max distance as 0.3
    distances = gdist.local_gdist_matrix(vertices, triangles, 0.3)
    # test if the obtained matrix is symmetric
    assert (abs(distances - distances.T) > epsilon).nnz == 0
    assert np.max(distances) <= 0.3

def test_hedgehog_mesh():
    data = np.loadtxt("data/hedgehog_mesh.txt", skiprows=1)
    vertices = data[0:300].astype(np.float64)
    triangles = data[300:].astype(np.uint32)
    distances = gdist.local_gdist_matrix(vertices, triangles)
    epsilon = 1e-6  # the default value used in `assert_array_almost_equal`
    # test if the obtained matrix is symmetric
    assert (abs(distances - distances.T) > epsilon).nnz == 0
    np.testing.assert_array_almost_equal(
        distances.toarray()[1][0], 1.40522
    )
    # set max distance as 1.45
    distances = gdist.local_gdist_matrix(vertices, triangles, 1.45)
    # test if the obtained matrix is symmetric
    assert (abs(distances - distances.T) > epsilon).nnz == 0
    assert np.max(distances) <= 1.45

If I run pytest on the code, the tests sometimes fail on Windows with the following error:

Windows fatal exception: access violation

Current thread 0x00001134 (most recent call first):

  File "c:\python38\lib\site-packages\gdist.py", line 92 in local_gdist_matrix
  File "c:\python38\lib\site-packages\gdist.py", line 146 in local_gdist_matrix
  File "C:\Users\travis\build\ayan-b\tvb-geodesic\tests\test_gdist.py", line 64 in test_hedgehog_mesh
  File "c:\python38\lib\site-packages\_pytest\python.py", line 182 in pytest_pyfunc_call
  File "c:\python38\lib\site-packages\pluggy\callers.py", line 187 in _multicall
  File "c:\python38\lib\site-packages\pluggy\manager.py", line 84 in <lambda>
  File "c:\python38\lib\site-packages\pluggy\manager.py", line 93 in _hookexec
  File "c:\python38\lib\site-packages\pluggy\hooks.py", line 286 in __call__
  File "c:\python38\lib\site-packages\_pytest\python.py", line 1477 in runtest
  File "c:\python38\lib\site-packages\_pytest\runner.py", line 135 in pytest_runtest_call
  File "c:\python38\lib\site-packages\pluggy\callers.py", line 187 in _multicall
  File "c:\python38\lib\site-packages\pluggy\manager.py", line 84 in <lambda>
  File "c:\python38\lib\site-packages\pluggy\manager.py", line 93 in _hookexec
  File "c:\python38\lib\site-packages\pluggy\hooks.py", line 286 in __call__
  File "c:\python38\lib\site-packages\_pytest\runner.py", line 217 in <lambda>
  File "c:\python38\lib\site-packages\_pytest\runner.py", line 244 in from_call
  File "c:\python38\lib\site-packages\_pytest\runner.py", line 216 in call_runtest_hook
  File "c:\python38\lib\site-packages\_pytest\runner.py", line 186 in call_and_report
  File "c:\python38\lib\site-packages\_pytest\runner.py", line 100 in runtestprotocol
  File "c:\python38\lib\site-packages\_pytest\runner.py", line 85 in pytest_runtest_protocol
  File "c:\python38\lib\site-packages\pluggy\callers.py", line 187 in _multicall
  File "c:\python38\lib\site-packages\pluggy\manager.py", line 84 in <lambda>
  File "c:\python38\lib\site-packages\pluggy\manager.py", line 93 in _hookexec
  File "c:\python38\lib\site-packages\pluggy\hooks.py", line 286 in __call__
  File "c:\python38\lib\site-packages\_pytest\main.py", line 272 in pytest_runtestloop
  File "c:\python38\lib\site-packages\pluggy\callers.py", line 187 in _multicall
  File "c:\python38\lib\site-packages\pluggy\manager.py", line 84 in <lambda>
  File "c:\python38\lib\site-packages\pluggy\manager.py", line 93 in _hookexec
  File "c:\python38\lib\site-packages\pluggy\hooks.py", line 286 in __call__
  File "c:\python38\lib\site-packages\_pytest\main.py", line 247 in _main
  File "c:\python38\lib\site-packages\_pytest\main.py", line 191 in wrap_session
  File "c:\python38\lib\site-packages\_pytest\main.py", line 240 in pytest_cmdline_main
  File "c:\python38\lib\site-packages\pluggy\callers.py", line 187 in _multicall
  File "c:\python38\lib\site-packages\pluggy\manager.py", line 84 in <lambda>
  File "c:\python38\lib\site-packages\pluggy\manager.py", line 93 in _hookexec
  File "c:\python38\lib\site-packages\pluggy\hooks.py", line 286 in __call__
  File "c:\python38\lib\site-packages\_pytest\config\__init__.py", line 124 in main
  File "C:\Python38\Scripts\pytest.exe\__main__.py", line 7 in <module>
  File "c:\python38\lib\runpy.py", line 85 in _run_code
  File "c:\python38\lib\runpy.py", line 192 in _run_module_as_main

Sometimes it also fails on macOS with the following error:

tests/test_equality_with_stable.py Fatal Python error: Segmentation fault

Current thread 0x0000000103ce05c0 (most recent call first):

  File "/usr/local/bin/gdist.py", line 98 in local_gdist_matrix
  File "/usr/local/bin/gdist.py", line 151 in local_gdist_matrix
  File "/Users/travis/build/ayan-b/tvb-geodesic/tests/test_equality_with_stable.py", line 19 in test_equality_with_stable
  File "/usr/local/lib/python3.7/site-packages/_pytest/python.py", line 182 in pytest_pyfunc_call
  File "/usr/local/lib/python3.7/site-packages/pluggy/callers.py", line 187 in _multicall
  File "/usr/local/lib/python3.7/site-packages/pluggy/manager.py", line 87 in <lambda>
  File "/usr/local/lib/python3.7/site-packages/pluggy/manager.py", line 93 in _hookexec
  File "/usr/local/lib/python3.7/site-packages/pluggy/hooks.py", line 286 in __call__
  File "/usr/local/lib/python3.7/site-packages/_pytest/python.py", line 1477 in runtest
  File "/usr/local/lib/python3.7/site-packages/_pytest/runner.py", line 135 in pytest_runtest_call
  File "/usr/local/lib/python3.7/site-packages/pluggy/callers.py", line 187 in _multicall
  File "/usr/local/lib/python3.7/site-packages/pluggy/manager.py", line 87 in <lambda>
  File "/usr/local/lib/python3.7/site-packages/pluggy/manager.py", line 93 in _hookexec
  File "/usr/local/lib/python3.7/site-packages/pluggy/hooks.py", line 286 in __call__
  File "/usr/local/lib/python3.7/site-packages/_pytest/runner.py", line 217 in <lambda>
  File "/usr/local/lib/python3.7/site-packages/_pytest/runner.py", line 244 in from_call
  File "/usr/local/lib/python3.7/site-packages/_pytest/runner.py", line 217 in call_runtest_hook
  File "/usr/local/lib/python3.7/site-packages/_pytest/runner.py", line 186 in call_and_report
  File "/usr/local/lib/python3.7/site-packages/_pytest/runner.py", line 100 in runtestprotocol
  File "/usr/local/lib/python3.7/site-packages/_pytest/runner.py", line 85 in pytest_runtest_protocol
  File "/usr/local/lib/python3.7/site-packages/pluggy/callers.py", line 187 in _multicall
  File "/usr/local/lib/python3.7/site-packages/pluggy/manager.py", line 87 in <lambda>
  File "/usr/local/lib/python3.7/site-packages/pluggy/manager.py", line 93 in _hookexec
  File "/usr/local/lib/python3.7/site-packages/pluggy/hooks.py", line 286 in __call__
  File "/usr/local/lib/python3.7/site-packages/_pytest/main.py", line 272 in pytest_runtestloop
  File "/usr/local/lib/python3.7/site-packages/pluggy/callers.py", line 187 in _multicall
  File "/usr/local/lib/python3.7/site-packages/pluggy/manager.py", line 87 in <lambda>
  File "/usr/local/lib/python3.7/site-packages/pluggy/manager.py", line 93 in _hookexec
  File "/usr/local/lib/python3.7/site-packages/pluggy/hooks.py", line 286 in __call__
  File "/usr/local/lib/python3.7/site-packages/_pytest/main.py", line 247 in _main
  File "/usr/local/lib/python3.7/site-packages/_pytest/main.py", line 191 in wrap_session
  File "/usr/local/lib/python3.7/site-packages/_pytest/main.py", line 240 in pytest_cmdline_main
  File "/usr/local/lib/python3.7/site-packages/pluggy/callers.py", line 187 in _multicall
  File "/usr/local/lib/python3.7/site-packages/pluggy/manager.py", line 87 in <lambda>
  File "/usr/local/lib/python3.7/site-packages/pluggy/manager.py", line 93 in _hookexec
  File "/usr/local/lib/python3.7/site-packages/pluggy/hooks.py", line 286 in __call__
  File "/usr/local/lib/python3.7/site-packages/_pytest/config/__init__.py", line 125 in main
  File "/usr/local/bin/pytest", line 8 in <module>

/Users/travis/.travis/functions: line 109:  2197 Segmentation fault: 11  pytest --cov=gdist

The command "pytest --cov=gdist" exited with 139.

However, the tests never fail on Linux.

Notes:

• I am not sure if the issue is with the testing framework I am using. With pytest the test failed around 50% of the times however with nose2 the test failed only once of 20+ runs.
  Log:
  • Link: Only once when nose2 failed (on Windows).
  • Link: macOS with pytest
  • Link: Windows with pytest
• I have tried with both CDLL and WinDLL but it didn't fix the issue.