How to prevent maximum recursion runtime error in python when using the libtiff library

1

In the past few years I have worked with the libtiff library and I have developed some classes and scripts around it. I felt confident to now use my insights in a larger project. Instead of opening one TIFF file, I had to now open many of them. The runtime error "maximum recursion depth exceeded" has continued to disturb me since. Of course, I tried opening, reading and closing one file after the other - to no avail. I tried using the "del" keyword as well as gc.collect() - to no avail. What happens is that the local function extender_pyfunc with argument tiff_struct is invoking itself recursively. This function is part of the method __init__of class TIFFExtender in module libtiff_ctypes. The first time a TIFF file is opened there's no problem and as long as the TIFF files are read within a main() function no problems seem to occur, but as soon as I invoke my code from a method the runtime error comes back. Here's my module read_multiple_tifs.py:

from libtiff.libtiff_ctypes import TIFFFieldInfo, TIFFDataType, FIELD_CUSTOM, add_tags, TIFF
from libtiff import libtiff
from lmgeo.formats import inmemoryraster
from math import floor, ceil
import os.path
import numpy as np

__ncols = 1
__nrows = 1
__tile_width = 1
__tile_length = 1
__nstrips = 1
__samples_per_pixel = 1
__bits_per_sample = 8
__sample_format = 16
__itemsize = 4
__tif = None

def dummy(self, *args): pass;
__ReadStrip = dummy(0, 0, 1)

def ReadStripOfTiles(strip, buf, size):
    result = False
    try:
        num_tiles_per_strip = int(ceil(__ncols / __tile_width))
        numpy_type = __tif.get_numpy_type(__bits_per_sample, __sample_format)
        if (strip == __nstrips-1): 
            length = __nrows - (strip*__tile_length)
        else:
            length = __tile_length
        buf = buf.reshape(length, __ncols, __samples_per_pixel)
        for k in range(num_tiles_per_strip): 
            if (k == num_tiles_per_strip-1):
                # We only need part of the tile because we are on the edge
                width = __ncols - (num_tiles_per_strip-1)*__tile_width
            else:
                width = __tile_width

            # Fill the buffer tile by tile 
            tmp_buf = np.ascontiguousarray(np.zeros((__tile_length, __tile_width), numpy_type))
            seq = libtiff.TIFFReadTile(__tif, tmp_buf.ctypes.data, k*__tile_width, strip*__tile_length, 0, 0)
            if seq != None:
                start = k*__tile_width
                buf[0:length, start:start+width, 0] = tmp_buf[0:length, 0:width]
        result = True
    except Exception as e:    
        print str(e)
    finally:
        return result

def nextRow(i):  
    global image

    # This is for reading tile TIFF files
    row_in_strip = i % __tile_length # also zero-based!
    if row_in_strip == 0: 
        # Retrieve new strip - not needed again for the next __tile_length rows  
        curstrip = int(floor(i / __tile_length)) 
        if curstrip == __nstrips-1:
            # Last strip
            length = __nrows - __tile_length * curstrip
            __layer_size = (__ncols) * length * (__samples_per_pixel) * (__itemsize) 
        else:
            length = __tile_length
            __layer_size = (__ncols) * length * (__samples_per_pixel) * (__itemsize) 

        # Before trying to read, initialise the buffer
        numpy_type = __tif.get_numpy_type(__bits_per_sample, __sample_format)
        image = np.zeros((length, __ncols, __samples_per_pixel), dtype=numpy_type)           
        image.fill(0.0)
        __ReadStrip(curstrip, image, int(__layer_size))
        image = image.reshape(length, __ncols, __samples_per_pixel)

    result = image[row_in_strip, :]
    return result

def main():
    global __ReadStrip
    global __tif
    global __ncols 
    global __nrows 
    global __tile_width 
    global __tile_length 
    global __nstrips 
    global __samples_per_pixel 
    global __bits_per_sample
    global __sample_format 
    global __itemsize 

    # Define constants
    datapath = r'D:\Userdata\username\prjname\subprj\tiffs'
    files = ['CRFVOL1', 'CRFVOL2', 'FC1', 'WP1', 'FC2', 'WP2']
    xll, yll = 20.4333013, 26.4595931
    cellsize = 0.002083333

    # Initialise some variables  
    ir = None    
    for j in range(len(files)):
        ir = None
        try:
            # Add extra tags
            extra_tags = [
                TIFFFieldInfo(297, 2, 2, TIFFDataType.TIFF_SHORT, FIELD_CUSTOM, True, False, "PageNumber"),          
                TIFFFieldInfo(33550, 3, 3, TIFFDataType.TIFF_DOUBLE, FIELD_CUSTOM, True, False, "ModelPixelScaleTag"),
                TIFFFieldInfo(33922, 6, 6, TIFFDataType.TIFF_DOUBLE, FIELD_CUSTOM, True, False, "ModelTiepointTag"),
                TIFFFieldInfo(34264, 16, 16, TIFFDataType.TIFF_DOUBLE, FIELD_CUSTOM, True, False, "ModelTransformationTag"),
                TIFFFieldInfo(34735, 32, 32, TIFFDataType.TIFF_SHORT, FIELD_CUSTOM, True, False, "GeoKeyDirectoryTag"),
                TIFFFieldInfo(34736, -1, -1, TIFFDataType.TIFF_DOUBLE, FIELD_CUSTOM, True, False, "GeoDoubleParamsTag"),
                TIFFFieldInfo(34737, -1, -1, TIFFDataType.TIFF_ASCII, FIELD_CUSTOM, True, False, "GeoAsciiParamsTag"),
                TIFFFieldInfo(42112, -1, -1, TIFFDataType.TIFF_ASCII, FIELD_CUSTOM, True, False, "GDAL_METADATA"),
                TIFFFieldInfo(42113, -1, -1, TIFFDataType.TIFF_ASCII, FIELD_CUSTOM, True, False, "GDAL_NODATA")
            ]
            add_tags(extra_tags)

            # Open the TIFF file
            fn = os.path.join(datapath, files[j] + '.tif')
            __tif = TIFF.open(fn, mode='r')

            # Retrieve some parameters
            __ncols = int(__tif.GetField("ImageWidth"))
            __nrows = int(__tif.GetField("ImageLength"))

            # Check what kind of TIFF it is
            __tile_width = __tif.GetField("TileWidth")
            __tile_length = __tif.GetField("TileLength")
            if (__tile_width != None) and (__tile_length != None):
                # Assume tile TIFF
                __ntiles = libtiff.TIFFNumberOfTiles(__tif).value
                __nstrips = int(__ntiles / ceil(__ncols / __tile_width))
                num_tiles_per_strip = int(ceil(__ncols / __tile_width))
                msg = "Number of tiles not in accordance with tile and image dimensions!"
                assert __ntiles == __nstrips * num_tiles_per_strip, msg
                planar_config = __tif.GetField("PlanarConfig")
                if (planar_config > 1):
                    raise NotImplementedError("Not yet able to deal with data organised in separate planes")
                __ReadStrip = ReadStripOfTiles
            else:
                # Assume row TIFF
                if __tif.GetField("RowsPerStrip") != 1:
                    raise Exception("Unable to read this kind of TIFF file!")
                if __tif.GetField("Compression") == 1: 
                    __ReadStrip = __tif.ReadRawStrip
                else:
                    __ReadStrip = __tif.ReadEncodedStrip

            __bits_per_sample = __tif.GetField("BitsPerSample")
            __sample_format = __tif.GetField("SampleFormat")
            __samples_per_pixel = __tif.GetField("SamplesPerPixel")
            numpy_type = __tif.get_numpy_type(__bits_per_sample, __sample_format)
            __itemsize = __bits_per_sample / 8
            layer_size = __ncols * __samples_per_pixel * __itemsize  
            objnodata = __tif.GetField("GDAL_NODATA")
            if objnodata != None:
                if files[j][:-1] == 'CRFVOL':
                    nodatavalue = int(__tif.GetField("GDAL_NODATA"))
                else:    
                    nodatavalue = float(__tif.GetField("GDAL_NODATA"))   
            else:
                if files[j][:-1] == 'CRFVOL':
                    nodatavalue = -32768
                else:    
                    nodatavalue = -3.4e+038         

            # Prepare a structure to receive the lines one by one
            if files[j][:-1] == 'CRFVOL':
                ir = inmemoryraster.InMemoryRaster(os.path.join(datapath, files[j] + '.tmp'), None, 'i')
            else:
                ir = inmemoryraster.InMemoryRaster(os.path.join(datapath, files[j] + '.tmp'), None, 'f')

            if not ir.open('w', __ncols, __nrows, xll, yll, cellsize, nodatavalue):
                raise Exception("Unable initialise InMemoryRaster!")

            if (__tile_width != None) and (__tile_length != None):
                for i in range(__nrows):
                    line = nextRow(i)
                    ir.writenext(np.reshape(line, (ir.ncols)))
            else:
                for i in range(__nrows):
                    # This is for reading row TIFF files
                    buf = np.zeros((__ncols, __samples_per_pixel), numpy_type)
                    __ReadStrip(i, buf.ctypes.data, int(layer_size))
                    ir.writenext(np.reshape(buf, (ir.ncols)))
        except Exception as e:
            print e
        finally:
            if (ir != None): print(ir.nrows, ir.nodatavalue)

if (__name__ == "__main__"):
    main()

In the TIFF files FC?.tif and WP?.tif floats are stored in blocks of 2048x1 and in the CRFVOL?.tif files integers are stored in tiles of 128x128 pixels. No problems when I run this code.

As soon as I invoke this code from a class, the runtime error occurs. E.g.

from read_multiple_tifs import main as rmt_main

class Mycls(object):
    __my_private_method = None
    def __init__(self):
        self.__my_private_method = rmt_main

    def my_public_method(self):
        self.__my_private_method()

def main():
    mycls = Mycls()
    mycls.my_public_method()

if (__name__ == "__main__"):
    main()

Then, this is part of the output:

Traceback (most recent call last):
  File "_ctypes/callbacks.c", line 314, in 'calling callback function'
  File "D:\UserData\username\Canopy32\User\lib\site-packages\libtiff\libtiff_ctypes.py", line 231, in extender_pyfunc
    self._ParentExtender(tiff_struct)
WindowsError: exception: access violation reading 0x00000005
Traceback (most recent call last):
Traceback (most recent call last):
  File "_ctypes/callbacks.c", line 314, in 'calling callback function'
  File "D:\UserData\username\Canopy32\User\lib\site-packages\libtiff\libtiff_ctypes.py", line 227, in extender_pyfunc
    def extender_pyfunc(tiff_struct):
  File "D:\usr\lib\Eclipse4_3\plugins\org.python.pydev_3.5.0.201405201709\pysrc\pydevd.py", line 1239, in trace_dispatch
    traceback.print_exc()
  File "C:\Users\username\AppData\Local\Enthought\Canopy32\App\appdata\canopy-1.4.0.1938.win-x86\lib\traceback.py", line 232, in print_exc
    print_exception(etype, value, tb, limit, file)
  File "C:\Users\username\AppData\Local\Enthought\Canopy32\App\appdata\canopy-1.4.0.1938.win-x86\lib\traceback.py", line 125, in print_exception
    print_tb(tb, limit, file)
  File "C:\Users\username\AppData\Local\Enthought\Canopy32\App\appdata\canopy-1.4.0.1938.win-x86\lib\traceback.py", line 67, in print_tb
    '  File "%s", line %d, in %s' % (filename, lineno, name))
RuntimeError: maximum recursion depth exceeded

Tha above behaviour makes libtiff less efficient than I expected it to be. What can be done to solve this??? BTW, I'm using Python 2.7.6 -- 32-bit and libtiff version 0.4.1.dev0, on Windows 7.

python
recursion
runtime-error
ctypes
libtiff
asked on Stack Overflow May 31, 2018 by Dobedani

0 Answers

Nobody has answered this question yet.


User contributions licensed under CC BY-SA 3.0