i'm make my first OpenCV Plugin using C++.

But, it has some issue.

1. Image glitch
2. Memory leak

This is what it running looks like.

Expected behavior

Get the document of drawing picture and export only draw lines.

Source

All source

Xcode OSX 12.4 or Android native Android Studio 4.1.2

• OpenCVPlugin.cpp

#include <iostream>
//https://www.vbflash.net/83
#include <opencv2/opencv.hpp>

using namespace std;
using namespace cv;

unsigned char* resultPicBuffer;
int picRows = 0;
int picCols = 0;

extern "C" {
    int ResultPicBufferRows();
    int ResultPicBufferCols();
    bool compareContourAreas (std::vector<cv::Point>, std::vector<cv::Point>);
    unsigned char* ExportPicFromDoc(int width, int height, unsigned char* buffer);
    void FreeBuffer();
}

void FreeBuffer() {
    if (picRows * picCols > 0) {
        fill_n(resultPicBuffer, picRows * picCols * 4, 0);
        delete [] resultPicBuffer;
    }
}

int ResultPicBufferRows() {
    return picRows;
}

int ResultPicBufferCols() {
    return picCols;
}

bool compareContourAreas (std::vector<cv::Point>, std::vector<cv::Point>);

unsigned char* ExportPicFromDoc(int width, int height, unsigned char* buffer) {

    Mat img(height, width, CV_8UC4, buffer);

    float maxImgWidth = 2000.0;
    float ratio = maxImgWidth / img.size().height;

    Mat smallImg;
    resize(img, smallImg, Size(int(img.size().width * ratio), maxImgWidth));

//        imshow("smallImg", smallImg);

    Mat gray;
    cvtColor(smallImg, gray, COLOR_BGR2GRAY);
//        imshow("gray", gray);

    Mat grayBlur;
    GaussianBlur(gray, grayBlur, Size(3, 3), BORDER_CONSTANT);
//        imshow("grayBlur", grayBlur);

    Mat edge;
    Canny(grayBlur, edge, 100, 200);
//        imshow("edge", edge);

    vector<vector<Point>> contours;
    vector<Vec4i> hierarchy;
    findContours( edge, contours, hierarchy, RETR_LIST, CHAIN_APPROX_SIMPLE );
    sort(contours.begin(), contours.end(), compareContourAreas);

    vector<vector<Point>> topContours = vector<vector<Point>>(contours.end() - 5, contours.end());
    Mat smallImg_copy = smallImg.clone();

    vector<Point> screenContours;

    for (unsigned long i = topContours.size() - 1; i >= 0; i--) {
        double peri = arcLength(topContours[i], true);
        vector<Point> approx;
        approxPolyDP(topContours[i], approx, 0.02 * peri, true);

        if (approx.size() == 4) {
            screenContours = approx;
            break;
        }
        
        vector<Point>().swap(approx);
    }

    if (screenContours.size() <= 0) {
        FreeBuffer();
        picRows = 0;
        picCols = 0;
        
        for (int i = 0; i < contours.size(); i++) {
            vector<Point>().swap(contours[i]);
        }
        vector<vector<Point>>().swap(contours);
        
        vector<Vec4i>().swap(hierarchy);
        
        for (int i = 0; i < topContours.size(); i++) {
            vector<Point>().swap(topContours[i]);
        }
        vector<vector<Point>>().swap(topContours);
        
        vector<Point>().swap(screenContours);
        
        smallImg_copy.release();
        edge.release();
        grayBlur.release();
        gray.release();
        smallImg.release();
        img.release();
        
        return 0;
    }

    vector<vector<Point>> screenContours_vec;
    screenContours_vec.push_back(screenContours);

    drawContours(smallImg_copy, screenContours_vec, -1, CV_RGB(0, 255, 0), 2);
//        imshow("contours", smallImg_copy);

    Point topLeft, topRight, bottomRight, bottomLeft;
    topLeft.x = 0x0fffffff;
    topLeft.y = 0x0fffffff;

    topRight.y = 0x7fffffff;

    for (unsigned long i = 0; i < screenContours.size(); i++) {
        if (topLeft.x + topLeft.y > screenContours[i].x + screenContours[i].y) {
            topLeft = screenContours[i];
        }

        if (topRight.y - topRight.x > screenContours[i].y - screenContours[i].x) {
            topRight = screenContours[i];
        }

        if (bottomRight.x + bottomRight.y < screenContours[i].x + screenContours[i].y) {
            bottomRight = screenContours[i];
        }

        if (bottomLeft.y - bottomLeft.x < screenContours[i].y - screenContours[i].x) {
            bottomLeft = screenContours[i];
        }
    }

    unsigned long padding = 10;
    topLeft.x += padding;
    topLeft.y += padding;
    topRight.x -= padding;
    topRight.y += padding;
    bottomLeft.x += padding;
    bottomLeft.y -= padding;
    bottomRight.x -= padding;
    bottomRight.y -= padding;

    unsigned long width1 = abs(topLeft.x - topRight.x), width2 = abs(bottomLeft.x - bottomRight.x),
            height1 = abs(topLeft.y - bottomLeft.y), height2 = abs(topRight.y - bottomRight.y);

    unsigned long maxWidth = max(width1, width2), maxHeight = max(height1, height2);

    vector<Point2f> srcRect;
    srcRect.push_back(topLeft);
    srcRect.push_back(topRight);
    srcRect.push_back(bottomLeft);
    srcRect.push_back(bottomRight);

    vector<Point2f> destRect;
    destRect.push_back(Point(0, 0));
    destRect.push_back(Point(maxWidth, 0));
    destRect.push_back(Point(0, maxHeight));
    destRect.push_back(Point(maxWidth, maxHeight));

    Mat perspectMat = getPerspectiveTransform(srcRect, destRect);
    Mat warpedImg;
    warpPerspective(smallImg, warpedImg, perspectMat, Size(maxWidth, maxHeight));
//        imshow("warpedImg", warpedImg);

    Mat warpedImgGray;
    cvtColor(warpedImg, warpedImgGray, COLOR_BGR2GRAY);
    adaptiveThreshold(warpedImgGray, warpedImgGray, 255, ADAPTIVE_THRESH_MEAN_C, THRESH_BINARY, 21, 10);
//        imshow("adapted", warpedImgGray);

    Mat edgePic;
    GaussianBlur(warpedImgGray, edgePic, Size(11, 11), BORDER_CONSTANT);
    Canny(edgePic, edgePic, 100, 200);
//        imshow("edgePic", edgePic);

    vector<vector<Point>> contoursPic;
    vector<Vec4i> hierarchyPic;
    findContours( edgePic, contoursPic, hierarchyPic, RETR_LIST, CHAIN_APPROX_SIMPLE );

    Mat edgePic_copy = warpedImg.clone();
    drawContours(edgePic_copy, contoursPic, -1, CV_RGB(0, 255, 0), 2);
//        imshow("edgePic_copy", edgePic_copy);

    Mat onlyContours = Mat(Size(edgePic_copy.cols, edgePic_copy.rows), CV_8UC4);
    drawContours(onlyContours, contoursPic, -1, CV_RGB(255, 255, 255), 2);
    
    cv::cvtColor(onlyContours, onlyContours, COLOR_RGB2BGRA);
//    std::vector<cv::Mat> bgra;
//    cv::split(onlyContours, bgra);
//    std::swap(bgra[0], bgra[3]);
//    std::swap(bgra[1], bgra[2]);
//    cvtColor(onlyContours, onlyContours, COLOR_BGR2GRAY);
//        imshow("onlyContours", onlyContours);
    int lastPicRows = picRows, lastPicCols = picCols;
    if (onlyContours.rows > 0 && onlyContours.rows != lastPicRows && onlyContours.cols > 0 && onlyContours.cols != lastPicCols) {
        FreeBuffer();
        picRows = onlyContours.rows;
        picCols = onlyContours.cols;
        resultPicBuffer = new unsigned char[picRows * picCols * 4];
    } else if (onlyContours.rows <= 0 || onlyContours.cols <= 0) {
        FreeBuffer();
        picRows = 0;
        picCols = 0;
        
        for (int i = 0; i < contours.size(); i++) {
            vector<Point>().swap(contours[i]);
        }
        vector<vector<Point>>().swap(contours);
        
        vector<Vec4i>().swap(hierarchy);
        
        for (int i = 0; i < topContours.size(); i++) {
            vector<Point>().swap(topContours[i]);
        }
        vector<vector<Point>>().swap(topContours);
        
        vector<Point>().swap(screenContours);
        
        for (int i = 0; i < screenContours_vec.size(); i++) {
            vector<Point>().swap(screenContours_vec[i]);
        }
        vector<vector<Point>>().swap(screenContours_vec);
        
        vector<Point2f>().swap(srcRect);
        
        vector<Point2f>().swap(destRect);
        
        for (int i = 0; i < contoursPic.size(); i++) {
            vector<Point>().swap(contoursPic[i]);
        }
        vector<vector<Point>>().swap(contoursPic);
        
        vector<Vec4i>().swap(hierarchyPic);

        onlyContours.release();
        edgePic_copy.release();
        edgePic.release();
        warpedImgGray.release();
        warpedImg.release();
        perspectMat.release();
        smallImg_copy.release();
        edge.release();
        grayBlur.release();
        gray.release();
        smallImg.release();
        img.release();
        
        return 0;
    }
//    picRows = onlyContours.rows;
//    picCols = onlyContours.cols;
//    resultPicBuffer = new unsigned char[picRows * picCols * 4];
    fill_n(resultPicBuffer, picRows * picCols * 4, 0);
    
//    globalMat = onlyContours.clone();
    
//    buffer = onlyContours.data;

//    size_t size = picRows * picCols * 3;
//    memcpy(resultPicBuffer, onlyContours.data, size);
//    memcpy(buffer, onlyContours.data, onlyContours.total() * onlyContours.elemSize());
    memcpy(resultPicBuffer, onlyContours.data, onlyContours.total() * onlyContours.elemSize());
    
    for (int i = 0; i < contours.size(); i++) {
        vector<Point>().swap(contours[i]);
    }
    vector<vector<Point>>().swap(contours);
    
    vector<Vec4i>().swap(hierarchy);
    
    for (int i = 0; i < topContours.size(); i++) {
        vector<Point>().swap(topContours[i]);
    }
    vector<vector<Point>>().swap(topContours);
    
    vector<Point>().swap(screenContours);
    
    for (int i = 0; i < screenContours_vec.size(); i++) {
        vector<Point>().swap(screenContours_vec[i]);
    }
    vector<vector<Point>>().swap(screenContours_vec);
    
    vector<Point2f>().swap(srcRect);
    
    vector<Point2f>().swap(destRect);
    
    for (int i = 0; i < contoursPic.size(); i++) {
        vector<Point>().swap(contoursPic[i]);
    }
    vector<vector<Point>>().swap(contoursPic);
    
    vector<Vec4i>().swap(hierarchyPic);

    onlyContours.release();
    edgePic_copy.release();
    edgePic.release();
    warpedImgGray.release();
    warpedImg.release();
    perspectMat.release();
    smallImg_copy.release();
    edge.release();
    grayBlur.release();
    gray.release();
    smallImg.release();
    img.release();

    return resultPicBuffer;
}

bool compareContourAreas ( std::vector<cv::Point> contour1, std::vector<cv::Point> contour2 ) {
    double i = fabs( contourArea(cv::Mat(contour1)) );
    double j = fabs( contourArea(cv::Mat(contour2)) );
    return ( i < j );
}

Unity project 2019.4.20f1

• TestController.cs

using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using UnityEngine.UI;
using System;
using System.Runtime.InteropServices;

public class TestController : MonoBehaviour
{
    public Text txt, txt2;
    public RawImage InImage;
    public RawImage OutImage;
    // Start is called before the first frame update
    void Start()
    {
        
    }

    // Update is called once per frame
    void Update()
    {
        txt.text = "Loading native lib";
        txt2.text = NativeAdapter.dllPath;
        GCHandle pixelHandle, resultPixelHandle;
        try {
            txt.text = "Native: " + NativeAdapter.FooTest().ToString();

            Texture2D rawImageTexture = (Texture2D)InImage.texture;
            Color32[] pixels = rawImageTexture.GetPixels32();

            pixelHandle = GCHandle.Alloc(pixels, GCHandleType.Pinned);
            IntPtr pixelPtr = pixelHandle.AddrOfPinnedObject();
            IntPtr testPtr = NativeAdapter.PicFromDoc(rawImageTexture.width, rawImageTexture.height, pixelPtr);

            int nativeH = NativeAdapter.PicBufferRows();
            int nativeW = NativeAdapter.PicBufferCols();
            int w = nativeW;
            int h = nativeH;

            txt.text = $"Result w: {w} h: {h} nativeW: {nativeW} nativeH: {nativeH}";

            Texture2D resultTexture = new Texture2D(w, h, TextureFormat.ARGB32, false);

            int bufferSize = w * h * 4;

            if (testPtr != IntPtr.Zero)
            {
                byte[] rawData = new byte[bufferSize];
                Marshal.Copy(testPtr, rawData, 0, bufferSize);

                resultTexture.LoadRawTextureData(rawData);
                resultTexture.Apply();
            }

            OutImage.texture = resultTexture;

        } catch (System.Exception e) {
            txt.text = e.Message;
            Debug.Log(e);
        } finally {
            if (pixelHandle != null) {
                pixelHandle.Free();
            }
            GC.Collect();
        }
    }
}

• NativeAdapter.cs

using System.Collections;
using System.Collections.Generic;
using System.Runtime.InteropServices;
using System;
using UnityEngine;
using System.IO;

public class NativeAdapter
{
    #if !UNITY_EDITOR
        [DllImport("OpenCVPlugin")]
        private static extern int FooTestFunction_Internal();

        [DllImport("OpenCVPlugin")]
        private static extern int ResultPicBufferRows();

        [DllImport("OpenCVPlugin")]
        private static extern int ResultPicBufferCols();

        [DllImport("OpenCVPlugin")]
        private static extern IntPtr ExportPicFromDoc(int width, int height, IntPtr bufferAddr);

        [DllImport ("OpenCVPlugin")]
        private static extern void FlipImage(ref Color32[] rawImage, int width, int height);

        [DllImport ("OpenCVPlugin")]
        private static extern void ReturnGlobalMat(IntPtr data);

        [DllImport ("OpenCVPlugin")]
        private static extern void FreeBuffer();
    #elif UNITY_EDITOR
        [DllImport ("UnityPlugin")]
        private static extern int FooTestFunction_Internal();

        [DllImport ("UnityPlugin")]
        private static extern int ResultPicBufferRows();

        [DllImport ("UnityPlugin")]
        private static extern int ResultPicBufferCols();

        [DllImport ("UnityPlugin")]
        private static extern IntPtr ExportPicFromDoc(int width, int height, IntPtr bufferAddr);

        [DllImport ("UnityPlugin")]
        private static extern void TestMat(int width, int height, IntPtr bufferAddr);

        [DllImport ("UnityPlugin")]
        private static extern void FlipImage(ref Color32[] rawImage, int width, int height);

        [DllImport ("UnityPlugin")]
        private static extern void ReturnGlobalMat(IntPtr data);

        [DllImport ("UnityPlugin")]
        private static extern IntPtr GetResultPicBuffer();

        [DllImport ("UnityPlugin")]
        private static extern void FreeBuffer();
    #endif

    public static string dllPath;

    static NativeAdapter() {
        string currentPath = Environment.GetEnvironmentVariable("PATH", EnvironmentVariableTarget.Process);
        dllPath = Path.DirectorySeparatorChar + "Assets" + Path.DirectorySeparatorChar + "Plugins";
        #if UNITY_ANDROID
            dllPath = Application.dataPath + Path.DirectorySeparatorChar + "Assets" + Path.DirectorySeparatorChar  + "Assets" + Path.DirectorySeparatorChar + "Plugins";
        #endif
        Debug.Log("dllPath " + dllPath);
        Debug.Log("currentPath " + currentPath);
        Debug.Log("process " + EnvironmentVariableTarget.Process);

        if(currentPath.Contains(dllPath) == false)
        {
            #if !UNITY_EDITOR
                Debug.Log("env " + Environment.GetEnvironmentVariable("PATH"));
            #elif UNITY_EDITOR
                Environment.SetEnvironmentVariable("PATH", currentPath + Path.PathSeparator + dllPath, EnvironmentVariableTarget.Process);
                Debug.Log("env " + Environment.GetEnvironmentVariable("PATH"));
            #endif
        }
    }

    public static int FooTest() {
        return FooTestFunction_Internal();
    }

    public static int PicBufferRows() {
        return ResultPicBufferRows();
    }

    public static int PicBufferCols() {
        return ResultPicBufferCols();
    }

    public static IntPtr PicFromDoc(int width, int height, IntPtr bufferAddr) {
        return ExportPicFromDoc(width, height, bufferAddr);
    }

    public static void _TestMat(int width, int height, IntPtr bufferAddr) {
        #if UNITY_EDITOR
            TestMat(width, height, bufferAddr);
        #endif
    }

    public static void _FlipImage(ref Color32[] rawImage, int width, int height) {
        FlipImage(ref rawImage, width, height);
    }

    public static void _ReturnGlobalMat(IntPtr bufferAddr) {
        ReturnGlobalMat(bufferAddr);
    }

    public static IntPtr _GetResultPicBuffer() {
        #if UNITY_EDITOR
            return GetResultPicBuffer();
        #else
            return IntPtr.Zero;
        #endif
    }

    public static void _FreeBuffer() {
        FreeBuffer();
    }
}

Edited

• Fixed Image glitch issue. But still memory leaking.

OpenCVPlugin.cpp

Mat onlyContours = Mat(Size(edgePic_copy.cols, edgePic_copy.rows), CV_8UC4, 0.0);
drawContours(onlyContours, contoursPic, -1, (255, 255, 255, 255), 2);

If you use Mat with Alpha channel. You should set alpha value or reset Mat as Zero.

i found the solution.

First @JohnFilleau said. I used some new X[] several times and might be this will cause memory leak.

So i removed it and use original buffer pointer.

In OpenCVPlugin.cpp

unsigned char* ExportPicFromDoc(int width, int height, unsigned char* buffer) {
...
memcpy(buffer, onlyContours.data, onlyContours.total() * onlyContours.elemSize());
...
}

Second in unity if i create Texture2D all the frames then it will cause OOM.

So plz use Resources.UnloadUnusedAssets(). Hint

In TestController.cs

void Update()
{
    ...
    Texture2D resultTexture = new Texture2D(w, h, TextureFormat.ARGB32, false);
    Resources.UnloadUnusedAssets(); // <-- Unload it *****

    int bufferSize = w * h * 4;

    if (bufferSize > 0)
    ...

So memory leak solved. My program works well. Ty everyone.