My main application reads a video file then sends the frame to be processed on a separate thread. The issue is that after so many frames it eventually starts saying the Mat is null inside Detector. What would cause this? I have tried mat.clone(), copying the bytes from a BufferedImage into byte[] then building the Mat again in the thread but the same issue happens. Everything works fine if I use a single thread.
EDIT:
So I’ve traced the issue to the need to call System.gc(); to prevent memory leaks… I have updated the code to allow you to duplicate the issue. I assume this has to do with the use of finalizers.
BlockingQueue<Frame> queue = new LinkedBlockingQueue<>(100000);
VideoCapture camera = new VideoCapture("/path/video.mp4");
Mat frame = new Mat();
Mat mat;
boolean finished = false;
Rect rectangle = new Rect(145, 0, 575, 426);
Runnable detection1Task = new ObjectDetection(queue);
Thread detection1Thread = new Thread(detection1Task);
detection1Thread.start();
detection1Thread.setName("Detection 1 Thread");
while ( !finished )
{
if (camera.read(frame))
{
mat = frame.submat(rectangle);
queue.add(new Frame(mat,0,0));//I do many more than just this mat this though
///Calling this on a timer will produce the same results eventually
///I have a task that needs to run 24/6 365 Days a year.
System.gc();//Removing this line will lead to a CRAZY amount of RAM consumption
}
else
{
finished = true;
}
}
My Thread:
public class ObjectDetection implements Runnable{
private final BlockingQueue<Object> queue;
private Frame object;
private Detector detector;
public ObjectDetection(BlockingQueue queue){
this.queue = queue;
}
@Override
public void run() {
while(true){
try{
object = queue.take();
Detector detect = new Detector("path/to/cfg", "path/to/weights", object.getMat(), 640,128,.9f);
object.clear();//Calls release
}
catch(Exception e){e.printStackTrace();}
}
}
Detector
private final String cfg;
private final String model;
private final Mat mat;
private final int networkWidth;
private final int networkHeight;
private final float threshold;
public Detector(String cfg, String model, Mat mat, int networkWidth, int networkHeight, float threshold){
this.cfg = cfg;
this.model = model;
this.mat = mat;
this.networkWidth = networkWidth;
this.networkHeight = networkHeight;
this.threshold = threshold;
}
public String detect(){
if(mat.empty())
{
System.out.println("EMPTY");
return "";
}
Net net = Dnn.readNetFromDarknet(cfg, model);
Size sz = new Size(networkWidth,networkHeight);
Mat blob = Dnn.blobFromImage(mat, 0.00392, sz, new Scalar(0), true, false);
net.setInput(blob);
List<Mat> result = new ArrayList<>();
List<String> outBlobNames = getOutputNames(net);
net.forward(result, outBlobNames);
List<Integer> clsIds = new ArrayList<>();
List<Float> confs = new ArrayList<>();
List<Rect2d> rects = new ArrayList<>();
for (int i = 0; i < result.size(); ++i)
{
// each row is a candidate detection, the 1st 4 numbers are
// [center_x, center_y, width, height], followed by (N-4) class probabilities
Mat level = result.get(i);
for (int j = 0; j < level.rows(); ++j)
{
Mat row = level.row(j);
Mat scores = row.colRange(5, level.cols());
Core.MinMaxLocResult mm = Core.minMaxLoc(scores);
float confidence = (float)mm.maxVal;
Point classIdPoint = mm.maxLoc;
if (confidence > threshold)
{
int centerX = (int)(row.get(0,0)[0] * mat.cols());
int centerY = (int)(row.get(0,1)[0] * mat.rows());
int width = (int)(row.get(0,2)[0] * mat.cols());
int height = (int)(row.get(0,3)[0] * mat.rows());
int left = centerX - width / 2;
int top = centerY - height / 2;
clsIds.add((int)classIdPoint.x);
confs.add((float)confidence);
rects.add(new Rect2d(left,top,width,height));
}
row.release();
scores.release();
}
level.release();
}
// Apply non-maximum suppression procedure.
float nmsThresh = .50f;
if (confs.isEmpty())//If no results were found return
{
blob.release();
return "";
}
MatOfFloat confidences = new MatOfFloat(Converters.vector_float_to_Mat(confs));
Rect2d[] boxesArray = rects.toArray(new Rect2d[0]);
MatOfRect2d bbox = new MatOfRect2d();
bbox.fromList(rects);
MatOfInt indices = new MatOfInt();
Dnn.NMSBoxes(bbox, confidences, threshold, nmsThresh, indices);
// Grab Results:
int [] ind = indices.toArray();
float confy = 0;
int classID = -1;
for (int i = 0; i < ind.length; ++i)
{
int idx = ind[i];
classID = clsIds.get(idx);
Float confidence = confs.get(idx);
confy = confidence;
}
//Release the mats
blob.release();
confidences.release();
bbox.release();
indices.release();
mat.release();
return classID + "," + confy;
}
private List<String> getOutputNames(Net net) {
List<String> names = new ArrayList<>();
List<Integer> outLayers = net.getUnconnectedOutLayers().toList();
List<String> layersNames = net.getLayerNames();
outLayers.forEach((item) -> names.add(layersNames.get(item - 1)));
return names;
}
After so many frames the program crashes and gives:
# A fatal error has been detected by the Java Runtime Environment:
#
# SIGSEGV (0xb) at pc=0x00007fda20e7bac1, pid=34593, tid=0x00007fd9ecaed640
#
# JRE version: OpenJDK Runtime Environment (8.0_362-b09) (build 1.8.0_362-8u362-ga-0ubuntu1~22.04-b09)
# Java VM: OpenJDK 64-Bit Server VM (25.362-b09 mixed mode linux-amd64 compressed oops)
# Problematic frame:
# C [libopencv_java3414.so+0xc7bac1] cv::dnn::experimental_dnn_34_v21::Net::Impl::forwardLayer(cv::dnn::experimental_dnn_34_v21::LayerData&)+0xc61