depthMap always initialize with empty values

I have a code that correlates a pixel and a point. I see that are 100 pixels in the image that doesn’t have any point, but all the others are not change and trying to write with cv::Mat.at(i, j).

Here is the code for clarification:

cv::Mat CloudPointProcessor::mapToPixel(const cv::Mat &image, const std::vector<Point> &points) {
    // Initialize depthMap with NaNs, representing uninitialized depth values.
    cv::Mat depthMap(image.rows, image.cols, CV_64F, cv::Scalar(std::numeric_limits<double>::quiet_NaN()));

    // Step 1: Create a matrix with lists of points for each image pixel
	std::vector<std::vector<std::vector<Point>>> pixelPointMatrix(image.rows, std::vector<std::vector<Point>>(image.cols));

    // Image parameters for mapping points to pixels
    double width = 6.17;     // Width of the mapped space in meters
    double height = 2.23;    // Height of the mapped space in meters
    double distance = 2.23;  // Distance for field of view


    double fovX = 2 * atan(width / (2 * distance)) * (180 / M_PI);
    double fovY = 2 * atan(height / (2 * distance)) * (180 / M_PI);


	for (const Point &point : points) {
		// Map point coordinates to pixel positions
		int xPixel = static_cast<int>((point.x / width + 0.5) * image.cols);
		int yPixel = static_cast<int>((point.y / height + 0.5) * image.rows);

		// Check that the calculated pixel is within the image boundaries
		std::cout << "Point (" << point.x << ", " << point.y << ") maps to pixel (" << xPixel << ", " << yPixel << ")\n";

		// Check that the calculated pixel is within the image boundaries
		if (xPixel >= 0 && xPixel < image.cols && yPixel >= 0 && yPixel < image.rows) {
			pixelPointMatrix[yPixel][xPixel].push_back(point);
		} else {
			std::cout << "Warning: Point (" << point.x << ", " << point.y << ") out of bounds.\n";
		}

    }


	long count = 0;
    #pragma omp parallel for collapse(2) reduction (+: count)
    for (int i = 0; i < 10; i++) {
        for (int j = 0; j < 10; j++) {
            const std::vector<Point>& pointsAtPixel = pixelPointMatrix[i][j];
            double minZdistance = 10;

			if(pointsAtPixel.empty()){
				count++;
				minZdistance = 0;
			} else {
				for (const Point &point : pointsAtPixel) {
                	if (point.z < minZdistance) {
	                    minZdistance = point.z;
            		}
            	}
			}

			depthMap.at<double>(i, j) = minZdistance;
        }

    }
    return depthMap;
}

All values, despite the minZdistance being assigned to position i,j are always NaN