Chroma subsampling errors in chromakeying

C++
1

Hello, I need to remove chroma subsampling errors in my code (it have to be smoother on contours and more accurate). I want proccesing of the mask to be in function errorKeying, then i use it in keying function to merge with background and my movie. i wrote a code in error keying but it apperas to be not very accurate. Do u have any suggestions?

Mat errorKeying(Mat mask)
{
    Mat edges;
    Canny(mask, edges, 30, 70); 

    Mat blurredMask;
    GaussianBlur(mask, blurredMask, Size(0, 0), 2); 

    Mat result = mask.clone();

    int influenceSize = 10; 
    int halfInfluenceSize = influenceSize / 2;

    for (int y = 0; y < mask.rows; ++y)
    {
        for (int x = 0; x < mask.cols; ++x)
        {
            if (edges.at<uchar>(y, x) > 0) 
            {
                int startY = max(0, y - halfInfluenceSize);
                int endY = min(mask.rows - 1, y + halfInfluenceSize);
                int startX = max(0, x - halfInfluenceSize);
                int endX = min(mask.cols - 1, x + halfInfluenceSize);

                uchar value = 0;
                int count = 0;

                for (int i = startY; i <= endY; ++i)
                {
                    for (int j = startX; j <= endX; ++j)
                    {
                        if (edges.at<uchar>(i, j) > 0)
                        {
                            value += blurredMask.at<uchar>(i, j);
                            count++;
                        }
                    }
                }

                if (count > 0)
                {
                    value /= count;
                    if (value < 128)
                        result.at<uchar>(y, x) = 0; 
                    else
                        result.at<uchar>(y, x) = 255; 
                }
            }
        }
    }

    return result;
}

void keying(VideoCapture& video, Mat& img, vector<int>& col)
{
    Info();

    Mat frame, keyingFrame, hsvFrame, mask, resizedMask;

    video.read(frame);
    resize(frame, frame, Size(sizex, sizey));
    resize(img, img, frame.size(), 0, 0, INTER_CUBIC);

    Scalar lower(col[0], col[2], col[4]);
    Scalar upper(col[1], col[3], col[5]);

    cvtColor(frame, hsvFrame, COLOR_BGR2HSV);

    inRange(hsvFrame, lower, upper, mask);

    Mat smoothedMask = errorKeying(mask);

    resize(mask, resizedMask, frame.size(), 0, 0, INTER_CUBIC);

    frame.copyTo(keyingFrame);
    keyingFrame.setTo(Scalar(0, 0, 0), smoothedMask);
    bitwise_or(keyingFrame, img, keyingFrame, smoothedMask);

    imshow("Wykluczowany obraz", keyingFrame);
    imshow("Oryginalny obraz", frame);
}

I want to remove those green edges after keying
obraz_2023-07-03_221633412