Qr detector Opencv.js Polylines from points

, , ,
1

I’m trying to replicate some code from rust/python-bindings to html with js however the points (cv.Mat) seems to be different than the what is returned in these other bindings. For example:

Python

import cv2
cap = cv2.VideoCapture(cv2.CAP_ANY)
gcd = cv2.QRCodeDetector()

while True: 
    ret, frame = cap.read()
    ret_qr, decoded_info, points, _ = gcd.detectAndDecodeMulti(frame) 
    if ret_qr:
        for s, p in zip(decoded_info, points):
            if s:
                print(s)
                color = (0, 255, 0)
            else:
                color = (0, 0, 255)
            frame = cv2.polylines(frame, [p.astype(int)], True, color, 8)
    cv2.imshow("QRCODEscanner", frame)     
    if cv2.waitKey(1) == ord("q"): 
        break
  
cap.release() 
cv2.destroyAllWindows()

Rust

use opencv::{
    Result,
    prelude::*,
    objdetect,
    imgproc,
    highgui,
    types,
    videoio,
    core,
};


fn main() -> Result<()> {
    let mut qr_detector = objdetect::QRCodeDetector::default()?;
    let mut res = types::VectorOfPoint::new();
    let mut camera = videoio::VideoCapture::new(0, videoio::CAP_ANY)?;
    let mut img = Mat::default();
    let mut recqr = Mat::default();
    loop{
        camera.read(&mut img)?;
        let ret = qr_detector.detect_and_decode(&img, &mut res, &mut recqr)?;
        let s = String::from_utf8_lossy(&ret);
        if !res.is_empty() & !s.is_empty() {
            println!("{:?}", res);
            println!("{:?}", s);
            imgproc::polylines(
                &mut img, 
                &res, 
                true, 
                core::Scalar::new(0f64,255f64,0f64,0f64), 
                1,
                1,
                0)?;
        };
        highgui::named_window("QR Code", highgui::WINDOW_NORMAL)?;
        if recqr.size()?.width > 0{
            highgui::imshow("QR Code", &recqr)?;
        }

        highgui::imshow("Frame", &img)?;
        let key = highgui::wait_key(1)?;
        if key == 'q' as i32 {
            break;
        }
    }
    Ok(())
}

These both work fine but when I try on opencv.js in html I get a conflict between points needing to be mat while polylines requiring MatVector and attempts to convert

html+js

<!DOCTYPE html>
<html>
<head>
    <title>Example of Reading a QR Code in OpenCV.js</title>
</head>
<body>
    <video id="video" width="500" height="350" autoplay></video>
    <canvas id="result" width="500" height="350"></canvas>
    <script type="text/javascript">
        let video, canvas, ctx, cvImg;
        // Load OpenCV.js
        let opencvjs = document.createElement("script");
        opencvjs.setAttribute("src", "https://docs.opencv.org/4.9.0/opencv.js");
        opencvjs.onload = function () {
            cv['onRuntimeInitialized'] = () => {
				video = document.getElementById("video");
				canvas = document.getElementById("result");
				ctx = canvas.getContext("2d");
				// Access the camera and start the video feed
				navigator.mediaDevices.getUserMedia({ video: true })
					.then(function (stream) {
						video.srcObject = stream;
						video.play();
						setInterval(captureFrame, 1000 / 30);
					})
					.catch(function (err) {
						console.log("An error occurred: " + err);
					});
            };
        }
        document.body.appendChild(opencvjs);

		function captureFrame() {
			// Draw the video frame onto the canvas
			ctx.drawImage(video, 0, 0, canvas.width, canvas.height);
			cvImg = cv.imread(canvas);

			// Create QR code detector and result variables outside the loop
			let qcd = new cv.QRCodeDetector();
			let decodedInfo = new cv.StringVector(); // Store decoded information
			let points = new cv.Mat();
			let straightQrCode = new cv.MatVector();
			let success = qcd.detectAndDecodeMulti(cvImg, decodedInfo, points, straightQrCode);
			// Draw detection only if a QR code is found
			if (success) {
                //points needs to be matvector but looks like reformat needed
				cv.polylines(cvImg, points, true, new cv.Scalar(255,0,0),4);
				cv.putText(cvImg, decodedInfo.get(0).toString(), new cv.Point(10, 20), cv.FONT_HERSHEY_SIMPLEX, 0.5, new cv.Scalar(0, 255, 0, 255), 1, cv.LINE_AA);
			}

			// Display the processed frame on the canvas
			cv.imshow('result', cvImg);

			// Clean up the OpenCV objects
			qcd.delete();
			points.delete();
			straightQrCode.delete();
		}
    </script>
</body>
</html>

Any help would be greatly appreciated.

3

As an update it seems that the datatype was issue and needed data32F; this appears to be working but looking to see if more optimal approach is needed:

<!DOCTYPE html>
<html>
<head>
    <title>Example of Reading a QR Code in OpenCV.js</title>
</head>
<body>
    <canvas id="video-canvas" width="500" height="350"></canvas>
    <script type="text/javascript">
        let videoCanvas, videoCtx, cvImg;
        let opencvjs;

        // Load OpenCV.js asynchronously
        async function loadOpenCVJS() {
            return new Promise((resolve, reject) => {
                opencvjs = document.createElement("script");
                opencvjs.setAttribute("src", "https://docs.opencv.org/4.9.0/opencv.js");
                opencvjs.onload = () => {
                    cv['onRuntimeInitialized'] = () => {
                        resolve();
                    };
                };
                opencvjs.onerror = () => {
                    reject(new Error("Failed to load OpenCV.js"));
                };
                document.body.appendChild(opencvjs);
            });
        }

        // Initialize the video and canvas
        async function initializeVideoAndCanvas() {
            try {
                await loadOpenCVJS();
                videoCanvas = document.getElementById("video-canvas");
                videoCtx = videoCanvas.getContext("2d");

                // Access the camera and start the video feed
                const stream = await navigator.mediaDevices.getUserMedia({ video: true });
                const video = document.createElement("video");
                video.srcObject = stream;
                video.play();
                setInterval(async () => await captureFrame(video), 1000 / 60);
            } catch (err) {
                console.error("An error occurred:", err);
            }
        }

        // Capture and process the video frame
        async function captureFrame(video) {
            // Draw the video frame onto the video canvas
            videoCtx.drawImage(video, 0, 0, videoCanvas.width, videoCanvas.height);

            // Create a new matrix from the video canvas
            cvImg = cv.imread(videoCanvas);

            // Create QR code detector and result variables
            const qcd = new cv.QRCodeDetector();
            const decodedInfo = new cv.StringVector();
            const points = new cv.Mat();
            const straightQrCode = new cv.MatVector();
            const success = qcd.detectAndDecodeMulti(cvImg, decodedInfo, points, straightQrCode);

            // Draw detection only if a QR code is found
            if (success) {
                for (let i = 0; i < decodedInfo.size(); i++) {
                    const s = decodedInfo.get(i);
                    const p = points.row(i).data32F;
                    const pointMatVector = new cv.MatVector();
                    pointMatVector.push_back(cv.matFromArray(4, 1, cv.CV_32SC2, p));
                    cv.polylines(cvImg, pointMatVector, true, new cv.Scalar(0, 255, 0, 255), 4,1, 0);
                    cv.putText(cvImg, s.toString(), new cv.Point(10, 20), cv.FONT_HERSHEY_SIMPLEX, 0.5, new cv.Scalar(0, 255, 0, 255), 1, cv.LINE_AA);
                }

                // Display the processed frame on the video canvas
                cv.imshow('video-canvas', cvImg);
            } else {
                // Display the original video frame on the canvas
                cv.imshow('video-canvas', cvImg);
            }

            // Clean up the OpenCV objects
            qcd.delete();
            points.delete();
            straightQrCode.delete();
            cvImg.delete();
        }

        // Initialize the video and canvas
        initializeVideoAndCanvas();
    </script>
</body>
</html>