Hi, I am using opencv js for the first time for doing video processing on the web, I have successfully set up open cv 4.5.0 https://docs.opencv.org/4.5.0/opencv.js in my Angular project.
Now the problem is, I have a 10 second mp4 video with 10 FPS, from which I want to extract all the frames and paint them into a canvas.
export async function extractFramesFromVideoV2(
cv,
media: DATA_DRIVE,
metaData: VIDEO_METADATA,
service: VideoOperatorService,
frameSize: FRAME_SIZE = { w: 50, h: 50 }
) {
const openCVInstance = cv;
const video = document.getElementsByTagName('video')[0]
video.height=metaData.height
video.width=metaData.width
const src = new openCVInstance.Mat(video.height, video.width, openCVInstance .CV_8UC4);
const dst = new openCVInstance.Mat(video.height, video.width, openCVInstance .CV_8UC1);
const cap = new openCVInstance.VideoCapture(video);
const canvas = createCanvas()
const context = canvas.getContext("2d");
canvas.width = video.width;
canvas.height = video.height;
let i=0
function processVideo() {
try {
const canvas = createCanvas() // this is a method which creates a blank canvas for painting the frames
let begin = Date.now();
// start processing.
cap.read(src);
canvas.toBlob((blob)=>{
const rawUrl = URL.createObjectURL(blob);
openCVInstance.imshow(canvas, dst );
console.log(rawUrl)
i++
})
// schedule the next one.
let delay = 1 / metaData.fps //- (Date.now() - begin);
setTimeout(processVideo, delay);
} catch (err) {
console.log(err)
}
};
}
The blob url that comes up from the above code, is always a black image. Not sure where I am doing wrong? Can someone please help me on this.
The video width is 1920 height is 1280, fps is 10
Hi @berak@matti.vuori thank you for your reply, really appreciate it. The way you suggested has worked, I have modified the code as per your suggestions. It seems the dst variable is not needed. I am now drawing into the canvas with the source matrix. It partially works. But now the problem is all the blob url that this is generating are basically the first frame of the video. How do I move to the next frame of the video with this setTimeout implementation?
Or should I have to do something with video.currentTime increment to move to the next frame after each call to the process video is complete , so that opencv understands that it needs to read next frame on every call to process video.
How does even opencvjs understand to move to next frame and how it understands that the video is now complete reading and the settimeout should terminate?
What am I doing wrong here?
export async function extractFramesFromVideoV2(
cv,
media: DATA_DRIVE,
metaData: VIDEO_METADATA,
service: VideoOperatorService,
frameSize: FRAME_SIZE = { w: 50, h: 50 }
) {
const openCVInstance = cv;
const video = document.getElementsByTagName("video")[0];
video.height = metaData.height;
video.width = metaData.width;
const canvas = createCanvas();
canvas.width = video.width;
canvas.height = video.height;
const src = new openCVInstance.Mat(video.height, video.width, openCVInstance.CV_8UC4);
const cap = new openCVInstance.VideoCapture(video);
let i = 0;
function processVideo() {
try {
let delay=0
const begin = Date.now();
// start processing.
cap.read(src); // read frame into the source matrix
openCVInstance.imshow(canvas, src); // draw the current source frame into the canvas
canvas.toBlob((blob) => {
const rawUrl = URL.createObjectURL(blob);
console.log(rawUrl);
});
if(metaData.fps===30){
// per docs if the fps is 30, the delay should be 1000/fps - processing time of each frame.
delay = 1000 / metaData.fps - (Date.now() - begin);
}
else{
delay = 1000 / metaData.fps
}
i++;
// schedule the next one.
setTimeout(processVideo, delay);
} catch (err) {
console.log(err);
}
}
// schedule the first one.
setTimeout(processVideo, 0);
}
@berak I was finally able to implement the opencv js version of implementing on how to capture all frames from a video source URL. Here is the solution I made:
const videoObjectUrl = URL.createObjectURL(metaData.blob);
const video = createVideo(); // creates a virtual VIDEO element in DOM
video.src = videoObjectUrl;
let seekComplete;
video.onseeked = async (event) => {
if (seekComplete) seekComplete();
/**
* The seeked event is fired when a seek operation completed,
* the current playback position has changed,
* and the Boolean seeking attribute is changed to false.
*/
};
video.onerror = (event) => {
// do not process further and stop
return;
};
//This workaround is needed to make sure the video is available to buffer. This is a //chrome bug and has to be there for seeking the video to the next second.
// workaround chromium metadata bug //(https://stackoverflow.com/q/38062864/993683)
while (
(video.duration === Infinity || isNaN(video.duration)) &&
video.readyState < 2
) {
await new Promise((r) => setTimeout(r, 1000));
}
let currentTime = 0;
let outputVideoFrame: VIDEO_FRAME = null;
const frame_name = "frame";
let i = 0;
let streaming = true;
const openCVInstance = cv;
// OpenCV implementation
if (openCVInstance) {
// anonymize the video virtually in DOM so that it does not get tainted with //cross origin blobs and hence keeps on painting the received video frames.
// Refer https://developer.mozilla.org/en-US/docs/Web/HTML/Attributes/crossorigin
video.crossOrigin = "Anonymous";
video.height = metaData.height;
video.width = metaData.width;
const canvas = createCanvas();
canvas.width = video.width;
canvas.height = video.height;
const sourceMatrix = new openCVInstance.Mat(
video.height,
video.width,
openCVInstance.CV_8UC4
);
const destinationMatrix = new openCVInstance.Mat(
video.height,
video.width,
openCVInstance.CV_8UC4
);
const openCVCaptureInstance = new openCVInstance.VideoCapture(video);
const intervalPerFrame = 1 / metaData.fps; // 0.1 sec
const processingDelayPerFrame = 1000 / metaData.fps;
currentTime = intervalPerFrame; // reset the currentTime to the first interval so that the video starts capturing from the 0.1 second(or exact first frame), otherwise if it starts capturing from 0 seconds, it often takes the last frame into account first and hence messes up the sequence of frames.
async function processVideo() {
try {
if (!streaming) {
// shut down the process if streaming is complete.
return;
}
let delay = 0;
const begin = metaData.fps === 30 ? Date.now() : null;
// start processing.
video.currentTime = currentTime; // seek video to next frame and wait until seeked to current time.
await once(video, "seeked");
/***
We use read (image) to get each frame of the video.
For performance reasons, the image material should be constructed with cv.CV_8UC4 type and same size as the video.
OpenCV material types can be read the following:
CV_
8U: Unsigned int 8-bit
C4: Four channels.
Thus mRgba = new Mat(height, width, CvType.CV_8UC4); creates a Matrix with four color channels and values in the range 0 to 255
These channels are the colour components. E.g. an ordinary RGB image has 3 channels, an RGBA (RGB + alpha) image has four channels, and a CMYK image has four channels.
Refer here https://docs.opencv.org/2.4/doc/tutorials/core/mat_the_basic_image_container/mat_the_basic_image_container.html#creating-a-mat-object-explicitly
for more explanation as to why we need to create the image material instance as a fourier channel instance.***/
openCVCaptureInstance.read(sourceMatrix); // read frame into the source matrix
sourceMatrix.copyTo(destinationMatrix);
openCVInstance.imshow(canvas, destinationMatrix); // draw the current source frame matrix into the destination frame matrix
i++;
canvas.toBlob((blob) => {
const rawUrl = URL.createObjectURL(blob);
currentTime += intervalPerFrame;
console.log(rawUrl, i);
if (i == metaData.numberOfFrames) {
// streaming of video completes
console.log("stream done");
streaming = false;
// clear the material from memory on stream complete
console.log("Clear memory");
sourceMatrix.delete();
destinationMatrix.delete();
video.remove();
} else {
if (streaming) {
// per docs if the fps is 30, the delay should be 1000/fps - processing time of each frame.
metaData.fps === 30
? (delay = 1000 / metaData.fps - (Date.now() - begin))
: (delay = processingDelayPerFrame);
setTimeout(processVideo, delay); // schedule the next frame and so on
}
}
});
} catch (err) {
console.log(err); //silently log error and move ahead
}
}
// schedule the first one.
setTimeout(processVideo, 0);
return;
}
This solution has worked for me and spits out 100 frames for a 10 fps 10 second video in 2 seconds.
Hi everyone, the video I am intending to capture frames from is coming from a different URL, and hence while painting to the canvas it is throwing me a security error
ERROR Error: Uncaught (in promise): SecurityError: Failed to execute 'getImageData' on 'CanvasRenderingContext2D': The canvas has been tainted by cross-origin data.
Error: Failed to execute 'getImageData' on 'CanvasRenderingContext2D': The canvas has been tainted by cross-origin data.
Here is the code I have:
export async function extractFramesFromVideoCV2(cv, media?: DATA_DRIVE,
metaData?: VIDEO_METADATA,
service?: VideoOperatorService,
frameSize: FRAME_SIZE = { w: 50, h: 50 },) {
const start = new Date().getTime();
const video:HTMLVideoElement=createVideo()
video.src=metaData.url
let seekComplete;
video.onseeked = async (event) => {
if (seekComplete) seekComplete();
/**
* The seeked event is fired when a seek operation completed,
* the current playback position has changed,
* and the Boolean seeking attribute is changed to false.
*/
};
// workaround chromium metadata bug (https://stackoverflow.com/q/38062864/993683)
while (
(video.duration === Infinity || isNaN(video.duration)) &&
video.readyState < 2
) {
await new Promise((r) => setTimeout(r, 1000));
}
video.crossOrigin = "Anonymous"; // here I set the cross origin as anonymous but //still getting the same error
video.height = metaData.height;
video.width = metaData.width;
const sourceMatrix = new cv.Mat(
video.height,
video.width,
cv.CV_8UC4
);
const destinationMatrix = new cv.Mat(
video.height,
video.width,
cv.CV_8UC4
);
const openCVCaptureInstance = new cv.VideoCapture(video);
const canvas = createCanvas();
canvas.width = video.width;
canvas.height = video.height;
const FPS= metaData.fps
const intervalPerFrame = 1 / FPS; // 0.1 sec
const processingDelayPerFrame = 1000 / FPS;
let currentTime = intervalPerFrame;
let streaming=true;
let i=0
async function processVideo() {
let begin = Date.now();
if (!streaming) {
// shut down the process if streaming is complete.
return;
}
// start processing.
video.currentTime = currentTime; // seek video to next frame and wait until seeked to current time.
await once(video, "seeked");
openCVCaptureInstance.read(sourceMatrix);
sourceMatrix.copyTo(destinationMatrix);
cv.imshow(canvas, destinationMatrix);
i++;
canvas.toBlob((blob) => {
const rawUrl = URL.createObjectURL(blob);
currentTime += intervalPerFrame;
console.log(rawUrl, i);
if (i == metaData.numberOfFrames) {
// streaming of video completes
console.log("stream done");
streaming = false;
// clear the material from memory on stream complete
console.log("Clear memory");
sourceMatrix.delete();
destinationMatrix.delete();
video.remove();
} else {
if (streaming) {
let delay = processingDelayPerFrame - (Date.now() - begin);
// schedule next one.
setTimeout(processVideo, delay);
}
}
});
}
// schedule first one.
setTimeout(processVideo, 0);
}
However, for videos on the same domain, the code works perfect. Pls help @matti.vuori@berak
@berak hey I understand you are just into computer vision, but have you not encountered this scenario ? Ok can you clarify me one thing about the VideoCapture method in opencv js ? It is very unclear with the docs. The way we use opencv in python is fairly easier but in opencvjs version it’s very unclear. In the web version implementation, It does not simply capture all the frames if video is not seeked, but in opencv python it captures all frames while running a while loop over the videocapture instance. Why are both so different implementations? There are no sufficient docs on opencv js as well.