CUDA Decoder issues

Python
, ,
1

I am trying to build OpenCV with CUDA support, it worked but now I wanted to add cudacodec as well, however I am having issues with installing the NVIDIA Video Codec SDK and get errors for the headers and libraries. I am building it on a Modal Image (similar to docker) for Cloud GPU use, so if anyone has any suggestions on how to get it working I’d be thankful. I even tried getting the .h files from PyNvVideoCodec.

image = (
    modal.Image.debian_slim(python_version="3.10")
    .apt_install(
        [
            "wget",
            "unzip",
            "ffmpeg",
            "libgl1-mesa-glx",
            "libglib2.0-0",
            "build-essential",
            "cmake",
            "pkg-config",
            "libgtk-3-dev",
            "libavcodec-dev",
            "libavformat-dev",
            "libswscale-dev",
            "libv4l-dev",
            "libxvidcore-dev",
            "libx264-dev",
            "ninja-build",
            "git",
            "libatlas-base-dev",  # Fix BLAS/LAPACK issue
            "libgflags-dev",  # Fix gflags issue
            "libgoogle-glog-dev",  # Fix glog issue
            "libhdf5-dev",  # Recommended for OpenCV ML/DNN modules
            "ca-certificates",
        ]
    )
    .pip_install(
        [
            "numpy",  # Fix missing NumPy error
            "moviepy==1.0.3",
            "Pillow",
            "spacy==3.7.2",
            "pandas",
        ]
    )
    .run_commands(
        # CUDA Installation
        "wget https://developer.download.nvidia.com/compute/cuda/12.1.0/local_installers/cuda_12.1.0_530.30.02_linux.run",
        "sh cuda_12.1.0_530.30.02_linux.run --toolkit --silent --override",
        # cuDNN Installation
        "mkdir -p /cudnn",
        "cd /cudnn",
        "wget https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-x86_64/cudnn-linux-x86_64-8.9.7.29_cuda12-archive.tar.xz -O cudnn.tar.xz --no-check-certificate",
        "tar -xf cudnn.tar.xz",
        "cp cudnn-linux-x86_64-8.9.7.29_cuda12-archive/include/cudnn*.h /usr/local/cuda/include",
        "cp -P cudnn-linux-x86_64-8.9.7.29_cuda12-archive/lib/libcudnn* /usr/local/cuda/lib64",
        "chmod a+r /usr/local/cuda/include/cudnn*.h /usr/local/cuda/lib64/libcudnn*",
        # NVIDIA VIDEO HEADERS
        "wget --content-disposition 'https://api.ngc.nvidia.com/v2/resources/org/nvidia/pynvvideocodec/1.0.2/files?redirect=true&path=PyNvVideoCodec_1.0.2.zip' -O PyNvVideoCodec_1.0.2.zip && unzip PyNvVideoCodec_1.0.2.zip -d ~/PyNvVideoCodec && mkdir -p /usr/local/include/nvidia-video-codec-sdk && cp -r ~/PyNvVideoCodec/src/VideoCodecSDKUtils/Interface/*.h /usr/local/include/nvidia-video-codec-sdk/",
        # OpenCV Installation
        "git clone https://github.com/opencv/opencv.git",
        "git clone https://github.com/opencv/opencv_contrib.git",
        "mkdir -p opencv/build",
        """cd opencv/build && cmake -GNinja \
                -D CMAKE_BUILD_TYPE=RELEASE \
                -D CMAKE_INSTALL_PREFIX=/usr/local \
                -D OPENCV_EXTRA_MODULES_PATH=../../opencv_contrib/modules \
                -D WITH_CUDA=ON \
                -D CUDA_ARCH_BIN=7.5 8.9 \
                -D CUDA_ARCH_PTX=8.9 \
                -D WITH_CUDNN=ON \
                -D CUDNN_INCLUDE_DIR=/usr/local/cuda/include \
                -D CUDNN_LIBRARY=/usr/local/cuda/lib64/libcudnn.so \
                -D OPENCV_DNN_CUDA=ON \
                -D WITH_NVCUVID=ON \
                -D WITH_NVCUVENC=ON \
                -D BUILD_OPENCV_CUDACODEC=ON \
                -D CUDA_nvcuvid_INCLUDE_DIR=/usr/local/include/nvidia-video-codec-sdk \
                -D CUDA_nvcuvid_LIBRARY=/usr/lib/x86_64-linux-gnu/libnvcuvid.so.1 \
                -D CUDA_nvidia_encode_LIBRARY=/usr/lib/x86_64-linux-gnu/libnvidia-encode.so.1 \                
                -D ENABLE_FAST_MATH=1 \
                -D CUDA_FAST_MATH=1 \
                -D WITH_CUBLAS=1 \
                -D WITH_V4L=ON \
                -D WITH_FFMPEG=ON \
                -D BUILD_opencv_python3=ON \
                -D PYTHON_EXECUTABLE=/usr/local/bin/python \
                -D OPENCV_ENABLE_NONFREE=ON \
                -D BUILD_EXAMPLES=OFF \
                -D BUILD_DOCS=OFF \
                -D BUILD_PERF_TESTS=OFF \
                -D BUILD_TESTS=OFF \
                -D INSTALL_PYTHON_EXAMPLES=OFF \
                -D INSTALL_C_EXAMPLES=OFF \
                ..""",
        "cd opencv/build && ninja",
        "cd opencv/build && ninja install",
        "ldconfig",
        "python -m spacy download es_core_news_sm",
        "export PATH=/usr/local/cuda/bin:$PATH && export CUDA_ROOT=/usr/local/cuda && export CUDA_INC_DIR=/usr/local/cuda/include && export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH && pip install 'pycuda>=2022.1'",
    )
)
2

I managed to add the NVIDIA Video Codec SDK, but when I copy the files it needs I get an error:

CMake Error: The following variables are used in this project, but they are set to NOTFOUND.
Please set them or make sure they are set and tested correctly in the CMake files:
CUDA_CUDA_LIBRARY (ADVANCED)
    linked by target "opencv_cudacodec" in directory /opencv_contrib/modules/cudacodec

Using this setup, where nvsdk contains the necessary .h and .so files from the SDK

image = (
    modal.Image.debian_slim(python_version="3.10")
    .apt_install(
        [
            "wget",
            "unzip",
            "ffmpeg",
            "libgl1-mesa-glx",
            "libglib2.0-0",
            "build-essential",
            "cmake",
            "pkg-config",
            "libgtk-3-dev",
            "libavcodec-dev",
            "libavformat-dev",
            "libswscale-dev",
            "libv4l-dev",
            "libxvidcore-dev",
            "libx264-dev",
            "ninja-build",
            "git",
            "libatlas-base-dev",  # Fix BLAS/LAPACK issue
            "libgflags-dev",  # Fix gflags issue
            "libgoogle-glog-dev",  # Fix glog issue
            "libhdf5-dev",  # Recommended for OpenCV ML/DNN modules
            "ca-certificates",
        ]
    )
    .pip_install(
        [
            "numpy",  # Fix missing NumPy error
            "moviepy==1.0.3",
            "Pillow",
            "spacy==3.7.2",
            "pandas",
        ]
    )
    .add_local_dir(
        "data/nvsdk",
        remote_path="/nvsdk",
        copy=True,
    )
    .run_commands(
        # CUDA Installation
        "wget https://developer.download.nvidia.com/compute/cuda/12.1.0/local_installers/cuda_12.1.0_530.30.02_linux.run",
        "sh cuda_12.1.0_530.30.02_linux.run --toolkit --silent --override",
        # cuDNN Installation
        "mkdir -p /cudnn",
        "cd /cudnn",
        "wget https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-x86_64/cudnn-linux-x86_64-8.9.7.29_cuda12-archive.tar.xz -O cudnn.tar.xz --no-check-certificate",
        "tar -xf cudnn.tar.xz",
        "cp cudnn-linux-x86_64-8.9.7.29_cuda12-archive/include/cudnn*.h /usr/local/cuda/include",
        "cp -P cudnn-linux-x86_64-8.9.7.29_cuda12-archive/lib/libcudnn* /usr/local/cuda/lib64",
        "chmod a+r /usr/local/cuda/include/cudnn*.h /usr/local/cuda/lib64/libcudnn*",
        "cp /nvsdk/*.h /usr/local/cuda-12.1/targets/x86_64-linux/include/",
        "cp /nvsdk/*.so /usr/local/cuda-12.1/targets/x86_64-linux/lib/",
        # OpenCV Installation
        "git clone https://github.com/opencv/opencv.git",
        "git clone https://github.com/opencv/opencv_contrib.git",
        "mkdir -p opencv/build",
        """cd opencv/build && cmake -GNinja \
                -D CMAKE_BUILD_TYPE=RELEASE \
                -D CMAKE_INSTALL_PREFIX=/usr/local \
                -D OPENCV_EXTRA_MODULES_PATH=../../opencv_contrib/modules \
                -D WITH_CUDA=ON \
                -D CUDA_ARCH_BIN=7.5 8.9 \
                -D CUDA_ARCH_PTX=8.9 \
                -D WITH_CUDNN=ON \
                -D CUDNN_INCLUDE_DIR=/usr/local/cuda/include \
                -D CUDNN_LIBRARY=/usr/local/cuda/lib64/libcudnn.so \
                -D OPENCV_DNN_CUDA=ON \
                -D WITH_NVCUVID=ON \
                -D WITH_NVCUVENC=ON \
                -D BUILD_OPENCV_CUDACODEC=ON \
                -D ENABLE_FAST_MATH=1 \
                -D CUDA_FAST_MATH=1 \
                -D WITH_CUBLAS=1 \
                -D WITH_V4L=ON \
                -D WITH_FFMPEG=ON \
                -D BUILD_opencv_python3=ON \
                -D PYTHON_EXECUTABLE=/usr/local/bin/python \
                -D OPENCV_ENABLE_NONFREE=ON \
                -D BUILD_EXAMPLES=OFF \
                -D BUILD_DOCS=OFF \
                -D BUILD_PERF_TESTS=OFF \
                -D BUILD_TESTS=OFF \
                -D INSTALL_PYTHON_EXAMPLES=OFF \
                -D INSTALL_C_EXAMPLES=OFF \
                ..""",
        "cd opencv/build && ninja",
        "cd opencv/build && ninja install",
        "ldconfig",
        "python -m spacy download es_core_news_sm",
        "export PATH=/usr/local/cuda/bin:$PATH && export CUDA_ROOT=/usr/local/cuda && export CUDA_INC_DIR=/usr/local/cuda/include && export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH && pip install 'pycuda>=2022.1'",
    )

If I just run it without copying the files, it just fails building cudacodec but the install works

CMake Warning at /opencv_contrib/modules/cudacodec/CMakeLists.txt:30 (message):
  cudacodec::VideoWriter requires Nvidia Video Codec SDK.  Please resolve
  dependency or disable WITH_NVCUVENC=OFF

Looking at the other posts I thought maybe you had some ideas on what to do @cudawarped ?

3

I have not seen that error before CMake can’t find the CUDA driver library. If its easy I would try building with -DENABLE_CUDA_FIRST_CLASS_LANGUAGE=ON to see if CMake can automatically find it. With this flag off CMake uses a really old script to find the CUDA toolkit. You could also try manually passing the location to libcuda.so which is probably /usr/lib/x86_64-linux-gnu/libcuda.so using CUDA_CUDA_LIBRARY when CUDA is not a first class language.

Don’t do this. When you run your program it will crash if this directory is on the path because the these stub libraries can be used instead of the ones which come with the driver (e.g. /usr/lib/x86_64-linux-gnu/libnvcuvid.so.1.

4

Thanks, I got it working now by doing this:

    .add_local_dir(
        "data/nvsdk",
        remote_path="/nvsdk",
        copy=True,
    )
    .run_commands(
        # CUDA Installation
        "wget https://developer.download.nvidia.com/compute/cuda/12.1.0/local_installers/cuda_12.1.0_530.30.02_linux.run",
        "sh cuda_12.1.0_530.30.02_linux.run --toolkit --silent --override",
        # cuDNN Installation
        "mkdir -p /cudnn",
        "cd /cudnn",
        "wget https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-x86_64/cudnn-linux-x86_64-8.9.7.29_cuda12-archive.tar.xz -O cudnn.tar.xz --no-check-certificate",
        "tar -xf cudnn.tar.xz",
        "cp cudnn-linux-x86_64-8.9.7.29_cuda12-archive/include/cudnn*.h /usr/local/cuda/include",
        "cp -P cudnn-linux-x86_64-8.9.7.29_cuda12-archive/lib/libcudnn* /usr/local/cuda/lib64",
        "chmod a+r /usr/local/cuda/include/cudnn*.h /usr/local/cuda/lib64/libcudnn*",
        "cp /nvsdk/include/*.h /usr/local/cuda-12.1/targets/x86_64-linux/include/",
        # OpenCV Installation
        "git clone https://github.com/opencv/opencv.git",
        "git clone https://github.com/opencv/opencv_contrib.git",
        "mkdir -p opencv/build",
        """cd opencv/build && cmake -GNinja \
                -D CMAKE_BUILD_TYPE=RELEASE \
                -D CMAKE_INSTALL_PREFIX=/usr/local \
                -D OPENCV_EXTRA_MODULES_PATH=../../opencv_contrib/modules \
                -D WITH_CUDA=ON \
                -D CUDA_ARCH_BIN=7.5 8.9 \
                -D CUDA_ARCH_PTX=8.9 \
                -D WITH_CUDNN=ON \
                -D CUDNN_INCLUDE_DIR=/usr/local/cuda/include \
                -D CUDNN_LIBRARY=/usr/local/cuda/lib64/libcudnn.so \
                -D OPENCV_DNN_CUDA=ON \
                -D CUDA_CUDA_LIBRARY=/usr/local/cuda-12.1/targets/x86_64-linux/lib/stubs/libcuda.so \
                -D CUDA_nvcuvid_LIBRARY=/nvsdk/lib/libnvcuvid.so \
                -D CUDA_nvcuvenc_LIBRARY=/nvsdk/lib/libnvcuvid.so \
                -D WITH_NVCUVID=ON \
                -D WITH_NVCUVENC=ON \
                -D BUILD_OPENCV_CUDACODEC=ON \
                -D ENABLE_FAST_MATH=1 \
                -D CUDA_FAST_MATH=1 \
                -D WITH_CUBLAS=1 \
                -D WITH_V4L=ON \
                -D WITH_FFMPEG=ON \
                -D BUILD_opencv_python3=ON \
                -D PYTHON_EXECUTABLE=/usr/local/bin/python \
                -D OPENCV_ENABLE_NONFREE=ON \
                -D BUILD_EXAMPLES=OFF \
                -D BUILD_DOCS=OFF \
                -D BUILD_PERF_TESTS=OFF \
                -D BUILD_TESTS=OFF \
                -D INSTALL_PYTHON_EXAMPLES=OFF \
                -D INSTALL_C_EXAMPLES=OFF \
                ..""",
        "cd opencv/build && ninja",
        "cd opencv/build && ninja install",
        "ldconfig",
        "python -m spacy download es_core_news_sm",
        "export PATH=/usr/local/cuda/bin:$PATH && export CUDA_ROOT=/usr/local/cuda && export CUDA_INC_DIR=/usr/local/cuda/include && export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH && pip install 'pycuda>=2022.1'",
    )
)

However, the decoding speed is about the same compared to cv2.VideoCapture, is this normal?

1 Like
5

It depends on the codec/resolution/etc. One thing to try is to increase the number of decode surfaces, e.g.

cv::cudacodec::VideoReaderInitParams params;
params.minNumDecodeSurfaces = 20;
cv::Ptr<cv::cudacodec::VideoReader> reader = cv::cudacodec::createVideoReader(inputFile, {}, params);
1 Like