It depends on the codec/resolution/etc. One thing to try is to increase the number of decode surfaces, e.g.
cv::cudacodec::VideoReaderInitParams params;
params.minNumDecodeSurfaces = 20;
cv::Ptr<cv::cudacodec::VideoReader> reader = cv::cudacodec::createVideoReader(inputFile, {}, params);