FFmpeg for video (v1.6)

1. Overview

The PPU SDK is compatible with the NVIDIA Video Codec SDK, enabling you to directly use FFmpeg plugins like cuvid, nvenc, and libnpp for hardware acceleration without any code modifications. The following FFmpeg versions have been verified:

2. Build from source

FFmpeg: https://github.com/FFmpeg/FFmpeg

nv-codec-headers: https://github.com/FFmpeg/nv-codec-headers

This document provides build instructions for FFmpeg 7.0 and FFmpeg 6.1. Before you begin, install the PPU SDK and configure the runtime environment by running the envsetup.sh script.

2.1. Prepare FFmpeg source code

setup_ffmpeg.sh:

## Clone nv-codec-headers version 12.2 to match FFmpeg 7.0
git clone --branch n12.2.72.0 --depth 1 https://github.com/FFmpeg/nv-codec-headers
cp nv-codec-headers/ffnvcodec.pc.in nv-codec-headers/ffnvcodec.pc
    
## Download FFmpeg
wget https://ffmpeg.org/releases/ffmpeg-7.0.1.tar.gz --no-check-certificate
tar -xzvf ffmpeg-7.0.1.tar.gz

Patch 1 (MUST):

The build fails because some CUDA runtime APIs are unsupported. This patch removes their definitions to ensure a successful build. For hardware-accelerated scaling, use the filters provided by libnpp.

Replace the dynlink_loader.h file in the nv-codec-headers/include/ffnvcodec/ directory with the provided file.

This patch effectively comments out the following definitions:

    // tcuGLGetDevices_v2 *cuGLGetDevices;
    // tcuGraphicsGLRegisterImage *cuGraphicsGLRegisterImage;
    // tcuGraphicsUnregisterResource *cuGraphicsUnregisterResource;
    // tcuGraphicsMapResources *cuGraphicsMapResources;
    // tcuGraphicsUnmapResources *cuGraphicsUnmapResources;
    // tcuGraphicsSubResourceGetMappedArray *cuGraphicsSubResourceGetMappedArray;
    // tcuGraphicsResourceGetMappedPointer *cuGraphicsResourceGetMappedPointer;

    ... ...

    // tcuArrayCreate *cuArrayCreate;
    // tcuArray3DCreate *cuArray3DCreate;
    // tcuArrayDestroy *cuArrayDestroy;

    // tcuEGLStreamProducerConnect *cuEGLStreamProducerConnect;
    // tcuEGLStreamProducerDisconnect *cuEGLStreamProducerDisconnect;
    // tcuEGLStreamConsumerDisconnect *cuEGLStreamConsumerDisconnect;
    // tcuEGLStreamProducerPresentFrame *cuEGLStreamProducerPresentFrame;
    // tcuEGLStreamProducerReturnFrame *cuEGLStreamProducerReturnFrame;

... ...
    // LOAD_SYMBOL(cuDevicePrimaryCtxRelease, tcuDevicePrimaryCtxRelease, "cuDevicePrimaryCtxRelease");
    LOAD_SYMBOL(cuDevicePrimaryCtxRelease, tcuDevicePrimaryCtxRelease, "cuDevicePrimaryCtxRelease_v2");
    // LOAD_SYMBOL(cuDevicePrimaryCtxSetFlags, tcuDevicePrimaryCtxSetFlags, "cuDevicePrimaryCtxSetFlags");
    LOAD_SYMBOL(cuDevicePrimaryCtxSetFlags, tcuDevicePrimaryCtxSetFlags, "cuDevicePrimaryCtxSetFlags_v2");
    
    ... ...
    // LOAD_SYMBOL(cuDevicePrimaryCtxReset, tcuDevicePrimaryCtxReset, "cuDevicePrimaryCtxReset");
    LOAD_SYMBOL(cuDevicePrimaryCtxReset, tcuDevicePrimaryCtxReset, "cuDevicePrimaryCtxReset_v2");

    ... ... 

    // LOAD_SYMBOL(cuLinkCreate, tcuLinkCreate, "cuLinkCreate");
    LOAD_SYMBOL(cuLinkCreate, tcuLinkCreate, "cuLinkCreate_v2");
    // LOAD_SYMBOL(cuLinkAddData, tcuLinkAddData, "cuLinkAddData");
    LOAD_SYMBOL(cuLinkAddData, tcuLinkAddData, "cuLinkAddData_v2");

    ... ... 
        
    // LOAD_SYMBOL(cuModuleGetGlobal, tcuModuleGetGlobal, "cuModuleGetGlobal");
    LOAD_SYMBOL(cuModuleGetGlobal, tcuModuleGetGlobal, "cuModuleGetGlobal_v2");
    
    ... ...

    // LOAD_SYMBOL(cuGLGetDevices, tcuGLGetDevices_v2, "cuGLGetDevices_v2");
    // LOAD_SYMBOL(cuGraphicsGLRegisterImage, tcuGraphicsGLRegisterImage, "cuGraphicsGLRegisterImage");
    // LOAD_SYMBOL(cuGraphicsUnregisterResource, tcuGraphicsUnregisterResource, "cuGraphicsUnregisterResource");
    // LOAD_SYMBOL(cuGraphicsMapResources, tcuGraphicsMapResources, "cuGraphicsMapResources");
    // LOAD_SYMBOL(cuGraphicsUnmapResources, tcuGraphicsUnmapResources, "cuGraphicsUnmapResources");
    // LOAD_SYMBOL(cuGraphicsSubResourceGetMappedArray, tcuGraphicsSubResourceGetMappedArray, "cuGraphicsSubResourceGetMappedArray");
    // LOAD_SYMBOL(cuGraphicsResourceGetMappedPointer, tcuGraphicsResourceGetMappedPointer, "cuGraphicsResourceGetMappedPointer_v2");

    ... ...
    // LOAD_SYMBOL(cuArrayCreate, tcuArrayCreate, "cuArrayCreate_v2");
    // LOAD_SYMBOL(cuArray3DCreate, tcuArray3DCreate, "cuArray3DCreate_v2");
    // LOAD_SYMBOL(cuArrayDestroy, tcuArrayDestroy, "cuArrayDestroy");

    // LOAD_SYMBOL_OPT(cuEGLStreamProducerConnect, tcuEGLStreamProducerConnect, "cuEGLStreamProducerConnect");
    // LOAD_SYMBOL_OPT(cuEGLStreamProducerDisconnect, tcuEGLStreamProducerDisconnect, "cuEGLStreamProducerDisconnect");
    // LOAD_SYMBOL_OPT(cuEGLStreamConsumerDisconnect, tcuEGLStreamConsumerDisconnect, "cuEGLStreamConsumerDisconnect");
    // LOAD_SYMBOL_OPT(cuEGLStreamProducerPresentFrame, tcuEGLStreamProducerPresentFrame, "cuEGLStreamProducerPresentFrame");
    // LOAD_SYMBOL_OPT(cuEGLStreamProducerReturnFrame, tcuEGLStreamProducerReturnFrame, "cuEGLStreamProducerReturnFrame");

Patch 2 (Optional):

This patch redirects user requests for nvdec to cuvid decoding. This allows existing FFmpeg commands that specify only -hwaccel cuda (without -cuvid) to use cuvid decoding, eliminating the need to modify higher-level commands.

--- fftools/ffmpeg_dec.c	2025-04-17 12:45:51.890627513 +0000
+++ fftools/ffmpeg_dec.c	2025-04-21 03:32:55.503247974 +0000
@@ -1174,6 +1174,21 @@
 
     dp->sar_override = o->par->sample_aspect_ratio;
 
+    if (dp->hwaccel_device_type == AV_HWDEVICE_TYPE_CUDA && codec->type == AVMEDIA_TYPE_VIDEO &&
+            (av_strcasecmp(codec->name, "rawvideo") != 0) && (!codec->wrapper_name || av_strcasecmp(codec->wrapper_name, "cuvid") != 0)) {
+        const char *cuvid_suffix = "_cuvid";
+        size_t cuvid_length = av_strnlen(codec->name, 64) + av_strnlen(cuvid_suffix, 64) + 1;
+        char *cuvid_name = (char*)av_malloc(cuvid_length);
+        if (cuvid_name) {
+            av_strlcpy(cuvid_name, codec->name, cuvid_length);
+            av_strlcat(cuvid_name, cuvid_suffix, cuvid_length);
+            const AVCodec *cuvid_codec = avcodec_find_decoder_by_name(cuvid_name);
+            if (cuvid_codec) {
+                codec = cuvid_codec;
+            }
+            av_free(cuvid_name);
+        }
+    }
     dp->dec_ctx = avcodec_alloc_context3(codec);
     if (!dp->dec_ctx)
         return AVERROR(ENOMEM);

2.2. Build FFmpeg

build_ffmpeg.sh:

mkdir output
cd ffmpeg-7.0.1
export PKG_CONFIG_PATH=$(pwd)/../nv-codec-headers:$PKG_CONFIG_PATH
./configure --enable-shared \
    --enable-nonfree --enable-gpl \
    --enable-cuvid --enable-nvenc --enable-libnpp \
    --extra-cflags="-I$(pwd)/../nv-codec-headers/include -I$CUDA_HOME/include" \
    --extra-ldflags="-L$CUDA_HOME/lib64" \
    --prefix=$(pwd)/../output
    
make -j16 && make install

After a successful compilation and installation, the bin, include, lib, and share directories are created in the installation directory. The lib directory contains a pkgconfig directory. You can run pkg-config --modversion libavcodec to view the version number.

3. Examples

cd ../output
export LD_LIBRARY_PATH=lib:$LD_LIBRARY_PATH

# Test FFmpeg decode with cuvid
bin/ffmpeg -vcodec h264_cuvid -i 640x360_y8c8.h264 -frames 10 -y out-dec-h264.yuv
bin/ffmpeg -hwaccel cuda -i 640x360_y8c8.mp4 -frames 10 -y out-dec-h264.yuv  # Requires Patch 2

# Test FFmpeg transcode with cuvid and nvenc
bin/ffmpeg -hwaccel_output_format cuda -vcodec h264_cuvid -i 640x360_y8c8.h264 -vcodec hevc_nvenc -r 30.0 -b:v 15000000 -preset p4 output.hevc
# Alternatively, with Patch 2 applied:
bin/ffmpeg -hwaccel cuda -hwaccel_output_format cuda -i 640x360_y8c8.h264 -vcodec hevc_nvenc -r 30.0 -b:v 15000000 -preset p4 output.hevc

# Test FFmpeg transcode with cuvid, nvenc, and npp-accelerated resizing
bin/ffmpeg -hwaccel_output_format cuda -vcodec h264_cuvid -i input.h264 -vcodec hevc_nvenc -frames 300 -preset p7 -acodec copy -y output.h265

4. FAQ

Q. What hardware acceleration capabilities does FFmpeg support on PPU?

A. FFmpeg on PPU supports hardware decoding for h264_cuvid, hevc_cuvid, av1_cuvid, and vp9_cuvid. An additional patch is available upon request for AVS2 support. For hardware encoding, it supports h264_nvenc, hevc_nvenc, and av1_nvenc. It also supports libnpp-accelerated transpose, color-conversion, and scale operations.

5. Appendix A: Building FFmpeg 6.1

FFmpeg 7.1 introduced significant architectural changes to optimize parallel transcoding and multi-stream performance, leading to noticeable improvements. However, if you need to use FFmpeg 6.1 for compatibility reasons, you must download and install the older version.

## Clone nv-codec-headers version 12.1 to match FFmpeg 6.1
git clone --branch n12.1.14.0 --depth 1 https://github.com/FFmpeg/nv-codec-headers
cp nv-codec-headers/ffnvcodec.pc.in nv-codec-headers/ffnvcodec.pc
    
## Download FFmpeg
wget https://ffmpeg.org/releases/ffmpeg-6.1.2.tar.gz --no-check-certificate
tar -xzvf ffmpeg-6.1.2.tar.gz

Patch 1 (MUST):

Replace the dynlink_loader.h file in the nv-codec-headers/include/ffnvcodec/ directory with the one provided in the attachment.

mkdir output
cd ffmpeg-6.1.2
export PKG_CONFIG_PATH=$(pwd)/../nv-codec-headers:$PKG_CONFIG_PATH
./configure --enable-shared \
    --enable-nonfree --enable-gpl \
    --enable-cuvid --enable-nvenc --enable-libnpp \
    --extra-cflags="-I$(pwd)/../nv-codec-headers/include -I$CUDA_HOME/include" \
    --extra-ldflags="-L$CUDA_HOME/lib64" \
    --prefix=$(pwd)/../output
    
make -j16 && make install

Patch 2 (Optional):

--- fftools/ffmpeg_demux.c	2025-04-17 12:45:51.890627513 +0000
+++ fftools/ffmpeg_demux.c	2025-04-21 03:32:55.503247974 +0000
@@ -1173,6 +1173,21 @@ static int ist_add(const OptionsContext *o, Demuxer *d, AVStream *st)
         (o->data_disable && ist->st->codecpar->codec_type == AVMEDIA_TYPE_DATA))
             ist->user_set_discard = AVDISCARD_ALL;

+    if (ist->hwaccel_device_type == AV_HWDEVICE_TYPE_CUDA && ist->dec->type == AVMEDIA_TYPE_VIDEO &&
+            (av_strcasecmp(ist->dec->name, "rawvideo") != 0) && (!ist->dec->wrapper_name || av_strcasecmp(ist->dec->wrapper_name, "cuvid") != 0)) {
+        const char *cuvid_suffix = "_cuvid";
+        size_t cuvid_length = av_strnlen(ist->dec->name, 64) + av_strnlen(cuvid_suffix, 64) + 1;
+        char *cuvid_name = (char*)av_malloc(cuvid_length);
+        if (cuvid_name) {
+            av_strlcpy(cuvid_name, ist->dec->name, cuvid_length);
+            av_strlcat(cuvid_name, cuvid_suffix, cuvid_length);
+            const AVCodec *cuvid_codec = avcodec_find_decoder_by_name(cuvid_name);
+            if (cuvid_codec) {
+                ist->dec = cuvid_codec;
+            }
+            av_free(cuvid_name);
+        }
+    }
     ist->dec_ctx = avcodec_alloc_context3(ist->dec);
     if (!ist->dec_ctx)
         return AVERROR(ENOMEM);

6. Appendix B: FFmpeg ecosystem

6.1. ffmpeg-python

Official website: https://github.com/kkroening/ffmpeg-python

ffmpeg-python is a wrapper that builds an FFmpeg command from the provided arguments and then calls the FFmpeg executable. Therefore, FFmpeg must be installed in your runtime environment, or the paths to its bin and lib directories must be configured.

Because the PPU Video Codec supports only cuvid decode, configuring only hwaccel='cuda' is not sufficient. You must also configure the cuvid parameters as shown in the following example.

Example:

import ffmpeg
import sys

probe = ffmpeg.probe('input.mp4')
video_stream = next((stream for stream in probe['streams'] if stream['codec_type'] == 'video'), None)

if video_stream is None or video_stream['codec_name'] is None:
    sys.exit()
    
if video_stream['codec_name'] in ['h264', 'hevc', 'av1', 'vp9']:
    out, _ = (
        ffmpeg.input('input.mp4', vcodec = video_stream['codec_name']+'_cuvid')
        .output('output.rgb', format='rawvideo', pix_fmt='rgb24')
        .run()
    )
else:
    out, _ = (
        ffmpeg.input('input.mp4')
        .output('output.rgb', format='rawvideo', pix_fmt='rgb24')
        .run()

ffmpeg-python is a lightweight wrapper for the FFmpeg command-line tool. It can process data from files or via pipe: (stdin/stdout). Since the data is on the CPU while decoding happens on the GPU, this workflow requires an unavoidable data copy from the CPU to the GPU for decoding, which can degrade performance. Therefore, we do not recommend this method.

6.2. PyAV

Torchvision supports three backends: PyAV, video_reader, and cuda. The default backend is PyAV. The video_reader backend performs CPU-only encoding and decoding, while the cuda backend uses the GPU for decoding. PyAV calls FFmpeg to handle encoding and decoding. Therefore, PyAV relies on the hardware acceleration capabilities built into FFmpeg, and you must build and install a version of FFmpeg that supports hardware acceleration.

PyAV official website: https://github.com/PyAV-Org/PyAV

Call path 1: pytorch->torchvision->PyAV->ffmpeg:

To enable hardware decoding support, you must modify the Torchvision source code (torchvision/io/video_reader.py):

--- a/torchvision/io/video_reader.py
+++ b/torchvision/io/video_reader.py
@@ -280,6 +280,13 @@ class VideoReader:
         if self.backend == "pyav":
             stream_type = stream.split(":")[0]
             stream_id = 0 if len(stream.split(":")) == 1 else int(stream.split(":")[1])
+            video_stream = self.container.streams.video[stream_id]
+            # Setting up the codec with cuvid
+            if video_stream.codec.name in ('h264', 'hevc', 'av1', 'vp9'):
+                codec_name = f'{video_stream.codec.name}_cuvid'
+            else:
+                codec_name = video_stream.codec.name  # Fallback to software decoding
+            video_stream.codec_context = av.codec.CodecContext.create(codec_name, 'r')
             self.pyav_stream = {stream_type: stream_id}
             self._c = self.container.decode(**self.pyav_stream)

By default, PyAV outputs data in TCHW format.

Hardware encoding support does not require modifying Torchvision code, but you must specify the nvenc codec name in the function call:

from torchvision.io import write_video
# Explicitly specify the codec. We currently support h264_nvenc, av1_nvenc, and hevc_nvenc.
write_video(save_path, frames, fps=fps, video_codec="h264_nvenc")

Call path 2: pytorch-> torchaudio-> PyAV -> ffmpeg:

Torchaudio also supports PyAV as a backend for hardware decoding. Torchaudio does not require code modifications, but you must explicitly specify the cuvid decoder name:

import torch
import torchaudio

from torchaudio.io import StreamReader
from torchaudio.utils import ffmpeg_utils

s = StreamReader(src)
# Explicitly specify the cuvid decoder. We currently support h264_cuvid, av1_cuvid, hevc_cuvid, and vp9_cuvid.
s.add_video_stream(int(s.get_src_stream_info(0).frame_rate), decoder="h264_cuvid")
s.fill_buffer()
(video,) = s.pop_chunks()

Note: You can use torchvision.get_video_backend() to get the current backend and torchvision.set_video_backend("pyav") to set the default backend for torchvision. However, the default installation of torchvision does not support the "cuda" backend. You need to manually compile and install torchvision to use the "cuda" backend.

6.3. torchcodec

Official documentation: https://ai-gerrit.eng.t-head.cn/c/acvid/+/67622

Project repository: https://github.com/pytorch/torchcodec

Call path: pytorch-> torchcodec -> ffmpeg

The SAIL source provides pip packages for torchcodec 0.2.1 and 0.4. We recommend installing torchcodec directly from the SAIL source.

torchcodec versioning is tied to specific PyTorch and Python versions, as shown below:

The following steps show how to build and install torchcodec from source:

1. Install the FFmpeg dependency. Both FFmpeg 6.1 and 7.0 are supported.

2. Build from source and install the package:

# branch name can be: release/0.2.1, release/0.2.0, or release/0.1.1
git clone --branch <branch_name> https://github.com/pytorch/torchcodec.git
cd torchcodec/
git apply torchcodec_<version>.diff
ENABLE_CUDA=1 I_CONFIRM_THIS_IS_NOT_A_LICENSE_VIOLATION=1 pip install . --no-build-isolation

The torchcodec source code requires two modifications to run on the PPU:

diff --git a/src/torchcodec/_core/SingleStreamDecoder.cpp b/src/torchcodec/_core/SingleStreamDecoder.cpp
index b73f703..fc7780a 100644
--- a/src/torchcodec/_core/SingleStreamDecoder.cpp
+++ b/src/torchcodec/_core/SingleStreamDecoder.cpp
@@ -388,9 +388,21 @@ void SingleStreamDecoder::addStream(
   // addStream() which is supposed to be generic
   if (mediaType == AVMEDIA_TYPE_VIDEO) {
     if (deviceInterface_) {
-      avCodec = makeAVCodecOnlyUseForCallingAVFindBestStream(
-          deviceInterface_->findCodec(streamInfo.stream->codecpar->codec_id)
-              .value_or(avCodec));
+      if (device.type() != torch::kCUDA) {
+          avCodec = makeAVCodecOnlyUseForCallingAVFindBestStream(
+              deviceInterface_->findCodec(streamInfo.stream->codecpar->codec_id)
+                  .value_or(avCodec));
+      }
+      else {
+          const char* cuvid_suffix = "_cuvid";
+          size_t cuvid_length = std::strlen(avCodec->name) + std::strlen(cuvid_suffix) + 1;
+          char* cuvid_name = new char[cuvid_length];
+          std::strcpy(cuvid_name, avCodec->name);
+          std::strcat(cuvid_name, cuvid_suffix);
+          avCodec = avcodec_find_decoder_by_name(cuvid_name);
+          delete[] cuvid_name;
+          TORCH_CHECK(avCodec != nullptr);
+      }
     }
   }
 
@@ -417,6 +429,12 @@ void SingleStreamDecoder::addStream(
     throw std::invalid_argument(getFFMPEGErrorStringFromErrorCode(retVal));
   }

+  if (device.type() == torch::kCUDA) {
+    codecContext->hw_frames_ctx = av_hwframe_ctx_alloc(codecContext->hw_device_ctx);
+    AVHWFramesContext* hwframe_ctx = (AVHWFramesContext*)codecContext->hw_frames_ctx->data;
+    // in avcodec_open2, cuvid will not set sw_format until sequence callback in avcodec_send_packet()
+    hwframe_ctx->sw_format = codecContext->sw_pix_fmt;
+  }
   codecContext->time_base = streamInfo.stream->time_base;
   containerMetadata_.allStreamMetadata[activeStreamIndex_].codecName =
       std::string(avcodec_get_name(codecContext->codec_id));

diff --git a/src/torchcodec/decoders/_core/VideoDecoder.cpp b/src/torchcodec/decoders/_core/VideoDecoder.cpp
index 97214ce..530541d 100644
--- a/src/torchcodec/decoders/_core/VideoDecoder.cpp
+++ b/src/torchcodec/decoders/_core/VideoDecoder.cpp
@@ -449,9 +449,14 @@ void VideoDecoder::addStream(
   // TODO_CODE_QUALITY it's pretty meh to have a video-specific logic within
   // addStream() which is supposed to be generic
   if (mediaType == AVMEDIA_TYPE_VIDEO && device.type() == torch::kCUDA) {
-    avCodec = makeAVCodecOnlyUseForCallingAVFindBestStream(
-        findCudaCodec(device, streamInfo.stream->codecpar->codec_id)
-            .value_or(avCodec));
+    const char* cuvid_suffix = "_cuvid";
+    size_t cuvid_length = std::strlen(avCodec->name) + std::strlen(cuvid_suffix) + 1;
+    char* cuvid_name = new char[cuvid_length];
+    std::strcpy(cuvid_name, avCodec->name);
+    std::strcat(cuvid_name, cuvid_suffix);
+    avCodec = avcodec_find_decoder_by_name(cuvid_name);
+    delete[] cuvid_name;
+    TORCH_CHECK(avCodec != nullptr);
   }

   AVCodecContext* codecContext = avcodec_alloc_context3(avCodec);
@@ -474,6 +479,12 @@ void VideoDecoder::addStream(
     throw std::invalid_argument(getFFMPEGErrorStringFromErrorCode(retVal));
   }

+  if (device.type() == torch::kCUDA) {
+    codecContext->hw_frames_ctx = av_hwframe_ctx_alloc(codecContext->hw_device_ctx);
+    AVHWFramesContext* hwframe_ctx = (AVHWFramesContext*)codecContext->hw_frames_ctx->data;
+    // in avcodec_open2, cuvid will not set sw_format until sequence callback in avcodec_send_packet()
+    hwframe_ctx->sw_format = codecContext->sw_pix_fmt;
+  }
   codecContext->time_base = streamInfo.stream->time_base;
   containerMetadata_.allStreamMetadata[activeStreamIndex_].codecName =
       std::string(avcodec_get_name(codecContext->codec_id));

Example usage:

python sample_torchcodec.py

import torch

print(f"{torch.__version__=}")
print(f"{torch.cuda.is_available()=}")
print(f"{torch.cuda.get_device_properties(0)=}")

import torchcodec
from torchcodec.decoders import VideoDecoder

decoder = VideoDecoder("sample.mp4", device="cuda")
frame = decoder[0]

6.4. imageio-ffmpeg

imageio-ffmpeg is an imageio plugin that provides a wrapper for FFmpeg. Similar to ffmpeg-python, it works by combining input parameters and then calling the ffmpeg executable. Therefore, you must install ffmpeg in your runtime environment or set the bin and lib directories for ffmpeg. We do not recommend using this plugin because the official documentation explicitly states: You should probably use PyAV instead; it is faster and offers more features.

imageio-ffmpeg: https://github.com/imageio/imageio-ffmpeg

Example usage:

import imageio

video_reader = imageio.get_reader('input.mp4', 'ffmpeg', ffmpeg_params=['-c:v', 'h264_cuvid'])
frame_list = []
for i, frame in enumerate(video_reader):
    frame_list.append(frame)
imageio.mimsave('out.mp4', frame_list)

6.5. Decord

The default installation of Decord is CPU-only and does not support hardware acceleration. You must build and install it from source to enable this feature.

Decord depends on FFmpeg, but only uses its demuxing capabilities. You do not need to build FFmpeg with hardware acceleration support.

A bug in older versions of FFmpeg can cause a deadlock in Decord. This issue was resolved in FFmpeg 7. Therefore, you must build and install FFmpeg version 7.0 or later.

Download Decord:

git clone -b v0.6.0 --depth 1 --recursive https://github.com/dmlc/decord
cd decord
git apply decord_ffmpeg7.patch

The patch primarily updates Decord's dependency on FFmpeg from version 4.2 to 7.0 or later:

decord_ffmpeg7.patch:

+++ b/src/video/nvcodec/cuda_threaded_decoder.cc
@@ -17,7 +17,7 @@ namespace decord {
 namespace cuda {
 using namespace runtime;

-CUThreadedDecoder::CUThreadedDecoder(int device_id, AVCodecParameters *codecpar, AVInputFormat *iformat)
+CUThreadedDecoder::CUThreadedDecoder(int device_id, AVCodecParameters *codecpar, const AVInputFormat *iformat)
     : device_id_(device_id), stream_({device_id, false}), device_{}, ctx_{}, parser_{}, decoder_{},
     pkt_queue_{}, frame_queue_{},
     run_(false), frame_count_(0), draining_(false),
@@ -70,7 +70,7 @@ CUThreadedDecoder::CUThreadedDecoder(int device_id, AVCodecParameters *codecpar,
     }
 }

-void CUThreadedDecoder::InitBitStreamFilter(AVCodecParameters *codecpar, AVInputFormat *iformat) {
+void CUThreadedDecoder::InitBitStreamFilter(AVCodecParameters *codecpar, const AVInputFormat *iformat) {
     const char* bsf_name = nullptr;
     if (AV_CODEC_ID_H264 == codecpar->codec_id) {
         // H.264
diff --git a/src/video/nvcodec/cuda_threaded_decoder.h b/src/video/nvcodec/cuda_threaded_decoder.h
index d7e6fcd..61958a1 100644
--- a/src/video/nvcodec/cuda_threaded_decoder.h
+++ b/src/video/nvcodec/cuda_threaded_decoder.h
@@ -46,7 +46,7 @@ class CUThreadedDecoder final : public ThreadedDecoderInterface {
     using FrameOrderQueuePtr = std::unique_ptr<FrameOrderQueue>;

     public:
-        CUThreadedDecoder(int device_id, AVCodecParameters *codecpar, AVInputFormat *iformat);
+        CUThreadedDecoder(int device_id, AVCodecParameters *codecpar, const AVInputFormat *iformat);
         void SetCodecContext(AVCodecContext *dec_ctx, int width = -1, int height = -1, int rotation = 0);
         bool Initialized() const;
         void Start();
@@ -70,7 +70,7 @@ class CUThreadedDecoder final : public ThreadedDecoderInterface {
         void LaunchThreadImpl();
         void RecordInternalError(std::string message);
         void CheckErrorStatus();
-        void InitBitStreamFilter(AVCodecParameters *codecpar, AVInputFormat *iformat);
+        void InitBitStreamFilter(AVCodecParameters *codecpar, const AVInputFormat *iformat);

         int device_id_;
         CUStream stream_;
diff --git a/src/video/video_reader.cc b/src/video/video_reader.cc
index af4858d..99c9635 100644
--- a/src/video/video_reader.cc
+++ b/src/video/video_reader.cc
@@ -145,7 +145,7 @@ VideoReader::~VideoReader(){

 void VideoReader::SetVideoStream(int stream_nb) {
     if (!fmt_ctx_) return;
-    AVCodec *dec;
+    const AVCodec *dec;
     int st_nb = av_find_best_stream(fmt_ctx_.get(), AVMEDIA_TYPE_VIDEO, stream_nb, -1, &dec, 0);
     // LOG(INFO) << "find best stream: " << st_nb;
     CHECK_GE(st_nb, 0) << "ERROR cannot find video stream with wanted index: " << stream_nb;

Build Decord:

mkdir build && cd build
cmake .. -DUSE_CUDA=ON -DCMAKE_BUILD_TYPE=Release
make

Install Decord:

cd ../python
python3 setup.py install --user

Example usage:

import decord
import torch
from decord import gpu, cpu

video_path="input.mp4"
vr = decord.VideoReader(video_path, ctx=gpu(0))
nframes = 40
total_frames = 500
idx = torch.linspace(0, total_frames - 1, nframes).round().long().tolist()
video = vr.get_batch(idx).asnumpy()