FFmpeg video (v1.7)

1. Overview

The PPU SDK is compatible with the Nvidia Video Codec SDK, allowing you to directly use FFmpeg plugins such as cuvid, nvenc, and libnpp for hardware acceleration without any code modifications. We have verified the following FFmpeg versions:

2. Build from source

FFmpeg: https://github.com/FFmpeg/FFmpeg

nv-codec-headers: https://github.com/FFmpeg/nv-codec-headers

The following sections provide build instructions for FFmpeg 7.0 and FFmpeg 6.1. Before you begin, ensure you have installed the PPU SDK and configured the runtime environment by running the envsetup.sh script.

2.1. Prepare the FFmpeg source code

setup_ffmpeg.sh:

## Clone version n12.2.72.0 of nv-codec-headers to match FFmpeg 7.0
git clone --branch n12.2.72.0 --depth 1 https://github.com/FFmpeg/nv-codec-headers
cp nv-codec-headers/ffnvcodec.pc.in nv-codec-headers/ffnvcodec.pc
    
## Download FFmpeg (7.1.1)
wget https://ffmpeg.org/releases/ffmpeg-7.1.1.tar.gz --no-check-certificate
tar -xzvf ffmpeg-7.1.1.tar.gz
mv ffmpeg-7.1.1 ffmpeg

## Option 2: Download FFmpeg 7.0.1
wget https://ffmpeg.org/releases/ffmpeg-7.0.1.tar.gz --no-check-certificate
tar -xzvf ffmpeg-7.0.1.tar.gz
mv ffmpeg-7.0.1 ffmpeg

Patch 1 (Required):

Some CUDA runtime APIs are not supported, which affects FFmpeg's cuda scale feature (npp scale is supported). This patch removes the definitions for unsupported CUDA runtime APIs to ensure a successful build.

Replace the file with the same name in the nv-codec-headers/include/ffnvcodec/ directory with dynlink_loader.h.

This patch comments out the following lines:

    // tcuGLGetDevices_v2 *cuGLGetDevices;
    // tcuGraphicsGLRegisterImage *cuGraphicsGLRegisterImage;
    // tcuGraphicsUnregisterResource *cuGraphicsUnregisterResource;
    // tcuGraphicsMapResources *cuGraphicsMapResources;
    // tcuGraphicsUnmapResources *cuGraphicsUnmapResources;
    // tcuGraphicsSubResourceGetMappedArray *cuGraphicsSubResourceGetMappedArray;
    // tcuGraphicsResourceGetMappedPointer *cuGraphicsResourceGetMappedPointer;

    ... ...

    // tcuArrayCreate *cuArrayCreate;
    // tcuArray3DCreate *cuArray3DCreate;
    // tcuArrayDestroy *cuArrayDestroy;

    // tcuEGLStreamProducerConnect *cuEGLStreamProducerConnect;
    // tcuEGLStreamProducerDisconnect *cuEGLStreamProducerDisconnect;
    // tcuEGLStreamConsumerDisconnect *cuEGLStreamConsumerDisconnect;
    // tcuEGLStreamProducerPresentFrame *cuEGLStreamProducerPresentFrame;
    // tcuEGLStreamProducerReturnFrame *cuEGLStreamProducerReturnFrame;

... ...
    // LOAD_SYMBOL(cuDevicePrimaryCtxRelease, tcuDevicePrimaryCtxRelease, "cuDevicePrimaryCtxRelease");
    LOAD_SYMBOL(cuDevicePrimaryCtxRelease, tcuDevicePrimaryCtxRelease, "cuDevicePrimaryCtxRelease_v2");
    // LOAD_SYMBOL(cuDevicePrimaryCtxSetFlags, tcuDevicePrimaryCtxSetFlags, "cuDevicePrimaryCtxSetFlags");
    LOAD_SYMBOL(cuDevicePrimaryCtxSetFlags, tcuDevicePrimaryCtxSetFlags, "cuDevicePrimaryCtxSetFlags_v2");
    
    ... ...
    // LOAD_SYMBOL(cuDevicePrimaryCtxReset, tcuDevicePrimaryCtxReset, "cuDevicePrimaryCtxReset");
    LOAD_SYMBOL(cuDevicePrimaryCtxReset, tcuDevicePrimaryCtxReset, "cuDevicePrimaryCtxReset_v2");

    ... ... 

    // LOAD_SYMBOL(cuLinkCreate, tcuLinkCreate, "cuLinkCreate");
    LOAD_SYMBOL(cuLinkCreate, tcuLinkCreate, "cuLinkCreate_v2");
    // LOAD_SYMBOL(cuLinkAddData, tcuLinkAddData, "cuLinkAddData");
    LOAD_SYMBOL(cuLinkAddData, tcuLinkAddData, "cuLinkAddData_v2");

    ... ... 
        
    // LOAD_SYMBOL(cuModuleGetGlobal, tcuModuleGetGlobal, "cuModuleGetGlobal");
    LOAD_SYMBOL(cuModuleGetGlobal, tcuModuleGetGlobal, "cuModuleGetGlobal_v2");
    
    ... ...

    // LOAD_SYMBOL(cuGLGetDevices, tcuGLGetDevices_v2, "cuGLGetDevices_v2");
    // LOAD_SYMBOL(cuGraphicsGLRegisterImage, tcuGraphicsGLRegisterImage, "cuGraphicsGLRegisterImage");
    // LOAD_SYMBOL(cuGraphicsUnregisterResource, tcuGraphicsUnregisterResource, "cuGraphicsUnregisterResource");
    // LOAD_SYMBOL(cuGraphicsMapResources, tcuGraphicsMapResources, "cuGraphicsMapResources");
    // LOAD_SYMBOL(cuGraphicsUnmapResources, tcuGraphicsUnmapResources, "cuGraphicsUnmapResources");
    // LOAD_SYMBOL(cuGraphicsSubResourceGetMappedArray, tcuGraphicsSubResourceGetMappedArray, "cuGraphicsSubResourceGetMappedArray");
    // LOAD_SYMBOL(cuGraphicsResourceGetMappedPointer, tcuGraphicsResourceGetMappedPointer, "cuGraphicsResourceGetMappedPointer_v2");

    ... ...
    // LOAD_SYMBOL(cuArrayCreate, tcuArrayCreate, "cuArrayCreate_v2");
    // LOAD_SYMBOL(cuArray3DCreate, tcuArray3DCreate, "cuArray3DCreate_v2");
    // LOAD_SYMBOL(cuArrayDestroy, tcuArrayDestroy, "cuArrayDestroy");

    // LOAD_SYMBOL_OPT(cuEGLStreamProducerConnect, tcuEGLStreamProducerConnect, "cuEGLStreamProducerConnect");
    // LOAD_SYMBOL_OPT(cuEGLStreamProducerDisconnect, tcuEGLStreamProducerDisconnect, "cuEGLStreamProducerDisconnect");
    // LOAD_SYMBOL_OPT(cuEGLStreamConsumerDisconnect, tcuEGLStreamConsumerDisconnect, "cuEGLStreamConsumerDisconnect");
    // LOAD_SYMBOL_OPT(cuEGLStreamProducerPresentFrame, tcuEGLStreamProducerPresentFrame, "cuEGLStreamProducerPresentFrame");
    // LOAD_SYMBOL_OPT(cuEGLStreamProducerReturnFrame, tcuEGLStreamProducerReturnFrame, "cuEGLStreamProducerReturnFrame");

Patch 2 (Optional):

This patch redirects nvdec requests to use cuvid for hardware decoding. This allows FFmpeg commands that specify -hwaccel cuda (without a _cuvid decoder) to work without modification.

--- fftools/ffmpeg_dec.c	2025-04-17 12:45:51.890627513 +0000
+++ fftools/ffmpeg_dec.c	2025-04-21 03:32:55.503247974 +0000
@@ -1174,6 +1174,21 @@
 
     dp->sar_override = o->par->sample_aspect_ratio;

+    if (dp->hwaccel_device_type == AV_HWDEVICE_TYPE_CUDA && codec->type == AVMEDIA_TYPE_VIDEO &&
+            (av_strcasecmp(codec->name, "rawvideo") != 0) && (!codec->wrapper_name || av_strcasecmp(codec->wrapper_name, "cuvid") != 0)) {
+        const char *cuvid_suffix = "_cuvid";
+        size_t cuvid_length = av_strnlen(codec->name, 64) + av_strnlen(cuvid_suffix, 64) + 1;
+        char *cuvid_name = (char*)av_malloc(cuvid_length);
+        if (cuvid_name) {
+            av_strlcpy(cuvid_name, codec->name, cuvid_length);
+            av_strlcat(cuvid_name, cuvid_suffix, cuvid_length);
+            const AVCodec *cuvid_codec = avcodec_find_decoder_by_name(cuvid_name);
+            if (cuvid_codec) {
+                codec = cuvid_codec;
+            }
+            av_free(cuvid_name);
+        }
+    }
     dp->dec_ctx = avcodec_alloc_context3(codec);
     if (!dp->dec_ctx)
         return AVERROR(ENOMEM);

Patch 3 (Optional):

This patch enables inline color conversion during PPU hardware decoding, allowing FFmpeg to output the rgb24 format during decode operations.

inline_csc.patch

git apply ../inline_csc.patch

Sample command:

ffmpeg -hwaccel cuvid -hwaccel_output_format cuda -c:v h264_cuvid -csc rgb24 -i test.mp4 -f rawvideo -

2.2. Build FFmpeg

build_ffmpeg.sh:

apt install nasm pkgconf 
apt install libdav1d-dev # On Ubuntu, install from the apt repository,
                         # or build from source: https://code.videolan.org/videolan/dav1d.git
mkdir output
cd ffmpeg
export PKG_CONFIG_PATH=$(pwd)/../nv-codec-headers:$PKG_CONFIG_PATH
./configure --enable-shared \
    --enable-nonfree --enable-gpl \
    --enable-cuvid --enable-nvenc --enable-libnpp --enable-libdav1d --enable-cuda-nvcc \
    --extra-cflags="-I$(pwd)/../nv-codec-headers/include -I$CUDA_HOME/include" \
    --extra-ldflags="-L$CUDA_HOME/lib64" \
    --prefix=$(pwd)/../output
    
make -j16 && make install

After a successful build and installation, the bin, include, lib, and share directories are created in the installation directory. The lib directory contains a pkgconfig directory. You can run the pkg-config --modversion libavcodec command to view the version number.

3. Run samples

cd ../output
export LD_LIBRARY_PATH=lib:$LD_LIBRARY_PATH

# ffmpeg decode test with cuvid
bin/ffmpeg -vcodec h264_cuvid -i 640x360_y8c8.h264 -frames 10 -y out-dec-h264.yuv
bin/ffmpeg -hwaccel cuda -i 640x360_y8c8.mp4 -frames 10 -y out-dec-h264.yuv  # Patch 2 must be applied for this command to work

# ffmpeg transcode test with cuvid and nvenc
bin/ffmpeg -hwaccel_output_format cuda -vcodec h264_cuvid -i 640x360_y8c8.h264 -vcodec hevc_nvenc -r 30.0 -b:v 15000000 -preset p4 output.hevc
# If Patch 2 is applied, you can also write the command as follows:
bin/ffmpeg -hwaccel cuda -hwaccel_output_format cuda -i 640x360_y8c8.h264 -vcodec hevc_nvenc -r 30.0 -b:v 15000000 -preset p4 output.hevc

# ffmpeg transcode test with cuvid and nvenc, resize with npp support
bin/ffmpeg -hwaccel_output_format cuda -vcodec h264_cuvid -i input.h264 -vcodec hevc_nvenc -frames 300 -preset p7 -acodec copy -y output.h265

4. FAQ

Q. What hardware acceleration capabilities does FFmpeg support on the PPU?

A. It supports hardware decoding for h264_cuvid, hevc_cuvid, av1_cuvid, and vp9_cuvid. An additional patch is available upon request for AVS2 support. It also supports hardware encoding for h264_nvenc, hevc_nvenc, and av1_nvenc, as well as NPP-accelerated transpose, color conversion, and scale operations.

Appendix A: Build FFmpeg 6.1

FFmpeg 7.1 introduces significant architectural changes that improve parallel performance for transcoding and multistream operations. However, if you need to use FFmpeg 6.1 for compatibility reasons, you must download and install the older version.

## Clone version n12.1.14.0 of nv-codec-headers to match FFmpeg 6.1
git clone --branch n12.1.14.0 --depth 1 https://github.com/FFmpeg/nv-codec-headers
cp nv-codec-headers/ffnvcodec.pc.in nv-codec-headers/ffnvcodec.pc
    
## Download FFmpeg
wget https://ffmpeg.org/releases/ffmpeg-6.1.2.tar.gz --no-check-certificate
tar -xzvf ffmpeg-6.1.2.tar.gz

Patch 1 (Required):

Use the attached dynlink_loader.h to replace the file of the same name in the nv-codec-headers/include/ffnvcodec/ directory. The subsequent build and run commands remain the same.

mkdir output
cd ffmpeg-6.1.2
export PKG_CONFIG_PATH=$(pwd)/../nv-codec-headers:$PKG_CONFIG_PATH
./configure --enable-shared \
    --enable-nonfree --enable-gpl \
    --enable-cuvid --enable-nvenc --enable-libnpp \
    --extra-cflags="-I$(pwd)/../nv-codec-headers/include -I$CUDA_HOME/include" \
    --extra-ldflags="-L$CUDA_HOME/lib64" \
    --prefix=$(pwd)/../output
    
make -j16 && make install

Patch 2 (Optional):

--- fftools/ffmpeg_demux.c	2025-04-17 12:45:51.890627513 +0000
+++ fftools/ffmpeg_demux.c	2025-04-21 03:32:55.503247974 +0000
@@ -1173,6 +1173,21 @@ static int ist_add(const OptionsContext *o, Demuxer *d, AVStream *st)
         (o->data_disable && ist->st->codecpar->codec_type == AVMEDIA_TYPE_DATA))
             ist->user_set_discard = AVDISCARD_ALL;

+    if (ist->hwaccel_device_type == AV_HWDEVICE_TYPE_CUDA && ist->dec->type == AVMEDIA_TYPE_VIDEO &&
+            (av_strcasecmp(ist->dec->name, "rawvideo") != 0) && (!ist->dec->wrapper_name || av_strcasecmp(ist->dec->wrapper_name, "cuvid") != 0)) {
+        const char *cuvid_suffix = "_cuvid";
+        size_t cuvid_length = av_strnlen(ist->dec->name, 64) + av_strnlen(cuvid_suffix, 64) + 1;
+        char *cuvid_name = (char*)av_malloc(cuvid_length);
+        if (cuvid_name) {
+            av_strlcpy(cuvid_name, ist->dec->name, cuvid_length);
+            av_strlcat(cuvid_name, cuvid_suffix, cuvid_length);
+            const AVCodec *cuvid_codec = avcodec_find_decoder_by_name(cuvid_name);
+            if (cuvid_codec) {
+                ist->dec = cuvid_codec;
+            }
+            av_free(cuvid_name);
+        }
+    }
     ist->dec_ctx = avcodec_alloc_context3(ist->dec);
     if (!ist->dec_ctx)
         return AVERROR(ENOMEM);

6. Appendix B: The FFmpeg ecosystem

6.1. ffmpeg-python

Official Website: https://github.com/kkroening/ffmpeg-python

ffmpeg-python is a wrapper that constructs and executes FFmpeg commands from your input parameters. Therefore, your runtime environment must have FFmpeg installed, or the bin and lib directories of your FFmpeg installation must be included in your system's path.

Because the PPU Video Codec supports only cuvid decoding, configuring only hwaccel='cuda' is not sufficient. You must also configure the cuvid parameters. The following is an example.

Sample:

import ffmpeg
import sys

probe = ffmpeg.probe('input.mp4')
video_stream = next((stream for stream in probe['streams'] if stream['codec_type'] == 'video'), None)

if video_stream is None or video_stream['codec_name'] is None:
    sys.exit()
    
if video_stream['codec_name'] in ['h264', 'hevc', 'av1', 'vp9']:
    out, _ = (
        ffmpeg.input('input.mp4', vcodec = video_stream['codec_name']+'_cuvid')
        .output('output.rgb', format='rawvideo', pix_fmt='rgb24')
        .run()
    )
else:
    out, _ = (
        ffmpeg.input('input.mp4')
        .output('output.rgb', format='rawvideo', pix_fmt='rgb24')
        .run()

ffmpeg-python is fundamentally a wrapper for FFmpeg commands. In addition to files, it can process data from pipes (such as stdin and stdout). This approach, however, involves data residing on the CPU while encoding and decoding occur on the GPU. This creates unavoidable data copy overhead and is therefore not recommended.

6.2. PyAV

Torchvision supports three backends: PyAV (the default), video_reader (CPU-only), and cuda (GPU decoder). Because the PyAV backend calls FFmpeg to handle encoding and decoding, it relies on the hardware acceleration capabilities of your FFmpeg installation, and you must build and install a version of FFmpeg with hardware acceleration enabled.

PyAV Official Website: https://github.com/PyAV-Org/PyAV

Call path 1: pytorch->torchvision->PyAV->ffmpeg:

Hardware decoding support (requires modifying Torchvision code in torchvision/io/video_reader.py):

--- a/torchvision/io/video_reader.py
+++ b/torchvision/io/video_reader.py
@@ -280,6 +280,13 @@ class VideoReader:
         if self.backend == "pyav":
             stream_type = stream.split(":")[0]
             stream_id = 0 if len(stream.split(":")) == 1 else int(stream.split(":")[1])
+            video_stream = self.container.streams.video[stream_id]
+            # Setting up the codec with cuvid
+            if video_stream.codec.name in ('h264', 'hevc', 'av1', 'vp9'):
+                codec_name = f'{video_stream.codec.name}_cuvid'
+            else:
+                codec_name = video_stream.codec.name  # Fallback to software decoding
+            video_stream.codec_context = av.codec.CodecContext.create(codec_name, 'r')
             self.pyav_stream = {stream_type: stream_id}
             self._c = self.container.decode(**self.pyav_stream)

By default, PyAV outputs data in TCHW format.

Hardware encoding support (no Torchvision code modification required, but users must specify an nvenc codec name when calling):

from torchvision.io import write_video
write_video(save_path, frames, fps=fps, video_codec="h264_nvenc")  # You must specify the codec explicitly. We currently support h264_nvenc, av1_nvenc, and hevc_nvenc.

Call path 2: pytorch-> torchaudio-> pyAV -> ffmpeg:

Torchaudio also supports PyAV as a backend for hardware decoding. No code changes to torchaudio are required, but you must explicitly specify the cuvid codec name:

import torch
import torchaudio

from torchaudio.io import StreamReader
from torchaudio.utils import ffmpeg_utils

s = StreamReader(src)
s.add_video_stream(int(s.get_src_stream_info(0).frame_rate), decoder="h264_cuvid") # You must explicitly specify cuvid. We currently support h264_cuvid, av1_cuvid, hevc_cuvid, and vp9_cuvid.
s.fill_buffer()
(video,) = s.pop_chunks()

Note: You can use torchvision.get_video_backend() to obtain the current backend, and torchvision.set_video_backend("pyav") to set the default backend for torchvision. However, the default installation of torchvision does not support the 'cuda' backend. You must manually build and install torchvision to use it.

6.3. torchcodec

Official Documentation: https://ai-gerrit.eng.t-head.cn/c/acvid/+/67622

Project Repo: https://github.com/pytorch/torchcodec

Call path: pytorch-> torchcodec -> ffmpeg

The SAIL repository includes pip packages for torchcodec 0.2.1 and 0.4. We recommend installing and using torchcodec directly from the SAIL repository.

The torchcodec and PyTorch version dependencies are as follows:

The following steps describe how to build and install torchcodec from source:

1. Install dependencies: FFmpeg (supports both FFmpeg 6.1 and FFmpeg 7.0, as described previously).

2. Build from source and install the package:

# branch name: release/0.2.1 release/0.2.0 release/0.1.1
git clone --branch <branch_name> https://github.com/pytorch/torchcodec.git
cd torchcodec/
git apply torchcodec_<version>.diff
ENABLE_CUDA=1 I_CONFIRM_THIS_IS_NOT_A_LICENSE_VIOLATION=1 pip install . --no-build-isolation

To run on a PPU, you must make two changes to the torchcodec source code:

diff --git a/src/torchcodec/_core/SingleStreamDecoder.cpp b/src/torchcodec/_core/SingleStreamDecoder.cpp
index b73f703..fc7780a 100644
--- a/src/torchcodec/_core/SingleStreamDecoder.cpp
+++ b/src/torchcodec/_core/SingleStreamDecoder.cpp
@@ -388,9 +388,21 @@ void SingleStreamDecoder::addStream(
   // addStream() which is supposed to be generic
   if (mediaType == AVMEDIA_TYPE_VIDEO) {
     if (deviceInterface_) {
-      avCodec = makeAVCodecOnlyUseForCallingAVFindBestStream(
-          deviceInterface_->findCodec(streamInfo.stream->codecpar->codec_id)
-              .value_or(avCodec));
+      if (device.type() != torch::kCUDA) {
+          avCodec = makeAVCodecOnlyUseForCallingAVFindBestStream(
+              deviceInterface_->findCodec(streamInfo.stream->codecpar->codec_id)
+                  .value_or(avCodec));
+      }
+      else {
+          const char* cuvid_suffix = "_cuvid";
+          size_t cuvid_length = std::strlen(avCodec->name) + std::strlen(cuvid_suffix) + 1;
+          char* cuvid_name = new char[cuvid_length];
+          std::strcpy(cuvid_name, avCodec->name);
+          std::strcat(cuvid_name, cuvid_suffix);
+          avCodec = avcodec_find_decoder_by_name(cuvid_name);
+          delete[] cuvid_name;
+          TORCH_CHECK(avCodec != nullptr);
+      }
     }
   }
 
@@ -417,6 +429,12 @@ void SingleStreamDecoder::addStream(
     throw std::invalid_argument(getFFMPEGErrorStringFromErrorCode(retVal));
   }

+  if (device.type() == torch::kCUDA) {
+    codecContext->hw_frames_ctx = av_hwframe_ctx_alloc(codecContext->hw_device_ctx);
+    AVHWFramesContext* hwframe_ctx = (AVHWFramesContext*)codecContext->hw_frames_ctx->data;
+    // in avcodec_open2, cuvid will not set sw_format until sequence callback in avcodec_send_packet()
+    hwframe_ctx->sw_format = codecContext->sw_pix_fmt;
+  }
   codecContext->time_base = streamInfo.stream->time_base;
   containerMetadata_.allStreamMetadata[activeStreamIndex_].codecName =
       std::string(avcodec_get_name(codecContext->codec_id));

diff --git a/src/torchcodec/decoders/_core/VideoDecoder.cpp b/src/torchcodec/decoders/_core/VideoDecoder.cpp
index 97214ce..530541d 100644
--- a/src/torchcodec/decoders/_core/VideoDecoder.cpp
+++ b/src/torchcodec/decoders/_core/VideoDecoder.cpp
@@ -449,9 +449,14 @@ void VideoDecoder::addStream(
   // TODO_CODE_QUALITY it's pretty meh to have a video-specific logic within
   // addStream() which is supposed to be generic
   if (mediaType == AVMEDIA_TYPE_VIDEO && device.type() == torch::kCUDA) {
-    avCodec = makeAVCodecOnlyUseForCallingAVFindBestStream(
-        findCudaCodec(device, streamInfo.stream->codecpar->codec_id)
-            .value_or(avCodec));
+    const char* cuvid_suffix = "_cuvid";
+    size_t cuvid_length = std::strlen(avCodec->name) + std::strlen(cuvid_suffix) + 1;
+    char* cuvid_name = new char[cuvid_length];
+    std::strcpy(cuvid_name, avCodec->name);
+    std::strcat(cuvid_name, cuvid_suffix);
+    avCodec = avcodec_find_decoder_by_name(cuvid_name);
+    delete[] cuvid_name;
+    TORCH_CHECK(avCodec != nullptr);
   }

   AVCodecContext* codecContext = avcodec_alloc_context3(avCodec);
@@ -474,6 +479,12 @@ void VideoDecoder::addStream(
     throw std::invalid_argument(getFFMPEGErrorStringFromErrorCode(retVal));
   }

+  if (device.type() == torch::kCUDA) {
+    codecContext->hw_frames_ctx = av_hwframe_ctx_alloc(codecContext->hw_device_ctx);
+    AVHWFramesContext* hwframe_ctx = (AVHWFramesContext*)codecContext->hw_frames_ctx->data;
+    // in avcodec_open2, cuvid will not set sw_format until sequence callback in avcodec_send_packet()
+    hwframe_ctx->sw_format = codecContext->sw_pix_fmt;
+  }
   codecContext->time_base = streamInfo.stream->time_base;
   containerMetadata_.allStreamMetadata[activeStreamIndex_].codecName =
       std::string(avcodec_get_name(codecContext->codec_id));

Sample usage:

python sample_torchcodec.py

import torch

print(f"{torch.__version__=}")
print(f"{torch.cuda.is_available()=}")
print(f"{torch.cuda.get_device_properties(0)=}")

import torchcodec
from torchcodec.decoders import VideoDecoder

decoder = VideoDecoder("sample.mp4", device="cuda")
frame = decoder[0]

6.4. imageio-ffmpeg

imageio-ffmpeg is a plugin for imageio that provides a wrapper for FFmpeg. Similar to ffmpeg-python, it constructs and executes FFmpeg commands from input parameters. This requires your runtime environment to have FFmpeg installed or to have the bin and lib directories of your FFmpeg installation in the system's path. We do not recommend using this plugin, as the official documentation states: You should probably use PyAV instead; it is faster and offers more features.

imageio-ffmpeg: https://github.com/imageio/imageio-ffmpeg

Sample usage:

import imageio

video_reader = imageio.get_reader('input.mp4', 'ffmpeg', ffmpeg_params=['-c:v', 'h264_cuvid'])
frame_list = []
for i, frame in enumerate(video_reader):
    frame_list.append(frame)
imageio.mimsave('out.mp4', frame_list)

6.5. Decord

The default installation of Decord is a CPU-only version and does not support hardware acceleration. To use hardware acceleration, you must build and install Decord from source.

Decord has a dependency on FFmpeg but only uses its demuxing capabilities. You do not need to build a special version of FFmpeg with hardware acceleration enabled.

Older versions of FFmpeg contain an issue that can cause a deadlock in Decord. This issue was resolved in FFmpeg 7.0. Therefore, you must build and install FFmpeg 7.0.

Download Decord:

git clone -b v0.6.0 --depth 1 --recursive https://github.com/dmlc/decord
cd decord
git apply decord_ffmpeg7.patch

This patch primarily updates Decord's FFmpeg dependency from version 4.2 to 7.0:

decord_ffmpeg7.patch:

+++ b/src/video/nvcodec/cuda_threaded_decoder.cc
@@ -17,7 +17,7 @@ namespace decord {
 namespace cuda {
 using namespace runtime;

-CUThreadedDecoder::CUThreadedDecoder(int device_id, AVCodecParameters *codecpar, AVInputFormat *iformat)
+CUThreadedDecoder::CUThreadedDecoder(int device_id, AVCodecParameters *codecpar, const AVInputFormat *iformat)
     : device_id_(device_id), stream_({device_id, false}), device_{}, ctx_{}, parser_{}, decoder_{},
     pkt_queue_{}, frame_queue_{},
     run_(false), frame_count_(0), draining_(false),
@@ -70,7 +70,7 @@ CUThreadedDecoder::CUThreadedDecoder(int device_id, AVCodecParameters *codecpar,
     }
 }

-void CUThreadedDecoder::InitBitStreamFilter(AVCodecParameters *codecpar, AVInputFormat *iformat) {
+void CUThreadedDecoder::InitBitStreamFilter(AVCodecParameters *codecpar, const AVInputFormat *iformat) {
     const char* bsf_name = nullptr;
     if (AV_CODEC_ID_H264 == codecpar->codec_id) {
         // H.264
diff --git a/src/video/nvcodec/cuda_threaded_decoder.h b/src/video/nvcodec/cuda_threaded_decoder.h
index d7e6fcd..61958a1 100644
--- a/src/video/nvcodec/cuda_threaded_decoder.h
+++ b/src/video/nvcodec/cuda_threaded_decoder.h
@@ -46,7 +46,7 @@ class CUThreadedDecoder final : public ThreadedDecoderInterface {
     using FrameOrderQueuePtr = std::unique_ptr<FrameOrderQueue>;

     public:
-        CUThreadedDecoder(int device_id, AVCodecParameters *codecpar, AVInputFormat *iformat);
+        CUThreadedDecoder(int device_id, AVCodecParameters *codecpar, const AVInputFormat *iformat);
         void SetCodecContext(AVCodecContext *dec_ctx, int width = -1, int height = -1, int rotation = 0);
         bool Initialized() const;
         void Start();
@@ -70,7 +70,7 @@ class CUThreadedDecoder final : public ThreadedDecoderInterface {
         void LaunchThreadImpl();
         void RecordInternalError(std::string message);
         void CheckErrorStatus();
-        void InitBitStreamFilter(AVCodecParameters *codecpar, AVInputFormat *iformat);
+        void InitBitStreamFilter(AVCodecParameters *codecpar, const AVInputFormat *iformat);

         int device_id_;
         CUStream stream_;
diff --git a/src/video/video_reader.cc b/src/video/video_reader.cc
index af4858d..99c9635 100644
--- a/src/video/video_reader.cc
+++ b/src/video/video_reader.cc
@@ -145,7 +145,7 @@ VideoReader::~VideoReader(){

 void VideoReader::SetVideoStream(int stream_nb) {
     if (!fmt_ctx_) return;
-    AVCodec *dec;
+    const AVCodec *dec;
     int st_nb = av_find_best_stream(fmt_ctx_.get(), AVMEDIA_TYPE_VIDEO, stream_nb, -1, &dec, 0);
     // LOG(INFO) << "find best stream: " << st_nb;
     CHECK_GE(st_nb, 0) << "ERROR cannot find video stream with wanted index: " << stream_nb;

Build Decord:

mkdir build && cd build
cmake .. -DUSE_CUDA=ON -DCMAKE_BUILD_TYPE=Release
make

Install Decord:

cd ../python
python3 setup.py install --user

Sample usage:

import decord
import torch
from decord import gpu, cpu

video_path="input.mp4"
vr = decord.VideoReader(video_path, ctx=gpu(0))
nframes = 40
total_frames = 500
idx = torch.linspace(0, total_frames - 1, nframes).round().long().tolist()
video = vr.get_batch(idx).asnumpy()