From: Jan Palus Date: Mon, 18 Jul 2022 18:43:33 +0000 (+0200) Subject: drmprime zero-copy output patch from libreelec X-Git-Tag: auto/th/ffmpeg-4.4.3-1~2 X-Git-Url: http://git.pld-linux.org/gitweb.cgi?a=commitdiff_plain;ds=sidebyside;h=6838be8a1ac89118df3db3771d44ad3734485b11;p=packages%2Fffmpeg.git drmprime zero-copy output patch from libreelec --- diff --git a/ffmpeg.spec b/ffmpeg.spec index 220a559..5ecc142 100644 --- a/ffmpeg.spec +++ b/ffmpeg.spec @@ -72,6 +72,7 @@ %bcond_with tesseract # OCR filter based on Tesseract %bcond_without theora # Theora encoding via libtheora %bcond_with uavs3d # AVS3 decoding via libuavs3d (TODO: enable when 1.1.41 released) +%bcond_with v4l2_drmprime # Zero-copy v4l2 decoding %bcond_with v4l2_request # V4L2 request API for stateless hw decoding %bcond_without va # VAAPI (Video Acceleration API) %bcond_without vapoursynth # VapourSynth demuxer @@ -94,7 +95,7 @@ %undefine with_opencv %undefine with_chromaprint %endif -%if %{with rkmpp} || %{with v4l2_request} +%if %{with rkmpp} || %{with v4l2_request} || %{with v4l2_drmprime} %define with_libdrm 1 %endif @@ -131,6 +132,7 @@ Patch1: %{name}-atadenoise.patch Patch2: opencv4.patch Patch3: v4l2-request-hwdec.patch Patch4: %{name}-glslang.patch +Patch5: v4l2-drmprime.patch URL: http://www.ffmpeg.org/ %{?with_decklink:BuildRequires: Blackmagic_DeckLink_SDK >= 10.10} %{?with_openal:BuildRequires: OpenAL-devel >= 1.1} @@ -538,6 +540,9 @@ Dokumentacja pakietu FFmpeg w formacie HTML. %patch3 -p1 %endif %patch4 -p1 +%if %{with v4l2_drmprime} +%patch5 -p1 +%endif # package the grep result for mplayer, the result formatted as ./mplayer/configure cat < ffmpeg-avconfig diff --git a/v4l2-drmprime.patch b/v4l2-drmprime.patch new file mode 100644 index 0000000..d6acd23 --- /dev/null +++ b/v4l2-drmprime.patch @@ -0,0 +1,855 @@ +From 40a990827399c05ad3ce3f8242321bd8a67aa0bd Mon Sep 17 00:00:00 2001 +From: Lukas Rusak +Date: Tue, 24 Apr 2018 23:00:23 -0700 +Subject: [PATCH 1/9] libavcodec: v4l2m2m: output AVDRMFrameDescriptor + +This allows for a zero-copy output by exporting the v4l2 buffer then wrapping that buffer +in the AVDRMFrameDescriptor like it is done in rkmpp. + +This has been in use for quite some time with great success on many platforms including: + - Amlogic S905 + - Raspberry Pi + - i.MX6 + - Dragonboard 410c + +This was developed in conjunction with Kodi to allow handling the zero-copy buffer rendering. +A simply utility for testing is also available here: https://github.com/BayLibre/ffmpeg-drm + +todo: + - allow selecting pixel format output from decoder + - allow configuring amount of output and capture buffers + +V2: + - allow selecting AV_PIX_FMT_DRM_PRIME + +V3: + - use get_format to select AV_PIX_FMT_DRM_PRIME + - use hw_configs + - add handling of AV_PIX_FMT_YUV420P format (for raspberry pi) + - add handling of AV_PIX_FMT_YUYV422 format (for i.MX6 coda decoder) + +V4: + - rebased on 4.2.2 + +V5: + - rebased on 4.3 +--- + libavcodec/v4l2_buffers.c | 155 ++++++++++++++++++++++++++++++++++++-- + libavcodec/v4l2_buffers.h | 4 + + libavcodec/v4l2_context.c | 40 +++++++++- + libavcodec/v4l2_m2m.h | 3 + + libavcodec/v4l2_m2m_dec.c | 23 ++++++ + 5 files changed, 213 insertions(+), 12 deletions(-) + +diff --git a/libavcodec/v4l2_buffers.c b/libavcodec/v4l2_buffers.c +index 4b2679eb38..cbd3e5680d 100644 +--- a/libavcodec/v4l2_buffers.c ++++ b/libavcodec/v4l2_buffers.c +@@ -21,6 +21,7 @@ + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + ++#include + #include + #include + #include +@@ -30,6 +31,7 @@ + #include "libavcodec/avcodec.h" + #include "libavcodec/internal.h" + #include "libavutil/pixdesc.h" ++#include "libavutil/hwcontext.h" + #include "v4l2_context.h" + #include "v4l2_buffers.h" + #include "v4l2_m2m.h" +@@ -210,7 +212,79 @@ static enum AVColorTransferCharacteristic v4l2_get_color_trc(V4L2Buffer *buf) + return AVCOL_TRC_UNSPECIFIED; + } + +-static void v4l2_free_buffer(void *opaque, uint8_t *unused) ++static uint8_t * v4l2_get_drm_frame(V4L2Buffer *avbuf) ++{ ++ AVDRMFrameDescriptor *drm_desc = &avbuf->drm_frame; ++ AVDRMLayerDescriptor *layer; ++ ++ /* fill the DRM frame descriptor */ ++ drm_desc->nb_objects = avbuf->num_planes; ++ drm_desc->nb_layers = 1; ++ ++ layer = &drm_desc->layers[0]; ++ layer->nb_planes = avbuf->num_planes; ++ ++ for (int i = 0; i < avbuf->num_planes; i++) { ++ layer->planes[i].object_index = i; ++ layer->planes[i].offset = 0; ++ layer->planes[i].pitch = avbuf->plane_info[i].bytesperline; ++ } ++ ++ switch (avbuf->context->av_pix_fmt) { ++ case AV_PIX_FMT_YUYV422: ++ ++ layer->format = DRM_FORMAT_YUYV; ++ layer->nb_planes = 1; ++ ++ break; ++ ++ case AV_PIX_FMT_NV12: ++ case AV_PIX_FMT_NV21: ++ ++ layer->format = avbuf->context->av_pix_fmt == AV_PIX_FMT_NV12 ? ++ DRM_FORMAT_NV12 : DRM_FORMAT_NV21; ++ ++ if (avbuf->num_planes > 1) ++ break; ++ ++ layer->nb_planes = 2; ++ ++ layer->planes[1].object_index = 0; ++ layer->planes[1].offset = avbuf->plane_info[0].bytesperline * ++ avbuf->context->format.fmt.pix.height; ++ layer->planes[1].pitch = avbuf->plane_info[0].bytesperline; ++ break; ++ ++ case AV_PIX_FMT_YUV420P: ++ ++ layer->format = DRM_FORMAT_YUV420; ++ ++ if (avbuf->num_planes > 1) ++ break; ++ ++ layer->nb_planes = 3; ++ ++ layer->planes[1].object_index = 0; ++ layer->planes[1].offset = avbuf->plane_info[0].bytesperline * ++ avbuf->context->format.fmt.pix.height; ++ layer->planes[1].pitch = avbuf->plane_info[0].bytesperline >> 1; ++ ++ layer->planes[2].object_index = 0; ++ layer->planes[2].offset = layer->planes[1].offset + ++ ((avbuf->plane_info[0].bytesperline * ++ avbuf->context->format.fmt.pix.height) >> 2); ++ layer->planes[2].pitch = avbuf->plane_info[0].bytesperline >> 1; ++ break; ++ ++ default: ++ drm_desc->nb_layers = 0; ++ break; ++ } ++ ++ return (uint8_t *) drm_desc; ++} ++ ++static void v4l2_free_buffer(void *opaque, uint8_t *data) + { + V4L2Buffer* avbuf = opaque; + V4L2m2mContext *s = buf_to_m2mctx(avbuf); +@@ -234,6 +308,36 @@ static void v4l2_free_buffer(void *opaque, uint8_t *unused) + } + } + ++static int v4l2_buffer_export_drm(V4L2Buffer* avbuf) ++{ ++ struct v4l2_exportbuffer expbuf; ++ int i, ret; ++ ++ for (i = 0; i < avbuf->num_planes; i++) { ++ memset(&expbuf, 0, sizeof(expbuf)); ++ ++ expbuf.index = avbuf->buf.index; ++ expbuf.type = avbuf->buf.type; ++ expbuf.plane = i; ++ ++ ret = ioctl(buf_to_m2mctx(avbuf)->fd, VIDIOC_EXPBUF, &expbuf); ++ if (ret < 0) ++ return AVERROR(errno); ++ ++ if (V4L2_TYPE_IS_MULTIPLANAR(avbuf->buf.type)) { ++ /* drm frame */ ++ avbuf->drm_frame.objects[i].size = avbuf->buf.m.planes[i].length; ++ avbuf->drm_frame.objects[i].fd = expbuf.fd; ++ } else { ++ /* drm frame */ ++ avbuf->drm_frame.objects[0].size = avbuf->buf.length; ++ avbuf->drm_frame.objects[0].fd = expbuf.fd; ++ } ++ } ++ ++ return 0; ++} ++ + static int v4l2_buf_increase_ref(V4L2Buffer *in) + { + V4L2m2mContext *s = buf_to_m2mctx(in); +@@ -254,6 +358,24 @@ static int v4l2_buf_increase_ref(V4L2Buffer *in) + return 0; + } + ++static int v4l2_buf_to_bufref_drm(V4L2Buffer *in, AVBufferRef **buf) ++{ ++ int ret; ++ ++ *buf = av_buffer_create((uint8_t *) &in->drm_frame, ++ sizeof(in->drm_frame), ++ v4l2_free_buffer, ++ in, AV_BUFFER_FLAG_READONLY); ++ if (!*buf) ++ return AVERROR(ENOMEM); ++ ++ ret = v4l2_buf_increase_ref(in); ++ if (ret) ++ av_buffer_unref(buf); ++ ++ return ret; ++} ++ + static int v4l2_buf_to_bufref(V4L2Buffer *in, int plane, AVBufferRef **buf) + { + int ret; +@@ -303,13 +425,24 @@ static int v4l2_buffer_buf_to_swframe(AVFrame *frame, V4L2Buffer *avbuf) + + frame->format = avbuf->context->av_pix_fmt; + +- for (i = 0; i < avbuf->num_planes; i++) { +- ret = v4l2_buf_to_bufref(avbuf, i, &frame->buf[i]); ++ if (buf_to_m2mctx(avbuf)->output_drm) { ++ /* 1. get references to the actual data */ ++ ret = v4l2_buf_to_bufref_drm(avbuf, &frame->buf[0]); + if (ret) + return ret; + +- frame->linesize[i] = avbuf->plane_info[i].bytesperline; +- frame->data[i] = frame->buf[i]->data; ++ frame->data[0] = (uint8_t *) v4l2_get_drm_frame(avbuf); ++ frame->format = AV_PIX_FMT_DRM_PRIME; ++ } else { ++ /* 1. get references to the actual data */ ++ for (i = 0; i < avbuf->num_planes; i++) { ++ ret = v4l2_buf_to_bufref(avbuf, i, &frame->buf[i]); ++ if (ret) ++ return ret; ++ ++ frame->linesize[i] = avbuf->plane_info[i].bytesperline; ++ frame->data[i] = frame->buf[i]->data; ++ } + } + + /* fixup special cases */ +@@ -543,9 +676,6 @@ int ff_v4l2_buffer_initialize(V4L2Buffer* avbuf, int index) + + avbuf->status = V4L2BUF_AVAILABLE; + +- if (V4L2_TYPE_IS_OUTPUT(ctx->type)) +- return 0; +- + if (V4L2_TYPE_IS_MULTIPLANAR(ctx->type)) { + avbuf->buf.m.planes = avbuf->planes; + avbuf->buf.length = avbuf->num_planes; +@@ -555,6 +685,15 @@ int ff_v4l2_buffer_initialize(V4L2Buffer* avbuf, int index) + avbuf->buf.length = avbuf->planes[0].length; + } + ++ if (V4L2_TYPE_IS_OUTPUT(ctx->type)) ++ return 0; ++ ++ if (buf_to_m2mctx(avbuf)->output_drm) { ++ ret = v4l2_buffer_export_drm(avbuf); ++ if (ret) ++ return ret; ++ } ++ + return ff_v4l2_buffer_enqueue(avbuf); + } + +diff --git a/libavcodec/v4l2_buffers.h b/libavcodec/v4l2_buffers.h +index 8dbc7fc104..037e667997 100644 +--- a/libavcodec/v4l2_buffers.h ++++ b/libavcodec/v4l2_buffers.h +@@ -27,6 +27,7 @@ + #include + #include + ++#include "libavutil/hwcontext_drm.h" + #include "avcodec.h" + + enum V4L2Buffer_status { +@@ -42,6 +43,9 @@ typedef struct V4L2Buffer { + /* each buffer needs to have a reference to its context */ + struct V4L2Context *context; + ++ /* DRM descriptor */ ++ AVDRMFrameDescriptor drm_frame; ++ + /* This object is refcounted per-plane, so we need to keep track + * of how many context-refs we are holding. */ + AVBufferRef *context_ref; +diff --git a/libavcodec/v4l2_context.c b/libavcodec/v4l2_context.c +index ff1ea8e57b..e9e8c27a54 100644 +--- a/libavcodec/v4l2_context.c ++++ b/libavcodec/v4l2_context.c +@@ -455,22 +455,54 @@ static int v4l2_release_buffers(V4L2Context* ctx) + struct v4l2_requestbuffers req = { + .memory = V4L2_MEMORY_MMAP, + .type = ctx->type, +- .count = 0, /* 0 -> unmaps buffers from the driver */ ++ .count = 0, /* 0 -> unmap all buffers from the driver */ + }; +- int i, j; ++ int ret, i, j; + + for (i = 0; i < ctx->num_buffers; i++) { + V4L2Buffer *buffer = &ctx->buffers[i]; + + for (j = 0; j < buffer->num_planes; j++) { + struct V4L2Plane_info *p = &buffer->plane_info[j]; ++ ++ if (V4L2_TYPE_IS_OUTPUT(ctx->type)) { ++ /* output buffers are not EXPORTED */ ++ goto unmap; ++ } ++ ++ if (ctx_to_m2mctx(ctx)->output_drm) { ++ /* use the DRM frame to close */ ++ if (buffer->drm_frame.objects[j].fd >= 0) { ++ if (close(buffer->drm_frame.objects[j].fd) < 0) { ++ av_log(logger(ctx), AV_LOG_ERROR, "%s close drm fd " ++ "[buffer=%2d, plane=%d, fd=%2d] - %s \n", ++ ctx->name, i, j, buffer->drm_frame.objects[j].fd, ++ av_err2str(AVERROR(errno))); ++ } ++ } ++ } ++unmap: + if (p->mm_addr && p->length) + if (munmap(p->mm_addr, p->length) < 0) +- av_log(logger(ctx), AV_LOG_ERROR, "%s unmap plane (%s))\n", ctx->name, av_err2str(AVERROR(errno))); ++ av_log(logger(ctx), AV_LOG_ERROR, "%s unmap plane (%s))\n", ++ ctx->name, av_err2str(AVERROR(errno))); + } + } + +- return ioctl(ctx_to_m2mctx(ctx)->fd, VIDIOC_REQBUFS, &req); ++ ret = ioctl(ctx_to_m2mctx(ctx)->fd, VIDIOC_REQBUFS, &req); ++ if (ret < 0) { ++ av_log(logger(ctx), AV_LOG_ERROR, "release all %s buffers (%s)\n", ++ ctx->name, av_err2str(AVERROR(errno))); ++ ++ if (ctx_to_m2mctx(ctx)->output_drm) ++ av_log(logger(ctx), AV_LOG_ERROR, ++ "Make sure the DRM client releases all FB/GEM objects before closing the codec (ie):\n" ++ "for all buffers: \n" ++ " 1. drmModeRmFB(..)\n" ++ " 2. drmIoctl(.., DRM_IOCTL_GEM_CLOSE,... )\n"); ++ } ++ ++ return ret; + } + + static inline int v4l2_try_raw_format(V4L2Context* ctx, enum AVPixelFormat pixfmt) +diff --git a/libavcodec/v4l2_m2m.h b/libavcodec/v4l2_m2m.h +index b67b216331..0fbd19a013 100644 +--- a/libavcodec/v4l2_m2m.h ++++ b/libavcodec/v4l2_m2m.h +@@ -66,6 +66,9 @@ typedef struct V4L2m2mContext { + + /* reference back to V4L2m2mPriv */ + void *priv; ++ ++ /* generate DRM frames */ ++ int output_drm; + } V4L2m2mContext; + + typedef struct V4L2m2mPriv { +diff --git a/libavcodec/v4l2_m2m_dec.c b/libavcodec/v4l2_m2m_dec.c +index ab07c0a24a..6bc7442702 100644 +--- a/libavcodec/v4l2_m2m_dec.c ++++ b/libavcodec/v4l2_m2m_dec.c +@@ -23,6 +23,9 @@ + + #include + #include ++ ++#include "libavutil/hwcontext.h" ++#include "libavutil/hwcontext_drm.h" + #include "libavutil/pixfmt.h" + #include "libavutil/pixdesc.h" + #include "libavutil/opt.h" +@@ -30,6 +33,9 @@ + #include "libavcodec/decode.h" + #include "libavcodec/internal.h" + ++#include "libavcodec/hwaccels.h" ++#include "libavcodec/internal.h" ++ + #include "v4l2_context.h" + #include "v4l2_m2m.h" + #include "v4l2_fmt.h" +@@ -201,6 +207,15 @@ static av_cold int v4l2_decode_init(AVCodecContext *avctx) + capture->av_codec_id = AV_CODEC_ID_RAWVIDEO; + capture->av_pix_fmt = avctx->pix_fmt; + ++ /* the client requests the codec to generate DRM frames: ++ * - data[0] will therefore point to the returned AVDRMFrameDescriptor ++ * check the ff_v4l2_buffer_to_avframe conversion function. ++ * - the DRM frame format is passed in the DRM frame descriptor layer. ++ * check the v4l2_get_drm_frame function. ++ */ ++ if (ff_get_format(avctx, avctx->codec->pix_fmts) == AV_PIX_FMT_DRM_PRIME) ++ s->output_drm = 1; ++ + s->avctx = avctx; + ret = ff_v4l2_m2m_codec_init(priv); + if (ret) { +@@ -226,6 +241,11 @@ static const AVOption options[] = { + { NULL}, + }; + ++static const AVCodecHWConfigInternal *v4l2_m2m_hw_configs[] = { ++ HW_CONFIG_INTERNAL(DRM_PRIME), ++ NULL ++}; ++ + #define M2MDEC_CLASS(NAME) \ + static const AVClass v4l2_m2m_ ## NAME ## _dec_class = { \ + .class_name = #NAME "_v4l2m2m_decoder", \ +@@ -249,6 +269,9 @@ static const AVOption options[] = { + .bsfs = bsf_name, \ + .capabilities = AV_CODEC_CAP_HARDWARE | AV_CODEC_CAP_DELAY | AV_CODEC_CAP_AVOID_PROBING, \ + .caps_internal = FF_CODEC_CAP_SETS_PKT_DTS | FF_CODEC_CAP_INIT_CLEANUP, \ ++ .pix_fmts = (const enum AVPixelFormat[]) { AV_PIX_FMT_DRM_PRIME, \ ++ AV_PIX_FMT_NONE}, \ ++ .hw_configs = v4l2_m2m_hw_configs, \ + .wrapper_name = "v4l2m2m", \ + } + + +From 9a2a361c2c84c8da54cd3a74b0d0bb966df8fe69 Mon Sep 17 00:00:00 2001 +From: Lukas Rusak +Date: Thu, 16 Aug 2018 21:09:40 -0700 +Subject: [PATCH 2/9] libavcodec: v4l2m2m: depends on libdrm + +--- + configure | 1 + + libavcodec/v4l2_buffers.c | 2 +- + 2 files changed, 2 insertions(+), 1 deletion(-) + +diff --git a/configure b/configure +index d7a3f507e8..d203f6f7da 100755 +--- a/configure ++++ b/configure +@@ -3437,6 +3437,7 @@ sndio_indev_deps="sndio" + sndio_outdev_deps="sndio" + v4l2_indev_deps_any="linux_videodev2_h sys_videoio_h" + v4l2_indev_suggest="libv4l2" ++v4l2_outdev_deps="libdrm" + v4l2_outdev_deps_any="linux_videodev2_h sys_videoio_h" + v4l2_outdev_suggest="libv4l2" + vfwcap_indev_deps="vfw32 vfwcap_defines" +diff --git a/libavcodec/v4l2_buffers.c b/libavcodec/v4l2_buffers.c +index cbd3e5680d..bebe2c1796 100644 +--- a/libavcodec/v4l2_buffers.c ++++ b/libavcodec/v4l2_buffers.c +@@ -21,7 +21,7 @@ + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +-#include ++#include + #include + #include + #include + +From 7b0fa2d859c12a8a129c884d16673ca731336c06 Mon Sep 17 00:00:00 2001 +From: Lukas Rusak +Date: Thu, 16 Aug 2018 21:10:13 -0700 +Subject: [PATCH 3/9] libavcodec: v4l2m2m: set format_modifier to + DRM_FORMAT_MOD_LINEAR + +--- + libavcodec/v4l2_buffers.c | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/libavcodec/v4l2_buffers.c b/libavcodec/v4l2_buffers.c +index bebe2c1796..12037d5d66 100644 +--- a/libavcodec/v4l2_buffers.c ++++ b/libavcodec/v4l2_buffers.c +@@ -328,10 +328,12 @@ static int v4l2_buffer_export_drm(V4L2Buffer* avbuf) + /* drm frame */ + avbuf->drm_frame.objects[i].size = avbuf->buf.m.planes[i].length; + avbuf->drm_frame.objects[i].fd = expbuf.fd; ++ avbuf->drm_frame.objects[i].format_modifier = DRM_FORMAT_MOD_LINEAR; + } else { + /* drm frame */ + avbuf->drm_frame.objects[0].size = avbuf->buf.length; + avbuf->drm_frame.objects[0].fd = expbuf.fd; ++ avbuf->drm_frame.objects[0].format_modifier = DRM_FORMAT_MOD_LINEAR; + } + } + + +From e1857456c8f24e40d5c898886f2f51014e59ee9d Mon Sep 17 00:00:00 2001 +From: Lukas Rusak +Date: Thu, 16 Aug 2018 21:10:53 -0700 +Subject: [PATCH 4/9] libavcodec: v4l2m2m: only mmap the buffer when it is + output type and drm prime is used + +--- + libavcodec/v4l2_buffers.c | 20 ++++++++++++++------ + 1 file changed, 14 insertions(+), 6 deletions(-) + +diff --git a/libavcodec/v4l2_buffers.c b/libavcodec/v4l2_buffers.c +index 12037d5d66..1adf518ab9 100644 +--- a/libavcodec/v4l2_buffers.c ++++ b/libavcodec/v4l2_buffers.c +@@ -662,14 +662,22 @@ int ff_v4l2_buffer_initialize(V4L2Buffer* avbuf, int index) + + if (V4L2_TYPE_IS_MULTIPLANAR(ctx->type)) { + avbuf->plane_info[i].length = avbuf->buf.m.planes[i].length; +- avbuf->plane_info[i].mm_addr = mmap(NULL, avbuf->buf.m.planes[i].length, +- PROT_READ | PROT_WRITE, MAP_SHARED, +- buf_to_m2mctx(avbuf)->fd, avbuf->buf.m.planes[i].m.mem_offset); ++ ++ if ((V4L2_TYPE_IS_OUTPUT(ctx->type) && buf_to_m2mctx(avbuf)->output_drm) || ++ !buf_to_m2mctx(avbuf)->output_drm) { ++ avbuf->plane_info[i].mm_addr = mmap(NULL, avbuf->buf.m.planes[i].length, ++ PROT_READ | PROT_WRITE, MAP_SHARED, ++ buf_to_m2mctx(avbuf)->fd, avbuf->buf.m.planes[i].m.mem_offset); ++ } + } else { + avbuf->plane_info[i].length = avbuf->buf.length; +- avbuf->plane_info[i].mm_addr = mmap(NULL, avbuf->buf.length, +- PROT_READ | PROT_WRITE, MAP_SHARED, +- buf_to_m2mctx(avbuf)->fd, avbuf->buf.m.offset); ++ ++ if ((V4L2_TYPE_IS_OUTPUT(ctx->type) && buf_to_m2mctx(avbuf)->output_drm) || ++ !buf_to_m2mctx(avbuf)->output_drm) { ++ avbuf->plane_info[i].mm_addr = mmap(NULL, avbuf->buf.length, ++ PROT_READ | PROT_WRITE, MAP_SHARED, ++ buf_to_m2mctx(avbuf)->fd, avbuf->buf.m.offset); ++ } + } + + if (avbuf->plane_info[i].mm_addr == MAP_FAILED) + +From c8fc3ea1b5777546f7ec72a54b053a2d4fa9fd59 Mon Sep 17 00:00:00 2001 +From: Lukas Rusak +Date: Thu, 16 Aug 2018 21:11:38 -0700 +Subject: [PATCH 5/9] libavcodec: v4l2m2m: allow using software pixel formats + +--- + libavcodec/v4l2_m2m_dec.c | 11 ++++++++++- + 1 file changed, 10 insertions(+), 1 deletion(-) + +diff --git a/libavcodec/v4l2_m2m_dec.c b/libavcodec/v4l2_m2m_dec.c +index 6bc7442702..4b9baf833c 100644 +--- a/libavcodec/v4l2_m2m_dec.c ++++ b/libavcodec/v4l2_m2m_dec.c +@@ -213,8 +213,16 @@ static av_cold int v4l2_decode_init(AVCodecContext *avctx) + * - the DRM frame format is passed in the DRM frame descriptor layer. + * check the v4l2_get_drm_frame function. + */ +- if (ff_get_format(avctx, avctx->codec->pix_fmts) == AV_PIX_FMT_DRM_PRIME) ++ switch (ff_get_format(avctx, avctx->codec->pix_fmts)) { ++ case AV_PIX_FMT_DRM_PRIME: + s->output_drm = 1; ++ break; ++ case AV_PIX_FMT_NONE: ++ return 0; ++ break; ++ default: ++ break; ++ } + + s->avctx = avctx; + ret = ff_v4l2_m2m_codec_init(priv); +@@ -270,6 +278,7 @@ static const AVCodecHWConfigInternal *v4l2_m2m_hw_configs[] = { + .capabilities = AV_CODEC_CAP_HARDWARE | AV_CODEC_CAP_DELAY | AV_CODEC_CAP_AVOID_PROBING, \ + .caps_internal = FF_CODEC_CAP_SETS_PKT_DTS | FF_CODEC_CAP_INIT_CLEANUP, \ + .pix_fmts = (const enum AVPixelFormat[]) { AV_PIX_FMT_DRM_PRIME, \ ++ AV_PIX_FMT_NV12, \ + AV_PIX_FMT_NONE}, \ + .hw_configs = v4l2_m2m_hw_configs, \ + .wrapper_name = "v4l2m2m", \ + +From 13f02e940f083f19dbe8b9ac8fc7df45700dd36e Mon Sep 17 00:00:00 2001 +From: Lukas Rusak +Date: Mon, 24 Sep 2018 13:39:31 -0700 +Subject: [PATCH 6/9] libavcodec: v4l2m2m: implement hwcontext + +--- + libavcodec/v4l2_buffers.c | 22 ++++++++++++++++++++++ + libavcodec/v4l2_context.h | 2 ++ + libavcodec/v4l2_m2m.h | 2 ++ + libavcodec/v4l2_m2m_dec.c | 11 +++++++++++ + 4 files changed, 37 insertions(+) + +diff --git a/libavcodec/v4l2_buffers.c b/libavcodec/v4l2_buffers.c +index 1adf518ab9..6e2a544394 100644 +--- a/libavcodec/v4l2_buffers.c ++++ b/libavcodec/v4l2_buffers.c +@@ -435,6 +435,7 @@ static int v4l2_buffer_buf_to_swframe(AVFrame *frame, V4L2Buffer *avbuf) + + frame->data[0] = (uint8_t *) v4l2_get_drm_frame(avbuf); + frame->format = AV_PIX_FMT_DRM_PRIME; ++ frame->hw_frames_ctx = av_buffer_ref(avbuf->context->frames_ref); + } else { + /* 1. get references to the actual data */ + for (i = 0; i < avbuf->num_planes; i++) { +@@ -635,6 +636,27 @@ int ff_v4l2_buffer_initialize(V4L2Buffer* avbuf, int index) + avbuf->buf.type = ctx->type; + avbuf->buf.index = index; + ++ if (buf_to_m2mctx(avbuf)->output_drm) { ++ AVHWFramesContext *hwframes; ++ ++ av_buffer_unref(&ctx->frames_ref); ++ ++ ctx->frames_ref = av_hwframe_ctx_alloc(buf_to_m2mctx(avbuf)->device_ref); ++ if (!ctx->frames_ref) { ++ ret = AVERROR(ENOMEM); ++ return ret; ++ } ++ ++ hwframes = (AVHWFramesContext*)ctx->frames_ref->data; ++ hwframes->format = AV_PIX_FMT_DRM_PRIME; ++ hwframes->sw_format = ctx->av_pix_fmt; ++ hwframes->width = ctx->width; ++ hwframes->height = ctx->height; ++ ret = av_hwframe_ctx_init(ctx->frames_ref); ++ if (ret < 0) ++ return ret; ++ } ++ + if (V4L2_TYPE_IS_MULTIPLANAR(ctx->type)) { + avbuf->buf.length = VIDEO_MAX_PLANES; + avbuf->buf.m.planes = avbuf->planes; +diff --git a/libavcodec/v4l2_context.h b/libavcodec/v4l2_context.h +index 22a9532444..e804e94131 100644 +--- a/libavcodec/v4l2_context.h ++++ b/libavcodec/v4l2_context.h +@@ -92,6 +92,8 @@ typedef struct V4L2Context { + */ + int done; + ++ AVBufferRef *frames_ref; ++ + } V4L2Context; + + /** +diff --git a/libavcodec/v4l2_m2m.h b/libavcodec/v4l2_m2m.h +index 0fbd19a013..adf5997bb5 100644 +--- a/libavcodec/v4l2_m2m.h ++++ b/libavcodec/v4l2_m2m.h +@@ -67,6 +67,8 @@ typedef struct V4L2m2mContext { + /* reference back to V4L2m2mPriv */ + void *priv; + ++ AVBufferRef *device_ref; ++ + /* generate DRM frames */ + int output_drm; + } V4L2m2mContext; +diff --git a/libavcodec/v4l2_m2m_dec.c b/libavcodec/v4l2_m2m_dec.c +index 4b9baf833c..6c23693137 100644 +--- a/libavcodec/v4l2_m2m_dec.c ++++ b/libavcodec/v4l2_m2m_dec.c +@@ -35,6 +35,7 @@ + + #include "libavcodec/hwaccels.h" + #include "libavcodec/internal.h" ++#include "libavcodec/hwconfig.h" + + #include "v4l2_context.h" + #include "v4l2_m2m.h" +@@ -224,6 +225,16 @@ static av_cold int v4l2_decode_init(AVCodecContext *avctx) + break; + } + ++ s->device_ref = av_hwdevice_ctx_alloc(AV_HWDEVICE_TYPE_DRM); ++ if (!s->device_ref) { ++ ret = AVERROR(ENOMEM); ++ return ret; ++ } ++ ++ ret = av_hwdevice_ctx_init(s->device_ref); ++ if (ret < 0) ++ return ret; ++ + s->avctx = avctx; + ret = ff_v4l2_m2m_codec_init(priv); + if (ret) { + +From 34be198b8039c9df434792f19f0985e45419407e Mon Sep 17 00:00:00 2001 +From: Lukas Rusak +Date: Mon, 4 May 2020 13:01:29 -0700 +Subject: [PATCH 7/9] libavcodec: v4l2m2m: allow lower minimum buffer values + +There is no reason to enforce a high minimum. In the context +of streaming only a few output buffers and capture buffers +are even needed for continuous playback. This also helps +alleviate memory pressure when decoding 4K media. +--- + libavcodec/v4l2_m2m.h | 2 +- + libavcodec/v4l2_m2m_dec.c | 2 +- + 2 files changed, 2 insertions(+), 2 deletions(-) + +diff --git a/libavcodec/v4l2_m2m.h b/libavcodec/v4l2_m2m.h +index adf5997bb5..1082b9dad2 100644 +--- a/libavcodec/v4l2_m2m.h ++++ b/libavcodec/v4l2_m2m.h +@@ -38,7 +38,7 @@ + + #define V4L_M2M_DEFAULT_OPTS \ + { "num_output_buffers", "Number of buffers in the output context",\ +- OFFSET(num_output_buffers), AV_OPT_TYPE_INT, { .i64 = 16 }, 6, INT_MAX, FLAGS } ++ OFFSET(num_output_buffers), AV_OPT_TYPE_INT, { .i64 = 16 }, 2, INT_MAX, FLAGS } + + typedef struct V4L2m2mContext { + char devname[PATH_MAX]; +diff --git a/libavcodec/v4l2_m2m_dec.c b/libavcodec/v4l2_m2m_dec.c +index 6c23693137..e323c37052 100644 +--- a/libavcodec/v4l2_m2m_dec.c ++++ b/libavcodec/v4l2_m2m_dec.c +@@ -256,7 +256,7 @@ static av_cold int v4l2_decode_close(AVCodecContext *avctx) + static const AVOption options[] = { + V4L_M2M_DEFAULT_OPTS, + { "num_capture_buffers", "Number of buffers in the capture context", +- OFFSET(num_capture_buffers), AV_OPT_TYPE_INT, {.i64 = 20}, 20, INT_MAX, FLAGS }, ++ OFFSET(num_capture_buffers), AV_OPT_TYPE_INT, {.i64 = 20}, 2, INT_MAX, FLAGS }, + { NULL}, + }; + + +From 2956fd1881d28abf6bf77bd9a57866c4ba81d199 Mon Sep 17 00:00:00 2001 +From: Lukas Rusak +Date: Wed, 6 May 2020 11:12:58 -0700 +Subject: [PATCH 8/9] libavcodec: v4l2m2m: add option to specify pixel format + used by the decoder + +--- + libavcodec/v4l2_context.c | 9 +++++++++ + libavcodec/v4l2_m2m.h | 2 ++ + libavcodec/v4l2_m2m_dec.c | 1 + + 3 files changed, 12 insertions(+) + +diff --git a/libavcodec/v4l2_context.c b/libavcodec/v4l2_context.c +index e9e8c27a54..a97b70e836 100644 +--- a/libavcodec/v4l2_context.c ++++ b/libavcodec/v4l2_context.c +@@ -531,6 +531,8 @@ static inline int v4l2_try_raw_format(V4L2Context* ctx, enum AVPixelFormat pixfm + + static int v4l2_get_raw_format(V4L2Context* ctx, enum AVPixelFormat *p) + { ++ V4L2m2mContext* s = ctx_to_m2mctx(ctx); ++ V4L2m2mPriv *priv = s->avctx->priv_data; + enum AVPixelFormat pixfmt = ctx->av_pix_fmt; + struct v4l2_fmtdesc fdesc; + int ret; +@@ -549,6 +551,13 @@ static int v4l2_get_raw_format(V4L2Context* ctx, enum AVPixelFormat *p) + if (ret) + return AVERROR(EINVAL); + ++ if (priv->pix_fmt != AV_PIX_FMT_NONE) { ++ if (fdesc.pixelformat != ff_v4l2_format_avfmt_to_v4l2(priv->pix_fmt)) { ++ fdesc.index++; ++ continue; ++ } ++ } ++ + pixfmt = ff_v4l2_format_v4l2_to_avfmt(fdesc.pixelformat, AV_CODEC_ID_RAWVIDEO); + ret = v4l2_try_raw_format(ctx, pixfmt); + if (ret){ +diff --git a/libavcodec/v4l2_m2m.h b/libavcodec/v4l2_m2m.h +index 1082b9dad2..943a8923c4 100644 +--- a/libavcodec/v4l2_m2m.h ++++ b/libavcodec/v4l2_m2m.h +@@ -30,6 +30,7 @@ + #include + + #include "libavcodec/avcodec.h" ++#include "libavutil/pixfmt.h" + #include "v4l2_context.h" + + #define container_of(ptr, type, member) ({ \ +@@ -81,6 +82,7 @@ typedef struct V4L2m2mPriv { + + int num_output_buffers; + int num_capture_buffers; ++ enum AVPixelFormat pix_fmt; + } V4L2m2mPriv; + + /** +diff --git a/libavcodec/v4l2_m2m_dec.c b/libavcodec/v4l2_m2m_dec.c +index e323c37052..363e998142 100644 +--- a/libavcodec/v4l2_m2m_dec.c ++++ b/libavcodec/v4l2_m2m_dec.c +@@ -257,6 +257,7 @@ static const AVOption options[] = { + V4L_M2M_DEFAULT_OPTS, + { "num_capture_buffers", "Number of buffers in the capture context", + OFFSET(num_capture_buffers), AV_OPT_TYPE_INT, {.i64 = 20}, 2, INT_MAX, FLAGS }, ++ { "pixel_format", "Pixel format to be used by the decoder", OFFSET(pix_fmt), AV_OPT_TYPE_PIXEL_FMT, {.i64 = AV_PIX_FMT_NONE}, AV_PIX_FMT_NONE, AV_PIX_FMT_NB, FLAGS }, + { NULL}, + }; + + +From 2bb6d0cb244590f0c70dd111ed978cd87fa3bee1 Mon Sep 17 00:00:00 2001 +From: Lukas Rusak +Date: Mon, 24 Sep 2018 13:39:56 -0700 +Subject: [PATCH 9/9] libavcodec: v4l2m2m: implement flush + +--- + libavcodec/v4l2_m2m_dec.c | 36 ++++++++++++++++++++++++++++++++++++ + 1 file changed, 36 insertions(+) + +diff --git a/libavcodec/v4l2_m2m_dec.c b/libavcodec/v4l2_m2m_dec.c +index 363e998142..52ec67cb59 100644 +--- a/libavcodec/v4l2_m2m_dec.c ++++ b/libavcodec/v4l2_m2m_dec.c +@@ -250,6 +250,41 @@ static av_cold int v4l2_decode_close(AVCodecContext *avctx) + return ff_v4l2_m2m_codec_end(avctx->priv_data); + } + ++static void v4l2_decode_flush(AVCodecContext *avctx) ++{ ++ V4L2m2mPriv *priv = avctx->priv_data; ++ V4L2m2mContext* s = priv->context; ++ V4L2Context* output = &s->output; ++ V4L2Context* capture = &s->capture; ++ int ret, i; ++ ++ ret = ff_v4l2_context_set_status(output, VIDIOC_STREAMOFF); ++ if (ret < 0) ++ av_log(avctx, AV_LOG_ERROR, "VIDIOC_STREAMOFF %s error: %d\n", output->name, ret); ++ ++ ret = ff_v4l2_context_set_status(output, VIDIOC_STREAMON); ++ if (ret < 0) ++ av_log(avctx, AV_LOG_ERROR, "VIDIOC_STREAMON %s error: %d\n", output->name, ret); ++ ++ for (i = 0; i < output->num_buffers; i++) { ++ if (output->buffers[i].status == V4L2BUF_IN_DRIVER) ++ output->buffers[i].status = V4L2BUF_AVAILABLE; ++ } ++ ++ struct v4l2_decoder_cmd cmd = { ++ .cmd = V4L2_DEC_CMD_START, ++ .flags = 0, ++ }; ++ ++ ret = ioctl(s->fd, VIDIOC_DECODER_CMD, &cmd); ++ if (ret < 0) ++ av_log(avctx, AV_LOG_ERROR, "VIDIOC_DECODER_CMD start error: %d\n", errno); ++ ++ s->draining = 0; ++ output->done = 0; ++ capture->done = 0; ++} ++ + #define OFFSET(x) offsetof(V4L2m2mPriv, x) + #define FLAGS AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_DECODING_PARAM + +@@ -286,6 +321,7 @@ static const AVCodecHWConfigInternal *v4l2_m2m_hw_configs[] = { + .init = v4l2_decode_init, \ + .receive_frame = v4l2_receive_frame, \ + .close = v4l2_decode_close, \ ++ .flush = v4l2_decode_flush, \ + .bsfs = bsf_name, \ + .capabilities = AV_CODEC_CAP_HARDWARE | AV_CODEC_CAP_DELAY | AV_CODEC_CAP_AVOID_PROBING, \ + .caps_internal = FF_CODEC_CAP_SETS_PKT_DTS | FF_CODEC_CAP_INIT_CLEANUP, \