1 From 771ef0eb163d23d172527e11ce1b587a03cfa745 Mon Sep 17 00:00:00 2001
2 From: Haihua Hu <jared.hu@nxp.com>
3 Date: Sun, 13 Nov 2016 00:45:29 +0800
4 Subject: [PATCH] MMFMWK-6930 [glplugin] Accelerate gldownload with
6 Content-Type: text/plain; charset=UTF-8
7 Content-Transfer-Encoding: 8bit
9 1) Propose a physical buffer pool to upstream in gldownload
10 2) Bind the physical buffer with texture via dirctviv
11 3) In gldownload, wrap the physical buffer to gstbuffer, pass to
13 4) Add some configure check for g2d and phymem
15 Upstream-Status: Inappropriate [i.MX specific]
17 Signed-off-by: Haihua Hu <jared.hu@nxp.com>
20 ext/gl/gstgldownloadelement.c | 99 +++++++++++++++
21 gst-libs/gst/gl/Makefile.am | 12 ++
22 gst-libs/gst/gl/gstglbufferpool.c | 16 +++
23 gst-libs/gst/gl/gstglphymemory.c | 254 ++++++++++++++++++++++++++++++++++++++
24 gst-libs/gst/gl/gstglphymemory.h | 43 +++++++
25 6 files changed, 435 insertions(+)
26 create mode 100644 gst-libs/gst/gl/gstglphymemory.c
27 create mode 100644 gst-libs/gst/gl/gstglphymemory.h
29 diff --git a/configure.ac b/configure.ac
30 index 3dc2b75..90bf768 100644
33 @@ -702,6 +702,7 @@ HAVE_GLES3_H=no
41 @@ -742,6 +743,7 @@ case $host in
42 dnl check fb backend for imx soc
43 AC_CHECK_LIB(EGL, fbGetDisplay, HAVE_FB_EGL=yes, HAVE_FB_EGL=no)
44 AC_CHECK_LIB(GLESv2, glTexDirectVIV, HAVE_DIRECTVIV=yes, HAVE_DIRECTVIV=no)
45 + AC_CHECK_HEADER(g2d.h, HAVE_G2D=yes, HAVE_G2D=no)
47 dnl FIXME: Mali EGL depends on GLESv1 or GLESv2
48 AC_CHECK_HEADER([EGL/fbdev_window.h],
49 @@ -854,8 +856,14 @@ fi
51 dnl specific for imx soc
52 GST_GL_HAVE_DIRECTVIV=0
54 if test "x$HAVE_DIRECTVIV" = "xyes"; then
55 GST_GL_HAVE_DIRECTVIV=1
56 + if test "x$HAVE_G2D" = "xyes"; then
57 + GST_GL_HAVE_PHYMEM=1
59 + AC_MSG_WARN([Physical memory do not support])
64 @@ -1317,6 +1325,7 @@ GL_CONFIG_DEFINES="$GL_CONFIG_DEFINES
65 GL_CONFIG_DEFINES="$GL_CONFIG_DEFINES
66 #define GST_GL_HAVE_DMABUF $GST_GL_HAVE_DMABUF
67 #define GST_GL_HAVE_DIRECTVIV $GST_GL_HAVE_DIRECTVIV
68 +#define GST_GL_HAVE_PHYMEM $GST_GL_HAVE_PHYMEM
71 dnl Check for no platforms/window systems
72 @@ -1354,6 +1363,7 @@ if test "x$GL_APIS" = "x" -o "x$GL_PLATFORMS" = "x" -o "x$GL_WINDOWS" = "x"; the
80 @@ -1371,6 +1381,7 @@ AM_CONDITIONAL(HAVE_WINDOW_ANDROID, test "x$HAVE_WINDOW_ANDROID" = "xyes")
81 AM_CONDITIONAL(HAVE_WINDOW_EAGL, test "x$HAVE_WINDOW_EAGL" = "xyes")
82 AM_CONDITIONAL(HAVE_WINDOW_FB, test "x$HAVE_WINDOW_FB" = "xyes")
83 AM_CONDITIONAL(HAVE_DIRECTVIV, test "x$HAVE_DIRECTVIV" = "xyes")
84 +AM_CONDITIONAL(HAVE_PHYMEM, test "x$HAVE_DIRECTVIV" = "xyes" -a "x$HAVE_G2D" = "xyes")
86 AM_CONDITIONAL(USE_OPENGL, test "x$USE_OPENGL" = "xyes")
87 AM_CONDITIONAL(USE_GLES2, test "x$USE_GLES2" = "xyes")
88 diff --git a/ext/gl/gstgldownloadelement.c b/ext/gl/gstgldownloadelement.c
89 index 058398b..a5ba6ab 100644
90 --- a/ext/gl/gstgldownloadelement.c
91 +++ b/ext/gl/gstgldownloadelement.c
93 #include <gst/gl/gl.h>
94 #include "gstgldownloadelement.h"
96 +#if GST_GL_HAVE_PHYMEM
97 +#include <gst/gl/gstglphymemory.h>
100 GST_DEBUG_CATEGORY_STATIC (gst_gl_download_element_debug);
101 #define GST_CAT_DEFAULT gst_gl_download_element_debug
103 @@ -45,6 +49,8 @@ gst_gl_download_element_prepare_output_buffer (GstBaseTransform * bt,
104 GstBuffer * buffer, GstBuffer ** outbuf);
105 static GstFlowReturn gst_gl_download_element_transform (GstBaseTransform * bt,
106 GstBuffer * buffer, GstBuffer * outbuf);
107 +static gboolean gst_gl_download_element_propose_allocation (GstBaseTransform *
108 + bt, GstQuery * decide_query, GstQuery * query);
110 static GstStaticPadTemplate gst_gl_download_element_src_pad_template =
111 GST_STATIC_PAD_TEMPLATE ("src",
112 @@ -70,6 +76,7 @@ gst_gl_download_element_class_init (GstGLDownloadElementClass * klass)
113 bt_class->prepare_output_buffer =
114 gst_gl_download_element_prepare_output_buffer;
115 bt_class->transform = gst_gl_download_element_transform;
116 + bt_class->propose_allocation = gst_gl_download_element_propose_allocation;
118 bt_class->passthrough_on_same_caps = TRUE;
120 @@ -160,9 +167,26 @@ static GstFlowReturn
121 gst_gl_download_element_prepare_output_buffer (GstBaseTransform * bt,
122 GstBuffer * inbuf, GstBuffer ** outbuf)
124 + GstGLDownloadElement *download = GST_GL_DOWNLOAD_ELEMENT (bt);
125 GstCaps *src_caps = gst_pad_get_current_caps (bt->srcpad);
126 GstCapsFeatures *features = NULL;
128 + GstGLMemory *glmem;
130 +#if GST_GL_HAVE_PHYMEM
131 + glmem = gst_buffer_peek_memory (inbuf, 0);
132 + if (gst_is_gl_physical_memory (glmem)) {
133 + GstGLContext *context = GST_GL_BASE_FILTER (bt)->context;
136 + gst_video_info_from_caps (&info, src_caps);
137 + *outbuf = gst_gl_phymem_buffer_to_gstbuffer (context, &info, inbuf);
139 + GST_DEBUG_OBJECT (download, "gl download with direct viv.");
141 + return GST_FLOW_OK;
143 +#endif /* GST_GL_HAVE_PHYMEM */
147 @@ -194,3 +218,78 @@ gst_gl_download_element_transform (GstBaseTransform * bt,
153 +gst_gl_download_element_propose_allocation (GstBaseTransform * bt,
154 + GstQuery * decide_query, GstQuery * query)
156 + GstGLContext *context = GST_GL_BASE_FILTER (bt)->context;
157 + GstGLDownloadElement *download = GST_GL_DOWNLOAD_ELEMENT (bt);
158 + GstAllocationParams params;
159 + GstAllocator *allocator = NULL;
160 + GstBufferPool *pool = NULL;
164 + GstStructure *config;
167 + gst_query_parse_allocation (query, &caps, NULL);
168 + if (!gst_video_info_from_caps (&info, caps)) {
169 + GST_WARNING_OBJECT (bt, "invalid caps specified");
173 + GST_DEBUG_OBJECT (bt, "video format is %s", gst_video_format_to_string (GST_VIDEO_INFO_FORMAT (&info)));
175 + gst_allocation_params_init (¶ms);
177 +#if GST_GL_HAVE_PHYMEM
178 + if (gst_is_gl_physical_memory_supported_fmt (&info)) {
179 + allocator = gst_phy_mem_allocator_obtain ();
180 + GST_DEBUG_OBJECT (bt, "obtain physical memory allocator %p.", allocator);
182 +#endif /* GST_GL_HAVE_PHYMEM */
185 + allocator = gst_allocator_find (GST_GL_MEMORY_ALLOCATOR_NAME);
188 + GST_ERROR_OBJECT (bt, "Can't obtain gl memory allocator.");
192 + gst_query_add_allocation_param (query, allocator, ¶ms);
193 + gst_object_unref (allocator);
195 + n_pools = gst_query_get_n_allocation_pools (query);
196 + for (i = 0; i < n_pools; i++) {
197 + gst_query_parse_nth_allocation_pool (query, i, &pool, NULL, NULL, NULL);
198 + gst_object_unref (pool);
203 + pool = gst_gl_buffer_pool_new (context);
204 + config = gst_buffer_pool_get_config (pool);
206 + /* the normal size of a frame */
208 + gst_buffer_pool_config_set_params (config, caps, size, 0, 0);
209 + gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_GL_SYNC_META);
211 + if (!gst_buffer_pool_set_config (pool, config)) {
212 + gst_object_unref (pool);
213 + GST_WARNING_OBJECT (bt, "failed setting config");
217 + GST_DEBUG_OBJECT (download, "create pool %p", pool);
219 + //propose 3 buffers for better performance
220 + gst_query_add_allocation_pool (query, pool, size, 3, 0);
222 + gst_object_unref (pool);
226 diff --git a/gst-libs/gst/gl/Makefile.am b/gst-libs/gst/gl/Makefile.am
227 index 4fec88a..0be4fa2 100644
228 --- a/gst-libs/gst/gl/Makefile.am
229 +++ b/gst-libs/gst/gl/Makefile.am
230 @@ -40,6 +40,10 @@ if HAVE_DIRECTVIV
231 libgstgl_@GST_API_VERSION@_la_SOURCES += gstglvivdirecttexture.c
235 +libgstgl_@GST_API_VERSION@_la_SOURCES += gstglphymemory.c
238 libgstgl_@GST_API_VERSION@includedir = $(includedir)/gstreamer-@GST_API_VERSION@/gst/gl
239 libgstgl_@GST_API_VERSION@include_HEADERS = \
241 @@ -78,6 +82,10 @@ if HAVE_DIRECTVIV
242 libgstgl_@GST_API_VERSION@include_HEADERS += gstglvivdirecttexture.h
246 +libgstgl_@GST_API_VERSION@include_HEADERS += gstglphymemory.h
251 utils/opengl_versions.h \
252 @@ -92,6 +100,10 @@ libgstgl_@GST_API_VERSION@_la_LIBADD = \
257 +libgstgl_@GST_API_VERSION@_la_LIBADD += -lg2d
262 libgstgl_@GST_API_VERSION@_la_LIBADD += win32/libgstgl-win32.la
263 diff --git a/gst-libs/gst/gl/gstglbufferpool.c b/gst-libs/gst/gl/gstglbufferpool.c
264 index 218169a..1ca6d8c 100644
265 --- a/gst-libs/gst/gl/gstglbufferpool.c
266 +++ b/gst-libs/gst/gl/gstglbufferpool.c
268 #include "gstglbufferpool.h"
269 #include "gstglutils.h"
271 +#if GST_GL_HAVE_PHYMEM
272 +#include <gst/gl/gstglphymemory.h>
276 * SECTION:gstglbufferpool
277 * @short_description: buffer pool for #GstGLMemory objects
278 @@ -261,6 +265,18 @@ gst_gl_buffer_pool_alloc (GstBufferPool * pool, GstBuffer ** buffer,
282 +#if GST_GL_HAVE_PHYMEM
283 + if ((g_strcmp0 (priv->allocator->mem_type, GST_GL_PHY_MEM_ALLOCATOR) == 0)) {
284 + GstAllocator* allocator = (GstAllocator*) gst_phy_mem_allocator_obtain ();
285 + if (!gst_gl_physical_memory_setup_buffer (allocator, buf, priv->gl_params)) {
286 + GST_ERROR_OBJECT (pool, "Can't create physcial buffer.");
287 + return GST_FLOW_ERROR;
290 + return GST_FLOW_OK;
294 alloc = GST_GL_MEMORY_ALLOCATOR (priv->allocator);
295 if (!gst_gl_memory_setup_buffer (alloc, buf, priv->gl_params, NULL, NULL, 0))
296 goto mem_create_failed;
297 diff --git a/gst-libs/gst/gl/gstglphymemory.c b/gst-libs/gst/gl/gstglphymemory.c
299 index 0000000..e28546c
301 +++ b/gst-libs/gst/gl/gstglphymemory.c
305 + * Copyright (c) 2015, Freescale Semiconductor, Inc.
307 + * This library is free software; you can redistribute it and/or
308 + * modify it under the terms of the GNU Library General Public
309 + * License as published by the Free Software Foundation; either
310 + * version 2 of the License, or (at your option) any later version.
312 + * This library is distributed in the hope that it will be useful,
313 + * but WITHOUT ANY WARRANTY; without even the implied warranty of
314 + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
315 + * Library General Public License for more details.
317 + * You should have received a copy of the GNU Library General Public
318 + * License along with this library; if not, write to the
319 + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
320 + * Boston, MA 02110-1301, USA.
323 +#ifdef HAVE_CONFIG_H
327 +#include "gstglvivdirecttexture.h"
328 +#include "gstglphymemory.h"
331 +GST_DEBUG_CATEGORY_STATIC (GST_CAT_GL_PHY_MEMORY);
332 +#define GST_CAT_DEFAULT GST_CAT_GL_PHY_MEMORY
334 +typedef struct _GstPhyMemAllocator GstPhyMemAllocator;
335 +typedef struct _GstPhyMemAllocatorClass GstPhyMemAllocatorClass;
337 +struct _GstPhyMemAllocator
339 + GstAllocatorPhyMem parent;
342 +struct _GstPhyMemAllocatorClass
344 + GstAllocatorPhyMemClass parent_class;
347 +GType gst_phy_mem_allocator_get_type (void);
348 +G_DEFINE_TYPE (GstPhyMemAllocator, gst_phy_mem_allocator, GST_TYPE_ALLOCATOR_PHYMEM);
351 +alloc_phymem (GstAllocatorPhyMem *allocator, PhyMemBlock *memblk)
353 + struct g2d_buf *pbuf = NULL;
355 + memblk->size = PAGE_ALIGN(memblk->size);
357 + pbuf = g2d_alloc (memblk->size, 0);
359 + GST_ERROR("G2D allocate %u bytes memory failed: %s",
360 + memblk->size, strerror(errno));
364 + memblk->vaddr = (guchar*) pbuf->buf_vaddr;
365 + memblk->paddr = (guchar*) pbuf->buf_paddr;
366 + memblk->user_data = (gpointer) pbuf;
367 + GST_DEBUG("G2D allocated memory (%p)", memblk->paddr);
373 +free_phymem (GstAllocatorPhyMem *allocator, PhyMemBlock *memblk)
375 + GST_DEBUG("G2D free memory (%p)", memblk->paddr);
376 + gint ret = g2d_free ((struct g2d_buf*)(memblk->user_data));
377 + memblk->user_data = NULL;
378 + memblk->vaddr = NULL;
379 + memblk->paddr = NULL;
386 +gst_phy_mem_allocator_class_init (GstPhyMemAllocatorClass * klass)
388 + GstAllocatorPhyMemClass *phy_allocator_klass = (GstAllocatorPhyMemClass *) klass;
390 + phy_allocator_klass->alloc_phymem = alloc_phymem;
391 + phy_allocator_klass->free_phymem = free_phymem;
395 +gst_phy_mem_allocator_init (GstPhyMemAllocator * allocator)
397 + GstAllocator *alloc = GST_ALLOCATOR_CAST (allocator);
399 + alloc->mem_type = GST_GL_PHY_MEM_ALLOCATOR;
404 +gst_phy_mem_allocator_init_instance (gpointer data)
406 + GstAllocator *allocator =
407 + g_object_new (gst_phy_mem_allocator_get_type (), NULL);
409 + GST_DEBUG_CATEGORY_INIT (GST_CAT_GL_PHY_MEMORY, "glphymemory", 0,
410 + "GLPhysical Memory");
412 + gst_allocator_register (GST_GL_PHY_MEM_ALLOCATOR, gst_object_ref (allocator));
418 +_finish_texture (GstGLContext * ctx, gpointer *data)
420 + GstGLFuncs *gl = ctx->gl_vtable;
426 +gst_gl_phy_mem_destroy (GstMemory *mem)
428 + gst_memory_unref (mem);
433 +gst_phy_mem_allocator_obtain (void)
435 + static GOnce once = G_ONCE_INIT;
437 + g_once (&once, gst_phy_mem_allocator_init_instance, NULL);
439 + g_return_val_if_fail (once.retval != NULL, NULL);
441 + return (GstAllocator *) (g_object_ref (once.retval));
445 +gst_is_gl_physical_memory (GstMemory * mem)
447 + GstGLBaseMemory *glmem;
448 + g_return_val_if_fail (gst_is_gl_memory (mem), FALSE);
450 + glmem = (GstGLBaseMemory*) mem;
452 + if (glmem->user_data
453 + && GST_IS_MINI_OBJECT_TYPE(glmem->user_data, GST_TYPE_MEMORY))
454 + return gst_memory_is_type ((GstMemory*)glmem->user_data, GST_GL_PHY_MEM_ALLOCATOR);
460 +gst_is_gl_physical_memory_supported_fmt (GstVideoInfo * info)
462 + if (GST_VIDEO_INFO_IS_RGB(info)
463 + && gst_gl_is_directviv_supported_format (GST_VIDEO_INFO_FORMAT (info))) {
471 +gst_gl_physical_memory_setup_buffer (GstAllocator * allocator, GstBuffer *buffer,
472 + GstGLVideoAllocationParams * params)
474 + GstGLBaseMemoryAllocator *gl_alloc;
475 + GstMemory *mem = NULL;
476 + PhyMemBlock *memblk = NULL;
477 + GstGLMemory *glmem = NULL;
480 + GstVideoInfo * info = params->v_info;
481 + GstVideoAlignment * valign = params->valign;
483 + GST_DEBUG ("glphymemory setup buffer format %s", gst_video_format_to_string (GST_VIDEO_INFO_FORMAT (info)));
485 + if (!gst_is_gl_physical_memory_supported_fmt (info)) {
486 + GST_DEBUG ("Not support format.");
490 + //allocator = (GstAllocator*) gst_phy_mem_allocator_obtain ();
491 + size = gst_gl_get_plane_data_size (info, valign, 0);
492 + mem = gst_allocator_alloc (allocator, size, params->parent.alloc_params);
494 + GST_DEBUG ("Can't allocate physical memory size %d", size);
498 + memblk = gst_memory_query_phymem_block (mem);
500 + GST_ERROR("Can't find physic memory block.");
505 + GST_GL_BASE_MEMORY_ALLOCATOR (gst_gl_memory_allocator_get_default
506 + (params->parent.context));
509 + params->parent.user_data = mem;
510 + params->parent.notify = gst_gl_phy_mem_destroy;
512 + glmem = (GstGLMemory *)gst_gl_base_memory_alloc (gl_alloc, (GstGLAllocationParams *) params);
514 + GST_ERROR("Can't get gl memory.");
518 + gst_buffer_append_memory (buffer, (GstMemory *) glmem);
520 + gst_buffer_add_video_meta_full (buffer, 0,
521 + GST_VIDEO_INFO_FORMAT (info), GST_VIDEO_INFO_WIDTH (info),
522 + GST_VIDEO_INFO_HEIGHT (info), 1, info->offset, info->stride);
524 + gst_gl_viv_direct_bind_data(params->parent.context, glmem->tex_id,
525 + GST_VIDEO_INFO_FORMAT (info), GST_VIDEO_INFO_WIDTH (info),
526 + GST_VIDEO_INFO_HEIGHT (info), memblk->vaddr, memblk->paddr);
532 +gst_gl_phymem_buffer_to_gstbuffer (GstGLContext * ctx,
533 + GstVideoInfo * info, GstBuffer *glbuf)
536 + GstGLBaseMemory *glmem;
538 + gst_gl_context_thread_add (ctx, (GstGLContextThreadFunc) _finish_texture, NULL);
540 + glmem = gst_buffer_peek_memory (glbuf, 0);
542 + buf = gst_buffer_new ();
543 + gst_buffer_append_memory (buf, (GstMemory *) glmem->user_data);
544 + gst_memory_ref ((GstMemory *)glmem->user_data);
546 + gst_buffer_add_video_meta_full (buf, 0,
547 + GST_VIDEO_INFO_FORMAT (info), GST_VIDEO_INFO_WIDTH (info),
548 + GST_VIDEO_INFO_HEIGHT (info), 1, info->offset, info->stride);
549 + GST_BUFFER_FLAGS (buf) = GST_BUFFER_FLAGS (glbuf);
550 + GST_BUFFER_PTS (buf) = GST_BUFFER_PTS (glbuf);
551 + GST_BUFFER_DTS (buf) = GST_BUFFER_DTS (glbuf);
552 + GST_BUFFER_DURATION (buf) = GST_BUFFER_DURATION (glbuf);
557 diff --git a/gst-libs/gst/gl/gstglphymemory.h b/gst-libs/gst/gl/gstglphymemory.h
559 index 0000000..ebb9911
561 +++ b/gst-libs/gst/gl/gstglphymemory.h
565 + * Copyright (c) 2015, Freescale Semiconductor, Inc.
567 + * This library is free software; you can redistribute it and/or
568 + * modify it under the terms of the GNU Library General Public
569 + * License as published by the Free Software Foundation; either
570 + * version 2 of the License, or (at your option) any later version.
572 + * This library is distributed in the hope that it will be useful,
573 + * but WITHOUT ANY WARRANTY; without even the implied warranty of
574 + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
575 + * Library General Public License for more details.
577 + * You should have received a copy of the GNU Library General Public
578 + * License along with this library; if not, write to the
579 + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
580 + * Boston, MA 02110-1301, USA.
583 +#ifndef _GST_GL_PHY_MEMORY_H_
584 +#define _GST_GL_PHY_MEMORY_H_
586 +#include <gst/gst.h>
587 +#include <gst/gstmemory.h>
588 +#include <gst/video/video.h>
589 +#include <gst/allocators/gstallocatorphymem.h>
591 +#include <gst/gl/gl.h>
595 +#define GST_GL_PHY_MEM_ALLOCATOR "GLPhyMemory"
597 +GstAllocator *gst_phy_mem_allocator_obtain (void);
598 +gboolean gst_is_gl_physical_memory (GstMemory * mem);
599 +gboolean gst_is_gl_physical_memory_supported_fmt (GstVideoInfo * info);
600 +gboolean gst_gl_physical_memory_setup_buffer (GstAllocator * allocator, GstBuffer *buffer, GstGLVideoAllocationParams * params);
601 +GstBuffer * gst_gl_phymem_buffer_to_gstbuffer (GstGLContext * ctx, GstVideoInfo * info, GstBuffer *glbuf);
605 +#endif /* _GST_GL_PHY_MEMORY_H_ */