1 From 989d63bdf9397ba3630a3d6b9fc934b78ef62823 Mon Sep 17 00:00:00 2001
2 From: Haihua Hu <b55597@freescale.com>
3 Date: Tue, 13 Oct 2015 09:33:54 +0800
4 Subject: [PATCH 12/26] Add directviv to glimagesink to improve playback
7 Content-Type: text/plain; charset=UTF-8
8 Content-Transfer-Encoding: 8bit
10 1.Add a physical buffer uploader in glupload plugin and using viv direct
11 texture to bind physical continious buffer with texture to avoid memory
12 copy from videobuffer to texture to gain good performance.
13 2.Reduce glimagesink load latency by override glimagesink ALLOCATION query to
15 3.Add configure check for directviv feature
17 Upstream-Status: Inappropriate [i.MX specific]
19 Signed-off-by: Haihua Hu <b55597@freescale.com>
22 ext/gl/gstglimagesink.c | 5 ++
23 gst-libs/gst/gl/Makefile.am | 8 ++
24 gst-libs/gst/gl/gstglapi.h | 1 +
25 gst-libs/gst/gl/gstglupload.c | 151 +++++++++++++++++++++++++++++++-
26 gst-libs/gst/gl/gstglvivdirecttexture.c | 149 +++++++++++++++++++++++++++++++
27 gst-libs/gst/gl/gstglvivdirecttexture.h | 38 ++++++++
28 7 files changed, 361 insertions(+), 3 deletions(-)
29 create mode 100644 gst-libs/gst/gl/gstglvivdirecttexture.c
30 create mode 100644 gst-libs/gst/gl/gstglvivdirecttexture.h
32 diff --git a/configure.ac b/configure.ac
33 index 991e7d0..3dc2b75 100644
36 @@ -701,6 +701,7 @@ HAVE_GLES2=no
44 @@ -740,6 +741,7 @@ case $host in
46 dnl check fb backend for imx soc
47 AC_CHECK_LIB(EGL, fbGetDisplay, HAVE_FB_EGL=yes, HAVE_FB_EGL=no)
48 + AC_CHECK_LIB(GLESv2, glTexDirectVIV, HAVE_DIRECTVIV=yes, HAVE_DIRECTVIV=no)
50 dnl FIXME: Mali EGL depends on GLESv1 or GLESv2
51 AC_CHECK_HEADER([EGL/fbdev_window.h],
52 @@ -850,6 +852,12 @@ if test "x$HAVE_GLES2" = "xno"; then
56 +dnl specific for imx soc
57 +GST_GL_HAVE_DIRECTVIV=0
58 +if test "x$HAVE_DIRECTVIV" = "xyes"; then
59 + GST_GL_HAVE_DIRECTVIV=1
63 if test "x$HAVE_X" = "xno"; then
64 if test "x$NEED_GLX" = "xyes"; then
65 @@ -1308,6 +1316,7 @@ GL_CONFIG_DEFINES="$GL_CONFIG_DEFINES
67 GL_CONFIG_DEFINES="$GL_CONFIG_DEFINES
68 #define GST_GL_HAVE_DMABUF $GST_GL_HAVE_DMABUF
69 +#define GST_GL_HAVE_DIRECTVIV $GST_GL_HAVE_DIRECTVIV
72 dnl Check for no platforms/window systems
73 @@ -1343,6 +1352,8 @@ if test "x$GL_APIS" = "x" -o "x$GL_PLATFORMS" = "x" -o "x$GL_WINDOWS" = "x"; the
82 @@ -1359,6 +1370,7 @@ AM_CONDITIONAL(HAVE_WINDOW_WAYLAND, test "x$HAVE_WINDOW_WAYLAND" = "xyes")
83 AM_CONDITIONAL(HAVE_WINDOW_ANDROID, test "x$HAVE_WINDOW_ANDROID" = "xyes")
84 AM_CONDITIONAL(HAVE_WINDOW_EAGL, test "x$HAVE_WINDOW_EAGL" = "xyes")
85 AM_CONDITIONAL(HAVE_WINDOW_FB, test "x$HAVE_WINDOW_FB" = "xyes")
86 +AM_CONDITIONAL(HAVE_DIRECTVIV, test "x$HAVE_DIRECTVIV" = "xyes")
88 AM_CONDITIONAL(USE_OPENGL, test "x$USE_OPENGL" = "xyes")
89 AM_CONDITIONAL(USE_GLES2, test "x$USE_GLES2" = "xyes")
90 diff --git a/ext/gl/gstglimagesink.c b/ext/gl/gstglimagesink.c
91 index 385e3e4..5da1012 100644
92 --- a/ext/gl/gstglimagesink.c
93 +++ b/ext/gl/gstglimagesink.c
94 @@ -1139,6 +1139,11 @@ gst_glimage_sink_query (GstBaseSink * bsink, GstQuery * query)
95 res = GST_BASE_SINK_CLASS (parent_class)->query (bsink, query);
98 + case GST_QUERY_ALLOCATION:
100 + gst_glimage_sink_propose_allocation(bsink, query);
104 res = GST_BASE_SINK_CLASS (parent_class)->query (bsink, query);
106 diff --git a/gst-libs/gst/gl/Makefile.am b/gst-libs/gst/gl/Makefile.am
107 index 01498e0..a6b9755 100644
108 --- a/gst-libs/gst/gl/Makefile.am
109 +++ b/gst-libs/gst/gl/Makefile.am
110 @@ -36,6 +36,10 @@ libgstgl_@GST_API_VERSION@_la_SOURCES = \
112 gstglcontrolbindingproxy.c
115 +libgstgl_@GST_API_VERSION@_la_SOURCES += gstglvivdirecttexture.c
118 libgstgl_@GST_API_VERSION@includedir = $(includedir)/gstreamer-@GST_API_VERSION@/gst/gl
119 libgstgl_@GST_API_VERSION@include_HEADERS = \
121 @@ -70,6 +74,10 @@ libgstgl_@GST_API_VERSION@include_HEADERS = \
126 +libgstgl_@GST_API_VERSION@include_HEADERS += gstglvivdirecttexture.h
131 utils/opengl_versions.h \
132 diff --git a/gst-libs/gst/gl/gstglapi.h b/gst-libs/gst/gl/gstglapi.h
133 index 3088920..2e9c561 100644
134 --- a/gst-libs/gst/gl/gstglapi.h
135 +++ b/gst-libs/gst/gl/gstglapi.h
137 # if GST_GL_HAVE_GLES3
138 # include <GLES3/gl3.h>
139 # include <GLES3/gl3ext.h>
140 +# include <GLES2/gl2ext.h>
142 # include <GLES2/gl2.h>
143 # include <GLES2/gl2ext.h>
144 diff --git a/gst-libs/gst/gl/gstglupload.c b/gst-libs/gst/gl/gstglupload.c
145 index bfa5f22..2e52c3e 100644
146 --- a/gst-libs/gst/gl/gstglupload.c
147 +++ b/gst-libs/gst/gl/gstglupload.c
150 #include "gstglupload.h"
152 +#if GST_GL_HAVE_DIRECTVIV
153 +#include <gst/gl/gstglvivdirecttexture.h>
155 #if GST_GL_HAVE_PLATFORM_EGL
156 #include "egl/gstglmemoryegl.h"
159 #define USING_GLES2(context) (gst_gl_context_check_gl_version (context, GST_GL_API_GLES2, 2, 0))
160 #define USING_GLES3(context) (gst_gl_context_check_gl_version (context, GST_GL_API_GLES2, 3, 0))
162 -GST_DEBUG_CATEGORY_STATIC (gst_gl_upload_debug);
163 +GST_DEBUG_CATEGORY (gst_gl_upload_debug);
164 #define GST_CAT_DEFAULT gst_gl_upload_debug
167 @@ -969,6 +972,144 @@ static const UploadMethod _upload_meta_upload = {
168 &_upload_meta_upload_free
171 +#if GST_GL_HAVE_DIRECTVIV
172 +struct PhyBufferUpload
174 + GstGLUpload *upload;
175 + GstGLVideoAllocationParams *params;
179 +_physical_buffer_upload_new(GstGLUpload *upload)
181 + struct PhyBufferUpload *phybuffer = g_new0 (struct PhyBufferUpload, 1);
183 + phybuffer->upload = upload;
189 +_physical_buffer_upload_transform_caps(gpointer impl, GstGLContext *context,
190 + GstPadDirection direction, GstCaps *caps)
192 + GstCapsFeatures *passthrough =
193 + gst_caps_features_from_string
194 + (GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION);
197 + if (direction == GST_PAD_SINK) {
201 + _set_caps_features_with_passthrough (caps,
202 + GST_CAPS_FEATURE_MEMORY_GL_MEMORY, passthrough);
204 + gst_caps_set_simple (ret, "format", G_TYPE_STRING, "RGBA", NULL);
205 + tmp = _caps_intersect_texture_target (ret, 1 << GST_GL_TEXTURE_TARGET_2D);
206 + gst_caps_unref (ret);
209 + ret = gst_caps_from_string (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
210 + (GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY,GST_GL_DIRECTVIV_FORMAT));
213 + gst_caps_features_free (passthrough);
218 +_physical_buffer_upload_accept(gpointer impl, GstBuffer *buffer,
219 + GstCaps *in_caps, GstCaps *out_caps)
221 + struct PhyBufferUpload *upload = impl;
222 + GstCapsFeatures *features;
224 + features = gst_caps_get_features (out_caps, 0);
225 + if (!gst_caps_features_contains (features, GST_CAPS_FEATURE_MEMORY_GL_MEMORY))
228 + if (upload->params)
229 + gst_gl_allocation_params_free ((GstGLAllocationParams *) upload->params);
230 + if (!(upload->params =
231 + gst_gl_video_allocation_params_new (upload->upload->context, NULL,
232 + &upload->upload->priv->out_info, -1, NULL,
233 + GST_GL_TEXTURE_TARGET_2D, GST_VIDEO_GL_TEXTURE_TYPE_RGBA)))
236 + return gst_is_physical_buffer(buffer);
240 +_physical_buffer_upload_propose_allocation(gpointer impl, GstQuery *decide_query,
243 + gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, 0);
246 +static GstGLUploadReturn
247 +_physical_buffer_upload_perform(gpointer impl, GstBuffer *buffer, GstBuffer **outbuf)
249 + struct PhyBufferUpload *phyBuffer = impl;
250 + GstGLMemoryAllocator *allocator;
251 + GstVideoInfo *info;
254 + info = &phyBuffer->upload->priv->out_info;
255 + n_mem = GST_VIDEO_INFO_N_PLANES (info);
256 + GST_LOG_OBJECT (phyBuffer->upload, "Attempting viv direct upload, out format %s",
257 + gst_video_format_to_string (GST_VIDEO_INFO_FORMAT (info)));
260 + GST_GL_MEMORY_ALLOCATOR (gst_allocator_find
261 + (GST_GL_MEMORY_PBO_ALLOCATOR_NAME));
263 + /* FIXME: buffer pool */
264 + *outbuf = gst_buffer_new ();
265 + gst_gl_memory_setup_buffer (allocator, *outbuf, phyBuffer->params, NULL,
267 + gst_object_unref (allocator);
269 + GstGLMemory *out_gl_mem =
270 + (GstGLMemory *) gst_buffer_peek_memory (*outbuf, 0);
272 + gst_gl_viv_direct_bind_gstbuffer(phyBuffer->upload->context, out_gl_mem->tex_id,
273 + &phyBuffer->upload->priv->in_info, buffer);
275 + gst_buffer_add_video_meta_full (*outbuf, 0,
276 + GST_VIDEO_INFO_FORMAT (info), GST_VIDEO_INFO_WIDTH (info),
277 + GST_VIDEO_INFO_HEIGHT (info), n_mem, info->offset, info->stride);
279 + return GST_GL_UPLOAD_DONE;
283 +_physical_buffer_upload_free(gpointer impl)
285 + struct PhyBufferUpload *phyBuffer = impl;
287 + if (phyBuffer->params)
288 + gst_gl_allocation_params_free ((GstGLAllocationParams *) phyBuffer->params);
293 +static GstStaticCaps _physical_buffer_upload_caps =
294 +GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE (GST_GL_DIRECTVIV_FORMAT));
296 +static const UploadMethod _physical_buffer_upload = {
299 + &_physical_buffer_upload_caps,
300 + &_physical_buffer_upload_new,
301 + &_physical_buffer_upload_transform_caps,
302 + &_physical_buffer_upload_accept,
303 + &_physical_buffer_upload_propose_allocation,
304 + &_physical_buffer_upload_perform,
305 + &_physical_buffer_upload_free
307 +#endif /* GST_GL_HAVE_DIRECTVIV */
309 struct RawUploadFrame
312 @@ -1191,7 +1332,11 @@ static const UploadMethod *upload_methods[] = { &_gl_memory_upload,
313 #if GST_GL_HAVE_DMABUF
316 - &_upload_meta_upload, &_raw_data_upload
317 + &_upload_meta_upload,
318 +#if GST_GL_HAVE_DIRECTVIV
319 + &_physical_buffer_upload,
324 static GMutex upload_global_lock;
325 @@ -1325,7 +1470,7 @@ gst_gl_upload_transform_caps (GstGLUpload * upload, GstGLContext * context,
329 - result = gst_caps_intersect_full (filter, tmp, GST_CAPS_INTERSECT_FIRST);
330 + result = gst_caps_intersect_full (tmp, filter, GST_CAPS_INTERSECT_FIRST);
331 gst_caps_unref (tmp);
334 diff --git a/gst-libs/gst/gl/gstglvivdirecttexture.c b/gst-libs/gst/gl/gstglvivdirecttexture.c
336 index 0000000..4c9dc69
338 +++ b/gst-libs/gst/gl/gstglvivdirecttexture.c
342 + * Copyright (c) 2015, Freescale Semiconductor, Inc.
344 + * This library is free software; you can redistribute it and/or
345 + * modify it under the terms of the GNU Library General Public
346 + * License as published by the Free Software Foundation; either
347 + * version 2 of the License, or (at your option) any later version.
349 + * This library is distributed in the hope that it will be useful,
350 + * but WITHOUT ANY WARRANTY; without even the implied warranty of
351 + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
352 + * Library General Public License for more details.
354 + * You should have received a copy of the GNU Library General Public
355 + * License along with this library; if not, write to the
356 + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
357 + * Boston, MA 02110-1301, USA.
360 +#ifdef HAVE_CONFIG_H
365 +#include "gstglvivdirecttexture.h"
366 +#include <gst/allocators/gstallocatorphymem.h>
368 +GST_DEBUG_CATEGORY_EXTERN (gst_gl_upload_debug);
369 +#define GST_CAT_DEFAULT gst_gl_upload_debug
372 + GstVideoFormat gst_fmt;
376 +static VIV_FMT_MAP viv_fmt_map_table[] = {
377 + {GST_VIDEO_FORMAT_I420, GL_VIV_I420},
378 + {GST_VIDEO_FORMAT_YV12, GL_VIV_YV12},
379 + {GST_VIDEO_FORMAT_NV12, GL_VIV_NV12},
380 + {GST_VIDEO_FORMAT_NV21, GL_VIV_NV21},
381 + {GST_VIDEO_FORMAT_YUY2, GL_VIV_YUY2},
382 + {GST_VIDEO_FORMAT_UYVY, GL_VIV_UYVY},
383 + {GST_VIDEO_FORMAT_RGBA, GL_RGBA},
384 + {GST_VIDEO_FORMAT_RGBx, GL_RGBA},
385 + {GST_VIDEO_FORMAT_BGRA, GL_BGRA_EXT},
386 + {GST_VIDEO_FORMAT_RGB16, GL_RGB565_OES}
397 +} GstVivDirectTexture;
400 +gst_is_physical_buffer (GstBuffer *buffer)
402 + return gst_buffer_is_phymem (buffer);
406 +_do_viv_direct_tex_bind_mem (GstGLContext * context, GstVivDirectTexture * viv_tex)
408 + GST_DEBUG ("viv direct upload, tex_id %d, fmt: %d, res: (%dx%d)", viv_tex->tex_id, viv_tex->fmt, viv_tex->w, viv_tex->h);
409 + GST_DEBUG ("Physical memory buffer, vaddr: %p, paddr: %p", viv_tex->vaddr, viv_tex->paddr);
411 + glBindTexture (GL_TEXTURE_2D, viv_tex->tex_id);
412 + glTexDirectVIVMap (GL_TEXTURE_2D, viv_tex->w, viv_tex->h, viv_tex->fmt, &viv_tex->vaddr, &viv_tex->paddr);
413 + glTexDirectInvalidateVIV (GL_TEXTURE_2D);
414 + viv_tex->ret = TRUE;
419 +gst_gl_is_directviv_supported_format (GstVideoFormat fmt)
422 + gboolean ret = FALSE;
424 + for (i=0; i<sizeof(viv_fmt_map_table)/sizeof(VIV_FMT_MAP); i++) {
425 + if (fmt == viv_fmt_map_table[i].gst_fmt) {
435 +gst_gl_viv_direct_bind_data (GstGLContext * context,
436 + guint tex_id, GstVideoFormat fmt, gint width, gint height,
437 + gpointer * vaddr, gpointer *paddr)
439 + guint viv_fmt = GL_NONE;
442 + for (i=0; i<sizeof(viv_fmt_map_table)/sizeof(VIV_FMT_MAP); i++) {
443 + if (fmt == viv_fmt_map_table[i].gst_fmt) {
444 + viv_fmt = viv_fmt_map_table[i].viv_fmt;
449 + if (viv_fmt == GL_NONE) {
450 + GST_ERROR ("Not supported format %d for viv direct texture upload.", fmt);
454 + GstVivDirectTexture viv_tex = {tex_id, width, height, viv_fmt, vaddr, paddr, FALSE};
455 + gst_gl_context_thread_add (context, (GstGLContextThreadFunc) _do_viv_direct_tex_bind_mem, &viv_tex);
457 + return viv_tex.ret;
461 +gst_gl_viv_direct_bind_gstbuffer (GstGLContext * context, guint tex_id, GstVideoInfo * info, GstBuffer * buffer)
463 + PhyMemBlock *memblk;
464 + GstVideoMeta *vmeta;
465 + GstVideoFormat fmt;
466 + gint width, height;
468 + memblk = gst_buffer_query_phymem_block (buffer);
472 + width = GST_VIDEO_INFO_WIDTH (info);
473 + height = GST_VIDEO_INFO_HEIGHT (info);
475 + vmeta = gst_buffer_get_video_meta (buffer);
476 + fmt = GST_VIDEO_INFO_FORMAT (info);
477 + if (vmeta && (fmt == GST_VIDEO_FORMAT_I420 || fmt == GST_VIDEO_FORMAT_NV12)) {
478 + width = vmeta->stride[0];
479 + height = vmeta->offset[1] / width;
482 + width = GST_VIDEO_INFO_WIDTH (info);
483 + height = GST_VIDEO_INFO_HEIGHT (info);
486 + return gst_gl_viv_direct_bind_data (context, tex_id, fmt, width, height, memblk->vaddr, memblk->paddr);
489 diff --git a/gst-libs/gst/gl/gstglvivdirecttexture.h b/gst-libs/gst/gl/gstglvivdirecttexture.h
491 index 0000000..9a2d123
493 +++ b/gst-libs/gst/gl/gstglvivdirecttexture.h
497 + * Copyright (c) 2015, Freescale Semiconductor, Inc.
499 + * This library is free software; you can redistribute it and/or
500 + * modify it under the terms of the GNU Library General Public
501 + * License as published by the Free Software Foundation; either
502 + * version 2 of the License, or (at your option) any later version.
504 + * This library is distributed in the hope that it will be useful,
505 + * but WITHOUT ANY WARRANTY; without even the implied warranty of
506 + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
507 + * Library General Public License for more details.
509 + * You should have received a copy of the GNU Library General Public
510 + * License along with this library; if not, write to the
511 + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
512 + * Boston, MA 02110-1301, USA.
515 +#ifndef __GST_GL_VIVDIRECT_H__
516 +#define __GST_GL_VIVDIRECT_H__
518 +#include <gst/video/video.h>
519 +#include <gst/gl/gstgl_fwd.h>
521 +#define GST_GL_DIRECTVIV_FORMAT "{RGBA, I420, YV12, NV12, NV21, YUY2, UYVY, BGRA, RGB16}"
524 +gboolean gst_is_physical_buffer (GstBuffer *buffer);
525 +gboolean gst_gl_is_directviv_supported_format (GstVideoFormat fmt);
526 +gboolean gst_gl_viv_direct_bind_data (GstGLContext * context, guint tex_id, GstVideoFormat fmt, gint width, gint height,
527 + gpointer * vaddr, gpointer *paddr);
528 +gboolean gst_gl_viv_direct_bind_gstbuffer (GstGLContext * context, guint tex_id, GstVideoInfo * info, GstBuffer * buffer);
532 +#endif /* __GST_GL_VIVDIRECT_H__ */