code.ossystems Code Review - meta-freescale.git/blob

   1 From 6136bc631e5e875ed6b75d994564dd5fd09c7bf3 Mon Sep 17 00:00:00 2001
   2 From: Carlos Rafael Giani <dv@pseudoterminal.org>
   3 Date: Wed, 7 Jun 2017 18:23:52 +0200
   4 Subject: [PATCH] gstgl: Add direct DMABUF uploader [WIP]
   5
   6 This is the first, work-in-progress version of a new "direct" DMABUF
   7 uploader. The idea is that some GPUs (like the Vivante series) can actually
   8 perform the YUV->RGB conversion internally, so no custom conversion shaders
   9 are needed. To make use of this feature, we need an additional uploader
  10 that can import DMABUF FDs and also directly pass the pixel format,
  11 relying on the GPU to do the conversion.
  12
  13 This patch is incomplete for these reasons:
  14 - It is currently unknown how to determine if the GPU can handle these
  15   conversions
  16 - Currently it is not possible to probe for what EGLImage DMABUF input
  17   formats the GPU can handle
  18 - The EGLImage cache is there, but cannot be used, because some drivers
  19   like etnaviv keep a cached copy of the texture inside, and it is
  20   currently not clear how to let etnaviv know that the cache has to be
  21   flushed once the associated DMABUF gets a new video frame
  22 ---
  23  gst-libs/gst/gl/gstglupload.c | 290 +++++++++++++++++++++++++++++++++++++-----
  24  1 file changed, 256 insertions(+), 34 deletions(-)
  25
  26 diff --git a/gst-libs/gst/gl/gstglupload.c b/gst-libs/gst/gl/gstglupload.c
  27 index feaf2f68e..28abc898d 100644
  28 --- a/gst-libs/gst/gl/gstglupload.c
  29 +++ b/gst-libs/gst/gl/gstglupload.c
  30 @@ -36,6 +36,7 @@
  31
  32  #if GST_GL_HAVE_DMABUF
  33  #include <gst/allocators/gstdmabuf.h>
  34 +#include <libdrm/drm_fourcc.h>
  35  #endif
  36
  37  #if GST_GL_HAVE_VIV_DIRECTVIV
  38 @@ -480,7 +481,256 @@ static const UploadMethod _gl_memory_upload = {
  39    &_gl_memory_upload_free
  40  };
  41
  42 +
  43  #if GST_GL_HAVE_DMABUF
  44 +
  45 +
  46 +static GQuark
  47 +_eglimage_quark (gint plane)
  48 +{
  49 +  static GQuark quark[4] = { 0 };
  50 +  static const gchar *quark_str[] = {
  51 +    "GstGLDMABufEGLImage0",
  52 +    "GstGLDMABufEGLImage1",
  53 +    "GstGLDMABufEGLImage2",
  54 +    "GstGLDMABufEGLImage3",
  55 +  };
  56 +
  57 +  if (!quark[plane])
  58 +    quark[plane] = g_quark_from_static_string (quark_str[plane]);
  59 +
  60 +  return quark[plane];
  61 +}
  62 +
  63 +static GstEGLImage *
  64 +_get_cached_eglimage (GstMiniObject * mobj, gint plane)
  65 +{
  66 +  return gst_mini_object_get_qdata (mobj, _eglimage_quark (plane));
  67 +}
  68 +
  69 +static void
  70 +_set_cached_eglimage (GstMiniObject * mobj, GstEGLImage * eglimage, gint plane)
  71 +{
  72 +  return gst_mini_object_set_qdata (mobj,
  73 +      _eglimage_quark (plane), eglimage, (GDestroyNotify) gst_egl_image_unref);
  74 +}
  75 +
  76 +/* TODO: the normal and direct dmabuf uploader share a lot of code.
  77 + * Perhaps they can be merged. */
  78 +struct DirectDmabufUpload
  79 +{
  80 +  GstGLUpload *upload;
  81 +
  82 +  GstEGLImage *eglimage;
  83 +  GstBuffer *outbuf;
  84 +  GstGLVideoAllocationParams *params;
  85 +};
  86 +
  87 +/* TODO add format probing (requires EGL_EXT_image_dma_buf_import_modifiers)
  88 + * Currently this list is just hardcoded */
  89 +#define GST_GL_DIRECT_DMABUF_FORMAT "{YUY2, NV12, I420}"
  90 +
  91 +static GstStaticCaps _direct_dma_buf_upload_caps =
  92 +GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE (GST_GL_DIRECT_DMABUF_FORMAT));
  93 +
  94 +static gpointer
  95 +_direct_dma_buf_upload_new (GstGLUpload * upload)
  96 +{
  97 +  struct DirectDmabufUpload *dmabuf = g_new0 (struct DirectDmabufUpload, 1);
  98 +  dmabuf->upload = upload;
  99 +  return dmabuf;
 100 +}
 101 +
 102 +static GstCaps *
 103 +_direct_dma_buf_upload_transform_caps (gpointer impl, GstGLContext * context,
 104 +    GstPadDirection direction, GstCaps * caps)
 105 +{
 106 +  GstCapsFeatures *passthrough =
 107 +      gst_caps_features_from_string
 108 +      (GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION);
 109 +  GstCaps *ret;
 110 +
 111 +  if (direction == GST_PAD_SINK) {
 112 +    GstCaps *tmp;
 113 +
 114 +    ret =
 115 +        _set_caps_features_with_passthrough (caps,
 116 +        GST_CAPS_FEATURE_MEMORY_GL_MEMORY, passthrough);
 117 +
 118 +    gst_caps_set_simple (ret, "format", G_TYPE_STRING, "RGBA", NULL);
 119 +    tmp = _caps_intersect_texture_target (ret, 1 << GST_GL_TEXTURE_TARGET_2D);
 120 +    gst_caps_unref (ret);
 121 +    ret = tmp;
 122 +  } else {
 123 +    GstCaps *tmp;
 124 +    tmp = gst_caps_from_string (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
 125 +        (GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY, GST_GL_DIRECT_DMABUF_FORMAT));
 126 +    ret =
 127 +        _set_caps_features_with_passthrough (tmp,
 128 +        GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY, passthrough);
 129 +    gst_caps_unref (tmp);
 130 +  }
 131 +
 132 +  gst_caps_features_free (passthrough);
 133 +  return ret;
 134 +}
 135 +
 136 +static gboolean
 137 +_direct_dma_buf_upload_accept (gpointer impl, GstBuffer * buffer,
 138 +    GstCaps * in_caps, GstCaps * out_caps)
 139 +{
 140 +  struct DirectDmabufUpload *dmabuf = impl;
 141 +  GstVideoInfo *in_info = &dmabuf->upload->priv->in_info;
 142 +  guint n_planes = GST_VIDEO_INFO_N_PLANES (in_info);
 143 +  GstVideoMeta *meta;
 144 +  guint n_mem;
 145 +  guint i;
 146 +
 147 +  n_mem = gst_buffer_n_memory (buffer);
 148 +  meta = gst_buffer_get_video_meta (buffer);
 149 +
 150 +  /* dmabuf upload is only supported with EGL contexts. */
 151 +  if (gst_gl_context_get_gl_platform (dmabuf->upload->context) !=
 152 +      GST_GL_PLATFORM_EGL)
 153 +    return FALSE;
 154 +
 155 +  if (!gst_gl_context_check_feature (dmabuf->upload->context,
 156 +          "EGL_KHR_image_base"))
 157 +    return FALSE;
 158 +
 159 +  /* This will eliminate most non-dmabuf out there */
 160 +  if (!gst_is_dmabuf_memory (gst_buffer_peek_memory (buffer, 0)))
 161 +    return FALSE;
 162 +
 163 +  /* We cannot have multiple dmabuf per plane */
 164 +  if (n_mem > n_planes)
 165 +    return FALSE;
 166 +
 167 +  /* Update video info based on video meta */
 168 +  if (meta) {
 169 +    in_info->width = meta->width;
 170 +    in_info->height = meta->height;
 171 +
 172 +    for (i = 0; i < meta->n_planes; i++) {
 173 +      in_info->offset[i] = meta->offset[i];
 174 +      in_info->stride[i] = meta->stride[i];
 175 +    }
 176 +  }
 177 +
 178 +  if (dmabuf->params)
 179 +    gst_gl_allocation_params_free ((GstGLAllocationParams *) dmabuf->params);
 180 +  if (!(dmabuf->params =
 181 +          gst_gl_video_allocation_params_new_wrapped_gl_handle (dmabuf->
 182 +              upload->context, NULL, &dmabuf->upload->priv->out_info, -1, NULL,
 183 +              GST_GL_TEXTURE_TARGET_2D, GST_VIDEO_GL_TEXTURE_TYPE_RGBA, NULL,
 184 +              NULL, NULL)))
 185 +    return FALSE;
 186 +
 187 +  /* Now create an EGLImage for each dmabufs */
 188 +  {
 189 +    /* check if one is cached */
 190 +    dmabuf->eglimage = _get_cached_eglimage (GST_MINI_OBJECT (buffer), 0);
 191 +    /* TODO: This is a hack that effectively disables caching.
 192 +     * Currently it is not possible to inform some GPU drivers (like etnaviv)
 193 +     * that they should invalidate whatever texture-related cache they have,
 194 +     * so even if the associated dmabuf changes, the texture doesn't.. So, as
 195 +     * a workaround, were disable caching here and force the reinitialization
 196 +     * of EGLImages. Fix the texture cache updates and then remove the #if
 197 +     * block below. */
 198 +#if 1
 199 +    dmabuf->eglimage = NULL;
 200 +#endif
 201 +
 202 +    if (!dmabuf->eglimage) {
 203 +      /* otherwise create one and cache it */
 204 +
 205 +      dmabuf->eglimage =
 206 +          gst_egl_image_from_dmabuf_direct (dmabuf->upload->context,
 207 +          buffer, in_info);
 208 +
 209 +      if (!dmabuf->eglimage)
 210 +        return FALSE;
 211 +
 212 +      _set_cached_eglimage (GST_MINI_OBJECT (buffer), dmabuf->eglimage, 0);
 213 +    }
 214 +
 215 +    GST_TRACE ("Got GstEGLImage %p", (gpointer) (dmabuf->eglimage));
 216 +  }
 217 +
 218 +  return TRUE;
 219 +}
 220 +
 221 +static void
 222 +_direct_dma_buf_upload_propose_allocation (gpointer impl,
 223 +    GstQuery * decide_query, GstQuery * query)
 224 +{
 225 +  /* nothing to do for now. */
 226 +}
 227 +
 228 +static void
 229 +_direct_dma_buf_upload_perform_gl_thread (GstGLContext * context,
 230 +    struct DirectDmabufUpload *dmabuf)
 231 +{
 232 +  GstGLMemoryAllocator *allocator;
 233 +  gpointer ptr;
 234 +
 235 +  allocator =
 236 +      GST_GL_MEMORY_ALLOCATOR (gst_allocator_find
 237 +      (GST_GL_MEMORY_EGL_ALLOCATOR_NAME));
 238 +
 239 +  /* FIXME: buffer pool */
 240 +  dmabuf->outbuf = gst_buffer_new ();
 241 +  ptr = (gpointer *) dmabuf->eglimage;
 242 +  gst_gl_memory_setup_buffer (allocator, dmabuf->outbuf, dmabuf->params, NULL,
 243 +      &ptr, 1);
 244 +  gst_object_unref (allocator);
 245 +}
 246 +
 247 +static GstGLUploadReturn
 248 +_direct_dma_buf_upload_perform (gpointer impl, GstBuffer * buffer,
 249 +    GstBuffer ** outbuf)
 250 +{
 251 +  struct DirectDmabufUpload *dmabuf = impl;
 252 +
 253 +  gst_gl_context_thread_add (dmabuf->upload->context,
 254 +      (GstGLContextThreadFunc) _direct_dma_buf_upload_perform_gl_thread,
 255 +      dmabuf);
 256 +
 257 +  if (!dmabuf->outbuf)
 258 +    return GST_GL_UPLOAD_ERROR;
 259 +
 260 +  gst_buffer_add_parent_buffer_meta (dmabuf->outbuf, buffer);
 261 +
 262 +  *outbuf = dmabuf->outbuf;
 263 +  dmabuf->outbuf = NULL;
 264 +
 265 +  return GST_GL_UPLOAD_DONE;
 266 +}
 267 +
 268 +static void
 269 +_direct_dma_buf_upload_free (gpointer impl)
 270 +{
 271 +  struct DirectDmabufUpload *dmabuf = impl;
 272 +
 273 +  if (dmabuf->params)
 274 +    gst_gl_allocation_params_free ((GstGLAllocationParams *) dmabuf->params);
 275 +
 276 +  g_free (impl);
 277 +}
 278 +
 279 +static const UploadMethod _direct_dma_buf_upload = {
 280 +  "DirectDmabuf",
 281 +  0,
 282 +  &_direct_dma_buf_upload_caps,
 283 +  &_direct_dma_buf_upload_new,
 284 +  &_direct_dma_buf_upload_transform_caps,
 285 +  &_direct_dma_buf_upload_accept,
 286 +  &_direct_dma_buf_upload_propose_allocation,
 287 +  &_direct_dma_buf_upload_perform,
 288 +  &_direct_dma_buf_upload_free
 289 +};
 290 +
 291 +
 292  struct DmabufUpload
 293  {
 294    GstGLUpload *upload;
 295 @@ -543,37 +793,6 @@ _dma_buf_upload_transform_caps (gpointer impl, GstGLContext * context,
 296    return ret;
 297  }
 298
 299 -static GQuark
 300 -_eglimage_quark (gint plane)
 301 -{
 302 -  static GQuark quark[4] = { 0 };
 303 -  static const gchar *quark_str[] = {
 304 -    "GstGLDMABufEGLImage0",
 305 -    "GstGLDMABufEGLImage1",
 306 -    "GstGLDMABufEGLImage2",
 307 -    "GstGLDMABufEGLImage3",
 308 -  };
 309 -
 310 -  if (!quark[plane])
 311 -    quark[plane] = g_quark_from_static_string (quark_str[plane]);
 312 -
 313 -  return quark[plane];
 314 -}
 315 -
 316 -static GstEGLImage *
 317 -_get_cached_eglimage (GstMemory * mem, gint plane)
 318 -{
 319 -  return gst_mini_object_get_qdata (GST_MINI_OBJECT (mem),
 320 -      _eglimage_quark (plane));
 321 -}
 322 -
 323 -static void
 324 -_set_cached_eglimage (GstMemory * mem, GstEGLImage * eglimage, gint plane)
 325 -{
 326 -  return gst_mini_object_set_qdata (GST_MINI_OBJECT (mem),
 327 -      _eglimage_quark (plane), eglimage, (GDestroyNotify) gst_egl_image_unref);
 328 -}
 329 -
 330  static gboolean
 331  _dma_buf_upload_accept (gpointer impl, GstBuffer * buffer, GstCaps * in_caps,
 332      GstCaps * out_caps)
 333 @@ -652,20 +871,20 @@ _dma_buf_upload_accept (gpointer impl, GstBuffer * buffer, GstCaps * in_caps,
 334    /* Now create an EGLImage for each dmabufs */
 335    for (i = 0; i < n_planes; i++) {
 336      /* check if one is cached */
 337 -    dmabuf->eglimage[i] = _get_cached_eglimage (mems[i], i);
 338 +    dmabuf->eglimage[i] = _get_cached_eglimage (GST_MINI_OBJECT (mems[i]), i);
 339      if (dmabuf->eglimage[i])
 340        continue;
 341
 342      /* otherwise create one and cache it */
 343      dmabuf->eglimage[i] =
 344 -        gst_egl_image_from_dmabuf (dmabuf->upload->context,
 345 +        gst_egl_image_from_dmabuf_as_rgba_plane (dmabuf->upload->context,
 346          gst_dmabuf_memory_get_fd (mems[i]), in_info, i,
 347          mems[i]->offset + mems_skip[i]);
 348
 349      if (!dmabuf->eglimage[i])
 350        return FALSE;
 351
 352 -    _set_cached_eglimage (mems[i], dmabuf->eglimage[i], i);
 353 +    _set_cached_eglimage (GST_MINI_OBJECT (mems[i]), dmabuf->eglimage[i], i);
 354    }
 355
 356    return TRUE;
 357 @@ -737,8 +956,10 @@ static const UploadMethod _dma_buf_upload = {
 358    &_dma_buf_upload_free
 359  };
 360
 361 +
 362  #endif /* GST_GL_HAVE_DMABUF */
 363
 364 +
 365  struct GLUploadMeta
 366  {
 367    GstGLUpload *upload;
 368 @@ -1523,6 +1744,7 @@ static const UploadMethod _directviv_upload = {
 369
 370  static const UploadMethod *upload_methods[] = { &_gl_memory_upload,
 371  #if GST_GL_HAVE_DMABUF
 372 +  &_direct_dma_buf_upload,
 373    &_dma_buf_upload,
 374  #endif
 375  #if GST_GL_HAVE_VIV_DIRECTVIV
 376 --
 377 2.14.1
 378