code.ossystems Code Review - openembedded-core.git/blob

   1 gst-ffmpeg: h264: set parameters from SPS whenever it changes
   2
   3 Fixes a crash in the fuzzed sample sample_varPAR.avi_s26638 with
   4 alternating bit depths.
   5
   6 Upstream-Status: Backport
   7
   8 Signed-off-by: Yue Tao <yue.tao@windriver.com>
   9
  10 diff --git a/gst-libs/ext/libav/libavcodec/h264.c.old b/gst-libs/ext/libav/libavcodec/h264.c
  11 index 3621f41..718906a 100644
  12 --- a/gst-libs/ext/libav/libavcodec/h264.c.old
  13 +++ b/gst-libs/ext/libav/libavcodec/h264.c
  14 @@ -2491,6 +2491,34 @@ int ff_h264_get_profile(SPS *sps)
  15      return profile;
  16  }
  17
  18 +static int h264_set_parameter_from_sps(H264Context *h)
  19 +{
  20 +    MpegEncContext *s = &h->s;
  21 +    AVCodecContext * avctx= s->avctx;
  22 +
  23 +    if (s->flags& CODEC_FLAG_LOW_DELAY ||
  24 +        (h->sps.bitstream_restriction_flag && !h->sps.num_reorder_frames))
  25 +        s->low_delay=1;
  26 +
  27 +    if(avctx->has_b_frames < 2)
  28 +        avctx->has_b_frames= !s->low_delay;
  29 +
  30 +    if (avctx->bits_per_raw_sample != h->sps.bit_depth_luma) {
  31 +        if (h->sps.bit_depth_luma >= 8 && h->sps.bit_depth_luma <= 10) {
  32 +            avctx->bits_per_raw_sample = h->sps.bit_depth_luma;
  33 +            h->pixel_shift = h->sps.bit_depth_luma > 8;
  34 +
  35 +            ff_h264dsp_init(&h->h264dsp, h->sps.bit_depth_luma);
  36 +            ff_h264_pred_init(&h->hpc, s->codec_id, h->sps.bit_depth_luma);
  37 +            dsputil_init(&s->dsp, s->avctx);
  38 +        } else {
  39 +            av_log(avctx, AV_LOG_DEBUG, "Unsupported bit depth: %d\n", h->sps.bit_depth_luma);
  40 +            return -1;
  41 +        }
  42 +    }
  43 +    return 0;
  44 +}
  45 +
  46  /**
  47   * decodes a slice header.
  48   * This will also call MPV_common_init() and frame_start() as needed.
  49 @@ -2505,7 +2533,7 @@ static int decode_slice_header(H264Context *h, H264Context *h0){
  50      MpegEncContext * const s0 = &h0->s;
  51      unsigned int first_mb_in_slice;
  52      unsigned int pps_id;
  53 -    int num_ref_idx_active_override_flag;
  54 +    int num_ref_idx_active_override_flag, ret;
  55      unsigned int slice_type, tmp, i, j;
  56      int default_ref_list_done = 0;
  57      int last_pic_structure;
  58 @@ -2569,7 +2597,17 @@ static int decode_slice_header(H264Context *h, H264Context *h0){
  59          av_log(h->s.avctx, AV_LOG_ERROR, "non-existing SPS %u referenced\n", h->pps.sps_id);
  60          return -1;
  61      }
  62 -    h->sps = *h0->sps_buffers[h->pps.sps_id];
  63 +
  64 +    if (h->pps.sps_id != h->current_sps_id ||
  65 +        h0->sps_buffers[h->pps.sps_id]->new) {
  66 +        h0->sps_buffers[h->pps.sps_id]->new = 0;
  67 +
  68 +        h->current_sps_id = h->pps.sps_id;
  69 +        h->sps            = *h0->sps_buffers[h->pps.sps_id];
  70 +
  71 +        if ((ret = h264_set_parameter_from_sps(h)) < 0)
  72 +            return ret;
  73 +    }
  74
  75      s->avctx->profile = ff_h264_get_profile(&h->sps);
  76      s->avctx->level   = h->sps.level_idc;
  77 @@ -3811,26 +3811,8 @@ static int decode_nal_units(H264Context *h, const uint8_t *buf, int buf_size){
  78          case NAL_SPS:
  79              init_get_bits(&s->gb, ptr, bit_length);
  80              ff_h264_decode_seq_parameter_set(h);
  81 -
  82 -            if (s->flags& CODEC_FLAG_LOW_DELAY ||
  83 -                (h->sps.bitstream_restriction_flag && !h->sps.num_reorder_frames))
  84 -                s->low_delay=1;
  85 -
  86 -            if(avctx->has_b_frames < 2)
  87 -                avctx->has_b_frames= !s->low_delay;
  88 -
  89 -            if (avctx->bits_per_raw_sample != h->sps.bit_depth_luma) {
  90 -                if (h->sps.bit_depth_luma >= 8 && h->sps.bit_depth_luma <= 10) {
  91 -                    avctx->bits_per_raw_sample = h->sps.bit_depth_luma;
  92 -                    h->pixel_shift = h->sps.bit_depth_luma > 8;
  93 -
  94 -                    ff_h264dsp_init(&h->h264dsp, h->sps.bit_depth_luma);
  95 -                    ff_h264_pred_init(&h->hpc, s->codec_id, h->sps.bit_depth_luma);
  96 -                    dsputil_init(&s->dsp, s->avctx);
  97 -                } else {
  98 -                    av_log(avctx, AV_LOG_DEBUG, "Unsupported bit depth: %d\n", h->sps.bit_depth_luma);
  99 -                    return -1;
 100 -                }
 101 +            if (h264_set_parameter_from_sps(h) < 0) {
 102 +                return -1;
 103              }
 104              break;
 105          case NAL_PPS:
 106 diff --git a/gst-libs/ext/libav/libavcodec/h264.h.old b/gst-libs/ext/libav/libavcodec/h264.h
 107 index e3cc815..b77ad98 100644
 108 --- a/gst-libs/ext/libav/libavcodec/h264.h.old
 109 +++ b/gst-libs/ext/libav/libavcodec/h264.h
 110 @@ -202,6 +202,7 @@ typedef struct SPS{
 111      int bit_depth_chroma;              ///< bit_depth_chroma_minus8 + 8
 112      int residual_color_transform_flag; ///< residual_colour_transform_flag
 113      int constraint_set_flags;          ///< constraint_set[0-3]_flag
 114 +    int new;                              ///< flag to keep track if the decoder context needs re-init due to changed SPS
 115  }SPS;
 116
 117  /**
 118 @@ -333,6 +334,7 @@ typedef struct H264Context{
 119      int emu_edge_width;
 120      int emu_edge_height;
 121
 122 +    unsigned current_sps_id; ///< id of the current SPS
 123      SPS sps; ///< current sps
 124
 125      /**
 126 diff --git a/gst-libs/ext/libav/libavcodec/h264_ps.c.old b/gst-libs/ext/libav/libavcodec/h264_ps.c
 127 index 7491807..0929098 100644
 128 --- a/gst-libs/ext/libav/libavcodec/h264_ps.c.old
 129 +++ b/gst-libs/ext/libav/libavcodec/h264_ps.c
 130 @@ -438,10 +438,13 @@ int ff_h264_decode_seq_parameter_set(H264Context *h){
 131                 sps->timing_info_present_flag ? sps->time_scale : 0
 132                 );
 133      }
 134 +    sps->new = 1;
 135
 136      av_free(h->sps_buffers[sps_id]);
 137 -    h->sps_buffers[sps_id]= sps;
 138 -    h->sps = *sps;
 139 +    h->sps_buffers[sps_id] = sps;
 140 +    h->sps                 = *sps;
 141 +    h->current_sps_id      = sps_id;
 142 +
 143      return 0;
 144  fail:
 145      av_free(sps);