FFMPEG vaapi_encoder 源码阅读
VAAPI是intel设计的一个视频硬件加速器的软件接口。FFMPEG也将其集成进来。这里通过对源码的分析来了解它的编码流程,尤其是参考帧是如何管理的。
一般情况,编码器的工作周期是一个GOP。GOP通常是封闭的,即下一个GOP不依赖于上一个GOP。这意味着各GOP之间是独立的。在每个GOP内部,每一帧的编码类型(I/P/B)常按照一定的模式来进行。比如,GOP的第一帧一般是I帧,(按编码顺序)第二帧一般是P帧,接着编码B帧。FFMPEG用两个参数来表示一个GOP的长度和模式。第一个参数是GOP size,即一个封闭GOP是由多少帧构成的。另一个参数是B帧的数量,即在两个P帧之间,有多少个B帧。这两个参数,在解码器的上下文中对应的分别是avctx->gope_size和avctx->b_per_p。
编码入口是ff_vaapi_encode2()。输入的原始图像是按顺序进入的,即每次调用编码的avctx->input_order都是递增的。
int ff_vaapi_encode2(AVCodecContext *avctx, AVPacket *pkt, /* 输出码流 */ const AVFrame *input_image, /*输入原始图像, null at endOfSeq */ int *got_packet) /* 如当前产生码流写1,否则写0 */ { if (input_image) { if (input_image->pict_type == AV_PICTURE_TYPE_I) { err = vaapi_encode_truncate_gop(avctx); if (err < 0) goto fail; ctx->force_idr = 1; } /* 获得当前要编码的输入图像。会设置相关的参考帧关系。 */ err = vaapi_encode_get_next(avctx, &pic); if (err) { av_log(avctx, AV_LOG_ERROR, "Input setup failed: %d.\n", err); return err; } pic->input_image = av_frame_alloc(); if (!pic->input_image) { err = AVERROR(ENOMEM); goto fail; } err = av_frame_ref(pic->input_image, input_image); if (err < 0) goto fail; pic->input_surface = (VASurfaceID)(uintptr_t)input_image->data[3]; pic->pts = input_image->pts; if (ctx->input_order == 0) ctx->first_pts = pic->pts; if (ctx->input_order == ctx->decode_delay) ctx->dts_pts_diff = pic->pts - ctx->first_pts; if (ctx->output_delay > 0) ctx->ts_ring[ctx->input_order % (3 * ctx->output_delay)] = pic->pts; pic->input_available = 1; } else { if (!ctx->end_of_stream) { err = vaapi_encode_truncate_gop(avctx); if (err < 0) goto fail; ctx->end_of_stream = 1; } } ++ctx->input_order; ++ctx->output_order; av_assert0(ctx->output_order + ctx->output_delay + 1 == ctx->input_order); /* 找出下一下编码帧。此帧会在step中完成编码,同时也会用递归的方式先完成其参考帧的编码. */ for (pic = ctx->pic_start; pic; pic = pic->next) if (pic->encode_order == ctx->output_order) break; // pic can be null here if we don't have a specific target in this // iteration. We might still issue encodes if things can be overlapped, // even though we don't intend to output anything. /* 编码当前帧及其参考帧 */ err = vaapi_encode_step(avctx, pic); if (!pic) { /* 当前没有有效的输出码流 */ *got_packet = 0; } else { /* 输出码流 */ err = vaapi_encode_output(avctx, pic, pkt); ... *got_packet = 1; } /* 清理上下文 */ err = vaapi_encode_clear_old(avctx); if (err < 0) { av_log(avctx, AV_LOG_ERROR, "List clearing failed: %d.\n", err); goto fail; } fail: /* 错误出口,现在没做什么 */ // Unclear what to clean up on failure. There are probably some things we // could do usefully clean up here, but for now just leave them for uninit() // to do instead. return err; }
当一个GOP结束时,其固定的编码类型模式会受到影响,这时需要一些特殊的处理。因此 代码中处理下一个I帧时要对上一个GOP作截断。
为了管理参考帧,由vaapi_encode_get_next()建立各帧的依赖关系。除了GOP中第一帧编码为I帧,接着会跳过ctx->b_per_p输入帧编一个P帧。然后,将中间跳过的帧全部编码为B帧。这些中间的B帧都以其前面的I帧或P帧为前向参考帧,以其后的P帧为后向参考帧。
static int vaapi_encode_get_next(AVCodecContext *avctx, VAAPIEncodePicture **pic_out) { VAAPIEncodeContext *ctx = avctx->priv_data; VAAPIEncodePicture *start, *end, *pic; int i; for (pic = ctx->pic_start; pic; pic = pic->next) { if (pic->next) av_assert0(pic->display_order + 1 == pic->next->display_order); if (pic->display_order == ctx->input_order) { *pic_out = pic; return 0; } } pic = vaapi_encode_alloc(); if (!pic) return AVERROR(ENOMEM); if (ctx->input_order == 0 || ctx->force_idr || ctx->gop_counter >= avctx->gop_size) { pic->type = PICTURE_TYPE_IDR; ctx->force_idr = 0; ctx->gop_counter = 1; ctx->p_counter = 0; } else if (ctx->p_counter >= ctx->p_per_i) { pic->type = PICTURE_TYPE_I; ++ctx->gop_counter; ctx->p_counter = 0; } else { /* P帧用前一组的最后一帧ctx->pic_end作为参考帧 */ pic->type = PICTURE_TYPE_P; pic->refs[0] = ctx->pic_end; pic->nb_refs = 1; ++ctx->gop_counter; ++ctx->p_counter; } start = end = pic; if (pic->type != PICTURE_TYPE_IDR) { // If that was not an IDR frame, add B-frames display-before and // encode-after it, but not exceeding the GOP size. for (i = 0; i < ctx->b_per_p && ctx->gop_counter < avctx->gop_size; i++) { pic = vaapi_encode_alloc(); if (!pic) goto fail; /* B帧用当前帧(end)和前一组的最后一帧ctx->pic_end作为参考帧 */ pic->type = PICTURE_TYPE_B; pic->refs[0] = ctx->pic_end; pic->refs[1] = end; pic->nb_refs = 2; pic->next = start; pic->display_order = ctx->input_order + ctx->b_per_p - i - 1; pic->encode_order = pic->display_order + 1; start = pic; ++ctx->gop_counter; } } if (ctx->input_order == 0) { pic->display_order = 0; pic->encode_order = 0; ctx->pic_start = ctx->pic_end = pic; } else { for (i = 0, pic = start; pic; i++, pic = pic->next) { pic->display_order = ctx->input_order + i; if (end->type == PICTURE_TYPE_IDR) pic->encode_order = ctx->input_order + i; else if (pic == end) pic->encode_order = ctx->input_order; else pic->encode_order = ctx->input_order + i + 1; } av_assert0(ctx->pic_end); ctx->pic_end->next = start; ctx->pic_end = end; } *pic_out = start; av_log(avctx, AV_LOG_DEBUG, "Pictures:"); for (pic = ctx->pic_start; pic; pic = pic->next) { av_log(avctx, AV_LOG_DEBUG, " %s (%"PRId64"/%"PRId64")", picture_type_name[pic->type], pic->display_order, pic->encode_order); } av_log(avctx, AV_LOG_DEBUG, "\n"); return 0; fail: while (start) { pic = start->next; vaapi_encode_free(avctx, start); start = pic; } return AVERROR(ENOMEM); }
根据以上分析,当前实现的参考帧管理是比较简单的一种,只是实现了封闭GOP中简单的IPBBB..PBBB..模式。要想实现其它更复杂的模式,只能自己定制相关的实现了。