VLC-Android视频显示流程

//vout中其实有两个线程，分别名为Thread和ThreadStep，视频正常播放走的是Thread，步进播放走的是ThreadStep（类似ffplay的s键），这里主要还是介绍Thread

/*****************************************************************************

 * Thread: video output thread

 *****************************************************************************

 * Video output thread. This function does only returns when the thread is

 * terminated. It handles the pictures arriving in the video heap and the

 * display device events.

 *****************************************************************************/

static void *Thread(void *object)

{

    vout_thread_t *vout = object;

    vout_thread_sys_t *sys = vout->p;

    mtime_t deadline = VLC_TS_INVALID;

    bool wait = false;

    //这里是一个大的死循环

    for (;;) {

        vout_control_cmd_t cmd;

        //对deadline进行修正

        if (wait)

        {

            const mtime_t max_deadline = mdate() + 100000;

            deadline = deadline <= VLC_TS_INVALID ? max_deadline : __MIN(deadline, max_deadline);

        } else {

            deadline = VLC_TS_INVALID;

        }

        //处理control

        while (!vout_control_Pop(&sys->control, &cmd, deadline))

            if (ThreadControl(vout, cmd))

                return NULL;

        //显示Picture

        deadline = VLC_TS_INVALID;

        wait = ThreadDisplayPicture(vout, &deadline) != VLC_SUCCESS;

        const bool picture_interlaced = sys->displayed.is_interlaced;

        vout_SetInterlacingState(vout, picture_interlaced);

        vout_ManageWrapper(vout);

    }

}

static int ThreadDisplayPicture(vout_thread_t *vout, mtime_t *deadline)

{

    bool frame_by_frame = !deadline;

    bool paused = vout->p->pause.is_on;

    //vout->p->displayed.current代表当前正在播放的帧（类比ffplay video_refresh函数中的lastvp）

    bool first = !vout->p->displayed.current;

    //调用ThreadDisplayPreparePicture准备一帧

    if (first)

        if (ThreadDisplayPreparePicture(vout, true, frame_by_frame)) /* FIXME not sure it is ok */

            return VLC_EGENERIC;

    //vout->p->displayed.next代表要播放的下一帧（类比ffplay video_refresh函数中的vp）

    if (!paused || frame_by_frame)

        while (!vout->p->displayed.next && !ThreadDisplayPreparePicture(vout, false, frame_by_frame))

            ;

    const mtime_t date = mdate();

    //这里计算render_delay来决定要提前多久显示

    //这里补充说明一点，Google在MediaCodec.java中建议提前两个VSYNC时间显示，原因可以自行百度Android的VSYNC显示机制

    //所以这里最好给render_delay再加33.3ms

    const mtime_t render_delay = vout_chrono_GetHigh(&vout->p->render) + VOUT_MWAIT_TOLERANCE/* + INT64_C(33333)*/;

    //这个drop_next_frame的名字起的很不恰当，我被这个变量名误导了很久，这里drop_next_frame代表的是丢弃当前播放的帧，显示下一帧

    //这个逻辑也就保证了正常情况下，视频会按照帧率稳定地刷新显示

    //如果是frame_by_frame步进模式，drop_next_frame恒为true

    bool drop_next_frame = frame_by_frame;

    mtime_t date_next = VLC_TS_INVALID;

    if (!paused && vout->p->displayed.next) {

        //计算下一帧需要显示的系统时间，如果当前时间已经到达，就将drop_next_frame置为true，丢弃当前帧

        date_next = vout->p->displayed.next->date - render_delay;

        if (date_next /* + 0 FIXME */ <= date)

            drop_next_frame = true;

    }

    /* FIXME/XXX we must redisplay the last decoded picture (because

     * of potential vout updated, or filters update or SPU update)

     * For now a high update period is needed but it could be removed

     * if and only if:

     * - vout module emits events from theselves.

     * - *and* SPU is modified to emit an event or a deadline when needed.

     *

     * So it will be done later.

     */

    bool refresh = false;

    //这里是强制刷新的逻辑，避免某一帧显示时间过长，如果某一帧显示时间超过(VOUT_REDISPLAY_DELAY-render_delay)，就强制刷新

    mtime_t date_refresh = VLC_TS_INVALID;

    if (vout->p->displayed.date > VLC_TS_INVALID) {

        date_refresh = vout->p->displayed.date + VOUT_REDISPLAY_DELAY - render_delay;

        refresh = date_refresh <= date;

    }

    bool force_refresh = !drop_next_frame && refresh;

    if (!frame_by_frame) {

        if (date_refresh != VLC_TS_INVALID)

            *deadline = date_refresh;

        if (date_next != VLC_TS_INVALID && date_next < *deadline)

            *deadline = date_next;

    }

    if (!first && !refresh && !drop_next_frame) {

        return VLC_EGENERIC;

    }

    //结合这里就能看懂drop_next_frame的含义了，其实是将当前帧释放掉了。。。

    if (drop_next_frame) {

        picture_Release(vout->p->displayed.current);

        vout->p->displayed.current = vout->p->displayed.next;

        vout->p->displayed.next    = NULL;

    }

    if (!vout->p->displayed.current)

        return VLC_EGENERIC;

    //调用ThreadDisplayRenderPicture函数显示Picture

    /* display the picture immediately */

    bool is_forced = frame_by_frame || force_refresh || vout->p->displayed.current->b_force;

    int ret = ThreadDisplayRenderPicture(vout, is_forced);

    return force_refresh ? VLC_EGENERIC : ret;

}

static int ThreadDisplayPreparePicture(vout_thread_t *vout, bool reuse, bool frame_by_frame)

{

    bool is_late_dropped = vout->p->is_late_dropped && !vout->p->pause.is_on && !frame_by_frame;

    vlc_mutex_lock(&vout->p->filter.lock);

    picture_t *picture = filter_chain_VideoFilter(vout->p->filter.chain_static, NULL);

    assert(!reuse || !picture);

    while (!picture) {

        picture_t *decoded;

        //vout->p->displayed.decoded代表解码后但没有经过VideoFilter的帧

        //重用上一帧

        if (reuse && vout->p->displayed.decoded) {

            decoded = picture_Hold(vout->p->displayed.decoded);

        } else {

            //从decoder fifo中取解码后的帧

            decoded = picture_fifo_Pop(vout->p->decoder_fifo);

            if (decoded) {

                //丢帧逻辑，如果当前帧晚了超过半帧的时长，就会被丢掉

                if (is_late_dropped && !decoded->b_force) {

                    mtime_t late_threshold;

                    if (decoded->format.i_frame_rate && decoded->format.i_frame_rate_base)

                        late_threshold = ((CLOCK_FREQ/2) * decoded->format.i_frame_rate_base) / decoded->format.i_frame_rate;

                    else

                        late_threshold = VOUT_DISPLAY_LATE_THRESHOLD;

                    const mtime_t predicted = mdate() + 0; /* TODO improve */

                    const mtime_t late = predicted - decoded->date;

                    if (late > late_threshold) {

                        msg_Warn(vout, "picture is too late to be displayed (missing %"PRId64" ms)", late/1000);

                        picture_Release(decoded);

                        vout_statistic_AddLost(&vout->p->statistic, 1);

                        continue;

                    //如果晚了不到半帧时长不会被丢掉

                    } else if (late > 0) {

                        msg_Dbg(vout, "picture might be displayed late (missing %"PRId64" ms)", late/1000);

                    }

                }

                if (!VideoFormatIsCropArEqual(&decoded->format, &vout->p->filter.format))

                    ThreadChangeFilters(vout, &decoded->format, vout->p->filter.configuration, -1, true);

            }

        }

        if (!decoded)

            break;

        reuse = false;

        //release上一个解码后的帧

        if (vout->p->displayed.decoded)

            picture_Release(vout->p->displayed.decoded);

        //更新vout->p->displayed.decoded

        vout->p->displayed.decoded       = picture_Hold(decoded);

        vout->p->displayed.timestamp     = decoded->date;

        vout->p->displayed.is_interlaced = !decoded->b_progressive;

        //VideoFilter处理

        picture = filter_chain_VideoFilter(vout->p->filter.chain_static, decoded);

    }

    vlc_mutex_unlock(&vout->p->filter.lock);

    if (!picture)

        return VLC_EGENERIC;

    //更新要显示的帧

    assert(!vout->p->displayed.next);

    if (!vout->p->displayed.current)

        vout->p->displayed.current = picture;

    else

        vout->p->displayed.next    = picture;

    return VLC_SUCCESS;

}

static int ThreadDisplayRenderPicture(vout_thread_t *vout, bool is_forced)

{

    vout_thread_sys_t *sys = vout->p;

    vout_display_t *vd = vout->p->display.vd;

    picture_t *torender = picture_Hold(vout->p->displayed.current);

    //字幕显示和filter相关的逻辑全部去掉了

    //这里显示调用vout_display_Prepare去做准备工作

    if (sys->display.use_dr) {

        vout_display_Prepare(vd, todisplay, subpic);

    } else {

        if (!do_dr_spu && !do_early_spu && vout->p->spu_blend && subpic)

            picture_BlendSubpicture(todisplay, vout->p->spu_blend, subpic);

        vout_display_Prepare(vd, todisplay, do_dr_spu ? subpic : NULL);

        if (!do_dr_spu && subpic)

        {

            subpicture_Delete(subpic);

            subpic = NULL;

        }

    }

    vout_chrono_Stop(&vout->p->render);

    //如果不是强制显示，需要等到当前帧需要显示的时间再显示

    if (!is_forced)

        mwait(todisplay->date);

    //显示当前picture

    /* Display the direct buffer returned by vout_RenderPicture */

    vout->p->displayed.date = mdate();

    vout_display_Display(vd, todisplay, subpic);

    vout_statistic_AddDisplayed(&vout->p->statistic, 1);

    return VLC_SUCCESS;

}

static void Prepare(vout_display_t *vd, picture_t *picture,

                    subpicture_t *subpicture)

{

    vout_display_sys_t *sys = vd->sys;

    VLC_UNUSED(picture);

    if (subpicture && sys->p_sub_window) {

        if (sys->b_sub_invalid) {

            sys->b_sub_invalid = false;

            if (sys->p_sub_pic) {

                picture_Release(sys->p_sub_pic);

                sys->p_sub_pic = NULL;

            }

            if (sys->p_spu_blend) {

                filter_DeleteBlend(sys->p_spu_blend);

                sys->p_spu_blend = NULL;

            }

            free(sys->p_sub_buffer_bounds);

            sys->p_sub_buffer_bounds = NULL;

        }

        if (!sys->p_sub_pic

         && AndroidWindow_Setup(sys, sys->p_sub_window, 1) == 0)

            sys->p_sub_pic = PictureAlloc(sys, &sys->p_sub_window->fmt, false);

        if (!sys->p_spu_blend && sys->p_sub_pic)

            sys->p_spu_blend = filter_NewBlend(VLC_OBJECT(vd),

                                               &sys->p_sub_pic->format);

        if (sys->p_sub_pic && sys->p_spu_blend)

            sys->b_has_subpictures = true;

    }

    /* As long as no subpicture was received, do not call

       SubpictureDisplay since JNI calls and clearing the subtitles

       surface are expensive operations. */

    if (sys->b_has_subpictures)

    {

        SubpicturePrepare(vd, subpicture);

        if (!subpicture)

        {

            /* The surface has been cleared and there is no new

               subpicture to upload, do not clear again until a new

               subpicture is received. */

            sys->b_has_subpictures = false;

        }

    }

    if (sys->p_window->b_opaque

     && AndroidOpaquePicture_CanReleaseAtTime(picture->p_sys))

    {

        mtime_t now = mdate();

        if (picture->date > now)

        {

            //实际在prepare时就已经调用NdkMediaCodec的releaseOutputBufferAtTime接口把这帧数据显示了

            //这就是显示流程的第二个坑，甚至根本没有注释说明为什么要这么改。。。

            if (picture->date - now <= INT64_C(1000000))

                AndroidOpaquePicture_ReleaseAtTime(picture->p_sys, picture->date);

            else /* The picture will be displayed from the Display callback */

                msg_Warn(vd, "picture way too early to release at time");

        }

    }

}

//这个函数其实是个摆设，啥也没干就返回了。。。

static void Display(vout_display_t *vd, picture_t *picture,

                    subpicture_t *subpicture)

{

    vout_display_sys_t *sys = vd->sys;

    if (sys->p_window->b_opaque)

        AndroidOpaquePicture_Release(picture->p_sys, true);

    else

        AndroidWindow_UnlockPicture(sys, sys->p_window, picture, true);

    picture_Release(picture);

    if (sys->p_sub_pic)

        AndroidWindow_UnlockPicture(sys, sys->p_sub_window, sys->p_sub_pic,

                                    true);

    if (subpicture)

        subpicture_Delete(subpicture);

    sys->b_displayed = true;

}