ffplay解复用线程分析

发表于 分类于 本文字数： 1.9k 阅读时长 ≈ 7 分钟

由主线程创建，负责媒体文件的解复用和读取，读取的数据根据流类型放入到对应的编码数据队列中

查询流信息

创建解复用上下文

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

// 创建解复用上下文
ic = avformat_alloc_context();
if (!ic) {
  ...
}
// 设置中断回调
ic->interrupt_callback.callback = decode_interrupt_cb;
// 回调函数的参数
ic->interrupt_callback.opaque = is;
...
// 打开输入
err = avformat_open_input(&ic, is->filename, is->iformat, &format_opts);
...

is->ic = ic;

if (genpts) ic->flags |= AVFMT_FLAG_GENPTS;

av_format_inject_global_side_data(ic);

if (find_stream_info) {
  AVDictionary **opts = setup_find_stream_info_opts(ic, codec_opts);
  int orig_nb_streams = ic->nb_streams;
  // 读取文件头，获取文件的流详细信息
  err = avformat_find_stream_info(ic, opts);

  ...
}

查找流索引

查找音频和视频流的索引

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
for (i = 0; i < ic->nb_streams; i++) {
  AVStream *st = ic->streams[i];
  enum AVMediaType type = st->codecpar->codec_type;
  st->discard = AVDISCARD_ALL;
  if (type >= 0 && wanted_stream_spec[type] && st_index[type] == -1)
    if (avformat_match_stream_specifier(ic, st, wanted_stream_spec[type]) > 0)
      st_index[type] = i;
}
for (i = 0; i < AVMEDIA_TYPE_NB; i++) {
  if (wanted_stream_spec[i] && st_index[i] == -1) {
    av_log(NULL, AV_LOG_ERROR,
           "Stream specifier %s does not match any %s stream\n",
           wanted_stream_spec[i], av_get_media_type_string(i));
    st_index[i] = INT_MAX;
  }
}
// 查找流索引
if (!video_disable)
  st_index[AVMEDIA_TYPE_VIDEO] = av_find_best_stream(
      ic, AVMEDIA_TYPE_VIDEO, st_index[AVMEDIA_TYPE_VIDEO], -1, NULL, 0);
if (!audio_disable)
  st_index[AVMEDIA_TYPE_AUDIO] = av_find_best_stream(
      ic, AVMEDIA_TYPE_AUDIO, st_index[AVMEDIA_TYPE_AUDIO],
      st_index[AVMEDIA_TYPE_VIDEO], NULL, 0);
if (!video_disable && !subtitle_disable)
  st_index[AVMEDIA_TYPE_SUBTITLE] = av_find_best_stream(
      ic, AVMEDIA_TYPE_SUBTITLE, st_index[AVMEDIA_TYPE_SUBTITLE],
      (st_index[AVMEDIA_TYPE_AUDIO] >= 0 ? st_index[AVMEDIA_TYPE_AUDIO]
                                         : st_index[AVMEDIA_TYPE_VIDEO]),
      NULL, 0);

开启输入流

创建解码上下文

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
// 创建解码上下文
avctx = avcodec_alloc_context3(NULL);
if (!avctx) return AVERROR(ENOMEM);
// 设置解码上下文参数
ret = avcodec_parameters_to_context(avctx, ic->streams[stream_index]->codecpar);
if (ret < 0) goto fail;
// 设置解码上下文时间基
avctx->pkt_timebase = ic->streams[stream_index]->time_base;
// 查找解码器
codec = avcodec_find_decoder(avctx->codec_id);

...

avctx->codec_id = codec->id;
...

// 开启编码器上下文
if ((ret = avcodec_open2(avctx, codec, &opts)) < 0) {
  goto fail;
}

开启音频设备

使用SDL_OpenAudioDevice开启音频设备，如果失败，则不断更换音频参数组合来重试

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
static int audio_open(void *opaque, int64_t wanted_channel_layout,
                      int wanted_nb_channels, int wanted_sample_rate,
                      struct AudioParams *audio_hw_params) {
  SDL_AudioSpec wanted_spec, spec;
  const char *env;
  static const int next_nb_channels[] = {0, 0, 1, 6, 2, 6, 4, 6};
  static const int next_sample_rates[] = {0, 44100, 48000, 96000, 192000};
  int next_sample_rate_idx = FF_ARRAY_ELEMS(next_sample_rates) - 1;
  // 设置SDL音频播放参数
  env = SDL_getenv("SDL_AUDIO_CHANNELS");
  if (env) {
    wanted_nb_channels = atoi(env);
    wanted_channel_layout = av_get_default_channel_layout(wanted_nb_channels);
  }
  if (!wanted_channel_layout ||
      wanted_nb_channels !=
          av_get_channel_layout_nb_channels(wanted_channel_layout)) {
    wanted_channel_layout = av_get_default_channel_layout(wanted_nb_channels);
    wanted_channel_layout &= ~AV_CH_LAYOUT_STEREO_DOWNMIX;
  }
  // 设置 采样率、声道数
  wanted_nb_channels = av_get_channel_layout_nb_channels(wanted_channel_layout);
  wanted_spec.channels = wanted_nb_channels;
  wanted_spec.freq = wanted_sample_rate;
  if (wanted_spec.freq <= 0 || wanted_spec.channels <= 0) {
    av_log(NULL, AV_LOG_ERROR, "Invalid sample rate or channel count!\n");
    return -1;
  }
  while (next_sample_rate_idx &&
         next_sample_rates[next_sample_rate_idx] >= wanted_spec.freq)
    next_sample_rate_idx--;
  // 采样位数
  wanted_spec.format = AUDIO_S16SYS;
  // 静音
  wanted_spec.silence = 0;
  // 样本数量
  wanted_spec.samples =
      FFMAX(SDL_AUDIO_MIN_BUFFER_SIZE,
            2 << av_log2(wanted_spec.freq / SDL_AUDIO_MAX_CALLBACKS_PER_SEC));
  // 回调函数
  wanted_spec.callback = sdl_audio_callback;
  // 回调函数参数
  wanted_spec.userdata = opaque;
  // 开启音频播放设备，如果开启失败，则更换参数不断重试
  while (
      !(audio_dev = SDL_OpenAudioDevice(NULL, 0, &wanted_spec, &spec,
                                        SDL_AUDIO_ALLOW_FREQUENCY_CHANGE |
                                            SDL_AUDIO_ALLOW_CHANNELS_CHANGE))) {
    av_log(NULL, AV_LOG_WARNING, "SDL_OpenAudio (%d channels, %d Hz): %s\n",
           wanted_spec.channels, wanted_spec.freq, SDL_GetError());
    wanted_spec.channels = next_nb_channels[FFMIN(7, wanted_spec.channels)];
    if (!wanted_spec.channels) {
      wanted_spec.freq = next_sample_rates[next_sample_rate_idx--];
      wanted_spec.channels = wanted_nb_channels;
      if (!wanted_spec.freq) {
        // 如果所有组合都重试完了还是打不开则直接退出
        av_log(NULL, AV_LOG_ERROR,
               "No more combinations to try, audio open failed\n");
        return -1;
      }
    }
    wanted_channel_layout = av_get_default_channel_layout(wanted_spec.channels);
  }
  // 如果开启音频设备的参数中采样位数不是16位，则退出，只支持16位
  if (spec.format != AUDIO_S16SYS) {
    av_log(NULL, AV_LOG_ERROR,
           "SDL advised audio format %d is not supported!\n", spec.format);
    return -1;
  }
  if (spec.channels != wanted_spec.channels) {
    wanted_channel_layout = av_get_default_channel_layout(spec.channels);
    if (!wanted_channel_layout) {
      av_log(NULL, AV_LOG_ERROR,
             "SDL advised channel count %d is not supported!\n", spec.channels);
      return -1;
    }
  }
  // 将真正开启音频设备的参数保存到 audio_hw_params 中
  // 只支持16位
  audio_hw_params->fmt = AV_SAMPLE_FMT_S16;
  // 采样率
  audio_hw_params->freq = spec.freq;
  // 声道布局
  audio_hw_params->channel_layout = wanted_channel_layout;
  // 声道数
  audio_hw_params->channels = spec.channels;
  //
  audio_hw_params->frame_size = av_samples_get_buffer_size(
      NULL, audio_hw_params->channels, 1, audio_hw_params->fmt, 1);
  // 1s数据的字节大小
  audio_hw_params->bytes_per_sec = av_samples_get_buffer_size(
      NULL, audio_hw_params->channels, audio_hw_params->freq,
      audio_hw_params->fmt, 1);
  if (audio_hw_params->bytes_per_sec <= 0 || audio_hw_params->frame_size <= 0) {
    av_log(NULL, AV_LOG_ERROR, "av_samples_get_buffer_size failed\n");
    return -1;
  }
  // 返回分配的缓冲区字节大小
  return spec.size;
}

启动解码线程

初始化Decoder结构体

1
2
3
4
5
6
7
8
9
static void decoder_init(Decoder *d, AVCodecContext *avctx, PacketQueue *queue,
                         SDL_cond *empty_queue_cond) {
  memset(d, 0, sizeof(Decoder));
  d->avctx = avctx;
  d->queue = queue;
  d->empty_queue_cond = empty_queue_cond;
  d->start_pts = AV_NOPTS_VALUE;
  d->pkt_serial = -1;
}

启动解码线程

1
2
3
4
5
6
7
8
9
10
11
12
static int decoder_start(Decoder *d, int (*fn)(void *), const char *thread_name,
                         void *arg) {
  // 启动编码数据队列
  packet_queue_start(d->queue);
  // 创建编码线程
  d->decoder_tid = SDL_CreateThread(fn, thread_name, arg);
  if (!d->decoder_tid) {
    av_log(NULL, AV_LOG_ERROR, "SDL_CreateThread(): %s\n", SDL_GetError());
    return AVERROR(ENOMEM);
  }
  return 0;
}

轮询读取流数据

流开启完毕后，就开始轮询从媒体文件中读取数据放入到Packet队列中，当Packet队列中数据足够的时候，会wait等待，

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
  // 轮询从文件中读取流数据
  for (;;) {
    // 是否退出
    if (is->abort_request) break;
    // 如果本次暂停事件没有处理，则处理暂停事件
    if (is->paused != is->last_paused) {
      // 更新最新处理的暂停事件
      is->last_paused = is->paused;
      // 如果是暂停，就停止读取，如果是播放，则开始读取
      if (is->paused)
        is->read_pause_return = av_read_pause(ic);
      else
        av_read_play(ic);
    }
#if CONFIG_RTSP_DEMUXER || CONFIG_MMSH_PROTOCOL
    // 如果是RTSP流，暂停就是等待10ms后再重试
    if (is->paused && (!strcmp(ic->iformat->name, "rtsp") ||
                       (ic->pb && !strncmp(input_filename, "mmsh:", 5)))) {
      /* wait 10 ms to avoid trying to get another packet */
      /* XXX: horrible */
      SDL_Delay(10);
      continue;
    }
#endif
    // seek操作
    if (is->seek_req) {
      // seek的位置
      int64_t seek_target = is->seek_pos;
      // seek的区间,上下行
      int64_t seek_min =
          is->seek_rel > 0 ? seek_target - is->seek_rel + 2 : INT64_MIN;
      int64_t seek_max =
          is->seek_rel < 0 ? seek_target - is->seek_rel - 2 : INT64_MAX;
      // FIXME the +-2 is due to rounding being not done in the correct
      // direction in generation
      //      of the seek_pos/seek_rel variables
      // seek
      ret = avformat_seek_file(is->ic, -1, seek_min, seek_target, seek_max,
                               is->seek_flags);
      if (ret < 0) {
        av_log(NULL, AV_LOG_ERROR, "%s: error while seeking\n", is->ic->url);
      } else {
        if (is->audio_stream >= 0) {
          // 重置音频编码数据队列
          packet_queue_flush(&is->audioq);
          // 队列中加入flush_pkt分割标记
          packet_queue_put(&is->audioq, &flush_pkt);
        }
        if (is->subtitle_stream >= 0) {
          // 重置字幕编码数据队列
          packet_queue_flush(&is->subtitleq);
          // 队列中加入flush_pkt分割标记
          packet_queue_put(&is->subtitleq, &flush_pkt);
        }
        if (is->video_stream >= 0) {
          // 重置视频编码数据队列
          packet_queue_flush(&is->videoq);
          // 队列中加入flush_pkt分割标记，让队列的序号+1
          packet_queue_put(&is->videoq, &flush_pkt);
        }
        // 更新外部时钟的PTS
        if (is->seek_flags & AVSEEK_FLAG_BYTE) {
          set_clock(&is->extclk, NAN, 0);
        } else {
          set_clock(&is->extclk, seek_target / (double)AV_TIME_BASE, 0);
        }
      }
      is->seek_req = 0;
      is->queue_attachments_req = 1;
      is->eof = 0;
      if (is->paused) step_to_next_frame(is);
    }
    if (is->queue_attachments_req) {
      if (is->video_st &&
          is->video_st->disposition & AV_DISPOSITION_ATTACHED_PIC) {
        AVPacket copy = {0};
        if ((ret = av_packet_ref(&copy, &is->video_st->attached_pic)) < 0)
          goto fail;
        packet_queue_put(&is->videoq, &copy);
        packet_queue_put_nullpacket(&is->videoq, is->video_stream);
      }
      is->queue_attachments_req = 0;
    }

    /* if the queue are full, no need to read more */
    // 如果音频编码数据队列+视频编码数据队列+字幕编码数据队列中字节数量超过了15
    // * 1024 * 1024（15M） 或者说
    // 音频流、视频流、字幕流中都已经读取了充足的Packet在队列中缓存着，则wait
    // 10ms等待
    // 也就是说三个队列中缓冲数据超过15M或者三个队列中都已经有了一定的数据(1s的数据)，那么就不能继续读下去了，等待10ms后再来重试
    if (infinite_buffer < 1 &&
        (is->audioq.size + is->videoq.size + is->subtitleq.size >
             MAX_QUEUE_SIZE ||
         (stream_has_enough_packets(is->audio_st, is->audio_stream,
                                    &is->audioq) &&
          stream_has_enough_packets(is->video_st, is->video_stream,
                                    &is->videoq) &&
          stream_has_enough_packets(is->subtitle_st, is->subtitle_stream,
                                    &is->subtitleq)))) {
      /* wait 10 ms */
      SDL_LockMutex(wait_mutex);
      // 等待10ms后，重新循环
      SDL_CondWaitTimeout(is->continue_read_thread, wait_mutex, 10);
      SDL_UnlockMutex(wait_mutex);
      continue;
    }
    // 处理播放结束
    if (!is->paused &&
        (!is->audio_st || (is->auddec.finished == is->audioq.serial &&
                           frame_queue_nb_remaining(&is->sampq) == 0)) &&
        (!is->video_st || (is->viddec.finished == is->videoq.serial &&
                           frame_queue_nb_remaining(&is->pictq) == 0))) {
      // loop = 0 表示无限循环， 否则表示循环次数，每次结束-1，直到等于1停止循环
      if (loop != 1 && (!loop || --loop)) {
        stream_seek(is, start_time != AV_NOPTS_VALUE ? start_time : 0, 0, 0);
      } else if (autoexit) {
        ret = AVERROR_EOF;
        goto fail;
      }
    }
    // 读取一个Packet
    ret = av_read_frame(ic, pkt);
    if (ret < 0) {
      // 读取失败
      if ((ret == AVERROR_EOF || avio_feof(ic->pb)) && !is->eof) {
        // 如果文件流已经结束，放入一个空的Packet到队列中
        if (is->video_stream >= 0)
          packet_queue_put_nullpacket(&is->videoq, is->video_stream);
        if (is->audio_stream >= 0)
          packet_queue_put_nullpacket(&is->audioq, is->audio_stream);
        if (is->subtitle_stream >= 0)
          packet_queue_put_nullpacket(&is->subtitleq, is->subtitle_stream);
        // EOF 流结束
        is->eof = 1;
      }
      if (ic->pb && ic->pb->error) break;
      SDL_LockMutex(wait_mutex);
      // 等待10ms
      SDL_CondWaitTimeout(is->continue_read_thread, wait_mutex, 10);
      SDL_UnlockMutex(wait_mutex);
      continue;
    } else {
      is->eof = 0;
    }
    /* check if packet is in play range specified by user, then queue, otherwise
     * discard */
    // 流的开始时间
    stream_start_time = ic->streams[pkt->stream_index]->start_time;
    // 当前Packet的PTS
    pkt_ts = pkt->pts == AV_NOPTS_VALUE ? pkt->dts : pkt->pts;
    // 当前Packet的PTS - 流开始时间 <= 播放时长duration
    // 则表明当前Packet是在播放范围内的
    pkt_in_play_range =
        duration == AV_NOPTS_VALUE ||
        (pkt_ts -
         (stream_start_time != AV_NOPTS_VALUE ? stream_start_time : 0)) *
                    av_q2d(ic->streams[pkt->stream_index]->time_base) -
                (double)(start_time != AV_NOPTS_VALUE ? start_time : 0) /
                    1000000 <=
            ((double)duration / 1000000);
    if (pkt->stream_index == is->audio_stream && pkt_in_play_range) {
      // 音频Packet，放入到音频编码数据队列中
      packet_queue_put(&is->audioq, pkt);
    } else if (pkt->stream_index == is->video_stream && pkt_in_play_range &&
               !(is->video_st->disposition & AV_DISPOSITION_ATTACHED_PIC)) {
      // 音频Packet，放入到音频编码数据队列中
      packet_queue_put(&is->videoq, pkt);
    } else if (pkt->stream_index == is->subtitle_stream && pkt_in_play_range) {
      // 音频Packet，放入到音频编码数据队列中
      packet_queue_put(&is->subtitleq, pkt);
    } else {
      // 类型都不符合，则释放不处理
      av_packet_unref(pkt);
    }
  }