前言
通过前面的实战，我们实现音视频解封装提取、音视频解码、音视频编码、音频重采样等的功能，今天我们就结合之前所做的功能，
来做一个短视频APP中常见的功能：
1、提取多个mp3文件中的音频，重新编码为合并为aac
2、提取mp4中的视频，重新编码合并为h264
3、h264与aac合并成新的mp4文件
因为我们的目的是以实战为主，为了囊括之前所学的一些知识点，在这个实战中我们不仅仅需要实现音视频解封装提取、音视频解码、音视频编码、音频重采样这些功能，
我们还需要结合多线程同步等知识点做好生产者消费者队列缓冲控制。还包含例如类成员函数作为线程执行函数的使用等知识点。
大致框架
这里要说明一个常识就是如果音频如果需要合并的话要保证两个音频的采样率、采样格式以及通道数一致，所以需要重采样，为了测试，笔者把音频都重采样为22050hz。
同时视频也一样，如果视频需要合并也需要保证两个视频的分辨率是一样的，这里笔者统一把尺寸转换为720x1280。
笔者文笔不好，经常一句卧槽走天下，直接看图吧。。。
代码实现
本来笔者想追求简单，希望用一个cpp文件实现的，后面写着写着发现代码量有点多，所以就拆分成了三个cpp文件，下面是代码详情：
AudioHandle.cpp
/**
* 音频处理
* 解码音频，并且重采样为22050，然后编码成aac
*/
#ifndef TARGET_AUDIO_SAMPLE_RATE
// 采样率
#define TARGET_AUDIO_SAMPLE_RATE 22050
#endif
#include <iostream>
#include <vector>
extern "C" {
#include <libavcodec/avcodec.h>
#include <libavformat/avformat.h>
#include <libavutil/opt.h>
#include <libavutil/channel_layout.h>
#include <libavutil/audio_fifo.h>
#include <libswresample/swresample.h>
};
class AudioHandle {
public:
void handle_audio(std::vector<char *> mp3_paths,
std::function<void(const AVCodecContext *, AVPacket *, bool)> callback) {
// 音频编码器相关
const AVCodec *avCodec = avcodec_find_encoder(AV_CODEC_ID_AAC);
audio_encoder_context = avcodec_alloc_context3(avCodec);
audio_encoder_context->sample_rate = TARGET_AUDIO_SAMPLE_RATE;
// 默认的aac编码器输入的PCM格式为:AV_SAMPLE_FMT_FLTP
audio_encoder_context->sample_fmt = AV_SAMPLE_FMT_FLTP;
audio_encoder_context->channel_layout = AV_CH_LAYOUT_STEREO;
// audio_encoder_context->bit_rate = 128 * 1024;
audio_encoder_context->codec_type = AVMEDIA_TYPE_AUDIO;
audio_encoder_context->channels = av_get_channel_layout_nb_channels(audio_encoder_context->channel_layout);
audio_encoder_context->profile = FF_PROFILE_AAC_LOW;
//ffmpeg默认的aac是不带adts，而fdk_aac默认带adts，这里我们强制不带
audio_encoder_context->flags = AV_CODEC_FLAG_GLOBAL_HEADER;
int ret = avcodec_open2(audio_encoder_context, avCodec, nullptr);
if (ret < 0) {
std::cout << "音频编码器打开失败" << std::endl;
return;
}
// 初始化audiofifo
audiofifo = av_audio_fifo_alloc(audio_encoder_context->sample_fmt, audio_encoder_context->channels,
audio_encoder_context->frame_size);
AVFormatContext *avFormatContext = nullptr;
AVCodecContext *decoder_context = nullptr;
AVPacket *avPacket = av_packet_alloc();
AVFrame *avFrame = av_frame_alloc();
std::vector<AVPacket *> pack_vector = std::vector<AVPacket *>();
while (!mp3_paths.empty()) {
// 先释放旧的
avcodec_free_context(&decoder_context);
avformat_free_context(avFormatContext);
const char *mp3 = mp3_paths.at(0);
mp3_paths.erase(mp3_paths.cbegin());
avFormatContext = avformat_alloc_context();
ret = avformat_open_input(&avFormatContext, mp3, nullptr, nullptr);
if (ret < 0) {
std::cout << "音频文件打开失败" << std::endl;
break;
}
int audio_index = av_find_best_stream(avFormatContext, AVMEDIA_TYPE_AUDIO, -1, -1, nullptr, 0);
if (audio_index < 0) {
for (int i = 0; i < avFormatContext->nb_streams; ++i) {
if (AVMEDIA_TYPE_AUDIO == avFormatContext->streams[i]->codecpar->codec_type) {
audio_index = i;
std::cout << "找到音频流，audio_index:" << audio_index << std::endl;
break;
}
}
if (audio_index < 0) {
std::cout << "没有找到音频流" << std::endl;
break;
}
}
const AVCodec *avCodec = avcodec_find_decoder(avFormatContext->streams[audio_index]->codecpar->codec_id);
decoder_context = avcodec_alloc_context3(avCodec);
avcodec_parameters_to_context(decoder_context, avFormatContext->streams[audio_index]->codecpar);
ret = avcodec_open2(decoder_context, avCodec, nullptr);
if (ret < 0) {
std::cout << "音频解码器打开失败" << std::endl;
break;
}
while (true) {
ret = av_read_frame(avFormatContext, avPacket);
if (ret < 0) {
std::cout << "音频包读取完毕" << std::endl;
break;
}
if (avPacket->stream_index != audio_index) {
av_packet_unref(avPacket);
continue;
}
ret = avcodec_send_packet(decoder_context, avPacket);
if (ret < 0) {
std::cout << "音频包发送解码失败" << std::endl;
break;
}
while (true) {
ret = avcodec_receive_frame(decoder_context, avFrame);
if (ret == AVERROR(EAGAIN)) {
std::cout << "音频包获取解码帧：EAGAIN" << std::endl;
break;
} else if (ret < 0) {
std::cout << "音频包获取解码帧：fail" << std::endl;
break;
} else {
std::cout << "重新编码音频" << std::endl;
// 先进行重采样
resample_audio(avFrame);
pack_vector.clear();
encode_audio(pack_vector, out_frame);
while (!pack_vector.empty()) {
AVPacket *packet = pack_vector.at(0);
pack_vector.erase(pack_vector.cbegin());
// 回调
callback(audio_encoder_context, packet, false);
}
}
}
av_packet_unref(avPacket);
}
}
avcodec_free_context(&decoder_context);
avformat_free_context(avFormatContext);
// 回调结束
callback(audio_encoder_context, nullptr, true);
}
private:
// 视频编码器
AVCodecContext *audio_encoder_context = nullptr;
AVFrame *encode_frame = nullptr;
AVAudioFifo *audiofifo = nullptr;
int64_t cur_pts = 0;
// 重采样相关
SwrContext *swrContext = nullptr;
AVFrame *out_frame = nullptr;
int64_t max_dst_nb_samples;
void init_out_frame(int64_t dst_nb_samples){
av_frame_free(&out_frame);
out_frame = av_frame_alloc();
out_frame->sample_rate = TARGET_AUDIO_SAMPLE_RATE;
out_frame->format = AV_SAMPLE_FMT_FLTP;
out_frame->channel_layout = AV_CH_LAYOUT_STEREO;
out_frame->nb_samples = dst_nb_samples;
// 分配buffer
av_frame_get_buffer(out_frame,0);
av_frame_make_writable(out_frame);
}
/**
* 重采样
* @param avFrame
*/
void resample_audio(AVFrame *avFrame){
if (nullptr == swrContext) {
/**
* 以下可以使用 swr_alloc、av_opt_set_channel_layout、av_opt_set_int、av_opt_set_sample_fmt
* 等API设置，更加灵活
*/
swrContext = swr_alloc_set_opts(nullptr, AV_CH_LAYOUT_STEREO, AV_SAMPLE_FMT_FLTP, TARGET_AUDIO_SAMPLE_RATE,
avFrame->channel_layout, static_cast<AVSampleFormat>(avFrame->format),
avFrame->sample_rate, 0, nullptr);
swr_init(swrContext);
}
// 进行音频重采样
int src_nb_sample = avFrame->nb_samples;
// 为了保持从采样后 dst_nb_samples / dest_sample = src_nb_sample / src_sample_rate
max_dst_nb_samples = av_rescale_rnd(src_nb_sample, TARGET_AUDIO_SAMPLE_RATE, avFrame->sample_rate, AV_ROUND_UP);
// 从采样器中会缓存一部分，获取缓存的长度
int64_t delay = swr_get_delay(swrContext, avFrame->sample_rate);
// 相当于a*b/c
int64_t dst_nb_samples = av_rescale_rnd(delay + avFrame->nb_samples, TARGET_AUDIO_SAMPLE_RATE, avFrame->sample_rate,
AV_ROUND_UP);
if(nullptr == out_frame){
init_out_frame(dst_nb_samples);
}
if (dst_nb_samples > max_dst_nb_samples) {
// 需要重新分配buffer
std::cout << "需要重新分配buffer" << std::endl;
init_out_frame(dst_nb_samples);
max_dst_nb_samples = dst_nb_samples;
}
// 重采样
int ret = swr_convert(swrContext, out_frame->data, dst_nb_samples,
const_cast<const uint8_t **>(avFrame->data), avFrame->nb_samples);
if(ret < 0){
std::cout << "重采样失败" << std::endl;
} else{
// 每帧音频数据量的大小
int data_size = av_get_bytes_per_sample(static_cast<AVSampleFormat>(out_frame->format));
// 返回值才是真正的重采样点数
out_frame->nb_samples = ret;
std::cout << "重采样成功：" << ret << "----dst_nb_samples:" << dst_nb_samples << "---data_size:" << data_size << std::endl;
}
}
void encode_audio(std::vector<AVPacket *> &pack_vector, AVFrame *avFrame) {
int cache_size = av_audio_fifo_size(audiofifo);
std::cout << "cache_size:" << cache_size << std::endl;
av_audio_fifo_realloc(audiofifo, cache_size + avFrame->nb_samples);
av_audio_fifo_write(audiofifo, reinterpret_cast<void **>(avFrame->data), avFrame->nb_samples);
if (nullptr == encode_frame) {
encode_frame = av_frame_alloc();
encode_frame->nb_samples = audio_encoder_context->frame_size;
encode_frame->sample_rate = audio_encoder_context->sample_rate;
encode_frame->channel_layout = audio_encoder_context->channel_layout;
encode_frame->channels = audio_encoder_context->channels;
encode_frame->format = audio_encoder_context->sample_fmt;
av_frame_get_buffer(encode_frame, 0);
}
av_frame_make_writable(encode_frame);
// todo 如果是冲刷最后几帧数据，不够的可以填充静音 av_samples_set_silence
while (av_audio_fifo_size(audiofifo) > audio_encoder_context->frame_size) {
int ret = av_audio_fifo_read(audiofifo, reinterpret_cast<void **>(encode_frame->data),
audio_encoder_context->frame_size);
if (ret < 0) {
std::cout << "audiofifo 读取数据失败" << std::endl;
return;
}
// 修改pts
cur_pts += encode_frame->nb_samples;
encode_frame->pts = cur_pts;
ret = avcodec_send_frame(audio_encoder_context, encode_frame);
if (ret < 0) {
std::cout << "发送编码失败" << std::endl;
return;
}
while (true) {
AVPacket *out_pack = av_packet_alloc();
ret = avcodec_receive_packet(audio_encoder_context, out_pack);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
std::cout << "avcodec_receive_packet end:" << ret << std::endl;
break;
} else if (ret < 0) {
std::cout << "avcodec_receive_packet fail:" << ret << std::endl;
return;
} else {
pack_vector.push_back(out_pack);
}
}
}
}
};
VideoHandle.cpp
/**
* 视频处理
* 解码视频，然后转换成720x1080，然后编码成h264
*/
#ifndef TARGET_VIDEO_WIDTH
#define TARGET_VIDEO_WIDTH 720
#define TARGET_VIDEO_HEIGHT 1280
#endif
#include <iostream>
#include <vector>
extern "C" {
#include <libavcodec/avcodec.h>
#include <libavformat/avformat.h>
#include <libavutil/opt.h>
#include <libswscale/swscale.h>
};
class VideoHandle {
public:
void handle_video(std::vector<char *> mp4_paths,
std::function<void(const AVCodecContext *, AVPacket *, bool)> callback) {
// 视频编码器相关
const AVCodec *video_codec = avcodec_find_encoder(AV_CODEC_ID_H264);
video_encoder_context = avcodec_alloc_context3(video_codec);
video_encoder_context->pix_fmt = AV_PIX_FMT_YUV420P;
video_encoder_context->width = TARGET_VIDEO_WIDTH;
video_encoder_context->height = TARGET_VIDEO_HEIGHT;
video_encoder_context->bit_rate = 2000 * 1024;
video_encoder_context->gop_size = 10;
video_encoder_context->time_base = {1, 25};
video_encoder_context->framerate = {25, 1};
// b帧的数量
video_encoder_context->max_b_frames = 1;
// 设置H264的编码器参数为延迟模式，提高编码质量，但是会造成编码速度下降
// av_opt_set(video_encoder_context->priv_data, "preset", "slow", 0);
int ret = avcodec_open2(video_encoder_context, video_codec, nullptr);
if (ret < 0) {
std::cout << "视频编码器打开失败" << std::endl;
return;
}
AVFormatContext *avFormatContext = nullptr;
AVCodecContext *decoder_context = nullptr;
AVPacket *avPacket = av_packet_alloc();
AVFrame *avFrame = av_frame_alloc();
std::vector<AVPacket *> pack_vector = std::vector<AVPacket *>();
// 前面视频的pts累计
int64_t previous_pts = 0;
// 但前视频最后的pts
int64_t last_pts = 0;
while (!mp4_paths.empty()) {
// 先释放旧的
previous_pts += last_pts;
avcodec_free_context(&decoder_context);
avformat_free_context(avFormatContext);
const char *mp4 = mp4_paths.at(0);
mp4_paths.erase(mp4_paths.cbegin());
avFormatContext = avformat_alloc_context();
ret = avformat_open_input(&avFormatContext, mp4, nullptr, nullptr);
if (ret < 0) {
std::cout << "视频文件打开失败" << std::endl;
break;
}
int video_index = av_find_best_stream(avFormatContext, AVMEDIA_TYPE_VIDEO, -1, -1, nullptr, 0);
if (video_index < 0) {
std::cout << "没有找到视频流" << std::endl;
break;
}
const AVCodec *avCodec = avcodec_find_decoder(avFormatContext->streams[video_index]->codecpar->codec_id);
decoder_context = avcodec_alloc_context3(avCodec);
avcodec_parameters_to_context(decoder_context, avFormatContext->streams[video_index]->codecpar);
ret = avcodec_open2(decoder_context, avCodec, nullptr);
if (ret < 0) {
std::cout << "视频解码器打开失败" << std::endl;
break;
}
while (true) {
ret = av_read_frame(avFormatContext, avPacket);
if (ret < 0) {
std::cout << "视频包读取完毕" << std::endl;
break;
}
if(avPacket->stream_index != video_index){
av_packet_unref(avPacket);
continue;
}
ret = avcodec_send_packet(decoder_context, avPacket);
if (ret < 0) {
char error[1024];
av_strerror(ret,error,1024);
std::cout << "视频包发送解码失败" << error << std::endl;
break;
}
while (true) {
ret = avcodec_receive_frame(decoder_context, avFrame);
if (ret == AVERROR(EAGAIN)) {
std::cout << "视频包获取解码帧：EAGAIN" << std::endl;
break;
} else if (ret < 0) {
std::cout << "视频包获取解码帧：fail" << std::endl;
} else {
std::cout << "重新编码视频" << std::endl;
pack_vector.clear();
// 转换成统一的pts
last_pts = av_rescale_q(avFrame->pts,avFormatContext->streams[video_index]->time_base,AV_TIME_BASE_Q);
avFrame->pts = previous_pts + last_pts;
// 尺寸转换
scale_yuv(avFrame);
// 重新编码
encode_video(pack_vector,out_frame);
while (!pack_vector.empty()){
AVPacket *packet = pack_vector.at(0);
// 回调
callback(video_encoder_context,packet, false);
pack_vector.erase(pack_vector.cbegin());
}
}
}
av_packet_unref(avPacket);
}
}
avcodec_free_context(&decoder_context);
avformat_free_context(avFormatContext);
// 回调结束
callback(video_encoder_context, nullptr, true);
}
void scale_yuv(AVFrame *frame){
swsContext = sws_getCachedContext(swsContext,frame->width,frame->height,AV_PIX_FMT_YUV420P,TARGET_VIDEO_WIDTH,TARGET_VIDEO_HEIGHT,AV_PIX_FMT_YUV420P,
SWS_BILINEAR,
nullptr, nullptr, nullptr);
if(nullptr == out_frame){
out_frame = av_frame_alloc();
out_frame->format = AV_PIX_FMT_YUV420P;
out_frame->width = TARGET_VIDEO_WIDTH;
out_frame->height = TARGET_VIDEO_HEIGHT;
av_frame_get_buffer(out_frame,0);
}
// 转换
int ret = sws_scale(swsContext,frame->data,frame->linesize,0,frame->height,out_frame->data,out_frame->linesize);
// pts
std::cout << "frame->pts:" << frame->pts << std::endl;
out_frame->pts = frame->pts;
if(ret < 0){
std::cout << "图像缩放失败" << std::endl;
return;
}
}
void encode_video(std::vector<AVPacket *>& pack_vector, AVFrame *frame) {
int ret = avcodec_send_frame(video_encoder_context, frame);
if (ret < 0) {
std::cout << "视频发送编码失败" << std::endl;
return;
}
while (true) {
AVPacket *packet = av_packet_alloc();
ret = avcodec_receive_packet(video_encoder_context,packet);
if(ret == AVERROR(EAGAIN)){
std::cout << "视频编码：EAGAIN" << std::endl;
break;
} else if(ret < 0){
std::cout << "视频编码：fail" << std::endl;
break;
} else{
pack_vector.push_back(packet);
}
}
}
// 视频编码器
AVCodecContext *video_encoder_context = nullptr;
// 视频转换专用
SwsContext *swsContext = nullptr;
AVFrame *out_frame = nullptr;
};
ComplexMuxerCore.cpp
/**
* 将视频和音频合并成mp4输出文件
*/
#ifndef MAX_QUEUE_SIZE
// 队列缓存最大值
#define MAX_QUEUE_SIZE 6
#endif
#include <iostream>
#include <vector>
#include <thread>
#include <AudioHandle.cpp>
#include <VideoHandle.cpp>
extern "C" {
#include <libavformat/avformat.h>
}
class ComplexMuxerCore {
public:
ComplexMuxerCore() : audio_queue(new std::vector<AVPacket *>()), video_queue(new std::vector<AVPacket *>()) {
}
/**
* 这个是主函数，也就是说在main中调用这个函数即可
* @param mp3_paths
* @param mp4_paths
* @param mp4_out
*/
void muxer_media(const std::vector<char *> mp3_paths, const std::vector<char *> mp4_paths,
const char *mp4_out) {
audioHandle = new AudioHandle();
auto a_call_back = std::bind(&ComplexMuxerCore::audio_callback, this, std::placeholders::_1,
std::placeholders::_2, std::placeholders::_3);
audio_thread = new std::thread(&AudioHandle::handle_audio, audioHandle, mp3_paths, a_call_back);
videoHandle = new VideoHandle();
auto v_call_back = std::bind(&ComplexMuxerCore::video_callback, this, std::placeholders::_1,
std::placeholders::_2, std::placeholders::_3);
video_thread = new std::thread(&VideoHandle::handle_video, videoHandle, mp4_paths, v_call_back);
muxer_thread = new std::thread(&ComplexMuxerCore::muxer_out, this, mp4_out);
muxer_thread->join();
}
~ComplexMuxerCore() {
// todo 释放资源
}
private:
VideoHandle *videoHandle = nullptr;
AudioHandle *audioHandle = nullptr;
// 音频处理线程
std::thread *audio_thread = nullptr;
// 视频处理线程
std::thread *video_thread = nullptr;
// 合并线程
std::thread *muxer_thread = nullptr;
// 音频队列
std::vector<AVPacket *> *audio_queue = nullptr;
// 视频队列
std::vector<AVPacket *> *video_queue = nullptr;
// 音频线程同步互斥量
std::mutex audio_mutex;
// 视频线程同步互斥量
std::mutex video_mutex;
// 合并线程同步互斥量
std::mutex muxer_mutex;
// 音频条件变量
std::condition_variable audio_conditionVariable;
// 视频条件变量
std::condition_variable video_conditionVariable;
// 合并条件变量
std::condition_variable conditionVariable;
// 输入音频是否处理完毕
volatile bool is_audio_end;
// 输入视频是否处理完毕
volatile bool is_video_end;
// 输出视频流的索引
int out_video_stream_index = -1;
// 输入视频流索引
int out_audio_stream_index = -1;
// 视频pts
double video_pts = 0;
// 音频pts
double audio_pts = 0;
AVFormatContext *out_format_context = nullptr;
void muxer_out(const char *mp4_out) {
out_format_context = avformat_alloc_context();
const AVOutputFormat *avOutputFormat = av_guess_format(nullptr, mp4_out, nullptr);
out_format_context->oformat = avOutputFormat;
while (out_video_stream_index < 0 || out_audio_stream_index < 0) {
std::cout << "视频流或音频流还没创建好，陷入等待" << std::endl;
std::unique_lock<std::mutex> muxer_lock(muxer_mutex);
conditionVariable.wait(muxer_lock);
}
int ret = avio_open(&out_format_context->pb, mp4_out, AVIO_FLAG_WRITE);
if (ret < 0) {
std::cout << "输出流打开失败" << std::endl;
return;
}
ret = avformat_write_header(out_format_context, nullptr);
if (ret < 0) {
std::cout << "文件头写入失败" << std::endl;
return;
}
while (!is_handle_end()) {
std::cout << "muxer while" << std::endl;
// 视频包的pts大于音频包或者视频包写完了则写音频包
if((video_pts > audio_pts && !audio_queue->empty()) ||
(is_video_end && video_queue->empty())){
// 写入音频包
write_audio();
} else {
// 写入视频包
write_video();
}
}
std::cout << "开始写入文件头" << std::endl;
ret = av_write_trailer(out_format_context);
if (ret < 0) {
std::cout << "文件尾写入失败" << std::endl;
} else {
std::cout << "合并完成" << std::endl;
}
}
void write_audio(){
while (audio_queue->empty() && !is_audio_end) {
std::cout << "等待音频包生产" << std::endl;
std::unique_lock<std::mutex> uniqueLock(audio_mutex);
audio_conditionVariable.wait(uniqueLock);
}
if (!audio_queue->empty()) {
// 锁住
std::lock_guard<std::mutex> lockGuard(audio_mutex);
AVPacket *pack = audio_queue->at(0);
// pts转换
// av_packet_rescale_ts(pack,pack->time_base,out_format_context->streams[out_audio_stream_index]->time_base);
audio_pts = pack->pts * av_q2d(out_format_context->streams[out_audio_stream_index]->time_base);
std::cout << "写入音频包 audio_pts:" << audio_pts << std::endl;
av_write_frame(out_format_context, pack);
av_packet_free(&pack);
audio_queue->erase(audio_queue->cbegin());
}
// 唤醒
audio_conditionVariable.notify_all();
// 休眠一下，模拟消费比生产慢
std::this_thread::sleep_for(std::chrono::milliseconds(10));
}
void write_video(){
while (video_queue->empty() && !is_video_end) {
std::cout << "等待视频包生产" << std::endl;
std::unique_lock<std::mutex> uniqueLock(video_mutex);
video_conditionVariable.wait(uniqueLock);
}
// 大括号括起来可以及时释放锁
if (!video_queue->empty()) {
// 加锁
std::lock_guard<std::mutex> lockGuard(video_mutex);
AVPacket *pack = video_queue->at(0);
// 之前在VideoHandle 中转换了统一的pts，现在要转换回去
av_packet_rescale_ts(pack,AV_TIME_BASE_Q,out_format_context->streams[out_video_stream_index]->time_base);
video_pts = pack->pts * av_q2d(out_format_context->streams[out_video_stream_index]->time_base);
std::cout << "写入视频包 video_pts:" << video_pts << std::endl;
// pts转换
// av_packet_rescale_ts(pack,pack->time_base,out_format_context->streams[out_video_stream_index]->time_base);
av_write_frame(out_format_context, pack);
av_packet_free(&pack);
video_queue->erase(video_queue->cbegin());
}
// 唤醒
video_conditionVariable.notify_all();
// 休眠一下，模拟消费比生产慢
std::this_thread::sleep_for(std::chrono::milliseconds(10));
}
void audio_callback(const AVCodecContext *codecContext, AVPacket *avPacket, bool is_end) {
if (nullptr == out_format_context) {
// 复用器还没初始化好
std::cout << "复用器还没初始化" << std::endl;
return;
}
if (out_audio_stream_index < 0) {
// 加锁
std::cout << "audio_callback" << std::endl;
std::lock_guard<std::mutex> lockGuard(muxer_mutex);
AVStream *audio_stream = avformat_new_stream(out_format_context, codecContext->codec);
avcodec_parameters_from_context(audio_stream->codecpar, codecContext);
out_audio_stream_index = audio_stream->index;
// 唤醒
conditionVariable.notify_all();
}
// 队列超了就阻塞在这里
while (audio_queue->size() >= MAX_QUEUE_SIZE) {
std::cout << "音频队列超出缓存，等待消费" << std::endl;
std::unique_lock<std::mutex> uniqueLock(audio_mutex);
audio_conditionVariable.wait(uniqueLock);
}
{
if (nullptr != avPacket) {
std::lock_guard<std::mutex> video_lock(audio_mutex);
avPacket->stream_index = out_audio_stream_index;
audio_queue->push_back(avPacket);
}
}
is_audio_end = is_end;
// 唤醒消费队列
audio_conditionVariable.notify_all();
}
void video_callback(const AVCodecContext *codecContext, AVPacket *avPacket, bool is_end) {
std::cout << "video_callback" << std::endl;
// 队列超了就阻塞在这里
if (nullptr == out_format_context) {
// 复用器还没初始化好
return;
}
if (out_video_stream_index < 0) {
// 加锁
std::cout << "video_callback" << std::endl;
std::lock_guard<std::mutex> lockGuard(muxer_mutex);
AVStream *video_stream = avformat_new_stream(out_format_context, codecContext->codec);
avcodec_parameters_from_context(video_stream->codecpar, codecContext);
out_video_stream_index = video_stream->index;
std::cout << "创建视频输出流:" << out_video_stream_index << std::endl;
// 唤醒
conditionVariable.notify_all();
}
std::cout << "video_callback：" << video_queue->size() << std::endl;
while (video_queue->size() >= MAX_QUEUE_SIZE) {
std::cout << "视频队列超出缓存，等待消费" << std::endl;
std::unique_lock<std::mutex> uniqueLock(video_mutex);
video_conditionVariable.wait(uniqueLock);
}
{
if (nullptr != avPacket) {
std::lock_guard<std::mutex> video_lock(video_mutex);
avPacket->stream_index = out_video_stream_index;
video_queue->push_back(avPacket);
}
}
is_video_end = is_end;
// 唤醒消费队列
video_conditionVariable.notify_all();
}
/**
* 是否处理完毕
* @return
*/
bool is_handle_end() {
return is_video_end &&
is_audio_end &&
(nullptr == audio_queue || audio_queue->empty()) &&
(nullptr == video_queue || video_queue->empty());
}
};
不得不说写技术博客是一个很耗时耗力的事情，本来为了更方便理解应该详细讲每个实现文件的细节问题的，由于时间问题和文笔表达不大好还是算了，有兴趣的私聊吧…
思考
上面的代码在视频合并的过程中我们通过pts的比较交叉写入音频包或视频包。为什么需要这样做呢？假设在不考虑多线程性能的前提下，能先写完音频再写入视频，或者能先写完视频再写入音频吗？
欢迎大家沟通交流…
系列推荐
FFmpeg连载1-开发环境搭建
FFmpeg连载2-分离视频和音频
FFmpeg连载3-视频解码
FFmpeg连载4-音频解码
FFmpeg连载5-音视频编码
FFmpeg连载6-音频重采样
FFmpeg连载7-mp3转码aac
关注我，一起进步，人生不止coding！！！
