FFmpeg+SDL实时解码和渲染H264视频流
前言
之前实现了Android手机摄像头数据的TCP实时传输，今天接着聊聊，如何在PC端把接收到的H264视频流实时解码并渲染出来。这次使用的语言是C++，框架有FFmpeg和SDL2。
解码
解码部分使用FFmpeg，首先，需要初始化H264解码器：
int H264Decoder::init() {
codec = avcodec_find_decoder(AV_CODEC_ID_H264);
if (codec == nullptr) {
printf("No H264 decoder found\n");
return -1;
}
codecCtx = avcodec_alloc_context3(codec);
codecCtx->flags |= AV_CODEC_FLAG_LOW_DELAY;
if (avcodec_open2(codecCtx, codec, nullptr) < 0) {
printf("Failed to open codec\n");
return -2;
}
packet = av_packet_alloc();
m_Frame = av_frame_alloc();
parser = av_parser_init(AV_CODEC_ID_H264);
return 0;
}
然后，使用创建TCP连接到我们的Android端，读取数据包：
bool read_data(SOCKET socket, void* data, unsigned int len) {
while (len > 0) {
int ret = recv(socket, (char*)data, len, 0);
if (ret <= 0) {
return false;
}
len -= ret;
data = (char*)data + ret;
}
return true;
}
bool read_int(SOCKET socket, ULONG* value) {
bool ret = read_data(socket, value, 4);
if (ret) {
*value = ntohl(*value);
}
return ret;
}
int PacketReceiver::readPacket(unsigned char** data, unsigned long* size) {
ULONG pkgSize = 0;
bool ret = read_int(m_Socket, &pkgSize);
if (!ret) {
printf("Failed to read packet size\n");
return -1;
}
if (m_DataLen < pkgSize) {
if (m_Data != nullptr) {
delete[] m_Data;
}
m_Data = new unsigned char[pkgSize];
m_DataLen = pkgSize;
}
if (!read_data(m_Socket, m_Data, pkgSize)) {
printf("Failed to read packet data\n");
return -2;
}
*data = m_Data;
*size = pkgSize;
return 0;
}
再把每个数据包传送给H264解码器解码
int H264Decoder::decode(unsigned char* data, int size, AVFrame** frame) {
int new_pkg_ret = av_new_packet(packet, size);
if (new_pkg_ret != 0) {
printf("Failed to create new packet\n");
return -1;
}
memcpy(packet->data, data, size);
int ret = avcodec_send_packet(codecCtx, packet);
if (ret < 0 && ret != AVERROR(EAGAIN)) {
printf("Failed to parse packet\n");
return -1;
}
ret = avcodec_receive_frame(codecCtx, m_Frame);
if (ret == AVERROR(EAGAIN)) {
*frame = nullptr;
return 0;
}
if (ret != 0) {
printf("Failed to read frame\n");
return -1;
}
*frame = m_Frame;
av_packet_unref(packet);
return 0;
}
解码器解码后，最终得到的是AVFrame对象，代表一帧画面，数据格式一般为YUV格式（跟编码端选择的像素格式有关）。
渲染
通过使用SDL2，我们可以直接渲染YUV数据，无需手动转成RGB。
首先，我们先初始化SDL2并创建渲染窗口：
int YuvRender::init(int video_width, int video_height) {
SDL_Init(SDL_INIT_VIDEO);
SDL_Rect bounds;
SDL_GetDisplayUsableBounds(0, &bounds);
int winWidth = video_width;
int winHeight = video_height;
if (winWidth > bounds.w || winHeight > bounds.h) {
float widthRatio = 1.0 * winWidth / bounds.w;
float heightRatio = 1.0 * winHeight / bounds.h;
float maxRatio = widthRatio > heightRatio ? widthRatio : heightRatio;
winWidth = int(winWidth / maxRatio);
winHeight = int(winHeight / maxRatio);
}
SDL_Window* window = SDL_CreateWindow(
"NetCameraViewer",
SDL_WINDOWPOS_UNDEFINED,
SDL_WINDOWPOS_UNDEFINED,
winWidth,
winHeight,
SDL_WINDOW_OPENGL
);
m_Renderer = SDL_CreateRenderer(window, -1, 0);
m_Texture = SDL_CreateTexture(
m_Renderer,
SDL_PIXELFORMAT_IYUV,
SDL_TEXTUREACCESS_STREAMING,
video_width,
video_height
);
m_VideoWidth = video_width;
m_VideoHeight = video_height;
m_Rect.x = 0;
m_Rect.y = 0;
m_Rect.w = winWidth;
m_Rect.h = winHeight;
return 0;
}
每次解码出一帧画面的时候，再调用render函数渲染：
int YuvRender::render(unsigned char* data[], int pitch[]) {
int uvHeight = m_VideoHeight / 2;
int ySize = pitch[0] * m_VideoHeight;
int uSize = pitch[1] * uvHeight;
int vSize = pitch[2] * uvHeight;
int buffSize = ySize + uSize + vSize;
if (m_FrameBufferSize < buffSize) {
if (m_FrameBuffer != nullptr) {
delete[] m_FrameBuffer;
}
m_FrameBuffer = new unsigned char[buffSize];
m_FrameBufferSize = buffSize;
}
SDL_memcpy(m_FrameBuffer, data[0], ySize);
SDL_memcpy(m_FrameBuffer + ySize, data[1], uSize);
SDL_memcpy(m_FrameBuffer + ySize + uSize, data[2], vSize);
SDL_UpdateTexture(m_Texture, NULL, m_FrameBuffer, pitch[0]);
SDL_RenderClear(m_Renderer);
SDL_RenderCopy(m_Renderer, m_Texture, NULL, &m_Rect);
SDL_RenderPresent(m_Renderer);
SDL_PollEvent(&m_Event);
if (m_Event.type == SDL_QUIT) {
exit(0);
}
return 0;
}
性能
在搭载AMD Ryzen 5 5600U的机器上，1800 x 1350的分辨率，解码一帧平均25ms, 渲染1~2ms，加上编码和传输延时，总体延时在70ms左右。
完整源码已上传至Github: https://github.com/kasonyang/net-camera/tree/main/viewer-app