FFmpeg+SDL实时解码和渲染H264视频流

前言

之前实现了Android手机摄像头数据的TCP实时传输，今天接着聊聊，如何在PC端把接收到的H264视频流实时解码并渲染出来。这次使用的语言是C++，框架有FFmpeg和SDL2。

解码

解码部分使用FFmpeg，首先，需要初始化H264解码器：

int H264Decoder::init() {
    codec = avcodec_find_decoder(AV_CODEC_ID_H264);
    if (codec == nullptr) {
        printf("No H264 decoder found\n");
        return -1;
    }
    codecCtx = avcodec_alloc_context3(codec);
    codecCtx->flags |= AV_CODEC_FLAG_LOW_DELAY;
    if (avcodec_open2(codecCtx, codec, nullptr) < 0) {
        printf("Failed to open codec\n");
        return -2;
    }
    packet = av_packet_alloc();
    m_Frame = av_frame_alloc();
    parser = av_parser_init(AV_CODEC_ID_H264);
    return 0;
}

然后，使用创建TCP连接到我们的Android端，读取数据包：

bool read_data(SOCKET socket, void* data, unsigned int len) {
    while (len > 0) {
        int ret = recv(socket, (char*)data, len, 0);
        if (ret <= 0) {
            return false;
        }
        len -= ret;
        data = (char*)data + ret;
    }
    return true;
}

bool read_int(SOCKET socket, ULONG* value) {
    bool ret = read_data(socket, value, 4);
    if (ret) {
        *value = ntohl(*value);
    }
    return ret;
}

int PacketReceiver::readPacket(unsigned char** data, unsigned long* size) {
    ULONG pkgSize = 0;
    bool ret = read_int(m_Socket, &pkgSize);
    if (!ret) {
        printf("Failed to read packet size\n");
        return -1;
    }
    if (m_DataLen < pkgSize) {
        if (m_Data != nullptr) {
            delete[] m_Data;
        }
        m_Data = new unsigned char[pkgSize];
        m_DataLen = pkgSize;
    }
    if (!read_data(m_Socket, m_Data, pkgSize)) {
        printf("Failed to read packet data\n");
        return -2;
    }
    *data = m_Data;
    *size = pkgSize;
    return 0;
}

再把每个数据包传送给H264解码器解码

int H264Decoder::decode(unsigned char* data, int size, AVFrame** frame) {
    int new_pkg_ret = av_new_packet(packet, size);
    if (new_pkg_ret != 0) {
        printf("Failed to create new packet\n");
        return -1;
    }
    memcpy(packet->data, data, size);
    int ret = avcodec_send_packet(codecCtx, packet);
    if (ret < 0 && ret != AVERROR(EAGAIN)) {
        printf("Failed to parse packet\n");
        return -1;
    }
    ret = avcodec_receive_frame(codecCtx, m_Frame);
    if (ret == AVERROR(EAGAIN)) {
        *frame = nullptr;
        return 0;
    }
    if (ret != 0) {
        printf("Failed to read frame\n");
        return -1;
    }
    *frame = m_Frame;
    av_packet_unref(packet);
    return 0;
}

解码器解码后，最终得到的是AVFrame对象，代表一帧画面，数据格式一般为YUV格式（跟编码端选择的像素格式有关）。

渲染

通过使用SDL2，我们可以直接渲染YUV数据，无需手动转成RGB。

首先，我们先初始化SDL2并创建渲染窗口：

int YuvRender::init(int video_width, int video_height) {
    SDL_Init(SDL_INIT_VIDEO);
    SDL_Rect bounds;
    SDL_GetDisplayUsableBounds(0, &bounds);
    int winWidth = video_width;
    int winHeight = video_height;
    if (winWidth > bounds.w || winHeight > bounds.h) {
        float widthRatio = 1.0 * winWidth / bounds.w;
        float heightRatio = 1.0 * winHeight / bounds.h;
        float maxRatio = widthRatio > heightRatio ? widthRatio : heightRatio;
        winWidth = int(winWidth / maxRatio);
        winHeight = int(winHeight / maxRatio);
    }
    SDL_Window* window = SDL_CreateWindow(
        "NetCameraViewer",
        SDL_WINDOWPOS_UNDEFINED,
        SDL_WINDOWPOS_UNDEFINED,
        winWidth,
        winHeight,
        SDL_WINDOW_OPENGL
    );
    m_Renderer = SDL_CreateRenderer(window, -1, 0);
    m_Texture = SDL_CreateTexture(
        m_Renderer,
        SDL_PIXELFORMAT_IYUV,
        SDL_TEXTUREACCESS_STREAMING,
        video_width,
        video_height
    );
    m_VideoWidth = video_width;
    m_VideoHeight = video_height;
    m_Rect.x = 0;
    m_Rect.y = 0;
    m_Rect.w = winWidth;
    m_Rect.h = winHeight;
    return 0;
}

每次解码出一帧画面的时候，再调用render函数渲染：

int YuvRender::render(unsigned char* data[], int pitch[]) {
    int uvHeight = m_VideoHeight / 2;
    int ySize = pitch[0] * m_VideoHeight;
    int uSize = pitch[1] * uvHeight;
    int vSize = pitch[2] * uvHeight;
    int buffSize =  ySize + uSize + vSize;
    if (m_FrameBufferSize < buffSize) {
        if (m_FrameBuffer != nullptr) {
            delete[] m_FrameBuffer;
        }
        m_FrameBuffer = new unsigned char[buffSize];
        m_FrameBufferSize = buffSize;
    }
    SDL_memcpy(m_FrameBuffer, data[0], ySize);
    SDL_memcpy(m_FrameBuffer + ySize, data[1], uSize);
    SDL_memcpy(m_FrameBuffer + ySize + uSize, data[2], vSize);
    SDL_UpdateTexture(m_Texture, NULL, m_FrameBuffer, pitch[0]);
    SDL_RenderClear(m_Renderer);
    SDL_RenderCopy(m_Renderer, m_Texture, NULL, &m_Rect);
    SDL_RenderPresent(m_Renderer);
    SDL_PollEvent(&m_Event);
    if (m_Event.type == SDL_QUIT) {
        exit(0);
    }
    return 0;
}

性能

在搭载AMD Ryzen 5 5600U的机器上，1800 x 1350的分辨率，解码一帧平均25ms, 渲染1~2ms，加上编码和传输延时，总体延时在70ms左右。

完整源码已上传至Github: https://github.com/kasonyang/net-camera/tree/main/viewer-app