Dynamic Face Detection: C++ OpenCV & FFmpeg 7

#include <iostream>
#include <vector>
#include <string>

// FFmpeg 7 C headers
extern "C" {
#include <libavformat/avformat.h>
#include <libavcodec/avcodec.h>
#include <libswscale/swscale.h>
#include <libavutil/imgutils.h>
}

// OpenCV headers
#include <opencv2/opencv.hpp>
#include <opencv2/objdetect.hpp>

// Struct: Record face info and precise timestamps
struct FaceRecord {
    double timestampSec;
    cv::Rect boundingBox;
};

int main(int argc, char** argv) {
    if (argc < 3) {
        std::cerr << "Usage: ./VideoFaceDetector <video_path> <cascade_xml_path>" << std::endl;
        return -1;
    }

    std::string videoPath = argv[1];
    std::string cascadePath = argv[2];

    // Initialize OpenCV Face Detector
    cv::CascadeClassifier faceCascade;
    if (!faceCascade.load(cascadePath)) {
        std::cerr << "Error: Cannot load OpenCV cascade file." << std::endl;
        return -1;
    }

    // FFmpeg: Open Video Stream
    AVFormatContext* formatCtx = nullptr;
    if (avformat_open_input(&formatCtx, videoPath.c_str(), nullptr, nullptr) != 0) {
        std::cerr << "Error: Cannot open video stream." << std::endl;
        return -1;
    }

    if (avformat_find_stream_info(formatCtx, nullptr) < 0) {
        std::cerr << "Error: Cannot find stream information." << std::endl;
        return -1;
    }

    // Find Video Stream Index
    int videoStreamIdx = -1;
    const AVCodec* codec = nullptr;
    for (unsigned int i = 0; i < formatCtx->nb_streams; i++) {
        if (formatCtx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
            videoStreamIdx = i;
            codec = avcodec_find_decoder(formatCtx->streams[i]->codecpar->codec_id);
            break;
        }
    }

    if (videoStreamIdx == -1 || !codec) {
        std::cerr << "Error: Video stream or codec not found." << std::endl;
        return -1;
    }

    // Configure Decoder Context
    AVCodecContext* codecCtx = avcodec_alloc_context3(codec);
    avcodec_parameters_to_context(codecCtx, formatCtx->streams[videoStreamIdx]->codecpar);
    if (avcodec_open2(codecCtx, codec, nullptr) < 0) {
        std::cerr << "Error: Cannot open codec." << std::endl;
        return -1;
    }

    // Memory Allocation: Packet and Frame
    AVPacket* packet = av_packet_alloc();
    AVFrame* frame = av_frame_alloc();
    AVFrame* rgbFrame = av_frame_alloc();

    // OpenCV requires BGR24 format, setup swscale conversion context
    int numBytes = av_image_get_buffer_size(AV_PIX_FMT_BGR24, codecCtx->width, codecCtx->height, 1);
    uint8_t* buffer = (uint8_t*)av_malloc(numBytes * sizeof(uint8_t));
    av_image_fill_arrays(rgbFrame->data, rgbFrame->linesize, buffer, AV_PIX_FMT_BGR24, codecCtx->width, codecCtx->height, 1);

    SwsContext* swsCtx = sws_getContext(
        codecCtx->width, codecCtx->height, codecCtx->pix_fmt,
        codecCtx->width, codecCtx->height, AV_PIX_FMT_BGR24,
        SWS_BILINEAR, nullptr, nullptr, nullptr
    );

    std::vector<FaceRecord> detectedFaces;
    AVRational timeBase = formatCtx->streams[videoStreamIdx]->time_base;

    // Start Video Frame Reading Loop
    while (av_read_frame(formatCtx, packet) >= 0) {
        if (packet->stream_index == videoStreamIdx) {
            // Send packet to decoder
            if (avcodec_send_packet(codecCtx, packet) == 0) {
                // Receive decoded raw frame (one packet might contain multiple frames)
                while (avcodec_receive_frame(codecCtx, frame) == 0) {
                    
                    // Core: Calculate absolute timestamp of current frame (seconds)
                    double timestamp = frame->pts * av_q2d(timeBase);

                    // Convert YUV to BGR for OpenCV compatibility
                    sws_scale(swsCtx, frame->data, frame->linesize, 0, codecCtx->height, rgbFrame->data, rgbFrame->linesize);

                    // Wrap RGB data into cv::Mat (Zero-copy mapping)
                    cv::Mat img(codecCtx->height, codecCtx->width, CV_8UC3, rgbFrame->data[0], rgbFrame->linesize[0]);
                    
                    // Convert to grayscale to improve detection efficiency
                    cv::Mat gray;
                    cv::cvtColor(img, gray, cv::COLOR_BGR2GRAY);
                    cv::equalizeHist(gray, gray);

                    // Execute OpenCV dynamic face detection
                    std::vector<cv::Rect> faces;
                    faceCascade.detectMultiScale(gray, faces, 1.1, 4, 0 | cv::CASCADE_SCALE_IMAGE, cv::Size(30, 30));

                    // Record all face feature points
                    for (const auto& face : faces) {
                        detectedFaces.push_back({timestamp, face});
                    }
                }
            }
        }
        av_packet_unref(packet);
    }

    // Output final analytics report
    std::cout << "\n[Analytics Complete] Total faces detected: " << detectedFaces.size() << std::endl;
    for (size_t i = 0; i < detectedFaces.size(); i++) {
        std::cout << "Time: " << detectedFaces[i].timestampSec << "s | "
                  << "Box: [x:" << detectedFaces[i].boundingBox.x << ", y:" << detectedFaces[i].boundingBox.y 
                  << ", w:" << detectedFaces[i].boundingBox.width << ", h:" << detectedFaces[i].boundingBox.height 
                  << "]" << std::endl;
    }

    // Safely release all resources
    av_free(buffer);
    av_frame_free(&rgbFrame);
    av_frame_free(&frame);
    av_packet_free(&packet);
    avcodec_free_context(&codecCtx);
    avformat_close_input(&formatCtx);
    sws_freeContext(swsCtx);

    return 0;
}