[Libav-user] AVFrame > OpenCV Mat > AVPacket conversion

classic Classic list List threaded Threaded
1 message Options
Reply | Threaded
Open this post in threaded view
|

[Libav-user] AVFrame > OpenCV Mat > AVPacket conversion

Davood Falahati
Dear all,

I am new to libav and I am writing a video manipulation software which uses opencv as its heart. What I did is briefly as below:

1- read the video packet 
2- decode the packet into AVFrame
3- convert the AVFrame to CV Mat
4- manipulate the Mat 
5- convert the CV Mat into AVFrame 
6- encode the AVFrame into AVPacket
7- write the packet 
8- goto 1

I read dranger tutorial in http://dranger.com/ffmpeg/tutorial01.html and I also used decoding_encoding example. I can read the video, extract video frames and convert them to CV Mat. My problem starts from converting from cv Mat to AVFrame and encode it to AVPacket. 

Would you please help me with this?

Here is my code :

int main(int argc, char **argv)
{
    AVOutputFormat *ofmt = NULL;
    AVFormatContext *ifmt_ctx = NULL, *ofmt_ctx = NULL;
    AVPacket pkt;
    AVCodecContext    *pCodecCtx = NULL;
    AVCodec           *pCodec = NULL;
    AVFrame           *pFrame = NULL;
    AVFrame           *pFrameRGB = NULL;
    int videoStream=-1;
    int audioStream=-1;
    int               frameFinished;
    int               numBytes;
    uint8_t           *buffer = NULL;
    struct SwsContext *sws_ctx = NULL;
    FrameManipulation *mal_frame;

    const char *in_filename, *out_filename;
    int ret, i;
    if (argc < 3) {
        printf("usage: %s input output\n"
               "API example program to remux a media file with libavformat and libavcodec.\n"
               "The output format is guessed according to the file extension.\n"
               "\n", argv[0]);
        return 1;
    }
    in_filename  = arg[1];
    out_filename = arg[2];
    av_register_all();
    if ((ret = avformat_open_input(&ifmt_ctx, in_filename, 0, 0)) < 0) {
        fprintf(stderr, "Could not open input file '%s'", in_filename);
        goto end;
    }

    if ((ret = avformat_find_stream_info(ifmt_ctx, 0)) < 0) {
        fprintf(stderr, "Failed to retrieve input stream information");
        goto end;
    }
    av_dump_format(ifmt_ctx, 0, in_filename, 0);
    avformat_alloc_output_context2(&ofmt_ctx, NULL, NULL, out_filename);

    if (!ofmt_ctx) {
        fprintf(stderr, "Could not create output context\n");
        ret = AVERROR_UNKNOWN;
        goto end;
    }
    ofmt = ofmt_ctx->oformat;

    for (i = 0; i < ifmt_ctx->nb_streams; i++) {
        AVStream *in_stream = ifmt_ctx->streams[i];
        AVStream *out_stream = avformat_new_stream(ofmt_ctx, in_stream->codec->codec);

        if(ifmt_ctx->streams[i]->codec->codec_type==AVMEDIA_TYPE_VIDEO &&
           videoStream < 0) {
               videoStream=i;
        }
        if(ifmt_ctx->streams[i]->codec->codec_type==AVMEDIA_TYPE_AUDIO &&
           audioStream < 0) {
                audioStream=i;
        }
        if (!out_stream) {
            fprintf(stderr, "Failed allocating output stream\n");
            ret = AVERROR_UNKNOWN;
            goto end;
        }
        ret = avcodec_copy_context(out_stream->codec, in_stream->codec);
        if (ret < 0) {
            fprintf(stderr, "Failed to copy context from input to output stream codec context\n");
            goto end;
        }
        out_stream->codec->codec_tag = 0;
        if (ofmt_ctx->oformat->flags & AVFMT_GLOBALHEADER)
           out_stream->codec->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
    }
    pCodec=avcodec_find_decoder(ifmt_ctx->streams[videoStream]->codec->codec_id);
    pCodecCtx = avcodec_alloc_context3(pCodec);

    if(avcodec_copy_context(pCodecCtx, ifmt_ctx->streams[videoStream]->codec) != 0) {
      fprintf(stderr, "Couldn't copy codec context");
      return -1; // Error copying codec context
    }

    // Open codec
     if(avcodec_open2(pCodecCtx, pCodec, NULL)<0)
       return -1; // Could not open codec

    // Allocate video frame
     pFrame=av_frame_alloc();

     // Allocate an AVFrame structure
     pFrameRGB=av_frame_alloc();

     // Determine required buffer size and allocate buffer
     numBytes=avpicture_get_size(AV_PIX_FMT_RGB24, ifmt_ctx->streams[videoStream]->codec->width,
                     ifmt_ctx->streams[videoStream]->codec->height);

     buffer=(uint8_t *)av_malloc(numBytes*sizeof(uint8_t));

     // Assign appropriate parts of buffer to image planes in pFrameRGB
     // Note that pFrameRGB is an AVFrame, but AVFrame is a superset
     // of AVPicture
     avpicture_fill((AVPicture *)pFrameRGB, buffer, AV_PIX_FMT_BGR24,
            ifmt_ctx->streams[videoStream]->codec->width, ifmt_ctx->streams[videoStream]->codec->height);

     av_dump_format(ofmt_ctx, 0, out_filename, 1);

     if (!(ofmt->flags & AVFMT_NOFILE)) {
        ret = avio_open(&ofmt_ctx->pb, out_filename, AVIO_FLAG_WRITE);
        if (ret < 0) {
            fprintf(stderr, "Could not open output file '%s'", out_filename);
            goto end;
        }
    }
    ret = avformat_write_header(ofmt_ctx, NULL);
    if (ret < 0) {
        fprintf(stderr, "Error occurred when opening output file\n");
        goto end;
    }
    // Assign appropriate parts of buffer to image planes in pFrameRGB
    // Note that pFrameRGB is an AVFrame, but AVFrame is a superset
    // of AVPicture
    avpicture_fill((AVPicture *)pFrameRGB, buffer, AV_PIX_FMT_BGR24,
                       ifmt_ctx->streams[videoStream]->codec->width,
                       ifmt_ctx->streams[videoStream]->codec->height);

    // initialize SWS context for software scaling
    sws_ctx = sws_getContext(
                 ifmt_ctx->streams[videoStream]->codec->width,
                 ifmt_ctx->streams[videoStream]->codec->height,
                 ifmt_ctx->streams[videoStream]->codec->pix_fmt,
                 ifmt_ctx->streams[videoStream]->codec->width,
                 ifmt_ctx->streams[videoStream]->codec->height,
                 AV_PIX_FMT_BGR24,
                 SWS_BICUBIC,
                 NULL,
                 NULL,
                 NULL
                 );
// Loop through packets
    while (1) {
        AVStream *in_stream, *out_stream;
        ret = av_read_frame(ifmt_ctx, &pkt);
        if(pkt.stream_index==videoStream)
         // Decode video frame
          avcodec_decode_video2(pCodecCtx, pFrame, &frameFinished, &pkt);
          if(frameFinished) {
                    sws_scale(sws_ctx, (uint8_t const * const *)pFrame->data,
                    pFrame->linesize, 0, pCodecCtx->height,
                    pFrameRGB->data, pFrameRGB->linesize);
                    cv::Mat img= mal_frame->process(
                              pFrameRGB,pFrame->width,pFrame->height);
/* My problem is Here ------------*/
        
        avpicture_fill((AVPicture*)pFrameRGB, 
                         img.data, 
                         PIX_FMT_BGR24, 
                         outStream->codec->width, 
                         outStream->codec->height);
        
        pFrameRGB->width =  ifmt_ctx->streams[videoStream]->codec->width;
        pFrameRGB->height = ifmt_ctx->streams[videoStream]->codec->height;
        
                avcodec_encode_video2(ifmt_ctx->streams[videoStream]->codec , 
                                                         &pkt , pFrameRGB , &gotPacket);
/*
I get this error
[swscaler @ 0x14b58a0] bad src image pointers
[swscaler @ 0x14b58a0] bad src image pointers
*/

/* My Problem Ends here ---------- */
                   
        }
        if (ret < 0)
            break;
        in_stream  = ifmt_ctx->streams[pkt.stream_index];
        out_stream = ofmt_ctx->streams[pkt.stream_index];

        //log_packet(ifmt_ctx, &pkt, "in");
        /* copy packet */
        pkt.pts = av_rescale_q_rnd(pkt.pts, in_stream->time_base, out_stream->time_base,
                                   AV_ROUND_NEAR_INF);

        pkt.dts = av_rescale_q_rnd(pkt.dts, in_stream->time_base, out_stream->time_base, AV_ROUND_NEAR_INF);
        pkt.duration = av_rescale_q(pkt.duration, in_stream->time_base, out_stream->time_base);
        pkt.pos = -1;
        log_packet(ofmt_ctx, &pkt, "out");
        ret = av_interleaved_write_frame(ofmt_ctx, &pkt);
        if (ret < 0) {
            fprintf(stderr, "Error muxing packet\n");
            break;
        }
        av_free_packet(&pkt);
    }
    av_write_trailer(ofmt_ctx);
end:
    avformat_close_input(&ifmt_ctx);
    /* close output */
    if (ofmt_ctx && !(ofmt->flags & AVFMT_NOFILE))
        avio_closep(&ofmt_ctx->pb);
    avformat_free_context(ofmt_ctx);
    if (ret < 0 && ret != AVERROR_EOF) {
//        fprintf(stderr, "Error occurred: %s\n", av_err2str(ret));
        return 1;
    }
    return 0;
}


______________________________________________________

Davood Falahati,


_______________________________________________
Libav-user mailing list
[hidden email]
http://ffmpeg.org/mailman/listinfo/libav-user