int init(const frame_type_ptr& frame) { ARLOG_DEBUG( "Initializing" ); if ( !avfilter_register_all_called ) { avfilter_register_all(); avfilter_register_all_called = true; } is_initialized_ = true; char args[512]; int ret = 0; AVFilter *buffersrc = avfilter_get_by_name("buffer"); AVFilter *buffersink = avfilter_get_by_name("buffersink"); AVFilterInOut *outputs = avfilter_inout_alloc( ); AVFilterInOut *inputs = avfilter_inout_alloc( ); boost::shared_ptr< AVFilterInOut * > i_ptr( &inputs, &destroy_avfilterinout ); boost::shared_ptr< AVFilterInOut * > o_ptr( &outputs, &destroy_avfilterinout ); enum AVPixelFormat pix_fmts[] = { AV_PIX_FMT_NONE }; filter_graph_ = avfilter_graph_alloc(); if (!outputs || !inputs || !filter_graph_) { ret = AVERROR(ENOMEM); return ret; } avfilter_graph_set_auto_convert( filter_graph_, AVFILTER_AUTO_CONVERT_ALL ); /* buffer video source: the decoded frames from the decoder will be inserted here. */ snprintf(args, sizeof(args), "video_size=%dx%d:pix_fmt=%d:time_base=%d/%d:pixel_aspect=%d/%d:frame_rate=%d/%d", frame->width(), frame->height(), image::ML_to_AV(frame->ml_pixel_format()), frame->get_fps_den(), frame->get_fps_num(), frame->get_sar_num(), frame->get_sar_den(), frame->get_fps_num(), frame->get_fps_den()); ret = avfilter_graph_create_filter(&buffersrc_ctx_, buffersrc, "in", args, NULL, filter_graph_); if (ret < 0) { ARLOG_ERR( "Cannot create buffer source with %s" )( args ); return ret; } /* buffer video sink: to terminate the filter chain. */ ret = avfilter_graph_create_filter(&buffersink_ctx_, buffersink, "out", NULL, NULL, filter_graph_); if (ret < 0) { ARLOG_ERR( "Cannot create buffer sink" ); return ret; } ret = av_opt_set_int_list(buffersink_ctx_, "pix_fmts", pix_fmts, AV_PIX_FMT_NONE, AV_OPT_SEARCH_CHILDREN); if (ret < 0) { ARLOG_ERR( "Cannot set output pixel format" ); return ret; } /* * Set the endpoints for the filter graph. The filter_graph will * be linked to the graph described by filters_descr. */ /* * The buffer source output must be connected to the input pad of * the first filter described by filters_descr; since the first * filter input label is not specified, it is set to "in" by * default. */ outputs->name = av_strdup("in"); outputs->filter_ctx = buffersrc_ctx_; outputs->pad_idx = 0; outputs->next = NULL; /* * The buffer sink input must be connected to the output pad of * the last filter described by filters_descr; since the last * filter output label is not specified, it is set to "out" by * default. */ inputs->name = av_strdup("out"); inputs->filter_ctx = buffersink_ctx_; inputs->pad_idx = 0; inputs->next = NULL; // Generate the full graph std::string filters_descr = generate_graph_string( ); ARLOG_DEBUG3( "Parsing '%s'" )( filters_descr ); if ((ret = avfilter_graph_parse_ptr(filter_graph_, filters_descr.c_str(), &inputs, &outputs, NULL)) < 0) return ret; if ((ret = avfilter_graph_config(filter_graph_, NULL)) < 0) return ret; fps_out_ = av_buffersink_get_frame_rate( buffersink_ctx_ ); frame->get_fps( fps_in_.num, fps_in_.den ); ARLOG_DEBUG3( "Input frame rate: %d:%d, Output frame rate: %d:%d" )( fps_in_.num )( fps_in_.den )( fps_out_.num )( fps_out_.den ); return ret; } ml::image_type_ptr flush_filter_graph( ) { ARLOG_DEBUG3( "Time to flush. (%1%, %2%)" )(get_position())(eof_position_); int limit = 5; ml::image_type_ptr image; while( !image ) { int ret = av_buffersrc_add_frame_flags(buffersrc_ctx_, NULL, AV_BUFFERSRC_FLAG_KEEP_REF); ARENFORCE_MSG( ret >= 0, "Error (%1%) while flushing the filtergraph")(ret); image = get_image_from_filter_graph(); ARENFORCE_MSG( image || -- limit, "Too many iterations - abandoning avfilter flush" ); } expected_position_ = -1; return image; } image_type_ptr get_image_from_filter_graph( ) { boost::shared_ptr< AVFrame > av_frame_out( av_frame_alloc( ), &destroy_avframe ); int ret = av_buffersink_get_frame(buffersink_ctx_, av_frame_out.get( )); std::cerr << "ret: " << ret << std::endl; if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) return image_type_ptr(); ARENFORCE( ret >= 0 ); return convert_to_oil(rescale_context_, av_frame_out.get( ), (AVPixelFormat)av_frame_out->format, av_frame_out->width, av_frame_out->height ); } void do_filter( frame_type_ptr& frame ) { if ( !is_initialized_ || frame->get_position() != expected_position_ ) { reset_context(); init( frame ); } ARENFORCE_MSG( is_initialized_, "filter graph is not initialized" ); if ( frame && frame->has_image( ) ) { std::cerr << "received: " << frame->get_position( ) << std::endl; AVFrame* av_frame = av_frame_alloc(); const image_type_ptr& image = frame->get_image(); const int num_planes = image->plane_count(); for( int p = 0; p < num_planes; ++p ) { av_frame->data[ p ] = static_cast< uint8_t * >( image->ptr( p ) ); av_frame->linesize[ p ] = image->pitch( p ); } av_frame->width = image->width(); av_frame->height= image->height(); av_frame->format = image::ML_to_AV( image->ml_pixel_format() ); AVRational sar = {image->get_sar_num(), image->get_sar_den()}; av_frame->sample_aspect_ratio = sar; av_frame->pts = frame->get_position(); av_frame->interlaced_frame = image->field_order() != image::progressive; av_frame->top_field_first = image->field_order() == image::top_field_first; int ret = av_buffersrc_add_frame_flags(buffersrc_ctx_, av_frame, AV_BUFFERSRC_FLAG_KEEP_REF); ARENFORCE_MSG( ret >= 0, "Error (%1%) while feeding the filtergraph with %2%")(ret)(av_frame); av_frame_free( &av_frame ); } expected_position_ = frame->get_position() + 1; } ml::fetch_status do_fetch( frame_type_ptr &frame ) { // Immediately handle the disabled case if( !prop_enable_.value< int >( ) ) { return fetch_from_slot( frame ); } // Immediately handle a re-request for the last frame if ( last_frame_ && get_position() == last_frame_->get_position() ) { frame = last_frame_->shallow(); return fetch_ok; } // The initialisation here creates an internal subgraph of: // // <slot 0> filter:callback callback=<do_filter> filter:frame_rate // // Where the frame_rate filter is configured according to the derived framerate // of the requested avfilter graph. // // Subsequent frames are fetched from the frame_rate filter, providing a queue of // output frames, while the original input frames are delivered to the callback for the // avfilter graph. When the avfilter graph delivers an output image, this is set on the // frame at the front of the output queue and this frame is returned to the caller. if ( !is_initialized_ ) { frame_type_ptr init_frame; AR_TRY_FETCH( fetch_slot( 0 )->fetch( init_frame, 0 ) ); int ret = init( init_frame ); ARENFORCE_MSG( ret >= 0, "Failed to initialize avfilter")(ret); callback_filter_.reset(new filter_callback(boost::bind(&filter_avvideo::do_filter, this, _1))); callback_filter_->connect( fetch_slot( 0 ) ); frame_rate_filter_ = ml::create_filter( L"frame_rate" ); frame_rate_filter_->connect( callback_filter_ ); frame_rate_filter_->properties( ).get_property_with_string( "fps_num" ) = fps_out_.num; frame_rate_filter_->properties( ).get_property_with_string( "fps_den" ) = fps_out_.den; } // If the requested position doesn't follow sequentially from the previous frame, we will clear // the output queue here. Note that the callback handles the state of the avfilter graph and we // don't touch it here. if ( queue_.size( ) && queue_.front( )->get_position( ) != get_position( ) ) queue_.clear( ); // Keep requesting frames from the frame rate filter until avfilter delivers an image. do { // Note that we may have already cached the frame for the current get_position, so we need to // restart from where we left off last time or from the current get_position as applicable int position = queue_.size( ) ? queue_.back( )->get_position( ) + 1 : get_position( ); // Obtain the frame indicated by the position ml::frame_type_ptr temp; ml::fetch_status status = frame_rate_filter_->fetch( temp, position ); // Return error on error, queue frame on ok, but only return eof if we have no frames pending if ( status == ml::fetch_error ) return status; else if ( status == ml::fetch_ok ) queue_.push_back( temp ); else if ( status == ml::fetch_eof && queue_.size( ) == 0 ) return ml::fetch_eof; // We must have pending frames at this point ARENFORCE_MSG( queue_.size( ), "Empty queue with non-eof condition" ); // Obtain the current avfilter image - we must only flush the filter graph if we are at eof image_type_ptr filtered_image = get_image_from_filter_graph(); if ( status == ml::fetch_eof && !filtered_image ) filtered_image = flush_filter_graph( ); // Set image on the front of the queue queue_.front( )->set_image( filtered_image ); } while( !queue_.front( )->get_image( ) ); // The image for the front of the queue should be here now and the front of the queue should // be the current get_position ARENFORCE_MSG( queue_.size( ), "Unexpected empty queue" ); frame = queue_.front( ); ARENFORCE_MSG( frame->get_position( ) == get_position( ), "Expected %d but got %d instead" )( get_position( ) )( frame->get_position( ) ); queue_.pop_front( ); last_frame_ = frame; return ml::fetch_ok; }