From 49a68184248b5613b46753298036767e2408d7f1 Mon Sep 17 00:00:00 2001 From: Diego Nieto Date: Sun, 15 Jun 2025 03:07:13 +0200 Subject: [PATCH 01/46] videostats: add video encoder stats elements --- Cargo.lock | 172 ++++++--- Cargo.toml | 2 + video/stats/Cargo.toml | 47 +++ video/stats/README.md | 16 + video/stats/build.rs | 4 + video/stats/examples/video-encoder-stats.rs | 34 ++ video/stats/src/comparemixer/imp.rs | 364 ++++++++++++++++++++ video/stats/src/comparemixer/mod.rs | 26 ++ video/stats/src/encoderstats/imp.rs | 293 ++++++++++++++++ video/stats/src/encoderstats/mod.rs | 26 ++ video/stats/src/lib.rs | 33 ++ video/stats/src/videoencoderstats.rs | 130 +++++++ video/stats/src/videoencoderstatsmeta.rs | 186 ++++++++++ 13 files changed, 1279 insertions(+), 54 deletions(-) create mode 100644 video/stats/Cargo.toml create mode 100644 video/stats/README.md create mode 100644 video/stats/build.rs create mode 100644 video/stats/examples/video-encoder-stats.rs create mode 100644 video/stats/src/comparemixer/imp.rs create mode 100644 video/stats/src/comparemixer/mod.rs create mode 100644 video/stats/src/encoderstats/imp.rs create mode 100644 video/stats/src/encoderstats/mod.rs create mode 100644 video/stats/src/lib.rs create mode 100644 video/stats/src/videoencoderstats.rs create mode 100644 video/stats/src/videoencoderstatsmeta.rs diff --git a/Cargo.lock b/Cargo.lock index 09870ced6..7aaf980ec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2789,7 +2789,7 @@ dependencies = [ "atomic_refcell", "byte-slice-cast", "ebur128", - "gst-plugin-version-helper", + "gst-plugin-version-helper 0.8.1", "gstreamer", "gstreamer-app", "gstreamer-audio", @@ -2831,7 +2831,7 @@ dependencies = [ "env_logger", "futures", "gio", - "gst-plugin-version-helper", + "gst-plugin-version-helper 0.8.1", "gstreamer", "gstreamer-audio", "gstreamer-base", @@ -2856,7 +2856,7 @@ version = "0.15.0-alpha.1" dependencies = [ "cdg", "cdg_renderer", - "gst-plugin-version-helper", + "gst-plugin-version-helper 0.8.1", "gstreamer", "gstreamer-app", "gstreamer-base", @@ -2871,7 +2871,7 @@ dependencies = [ "atomic_refcell", "byte-slice-cast", "claxon", - "gst-plugin-version-helper", + "gst-plugin-version-helper 0.8.1", "gstreamer", "gstreamer-audio", "gstreamer-check", @@ -2892,7 +2892,7 @@ dependencies = [ "chrono", "clap", "either", - "gst-plugin-version-helper", + "gst-plugin-version-helper 0.8.1", "gstreamer", "gstreamer-base", "gstreamer-check", @@ -2915,7 +2915,7 @@ version = "0.15.0-alpha.1" dependencies = [ "byte-slice-cast", "csound", - "gst-plugin-version-helper", + "gst-plugin-version-helper 0.8.1", "gstreamer", "gstreamer-audio", "gstreamer-base", @@ -2927,7 +2927,7 @@ name = "gst-plugin-dav1d" version = "0.15.0-alpha.1" dependencies = [ "dav1d", - "gst-plugin-version-helper", + "gst-plugin-version-helper 0.8.1", "gstreamer", "gstreamer-base", "gstreamer-video", @@ -2959,7 +2959,7 @@ version = "0.15.0-alpha.1" dependencies = [ "gio", "gst-plugin-gtk4", - "gst-plugin-version-helper", + "gst-plugin-version-helper 0.8.1", "gstreamer", "gstreamer-app", "gstreamer-audio", @@ -2977,7 +2977,7 @@ version = "0.15.0-alpha.1" dependencies = [ "byte-slice-cast", "ffv1", - "gst-plugin-version-helper", + "gst-plugin-version-helper 0.8.1", "gstreamer", "gstreamer-check", "gstreamer-video", @@ -2987,7 +2987,7 @@ dependencies = [ name = "gst-plugin-file" version = "0.15.0-alpha.1" dependencies = [ - "gst-plugin-version-helper", + "gst-plugin-version-helper 0.8.1", "gstreamer", "gstreamer-base", "url", @@ -2999,7 +2999,7 @@ version = "0.15.0-alpha.1" dependencies = [ "byteorder", "flavors", - "gst-plugin-version-helper", + "gst-plugin-version-helper 0.8.1", "gstreamer", "gstreamer-audio", "gstreamer-base", @@ -3016,7 +3016,7 @@ dependencies = [ "bitstream-io", "chrono", "dash-mpd", - "gst-plugin-version-helper", + "gst-plugin-version-helper 0.8.1", "gstreamer", "gstreamer-app", "gstreamer-audio", @@ -3036,7 +3036,7 @@ version = "0.15.0-alpha.1" dependencies = [ "atomic_refcell", "gif", - "gst-plugin-version-helper", + "gst-plugin-version-helper 0.8.1", "gstreamer", "gstreamer-check", "gstreamer-video", @@ -3047,7 +3047,7 @@ name = "gst-plugin-gopbuffer" version = "0.15.0-alpha.1" dependencies = [ "anyhow", - "gst-plugin-version-helper", + "gst-plugin-version-helper 0.8.1", "gstreamer", "gstreamer-app", "gstreamer-check", @@ -3062,7 +3062,7 @@ dependencies = [ "gdk4-wayland", "gdk4-win32", "gdk4-x11", - "gst-plugin-version-helper", + "gst-plugin-version-helper 0.8.1", "gstreamer", "gstreamer-allocators", "gstreamer-base", @@ -3088,7 +3088,7 @@ dependencies = [ "gio", "gst-plugin-fmp4", "gst-plugin-hlssink3", - "gst-plugin-version-helper", + "gst-plugin-version-helper 0.8.1", "gstreamer", "gstreamer-app", "gstreamer-audio", @@ -3107,8 +3107,7 @@ dependencies = [ "anyhow", "chrono", "gio", - "gst-plugin-fmp4", - "gst-plugin-version-helper", + "gst-plugin-version-helper 0.8.1", "gstreamer", "gstreamer-app", "gstreamer-audio", @@ -3124,7 +3123,7 @@ name = "gst-plugin-hsv" version = "0.15.0-alpha.1" dependencies = [ "byte-slice-cast", - "gst-plugin-version-helper", + "gst-plugin-version-helper 0.8.1", "gstreamer", "gstreamer-audio", "gstreamer-base", @@ -3139,7 +3138,7 @@ version = "0.15.0-alpha.1" dependencies = [ "anyhow", "futures", - "gst-plugin-version-helper", + "gst-plugin-version-helper 0.8.1", "gstreamer", "gstreamer-app", "gstreamer-check", @@ -3154,7 +3153,7 @@ dependencies = [ name = "gst-plugin-json" version = "0.15.0-alpha.1" dependencies = [ - "gst-plugin-version-helper", + "gst-plugin-version-helper 0.8.1", "gstreamer", "gstreamer-check", "serde", @@ -3167,7 +3166,7 @@ version = "0.15.0-alpha.1" dependencies = [ "atomic_refcell", "byte-slice-cast", - "gst-plugin-version-helper", + "gst-plugin-version-helper 0.8.1", "gstreamer", "gstreamer-audio", "gstreamer-check", @@ -3180,7 +3179,7 @@ version = "0.15.0-alpha.1" dependencies = [ "gio", "gst-plugin-gtk4", - "gst-plugin-version-helper", + "gst-plugin-version-helper 0.8.1", "gstreamer", "gstreamer-audio", "gstreamer-check", @@ -3195,7 +3194,7 @@ version = "0.15.0-alpha.1" dependencies = [ "anyhow", "bitstream-io", - "gst-plugin-version-helper", + "gst-plugin-version-helper 0.8.1", "gstreamer", "gstreamer-audio", "gstreamer-base", @@ -3214,7 +3213,7 @@ version = "0.15.0-alpha.1" dependencies = [ "anyhow", "bitstream-io", - "gst-plugin-version-helper", + "gst-plugin-version-helper 0.8.1", "gstreamer", "smallvec", ] @@ -3228,7 +3227,7 @@ dependencies = [ "byteorder", "data-encoding", "glib", - "gst-plugin-version-helper", + "gst-plugin-version-helper 0.8.1", "gstreamer", "gstreamer-audio", "gstreamer-base", @@ -3245,7 +3244,7 @@ version = "0.15.0-alpha.1" dependencies = [ "cairo-rs", "chrono", - "gst-plugin-version-helper", + "gst-plugin-version-helper 0.8.1", "gstreamer", "gstreamer-base", "gstreamer-rtp", @@ -3262,7 +3261,7 @@ version = "0.15.0-alpha.1" dependencies = [ "atomic_refcell", "glib", - "gst-plugin-version-helper", + "gst-plugin-version-helper 0.8.1", "gstreamer", "gstreamer-video", ] @@ -3271,7 +3270,7 @@ dependencies = [ name = "gst-plugin-png" version = "0.15.0-alpha.1" dependencies = [ - "gst-plugin-version-helper", + "gst-plugin-version-helper 0.8.1", "gstreamer", "gstreamer-check", "gstreamer-video", @@ -3290,7 +3289,7 @@ dependencies = [ "env_logger", "futures", "glib", - "gst-plugin-version-helper", + "gst-plugin-version-helper 0.8.1", "gstreamer", "gstreamer-base", "gstreamer-check", @@ -3312,7 +3311,7 @@ dependencies = [ name = "gst-plugin-raptorq" version = "0.15.0-alpha.1" dependencies = [ - "gst-plugin-version-helper", + "gst-plugin-version-helper 0.8.1", "gstreamer", "gstreamer-base", "gstreamer-check", @@ -3326,7 +3325,7 @@ name = "gst-plugin-rav1e" version = "0.15.0-alpha.1" dependencies = [ "atomic_refcell", - "gst-plugin-version-helper", + "gst-plugin-version-helper 0.8.1", "gstreamer", "gstreamer-check", "gstreamer-video", @@ -3337,19 +3336,34 @@ dependencies = [ name = "gst-plugin-regex" version = "0.15.0-alpha.1" dependencies = [ - "gst-plugin-version-helper", + "gst-plugin-version-helper 0.8.1", "gstreamer", "gstreamer-check", "regex", ] +[[package]] +name = "gst-plugin-relationmeta" +version = "0.14.0-alpha.1" +dependencies = [ + "chrono", + "glib", + "gst-plugin-version-helper 0.8.1", + "gstreamer", + "gstreamer-analytics", + "gstreamer-base", + "gstreamer-rtp", + "gstreamer-video", + "xmltree", +] + [[package]] name = "gst-plugin-reqwest" version = "0.15.0-alpha.1" dependencies = [ "bytes", "futures", - "gst-plugin-version-helper", + "gst-plugin-version-helper 0.8.1", "gstreamer", "gstreamer-base", "headers", @@ -3374,8 +3388,7 @@ dependencies = [ "chrono", "futures", "gio", - "glib", - "gst-plugin-version-helper", + "gst-plugin-version-helper 0.8.1", "gstreamer", "gstreamer-app", "gstreamer-audio", @@ -3406,7 +3419,7 @@ dependencies = [ "atomic_refcell", "data-encoding", "futures", - "gst-plugin-version-helper", + "gst-plugin-version-helper 0.8.1", "gstreamer", "gstreamer-app", "gstreamer-net", @@ -3438,7 +3451,7 @@ name = "gst-plugin-sodium" version = "0.15.0-alpha.1" dependencies = [ "clap", - "gst-plugin-version-helper", + "gst-plugin-version-helper 0.8.1", "gstreamer", "gstreamer-app", "gstreamer-base", @@ -3459,7 +3472,7 @@ dependencies = [ "async-tungstenite", "atomic_refcell", "futures", - "gst-plugin-version-helper", + "gst-plugin-version-helper 0.8.1", "gstreamer", "gstreamer-audio", "gstreamer-base", @@ -3477,7 +3490,7 @@ version = "0.15.0-alpha.1" dependencies = [ "anyhow", "futures", - "gst-plugin-version-helper", + "gst-plugin-version-helper 0.8.1", "gstreamer", "gstreamer-base", "librespot-core", @@ -3491,7 +3504,7 @@ dependencies = [ name = "gst-plugin-streamgrouper" version = "0.15.0-alpha.1" dependencies = [ - "gst-plugin-version-helper", + "gst-plugin-version-helper 0.8.1", "gstreamer", "gstreamer-check", ] @@ -3514,7 +3527,7 @@ dependencies = [ name = "gst-plugin-textahead" version = "0.15.0-alpha.1" dependencies = [ - "gst-plugin-version-helper", + "gst-plugin-version-helper 0.8.1", "gstreamer", ] @@ -3522,7 +3535,7 @@ dependencies = [ name = "gst-plugin-textwrap" version = "0.15.0-alpha.1" dependencies = [ - "gst-plugin-version-helper", + "gst-plugin-version-helper 0.8.1", "gstreamer", "gstreamer-check", "hyphenation", @@ -3543,8 +3556,7 @@ dependencies = [ "futures", "getifaddrs", "gio", - "gst-plugin-rtp", - "gst-plugin-version-helper", + "gst-plugin-version-helper 0.8.1", "gstreamer", "gstreamer-app", "gstreamer-audio", @@ -3571,7 +3583,7 @@ dependencies = [ "either", "gio", "gst-plugin-gtk4", - "gst-plugin-version-helper", + "gst-plugin-version-helper 0.8.1", "gstreamer", "gstreamer-audio", "gstreamer-check", @@ -3590,7 +3602,7 @@ dependencies = [ "chrono", "etherparse", "futures", - "gst-plugin-version-helper", + "gst-plugin-version-helper 0.8.1", "gstreamer", "pcap-file", "regex", @@ -3608,7 +3620,7 @@ name = "gst-plugin-tutorial" version = "0.15.0-alpha.1" dependencies = [ "byte-slice-cast", - "gst-plugin-version-helper", + "gst-plugin-version-helper 0.8.1", "gstreamer", "gstreamer-audio", "gstreamer-base", @@ -3622,7 +3634,7 @@ version = "0.15.0-alpha.1" dependencies = [ "anyhow", "clap", - "gst-plugin-version-helper", + "gst-plugin-version-helper 0.8.1", "gstreamer", "gstreamer-app", "more-asserts", @@ -3640,6 +3652,15 @@ dependencies = [ "toml_edit 0.23.5", ] +[[package]] +name = "gst-plugin-version-helper" +version = "0.8.1" +source = "git+https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs#c580400d5c96439b9451ccb669449859e36ebdc2" +dependencies = [ + "chrono", + "toml_edit", +] + [[package]] name = "gst-plugin-videofx" version = "0.15.0-alpha.1" @@ -3649,7 +3670,7 @@ dependencies = [ "color-name", "color-thief", "dssim-core", - "gst-plugin-version-helper", + "gst-plugin-version-helper 0.8.1", "gstreamer", "gstreamer-base", "gstreamer-check", @@ -3659,11 +3680,26 @@ dependencies = [ "rgb", ] +[[package]] +name = "gst-plugin-videostats" +version = "0.1.0" +dependencies = [ + "anyhow", + "atomic_refcell", + "glib", + "gst-plugin-version-helper 0.8.1 (git+https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs)", + "gstreamer", + "gstreamer-video", + "hound", + "human_bytes", + "procfs", +] + [[package]] name = "gst-plugin-vvdec" version = "0.15.0-alpha.1" dependencies = [ - "gst-plugin-version-helper", + "gst-plugin-version-helper 0.8.1", "gstreamer", "gstreamer-audio", "gstreamer-check", @@ -3675,7 +3711,7 @@ dependencies = [ name = "gst-plugin-webp" version = "0.15.0-alpha.1" dependencies = [ - "gst-plugin-version-helper", + "gst-plugin-version-helper 0.8.1", "gstreamer", "gstreamer-check", "gstreamer-video", @@ -3708,7 +3744,7 @@ dependencies = [ "fastrand", "futures", "gst-plugin-rtp", - "gst-plugin-version-helper", + "gst-plugin-version-helper 0.8.1", "gst-plugin-webrtc-signalling", "gst-plugin-webrtc-signalling-protocol", "gstreamer", @@ -3789,7 +3825,7 @@ dependencies = [ "async-recursion", "bytes", "futures", - "gst-plugin-version-helper", + "gst-plugin-version-helper 0.8.1", "gstreamer", "gstreamer-sdp", "gstreamer-webrtc", @@ -4431,6 +4467,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "hound" +version = "3.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62adaabb884c94955b19907d60019f4e145d091c75345379e70d1ee696f7854f" + [[package]] name = "hrtf" version = "0.8.1" @@ -6435,6 +6477,28 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "procfs" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc5b72d8145275d844d4b5f6d4e1eef00c8cd889edb6035c21675d1bb1f45c9f" +dependencies = [ + "bitflags 2.9.0", + "hex", + "procfs-core", + "rustix", +] + +[[package]] +name = "procfs-core" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "239df02d8349b06fc07398a3a1697b06418223b1c7725085e801e7c0fc6a12ec" +dependencies = [ + "bitflags 2.9.0", + "hex", +] + [[package]] name = "profiling" version = "1.0.17" diff --git a/Cargo.toml b/Cargo.toml index 5a1b3ad68..c3a2ab6b4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -69,6 +69,7 @@ members = [ "video/videofx", "video/vvdec", "video/webp", + "video/stats", ] # Only plugins without external dependencies @@ -124,6 +125,7 @@ default-members = [ "video/hsv", "video/png", "video/rav1e", + "video/stats", ] [profile.release] diff --git a/video/stats/Cargo.toml b/video/stats/Cargo.toml new file mode 100644 index 000000000..244e706f5 --- /dev/null +++ b/video/stats/Cargo.toml @@ -0,0 +1,47 @@ +[package] +name = "gst-plugin-videostats" +version = "0.1.0" +authors = ["Diego Nieto "] +repository = "https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs" +license = "MPL-2.0" +edition = "2021" +description = "videostats plugin" + +[dependencies] +hound = "3" +anyhow = "1" +glib.workspace = true +gst.workspace = true +gst-video.workspace = true +human_bytes = { version = "0.4", default-features = false } +atomic_refcell = "0.1" + +[lib] +name = "gstvideostats" +crate-type = ["cdylib", "rlib"] +path = "src/lib.rs" + +[build-dependencies] +gst-plugin-version-helper = { git = "https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs" } + +[features] +capi = [] +static = [] +doc = ["gst/v1_16"] + +[package.metadata.capi] +min_version = "0.9.21" + +[package.metadata.capi.header] +enabled = false + +[package.metadata.capi.library] +install_subdir = "gstreamer-1.0" +versioning = false +import_library = false + +[target.'cfg(target_os = "linux")'.dependencies] +procfs = { version = "0.17", default-features = false } + +[package.metadata.capi.pkg_config] +requires_private = "gstreamer-1.0, gstreamer-base-1.0, gobject-2.0, glib-2.0, gmodule-2.0" diff --git a/video/stats/README.md b/video/stats/README.md new file mode 100644 index 000000000..a1a9536d6 --- /dev/null +++ b/video/stats/README.md @@ -0,0 +1,16 @@ +# Video Encoder Stats + +- `video-encoder-stats`: + The element that collects statistics from a video encoder, and attach them onto the `GstBuffers` as metadata. It helps analyze encoding performance and quality metrics. + +- `video-compare-mixer`: + The element in charge of comparing and mixing multiple video streams. Useful for side-by-side quality comparisons or blending outputs from different encoders. + +- `videoencoderstatsmeta`: + Defines metadata structures and logic for handling video encoder statistics along the pipeline. It has been defined as `GstVideoEncoderStatsMetaAPI`. + + +**`video-encoder-stats`** example: +``` +cargo r --example video-encoder-stats +``` diff --git a/video/stats/build.rs b/video/stats/build.rs new file mode 100644 index 000000000..056534a8f --- /dev/null +++ b/video/stats/build.rs @@ -0,0 +1,4 @@ + +fn main() { + gst_plugin_version_helper::info() +} diff --git a/video/stats/examples/video-encoder-stats.rs b/video/stats/examples/video-encoder-stats.rs new file mode 100644 index 000000000..ddd461221 --- /dev/null +++ b/video/stats/examples/video-encoder-stats.rs @@ -0,0 +1,34 @@ +// Copyright (C) 2025, Fluendo S.A. +// Author: Diego Nieto +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +use gst::prelude::*; +use anyhow::Error; + +fn main() -> Result<(), Error> { + gst::init()?; + + let pipeline = gst::parse::launch("videotestsrc ! video/x-raw,width=640,height=480 ! videoconvert ! tee name=tee ! queue name=encq0 ! video-encoder-stats encoder=\"x264enc\" ! decodebin3 name=dec0 tee. ! queue name=encq1 ! video-encoder-stats encoder=\"x264enc bitrate=32\" ! decodebin3 name=dec1 video-compare-mixer name=mixer backend=CPU dec0. ! mixer. dec1. ! mixer. mixer. ! videoconvert ! autovideosink")?; + pipeline.set_state(gst::State::Playing)?; + + let bus = pipeline.bus().unwrap(); + while let Some(msg) = bus.timed_pop(gst::ClockTime::NONE) { + use gst::MessageView; + match msg.view() { + MessageView::Eos(..) => { + break; + } + MessageView::Error(..) => unreachable!(), + _ => (), + } + } + + pipeline.set_state(gst::State::Null)?; + + Ok(()) +} diff --git a/video/stats/src/comparemixer/imp.rs b/video/stats/src/comparemixer/imp.rs new file mode 100644 index 000000000..e6471efd7 --- /dev/null +++ b/video/stats/src/comparemixer/imp.rs @@ -0,0 +1,364 @@ +// Copyright (C) 2025, Fluendo S.A. +// Author: Diego Nieto +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +use gst::glib; +use gst::prelude::*; +use gst::subclass::prelude::*; + +use crate::videoencoderstatsmeta::VideoEncoderStatsMeta; + +use std::sync::{LazyLock, Mutex}; +use std::vec::Vec; + +static CAT: LazyLock = LazyLock::new(|| { + gst::DebugCategory::new( + "video-compare-mixer", + gst::DebugColorFlags::empty(), + Some("GstVideoCompareMixer"), + ) +}); + +#[derive(Default, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Clone, Copy, glib::Enum)] +#[enum_type(name = "GstVideoCompareMixerBackend")] +#[repr(u32)] +#[non_exhaustive] +pub enum Backend { + #[default] + #[enum_value(name = "OpenGL", nick = "OpenGL")] + GL, + #[enum_value(name = "VAAPI", nick = "VAAPI")] + #[cfg(target_os = "linux")] + VAAPI, + #[enum_value(name = "CPU", nick = "CPU")] + CPU, + #[enum_value(name = "D3D12", nick = "D3D12")] + #[cfg(target_os = "windows")] + D3D12, +} + +struct Settings { + backend: Backend, +} + +pub struct VideoCompareMixer { + srcpad: gst::GhostPad, + sinkpad0: gst::GhostPad, + sinkpad1: gst::GhostPad, + queue0: gst::Element, + queue1: gst::Element, + overlay0: gst::Element, + overlay1: gst::Element, + settings: Mutex, +} + +impl Default for Settings { + fn default() -> Self { + Self { + backend: Backend::default(), + } + } +} + +impl VideoCompareMixer { + fn get_pipeline_compositor(&self, backend: Backend) -> &str { + match backend { + Backend::GL => "glvideomixer", + #[cfg(target_os = "linux")] + Backend::VAAPI => "vacompositor", + Backend::CPU => "compositor", + #[cfg(target_os = "windows")] + Backend::D3D12 => "d3d12compositor", + } + } + + fn add_overlay_probe(&self, overlay: &gst::Element) { + let overlay_src_pad = overlay.static_pad("video_sink").unwrap(); + let overlay_clone = overlay.clone(); + overlay_src_pad.add_probe(gst::PadProbeType::BUFFER, move |_: &gst::Pad, probe_info| { + let Some(buffer) = probe_info.buffer_mut() else { + return gst::PadProbeReturn::Ok; + }; + + if let Some(statsmeta) = buffer.meta::() { + let stats = statsmeta.stats(); + let stats_string = format!("{stats}"); + overlay_clone.set_property("text", stats_string); + } + + gst::PadProbeReturn::Ok + }); + } + + fn link_elements( + &self, + compositor: &gst::Element, + ) -> Result<(), gst::ErrorMessage> { + self.overlay0.set_property_from_str("line-alignment", "left"); + self.overlay0.set_property_from_str("halignment", "left"); + self.overlay1.set_property_from_str("line-alignment", "right"); + self.overlay1.set_property_from_str("halignment", "right"); + + let compositor_pad0 = compositor + .request_pad_simple("sink_0") + .expect("Failed to request pad sink_0"); + let compositor_pad1 = compositor + .request_pad_simple("sink_1") + .expect("Failed to request pad sink_1"); + + self.obj() + .add(compositor) + .expect("Failed to add compositor element"); + self.obj() + .add(&self.queue0) + .expect("Failed to add queue0 element"); + self.obj() + .add(&self.queue1) + .expect("Failed to add queue1 element"); + self.obj() + .add(&self.overlay0) + .expect("Failed to add overlay0 element"); + self.obj() + .add(&self.overlay1) + .expect("Failed to add overlay1 element"); + + self.sinkpad0 + .set_target(Some(&self.queue0.static_pad("sink").unwrap())) + .expect("Failed to link sinkpad0 to queue0"); + self.sinkpad1 + .set_target(Some(&self.queue1.static_pad("sink").unwrap())) + .expect("Failed to link sinkpad1 to queue1"); + self.queue0 + .static_pad( "src") + .unwrap() + .link(&self.overlay0.static_pad("video_sink").unwrap()) + .expect("Failed to link queue0 to compositor"); + self.queue1 + .static_pad("src") + .unwrap() + .link( &self.overlay1.static_pad( "video_sink").unwrap()) + .expect("Failed to link queue1 to compositor"); + self.overlay0 + .static_pad("src") + .unwrap() + .link(&compositor_pad0) + .expect("Failed to link overlay0 to compositor"); + self.overlay1 + .static_pad("src") + .unwrap() + .link(&compositor_pad1) + .expect("Failed to link overlay1 to compositor"); + self.srcpad + .set_target(Some(&compositor.static_pad("src").unwrap())) + .expect("Failed to link srcpad to compositor"); + Ok(()) + } + + fn sink_event(&self, pad: &gst::Pad, event: gst::Event) -> bool { + gst::log!(CAT, obj = pad, "Handling sink event {:?}", event); + + use gst::EventView::*; + match event.view() { + Caps(c) => { + let caps = c.caps(); + let s = caps.structure(0).unwrap(); + let compositor_sink1_pad = self.obj().by_name("compositor").unwrap().static_pad("sink_1").unwrap(); + compositor_sink1_pad.set_property("xpos", s.get::("width").unwrap()); + gst::info!(CAT, "Received caps {caps:?}"); + } + _ => { + gst::info!(CAT, "Other event"); + } + } + gst::Pad::event_default(pad, Some(&*self.obj()), event); + true + } +} + +#[glib::object_subclass] +impl ObjectSubclass for VideoCompareMixer { + const NAME: &'static str = "GstVideoCompareMixer"; + type Type = super::VideoCompareMixer; + type ParentType = gst::Bin; + + fn with_class(klass: &Self::Class) -> Self { + let templ = klass.pad_template("sink_0").unwrap(); + let sinkpad0 = gst::GhostPad::from_template(&templ); + + let templ = klass.pad_template("sink_1").unwrap(); + let sinkpad1 = gst::GhostPad::from_template(&templ); + + let templ = klass.pad_template("src").unwrap(); + let srcpad = gst::GhostPad::from_template(&templ); + + let queue0 = gst::ElementFactory::make("queue") + .build() + .expect("Failed to create queue0"); + queue0.set_property("name", "queue0"); + + let queue1 = gst::ElementFactory::make("queue") + .build() + .expect("Failed to create queue1"); + queue1.set_property("name", "queue1"); + + let overlay0 = gst::ElementFactory::make("textoverlay") + .build() + .expect("Failed to create overlay0"); + overlay0.set_property("name", "overlay0"); + + let overlay1 = gst::ElementFactory::make("textoverlay") + .build() + .expect("Failed to create overlay1"); + overlay1.set_property("name", "overlay1"); + + Self { + srcpad, + sinkpad0, + sinkpad1, + queue0, + queue1, + overlay0, + overlay1, + settings: Mutex::new(Settings::default()), + } + } +} + +impl ObjectImpl for VideoCompareMixer { + // TODO + // navigation-evets = default true + + fn properties() -> &'static [glib::ParamSpec] { + static PROPERTIES: LazyLock> = LazyLock::new(|| { + vec![ + glib::ParamSpecEnum::builder_with_default("backend", Backend::default()) + .nick("The backend to use for mixing the video") + .blurb("The backend to use for mixing the video") + .mutable_ready() + .build(), + ] + }); + + PROPERTIES.as_ref() + } + + fn set_property(&self, _id: usize, value: &glib::Value, pspec: &glib::ParamSpec) { + let mut settings = self.settings.lock().unwrap(); + match pspec.name() { + "backend" => { + settings.backend = value.get().expect("type checked upstream"); + + let compositor = + gst::ElementFactory::make(self.get_pipeline_compositor(settings.backend)) + .build() + .expect("Failed to create identity element"); + compositor.set_property("name", "compositor"); + + self.link_elements(&compositor) + .expect("Failed to link elements"); + + self.add_overlay_probe(&self.overlay0); + self.add_overlay_probe(&self.overlay1); + + unsafe + { + self.sinkpad0.set_event_full_function(|pad, parent, event| { + VideoCompareMixer::catch_panic_pad_function( + parent, + || false, + |video_compare_mixer| video_compare_mixer.sink_event(&pad.clone().upcast::(), event), + ); + Ok(gst::FlowSuccess::Ok) + }); + } + + gst::info!( + CAT, + imp = self, + "Set backend to {:?} and linked pads", + settings.backend + ); + } + _ => unimplemented!(), + } + } + + fn property(&self, _id: usize, pspec: &glib::ParamSpec) -> glib::Value { + let settings = self.settings.lock().unwrap(); + match pspec.name() { + "backend" => settings.backend.to_value(), + _ => unimplemented!(), + } + } + + fn constructed(&self) { + self.parent_constructed(); + + let obj = self.obj(); + obj.add_pad(&self.sinkpad0).unwrap(); + obj.add_pad(&self.sinkpad1).unwrap(); + obj.add_pad(&self.srcpad).unwrap(); + } +} + +impl GstObjectImpl for VideoCompareMixer {} + +impl ElementImpl for VideoCompareMixer { + fn metadata() -> Option<&'static gst::subclass::ElementMetadata> { + static ELEMENT_METADATA: LazyLock = LazyLock::new(|| { + gst::subclass::ElementMetadata::new( + "VideoCompareMixer", + "Video/Mixer/Filter", + "Video Compare Mixer Wrapper", + "Diego Nieto ", + ) + }); + + Some(&*ELEMENT_METADATA) + } + + fn pad_templates() -> &'static [gst::PadTemplate] { + static PAD_TEMPLATES: LazyLock> = LazyLock::new(|| { + let caps = gst_video::VideoCapsBuilder::new().build(); + + let video_src_pad_template = gst::PadTemplate::new( + "src", + gst::PadDirection::Src, + gst::PadPresence::Always, + &caps, + ) + .unwrap(); + + let video_sink_0_pad_template = gst::PadTemplate::new( + "sink_0", + gst::PadDirection::Sink, + gst::PadPresence::Always, + &caps, + ) + .unwrap(); + + let video_sink_1_pad_template = gst::PadTemplate::new( + "sink_1", + gst::PadDirection::Sink, + gst::PadPresence::Always, + &caps, + ) + .unwrap(); + + vec![ + video_src_pad_template, + video_sink_0_pad_template, + video_sink_1_pad_template, + ] + }); + + PAD_TEMPLATES.as_ref() + } +} + +impl BinImpl for VideoCompareMixer {} diff --git a/video/stats/src/comparemixer/mod.rs b/video/stats/src/comparemixer/mod.rs new file mode 100644 index 000000000..3b25d5b84 --- /dev/null +++ b/video/stats/src/comparemixer/mod.rs @@ -0,0 +1,26 @@ +// Copyright (C) 2025, Fluendo S.A. +// Author: Diego Nieto +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +use gst::glib; +use gst::prelude::*; + +mod imp; + +glib::wrapper! { + pub struct VideoCompareMixer(ObjectSubclass) @extends gst::Bin, gst::Element, gst::Object; +} + +pub fn register(plugin: &gst::Plugin) -> Result<(), glib::BoolError> { + gst::Element::register( + Some(plugin), + "video-compare-mixer", + gst::Rank::NONE, + VideoCompareMixer::static_type(), + ) +} diff --git a/video/stats/src/encoderstats/imp.rs b/video/stats/src/encoderstats/imp.rs new file mode 100644 index 000000000..6e41d1228 --- /dev/null +++ b/video/stats/src/encoderstats/imp.rs @@ -0,0 +1,293 @@ +// Copyright (C) 2025, Fluendo S.A. +// Author: Diego Nieto +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +use gst::glib; +use gst::prelude::*; +use gst::subclass::prelude::*; + +use crate::videoencoderstats::*; +use crate::videoencoderstatsmeta::VideoEncoderStatsMeta; + +use std::sync::{LazyLock, Mutex}; +use std::vec::Vec; +use std::sync::Arc; + +static CAT: LazyLock = LazyLock::new(|| { + gst::DebugCategory::new( + "video-encoder-stats", + gst::DebugColorFlags::empty(), + Some("GstVideoEncoderStats"), + ) +}); + +pub struct EncoderStats { + srcpad: gst::GhostPad, + sinkpad: gst::GhostPad, + identity: gst::Element, + stats: Arc>, +} + +impl EncoderStats { + fn add_identity_probe( + &self, + ) { + let identity = self.obj().by_name("identity").expect("expected identity"); + let identity_src_pad = identity.static_pad("src").unwrap(); + let encoder = self.obj().by_name("enc").expect("expected encoder"); + let encoder_factory = encoder.factory().expect("encoder should have a factory"); + let encoder_name = encoder_factory.name(); + + let stats = self.stats.clone(); + let obj_name = self.obj().name().to_string(); + identity_src_pad.add_probe(gst::PadProbeType::BUFFER, move |_pad, probe_info| { + let Some(buffer) = probe_info.buffer_mut() else { + return gst::PadProbeReturn::Ok; + }; + + let identity_stats = identity.property::("stats"); + let num_bytes = identity_stats.get::("num-bytes").unwrap(); + let num_buffers = identity_stats.get::("num-buffers").unwrap(); + + let mut stats = stats.lock().unwrap(); + let fps_n: i32; + if let Some(fps) = stats.framerate { + fps_n = fps.numer(); + } else { + return gst::PadProbeReturn::Ok; + } + + if num_buffers % (fps_n as u64) != 0 { + gst::log!(CAT, "Skipping probe for buffer {num_buffers} as it is not a multiple of framerate {fps_n}"); + return gst::PadProbeReturn::Ok; + } + + // FIXME: integrates queues internally to calculate the CPU usage + let thread_name = if obj_name.contains("0") { + "encq0:src" + } else { + "encq1:src" + }; + let (total_utime, total_stime) = get_cpu_usage(thread_name.to_string()); + + stats.threads_utime = total_utime; + stats.threads_stime = total_stime; + stats.num_bytes = num_bytes; + stats.num_buffers = num_buffers; + stats.name = encoder_name.to_string(); + + let buffer = buffer.make_mut(); + + // Add the VideoEncoderStatsMeta to the buffer + VideoEncoderStatsMeta::add( + buffer, + stats.clone(), + ); + + gst::PadProbeReturn::Ok + }); + } + + fn add_encoder_probes(&self) { + let encoder = self.obj().by_name("enc").expect("expected identity"); + let encoder_sink_pad = encoder.static_pad("sink").unwrap(); + let encoder_src_pad = encoder.static_pad("src").unwrap(); + + let stats = self.stats.clone(); + encoder_sink_pad.add_probe(gst::PadProbeType::BUFFER, move |_, probe_info| { + let Some(_) = probe_info.buffer() else { + return gst::PadProbeReturn::Ok; + }; + stats.lock().unwrap().buffer_in(); + gst::log!(CAT, "Buffer in encoder sink pad"); + gst::PadProbeReturn::Ok + }); + + let stats = self.stats.clone(); + encoder_src_pad.add_probe(gst::PadProbeType::BUFFER, move |_, probe_info| { + let Some(_) = probe_info.buffer() else { + return gst::PadProbeReturn::Ok; + }; + stats.lock().unwrap().buffer_out(); + gst::log!(CAT, "Buffer out encoder src pad"); + gst::PadProbeReturn::Ok + }); + } + + fn sink_event(&self, pad: &gst::Pad, event: gst::Event) -> bool { + gst::log!(CAT, obj = pad, "Handling sink event {:?}", event); + + use gst::EventView::*; + match event.view() { + Caps(event) => { + let caps = event.caps(); + let s = caps.structure(0).unwrap(); + let fps = s.get::("framerate").ok(); + self.stats.lock().unwrap().framerate = fps; + gst::info!(CAT, "Received caps {caps:?}"); + } + _ => { + gst::info!(CAT, "Other event"); + } + } + gst::Pad::event_default(pad, Some(&*self.obj()), event); + true + } +} + +#[glib::object_subclass] +impl ObjectSubclass for EncoderStats { + const NAME: &'static str = "GstEncoderStats"; + type Type = super::VideoEncoderStats; + type ParentType = gst::Bin; + + fn with_class(klass: &Self::Class) -> Self { + let templ = klass.pad_template("sink").unwrap(); + let sinkpad = gst::GhostPad::from_template(&templ); + + let templ = klass.pad_template("src").unwrap(); + let srcpad = gst::GhostPad::from_template(&templ); + + let identity = gst::ElementFactory::make("identity") + .build() + .expect("Failed to create identity element"); + identity.set_property("name", "identity"); + + Self { + srcpad, + sinkpad, + identity, + stats: Arc::new(Mutex::new(VideoEncoderStats::default())), + } + } +} + +impl ObjectImpl for EncoderStats { + fn properties() -> &'static [glib::ParamSpec] { + static PROPERTIES: LazyLock> = LazyLock::new(|| { + vec![ + glib::ParamSpecObject::builder::("encoder") + .nick("The encoder stats") + .blurb("The encoder name to use") + .construct_only() + .build(), + ] + }); + + PROPERTIES.as_ref() + } + + fn property(&self, _id: usize, pspec: &glib::ParamSpec) -> glib::Value { + match pspec.name() { + "encoder" => { + self.obj().by_name("enc").to_value() + } + _ => unimplemented!(), + } + } + + fn set_property(&self, _id: usize, value: &glib::Value, pspec: &glib::ParamSpec) { + match pspec.name() { + "encoder" => { + if let Ok(Some(enc_obj)) = value.get::>() { + let factory = enc_obj + .factory() + .expect("Element should have a factory"); + + if !factory.has_type(gst::ElementFactoryType::VIDEO_ENCODER) + { + gst::error!(CAT, "The element is not a video encoder"); + panic!("The element is not a video encoder"); + } + enc_obj.set_property("name", "enc"); + + self.obj().add(&self.identity).unwrap(); + self.srcpad + .set_target(Some(&self.identity.static_pad("src").unwrap())) + .unwrap(); + + self.obj().add(&enc_obj).expect("Failed to add encoder element"); + self.sinkpad + .set_target(Some(&enc_obj.static_pad( "sink").unwrap())) + .expect("Failed to link sink pad to encoder element"); + enc_obj.link(&self.obj().by_name("identity").expect("expected identity")) + .expect("Failed to link encoder to identity"); + + unsafe + { + self.sinkpad.set_event_full_function(|pad, parent, event| { + EncoderStats::catch_panic_pad_function( + parent, + || false, + |video_encoder_stats| video_encoder_stats.sink_event(&pad.clone().upcast::(), event), + ); + Ok(gst::FlowSuccess::Ok) + }); + } + + self.add_identity_probe(); + self.add_encoder_probes(); + } + } + _ => unimplemented!(), + } + } + + fn constructed(&self) { + self.parent_constructed(); + let obj = self.obj(); + + obj.add_pad(&self.sinkpad).unwrap(); + obj.add_pad(&self.srcpad).unwrap(); + } +} + +impl GstObjectImpl for EncoderStats {} + +impl ElementImpl for EncoderStats { + fn metadata() -> Option<&'static gst::subclass::ElementMetadata> { + static ELEMENT_METADATA: LazyLock = LazyLock::new(|| { + gst::subclass::ElementMetadata::new( + "EncoderStats", + "Video/Encoder/Filter", + "Video Encoder Stats Wrapper", + "Diego Nieto ", + ) + }); + + Some(&*ELEMENT_METADATA) + } + + fn pad_templates() -> &'static [gst::PadTemplate] { + static PAD_TEMPLATES: LazyLock> = LazyLock::new(|| { + let sink_caps = gst_video::VideoCapsBuilder::new() + .build(); + let src_caps = gst::Caps::new_any(); + let video_src_pad_template = gst::PadTemplate::new( + "src", + gst::PadDirection::Src, + gst::PadPresence::Always, + &src_caps, + ) + .unwrap(); + let video_sink_pad_template = gst::PadTemplate::new( + "sink", + gst::PadDirection::Sink, + gst::PadPresence::Always, + &sink_caps, + ) + .unwrap(); + + vec![video_src_pad_template, video_sink_pad_template] + }); + + PAD_TEMPLATES.as_ref() + } +} + +impl BinImpl for EncoderStats {} diff --git a/video/stats/src/encoderstats/mod.rs b/video/stats/src/encoderstats/mod.rs new file mode 100644 index 000000000..a25fd03dc --- /dev/null +++ b/video/stats/src/encoderstats/mod.rs @@ -0,0 +1,26 @@ +// Copyright (C) 2025, Fluendo S.A. +// Author: Diego Nieto +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +use gst::glib; +use gst::prelude::*; + +mod imp; + +glib::wrapper! { + pub struct VideoEncoderStats(ObjectSubclass) @extends gst::Bin, gst::Element, gst::Object; +} + +pub fn register(plugin: &gst::Plugin) -> Result<(), glib::BoolError> { + gst::Element::register( + Some(plugin), + "video-encoder-stats", + gst::Rank::NONE, + VideoEncoderStats::static_type(), + ) +} diff --git a/video/stats/src/lib.rs b/video/stats/src/lib.rs new file mode 100644 index 000000000..a16b6cd86 --- /dev/null +++ b/video/stats/src/lib.rs @@ -0,0 +1,33 @@ +// Copyright (C) 2025, Fluendo S.A. +// Author: Diego Nieto +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +use gst::glib; + +mod videoencoderstats; +mod videoencoderstatsmeta; +mod comparemixer; +mod encoderstats; + +fn plugin_init(plugin: &gst::Plugin) -> Result<(), glib::BoolError> { + comparemixer::register(plugin)?; + encoderstats::register(plugin)?; + Ok(()) +} + +gst::plugin_define!( + videostats, + env!("CARGO_PKG_DESCRIPTION"), + plugin_init, + concat!(env!("CARGO_PKG_VERSION"), "-", "commit-id"), + "MPL/X11", + env!("CARGO_PKG_NAME"), + env!("CARGO_PKG_NAME"), + env!("CARGO_PKG_REPOSITORY"), + env!("BUILD_REL_DATE") +); diff --git a/video/stats/src/videoencoderstats.rs b/video/stats/src/videoencoderstats.rs new file mode 100644 index 000000000..80778f9ad --- /dev/null +++ b/video/stats/src/videoencoderstats.rs @@ -0,0 +1,130 @@ +// Copyright (C) 2025, Fluendo S.A. +// Author: Diego Nieto +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +use std::collections::VecDeque; +use std::time::Instant; +use std::time::Duration; +use std::fmt; + +use procfs::process::Process; +use human_bytes::human_bytes; + +#[derive(Default, Clone, PartialEq, Debug)] +pub struct VideoEncoderStats { + pub name: String, + pub num_buffers: u64, + pub num_bytes: u64, + pub time_last_buffers: VecDeque, + pub max_buffers_inside: usize, + pub total_processing_time: Duration, + pub threads_utime: u64, + pub threads_stime: u64, + pub framerate: Option, +} + +impl VideoEncoderStats { + pub fn buffer_in(&mut self) { + self.time_last_buffers.push_back(Instant::now()); + if self.time_last_buffers.len() > self.max_buffers_inside { + self.max_buffers_inside = self.time_last_buffers.len(); + } + } + + pub fn buffer_out(&mut self) { + if let Some(arrive) = self.time_last_buffers.pop_front() { + let diff = arrive.elapsed(); + self.total_processing_time += diff; + } else { + panic!("output buffer w/o input"); + } + } + + pub fn avg_processing_time(&self) -> Duration { + if self.num_buffers != 0 { + self.total_processing_time / self.num_buffers as u32 + } else { + Duration::ZERO + } + } +} + +impl fmt::Display for VideoEncoderStats { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + if self.framerate.unwrap().denom() != 1 { + unimplemented!(); + } + + writeln!( + f, + "Encoder: {}", + &self.name + )?; + writeln!( + f, + "Buffers: {}", + self.num_buffers, + )?; + writeln!( + f, + "Bytes: {}", + self.num_bytes, + )?; + + let framerate = self.framerate.unwrap(); + let total_time_secs = self.num_buffers as f64 / framerate.numer() as f64; + let bitrate = if total_time_secs > 0.0 { + (self.num_bytes as f64 * 8.0) / total_time_secs + } else { + 0.0 + }; + let bitrate_str = human_bytes(bitrate); + + writeln!(f, "Bitrate: {}b/s", bitrate_str)?; + + let processing_time = self.avg_processing_time(); + writeln!( + f, + "Processing time: {}", + processing_time.as_secs_f64() + )?; + + let cpu_time = self.threads_utime + self.threads_stime; + writeln!( + f, + "CPU time: {}", + cpu_time + ) + } +} + +#[cfg(target_os = "linux")] +pub fn get_cpu_usage(name: String) -> (u64, u64) { + let my_pid = std::process::id() as i32; + let process = Process::new(my_pid).unwrap(); + + let mut total_utime: u64 = 0; + let mut total_stime: u64 = 0; + + for thread in process.tasks().unwrap().flatten() { + let stat = thread.stat().unwrap(); + // FIXME + //println!("Thread: {}, Comm: {}, Utime: {}, Stime: {}", thread.tid, stat.comm, stat.utime, stat.stime); + if stat.comm == name { + total_utime += stat.utime; + total_stime += stat.stime; + } + } + + (total_utime, total_stime) +} + +#[cfg(not(target_os = "linux"))] +pub fn get_cpu_usage(name: String) -> (u64, u64) { + (0, 0) +} diff --git a/video/stats/src/videoencoderstatsmeta.rs b/video/stats/src/videoencoderstatsmeta.rs new file mode 100644 index 000000000..8751afe7c --- /dev/null +++ b/video/stats/src/videoencoderstatsmeta.rs @@ -0,0 +1,186 @@ +// Copyright (C) 2025, Fluendo S.A. +// Author: Diego Nieto +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +use gst::prelude::*; +use std::fmt; +use std::mem; + +use crate::videoencoderstats::*; + +#[repr(transparent)] +pub struct VideoEncoderStatsMeta(imp::VideoEncoderStatsMeta); + +unsafe impl Send for VideoEncoderStatsMeta {} +unsafe impl Sync for VideoEncoderStatsMeta {} + +impl VideoEncoderStatsMeta { + pub fn add( + buffer: &mut gst::BufferRef, + stats: VideoEncoderStats, + ) -> gst::MetaRefMut<'_, Self, gst::meta::Standalone> { + unsafe { + let mut params = + mem::ManuallyDrop::new(imp::VideoEncoderStatsMetaParams { stats }); + + let meta = gst::ffi::gst_buffer_add_meta( + buffer.as_mut_ptr(), + imp::video_encoder_stats_meta_get_info(), + &mut *params as *mut imp::VideoEncoderStatsMetaParams as gst::glib::ffi::gpointer, + ) as *mut imp::VideoEncoderStatsMeta; + + Self::from_mut_ptr(buffer, meta) + } + } + + pub fn stats(&self) -> &VideoEncoderStats { + &self.0.stats + } +} + +unsafe impl MetaAPI for VideoEncoderStatsMeta { + type GstType = imp::VideoEncoderStatsMeta; + + fn meta_api() -> gst::glib::Type { + imp::video_encoder_stats_meta_api_get_type() + } +} + +impl fmt::Debug for VideoEncoderStatsMeta { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("VideoEncoderStatsMeta") + .finish() + } +} + +mod imp { + use gst::glib::translate::*; + use std::mem; + use std::ptr; + use std::sync::LazyLock; + + pub(super) struct VideoEncoderStatsMetaParams { + pub stats: super::VideoEncoderStats, + } + + #[repr(C)] + pub struct VideoEncoderStatsMeta { + parent: gst::ffi::GstMeta, + pub(super) stats: super::VideoEncoderStats, + } + + pub(super) fn video_encoder_stats_meta_api_get_type() -> glib::Type { + static TYPE: LazyLock = LazyLock::new(|| unsafe { + let t = from_glib(gst::ffi::gst_meta_api_type_register( + c"GstVideoEncoderStatsMetaAPI".as_ptr() as *const _, + [ptr::null::()].as_ptr() as *mut *const _, + )); + + assert_ne!(t, glib::Type::INVALID); + + t + }); + + *TYPE + } + + unsafe extern "C" fn video_encoder_stats_meta_init( + meta: *mut gst::ffi::GstMeta, + params: glib::ffi::gpointer, + _buffer: *mut gst::ffi::GstBuffer, + ) -> glib::ffi::gboolean { + assert!(!params.is_null()); + let meta = &mut *(meta as *mut VideoEncoderStatsMeta); + let params = ptr::read(params as *const VideoEncoderStatsMetaParams); + + let VideoEncoderStatsMetaParams { stats } = params; + + ptr::write(&mut meta.stats, stats); + + true.into_glib() + } + + unsafe extern "C" fn video_encoder_stats_meta_free( + meta: *mut gst::ffi::GstMeta, + _buffer: *mut gst::ffi::GstBuffer, + ) { + let meta = &mut *(meta as *mut VideoEncoderStatsMeta); + meta.stats = super::VideoEncoderStats::default(); + } + + unsafe extern "C" fn video_encoder_stats_meta_transform( + dest: *mut gst::ffi::GstBuffer, + meta: *mut gst::ffi::GstMeta, + _buffer: *mut gst::ffi::GstBuffer, + _type_: glib::ffi::GQuark, + _data: glib::ffi::gpointer, + ) -> glib::ffi::gboolean { + let dest = gst::BufferRef::from_mut_ptr(dest); + let meta = &*(meta as *const VideoEncoderStatsMeta); + + if dest.meta::().is_some() { + return true.into_glib(); + } + super::VideoEncoderStatsMeta::add( + dest, + meta.stats.clone(), + ); + + true.into_glib() + } + + pub(super) fn video_encoder_stats_meta_get_info() -> *const gst::ffi::GstMetaInfo { + struct MetaInfo(ptr::NonNull); + unsafe impl Send for MetaInfo {} + unsafe impl Sync for MetaInfo {} + + static META_INFO: LazyLock = LazyLock::new(|| unsafe { + MetaInfo( + ptr::NonNull::new(gst::ffi::gst_meta_register( + video_encoder_stats_meta_api_get_type().into_glib(), + c"VideoEncoderStatsMeta".as_ptr() as *const _, + mem::size_of::(), + Some(video_encoder_stats_meta_init), + Some(video_encoder_stats_meta_free), + Some(video_encoder_stats_meta_transform), + ) as *mut gst::ffi::GstMetaInfo) + .expect("Failed to register meta API"), + ) + }); + + META_INFO.0.as_ptr() + } +} + +#[test] +fn test() { + gst::init().unwrap(); + let stats = VideoEncoderStats { + name: "test_encoder".to_string(), + num_buffers: 8, + num_bytes: 16, + time_last_buffers: std::collections::VecDeque::new(), + max_buffers_inside: 5, + total_processing_time: std::time::Duration::ZERO, + threads_utime: 0, + threads_stime: 0, + framerate: None, + }; + let mut b = gst::Buffer::with_size(10).unwrap(); + let m = VideoEncoderStatsMeta::add(b.make_mut(), stats.clone()); + assert_eq!(m.stats().name, "test_encoder"); + let b2: gst::Buffer = b.copy_deep().unwrap(); + let m = b.meta::().unwrap(); + assert_eq!(m.stats().num_buffers, 8); + let m = b2.meta::().unwrap(); + assert_eq!(m.stats(), &stats); + let b3: gst::Buffer = b2.copy_deep().unwrap(); + drop(b2); + let m = b3.meta::().unwrap(); + assert_eq!(m.stats(), &stats); +} From 46d5eab2915d847d235b7961b7aa9ece7bac68b9 Mon Sep 17 00:00:00 2001 From: Diego Nieto Date: Wed, 23 Jul 2025 15:08:57 +0200 Subject: [PATCH 02/46] video-encoder-stats: add vmaf metrics --- video/stats/src/encoderstats/imp.rs | 84 +++++++++++++++++++++++++--- video/stats/src/videoencoderstats.rs | 10 +++- 2 files changed, 86 insertions(+), 8 deletions(-) diff --git a/video/stats/src/encoderstats/imp.rs b/video/stats/src/encoderstats/imp.rs index 6e41d1228..5689bf270 100644 --- a/video/stats/src/encoderstats/imp.rs +++ b/video/stats/src/encoderstats/imp.rs @@ -206,17 +206,87 @@ impl ObjectImpl for EncoderStats { } enc_obj.set_property("name", "enc"); + let originalbuffersave = gst::ElementFactory::make("originalbuffersave") + .build() + .expect("Failed to create originalbuffersave element"); + self.obj().add(&originalbuffersave).expect("Failed to add originalbuffersave element"); + self.obj().add(&self.identity).unwrap(); - self.srcpad - .set_target(Some(&self.identity.static_pad("src").unwrap())) - .unwrap(); + + let tee0 = gst::ElementFactory::make("tee") + .name("tee0") + .build() + .expect("Failed to create tee0 element"); + self.obj().add(&tee0).unwrap(); self.obj().add(&enc_obj).expect("Failed to add encoder element"); + originalbuffersave.link(&enc_obj).expect("Failed to link originalbuffersave to encoder"); + enc_obj.link(&self.identity).expect("Failed to link encoder to identity"); + self.identity.link(&tee0).expect("Failed to link identity to tee0"); + + let tee0_src_0 = tee0.request_pad_simple("src_%u").expect("tee0 src pad"); + self.srcpad.set_target(Some(&tee0_src_0)).unwrap(); + self.sinkpad - .set_target(Some(&enc_obj.static_pad( "sink").unwrap())) - .expect("Failed to link sink pad to encoder element"); - enc_obj.link(&self.obj().by_name("identity").expect("expected identity")) - .expect("Failed to link encoder to identity"); + .set_target(Some(&originalbuffersave.static_pad("sink").unwrap())) + .expect("Failed to link sink pad to originalbuffersave element"); + + let tee0_src_1 = tee0.request_pad_simple("src_%u").expect("tee0 src_1"); + let decodebin3 = gst::ElementFactory::make("decodebin3") + .build() + .expect("Failed to create decodebin3"); + let tee2 = gst::ElementFactory::make("tee") + .name("tee2") + .build() + .expect("Failed to create tee2"); + let originalbufferstore = gst::ElementFactory::make("originalbufferrestore") + .build() + .expect("Failed to create originalbufferrestore"); + let vmaf = gst::ElementFactory::make("vmaf") + .name("vmaf0") + .build() + .expect("Failed to create vmaf"); + vmaf.set_property("signal-scores", true); + { + let stats = self.stats.clone(); + vmaf.connect_closure( + "score", + false, + glib::closure!( + move |_vmaf: &gst::Element, score: f64| { + let mut stats = stats.lock().unwrap(); + stats.vmaf_score = score; + } + ), + ); + } + let fakesink = gst::ElementFactory::make("fakesink") + .build() + .expect("Failed to create fakesink"); + + self.obj().add_many([ + &decodebin3, &tee2, &originalbufferstore, &vmaf, &fakesink, + ].as_ref()).expect("Failed to add vmaf branch elements"); + + tee0_src_1.link(&decodebin3.static_pad("sink").unwrap()).expect("tee0.src_1 -> decodebin3"); + + let tee2_clone = tee2.clone(); + let originalbufferstore_clone = originalbufferstore.clone(); + let vmaf_clone = vmaf.clone(); + let fakesink_clone = fakesink.clone(); + decodebin3.connect_pad_added(move |_dbin, src_pad| { + let tee2_sink = tee2_clone.static_pad("sink").unwrap(); + if src_pad.link(&tee2_sink).is_ok() { + let tee2_src_0 = tee2_clone.request_pad_simple("src_%u").expect("tee2 src_0"); + tee2_src_0.link(&originalbufferstore_clone.static_pad("sink").unwrap()).expect("tee2.src_0 -> originalbufferstore"); + originalbufferstore_clone.link(&vmaf_clone).expect("originalbufferrestore -> vmaf"); + vmaf_clone.link(&fakesink_clone).expect("vmaf -> fakesink"); + + let tee2_src_1 = tee2_clone.request_pad_simple("src_%u").expect("tee2 src_1"); + let vmaf_sink_1 = vmaf_clone.request_pad_simple("sink_1").expect("vmaf sink_1"); + tee2_src_1.link(&vmaf_sink_1).expect("tee2.src_1 -> vmaf.sink_1"); + } + }); unsafe { diff --git a/video/stats/src/videoencoderstats.rs b/video/stats/src/videoencoderstats.rs index 80778f9ad..354debf81 100644 --- a/video/stats/src/videoencoderstats.rs +++ b/video/stats/src/videoencoderstats.rs @@ -26,6 +26,7 @@ pub struct VideoEncoderStats { pub threads_utime: u64, pub threads_stime: u64, pub framerate: Option, + pub vmaf_score: f64, } impl VideoEncoderStats { @@ -90,7 +91,7 @@ impl fmt::Display for VideoEncoderStats { let processing_time = self.avg_processing_time(); writeln!( f, - "Processing time: {}", + "Processing time: {:.3}", processing_time.as_secs_f64() )?; @@ -99,6 +100,13 @@ impl fmt::Display for VideoEncoderStats { f, "CPU time: {}", cpu_time + )?; + + let vmaf_score = self.vmaf_score; + writeln!( + f, + "VMAF score: {:.3}", + vmaf_score ) } } From 78fd66e4a4624875e67c59479dd7b0594c9ae83a Mon Sep 17 00:00:00 2001 From: Diego Nieto Date: Thu, 24 Jul 2025 10:51:37 +0200 Subject: [PATCH 03/46] video-compare-stats: show half decoded outputs for mixing --- video/stats/examples/video-encoder-stats.rs | 4 +- video/stats/src/comparemixer/imp.rs | 57 +++++++++++++++++---- 2 files changed, 50 insertions(+), 11 deletions(-) diff --git a/video/stats/examples/video-encoder-stats.rs b/video/stats/examples/video-encoder-stats.rs index ddd461221..ab3de7ea3 100644 --- a/video/stats/examples/video-encoder-stats.rs +++ b/video/stats/examples/video-encoder-stats.rs @@ -7,13 +7,13 @@ // // SPDX-License-Identifier: MPL-2.0 -use gst::prelude::*; use anyhow::Error; +use gst::prelude::*; fn main() -> Result<(), Error> { gst::init()?; - let pipeline = gst::parse::launch("videotestsrc ! video/x-raw,width=640,height=480 ! videoconvert ! tee name=tee ! queue name=encq0 ! video-encoder-stats encoder=\"x264enc\" ! decodebin3 name=dec0 tee. ! queue name=encq1 ! video-encoder-stats encoder=\"x264enc bitrate=32\" ! decodebin3 name=dec1 video-compare-mixer name=mixer backend=CPU dec0. ! mixer. dec1. ! mixer. mixer. ! videoconvert ! autovideosink")?; + let pipeline = gst::parse::launch("souphttpsrc location=\"https://ftp.nluug.nl/pub/graphics/blender/demo/movies/ToS/tears_of_steel_1080p.mov\" ! qtdemux name=demux demux.video_0 ! queue ! decodebin3 ! videoconvertscale ! capsfilter caps=\"video/x-raw,width=1280,aspect-ratio=1/1\" ! tee name=tee ! queue name=encq0 ! video-encoder-stats encoder=\"x264enc\" ! decodebin3 name=dec0 tee. ! queue name=encq1 ! video-encoder-stats encoder=\"x264enc bitrate=512\" ! decodebin3 name=dec1 video-compare-mixer name=mixer backend=CPU dec0. ! mixer. dec1. ! mixer. mixer. ! videoconvert ! autovideosink")?; pipeline.set_state(gst::State::Playing)?; let bus = pipeline.bus().unwrap(); diff --git a/video/stats/src/comparemixer/imp.rs b/video/stats/src/comparemixer/imp.rs index e6471efd7..92a3ef380 100644 --- a/video/stats/src/comparemixer/imp.rs +++ b/video/stats/src/comparemixer/imp.rs @@ -54,6 +54,8 @@ pub struct VideoCompareMixer { queue1: gst::Element, overlay0: gst::Element, overlay1: gst::Element, + crop0: gst::Element, + crop1: gst::Element, settings: Mutex, } @@ -101,8 +103,10 @@ impl VideoCompareMixer { ) -> Result<(), gst::ErrorMessage> { self.overlay0.set_property_from_str("line-alignment", "left"); self.overlay0.set_property_from_str("halignment", "left"); + self.overlay0.set_property_from_str("valignment", "top"); self.overlay1.set_property_from_str("line-alignment", "right"); self.overlay1.set_property_from_str("halignment", "right"); + self.overlay1.set_property_from_str("valignment", "top"); let compositor_pad0 = compositor .request_pad_simple("sink_0") @@ -126,6 +130,12 @@ impl VideoCompareMixer { self.obj() .add(&self.overlay1) .expect("Failed to add overlay1 element"); + self.obj() + .add(&self.crop0) + .expect("Failed to add crop0 element"); + self.obj() + .add(&self.crop1) + .expect("Failed to add crop1 element"); self.sinkpad0 .set_target(Some(&self.queue0.static_pad("sink").unwrap())) @@ -134,25 +144,35 @@ impl VideoCompareMixer { .set_target(Some(&self.queue1.static_pad("sink").unwrap())) .expect("Failed to link sinkpad1 to queue1"); self.queue0 - .static_pad( "src") + .static_pad("src") .unwrap() .link(&self.overlay0.static_pad("video_sink").unwrap()) - .expect("Failed to link queue0 to compositor"); - self.queue1 + .expect("Failed to link queue0 to overlay0"); + self.overlay0 .static_pad("src") .unwrap() - .link( &self.overlay1.static_pad( "video_sink").unwrap()) - .expect("Failed to link queue1 to compositor"); - self.overlay0 + .link(&self.crop0.static_pad("sink").unwrap()) + .expect("Failed to link overlay0 to crop0"); + self.crop0 .static_pad("src") .unwrap() .link(&compositor_pad0) - .expect("Failed to link overlay0 to compositor"); + .expect("Failed to link crop0 to compositor"); + self.queue1 + .static_pad("src") + .unwrap() + .link(&self.overlay1.static_pad("video_sink").unwrap()) + .expect("Failed to link queue1 to overlay1"); self.overlay1 + .static_pad("src") + .unwrap() + .link(&self.crop1.static_pad("sink").unwrap()) + .expect("Failed to link overlay1 to crop1"); + self.crop1 .static_pad("src") .unwrap() .link(&compositor_pad1) - .expect("Failed to link overlay1 to compositor"); + .expect("Failed to link crop1 to compositor"); self.srcpad .set_target(Some(&compositor.static_pad("src").unwrap())) .expect("Failed to link srcpad to compositor"); @@ -167,8 +187,15 @@ impl VideoCompareMixer { Caps(c) => { let caps = c.caps(); let s = caps.structure(0).unwrap(); + let width = s.get::("width").unwrap(); + let half_width = width / 2; + + // Set crop properties for both crops + self.crop0.set_property("right", half_width); + self.crop1.set_property("left", half_width); + let compositor_sink1_pad = self.obj().by_name("compositor").unwrap().static_pad("sink_1").unwrap(); - compositor_sink1_pad.set_property("xpos", s.get::("width").unwrap()); + compositor_sink1_pad.set_property("xpos", half_width); gst::info!(CAT, "Received caps {caps:?}"); } _ => { @@ -216,6 +243,16 @@ impl ObjectSubclass for VideoCompareMixer { .expect("Failed to create overlay1"); overlay1.set_property("name", "overlay1"); + let crop0 = gst::ElementFactory::make("videocrop") + .build() + .expect("Failed to create crop0"); + crop0.set_property("name", "crop0"); + + let crop1 = gst::ElementFactory::make("videocrop") + .build() + .expect("Failed to create crop1"); + crop1.set_property("name", "crop1"); + Self { srcpad, sinkpad0, @@ -224,6 +261,8 @@ impl ObjectSubclass for VideoCompareMixer { queue1, overlay0, overlay1, + crop0, + crop1, settings: Mutex::new(Settings::default()), } } From fe14dbb347ed3fb49b23af06da26686e80004bc4 Mon Sep 17 00:00:00 2001 From: Diego Nieto Date: Thu, 31 Jul 2025 19:49:47 +0200 Subject: [PATCH 04/46] video-encoder-stats: fix issue for different encoders Different encoders introduce different latencies and the pipelines where not starting properly. Introduce multiples queues in bifurcations and convergences. Introduce also videoconvert and fix the video caps format to I420 --- video/stats/src/encoderstats/imp.rs | 90 +++++++++++++++++++++++------ 1 file changed, 71 insertions(+), 19 deletions(-) diff --git a/video/stats/src/encoderstats/imp.rs b/video/stats/src/encoderstats/imp.rs index 5689bf270..f07296aea 100644 --- a/video/stats/src/encoderstats/imp.rs +++ b/video/stats/src/encoderstats/imp.rs @@ -218,30 +218,63 @@ impl ObjectImpl for EncoderStats { .build() .expect("Failed to create tee0 element"); self.obj().add(&tee0).unwrap(); - + self.obj().add(&enc_obj).expect("Failed to add encoder element"); originalbuffersave.link(&enc_obj).expect("Failed to link originalbuffersave to encoder"); enc_obj.link(&self.identity).expect("Failed to link encoder to identity"); self.identity.link(&tee0).expect("Failed to link identity to tee0"); - + let tee0_src_0 = tee0.request_pad_simple("src_%u").expect("tee0 src pad"); - self.srcpad.set_target(Some(&tee0_src_0)).unwrap(); + let queue0 = gst::ElementFactory::make("queue") + .name("encq0") + .build() + .expect("Failed to create queue encq0"); + self.obj().add(&queue0).expect("Failed to add queue encq0"); + let queue0_sink_pad = queue0.static_pad("sink").unwrap(); + let queue0_src_pad = queue0.static_pad("src").unwrap(); + tee0_src_0.link(&queue0_sink_pad).expect("tee0.src_0 -> encq0.sink"); + self.srcpad.set_target(Some(&queue0_src_pad)).unwrap(); self.sinkpad .set_target(Some(&originalbuffersave.static_pad("sink").unwrap())) .expect("Failed to link sink pad to originalbuffersave element"); let tee0_src_1 = tee0.request_pad_simple("src_%u").expect("tee0 src_1"); + let queue1 = gst::ElementFactory::make("queue") + .name("encq1") + .build() + .expect("Failed to create queue encq1"); let decodebin3 = gst::ElementFactory::make("decodebin3") .build() .expect("Failed to create decodebin3"); - let tee2 = gst::ElementFactory::make("tee") - .name("tee2") + // Add videoconvert after decodebin3 and before capsfilter + let videoconvert = gst::ElementFactory::make("videoconvert") + .build() + .expect("Failed to create videoconvert"); + let capsfilter = gst::ElementFactory::make("capsfilter") .build() - .expect("Failed to create tee2"); + .expect("Failed to create capsfilter"); + let caps = gst::Caps::builder("video/x-raw") + .field("format", &"I420") + .build(); + capsfilter.set_property("caps", &caps); + let tee1 = gst::ElementFactory::make("tee") + .name("tee1") + .build() + .expect("Failed to create tee1 element"); let originalbufferstore = gst::ElementFactory::make("originalbufferrestore") .build() .expect("Failed to create originalbufferrestore"); + // Add queue before originalbufferrestore -> vmaf + let queue_vmaf_0 = gst::ElementFactory::make("queue") + .name("queue_vmaf_0") + .build() + .expect("Failed to create queue_vmaf_0"); + // Add queue before vmaf sink_1 + let queue_vmaf_1 = gst::ElementFactory::make("queue") + .name("queue_vmaf_1") + .build() + .expect("Failed to create queue_vmaf_1"); let vmaf = gst::ElementFactory::make("vmaf") .name("vmaf0") .build() @@ -265,26 +298,45 @@ impl ObjectImpl for EncoderStats { .expect("Failed to create fakesink"); self.obj().add_many([ - &decodebin3, &tee2, &originalbufferstore, &vmaf, &fakesink, + &queue1, &decodebin3, &videoconvert, &capsfilter, &tee1, + &originalbufferstore, &queue_vmaf_0, &vmaf, &queue_vmaf_1, &fakesink, ].as_ref()).expect("Failed to add vmaf branch elements"); - tee0_src_1.link(&decodebin3.static_pad("sink").unwrap()).expect("tee0.src_1 -> decodebin3"); + tee0_src_1.link(&queue1.static_pad("sink").unwrap()).expect("tee0.src_1 -> decodebin3"); + queue1.static_pad("src").unwrap().link(&decodebin3.static_pad("sink").unwrap()).expect("encq1.src -> decodebin3.sink"); - let tee2_clone = tee2.clone(); + let tee1_clone = tee1.clone(); let originalbufferstore_clone = originalbufferstore.clone(); + let queue_vmaf_0_clone = queue_vmaf_0.clone(); let vmaf_clone = vmaf.clone(); + let queue_vmaf_1_clone = queue_vmaf_1.clone(); let fakesink_clone = fakesink.clone(); + let videoconvert_clone = videoconvert.clone(); + let capsfilter_clone = capsfilter.clone(); decodebin3.connect_pad_added(move |_dbin, src_pad| { - let tee2_sink = tee2_clone.static_pad("sink").unwrap(); - if src_pad.link(&tee2_sink).is_ok() { - let tee2_src_0 = tee2_clone.request_pad_simple("src_%u").expect("tee2 src_0"); - tee2_src_0.link(&originalbufferstore_clone.static_pad("sink").unwrap()).expect("tee2.src_0 -> originalbufferstore"); - originalbufferstore_clone.link(&vmaf_clone).expect("originalbufferrestore -> vmaf"); - vmaf_clone.link(&fakesink_clone).expect("vmaf -> fakesink"); - - let tee2_src_1 = tee2_clone.request_pad_simple("src_%u").expect("tee2 src_1"); - let vmaf_sink_1 = vmaf_clone.request_pad_simple("sink_1").expect("vmaf sink_1"); - tee2_src_1.link(&vmaf_sink_1).expect("tee2.src_1 -> vmaf.sink_1"); + // Link decodebin3 src_pad -> videoconvert -> capsfilter -> tee1 + let videoconvert_sink = videoconvert_clone.static_pad("sink").unwrap(); + if src_pad.link(&videoconvert_sink).is_ok() { + let videoconvert_src = videoconvert_clone.static_pad("src").unwrap(); + let capsfilter_sink = capsfilter_clone.static_pad("sink").unwrap(); + if videoconvert_src.link(&capsfilter_sink).is_ok() { + let capsfilter_src = capsfilter_clone.static_pad("src").unwrap(); + let tee1_sink = tee1_clone.static_pad("sink").unwrap(); + if capsfilter_src.link(&tee1_sink).is_ok() { + let tee1_src_0 = tee1_clone.request_pad_simple("src_%u").expect("tee1 src_0"); + // Link: tee1.src_0 -> originalbufferstore -> queue_vmaf_0 -> vmaf -> fakesink + tee1_src_0.link(&originalbufferstore_clone.static_pad("sink").unwrap()).expect("tee1.src_0 -> originalbufferstore"); + originalbufferstore_clone.link(&queue_vmaf_0_clone).expect("originalbufferrestore -> queue_vmaf_0"); + queue_vmaf_0_clone.link(&vmaf_clone).expect("queue_vmaf_0 -> vmaf"); + vmaf_clone.link(&fakesink_clone).expect("vmaf -> fakesink"); + + let tee1_src_1 = tee1_clone.request_pad_simple("src_%u").expect("tee1 src_1"); + let vmaf_sink_1 = vmaf_clone.request_pad_simple("sink_1").expect("vmaf sink_1"); + // Link: tee1.src_1 -> queue_vmaf_1 -> vmaf.sink_1 + tee1_src_1.link(&queue_vmaf_1_clone.static_pad("sink").unwrap()).expect("tee1.src_1 -> queue_vmaf_1"); + queue_vmaf_1_clone.static_pad("src").unwrap().link(&vmaf_sink_1).expect("queue_vmaf_1.src -> vmaf.sink_1"); + } + } } }); From 683a3a7c239acdb39d49c87e2bebe52ccaabc1d8 Mon Sep 17 00:00:00 2001 From: Diego Nieto Date: Mon, 4 Aug 2025 09:42:25 +0200 Subject: [PATCH 05/46] video-compare-mixer: set CPU as default mixer --- video/stats/examples/video-encoder-stats.rs | 2 +- video/stats/src/comparemixer/imp.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/video/stats/examples/video-encoder-stats.rs b/video/stats/examples/video-encoder-stats.rs index ab3de7ea3..4c9e1ae99 100644 --- a/video/stats/examples/video-encoder-stats.rs +++ b/video/stats/examples/video-encoder-stats.rs @@ -13,7 +13,7 @@ use gst::prelude::*; fn main() -> Result<(), Error> { gst::init()?; - let pipeline = gst::parse::launch("souphttpsrc location=\"https://ftp.nluug.nl/pub/graphics/blender/demo/movies/ToS/tears_of_steel_1080p.mov\" ! qtdemux name=demux demux.video_0 ! queue ! decodebin3 ! videoconvertscale ! capsfilter caps=\"video/x-raw,width=1280,aspect-ratio=1/1\" ! tee name=tee ! queue name=encq0 ! video-encoder-stats encoder=\"x264enc\" ! decodebin3 name=dec0 tee. ! queue name=encq1 ! video-encoder-stats encoder=\"x264enc bitrate=512\" ! decodebin3 name=dec1 video-compare-mixer name=mixer backend=CPU dec0. ! mixer. dec1. ! mixer. mixer. ! videoconvert ! autovideosink")?; + let pipeline = gst::parse::launch("souphttpsrc location=\"https://ftp.nluug.nl/pub/graphics/blender/demo/movies/ToS/tears_of_steel_1080p.mov\" ! qtdemux name=demux demux.video_0 ! queue ! decodebin3 ! videoconvertscale ! capsfilter caps=\"video/x-raw,width=1280,aspect-ratio=1/1\" ! tee name=tee ! queue name=encq0 ! video-encoder-stats encoder=\"x264enc\" ! decodebin3 name=dec0 tee. ! queue name=encq1 ! video-encoder-stats encoder=\"x264enc bitrate=512\" ! decodebin3 name=dec1 video-compare-mixer name=mixer dec0. ! mixer. dec1. ! mixer. mixer. ! videoconvert ! autovideosink")?; pipeline.set_state(gst::State::Playing)?; let bus = pipeline.bus().unwrap(); diff --git a/video/stats/src/comparemixer/imp.rs b/video/stats/src/comparemixer/imp.rs index 92a3ef380..d92e49061 100644 --- a/video/stats/src/comparemixer/imp.rs +++ b/video/stats/src/comparemixer/imp.rs @@ -29,12 +29,12 @@ static CAT: LazyLock = LazyLock::new(|| { #[repr(u32)] #[non_exhaustive] pub enum Backend { - #[default] #[enum_value(name = "OpenGL", nick = "OpenGL")] GL, #[enum_value(name = "VAAPI", nick = "VAAPI")] #[cfg(target_os = "linux")] VAAPI, + #[default] #[enum_value(name = "CPU", nick = "CPU")] CPU, #[enum_value(name = "D3D12", nick = "D3D12")] From 2f396aff369678b3942758d6c6a7240e990d0d90 Mon Sep 17 00:00:00 2001 From: Diego Nieto Date: Wed, 6 Aug 2025 16:22:38 +0200 Subject: [PATCH 06/46] video-compare-stats: init element without the need of specify a backend Change start-up sequence --- video/stats/examples/video-encoder-stats.rs | 2 +- video/stats/src/comparemixer/imp.rs | 73 +++++++++++++-------- 2 files changed, 48 insertions(+), 27 deletions(-) diff --git a/video/stats/examples/video-encoder-stats.rs b/video/stats/examples/video-encoder-stats.rs index 4c9e1ae99..341cf2443 100644 --- a/video/stats/examples/video-encoder-stats.rs +++ b/video/stats/examples/video-encoder-stats.rs @@ -13,7 +13,7 @@ use gst::prelude::*; fn main() -> Result<(), Error> { gst::init()?; - let pipeline = gst::parse::launch("souphttpsrc location=\"https://ftp.nluug.nl/pub/graphics/blender/demo/movies/ToS/tears_of_steel_1080p.mov\" ! qtdemux name=demux demux.video_0 ! queue ! decodebin3 ! videoconvertscale ! capsfilter caps=\"video/x-raw,width=1280,aspect-ratio=1/1\" ! tee name=tee ! queue name=encq0 ! video-encoder-stats encoder=\"x264enc\" ! decodebin3 name=dec0 tee. ! queue name=encq1 ! video-encoder-stats encoder=\"x264enc bitrate=512\" ! decodebin3 name=dec1 video-compare-mixer name=mixer dec0. ! mixer. dec1. ! mixer. mixer. ! videoconvert ! autovideosink")?; + let pipeline = gst::parse::launch("souphttpsrc location=\"https://ftp.nluug.nl/pub/graphics/blender/demo/movies/ToS/tears_of_steel_1080p.mov\" ! qtdemux name=demux demux.video_0 ! queue ! decodebin3 ! videoconvertscale ! capsfilter caps=\"video/x-raw,width=640,aspect-ratio=1/1\" ! tee name=tee ! queue name=encq0 ! video-encoder-stats encoder=\"x264enc\" ! decodebin3 name=dec0 tee. ! queue name=encq1 ! video-encoder-stats encoder=\"x264enc bitrate=512\" ! decodebin3 name=dec1 video-compare-mixer name=mixer dec0. ! mixer. dec1. ! mixer. mixer. ! videoconvert ! autovideosink")?; pipeline.set_state(gst::State::Playing)?; let bus = pipeline.bus().unwrap(); diff --git a/video/stats/src/comparemixer/imp.rs b/video/stats/src/comparemixer/imp.rs index d92e49061..b32541010 100644 --- a/video/stats/src/comparemixer/imp.rs +++ b/video/stats/src/comparemixer/imp.rs @@ -79,6 +79,31 @@ impl VideoCompareMixer { } } + fn prepare_pipeline(&self, backend: Backend) -> Result<(), gst::ErrorMessage> { + let compositor = gst::ElementFactory::make(self.get_pipeline_compositor(backend)) + .build() + .expect("Failed to create compositor element"); + compositor.set_property("name", "compositor"); + + self.link_elements(&compositor)?; + + self.add_overlay_probe(&self.overlay0); + self.add_overlay_probe(&self.overlay1); + + unsafe { + self.sinkpad0.set_event_full_function(|pad, parent, event| { + VideoCompareMixer::catch_panic_pad_function( + parent, + || false, + |video_compare_mixer| video_compare_mixer.sink_event(&pad.clone().upcast::(), event), + ); + Ok(gst::FlowSuccess::Ok) + }); + } + + Ok(()) + } + fn add_overlay_probe(&self, overlay: &gst::Element) { let overlay_src_pad = overlay.static_pad("video_sink").unwrap(); let overlay_clone = overlay.clone(); @@ -292,34 +317,10 @@ impl ObjectImpl for VideoCompareMixer { "backend" => { settings.backend = value.get().expect("type checked upstream"); - let compositor = - gst::ElementFactory::make(self.get_pipeline_compositor(settings.backend)) - .build() - .expect("Failed to create identity element"); - compositor.set_property("name", "compositor"); - - self.link_elements(&compositor) - .expect("Failed to link elements"); - - self.add_overlay_probe(&self.overlay0); - self.add_overlay_probe(&self.overlay1); - - unsafe - { - self.sinkpad0.set_event_full_function(|pad, parent, event| { - VideoCompareMixer::catch_panic_pad_function( - parent, - || false, - |video_compare_mixer| video_compare_mixer.sink_event(&pad.clone().upcast::(), event), - ); - Ok(gst::FlowSuccess::Ok) - }); - } - gst::info!( CAT, imp = self, - "Set backend to {:?} and linked pads", + "Set backend to {:?}", settings.backend ); } @@ -398,6 +399,26 @@ impl ElementImpl for VideoCompareMixer { PAD_TEMPLATES.as_ref() } + + fn change_state( + &self, + transition: gst::StateChange, + ) -> Result { + match transition { + gst::StateChange::ReadyToPaused => { + let settings = self.settings.lock().unwrap(); + if let Err(err) = self.prepare_pipeline(settings.backend) { + gst::error!(CAT, imp = self, "Failed to prepare pipeline: {}", err); + return Err(gst::StateChangeError); + } + gst::info!(CAT, imp = self, "Pipeline prepared for backend {:?}", settings.backend); + } + _ => {} + } + + self.parent_change_state(transition) + } } -impl BinImpl for VideoCompareMixer {} +impl BinImpl for VideoCompareMixer { +} From f90dc64650b33570e9264196c3dcdb0973a2a068 Mon Sep 17 00:00:00 2001 From: Diego Nieto Date: Thu, 7 Aug 2025 13:32:17 +0200 Subject: [PATCH 07/46] video-compare-mixer: add property to allow split the screen or duplicate --- video/stats/src/comparemixer/imp.rs | 183 +++++++++++++++++++--------- 1 file changed, 126 insertions(+), 57 deletions(-) diff --git a/video/stats/src/comparemixer/imp.rs b/video/stats/src/comparemixer/imp.rs index b32541010..f5f129301 100644 --- a/video/stats/src/comparemixer/imp.rs +++ b/video/stats/src/comparemixer/imp.rs @@ -44,6 +44,7 @@ pub enum Backend { struct Settings { backend: Backend, + split_screen: bool, } pub struct VideoCompareMixer { @@ -54,8 +55,6 @@ pub struct VideoCompareMixer { queue1: gst::Element, overlay0: gst::Element, overlay1: gst::Element, - crop0: gst::Element, - crop1: gst::Element, settings: Mutex, } @@ -63,6 +62,7 @@ impl Default for Settings { fn default() -> Self { Self { backend: Backend::default(), + split_screen: false, } } } @@ -85,7 +85,26 @@ impl VideoCompareMixer { .expect("Failed to create compositor element"); compositor.set_property("name", "compositor"); - self.link_elements(&compositor)?; + let settings = self.settings.lock().unwrap(); + let split_screen = settings.split_screen; + drop(settings); + + if split_screen { + let crop0 = gst::ElementFactory::make("videocrop") + .build() + .expect("Failed to create crop0"); + crop0.set_property("name", "crop0"); + + let crop1 = gst::ElementFactory::make("videocrop") + .build() + .expect("Failed to create crop1"); + crop1.set_property("name", "crop1"); + + self.obj().add(&crop0).expect("Failed to add crop0 element"); + self.obj().add(&crop1).expect("Failed to add crop1 element"); + } + + self.link_elements(&compositor, split_screen)?; self.add_overlay_probe(&self.overlay0); self.add_overlay_probe(&self.overlay1); @@ -125,6 +144,7 @@ impl VideoCompareMixer { fn link_elements( &self, compositor: &gst::Element, + split_screen: bool, ) -> Result<(), gst::ErrorMessage> { self.overlay0.set_property_from_str("line-alignment", "left"); self.overlay0.set_property_from_str("halignment", "left"); @@ -155,12 +175,6 @@ impl VideoCompareMixer { self.obj() .add(&self.overlay1) .expect("Failed to add overlay1 element"); - self.obj() - .add(&self.crop0) - .expect("Failed to add crop0 element"); - self.obj() - .add(&self.crop1) - .expect("Failed to add crop1 element"); self.sinkpad0 .set_target(Some(&self.queue0.static_pad("sink").unwrap())) @@ -168,39 +182,75 @@ impl VideoCompareMixer { self.sinkpad1 .set_target(Some(&self.queue1.static_pad("sink").unwrap())) .expect("Failed to link sinkpad1 to queue1"); - self.queue0 - .static_pad("src") - .unwrap() - .link(&self.overlay0.static_pad("video_sink").unwrap()) - .expect("Failed to link queue0 to overlay0"); - self.overlay0 - .static_pad("src") - .unwrap() - .link(&self.crop0.static_pad("sink").unwrap()) - .expect("Failed to link overlay0 to crop0"); - self.crop0 - .static_pad("src") - .unwrap() - .link(&compositor_pad0) - .expect("Failed to link crop0 to compositor"); - self.queue1 - .static_pad("src") - .unwrap() - .link(&self.overlay1.static_pad("video_sink").unwrap()) - .expect("Failed to link queue1 to overlay1"); - self.overlay1 - .static_pad("src") - .unwrap() - .link(&self.crop1.static_pad("sink").unwrap()) - .expect("Failed to link overlay1 to crop1"); - self.crop1 - .static_pad("src") - .unwrap() - .link(&compositor_pad1) - .expect("Failed to link crop1 to compositor"); + self.srcpad .set_target(Some(&compositor.static_pad("src").unwrap())) .expect("Failed to link srcpad to compositor"); + + if split_screen { + // Get crop elements by name since we can't store them in struct easily + let crop0 = self.obj().by_name("crop0").expect("crop0 should exist"); + let crop1 = self.obj().by_name("crop1").expect("crop1 should exist"); + + self.queue0 + .static_pad("src") + .unwrap() + .link(&self.overlay0.static_pad("video_sink").unwrap()) + .expect("Failed to link queue0 to overlay0"); + self.overlay0 + .static_pad("src") + .unwrap() + .link(&crop0.static_pad("sink").unwrap()) + .expect("Failed to link overlay0 to crop0"); + crop0 + .static_pad("src") + .unwrap() + .link(&compositor_pad0) + .expect("Failed to link crop0 to queue2"); + self.queue1 + .static_pad("src") + .unwrap() + .link(&self.overlay1.static_pad("video_sink").unwrap()) + .expect("Failed to link queue1 to overlay1"); + self.overlay1 + .static_pad("src") + .unwrap() + .link(&crop1.static_pad("sink").unwrap()) + .expect("Failed to link overlay1 to crop1"); + crop1 + .static_pad("src") + .unwrap() + .link(&compositor_pad1) + .expect("Failed to link crop1 to queue3"); + } else { + // Direct connection without crops - overlay mode + self.queue0 + .static_pad("src") + .unwrap() + .link(&self.overlay0.static_pad("video_sink").unwrap()) + .expect("Failed to link queue0 to overlay0"); + self.overlay0 + .static_pad("src") + .unwrap() + .link(&compositor_pad0) + .expect("Failed to link overlay0 to queue2"); + self.queue1 + .static_pad("src") + .unwrap() + .link(&self.overlay1.static_pad("video_sink").unwrap()) + .expect("Failed to link queue1 to overlay1"); + self.overlay1 + .static_pad("src") + .unwrap() + .link(&compositor_pad1) + .expect("Failed to link overlay1 to queue3"); + } + + self.queue0.sync_state_with_parent().unwrap(); + self.queue1.sync_state_with_parent().unwrap(); + self.overlay0.sync_state_with_parent().unwrap(); + self.overlay1.sync_state_with_parent().unwrap(); + self.obj().by_name("compositor").unwrap().sync_state_with_parent().unwrap(); Ok(()) } @@ -215,12 +265,23 @@ impl VideoCompareMixer { let width = s.get::("width").unwrap(); let half_width = width / 2; - // Set crop properties for both crops - self.crop0.set_property("right", half_width); - self.crop1.set_property("left", half_width); + let settings = self.settings.lock().unwrap(); + let split_screen = settings.split_screen; + drop(settings); let compositor_sink1_pad = self.obj().by_name("compositor").unwrap().static_pad("sink_1").unwrap(); - compositor_sink1_pad.set_property("xpos", half_width); + if split_screen { + // Set crop properties for both crops + if let Some(crop0) = self.obj().by_name("crop0") { + crop0.set_property("right", half_width); + } + if let Some(crop1) = self.obj().by_name("crop1") { + crop1.set_property("left", half_width); + } + compositor_sink1_pad.set_property("xpos", half_width); + } else { + compositor_sink1_pad.set_property("xpos", width); + } gst::info!(CAT, "Received caps {caps:?}"); } _ => { @@ -268,16 +329,6 @@ impl ObjectSubclass for VideoCompareMixer { .expect("Failed to create overlay1"); overlay1.set_property("name", "overlay1"); - let crop0 = gst::ElementFactory::make("videocrop") - .build() - .expect("Failed to create crop0"); - crop0.set_property("name", "crop0"); - - let crop1 = gst::ElementFactory::make("videocrop") - .build() - .expect("Failed to create crop1"); - crop1.set_property("name", "crop1"); - Self { srcpad, sinkpad0, @@ -286,8 +337,6 @@ impl ObjectSubclass for VideoCompareMixer { queue1, overlay0, overlay1, - crop0, - crop1, settings: Mutex::new(Settings::default()), } } @@ -305,6 +354,12 @@ impl ObjectImpl for VideoCompareMixer { .blurb("The backend to use for mixing the video") .mutable_ready() .build(), + glib::ParamSpecBoolean::builder("split-screen") + .nick("Split Screen Mode") + .blurb("Enable split-screen mode with cropping") + .default_value(false) + .mutable_ready() + .build(), ] }); @@ -324,6 +379,16 @@ impl ObjectImpl for VideoCompareMixer { settings.backend ); } + "split-screen" => { + settings.split_screen = value.get().expect("type checked upstream"); + + gst::info!( + CAT, + imp = self, + "Set split-screen to {:?}", + settings.split_screen + ); + } _ => unimplemented!(), } } @@ -332,11 +397,13 @@ impl ObjectImpl for VideoCompareMixer { let settings = self.settings.lock().unwrap(); match pspec.name() { "backend" => settings.backend.to_value(), + "split-screen" => settings.split_screen.to_value(), _ => unimplemented!(), } } fn constructed(&self) { + gst::info!(CAT, "Constructing VideoCompareMixer"); self.parent_constructed(); let obj = self.obj(); @@ -407,11 +474,13 @@ impl ElementImpl for VideoCompareMixer { match transition { gst::StateChange::ReadyToPaused => { let settings = self.settings.lock().unwrap(); - if let Err(err) = self.prepare_pipeline(settings.backend) { + let backend = settings.backend; + drop(settings); + if let Err(err) = self.prepare_pipeline(backend) { gst::error!(CAT, imp = self, "Failed to prepare pipeline: {}", err); return Err(gst::StateChangeError); } - gst::info!(CAT, imp = self, "Pipeline prepared for backend {:?}", settings.backend); + gst::info!(CAT, imp = self, "Pipeline prepared for backend {:?}", backend); } _ => {} } From 98ba55db0664c7f42611866945a0622d6cd99c7f Mon Sep 17 00:00:00 2001 From: Diego Nieto Date: Thu, 7 Aug 2025 14:27:10 +0200 Subject: [PATCH 08/46] video-compare-mixer: handle crop for glvideomixer --- video/stats/examples/video-encoder-stats.rs | 2 +- video/stats/src/comparemixer/imp.rs | 44 ++++++++++++--------- 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/video/stats/examples/video-encoder-stats.rs b/video/stats/examples/video-encoder-stats.rs index 341cf2443..234ab0732 100644 --- a/video/stats/examples/video-encoder-stats.rs +++ b/video/stats/examples/video-encoder-stats.rs @@ -13,7 +13,7 @@ use gst::prelude::*; fn main() -> Result<(), Error> { gst::init()?; - let pipeline = gst::parse::launch("souphttpsrc location=\"https://ftp.nluug.nl/pub/graphics/blender/demo/movies/ToS/tears_of_steel_1080p.mov\" ! qtdemux name=demux demux.video_0 ! queue ! decodebin3 ! videoconvertscale ! capsfilter caps=\"video/x-raw,width=640,aspect-ratio=1/1\" ! tee name=tee ! queue name=encq0 ! video-encoder-stats encoder=\"x264enc\" ! decodebin3 name=dec0 tee. ! queue name=encq1 ! video-encoder-stats encoder=\"x264enc bitrate=512\" ! decodebin3 name=dec1 video-compare-mixer name=mixer dec0. ! mixer. dec1. ! mixer. mixer. ! videoconvert ! autovideosink")?; + let pipeline = gst::parse::launch("souphttpsrc location=\"https://ftp.nluug.nl/pub/graphics/blender/demo/movies/ToS/tears_of_steel_1080p.mov\" ! qtdemux name=demux demux.video_0 ! queue ! decodebin3 ! videoconvertscale ! capsfilter caps=\"video/x-raw,width=640,aspect-ratio=1/1\" ! tee name=tee ! queue name=encq0 ! video-encoder-stats encoder=\"x264enc\" ! decodebin3 name=dec0 tee. ! queue name=encq1 ! video-encoder-stats encoder=\"x264enc bitrate=512\" ! decodebin3 name=dec1 video-compare-mixer split-screen=false backend=OpenGL name=mixer dec0. ! mixer. dec1. ! mixer. mixer. ! autovideosink")?; pipeline.set_state(gst::State::Playing)?; let bus = pipeline.bus().unwrap(); diff --git a/video/stats/src/comparemixer/imp.rs b/video/stats/src/comparemixer/imp.rs index f5f129301..c1c7a65e1 100644 --- a/video/stats/src/comparemixer/imp.rs +++ b/video/stats/src/comparemixer/imp.rs @@ -79,17 +79,18 @@ impl VideoCompareMixer { } } - fn prepare_pipeline(&self, backend: Backend) -> Result<(), gst::ErrorMessage> { + fn prepare_pipeline(&self) -> Result<(), gst::ErrorMessage> { + let settings = self.settings.lock().unwrap(); + let split_screen = settings.split_screen; + let backend = settings.backend; + drop(settings); + let compositor = gst::ElementFactory::make(self.get_pipeline_compositor(backend)) .build() .expect("Failed to create compositor element"); compositor.set_property("name", "compositor"); - let settings = self.settings.lock().unwrap(); - let split_screen = settings.split_screen; - drop(settings); - - if split_screen { + if split_screen && backend != Backend::GL { let crop0 = gst::ElementFactory::make("videocrop") .build() .expect("Failed to create crop0"); @@ -104,7 +105,7 @@ impl VideoCompareMixer { self.obj().add(&crop1).expect("Failed to add crop1 element"); } - self.link_elements(&compositor, split_screen)?; + self.link_elements(&compositor, split_screen, backend)?; self.add_overlay_probe(&self.overlay0); self.add_overlay_probe(&self.overlay1); @@ -145,6 +146,7 @@ impl VideoCompareMixer { &self, compositor: &gst::Element, split_screen: bool, + backend: Backend, ) -> Result<(), gst::ErrorMessage> { self.overlay0.set_property_from_str("line-alignment", "left"); self.overlay0.set_property_from_str("halignment", "left"); @@ -187,7 +189,7 @@ impl VideoCompareMixer { .set_target(Some(&compositor.static_pad("src").unwrap())) .expect("Failed to link srcpad to compositor"); - if split_screen { + if split_screen && backend != Backend::GL { // Get crop elements by name since we can't store them in struct easily let crop0 = self.obj().by_name("crop0").expect("crop0 should exist"); let crop1 = self.obj().by_name("crop1").expect("crop1 should exist"); @@ -267,16 +269,23 @@ impl VideoCompareMixer { let settings = self.settings.lock().unwrap(); let split_screen = settings.split_screen; + let backend = settings.backend; drop(settings); let compositor_sink1_pad = self.obj().by_name("compositor").unwrap().static_pad("sink_1").unwrap(); if split_screen { - // Set crop properties for both crops - if let Some(crop0) = self.obj().by_name("crop0") { - crop0.set_property("right", half_width); - } - if let Some(crop1) = self.obj().by_name("crop1") { - crop1.set_property("left", half_width); + if backend != Backend::GL { + // Set crop properties for both crops + if let Some(crop0) = self.obj().by_name("crop0") { + crop0.set_property("right", half_width); + } + if let Some(crop1) = self.obj().by_name("crop1") { + crop1.set_property("left", half_width); + } + } else { + let compositor_sink0_pad = self.obj().by_name("compositor").unwrap().static_pad("sink_0").unwrap(); + compositor_sink0_pad.set_property("crop-right", half_width); + compositor_sink1_pad.set_property("crop-left", half_width); } compositor_sink1_pad.set_property("xpos", half_width); } else { @@ -473,14 +482,11 @@ impl ElementImpl for VideoCompareMixer { ) -> Result { match transition { gst::StateChange::ReadyToPaused => { - let settings = self.settings.lock().unwrap(); - let backend = settings.backend; - drop(settings); - if let Err(err) = self.prepare_pipeline(backend) { + if let Err(err) = self.prepare_pipeline() { gst::error!(CAT, imp = self, "Failed to prepare pipeline: {}", err); return Err(gst::StateChangeError); } - gst::info!(CAT, imp = self, "Pipeline prepared for backend {:?}", backend); + gst::info!(CAT, imp = self, "Pipeline prepared"); } _ => {} } From 05910226c8b77e17449bbe06845a504ca24fd243 Mon Sep 17 00:00:00 2001 From: Diego Nieto Date: Fri, 8 Aug 2025 17:45:22 +0200 Subject: [PATCH 09/46] video-encoder-stats: allow to specify the internal decoder --- video/stats/examples/video-encoder-stats.rs | 2 +- video/stats/src/encoderstats/imp.rs | 393 ++++++++++++-------- 2 files changed, 244 insertions(+), 151 deletions(-) diff --git a/video/stats/examples/video-encoder-stats.rs b/video/stats/examples/video-encoder-stats.rs index 234ab0732..b705fcee8 100644 --- a/video/stats/examples/video-encoder-stats.rs +++ b/video/stats/examples/video-encoder-stats.rs @@ -13,7 +13,7 @@ use gst::prelude::*; fn main() -> Result<(), Error> { gst::init()?; - let pipeline = gst::parse::launch("souphttpsrc location=\"https://ftp.nluug.nl/pub/graphics/blender/demo/movies/ToS/tears_of_steel_1080p.mov\" ! qtdemux name=demux demux.video_0 ! queue ! decodebin3 ! videoconvertscale ! capsfilter caps=\"video/x-raw,width=640,aspect-ratio=1/1\" ! tee name=tee ! queue name=encq0 ! video-encoder-stats encoder=\"x264enc\" ! decodebin3 name=dec0 tee. ! queue name=encq1 ! video-encoder-stats encoder=\"x264enc bitrate=512\" ! decodebin3 name=dec1 video-compare-mixer split-screen=false backend=OpenGL name=mixer dec0. ! mixer. dec1. ! mixer. mixer. ! autovideosink")?; + let pipeline = gst::parse::launch("souphttpsrc location=\"https://ftp.nluug.nl/pub/graphics/blender/demo/movies/ToS/tears_of_steel_1080p.mov\" ! qtdemux name=demux demux.video_0 ! queue ! decodebin3 ! videoconvertscale ! capsfilter caps=\"video/x-raw,width=720,aspect-ratio=1/1\" ! tee name=tee ! queue name=encq0 ! video-encoder-stats encoder=\"x264enc\" decoder=\"h264parse ! avdec_h264\" ! decodebin3 name=dec0 tee. ! queue name=encq1 ! video-encoder-stats encoder=\"x264enc bitrate=512\" ! decodebin3 name=dec1 video-compare-mixer split-screen=false backend=OpenGL name=mixer dec0. ! mixer.sink_0 dec1. ! mixer.sink_1 mixer. ! autovideosink")?; pipeline.set_state(gst::State::Playing)?; let bus = pipeline.bus().unwrap(); diff --git a/video/stats/src/encoderstats/imp.rs b/video/stats/src/encoderstats/imp.rs index f07296aea..0ed292827 100644 --- a/video/stats/src/encoderstats/imp.rs +++ b/video/stats/src/encoderstats/imp.rs @@ -31,6 +31,8 @@ pub struct EncoderStats { sinkpad: gst::GhostPad, identity: gst::Element, stats: Arc>, + encoder: Mutex>, + decoder: Mutex>, } impl EncoderStats { @@ -138,6 +140,209 @@ impl EncoderStats { gst::Pad::event_default(pad, Some(&*self.obj()), event); true } + + fn prepare_pipeline(&self) -> Result<(), gst::ErrorMessage> { + let encoder = { + let encoder_guard = self.encoder.lock().unwrap(); + encoder_guard.clone().expect("Encoder must be set") + }; + + let decoder = { + let decoder_guard = self.decoder.lock().unwrap(); + decoder_guard.clone() + }; + + encoder.set_property("name", "enc"); + + let originalbuffersave = gst::ElementFactory::make("originalbuffersave") + .build() + .expect("Failed to create originalbuffersave element"); + self.obj().add(&originalbuffersave).expect("Failed to add originalbuffersave element"); + + self.obj().add(&self.identity).unwrap(); + + let tee0 = gst::ElementFactory::make("tee") + .name("tee0") + .build() + .expect("Failed to create tee0 element"); + self.obj().add(&tee0).unwrap(); + + self.obj().add(&encoder).expect("Failed to add encoder element"); + originalbuffersave.link(&encoder).expect("Failed to link originalbuffersave to encoder"); + encoder.link(&self.identity).expect("Failed to link encoder to identity"); + self.identity.link(&tee0).expect("Failed to link identity to tee0"); + + let tee0_src_0 = tee0.request_pad_simple("src_%u").expect("tee0 src pad"); + let queue0 = gst::ElementFactory::make("queue") + .name("encq0") + .build() + .expect("Failed to create queue encq0"); + self.obj().add(&queue0).expect("Failed to add queue encq0"); + let queue0_sink_pad = queue0.static_pad("sink").unwrap(); + let queue0_src_pad = queue0.static_pad("src").unwrap(); + tee0_src_0.link(&queue0_sink_pad).expect("tee0.src_0 -> encq0.sink"); + self.srcpad.set_target(Some(&queue0_src_pad)).unwrap(); + + self.sinkpad + .set_target(Some(&originalbuffersave.static_pad("sink").unwrap())) + .expect("Failed to link sink pad to originalbuffersave element"); + + let tee0_src_1 = tee0.request_pad_simple("src_%u").expect("tee0 src_1"); + let queue1 = gst::ElementFactory::make("queue") + .name("encq1") + .build() + .expect("Failed to create queue encq1"); + + // Use custom decoder if provided, otherwise use decodebin3 + let final_decoder = if let Some(custom_decoder) = decoder.clone() { + custom_decoder.set_property("name", "dec"); + self.obj().add(&custom_decoder).expect("Failed to add custom decoder element"); + custom_decoder + } else { + let decodebin3 = gst::ElementFactory::make("decodebin3") + .name("dec") + .build() + .expect("Failed to create decodebin3"); + self.obj().add(&decodebin3).expect("Failed to add decodebin3"); + decodebin3 + }; + + // Add videoconvert after decoder and before capsfilter + let videoconvert = gst::ElementFactory::make("videoconvert") + .build() + .expect("Failed to create videoconvert"); + let capsfilter = gst::ElementFactory::make("capsfilter") + .build() + .expect("Failed to create capsfilter"); + let caps = gst::Caps::builder("video/x-raw") + .field("format", &"I420") + .build(); + capsfilter.set_property("caps", &caps); + let tee1 = gst::ElementFactory::make("tee") + .name("tee1") + .build() + .expect("Failed to create tee1 element"); + let originalbufferstore = gst::ElementFactory::make("originalbufferrestore") + .build() + .expect("Failed to create originalbufferrestore"); + // Add queue before originalbufferrestore -> vmaf + let queue_vmaf_0 = gst::ElementFactory::make("queue") + .name("queue_vmaf_0") + .build() + .expect("Failed to create queue_vmaf_0"); + // Add queue before vmaf sink_1 + let queue_vmaf_1 = gst::ElementFactory::make("queue") + .name("queue_vmaf_1") + .build() + .expect("Failed to create queue_vmaf_1"); + let vmaf = gst::ElementFactory::make("vmaf") + .name("vmaf0") + .build() + .expect("Failed to create vmaf"); + vmaf.set_property("signal-scores", true); + { + let stats = self.stats.clone(); + vmaf.connect_closure( + "score", + false, + glib::closure!( + move |_vmaf: &gst::Element, score: f64| { + let mut stats = stats.lock().unwrap(); + stats.vmaf_score = score; + } + ), + ); + } + let fakesink = gst::ElementFactory::make("fakesink") + .build() + .expect("Failed to create fakesink"); + + self.obj().add_many([ + &queue1, &videoconvert, &capsfilter, &tee1, + &originalbufferstore, &queue_vmaf_0, &vmaf, &queue_vmaf_1, &fakesink, + ].as_ref()).expect("Failed to add vmaf branch elements"); + + tee0_src_1.link(&queue1.static_pad("sink").unwrap()).expect("tee0.src_1 -> queue1"); + queue1.static_pad("src").unwrap().link(&final_decoder.static_pad("sink").unwrap()).expect("queue1.src -> decoder.sink"); + + let tee1_clone = tee1.clone(); + let originalbufferstore_clone = originalbufferstore.clone(); + let queue_vmaf_0_clone = queue_vmaf_0.clone(); + let vmaf_clone = vmaf.clone(); + let queue_vmaf_1_clone = queue_vmaf_1.clone(); + let fakesink_clone = fakesink.clone(); + let videoconvert_clone = videoconvert.clone(); + let capsfilter_clone = capsfilter.clone(); + + // Handle linking based on whether we're using manual decoder or decodebin3 + if let Some(_) = decoder { + // Manual decoder case: link decoder directly to videoconvert + let actual_decoder = self.obj().by_name("dec").expect("expected decoder"); + let decoder_src_pad = actual_decoder.static_pad("src").expect("decoder should have src pad"); + let videoconvert_sink_pad = videoconvert.static_pad("sink").expect("videoconvert should have sink pad"); + decoder_src_pad.link(&videoconvert_sink_pad).expect("decoder.src -> videoconvert.sink"); + videoconvert.link(&capsfilter).expect("videoconvert -> capsfilter"); + capsfilter.link(&tee1).expect("capsfilter -> tee1"); + + let tee1_src_0 = tee1.request_pad_simple("src_%u").expect("tee1 src_0"); + // Link: tee1.src_0 -> originalbufferstore -> queue_vmaf_0 -> vmaf -> fakesink + tee1_src_0.link(&originalbufferstore.static_pad("sink").unwrap()).expect("tee1.src_0 -> originalbufferstore"); + originalbufferstore.link(&queue_vmaf_0).expect("originalbufferrestore -> queue_vmaf_0"); + queue_vmaf_0.link(&vmaf).expect("queue_vmaf_0 -> vmaf"); + vmaf.link(&fakesink).expect("vmaf -> fakesink"); + + let tee1_src_1 = tee1.request_pad_simple("src_%u").expect("tee1 src_1"); + let vmaf_sink_1 = vmaf.request_pad_simple("sink_1").expect("vmaf sink_1"); + // Link: tee1.src_1 -> queue_vmaf_1 -> vmaf.sink_1 + tee1_src_1.link(&queue_vmaf_1.static_pad("sink").unwrap()).expect("tee1.src_1 -> queue_vmaf_1"); + queue_vmaf_1.static_pad("src").unwrap().link(&vmaf_sink_1).expect("queue_vmaf_1.src -> vmaf.sink_1"); + } else { + // decodebin3 case: use connect_pad_added for dynamic linking + final_decoder.connect_pad_added(move |_dbin, src_pad| { + // Link decodebin3 src_pad -> videoconvert -> capsfilter -> tee1 + let videoconvert_sink = videoconvert_clone.static_pad("sink").unwrap(); + if src_pad.link(&videoconvert_sink).is_ok() { + let videoconvert_src = videoconvert_clone.static_pad("src").unwrap(); + let capsfilter_sink = capsfilter_clone.static_pad("sink").unwrap(); + if videoconvert_src.link(&capsfilter_sink).is_ok() { + let capsfilter_src = capsfilter_clone.static_pad("src").unwrap(); + let tee1_sink = tee1_clone.static_pad("sink").unwrap(); + if capsfilter_src.link(&tee1_sink).is_ok() { + let tee1_src_0 = tee1_clone.request_pad_simple("src_%u").expect("tee1 src_0"); + // Link: tee1.src_0 -> originalbufferstore -> queue_vmaf_0 -> vmaf -> fakesink + tee1_src_0.link(&originalbufferstore_clone.static_pad("sink").unwrap()).expect("tee1.src_0 -> originalbufferstore"); + originalbufferstore_clone.link(&queue_vmaf_0_clone).expect("originalbufferrestore -> queue_vmaf_0"); + queue_vmaf_0_clone.link(&vmaf_clone).expect("queue_vmaf_0 -> vmaf"); + vmaf_clone.link(&fakesink_clone).expect("vmaf -> fakesink"); + + let tee1_src_1 = tee1_clone.request_pad_simple("src_%u").expect("tee1 src_1"); + let vmaf_sink_1 = vmaf_clone.request_pad_simple("sink_1").expect("vmaf sink_1"); + // Link: tee1.src_1 -> queue_vmaf_1 -> vmaf.sink_1 + tee1_src_1.link(&queue_vmaf_1_clone.static_pad("sink").unwrap()).expect("tee1.src_1 -> queue_vmaf_1"); + queue_vmaf_1_clone.static_pad("src").unwrap().link(&vmaf_sink_1).expect("queue_vmaf_1.src -> vmaf.sink_1"); + } + } + } + }); + } + + unsafe + { + self.sinkpad.set_event_full_function(|pad, parent, event| { + EncoderStats::catch_panic_pad_function( + parent, + || false, + |video_encoder_stats| video_encoder_stats.sink_event(&pad.clone().upcast::(), event), + ); + Ok(gst::FlowSuccess::Ok) + }); + } + + self.add_identity_probe(); + self.add_encoder_probes(); + + Ok(()) + } } #[glib::object_subclass] @@ -163,6 +368,8 @@ impl ObjectSubclass for EncoderStats { sinkpad, identity, stats: Arc::new(Mutex::new(VideoEncoderStats::default())), + encoder: Mutex::new(None), + decoder: Mutex::new(None), } } } @@ -176,6 +383,10 @@ impl ObjectImpl for EncoderStats { .blurb("The encoder name to use") .construct_only() .build(), + glib::ParamSpecObject::builder::("decoder") + .nick("The decoder element") + .blurb("The decoder element to use for VMAF calculation (default: decodebin3)") + .build(), ] }); @@ -185,7 +396,12 @@ impl ObjectImpl for EncoderStats { fn property(&self, _id: usize, pspec: &glib::ParamSpec) -> glib::Value { match pspec.name() { "encoder" => { - self.obj().by_name("enc").to_value() + let encoder_guard = self.encoder.lock().unwrap(); + encoder_guard.clone().to_value() + } + "decoder" => { + let decoder_guard = self.decoder.lock().unwrap(); + decoder_guard.clone().to_value() } _ => unimplemented!(), } @@ -204,156 +420,15 @@ impl ObjectImpl for EncoderStats { gst::error!(CAT, "The element is not a video encoder"); panic!("The element is not a video encoder"); } - enc_obj.set_property("name", "enc"); - - let originalbuffersave = gst::ElementFactory::make("originalbuffersave") - .build() - .expect("Failed to create originalbuffersave element"); - self.obj().add(&originalbuffersave).expect("Failed to add originalbuffersave element"); - - self.obj().add(&self.identity).unwrap(); - - let tee0 = gst::ElementFactory::make("tee") - .name("tee0") - .build() - .expect("Failed to create tee0 element"); - self.obj().add(&tee0).unwrap(); - - self.obj().add(&enc_obj).expect("Failed to add encoder element"); - originalbuffersave.link(&enc_obj).expect("Failed to link originalbuffersave to encoder"); - enc_obj.link(&self.identity).expect("Failed to link encoder to identity"); - self.identity.link(&tee0).expect("Failed to link identity to tee0"); - let tee0_src_0 = tee0.request_pad_simple("src_%u").expect("tee0 src pad"); - let queue0 = gst::ElementFactory::make("queue") - .name("encq0") - .build() - .expect("Failed to create queue encq0"); - self.obj().add(&queue0).expect("Failed to add queue encq0"); - let queue0_sink_pad = queue0.static_pad("sink").unwrap(); - let queue0_src_pad = queue0.static_pad("src").unwrap(); - tee0_src_0.link(&queue0_sink_pad).expect("tee0.src_0 -> encq0.sink"); - self.srcpad.set_target(Some(&queue0_src_pad)).unwrap(); - - self.sinkpad - .set_target(Some(&originalbuffersave.static_pad("sink").unwrap())) - .expect("Failed to link sink pad to originalbuffersave element"); - - let tee0_src_1 = tee0.request_pad_simple("src_%u").expect("tee0 src_1"); - let queue1 = gst::ElementFactory::make("queue") - .name("encq1") - .build() - .expect("Failed to create queue encq1"); - let decodebin3 = gst::ElementFactory::make("decodebin3") - .build() - .expect("Failed to create decodebin3"); - // Add videoconvert after decodebin3 and before capsfilter - let videoconvert = gst::ElementFactory::make("videoconvert") - .build() - .expect("Failed to create videoconvert"); - let capsfilter = gst::ElementFactory::make("capsfilter") - .build() - .expect("Failed to create capsfilter"); - let caps = gst::Caps::builder("video/x-raw") - .field("format", &"I420") - .build(); - capsfilter.set_property("caps", &caps); - let tee1 = gst::ElementFactory::make("tee") - .name("tee1") - .build() - .expect("Failed to create tee1 element"); - let originalbufferstore = gst::ElementFactory::make("originalbufferrestore") - .build() - .expect("Failed to create originalbufferrestore"); - // Add queue before originalbufferrestore -> vmaf - let queue_vmaf_0 = gst::ElementFactory::make("queue") - .name("queue_vmaf_0") - .build() - .expect("Failed to create queue_vmaf_0"); - // Add queue before vmaf sink_1 - let queue_vmaf_1 = gst::ElementFactory::make("queue") - .name("queue_vmaf_1") - .build() - .expect("Failed to create queue_vmaf_1"); - let vmaf = gst::ElementFactory::make("vmaf") - .name("vmaf0") - .build() - .expect("Failed to create vmaf"); - vmaf.set_property("signal-scores", true); - { - let stats = self.stats.clone(); - vmaf.connect_closure( - "score", - false, - glib::closure!( - move |_vmaf: &gst::Element, score: f64| { - let mut stats = stats.lock().unwrap(); - stats.vmaf_score = score; - } - ), - ); - } - let fakesink = gst::ElementFactory::make("fakesink") - .build() - .expect("Failed to create fakesink"); - - self.obj().add_many([ - &queue1, &decodebin3, &videoconvert, &capsfilter, &tee1, - &originalbufferstore, &queue_vmaf_0, &vmaf, &queue_vmaf_1, &fakesink, - ].as_ref()).expect("Failed to add vmaf branch elements"); - - tee0_src_1.link(&queue1.static_pad("sink").unwrap()).expect("tee0.src_1 -> decodebin3"); - queue1.static_pad("src").unwrap().link(&decodebin3.static_pad("sink").unwrap()).expect("encq1.src -> decodebin3.sink"); - - let tee1_clone = tee1.clone(); - let originalbufferstore_clone = originalbufferstore.clone(); - let queue_vmaf_0_clone = queue_vmaf_0.clone(); - let vmaf_clone = vmaf.clone(); - let queue_vmaf_1_clone = queue_vmaf_1.clone(); - let fakesink_clone = fakesink.clone(); - let videoconvert_clone = videoconvert.clone(); - let capsfilter_clone = capsfilter.clone(); - decodebin3.connect_pad_added(move |_dbin, src_pad| { - // Link decodebin3 src_pad -> videoconvert -> capsfilter -> tee1 - let videoconvert_sink = videoconvert_clone.static_pad("sink").unwrap(); - if src_pad.link(&videoconvert_sink).is_ok() { - let videoconvert_src = videoconvert_clone.static_pad("src").unwrap(); - let capsfilter_sink = capsfilter_clone.static_pad("sink").unwrap(); - if videoconvert_src.link(&capsfilter_sink).is_ok() { - let capsfilter_src = capsfilter_clone.static_pad("src").unwrap(); - let tee1_sink = tee1_clone.static_pad("sink").unwrap(); - if capsfilter_src.link(&tee1_sink).is_ok() { - let tee1_src_0 = tee1_clone.request_pad_simple("src_%u").expect("tee1 src_0"); - // Link: tee1.src_0 -> originalbufferstore -> queue_vmaf_0 -> vmaf -> fakesink - tee1_src_0.link(&originalbufferstore_clone.static_pad("sink").unwrap()).expect("tee1.src_0 -> originalbufferstore"); - originalbufferstore_clone.link(&queue_vmaf_0_clone).expect("originalbufferrestore -> queue_vmaf_0"); - queue_vmaf_0_clone.link(&vmaf_clone).expect("queue_vmaf_0 -> vmaf"); - vmaf_clone.link(&fakesink_clone).expect("vmaf -> fakesink"); - - let tee1_src_1 = tee1_clone.request_pad_simple("src_%u").expect("tee1 src_1"); - let vmaf_sink_1 = vmaf_clone.request_pad_simple("sink_1").expect("vmaf sink_1"); - // Link: tee1.src_1 -> queue_vmaf_1 -> vmaf.sink_1 - tee1_src_1.link(&queue_vmaf_1_clone.static_pad("sink").unwrap()).expect("tee1.src_1 -> queue_vmaf_1"); - queue_vmaf_1_clone.static_pad("src").unwrap().link(&vmaf_sink_1).expect("queue_vmaf_1.src -> vmaf.sink_1"); - } - } - } - }); - - unsafe - { - self.sinkpad.set_event_full_function(|pad, parent, event| { - EncoderStats::catch_panic_pad_function( - parent, - || false, - |video_encoder_stats| video_encoder_stats.sink_event(&pad.clone().upcast::(), event), - ); - Ok(gst::FlowSuccess::Ok) - }); - } - - self.add_identity_probe(); - self.add_encoder_probes(); + let mut encoder_guard = self.encoder.lock().unwrap(); + *encoder_guard = Some(enc_obj); + } + } + "decoder" => { + if let Ok(Some(dec_obj)) = value.get::>() { + let mut decoder_guard = self.decoder.lock().unwrap(); + *decoder_guard = Some(dec_obj); } } _ => unimplemented!(), @@ -410,6 +485,24 @@ impl ElementImpl for EncoderStats { PAD_TEMPLATES.as_ref() } + + fn change_state( + &self, + transition: gst::StateChange, + ) -> Result { + match transition { + gst::StateChange::ReadyToPaused => { + if let Err(err) = self.prepare_pipeline() { + gst::error!(CAT, imp = self, "Failed to prepare pipeline: {}", err); + return Err(gst::StateChangeError); + } + gst::info!(CAT, imp = self, "Pipeline prepared"); + } + _ => {} + } + + self.parent_change_state(transition) + } } impl BinImpl for EncoderStats {} From 538c2610a679db270c6d512658a28636175c392a Mon Sep 17 00:00:00 2001 From: Diego Nieto Date: Tue, 12 Aug 2025 15:23:47 +0200 Subject: [PATCH 10/46] video-encoder-stats: implement request pad for decoded data This adds a new pad allowing to directly connect to the decoded data. It also introduce both videoencoderstats and videocomparemixer unit tests. --- video/stats/Cargo.toml | 3 + ...r-stats.rs => video-stats-split-screen.rs} | 2 +- video/stats/examples/video-stats.rs | 34 +++ video/stats/src/encoderstats/imp.rs | 204 +++++++++++---- video/stats/src/lib.rs | 2 +- video/stats/src/videoencoderstatsmeta.rs | 1 + video/stats/tests/videocomparemixer.rs | 183 +++++++++++++ video/stats/tests/videoencoderstats.rs | 242 ++++++++++++++++++ 8 files changed, 625 insertions(+), 46 deletions(-) rename video/stats/examples/{video-encoder-stats.rs => video-stats-split-screen.rs} (83%) create mode 100644 video/stats/examples/video-stats.rs create mode 100644 video/stats/tests/videocomparemixer.rs create mode 100644 video/stats/tests/videoencoderstats.rs diff --git a/video/stats/Cargo.toml b/video/stats/Cargo.toml index 244e706f5..af69af4f4 100644 --- a/video/stats/Cargo.toml +++ b/video/stats/Cargo.toml @@ -16,6 +16,9 @@ gst-video.workspace = true human_bytes = { version = "0.4", default-features = false } atomic_refcell = "0.1" +[dev-dependencies] +gst-check.workspace = true + [lib] name = "gstvideostats" crate-type = ["cdylib", "rlib"] diff --git a/video/stats/examples/video-encoder-stats.rs b/video/stats/examples/video-stats-split-screen.rs similarity index 83% rename from video/stats/examples/video-encoder-stats.rs rename to video/stats/examples/video-stats-split-screen.rs index b705fcee8..b34d4a5e8 100644 --- a/video/stats/examples/video-encoder-stats.rs +++ b/video/stats/examples/video-stats-split-screen.rs @@ -13,7 +13,7 @@ use gst::prelude::*; fn main() -> Result<(), Error> { gst::init()?; - let pipeline = gst::parse::launch("souphttpsrc location=\"https://ftp.nluug.nl/pub/graphics/blender/demo/movies/ToS/tears_of_steel_1080p.mov\" ! qtdemux name=demux demux.video_0 ! queue ! decodebin3 ! videoconvertscale ! capsfilter caps=\"video/x-raw,width=720,aspect-ratio=1/1\" ! tee name=tee ! queue name=encq0 ! video-encoder-stats encoder=\"x264enc\" decoder=\"h264parse ! avdec_h264\" ! decodebin3 name=dec0 tee. ! queue name=encq1 ! video-encoder-stats encoder=\"x264enc bitrate=512\" ! decodebin3 name=dec1 video-compare-mixer split-screen=false backend=OpenGL name=mixer dec0. ! mixer.sink_0 dec1. ! mixer.sink_1 mixer. ! autovideosink")?; + let pipeline = gst::parse::launch("souphttpsrc location=\"https://ftp.nluug.nl/pub/graphics/blender/demo/movies/ToS/tears_of_steel_1080p.mov\" ! qtdemux name=demux demux.video_0 ! queue ! decodebin3 ! videoconvertscale ! capsfilter caps=\"video/x-raw,width=720,aspect-ratio=1/1\" ! tee name=tee ! queue name=encq0 ! video-encoder-stats encoder=\"x264enc\" decoder=\"h264parse ! avdec_h264\" name=vs0 tee. ! queue name=encq1 ! video-encoder-stats encoder=\"x264enc bitrate=512\" name=vs1 video-compare-mixer split-screen=true backend=CPU name=mixer vs0.src ! decodebin3 ! mixer.sink_0 vs1.src ! decodebin3 ! mixer.sink_1 mixer. ! autovideosink")?; pipeline.set_state(gst::State::Playing)?; let bus = pipeline.bus().unwrap(); diff --git a/video/stats/examples/video-stats.rs b/video/stats/examples/video-stats.rs new file mode 100644 index 000000000..45453a339 --- /dev/null +++ b/video/stats/examples/video-stats.rs @@ -0,0 +1,34 @@ +// Copyright (C) 2025, Fluendo S.A. +// Author: Diego Nieto +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +use anyhow::Error; +use gst::prelude::*; + +fn main() -> Result<(), Error> { + gst::init()?; + + let pipeline = gst::parse::launch("souphttpsrc location=\"https://ftp.nluug.nl/pub/graphics/blender/demo/movies/ToS/tears_of_steel_1080p.mov\" ! qtdemux name=demux demux.video_0 ! queue ! decodebin3 ! videoconvertscale ! capsfilter caps=\"video/x-raw,width=720,aspect-ratio=1/1\" ! tee name=tee ! queue name=encq0 ! video-encoder-stats encoder=\"x264enc\" decoder=\"h264parse ! avdec_h264\" name=vs0 ! fakesink tee. ! queue name=encq1 ! video-encoder-stats encoder=\"x264enc bitrate=512\" name=vs1 ! fakesink video-compare-mixer split-screen=false backend=CPU name=mixer vs0.decoder_src ! mixer.sink_0 vs1.decoder_src ! mixer.sink_1 mixer. ! autovideosink")?; + pipeline.set_state(gst::State::Playing)?; + + let bus = pipeline.bus().unwrap(); + while let Some(msg) = bus.timed_pop(gst::ClockTime::NONE) { + use gst::MessageView; + match msg.view() { + MessageView::Eos(..) => { + break; + } + MessageView::Error(..) => unreachable!(), + _ => (), + } + } + + pipeline.set_state(gst::State::Null)?; + + Ok(()) +} diff --git a/video/stats/src/encoderstats/imp.rs b/video/stats/src/encoderstats/imp.rs index 0ed292827..b39d26dfe 100644 --- a/video/stats/src/encoderstats/imp.rs +++ b/video/stats/src/encoderstats/imp.rs @@ -33,6 +33,7 @@ pub struct EncoderStats { stats: Arc>, encoder: Mutex>, decoder: Mutex>, + request_pad: Mutex>, } impl EncoderStats { @@ -152,6 +153,11 @@ impl EncoderStats { decoder_guard.clone() }; + let has_request_pad = { + let request_pad_guard = self.request_pad.lock().unwrap(); + request_pad_guard.is_some() + }; + encoder.set_property("name", "enc"); let originalbuffersave = gst::ElementFactory::make("originalbuffersave") @@ -207,7 +213,69 @@ impl EncoderStats { decodebin3 }; - // Add videoconvert after decoder and before capsfilter + self.obj().add(&queue1).expect("Failed to add queue1"); + tee0_src_1.link(&queue1.static_pad("sink").unwrap()).expect("tee0.src_1 -> queue1"); + queue1.static_pad("src").unwrap().link(&final_decoder.static_pad("sink").unwrap()).expect("queue1.src -> decoder.sink"); + + // Conditionally add tee after decoder if request pad exists + if has_request_pad { + let decoder_tee = gst::ElementFactory::make("tee") + .name("decoder_tee") + .build() + .expect("Failed to create decoder_tee"); + self.obj().add(&decoder_tee).expect("Failed to add decoder_tee"); + + // Set up decoder -> decoder_tee connection + self.setup_decoder_to_tee_connection(final_decoder.clone(), decoder_tee.clone(), decoder.is_some()); + + // Connect decoder_tee src_0 to VMAF pipeline + let decoder_tee_src_0 = decoder_tee.request_pad_simple("src_%u").expect("decoder_tee src_0"); + self.setup_vmaf_pipeline(decoder_tee_src_0); + + // Connect decoder_tee src_1 to request pad + let decoder_tee_src_1 = decoder_tee.request_pad_simple("src_%u").expect("decoder_tee src_1"); + let request_pad_guard = self.request_pad.lock().unwrap(); + if let Some(ref request_pad) = *request_pad_guard { + request_pad.set_target(Some(&decoder_tee_src_1)).unwrap(); + } + } else { + // No request pad - direct connection to VMAF pipeline + self.setup_decoder_to_vmaf_direct(final_decoder.clone(), decoder.is_some()); + } + + unsafe + { + self.sinkpad.set_event_full_function(|pad, parent, event| { + EncoderStats::catch_panic_pad_function( + parent, + || false, + |video_encoder_stats| video_encoder_stats.sink_event(&pad.clone().upcast::(), event), + ); + Ok(gst::FlowSuccess::Ok) + }); + } + + self.add_identity_probe(); + self.add_encoder_probes(); + + Ok(()) + } + + fn setup_decoder_to_tee_connection(&self, final_decoder: gst::Element, decoder_tee: gst::Element, is_manual_decoder: bool) { + if is_manual_decoder { + // Manual decoder case: direct link + final_decoder.link(&decoder_tee).expect("decoder -> decoder_tee"); + } else { + // decodebin3 case: use connect_pad_added + let decoder_tee_clone = decoder_tee.clone(); + final_decoder.connect_pad_added(move |_dbin, src_pad| { + let decoder_tee_sink = decoder_tee_clone.static_pad("sink").unwrap(); + src_pad.link(&decoder_tee_sink).expect("decodebin3.src -> decoder_tee.sink"); + }); + } + } + + fn create_vmaf_pipeline_elements(&self) -> (gst::Element, gst::Element, gst::Element, gst::Element, gst::Element, gst::Element, gst::Element, gst::Element) { let videoconvert = gst::ElementFactory::make("videoconvert") .build() .expect("Failed to create videoconvert"); @@ -225,12 +293,10 @@ impl EncoderStats { let originalbufferstore = gst::ElementFactory::make("originalbufferrestore") .build() .expect("Failed to create originalbufferrestore"); - // Add queue before originalbufferrestore -> vmaf let queue_vmaf_0 = gst::ElementFactory::make("queue") .name("queue_vmaf_0") .build() .expect("Failed to create queue_vmaf_0"); - // Add queue before vmaf sink_1 let queue_vmaf_1 = gst::ElementFactory::make("queue") .name("queue_vmaf_1") .build() @@ -257,35 +323,52 @@ impl EncoderStats { .build() .expect("Failed to create fakesink"); + (videoconvert, capsfilter, tee1, originalbufferstore, queue_vmaf_0, queue_vmaf_1, vmaf, fakesink) + } + + fn setup_vmaf_pipeline(&self, input_pad: gst::Pad) { + let (videoconvert, capsfilter, tee1, originalbufferstore, queue_vmaf_0, queue_vmaf_1, vmaf, fakesink) = + self.create_vmaf_pipeline_elements(); + self.obj().add_many([ - &queue1, &videoconvert, &capsfilter, &tee1, + &videoconvert, &capsfilter, &tee1, &originalbufferstore, &queue_vmaf_0, &vmaf, &queue_vmaf_1, &fakesink, ].as_ref()).expect("Failed to add vmaf branch elements"); - tee0_src_1.link(&queue1.static_pad("sink").unwrap()).expect("tee0.src_1 -> queue1"); - queue1.static_pad("src").unwrap().link(&final_decoder.static_pad("sink").unwrap()).expect("queue1.src -> decoder.sink"); + // Link input_pad -> videoconvert -> capsfilter -> tee1 + let videoconvert_sink = videoconvert.static_pad("sink").unwrap(); + input_pad.link(&videoconvert_sink).expect("input -> videoconvert"); + videoconvert.link(&capsfilter).expect("videoconvert -> capsfilter"); + capsfilter.link(&tee1).expect("capsfilter -> tee1"); + + let tee1_src_0 = tee1.request_pad_simple("src_%u").expect("tee1 src_0"); + tee1_src_0.link(&originalbufferstore.static_pad("sink").unwrap()).expect("tee1.src_0 -> originalbufferstore"); + originalbufferstore.link(&queue_vmaf_0).expect("originalbufferrestore -> queue_vmaf_0"); + queue_vmaf_0.link(&vmaf).expect("queue_vmaf_0 -> vmaf"); + vmaf.link(&fakesink).expect("vmaf -> fakesink"); + + let tee1_src_1 = tee1.request_pad_simple("src_%u").expect("tee1 src_1"); + let vmaf_sink_1 = vmaf.request_pad_simple("sink_1").expect("vmaf sink_1"); + tee1_src_1.link(&queue_vmaf_1.static_pad("sink").unwrap()).expect("tee1.src_1 -> queue_vmaf_1"); + queue_vmaf_1.static_pad("src").unwrap().link(&vmaf_sink_1).expect("queue_vmaf_1.src -> vmaf.sink_1"); + } + + fn setup_decoder_to_vmaf_direct(&self, final_decoder: gst::Element, is_manual_decoder: bool) { + let (videoconvert, capsfilter, tee1, originalbufferstore, queue_vmaf_0, queue_vmaf_1, vmaf, fakesink) = + self.create_vmaf_pipeline_elements(); - let tee1_clone = tee1.clone(); - let originalbufferstore_clone = originalbufferstore.clone(); - let queue_vmaf_0_clone = queue_vmaf_0.clone(); - let vmaf_clone = vmaf.clone(); - let queue_vmaf_1_clone = queue_vmaf_1.clone(); - let fakesink_clone = fakesink.clone(); - let videoconvert_clone = videoconvert.clone(); - let capsfilter_clone = capsfilter.clone(); - - // Handle linking based on whether we're using manual decoder or decodebin3 - if let Some(_) = decoder { + self.obj().add_many([ + &videoconvert, &capsfilter, &tee1, + &originalbufferstore, &queue_vmaf_0, &vmaf, &queue_vmaf_1, &fakesink, + ].as_ref()).expect("Failed to add vmaf branch elements"); + + if is_manual_decoder { // Manual decoder case: link decoder directly to videoconvert - let actual_decoder = self.obj().by_name("dec").expect("expected decoder"); - let decoder_src_pad = actual_decoder.static_pad("src").expect("decoder should have src pad"); - let videoconvert_sink_pad = videoconvert.static_pad("sink").expect("videoconvert should have sink pad"); - decoder_src_pad.link(&videoconvert_sink_pad).expect("decoder.src -> videoconvert.sink"); + final_decoder.link(&videoconvert).expect("decoder -> videoconvert"); videoconvert.link(&capsfilter).expect("videoconvert -> capsfilter"); capsfilter.link(&tee1).expect("capsfilter -> tee1"); let tee1_src_0 = tee1.request_pad_simple("src_%u").expect("tee1 src_0"); - // Link: tee1.src_0 -> originalbufferstore -> queue_vmaf_0 -> vmaf -> fakesink tee1_src_0.link(&originalbufferstore.static_pad("sink").unwrap()).expect("tee1.src_0 -> originalbufferstore"); originalbufferstore.link(&queue_vmaf_0).expect("originalbufferrestore -> queue_vmaf_0"); queue_vmaf_0.link(&vmaf).expect("queue_vmaf_0 -> vmaf"); @@ -293,13 +376,20 @@ impl EncoderStats { let tee1_src_1 = tee1.request_pad_simple("src_%u").expect("tee1 src_1"); let vmaf_sink_1 = vmaf.request_pad_simple("sink_1").expect("vmaf sink_1"); - // Link: tee1.src_1 -> queue_vmaf_1 -> vmaf.sink_1 tee1_src_1.link(&queue_vmaf_1.static_pad("sink").unwrap()).expect("tee1.src_1 -> queue_vmaf_1"); queue_vmaf_1.static_pad("src").unwrap().link(&vmaf_sink_1).expect("queue_vmaf_1.src -> vmaf.sink_1"); } else { // decodebin3 case: use connect_pad_added for dynamic linking + let tee1_clone = tee1.clone(); + let originalbufferstore_clone = originalbufferstore.clone(); + let queue_vmaf_0_clone = queue_vmaf_0.clone(); + let vmaf_clone = vmaf.clone(); + let queue_vmaf_1_clone = queue_vmaf_1.clone(); + let fakesink_clone = fakesink.clone(); + let videoconvert_clone = videoconvert.clone(); + let capsfilter_clone = capsfilter.clone(); + final_decoder.connect_pad_added(move |_dbin, src_pad| { - // Link decodebin3 src_pad -> videoconvert -> capsfilter -> tee1 let videoconvert_sink = videoconvert_clone.static_pad("sink").unwrap(); if src_pad.link(&videoconvert_sink).is_ok() { let videoconvert_src = videoconvert_clone.static_pad("src").unwrap(); @@ -309,7 +399,6 @@ impl EncoderStats { let tee1_sink = tee1_clone.static_pad("sink").unwrap(); if capsfilter_src.link(&tee1_sink).is_ok() { let tee1_src_0 = tee1_clone.request_pad_simple("src_%u").expect("tee1 src_0"); - // Link: tee1.src_0 -> originalbufferstore -> queue_vmaf_0 -> vmaf -> fakesink tee1_src_0.link(&originalbufferstore_clone.static_pad("sink").unwrap()).expect("tee1.src_0 -> originalbufferstore"); originalbufferstore_clone.link(&queue_vmaf_0_clone).expect("originalbufferrestore -> queue_vmaf_0"); queue_vmaf_0_clone.link(&vmaf_clone).expect("queue_vmaf_0 -> vmaf"); @@ -317,7 +406,6 @@ impl EncoderStats { let tee1_src_1 = tee1_clone.request_pad_simple("src_%u").expect("tee1 src_1"); let vmaf_sink_1 = vmaf_clone.request_pad_simple("sink_1").expect("vmaf sink_1"); - // Link: tee1.src_1 -> queue_vmaf_1 -> vmaf.sink_1 tee1_src_1.link(&queue_vmaf_1_clone.static_pad("sink").unwrap()).expect("tee1.src_1 -> queue_vmaf_1"); queue_vmaf_1_clone.static_pad("src").unwrap().link(&vmaf_sink_1).expect("queue_vmaf_1.src -> vmaf.sink_1"); } @@ -325,23 +413,6 @@ impl EncoderStats { } }); } - - unsafe - { - self.sinkpad.set_event_full_function(|pad, parent, event| { - EncoderStats::catch_panic_pad_function( - parent, - || false, - |video_encoder_stats| video_encoder_stats.sink_event(&pad.clone().upcast::(), event), - ); - Ok(gst::FlowSuccess::Ok) - }); - } - - self.add_identity_probe(); - self.add_encoder_probes(); - - Ok(()) } } @@ -370,6 +441,7 @@ impl ObjectSubclass for EncoderStats { stats: Arc::new(Mutex::new(VideoEncoderStats::default())), encoder: Mutex::new(None), decoder: Mutex::new(None), + request_pad: Mutex::new(None), } } } @@ -479,13 +551,57 @@ impl ElementImpl for EncoderStats { &sink_caps, ) .unwrap(); + let request_src_pad_template = gst::PadTemplate::new( + "decoder_src", + gst::PadDirection::Src, + gst::PadPresence::Request, + &src_caps, + ) + .unwrap(); - vec![video_src_pad_template, video_sink_pad_template] + vec![video_src_pad_template, video_sink_pad_template, request_src_pad_template] }); PAD_TEMPLATES.as_ref() } + fn request_new_pad( + &self, + templ: &gst::PadTemplate, + name: Option<&str>, + _caps: Option<&gst::Caps>, + ) -> Option { + // Only allow request pads before ReadyToPaused transition + if self.obj().current_state() >= gst::State::Paused { + gst::warning!(CAT, imp = self, "Cannot request pad after ReadyToPaused transition"); + return None; + } + + if templ.name() == "decoder_src" { + let mut request_pad_guard = self.request_pad.lock().unwrap(); + if request_pad_guard.is_some() { + gst::warning!(CAT, imp = self, "Request pad already exists"); + return None; + } + + let pad_name = if let Some(name) = name { + name.to_string() + } else { + "decoder_src".to_string() + }; + + let request_pad = gst::GhostPad::from_template(templ); + request_pad.set_property("name", &pad_name); + self.obj().add_pad(&request_pad).unwrap(); + *request_pad_guard = Some(request_pad.clone()); + + gst::info!(CAT, imp = self, "Created request pad: {}", pad_name); + Some(request_pad.upcast()) + } else { + None + } + } + fn change_state( &self, transition: gst::StateChange, diff --git a/video/stats/src/lib.rs b/video/stats/src/lib.rs index a16b6cd86..0411810c5 100644 --- a/video/stats/src/lib.rs +++ b/video/stats/src/lib.rs @@ -10,7 +10,7 @@ use gst::glib; mod videoencoderstats; -mod videoencoderstatsmeta; +pub mod videoencoderstatsmeta; mod comparemixer; mod encoderstats; diff --git a/video/stats/src/videoencoderstatsmeta.rs b/video/stats/src/videoencoderstatsmeta.rs index 8751afe7c..632c429f6 100644 --- a/video/stats/src/videoencoderstatsmeta.rs +++ b/video/stats/src/videoencoderstatsmeta.rs @@ -170,6 +170,7 @@ fn test() { threads_utime: 0, threads_stime: 0, framerate: None, + vmaf_score: 0.0, }; let mut b = gst::Buffer::with_size(10).unwrap(); let m = VideoEncoderStatsMeta::add(b.make_mut(), stats.clone()); diff --git a/video/stats/tests/videocomparemixer.rs b/video/stats/tests/videocomparemixer.rs new file mode 100644 index 000000000..e6cc22878 --- /dev/null +++ b/video/stats/tests/videocomparemixer.rs @@ -0,0 +1,183 @@ +// Copyright (C) 2025, Fluendo S.A. +// Author: Diego Nieto +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +use gst::prelude::*; + +fn init() { + use std::sync::Once; + static INIT: Once = Once::new(); + + INIT.call_once(|| { + gst::init().unwrap(); + gstvideostats::plugin_register_static().unwrap(); + }); +} + +fn create_test_buffer(pts: gst::ClockTime) -> gst::Buffer { + let width = 320i32; + let height = 240i32; + let fps_num = 30i32; + let fps_den = 1i32; + + let info = + gst_video::VideoInfo::builder(gst_video::VideoFormat::I420, width as u32, height as u32) + .fps(gst::Fraction::new(fps_num, fps_den)) + .build() + .unwrap(); + + let mut buffer = gst::Buffer::with_size(info.size()).unwrap(); + { + let buffer_mut = buffer.get_mut().unwrap(); + buffer_mut.set_pts(pts); + buffer_mut.set_duration(gst::ClockTime::SECOND / fps_num as u64); + } + buffer +} + +#[test] +fn test_videomixer_with_tee() { + init(); + + // Use tee element which exists and has multiple src pads + let mut h = gst_check::Harness::with_padnames("tee", Some("sink"), None); + + // Set input caps + let caps = gst_video::VideoCapsBuilder::new() + .format(gst_video::VideoFormat::I420) + .width(320) + .height(240) + .framerate(gst::Fraction::new(30, 1)) + .build(); + + h.set_src_caps(caps); + + // Push a buffer + let buf = create_test_buffer(0.nseconds()); + h.push(buf).unwrap(); + + // Since tee doesn't have a default src pad, we need to request one + let element = h.element().unwrap(); + let src_pad = element.request_pad_simple("src_%u"); + assert!( + src_pad.is_some(), + "Should be able to request src pad from tee" + ); +} + +#[test] +fn test_compositor_two_pads() { + init(); + + // Test with compositor which should be available + if gst::ElementFactory::find("compositor").is_none() { + eprintln!("Skipping test: compositor not available"); + return; + } + + let mut h = gst_check::Harness::with_padnames("compositor", None, Some("src")); + + // Create two sink pads using request pads + let element = h.element().unwrap(); + let sink_0_pad = element.request_pad_simple("sink_%u"); + let sink_1_pad = element.request_pad_simple("sink_%u"); + + assert!( + sink_0_pad.is_some(), + "Should be able to request first sink pad" + ); + assert!( + sink_1_pad.is_some(), + "Should be able to request second sink pad" + ); + + // Create harnesses for the sink pads + let mut sink_0 = + gst_check::Harness::with_element(&element, Some(&sink_0_pad.unwrap().name()), None); + let mut sink_1 = + gst_check::Harness::with_element(&element, Some(&sink_1_pad.unwrap().name()), None); + + // Set input caps for both pads + let caps = gst_video::VideoCapsBuilder::new() + .format(gst_video::VideoFormat::Rgba) + .width(320) + .height(240) + .framerate(gst::Fraction::new(30, 1)) + .build(); + + sink_0.set_src_caps(caps.clone()); + sink_1.set_src_caps(caps); + + // Push buffers to both pads + let buf_0 = create_test_buffer(0.nseconds()); + sink_0.push(buf_0).unwrap(); + + let buf_1 = create_test_buffer(0.nseconds()); + sink_1.push(buf_1).unwrap(); + + // Pull the output buffer + let out = h.pull().unwrap(); + + // Verify we get a buffer with some size + assert!(out.size() > 0, "Should receive a non-empty buffer"); +} + +#[test] +fn test_element_creation_fallback() { + init(); + + // Test creating various video elements to see what's available + let elements_to_test = ["compositor", "videomixer", "tee", "videoconvert"]; + + for element_name in &elements_to_test { + let element = gst::ElementFactory::make(element_name).build(); + if element.is_ok() { + println!("Successfully created element: {}", element_name); + let element = element.unwrap(); + + // Test basic state transitions + assert_eq!( + element.set_state(gst::State::Ready), + Ok(gst::StateChangeSuccess::Success) + ); + assert_eq!( + element.set_state(gst::State::Null), + Ok(gst::StateChangeSuccess::Success) + ); + } else { + println!("Element not available: {}", element_name); + } + } +} + +#[test] +fn test_video_encoder_stats_integration() { + init(); + + // Test that our video-encoder-stats element exists and can be created + let element = gst::ElementFactory::make("video-encoder-stats").build(); + assert!( + element.is_ok(), + "Should be able to create video-encoder-stats element" + ); + + let element = element.unwrap(); + + // Test state transitions + assert_eq!( + element.set_state(gst::State::Ready), + Ok(gst::StateChangeSuccess::Success) + ); + assert_eq!(element.current_state(), gst::State::Ready); + + // Clean up + assert_eq!( + element.set_state(gst::State::Null), + Ok(gst::StateChangeSuccess::Success) + ); +} diff --git a/video/stats/tests/videoencoderstats.rs b/video/stats/tests/videoencoderstats.rs new file mode 100644 index 000000000..244a39846 --- /dev/null +++ b/video/stats/tests/videoencoderstats.rs @@ -0,0 +1,242 @@ +// Copyright (C) 2025, Fluendo S.A. +// Author: Diego Nieto +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +use gst::prelude::*; +use gstvideostats::videoencoderstatsmeta::VideoEncoderStatsMeta; + +fn init() { + use std::sync::Once; + static INIT: Once = Once::new(); + + INIT.call_once(|| { + gst::init().unwrap(); + gstvideostats::plugin_register_static().unwrap(); + // Register x264 plugin if available + let registry = gst::Registry::get(); + let _ = registry.find_plugin("x264"); + }); +} + +fn create_test_buffer() -> gst::Buffer { + let width = 320i32; + let height = 240i32; + let fps_num = 30i32; + let fps_den = 1i32; + + let info = + gst_video::VideoInfo::builder(gst_video::VideoFormat::I420, width as u32, height as u32) + .fps(gst::Fraction::new(fps_num, fps_den)) + .build() + .unwrap(); + + let mut buffer = gst::Buffer::with_size(info.size()).unwrap(); + { + let buffer_mut = buffer.get_mut().unwrap(); + buffer_mut.set_pts(0.nseconds()); + buffer_mut.set_duration(gst::ClockTime::SECOND / fps_num as u64); + } + buffer +} + +#[test] +fn test_video_encoder_stats_single_buffer() { + init(); + + // Skip test if x264enc is not available + if gst::ElementFactory::find("x264enc").is_none() { + eprintln!("Skipping test: x264enc not available"); + return; + } + + let mut h = gst_check::Harness::with_padnames("video-encoder-stats", Some("sink"), Some("src")); + + // Create and configure the encoder + let encoder = gst::ElementFactory::make("x264enc") + .property("bitrate", 256u32) + .build() + .unwrap(); + + // Set the encoder property on the video-encoder-stats element + h.element().unwrap().set_property("encoder", &encoder); + + // Set input caps (raw video) + let caps = gst_video::VideoCapsBuilder::new() + .format(gst_video::VideoFormat::I420) + .width(320) + .height(240) + .framerate(gst::Fraction::new(30, 1)) + .build(); + h.set_src_caps(caps); + + // Create and push a test buffer + let buf = create_test_buffer(); + h.push(buf).unwrap(); + + // Pull the output buffer + let out = h.pull().unwrap(); + + // Verify we get a buffer with some size + assert!(out.size() > 0, "Should receive a non-empty buffer"); + + // Check that VideoEncoderStatsMeta is present + let meta = out.meta::(); + assert!( + meta.is_some(), + "VideoEncoderStatsMeta should be present on output buffer" + ); + + let meta = meta.unwrap(); + let stats = meta.stats(); + + // Verify basic stats properties + assert_eq!(stats.name, "x264enc", "Encoder name should match"); + assert!( + stats.num_buffers > 0, + "Buffer count should be greater than 0" + ); + assert!(stats.num_bytes > 0, "Byte count should be greater than 0"); +} + +#[test] +fn test_video_encoder_stats_multiple_buffers() { + init(); + + // Skip test if x264enc is not available + if gst::ElementFactory::find("x264enc").is_none() { + eprintln!("Skipping test: x264enc not available"); + return; + } + + let mut h = gst_check::Harness::with_padnames("video-encoder-stats", Some("sink"), Some("src")); + + // Create and configure the encoder + let encoder = gst::ElementFactory::make("x264enc") + .property("bitrate", 256u32) + .build() + .unwrap(); + + h.element().unwrap().set_property("encoder", &encoder); + + // Set input caps + let caps = gst_video::VideoCapsBuilder::new() + .format(gst_video::VideoFormat::I420) + .width(320) + .height(240) + .framerate(gst::Fraction::new(30, 1)) + .build(); + h.set_src_caps(caps); + + // Push multiple buffers + for i in 0..5 { + let mut buf = create_test_buffer(); + { + let buf_mut = buf.get_mut().unwrap(); + buf_mut.set_pts((i as u64) * gst::ClockTime::SECOND / 30); + } + h.push(buf).unwrap(); + } + + // Pull and verify each output buffer + for i in 0..5 { + let out = h.pull().unwrap(); + assert!(out.size() > 0, "Should receive non-empty buffers"); + + // Check that VideoEncoderStatsMeta is present + let meta = out.meta::(); + assert!( + meta.is_some(), + "VideoEncoderStatsMeta should be present on buffer {}", + i + ); + + let meta = meta.unwrap(); + let stats = meta.stats(); + + assert_eq!(stats.name, "x264enc"); + assert!( + stats.num_buffers >= (i + 1) as u64, + "Buffer count should increase with each buffer" + ); + assert!(stats.num_bytes > 0); + + // Verify framerate is set from caps + assert!( + stats.framerate.is_some(), + "Framerate should be set from caps" + ); + if let Some(fps) = stats.framerate { + assert_eq!(fps.numer(), 30); + assert_eq!(fps.denom(), 1); + } + } +} + +#[test] +fn test_video_encoder_stats_with_request_pad() { + init(); + + // Skip test if x264enc is not available + if gst::ElementFactory::find("x264enc").is_none() { + eprintln!("Skipping test: x264enc not available"); + return; + } + + let element = gst::ElementFactory::make("video-encoder-stats") + .build() + .unwrap(); + + // Request the decoder src pad before going to PAUSED + let request_pad = element.request_pad_simple("decoder_src_%u"); + assert!( + request_pad.is_some(), + "Should be able to request decoder_src pad" + ); + + let mut h = gst_check::Harness::with_element(&element, Some("sink"), Some("src")); + + // Create and configure the encoder + let encoder = gst::ElementFactory::make("x264enc") + .property("bitrate", 256u32) + .build() + .unwrap(); + + element.set_property("encoder", &encoder); + + // Set input caps + let caps = gst_video::VideoCapsBuilder::new() + .format(gst_video::VideoFormat::I420) + .width(320) + .height(240) + .framerate(gst::Fraction::new(30, 1)) + .build(); + h.set_src_caps(caps); + + // Push a buffer + let buf = create_test_buffer(); + h.push(buf).unwrap(); + + // Pull the output buffer from main src pad + let out = h.pull().unwrap(); + + // Verify we get a buffer with some size + assert!(out.size() > 0, "Should receive a non-empty buffer"); + + // Check that VideoEncoderStatsMeta is present even with request pad + let meta = out.meta::(); + assert!( + meta.is_some(), + "VideoEncoderStatsMeta should be present with request pad" + ); + + let meta = meta.unwrap(); + let stats = meta.stats(); + assert_eq!(stats.name, "x264enc"); + assert!(stats.num_buffers > 0); + assert!(stats.num_bytes > 0); +} From 41b47217e3b06aed664a808df09f01fd970fdcdf Mon Sep 17 00:00:00 2001 From: Diego Nieto Date: Thu, 14 Aug 2025 11:38:14 +0200 Subject: [PATCH 11/46] video-encoder-stats: calculate score every framerate frames This reduces drastically the processing time taken by the vmaf element, avoiding to post scores for each frame. --- video/stats/src/encoderstats/imp.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/video/stats/src/encoderstats/imp.rs b/video/stats/src/encoderstats/imp.rs index b39d26dfe..f1bb3d6eb 100644 --- a/video/stats/src/encoderstats/imp.rs +++ b/video/stats/src/encoderstats/imp.rs @@ -129,10 +129,11 @@ impl EncoderStats { match event.view() { Caps(event) => { let caps = event.caps(); + gst::info!(CAT, "Received caps {caps:?}"); let s = caps.structure(0).unwrap(); let fps = s.get::("framerate").ok(); self.stats.lock().unwrap().framerate = fps; - gst::info!(CAT, "Received caps {caps:?}"); + self.obj().by_name("vmaf0").unwrap().set_property("subsample", fps.unwrap().numer() as u32); } _ => { gst::info!(CAT, "Other event"); From d58fdb991c726f2859db4e2d02413f1648ae0ea7 Mon Sep 17 00:00:00 2001 From: Diego Nieto Date: Mon, 18 Aug 2025 14:42:20 +0200 Subject: [PATCH 12/46] videostats: add originalbuffer internal dependency for building --- video/stats/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/video/stats/Cargo.toml b/video/stats/Cargo.toml index af69af4f4..df96f143a 100644 --- a/video/stats/Cargo.toml +++ b/video/stats/Cargo.toml @@ -15,6 +15,7 @@ gst.workspace = true gst-video.workspace = true human_bytes = { version = "0.4", default-features = false } atomic_refcell = "0.1" +gst-plugin-originalbuffer = { path = "../../generic/originalbuffer" } [dev-dependencies] gst-check.workspace = true From 1072f15a834bf4a5967a2b7cad60ada2757d14d7 Mon Sep 17 00:00:00 2001 From: Diego Nieto Date: Wed, 20 Aug 2025 11:09:10 +0200 Subject: [PATCH 13/46] encoderstats: add max. internal buffers metric --- video/stats/src/videoencoderstats.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/video/stats/src/videoencoderstats.rs b/video/stats/src/videoencoderstats.rs index 354debf81..e5e1bb4cc 100644 --- a/video/stats/src/videoencoderstats.rs +++ b/video/stats/src/videoencoderstats.rs @@ -11,10 +11,19 @@ use std::collections::VecDeque; use std::time::Instant; use std::time::Duration; use std::fmt; +use std::sync::LazyLock; use procfs::process::Process; use human_bytes::human_bytes; +static CAT: LazyLock = LazyLock::new(|| { + gst::DebugCategory::new( + "VideoEncoderStats", + gst::DebugColorFlags::empty(), + Some("VideoEncoderStats"), + ) +}); + #[derive(Default, Clone, PartialEq, Debug)] pub struct VideoEncoderStats { pub name: String, @@ -35,6 +44,7 @@ impl VideoEncoderStats { if self.time_last_buffers.len() > self.max_buffers_inside { self.max_buffers_inside = self.time_last_buffers.len(); } + gst::log!(CAT, "Current buffers lenght {}", self.time_last_buffers.len()); } pub fn buffer_out(&mut self) { @@ -95,6 +105,12 @@ impl fmt::Display for VideoEncoderStats { processing_time.as_secs_f64() )?; + writeln!( + f, + "Max internal buffers: {}", + self.max_buffers_inside + )?; + let cpu_time = self.threads_utime + self.threads_stime; writeln!( f, From a1b9e9fbf661217816ed880f50bbbd1cecec695b Mon Sep 17 00:00:00 2001 From: Diego Nieto Date: Thu, 21 Aug 2025 09:49:03 +0200 Subject: [PATCH 14/46] videostats: update data layout --- .../examples/video-stats-split-screen.rs | 2 +- video/stats/examples/video-stats.rs | 2 +- video/stats/src/encoderstats/imp.rs | 26 +++++++++++- video/stats/src/videoencoderstats.rs | 42 ++++++++----------- 4 files changed, 44 insertions(+), 28 deletions(-) diff --git a/video/stats/examples/video-stats-split-screen.rs b/video/stats/examples/video-stats-split-screen.rs index b34d4a5e8..486850283 100644 --- a/video/stats/examples/video-stats-split-screen.rs +++ b/video/stats/examples/video-stats-split-screen.rs @@ -13,7 +13,7 @@ use gst::prelude::*; fn main() -> Result<(), Error> { gst::init()?; - let pipeline = gst::parse::launch("souphttpsrc location=\"https://ftp.nluug.nl/pub/graphics/blender/demo/movies/ToS/tears_of_steel_1080p.mov\" ! qtdemux name=demux demux.video_0 ! queue ! decodebin3 ! videoconvertscale ! capsfilter caps=\"video/x-raw,width=720,aspect-ratio=1/1\" ! tee name=tee ! queue name=encq0 ! video-encoder-stats encoder=\"x264enc\" decoder=\"h264parse ! avdec_h264\" name=vs0 tee. ! queue name=encq1 ! video-encoder-stats encoder=\"x264enc bitrate=512\" name=vs1 video-compare-mixer split-screen=true backend=CPU name=mixer vs0.src ! decodebin3 ! mixer.sink_0 vs1.src ! decodebin3 ! mixer.sink_1 mixer. ! autovideosink")?; + let pipeline = gst::parse::launch("souphttpsrc location=\"https://ftp.nluug.nl/pub/graphics/blender/demo/movies/ToS/tears_of_steel_1080p.mov\" ! qtdemux name=demux demux.video_0 ! queue ! decodebin3 ! videoconvertscale ! capsfilter caps=\"video/x-raw,aspect-ratio=1/1\" ! tee name=tee ! queue name=encq0 ! video-encoder-stats encoder=\"x264enc\" decoder=\"h264parse ! avdec_h264\" name=vs0 tee. ! queue name=encq1 ! video-encoder-stats encoder=\"x264enc bitrate=512\" name=vs1 video-compare-mixer split-screen=true backend=CPU name=mixer vs0.src ! decodebin3 ! mixer.sink_0 vs1.src ! decodebin3 ! mixer.sink_1 mixer. ! autovideosink")?; pipeline.set_state(gst::State::Playing)?; let bus = pipeline.bus().unwrap(); diff --git a/video/stats/examples/video-stats.rs b/video/stats/examples/video-stats.rs index 45453a339..137ce279d 100644 --- a/video/stats/examples/video-stats.rs +++ b/video/stats/examples/video-stats.rs @@ -13,7 +13,7 @@ use gst::prelude::*; fn main() -> Result<(), Error> { gst::init()?; - let pipeline = gst::parse::launch("souphttpsrc location=\"https://ftp.nluug.nl/pub/graphics/blender/demo/movies/ToS/tears_of_steel_1080p.mov\" ! qtdemux name=demux demux.video_0 ! queue ! decodebin3 ! videoconvertscale ! capsfilter caps=\"video/x-raw,width=720,aspect-ratio=1/1\" ! tee name=tee ! queue name=encq0 ! video-encoder-stats encoder=\"x264enc\" decoder=\"h264parse ! avdec_h264\" name=vs0 ! fakesink tee. ! queue name=encq1 ! video-encoder-stats encoder=\"x264enc bitrate=512\" name=vs1 ! fakesink video-compare-mixer split-screen=false backend=CPU name=mixer vs0.decoder_src ! mixer.sink_0 vs1.decoder_src ! mixer.sink_1 mixer. ! autovideosink")?; + let pipeline = gst::parse::launch("souphttpsrc location=\"https://ftp.nluug.nl/pub/graphics/blender/demo/movies/ToS/tears_of_steel_1080p.mov\" ! qtdemux name=demux demux.video_0 ! queue ! decodebin3 ! videoconvertscale ! capsfilter caps=\"video/x-raw,aspect-ratio=1/1\" ! tee name=tee ! queue name=encq0 ! video-encoder-stats encoder=\"x264enc bitrate=1024\" decoder=\"h264parse ! avdec_h264\" name=vs0 ! fakesink tee. ! queue name=encq1 ! video-encoder-stats encoder=\"x264enc bitrate=512\" name=vs1 ! fakesink video-compare-mixer split-screen=false backend=CPU name=mixer vs0.decoder_src ! mixer.sink_0 vs1.decoder_src ! mixer.sink_1 mixer. ! autovideosink")?; pipeline.set_state(gst::State::Playing)?; let bus = pipeline.bus().unwrap(); diff --git a/video/stats/src/encoderstats/imp.rs b/video/stats/src/encoderstats/imp.rs index f1bb3d6eb..cd50f992e 100644 --- a/video/stats/src/encoderstats/imp.rs +++ b/video/stats/src/encoderstats/imp.rs @@ -76,7 +76,31 @@ impl EncoderStats { } else { "encq1:src" }; - let (total_utime, total_stime) = get_cpu_usage(thread_name.to_string()); + let (mut total_utime, mut total_stime) = get_cpu_usage(thread_name.to_string()); + + if encoder_name == "flulcevch264enc" { + // Fixme flulcevc uses multiple threads, so we need to get the CPU usage of all threads + let (utime, stime) = get_cpu_usage("lcevc".to_string()); + gst::log!(CAT, "flulcevc lcevc utime: {}, stime: {}", utime, stime); + // Add the CPU usage to the total CPU usage + total_utime += utime; + total_stime += stime; + + let (utime, stime) = get_cpu_usage("pool.".to_string()); + gst::log!(CAT, "flulcevc pool utime: {}, stime: {}", utime, stime); + // Add the CPU usage to the total CPU usage + total_utime += utime; + total_stime += stime; + } + + if encoder_name == "lcevch264enc" { + // Fixme lcevc uses multiple threads, so we need to get the CPU usage of all threads + let (utime, stime) = get_cpu_usage("pool.".to_string()); + gst::log!(CAT, "lcevc pool utime: {}, stime: {}", utime, stime); + // Add the CPU usage to the total CPU usage + total_utime += utime; + total_stime += stime; + } stats.threads_utime = total_utime; stats.threads_stime = total_stime; diff --git a/video/stats/src/videoencoderstats.rs b/video/stats/src/videoencoderstats.rs index e5e1bb4cc..599ff5dc3 100644 --- a/video/stats/src/videoencoderstats.rs +++ b/video/stats/src/videoencoderstats.rs @@ -14,7 +14,6 @@ use std::fmt; use std::sync::LazyLock; use procfs::process::Process; -use human_bytes::human_bytes; static CAT: LazyLock = LazyLock::new(|| { gst::DebugCategory::new( @@ -78,13 +77,8 @@ impl fmt::Display for VideoEncoderStats { )?; writeln!( f, - "Buffers: {}", - self.num_buffers, - )?; - writeln!( - f, - "Bytes: {}", - self.num_bytes, + "Output size: {} KB", + self.num_bytes / 1024, // Convert to KB )?; let framerate = self.framerate.unwrap(); @@ -94,34 +88,33 @@ impl fmt::Display for VideoEncoderStats { } else { 0.0 }; - let bitrate_str = human_bytes(bitrate); - - writeln!(f, "Bitrate: {}b/s", bitrate_str)?; + let bitrate_str = bitrate/1024.0; // Convert to kbps - let processing_time = self.avg_processing_time(); - writeln!( - f, - "Processing time: {:.3}", - processing_time.as_secs_f64() - )?; + writeln!(f, "Bitrate: {:.3} kbps", bitrate_str)?; + let throughput = (1.0 + self.time_last_buffers.len() as f64)/self.avg_processing_time().as_secs_f64(); writeln!( f, - "Max internal buffers: {}", - self.max_buffers_inside + "Throughput: {:.2} fps", + throughput )?; let cpu_time = self.threads_utime + self.threads_stime; + #[cfg(target_os = "linux")] + let cpu_time_seconds = { + let ticks_per_second = procfs::ticks_per_second() as u64; + cpu_time as f64 / ticks_per_second as f64 + }; writeln!( f, - "CPU time: {}", - cpu_time + "CPU: {} s", + cpu_time_seconds )?; let vmaf_score = self.vmaf_score; writeln!( f, - "VMAF score: {:.3}", + "VMAF: {:.3}", vmaf_score ) } @@ -137,9 +130,8 @@ pub fn get_cpu_usage(name: String) -> (u64, u64) { for thread in process.tasks().unwrap().flatten() { let stat = thread.stat().unwrap(); - // FIXME - //println!("Thread: {}, Comm: {}, Utime: {}, Stime: {}", thread.tid, stat.comm, stat.utime, stat.stime); - if stat.comm == name { + if stat.comm.contains(&name) { + gst::log!(CAT, "Thread: {}, Comm: {}, Utime: {}, Stime: {}", thread.tid, stat.comm, stat.utime, stat.stime); total_utime += stat.utime; total_stime += stat.stime; } From cc73a8431cf344a7058de05bfda050db7c1d2ae6 Mon Sep 17 00:00:00 2001 From: Diego Nieto Date: Mon, 25 Aug 2025 14:12:09 +0200 Subject: [PATCH 15/46] encoderstats: fix name causing problems CPU stats --- video/stats/src/encoderstats/imp.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/video/stats/src/encoderstats/imp.rs b/video/stats/src/encoderstats/imp.rs index cd50f992e..c73ff6ad8 100644 --- a/video/stats/src/encoderstats/imp.rs +++ b/video/stats/src/encoderstats/imp.rs @@ -205,13 +205,13 @@ impl EncoderStats { let tee0_src_0 = tee0.request_pad_simple("src_%u").expect("tee0 src pad"); let queue0 = gst::ElementFactory::make("queue") - .name("encq0") + .name("encintq0") .build() - .expect("Failed to create queue encq0"); - self.obj().add(&queue0).expect("Failed to add queue encq0"); + .expect("Failed to create queue encintq0"); + self.obj().add(&queue0).expect("Failed to add queue encintq0"); let queue0_sink_pad = queue0.static_pad("sink").unwrap(); let queue0_src_pad = queue0.static_pad("src").unwrap(); - tee0_src_0.link(&queue0_sink_pad).expect("tee0.src_0 -> encq0.sink"); + tee0_src_0.link(&queue0_sink_pad).expect("tee0.src_0 -> encintq0.sink"); self.srcpad.set_target(Some(&queue0_src_pad)).unwrap(); self.sinkpad @@ -220,9 +220,9 @@ impl EncoderStats { let tee0_src_1 = tee0.request_pad_simple("src_%u").expect("tee0 src_1"); let queue1 = gst::ElementFactory::make("queue") - .name("encq1") + .name("encintq1") .build() - .expect("Failed to create queue encq1"); + .expect("Failed to create queue encintq1"); // Use custom decoder if provided, otherwise use decodebin3 let final_decoder = if let Some(custom_decoder) = decoder.clone() { From bb37bbbbdbfc09380b59566153ae98d831138783 Mon Sep 17 00:00:00 2001 From: Diego Nieto Date: Tue, 26 Aug 2025 13:45:43 +0200 Subject: [PATCH 16/46] videostats: fix originalbuffer version --- video/stats/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/video/stats/Cargo.toml b/video/stats/Cargo.toml index df96f143a..ca0d3116f 100644 --- a/video/stats/Cargo.toml +++ b/video/stats/Cargo.toml @@ -15,7 +15,7 @@ gst.workspace = true gst-video.workspace = true human_bytes = { version = "0.4", default-features = false } atomic_refcell = "0.1" -gst-plugin-originalbuffer = { path = "../../generic/originalbuffer" } +gst-plugin-originalbuffer = { git = "https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs", tag = "0.13.0" } [dev-dependencies] gst-check.workspace = true From 6dc303a2ae6ab3be7deb08a8103a7f5e451383df Mon Sep 17 00:00:00 2001 From: Diego Nieto Date: Wed, 27 Aug 2025 14:19:30 +0200 Subject: [PATCH 17/46] video-compare-mixer: add navigation events --- video/stats/README.md | 21 + video/stats/src/comparemixer/compositor.rs | 1438 ++++++++++++++++++++ video/stats/src/comparemixer/imp.rs | 380 +++++- video/stats/src/comparemixer/mod.rs | 1 + 4 files changed, 1790 insertions(+), 50 deletions(-) create mode 100644 video/stats/src/comparemixer/compositor.rs diff --git a/video/stats/README.md b/video/stats/README.md index a1a9536d6..61ecfe4bf 100644 --- a/video/stats/README.md +++ b/video/stats/README.md @@ -6,6 +6,27 @@ - `video-compare-mixer`: The element in charge of comparing and mixing multiple video streams. Useful for side-by-side quality comparisons or blending outputs from different encoders. + User can change the video showed using the next keys: + + 1: Only first video + 2: Only second video + 3: First and second videos split mode (default) + 4: First and second videos side by side mode (default) + 5: Move side by side border left + 6: Move side by side border right + +Also click in the botton of the video can be done to change the side by side border + +User can change the video player zoom using the next keys: + + +: Zoom in + -: Zoom out + Up/Down/Right/Left: Move the frame + r: reset the zoom position + R: reset the zoom + +Also mouse navigation events can be used for a better UX. + - `videoencoderstatsmeta`: Defines metadata structures and logic for handling video encoder statistics along the pipeline. It has been defined as `GstVideoEncoderStatsMetaAPI`. diff --git a/video/stats/src/comparemixer/compositor.rs b/video/stats/src/comparemixer/compositor.rs new file mode 100644 index 000000000..7715a48a2 --- /dev/null +++ b/video/stats/src/comparemixer/compositor.rs @@ -0,0 +1,1438 @@ +// Copyright (C) 2025, Fluendo S.A. +// Author: Ruben Gonzalez +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +#[derive(Debug, Clone, Copy, PartialEq, Default)] +pub enum Mode { + #[default] + Split, + SideBySide, +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct Compositor { + pub mode: Mode, + pub zoom: usize, + pub offset_x: i32, + pub offset_y: i32, + pub border: i32, + pub width: i32, + pub height: i32, +} + +#[derive(Debug, Clone, Copy)] +pub struct Position { + pub xpos: i32, + pub ypos: i32, + pub width: i32, + pub height: i32, + pub crop_right: i32, + pub crop_left: i32, +} + +const BORDER_STEP: usize = 10; +const WIDTH: i32 = 1280; +const HEIGHT: i32 = 720; +const HALF_WIDTH: i32 = WIDTH / 2; + +impl Default for Compositor { + fn default() -> Self { + Self { + mode: Mode::default(), + zoom: 100, + offset_x: 0, + offset_y: 0, + border: HALF_WIDTH, + width: WIDTH, + height: HEIGHT, + } + } +} + +impl Compositor { + #[allow(dead_code)] + pub fn new(mode: Mode, width: i32, height: i32) -> Self { + Self { + mode, + width, + height, + border: width / 2, + ..Default::default() + } + } + + #[allow(dead_code)] + pub fn new_side_by_side(width: i32, height: i32) -> Self { + Self { + mode: Mode::SideBySide, + width, + height, + border: width / 2, + ..Default::default() + } + } + + #[allow(dead_code)] + pub fn new_split(width: i32, height: i32) -> Self { + Self { + mode: Mode::Split, + width, + height, + border: width / 2, + ..Default::default() + } + } + + /// Set side_by_side mode + pub fn split_mode(&mut self) { + self.mode = Mode::Split; + } + + /// Set side_by_side mode + pub fn side_by_side_mode(&mut self) { + self.mode = Mode::SideBySide; + } + + /// Set side_by_side mode + #[allow(dead_code)] + pub fn is_split_mode(&self) -> bool { + self.mode == Mode::Split + } + + /// Set side_by_side mode + #[allow(dead_code)] + pub fn is_side_by_side_mode(&self) -> bool { + self.mode == Mode::SideBySide + } + + /// Reset default values + pub fn reset(&mut self) { + let d = Compositor::default(); + self.zoom = d.zoom; + self.offset_x = d.offset_x; + self.offset_y = d.offset_y; + self.border = d.border; + } + + /// Reset only border to default values + pub fn reset_border(&mut self) { + let d = Compositor::default(); + self.border = d.border; + } + + /// Reset only border to default values + pub fn reset_position(&mut self) { + let d = Compositor::default(); + self.zoom = d.zoom; + self.offset_x = d.offset_x; + self.offset_y = d.offset_y; + } + + /// Moves the viewport by `x_step` pixels horizontally and `y_step` pixels vertically. + /// Does not clamp the values, allowing offsets to exceed valid bounds. + pub fn move_pos(&mut self, x_step: i32, y_step: i32) { + self.offset_x += x_step; + self.offset_y += y_step; + } + + /// Set offset_x and offset_y + pub fn move_pos_to(&mut self, x: i32, y: i32) { + self.offset_x = x; + self.offset_y = y; + } + + /// Offsets the border inside the bounds. + pub fn move_border(&mut self, offset: i32) { + self.move_border_to(self.border + offset); + } + + /// Set border position inside the bounds. + pub fn move_border_to(&mut self, new_border: i32) { + if new_border < 0 { + self.border = 0 + } else if new_border > self.width { + self.border = self.width + } else { + self.border = new_border + } + } + + /// Increases the zoom level, capping it at a sensible maximum (e.g., 1000000) + pub fn zoom_in(&mut self) { + let scale = if self.is_split_mode() { 2 } else { 4 }; + self.zoom_in_center_at(self.width / scale, self.height / 2); + } + + /// Decreases the zoom level, ensuring it stays at a minimum of 1 + pub fn zoom_out(&mut self) { + let scale = if self.is_split_mode() { 2 } else { 4 }; + self.zoom_out_center_at(self.width / scale, self.height / 2); + } + + /// Increases the zoom level, capping it at a sensible maximum (e.g., 1000000) + /// Update offset to keep centered + pub fn zoom_in_center_at(&mut self, x: i32, y: i32) { + self.zoom = (self.zoom + BORDER_STEP).min(1000000); + self.fix_offset_when_zoom(x, y, true); + } + + /// Decreases the zoom level, ensuring it stays at a minimum of 1, + /// Update offset to keep centered + pub fn zoom_out_center_at(&mut self, x: i32, y: i32) { + self.zoom = (self.zoom.saturating_sub(BORDER_STEP)).max(1); + self.fix_offset_when_zoom(x, y, false); + } + + fn fix_offset_when_zoom(&mut self, x: i32, y: i32, inside: bool) { + match self.mode { + Mode::Split => { + self.fix_offset_when_zoom_split(x, y, inside); + } + Mode::SideBySide => { + self.fix_offset_when_zoom_side_by_side(x, y, inside); + } + } + } + + fn fix_offset_when_zoom_split(&mut self, x: i32, y: i32, inside: bool) { + let diff = x - (self.width / 2); + let new_offset = diff / (BORDER_STEP as i32); + if inside { + self.offset_x -= new_offset; + } else { + self.offset_x += new_offset; + } + + let diff = y - (self.height / 2); + let new_offset = diff / (BORDER_STEP as i32); + if inside { + self.offset_y -= new_offset; + } else { + self.offset_y += new_offset; + } + } + + fn fix_offset_when_zoom_side_by_side(&mut self, x: i32, y: i32, inside: bool) { + let x = x % (self.width / 2); + let diff = x - (self.width / 4); + let new_offset = diff / (BORDER_STEP as i32); + if inside { + self.offset_x -= new_offset; + } else { + self.offset_x += new_offset; + } + + let diff = y - (self.height / 2); + let new_offset = diff / (BORDER_STEP as i32); + if inside { + self.offset_y -= new_offset; + } else { + self.offset_y += new_offset; + } + } + + /// Calculates the two `Position`s for the input videos based on the compositor values + pub fn get_positions(&self) -> (Position, Position) { + match self.mode { + Mode::Split => self.get_positions_split(), + Mode::SideBySide => self.get_positions_side_by_side(), + } + } + + //here impl + fn get_positions_side_by_side(&self) -> (Position, Position) { + let zoom_factor = (self.zoom as f32) / 100.0; + let viewport_width = (self.width as f32 * zoom_factor) as i32; + let viewport_height = (self.height as f32 * zoom_factor) as i32; + + let half_width = self.width / 2; + let half_viewport_width = viewport_width / 2; + + let pos_height = viewport_height / 2; + let pos_ypos = self.offset_y + (self.height - pos_height) / 2; + + let pos_xpos = self.offset_x + (self.width - viewport_width) / 4; + + let unscaling = |w: i32| -> i32 { + // crop is done over the original image + let u_w = w * self.width / half_viewport_width; + if u_w < self.width { + u_w + } else { + 0 + } + }; + + let pos0 = Position { + xpos: if pos_xpos > half_width { 0 } else { pos_xpos }, + ypos: pos_ypos, + width: if pos_xpos + half_viewport_width > half_width { + if pos_xpos < half_width { + half_width - pos_xpos + } else { + 0 + } + } else { + half_viewport_width + }, + height: pos_height, + crop_right: if (pos_xpos + half_viewport_width) > half_width { + let crop = pos_xpos + half_viewport_width - half_width; + unscaling(crop) + } else { + 0 + }, + crop_left: 0, + }; + + let pos1 = Position { + xpos: if pos_xpos < 0 { + half_width + } else { + pos_xpos + half_width + }, + ypos: pos_ypos, + width: if pos_xpos < 0 { + if half_viewport_width > pos_xpos && half_viewport_width + pos_xpos > 0 { + half_viewport_width + pos_xpos + } else { + 0 + } + } else { + half_viewport_width + }, + height: pos_height, + crop_right: 0, + crop_left: if pos_xpos > 0 || pos_xpos < -half_viewport_width { + 0 + } else { + let crop = -pos_xpos; + unscaling(crop) + }, + }; + + (pos0, pos1) + } + + fn get_positions_split(&self) -> (Position, Position) { + let zoom_factor = (self.zoom as f32) / 100.0; + let viewport_width = (self.width as f32 * zoom_factor) as i32; + let viewport_height = (self.height as f32 * zoom_factor) as i32; + let viewport_offset_x = self.offset_x - (viewport_width - self.width) / 2; + let viewport_offset_y = self.offset_y - (viewport_height - self.height) / 2; + + let pos0 = Position { + xpos: if viewport_offset_x < self.border { + viewport_offset_x + } else { + 0 + }, + ypos: viewport_offset_y, + width: { + if viewport_offset_x < self.border { + if self.border - viewport_offset_x > viewport_width { + viewport_width + } else { + self.border - viewport_offset_x + } + } else { + 0 + } + }, + height: viewport_height, + crop_right: { + if viewport_width + viewport_offset_x < self.border { + 0 + } else if viewport_offset_x > self.border { + self.width + } else { + // Note crop before zoom scaling (because glvideomixer implementation)w + let scale = self.width as f32 / viewport_width as f32; + let crop_right_scaled = + (viewport_width + viewport_offset_x - self.border) as f32; + (crop_right_scaled * scale) as i32 + } + }, + crop_left: 0, + }; + + let pos1 = Position { + xpos: if viewport_offset_x > self.border { + viewport_offset_x + } else { + self.border + }, + ypos: viewport_offset_y, + width: { + //TODO refactor + if self.border < (viewport_width - viewport_offset_x) { + if viewport_width > self.border - viewport_offset_x { + if self.border < viewport_offset_x { + viewport_width + } else { + viewport_width - self.border + viewport_offset_x + } + } else { + 0 + } + } else if self.border < viewport_offset_x { + viewport_width + } else { + viewport_width - self.border + viewport_offset_x + } + }, + height: viewport_height, + crop_right: 0, + crop_left: { + if viewport_width + viewport_offset_x < self.border { + self.width + } else if viewport_offset_x > self.border { + 0 + } else { + // Note crop before zoom scaling (because glvideomixer implementation)w + let scale = self.width as f32 / viewport_width as f32; + let crop_right_scaled = (self.border - viewport_offset_x) as f32; + (crop_right_scaled * scale) as i32 + } + }, + }; + + (pos0, pos1) + } +} + +#[cfg(test)] +mod tests { + const HALF_HEIGHT: i32 = HEIGHT / 2; + + use super::*; + + #[test] + fn test_compositor_default() { + let compositor = Compositor::default(); + + assert_eq!(compositor.zoom, 100, "compositor.zoom"); + assert_eq!(compositor.offset_x, 0, "compositor.offset_x"); + assert_eq!(compositor.offset_y, 0, "compositor.offset_y"); + assert_eq!(compositor.border, HALF_WIDTH, "compositor.border"); + assert_eq!(compositor.width, WIDTH, "compositor.width"); + assert_eq!(compositor.height, HEIGHT, "compositor.height"); + } + + #[test] + fn test_compositor_new() { + let width = 14; + let height = 11; + let compositor = Compositor::new(Mode::SideBySide, width, height); + assert_eq!(compositor.mode, Mode::SideBySide, "compositor.mode"); + assert!( + compositor.is_side_by_side_mode(), + "compositor.is_side_by_side_mode" + ); + assert_eq!(compositor.width, width, "compositor.width"); + assert_eq!(compositor.height, height, "compositor.height"); + + let compositor = Compositor::new(Mode::Split, width, height); + assert_eq!(compositor.mode, Mode::Split, "compositor.mode"); + assert!(compositor.is_split_mode(), "compositor.is_split_mode"); + assert_eq!(compositor.width, width, "compositor.width"); + assert_eq!(compositor.height, height, "compositor.height"); + + let compositor = Compositor::new_split(width, height); + assert!(compositor.is_split_mode(), "compositor.is_split_mode"); + + let mut compositor = Compositor::new_side_by_side(width, height); + assert!( + compositor.is_side_by_side_mode(), + "compositor.is_side_by_side_mode" + ); + compositor.split_mode(); + assert!(compositor.is_split_mode(), "compositor.is_split_mode"); + compositor.side_by_side_mode(); + assert!( + compositor.is_side_by_side_mode(), + "compositor.is_side_by_side_mode" + ); + } + + #[test] + fn test_split_get_positions_default() { + let compositor = Compositor::default(); + let (pos0, pos1) = compositor.get_positions(); + + assert_eq!(pos0.xpos, 0, "pos0.xpos"); + assert_eq!(pos0.ypos, 0, "pos0.ypos"); + assert_eq!(pos0.width, HALF_WIDTH, "pos0.width"); + assert_eq!(pos0.height, HEIGHT, "pos0.height"); + assert_eq!(pos0.crop_right, HALF_WIDTH, "pos0.crop_right"); + assert_eq!(pos0.crop_left, 0, "pos0.crop_left"); + + assert_eq!(pos1.xpos, HALF_WIDTH, "pos1.xpos"); + assert_eq!(pos1.ypos, 0, "pos1.ypos"); + assert_eq!(pos1.width, HALF_WIDTH, "pos1.width"); + assert_eq!(pos1.height, HEIGHT, "pos1.height"); + assert_eq!(pos1.crop_right, 0, "pos1.crop_right"); + assert_eq!(pos1.crop_left, HALF_WIDTH, "pos1.crop_left"); + } + + #[test] + fn test_split_move_pos_left() { + let mut compositor = Compositor::default(); + compositor.move_pos(-10, 0); + let (pos0, pos1) = compositor.get_positions(); + + assert_eq!(compositor.zoom, 100, "compositor.zoom"); + assert_eq!(compositor.offset_x, -10, "compositor.offset_x"); + assert_eq!(compositor.offset_y, 0, "compositor.offset_y"); + assert_eq!(compositor.border, HALF_WIDTH, "compositor.border"); + assert_eq!(compositor.width, WIDTH, "compositor.width"); + assert_eq!(compositor.height, HEIGHT, "compositor.height"); + + assert_eq!(pos0.xpos, -10, "pos0.xpos"); + assert_eq!(pos0.ypos, 0, "pos0.ypos"); + assert_eq!(pos0.width, HALF_WIDTH + 10, "pos0.width"); + assert_eq!(pos0.height, HEIGHT, "pos0.height"); + assert_eq!(pos0.crop_right, HALF_WIDTH - 10, "pos0.crop_right"); + assert_eq!(pos0.crop_left, 0, "pos0.crop_left"); + + assert_eq!(pos1.xpos, HALF_WIDTH, "pos1.xpos"); + assert_eq!(pos1.ypos, 0, "pos1.ypos"); + assert_eq!(pos1.width, HALF_WIDTH - 10, "pos1.width"); + assert_eq!(pos1.height, HEIGHT, "pos1.height"); + assert_eq!(pos1.crop_right, 0, "pos1.crop_right"); + assert_eq!(pos1.crop_left, HALF_WIDTH + 10, "pos1.crop_left"); + + compositor.reset_position(); + assert_eq!(compositor.zoom, 100, "compositor.zoom"); + assert_eq!(compositor.offset_x, 0, "compositor.offset_x"); + assert_eq!(compositor.offset_y, 0, "compositor.offset_y"); + assert_eq!(compositor.border, HALF_WIDTH, "compositor.border"); + assert_eq!(compositor.width, WIDTH, "compositor.width"); + assert_eq!(compositor.height, HEIGHT, "compositor.height"); + } + + #[test] + fn test_split_move_pos_left_out_of_border() { + let mut compositor = Compositor::default(); + compositor.move_pos(-1000, 0); + let (pos0, pos1) = compositor.get_positions(); + + assert_eq!(compositor.zoom, 100, "compositor.zoom"); + assert_eq!(compositor.offset_x, -1000, "compositor.offset_x"); + assert_eq!(compositor.offset_y, 0, "compositor.offset_y"); + assert_eq!(compositor.border, HALF_WIDTH, "compositor.border"); + assert_eq!(compositor.width, WIDTH, "compositor.width"); + assert_eq!(compositor.height, HEIGHT, "compositor.height"); + + assert_eq!(pos0.xpos, -1000, "pos0.xpos"); + assert_eq!(pos0.ypos, 0, "pos0.ypos"); + assert_eq!(pos0.width, WIDTH, "pos0.width"); + assert_eq!(pos0.height, HEIGHT, "pos0.height"); + assert_eq!(pos0.crop_right, 0, "pos0.crop_right"); + assert_eq!(pos0.crop_left, 0, "pos0.crop_left"); + + assert_eq!(pos1.xpos, HALF_WIDTH, "pos1.xpos"); + assert_eq!(pos1.ypos, 0, "pos1.ypos"); + assert_eq!(pos1.width, 0, "pos1.width"); + assert_eq!(pos1.height, HEIGHT, "pos1.height"); + assert_eq!(pos1.crop_right, 0, "pos1.crop_right"); + assert_eq!(pos1.crop_left, WIDTH, "pos1.crop_left"); + } + + #[test] + fn test_split_move_pos_right_out_of_border() { + let mut compositor = Compositor::default(); + compositor.move_pos(1000, 0); + let (pos0, pos1) = compositor.get_positions(); + + assert_eq!(compositor.zoom, 100, "compositor.zoom"); + assert_eq!(compositor.offset_x, 1000, "compositor.offset_x"); + assert_eq!(compositor.offset_y, 0, "compositor.offset_y"); + assert_eq!(compositor.border, HALF_WIDTH, "compositor.border"); + assert_eq!(compositor.width, WIDTH, "compositor.width"); + assert_eq!(compositor.height, HEIGHT, "compositor.height"); + + assert_eq!(pos0.xpos, 0, "pos0.xpos"); + assert_eq!(pos0.ypos, 0, "pos0.ypos"); + assert_eq!(pos0.width, 0, "pos0.width"); + assert_eq!(pos0.height, HEIGHT, "pos0.height"); + assert_eq!(pos0.crop_right, WIDTH, "pos0.crop_right"); + assert_eq!(pos0.crop_left, 0, "pos0.crop_left"); + + assert_eq!(pos1.xpos, 1000, "pos1.xpos"); + assert_eq!(pos1.ypos, 0, "pos1.ypos"); + assert_eq!(pos1.width, WIDTH, "pos1.width"); + assert_eq!(pos1.height, HEIGHT, "pos1.height"); + assert_eq!(pos1.crop_right, 0, "pos1.crop_right"); + assert_eq!(pos1.crop_left, 0, "pos1.crop_left"); + } + + #[test] + fn test_split_move_border_left() { + let mut compositor = Compositor::default(); + compositor.move_border(10); + let (pos0, pos1) = compositor.get_positions(); + + assert_eq!(compositor.zoom, 100, "compositor.zoom"); + assert_eq!(compositor.offset_x, 0, "compositor.offset_x"); + assert_eq!(compositor.offset_y, 0, "compositor.offset_y"); + assert_eq!(compositor.border, HALF_WIDTH + 10, "compositor.border"); + assert_eq!(compositor.width, WIDTH, "compositor.width"); + assert_eq!(compositor.height, HEIGHT, "compositor.height"); + + assert_eq!(pos0.xpos, 0, "pos0.xpos"); + assert_eq!(pos0.ypos, 0, "pos0.ypos"); + assert_eq!(pos0.width, HALF_WIDTH + 10, "pos0.width"); + assert_eq!(pos0.height, HEIGHT, "pos0.height"); + assert_eq!(pos0.crop_right, HALF_WIDTH - 10, "pos0.crop_right"); + assert_eq!(pos0.crop_left, 0, "pos0.crop_left"); + + assert_eq!(pos1.xpos, HALF_WIDTH + 10, "pos1.xpos"); + assert_eq!(pos1.ypos, 0, "pos1.ypos"); + assert_eq!(pos1.width, HALF_WIDTH - 10, "pos1.width"); + assert_eq!(pos1.height, HEIGHT, "pos1.height"); + assert_eq!(pos1.crop_right, 0, "pos1.crop_right"); + assert_eq!(pos1.crop_left, HALF_WIDTH + 10, "pos1.crop_left"); + + compositor.reset_position(); + assert_eq!(compositor.zoom, 100, "compositor.zoom"); + assert_eq!(compositor.offset_x, 0, "compositor.offset_x"); + assert_eq!(compositor.offset_y, 0, "compositor.offset_y"); + assert_eq!(compositor.border, HALF_WIDTH + 10, "compositor.border"); + assert_eq!(compositor.width, WIDTH, "compositor.width"); + assert_eq!(compositor.height, HEIGHT, "compositor.height"); + } + + #[test] + fn test_split_zoom_in() { + let mut compositor = Compositor::default(); + compositor.zoom_in(); + let (pos0, pos1) = compositor.get_positions(); + + assert_eq!(compositor.zoom, 110, "compositor.zoom"); + assert_eq!(compositor.offset_x, 0, "compositor.offset_x"); + assert_eq!(compositor.offset_y, 0, "compositor.offset_y"); + assert_eq!(compositor.border, HALF_WIDTH, "compositor.border"); + assert_eq!(compositor.width, WIDTH, "compositor.width"); + assert_eq!(compositor.height, HEIGHT, "compositor.height"); + + assert_eq!(pos0.xpos, -64, "pos0.xpos"); + assert_eq!(pos0.ypos, -36, "pos0.ypos"); + assert_eq!(pos0.width, 704, "pos0.width"); + assert_eq!(pos0.height, 792, "pos0.height"); + assert_eq!(pos0.crop_right, 640, "pos0.crop_right"); // Note crop before zoom scaling (because glvideomixer implementation) TODO delete + assert_eq!(pos0.crop_left, 0, "pos0.crop_left"); + + assert_eq!(pos1.xpos, 640, "pos1.xpos"); + assert_eq!(pos1.ypos, -36, "pos1.ypos"); + assert_eq!(pos1.width, 704, "pos1.width"); + assert_eq!(pos1.height, 792, "pos1.height"); + assert_eq!(pos1.crop_right, 0, "pos1.crop_right"); + assert_eq!(pos1.crop_left, 640, "pos1.crop_left"); // Note crop before zoom scaling (because glvideomixer implementation) TODO delete + } + + #[test] + fn test_split_zoom_out_five_times() { + let mut compositor = Compositor::default(); + compositor.zoom_out(); + compositor.zoom_out(); + compositor.zoom_out(); + compositor.zoom_out(); + compositor.zoom_out(); + compositor.zoom_out(); + let (pos0, pos1) = compositor.get_positions(); + + assert_eq!(compositor.zoom, 40, "compositor.zoom"); + assert_eq!(compositor.offset_x, 0, "compositor.offset_x"); + assert_eq!(compositor.offset_y, 0, "compositor.offset_y"); + assert_eq!(compositor.border, HALF_WIDTH, "compositor.border"); + assert_eq!(compositor.width, WIDTH, "compositor.width"); + assert_eq!(compositor.height, HEIGHT, "compositor.height"); + + assert_eq!(pos0.xpos, 384, "pos0.xpos"); + assert_eq!(pos0.ypos, 216, "pos0.ypos"); + assert_eq!(pos0.width, 256, "pos0.width"); + assert_eq!(pos0.height, 288, "pos0.height"); + assert_eq!(pos0.crop_right, 640, "pos0.crop_right"); // Note crop before zoom scaling (because glvideomixer implementation) + assert_eq!(pos0.crop_left, 0, "pos0.crop_left"); + + assert_eq!(pos1.xpos, 640, "pos1.xpos"); + assert_eq!(pos1.ypos, 216, "pos1.ypos"); + assert_eq!(pos1.width, 256, "pos1.width"); + assert_eq!(pos1.height, 288, "pos1.height"); + assert_eq!(pos1.crop_right, 0, "pos1.crop_right"); + assert_eq!(pos1.crop_left, 640, "pos1.crop_left"); + } + + #[test] + fn test_split_zoom_out_five_times_and_move() { + let mut compositor = Compositor::default(); + compositor.zoom_out(); + compositor.zoom_out(); + compositor.zoom_out(); + compositor.zoom_out(); + compositor.zoom_out(); + compositor.zoom_out(); + let (pos0, pos1) = compositor.get_positions(); + + assert_eq!(compositor.zoom, 40, "compositor.zoom"); + assert_eq!(compositor.offset_x, 0, "compositor.offset_x"); + assert_eq!(compositor.offset_y, 0, "compositor.offset_y"); + assert_eq!(compositor.border, HALF_WIDTH, "compositor.border"); + assert_eq!(compositor.width, WIDTH, "compositor.width"); + assert_eq!(compositor.height, HEIGHT, "compositor.height"); + + assert_eq!(pos0.xpos, 384, "pos0.xpos"); + assert_eq!(pos0.ypos, 216, "pos0.ypos"); + assert_eq!(pos0.width, 256, "pos0.width"); + assert_eq!(pos0.height, 288, "pos0.height"); + assert_eq!(pos0.crop_right, 640, "pos0.crop_right"); + assert_eq!(pos0.crop_left, 0, "pos0.crop_left"); + + assert_eq!(pos1.xpos, 640, "pos1.xpos"); + assert_eq!(pos1.ypos, 216, "pos1.ypos"); + assert_eq!(pos1.width, 256, "pos1.width"); + assert_eq!(pos1.height, 288, "pos1.height"); + assert_eq!(pos1.crop_right, 0, "pos1.crop_right"); + assert_eq!(pos1.crop_left, 640, "pos1.crop_left"); + + let current_width = pos0.width + pos1.width; + compositor.move_pos(-10, 0); + let (pos0, pos1) = compositor.get_positions(); + + assert_eq!(compositor.zoom, 40, "compositor.zoom"); + assert_eq!(compositor.offset_x, -10, "compositor.offset_x"); + assert_eq!( + pos0.width + pos1.width, + current_width, + "pos0.width + pos1.width" + ); + assert_eq!(pos0.crop_right, 615, "pos0.crop_right"); + assert_eq!(pos1.crop_left, 665, "pos1.crop_left"); + + compositor.move_pos(-10, 0); + let (pos0, pos1) = compositor.get_positions(); + + assert_eq!(compositor.zoom, 40, "compositor.zoom"); + assert_eq!(compositor.offset_x, -20, "compositor.offset_x"); + assert_eq!( + pos0.width + pos1.width, + current_width, + "pos0.width + pos1.width" + ); + assert_eq!(pos0.crop_right, 590, "pos0.crop_right"); + assert_eq!(pos1.crop_left, 690, "pos1.crop_left"); + + compositor.move_pos(-10, 0); + let (pos0, pos1) = compositor.get_positions(); + + assert_eq!(compositor.zoom, 40, "compositor.zoom"); + assert_eq!(compositor.offset_x, -30, "compositor.offset_x"); + assert_eq!( + pos0.width + pos1.width, + current_width, + "pos0.width + pos1.width" + ); + assert_eq!(pos0.crop_right, 565, "pos0.crop_right"); + assert_eq!(pos1.crop_left, 715, "pos1.crop_left"); + + compositor.move_pos(-10, 0); + let (pos0, pos1) = compositor.get_positions(); + + assert_eq!(compositor.zoom, 40, "compositor.zoom"); + assert_eq!(compositor.offset_x, -40, "compositor.offset_x"); + assert_eq!( + pos0.width + pos1.width, + current_width, + "pos0.width + pos1.width" + ); + assert_eq!(pos0.crop_right, 540, "pos0.crop_right"); + assert_eq!(pos1.crop_left, 740, "pos1.crop_left"); + } + + #[test] + fn test_split_zoom_out_five_times_only_one_video() { + let mut compositor = Compositor::default(); + compositor.zoom_out(); + compositor.zoom_out(); + compositor.zoom_out(); + compositor.zoom_out(); + compositor.zoom_out(); + compositor.zoom_out(); + let (pos0, pos1) = compositor.get_positions(); + + assert_eq!(compositor.zoom, 40, "compositor.zoom"); + assert_eq!(compositor.offset_x, 0, "compositor.offset_x"); + assert_eq!(compositor.offset_y, 0, "compositor.offset_y"); + assert_eq!(compositor.border, HALF_WIDTH, "compositor.border"); + assert_eq!(compositor.width, WIDTH, "compositor.width"); + assert_eq!(compositor.height, HEIGHT, "compositor.height"); + + assert_eq!(pos0.xpos, 384, "pos0.xpos"); + assert_eq!(pos0.ypos, 216, "pos0.ypos"); + assert_eq!(pos0.width, 256, "pos0.width"); + assert_eq!(pos0.height, 288, "pos0.height"); + assert_eq!(pos0.crop_right, 640, "pos0.crop_right"); + assert_eq!(pos0.crop_left, 0, "pos0.crop_left"); + + assert_eq!(pos1.xpos, 640, "pos1.xpos"); + assert_eq!(pos1.ypos, 216, "pos1.ypos"); + assert_eq!(pos1.width, 256, "pos1.width"); + assert_eq!(pos1.height, 288, "pos1.height"); + assert_eq!(pos1.crop_right, 0, "pos1.crop_right"); + assert_eq!(pos1.crop_left, 640, "pos1.crop_left"); + + compositor.move_border_to(0); + let (pos0, pos1) = compositor.get_positions(); + assert_eq!(compositor.zoom, 40, "compositor.zoom"); + assert_eq!(compositor.offset_x, 0, "compositor.offset_x"); + assert_eq!(compositor.offset_y, 0, "compositor.offset_y"); + assert_eq!(compositor.border, 0, "compositor.border"); + assert_eq!(compositor.width, WIDTH, "compositor.width"); + assert_eq!(compositor.height, HEIGHT, "compositor.height"); + + assert_eq!(pos0.xpos, 0, "pos0.xpos"); + assert_eq!(pos0.ypos, 216, "pos0.ypos"); + assert_eq!(pos0.width, 0, "pos0.width"); + assert_eq!(pos0.height, 288, "pos0.height"); + assert_eq!(pos0.crop_right, 1280, "pos0.crop_right"); + assert_eq!(pos0.crop_left, 0, "pos0.crop_left"); + + assert_eq!(pos1.xpos, 384, "pos1.xpos"); + assert_eq!(pos1.ypos, 216, "pos1.ypos"); + assert_eq!(pos1.width, 512, "pos1.width"); + assert_eq!(pos1.height, 288, "pos1.height"); + assert_eq!(pos1.crop_right, 0, "pos1.crop_right"); + assert_eq!(pos1.crop_left, 0, "pos1.crop_left"); + } + + #[test] + fn test_split_zoom_inout_center_at() { + let mut compositor = Compositor::default(); + compositor.zoom_in_center_at(0, 0); + + assert_eq!(compositor.zoom, 110, "compositor.zoom"); + assert_eq!(compositor.offset_x, 64, "compositor.offset_x"); + assert_eq!(compositor.offset_y, 36, "compositor.offset_y"); + assert_eq!(compositor.border, HALF_WIDTH, "compositor.border"); + assert_eq!(compositor.width, WIDTH, "compositor.width"); + assert_eq!(compositor.height, HEIGHT, "compositor.height"); + + compositor.zoom_out_center_at(0, 0); + assert_eq!(compositor.zoom, 100, "compositor.zoom"); + assert_eq!(compositor.offset_x, 0, "compositor.offset_x"); + assert_eq!(compositor.offset_y, 0, "compositor.offset_y"); + assert_eq!(compositor.border, HALF_WIDTH, "compositor.border"); + assert_eq!(compositor.width, WIDTH, "compositor.width"); + assert_eq!(compositor.height, HEIGHT, "compositor.height"); + + compositor.zoom_out_center_at(0, 0); + assert_eq!(compositor.zoom, 90, "compositor.zoom"); + assert_eq!(compositor.offset_x, -64, "compositor.offset_x"); + assert_eq!(compositor.offset_y, -36, "compositor.offset_y"); + assert_eq!(compositor.border, HALF_WIDTH, "compositor.border"); + assert_eq!(compositor.width, WIDTH, "compositor.width"); + assert_eq!(compositor.height, HEIGHT, "compositor.height"); + } + + #[test] + fn test_split_new_zoom_inout_center_at() { + let width = 12800; + let height = 7200; + let half_width = 12800 / 2; + + let mut compositor = Compositor::new(Mode::Split, width, height); + assert_eq!(compositor.zoom, 100, "compositor.zoom"); + assert_eq!(compositor.offset_x, 0, "compositor.offset_x"); + assert_eq!(compositor.offset_y, 0, "compositor.offset_y"); + assert_eq!(compositor.border, half_width, "compositor.border"); + assert_eq!(compositor.width, width, "compositor.width"); + assert_eq!(compositor.height, height, "compositor.height"); + + compositor.zoom_in_center_at(0, 0); + + assert_eq!(compositor.zoom, 110, "compositor.zoom"); + assert_eq!(compositor.offset_x, 640, "compositor.offset_x"); + assert_eq!(compositor.offset_y, 360, "compositor.offset_y"); + assert_eq!(compositor.border, half_width, "compositor.border"); + assert_eq!(compositor.width, width, "compositor.width"); + assert_eq!(compositor.height, height, "compositor.height"); + + compositor.zoom_out_center_at(0, 0); + assert_eq!(compositor.zoom, 100, "compositor.zoom"); + assert_eq!(compositor.offset_x, 0, "compositor.offset_x"); + assert_eq!(compositor.offset_y, 0, "compositor.offset_y"); + assert_eq!(compositor.border, half_width, "compositor.border"); + assert_eq!(compositor.width, width, "compositor.width"); + assert_eq!(compositor.height, height, "compositor.height"); + + compositor.zoom_out_center_at(0, 0); + assert_eq!(compositor.zoom, 90, "compositor.zoom"); + assert_eq!(compositor.offset_x, -640, "compositor.offset_x"); + assert_eq!(compositor.offset_y, -360, "compositor.offset_y"); + assert_eq!(compositor.border, half_width, "compositor.border"); + assert_eq!(compositor.width, width, "compositor.width"); + assert_eq!(compositor.height, height, "compositor.height"); + } + + #[test] + fn test_sidebyside_get_positions_default() { + let mut compositor = Compositor::default(); + compositor.side_by_side_mode(); + + let (pos0, pos1) = compositor.get_positions(); + + assert_eq!(pos0.xpos, 0, "pos0.xpos"); + assert_eq!(pos0.ypos, HEIGHT / 4, "pos0.ypos"); + assert_eq!(pos0.width, HALF_WIDTH, "pos0.width"); + assert_eq!(pos0.height, HEIGHT / 2, "pos0.height"); + assert_eq!(pos0.crop_right, 0, "pos0.crop_right"); + assert_eq!(pos0.crop_left, 0, "pos0.crop_left"); + + assert_eq!(pos1.xpos, HALF_WIDTH, "pos1.xpos"); + assert_eq!(pos1.ypos, HEIGHT / 4, "pos1.ypos"); + assert_eq!(pos1.width, HALF_WIDTH, "pos1.width"); + assert_eq!(pos1.height, HEIGHT / 2, "pos1.height"); + assert_eq!(pos1.crop_right, 0, "pos1.crop_right"); + assert_eq!(pos1.crop_left, 0, "pos1.crop_left"); + } + + #[test] + fn test_sidebyside_move_pos_up_reset_down() { + let mut compositor = Compositor::default(); + compositor.side_by_side_mode(); + + compositor.move_pos(0, -10); + let (pos0, pos1) = compositor.get_positions(); + + assert_eq!(compositor.zoom, 100, "compositor.zoom"); + assert_eq!(compositor.offset_x, 0, "compositor.offset_x"); + assert_eq!(compositor.offset_y, -10, "compositor.offset_y"); + assert_eq!(compositor.border, HALF_WIDTH, "compositor.border"); + assert_eq!(compositor.width, WIDTH, "compositor.width"); + assert_eq!(compositor.height, HEIGHT, "compositor.height"); + + assert_eq!(pos0.xpos, 0, "pos0.xpos"); + assert_eq!(pos0.ypos, 170, "pos0.ypos"); + assert_eq!(pos0.width, 640, "pos0.width"); + assert_eq!(pos0.height, HALF_HEIGHT, "pos0.height"); + assert_eq!(pos0.crop_right, 0, "pos0.crop_right"); + assert_eq!(pos0.crop_left, 0, "pos0.crop_left"); + + assert_eq!(pos1.xpos, HALF_WIDTH, "pos1.xpos"); + assert_eq!(pos1.ypos, 170, "pos1.ypos"); + assert_eq!(pos1.width, 640, "pos1.width"); + assert_eq!(pos1.height, HALF_HEIGHT, "pos1.height"); + assert_eq!(pos1.crop_right, 0, "pos1.crop_right"); + assert_eq!(pos1.crop_left, 0, "pos1.crop_left"); + + compositor.reset_position(); + assert!( + compositor.is_side_by_side_mode(), + "compositor.is_side_by_side_mode" + ); + assert_eq!(compositor.zoom, 100, "compositor.zoom"); + assert_eq!(compositor.offset_x, 0, "compositor.offset_x"); + assert_eq!(compositor.offset_y, 0, "compositor.offset_y"); + assert_eq!(compositor.border, HALF_WIDTH, "compositor.border"); + assert_eq!(compositor.width, WIDTH, "compositor.width"); + assert_eq!(compositor.height, HEIGHT, "compositor.height"); + + compositor.move_pos(0, 10); + let (pos0, pos1) = compositor.get_positions(); + + assert_eq!(compositor.zoom, 100, "compositor.zoom"); + assert_eq!(compositor.offset_x, 0, "compositor.offset_x"); + assert_eq!(compositor.offset_y, 10, "compositor.offset_y"); + assert_eq!(compositor.border, HALF_WIDTH, "compositor.border"); + assert_eq!(compositor.width, WIDTH, "compositor.width"); + assert_eq!(compositor.height, HEIGHT, "compositor.height"); + + assert_eq!(pos0.xpos, 0, "pos0.xpos"); + assert_eq!(pos0.ypos, 190, "pos0.ypos"); + assert_eq!(pos0.width, 640, "pos0.width"); + assert_eq!(pos0.height, HALF_HEIGHT, "pos0.height"); + assert_eq!(pos0.crop_right, 0, "pos0.crop_right"); + assert_eq!(pos0.crop_left, 0, "pos0.crop_left"); + + assert_eq!(pos1.xpos, HALF_WIDTH, "pos1.xpos"); + assert_eq!(pos1.ypos, 190, "pos1.ypos"); + assert_eq!(pos1.width, 640, "pos1.width"); + assert_eq!(pos1.height, HALF_HEIGHT, "pos1.height"); + assert_eq!(pos1.crop_right, 0, "pos1.crop_right"); + assert_eq!(pos1.crop_left, 0, "pos1.crop_left"); + } + + #[test] + fn test_sidebyside_move_pos_left_reset_right() { + let mut compositor = Compositor::default(); + compositor.side_by_side_mode(); + + compositor.move_pos(-10, 0); + let (pos0, pos1) = compositor.get_positions(); + + assert_eq!(compositor.zoom, 100, "compositor.zoom"); + assert_eq!(compositor.offset_x, -10, "compositor.offset_x"); + assert_eq!(compositor.offset_y, 0, "compositor.offset_y"); + assert_eq!(compositor.border, HALF_WIDTH, "compositor.border"); + assert_eq!(compositor.width, WIDTH, "compositor.width"); + assert_eq!(compositor.height, HEIGHT, "compositor.height"); + + assert_eq!(pos0.xpos, -10, "pos0.xpos"); + assert_eq!(pos0.ypos, 180, "pos0.ypos"); + assert_eq!(pos0.width, 640, "pos0.width"); + assert_eq!(pos0.height, HALF_HEIGHT, "pos0.height"); + assert_eq!(pos0.crop_right, 0, "pos0.crop_right"); + assert_eq!(pos0.crop_left, 0, "pos0.crop_left"); + + assert_eq!(pos1.xpos, HALF_WIDTH, "pos1.xpos"); + assert_eq!(pos1.ypos, 180, "pos1.ypos"); + assert_eq!(pos1.width, 630, "pos1.width"); + assert_eq!(pos1.height, HALF_HEIGHT, "pos1.height"); + assert_eq!(pos1.crop_right, 0, "pos1.crop_right"); + assert_eq!(pos1.crop_left, 20, "pos1.crop_left"); + + compositor.reset_position(); + assert!( + compositor.is_side_by_side_mode(), + "compositor.is_side_by_side_mode" + ); + assert_eq!(compositor.zoom, 100, "compositor.zoom"); + assert_eq!(compositor.offset_x, 0, "compositor.offset_x"); + assert_eq!(compositor.offset_y, 0, "compositor.offset_y"); + assert_eq!(compositor.border, HALF_WIDTH, "compositor.border"); + assert_eq!(compositor.width, WIDTH, "compositor.width"); + assert_eq!(compositor.height, HEIGHT, "compositor.height"); + + compositor.move_pos(10, 0); + let (pos0, pos1) = compositor.get_positions(); + + assert_eq!(compositor.zoom, 100, "compositor.zoom"); + assert_eq!(compositor.offset_x, 10, "compositor.offset_x"); + assert_eq!(compositor.offset_y, 0, "compositor.offset_y"); + assert_eq!(compositor.border, HALF_WIDTH, "compositor.border"); + assert_eq!(compositor.width, WIDTH, "compositor.width"); + assert_eq!(compositor.height, HEIGHT, "compositor.height"); + + assert_eq!(pos0.xpos, 10, "pos0.xpos"); + assert_eq!(pos0.ypos, 180, "pos0.ypos"); + assert_eq!(pos0.width, 630, "pos0.width"); + assert_eq!(pos0.height, HALF_HEIGHT, "pos0.height"); + assert_eq!(pos0.crop_right, 20, "pos0.crop_right"); + assert_eq!(pos0.crop_left, 0, "pos0.crop_left"); + + assert_eq!(pos1.xpos, HALF_WIDTH + 10, "pos1.xpos"); + assert_eq!(pos1.ypos, 180, "pos1.ypos"); + assert_eq!(pos1.width, 640, "pos1.width"); + assert_eq!(pos1.height, HALF_HEIGHT, "pos1.height"); + assert_eq!(pos1.crop_right, 0, "pos1.crop_right"); + assert_eq!(pos1.crop_left, 0, "pos1.crop_left"); + } + + #[test] + fn test_sidebyside_move_pos_left_out_of_border() { + let mut compositor = Compositor::default(); + compositor.side_by_side_mode(); + + compositor.move_pos(-1000, 0); + let (pos0, pos1) = compositor.get_positions(); + + assert_eq!(compositor.zoom, 100, "compositor.zoom"); + assert_eq!(compositor.offset_x, -1000, "compositor.offset_x"); + assert_eq!(compositor.offset_y, 0, "compositor.offset_y"); + assert_eq!(compositor.border, HALF_WIDTH, "compositor.border"); + assert_eq!(compositor.width, WIDTH, "compositor.width"); + assert_eq!(compositor.height, HEIGHT, "compositor.height"); + + assert_eq!(pos0.xpos, -1000, "pos0.xpos"); + assert_eq!(pos0.ypos, 180, "pos0.ypos"); + assert_eq!(pos0.width, HALF_WIDTH, "pos0.width"); + assert_eq!(pos0.height, HALF_HEIGHT, "pos0.height"); + assert_eq!(pos0.crop_right, 0, "pos0.crop_right"); + assert_eq!(pos0.crop_left, 0, "pos0.crop_left"); + + assert_eq!(pos1.xpos, HALF_WIDTH, "pos1.xpos"); + assert_eq!(pos1.ypos, 180, "pos1.ypos"); + assert_eq!(pos1.width, 0, "pos1.width"); + assert_eq!(pos1.height, HALF_HEIGHT, "pos1.height"); + assert_eq!(pos1.crop_right, 0, "pos1.crop_right"); + assert_eq!(pos1.crop_left, 0, "pos1.crop_left"); + } + + #[test] + fn test_sidebyside_move_pos_right_out_of_border() { + let mut compositor = Compositor::default(); + compositor.side_by_side_mode(); + + compositor.move_pos(1000, 0); + let (pos0, pos1) = compositor.get_positions(); + + assert_eq!(compositor.zoom, 100, "compositor.zoom"); + assert_eq!(compositor.offset_x, 1000, "compositor.offset_x"); + assert_eq!(compositor.offset_y, 0, "compositor.offset_y"); + assert_eq!(compositor.border, HALF_WIDTH, "compositor.border"); + assert_eq!(compositor.width, WIDTH, "compositor.width"); + assert_eq!(compositor.height, HEIGHT, "compositor.height"); + + assert_eq!(pos0.xpos, 0, "pos0.xpos"); + assert_eq!(pos0.ypos, 180, "pos0.ypos"); + assert_eq!(pos0.width, 0, "pos0.width"); + assert_eq!(pos0.height, HALF_HEIGHT, "pos0.height"); + assert_eq!(pos0.crop_right, 0, "pos0.crop_right"); + assert_eq!(pos0.crop_left, 0, "pos0.crop_left"); + + assert_eq!(pos1.xpos, 1640, "pos1.xpos"); + assert_eq!(pos1.ypos, 180, "pos1.ypos"); + assert_eq!(pos1.width, HALF_WIDTH, "pos1.width"); + assert_eq!(pos1.height, HALF_HEIGHT, "pos1.height"); + assert_eq!(pos1.crop_right, 0, "pos1.crop_right"); + assert_eq!(pos1.crop_left, 0, "pos1.crop_left"); + } + + #[test] + fn test_sidebyside_zoom_in() { + let mut compositor = Compositor::default(); + compositor.side_by_side_mode(); + + compositor.zoom_in(); + let (pos0, pos1) = compositor.get_positions(); + + assert_eq!(compositor.zoom, 110, "compositor.zoom"); + assert_eq!(compositor.offset_x, 0, "compositor.offset_x"); + assert_eq!(compositor.offset_y, 0, "compositor.offset_y"); + assert_eq!(compositor.border, HALF_WIDTH, "compositor.border"); + assert_eq!(compositor.width, WIDTH, "compositor.width"); + assert_eq!(compositor.height, HEIGHT, "compositor.height"); + + assert_eq!(pos0.xpos, -32, "pos0.xpos"); + assert_eq!(pos0.ypos, 162, "pos0.ypos"); + assert_eq!(pos0.width, 704 - 32, "pos0.width"); + assert_eq!(pos0.height, 396, "pos0.height"); + assert_eq!(pos0.crop_right, 58, "pos0.crop_right"); + assert_eq!(pos0.crop_left, 0, "pos0.crop_left"); + + assert_eq!(pos1.xpos, 640, "pos1.xpos"); + assert_eq!(pos1.ypos, 162, "pos1.ypos"); + assert_eq!(pos1.width, 672, "pos1.width"); + assert_eq!(pos1.height, 396, "pos1.height"); + assert_eq!(pos1.crop_right, 0, "pos1.crop_right"); + assert_eq!(pos1.crop_left, 58, "pos1.crop_left"); + } + + #[test] + fn test_sidebyside_zoom_out_five_times() { + let mut compositor = Compositor::default(); + compositor.side_by_side_mode(); + + compositor.zoom_out(); + let (pos0, pos1) = compositor.get_positions(); + + assert_eq!(compositor.zoom, 90, "compositor.zoom"); + assert_eq!(compositor.offset_x, 0, "compositor.offset_x"); + assert_eq!(compositor.offset_y, 0, "compositor.offset_y"); + assert_eq!(compositor.border, HALF_WIDTH, "compositor.border"); + assert_eq!(compositor.width, WIDTH, "compositor.width"); + assert_eq!(compositor.height, HEIGHT, "compositor.height"); + + assert_eq!(pos0.xpos, 32, "pos0.xpos"); + assert_eq!(pos0.ypos, 198, "pos0.ypos"); + assert_eq!(pos0.width, 576, "pos0.width"); + assert_eq!(pos0.height, 324, "pos0.height"); + assert_eq!(pos0.crop_right, 0, "pos0.crop_right"); + assert_eq!(pos0.crop_left, 0, "pos0.crop_left"); + + assert_eq!(pos1.xpos, 640 + 32, "pos1.xpos"); + assert_eq!(pos1.ypos, 198, "pos1.ypos"); + assert_eq!(pos1.width, 576, "pos1.width"); + assert_eq!(pos1.height, 324, "pos1.height"); + assert_eq!(pos1.crop_right, 0, "pos1.crop_right"); + assert_eq!(pos1.crop_left, 0, "pos1.crop_left"); + + compositor.zoom_out(); + compositor.zoom_out(); + compositor.zoom_out(); + compositor.zoom_out(); + compositor.zoom_out(); + let (pos0, pos1) = compositor.get_positions(); + + assert_eq!(compositor.zoom, 40, "compositor.zoom"); + assert_eq!(compositor.offset_x, 0, "compositor.offset_x"); + assert_eq!(compositor.offset_y, 0, "compositor.offset_y"); + assert_eq!(compositor.border, HALF_WIDTH, "compositor.border"); + assert_eq!(compositor.width, WIDTH, "compositor.width"); + assert_eq!(compositor.height, HEIGHT, "compositor.height"); + + assert_eq!(pos0.xpos, 192, "pos0.xpos"); + assert_eq!(pos0.ypos, 288, "pos0.ypos"); + assert_eq!(pos0.width, 256, "pos0.width"); + assert_eq!(pos0.height, 144, "pos0.height"); + assert_eq!(pos0.crop_right, 0, "pos0.crop_right"); + assert_eq!(pos0.crop_left, 0, "pos0.crop_left"); + + assert_eq!(pos1.xpos, 640 + 192, "pos1.xpos"); + assert_eq!(pos1.ypos, 288, "pos1.ypos"); + assert_eq!(pos1.width, 256, "pos1.width"); + assert_eq!(pos1.height, 144, "pos1.height"); + assert_eq!(pos1.crop_right, 0, "pos1.crop_right"); + assert_eq!(pos1.crop_left, 0, "pos1.crop_left"); + } + + #[test] + fn test_sidebyside_zoom_in_and_move() { + let mut compositor = Compositor::default(); + compositor.side_by_side_mode(); + + compositor.zoom_in(); + compositor.move_pos(-20, 0); + let (pos0, pos1) = compositor.get_positions(); + + assert_eq!(compositor.zoom, 110, "compositor.zoom"); + assert_eq!(compositor.offset_x, -20, "compositor.offset_x"); + assert_eq!(compositor.offset_y, 0, "compositor.offset_y"); + assert_eq!(compositor.border, HALF_WIDTH, "compositor.border"); + assert_eq!(compositor.width, WIDTH, "compositor.width"); + assert_eq!(compositor.height, HEIGHT, "compositor.height"); + + assert_eq!(pos0.xpos, -52, "pos0.xpos"); + assert_eq!(pos0.ypos, 162, "pos0.ypos"); + assert_eq!(pos0.width, 692, "pos0.width"); + assert_eq!(pos0.height, 396, "pos0.height"); + assert_eq!(pos0.crop_right, 21, "pos0.crop_right"); + assert_eq!(pos0.crop_left, 0, "pos0.crop_left"); + + assert_eq!(pos1.xpos, 640, "pos1.xpos"); + assert_eq!(pos1.ypos, 162, "pos1.ypos"); + assert_eq!(pos1.width, 704 - 52, "pos1.width"); + assert_eq!(pos1.height, 396, "pos1.height"); + assert_eq!(pos1.crop_right, 0, "pos1.crop_right"); + assert_eq!(pos1.crop_left, 94, "pos1.crop_left"); + } + + #[test] + fn test_sidebyside_zoom_out_five_times_and_move() { + let mut compositor = Compositor::default(); + compositor.side_by_side_mode(); + + compositor.zoom_out(); + compositor.zoom_out(); + compositor.zoom_out(); + compositor.zoom_out(); + compositor.zoom_out(); + compositor.zoom_out(); + let (pos0, pos1) = compositor.get_positions(); + + assert_eq!(compositor.zoom, 40, "compositor.zoom"); + assert_eq!(compositor.offset_x, 0, "compositor.offset_x"); + assert_eq!(compositor.offset_y, 0, "compositor.offset_y"); + assert_eq!(compositor.border, HALF_WIDTH, "compositor.border"); + assert_eq!(compositor.width, WIDTH, "compositor.width"); + assert_eq!(compositor.height, HEIGHT, "compositor.height"); + + assert_eq!(pos0.xpos, 192, "pos0.xpos"); + assert_eq!(pos0.ypos, 288, "pos0.ypos"); + assert_eq!(pos0.width, 256, "pos0.width"); + assert_eq!(pos0.height, 144, "pos0.height"); + assert_eq!(pos0.crop_right, 0, "pos0.crop_right"); + assert_eq!(pos0.crop_left, 0, "pos0.crop_left"); + + assert_eq!(pos1.xpos, 640 + 192, "pos1.xpos"); + assert_eq!(pos1.ypos, 288, "pos1.ypos"); + assert_eq!(pos1.width, 256, "pos1.width"); + assert_eq!(pos1.height, 144, "pos1.height"); + assert_eq!(pos1.crop_right, 0, "pos1.crop_right"); + assert_eq!(pos1.crop_left, 0, "pos1.crop_left"); + + let current_width = pos0.width + pos1.width; + compositor.move_pos(-10, 0); + let (pos0, pos1) = compositor.get_positions(); + + assert_eq!(compositor.zoom, 40, "compositor.zoom"); + assert_eq!(compositor.offset_x, -10, "compositor.offset_x"); + assert_eq!( + pos0.width + pos1.width, + current_width, + "pos0.width + pos1.width" + ); + assert_eq!(pos1.xpos, 640 + 192 - 10, "pos1.xpos"); + assert_eq!(pos0.crop_right, 0, "pos0.crop_right"); + assert_eq!(pos1.crop_left, 0, "pos1.crop_left"); + + compositor.move_pos(-10, 0); + let (pos0, pos1) = compositor.get_positions(); + + assert_eq!(compositor.zoom, 40, "compositor.zoom"); + assert_eq!(compositor.offset_x, -20, "compositor.offset_x"); + assert_eq!( + pos0.width + pos1.width, + current_width, + "pos0.width + pos1.width" + ); + assert_eq!(pos0.crop_right, 0, "pos0.crop_right"); + assert_eq!(pos1.crop_left, 0, "pos1.crop_left"); + + compositor.move_pos(-10, 0); + let (pos0, pos1) = compositor.get_positions(); + + assert_eq!(compositor.zoom, 40, "compositor.zoom"); + assert_eq!(compositor.offset_x, -30, "compositor.offset_x"); + assert_eq!( + pos0.width + pos1.width, + current_width, + "pos0.width + pos1.width" + ); + assert_eq!(pos0.crop_right, 0, "pos0.crop_right"); + assert_eq!(pos1.crop_left, 0, "pos1.crop_left"); + + compositor.move_pos(-200, 0); + let (pos0, pos1) = compositor.get_positions(); + + assert_eq!(compositor.zoom, 40, "compositor.zoom"); + assert_eq!(compositor.offset_x, -230, "compositor.offset_x"); + + assert_eq!(pos0.xpos, -38, "pos0.xpos"); + assert_eq!(pos0.ypos, 288, "pos0.ypos"); + assert_eq!(pos0.width, 256, "pos0.width"); + assert_eq!(pos0.height, 144, "pos0.height"); + assert_eq!(pos0.crop_right, 0, "pos0.crop_right"); + assert_eq!(pos0.crop_left, 0, "pos0.crop_left"); + + assert_eq!(pos1.xpos, 640, "pos1.xpos"); + assert_eq!(pos1.ypos, 288, "pos1.ypos"); + assert_eq!(pos1.width, 218, "pos1.width"); + assert_eq!(pos1.height, 144, "pos1.height"); + assert_eq!(pos1.crop_right, 0, "pos1.crop_right"); + assert_eq!(pos1.crop_left, 190, "pos1.crop_left"); + } + + #[test] + fn test_sidebyside_zoom_inout_center_at() { + let mut compositor = Compositor::default(); + compositor.side_by_side_mode(); + + compositor.zoom_in_center_at(0, 0); + + assert_eq!(compositor.zoom, 110, "compositor.zoom"); + assert_eq!(compositor.offset_x, 32, "compositor.offset_x"); + assert_eq!(compositor.offset_y, 36, "compositor.offset_y"); + assert_eq!(compositor.border, HALF_WIDTH, "compositor.border"); + assert_eq!(compositor.width, WIDTH, "compositor.width"); + assert_eq!(compositor.height, HEIGHT, "compositor.height"); + + compositor.zoom_out_center_at(0, 0); + assert_eq!(compositor.zoom, 100, "compositor.zoom"); + assert_eq!(compositor.offset_x, 0, "compositor.offset_x"); + assert_eq!(compositor.offset_y, 0, "compositor.offset_y"); + assert_eq!(compositor.border, HALF_WIDTH, "compositor.border"); + assert_eq!(compositor.width, WIDTH, "compositor.width"); + assert_eq!(compositor.height, HEIGHT, "compositor.height"); + + compositor.zoom_out_center_at(0, 0); + assert_eq!(compositor.zoom, 90, "compositor.zoom"); + assert_eq!(compositor.offset_x, -32, "compositor.offset_x"); + assert_eq!(compositor.offset_y, -36, "compositor.offset_y"); + assert_eq!(compositor.border, HALF_WIDTH, "compositor.border"); + assert_eq!(compositor.width, WIDTH, "compositor.width"); + assert_eq!(compositor.height, HEIGHT, "compositor.height"); + } + + #[test] + fn test_sidebyside_new_zoom_inout_center_at() { + let width = 12800; + let height = 7200; + let half_width = 12800 / 2; + + let mut compositor = Compositor::new(Mode::Split, width, height); + assert_eq!(compositor.zoom, 100, "compositor.zoom"); + assert_eq!(compositor.offset_x, 0, "compositor.offset_x"); + assert_eq!(compositor.offset_y, 0, "compositor.offset_y"); + assert_eq!(compositor.border, half_width, "compositor.border"); + assert_eq!(compositor.width, width, "compositor.width"); + assert_eq!(compositor.height, height, "compositor.height"); + + compositor.zoom_in_center_at(0, 0); + + assert_eq!(compositor.zoom, 110, "compositor.zoom"); + assert_eq!(compositor.offset_x, 640, "compositor.offset_x"); + assert_eq!(compositor.offset_y, 360, "compositor.offset_y"); + assert_eq!(compositor.border, half_width, "compositor.border"); + assert_eq!(compositor.width, width, "compositor.width"); + assert_eq!(compositor.height, height, "compositor.height"); + + compositor.zoom_out_center_at(0, 0); + assert_eq!(compositor.zoom, 100, "compositor.zoom"); + assert_eq!(compositor.offset_x, 0, "compositor.offset_x"); + assert_eq!(compositor.offset_y, 0, "compositor.offset_y"); + assert_eq!(compositor.border, half_width, "compositor.border"); + assert_eq!(compositor.width, width, "compositor.width"); + assert_eq!(compositor.height, height, "compositor.height"); + + compositor.zoom_out_center_at(0, 0); + assert_eq!(compositor.zoom, 90, "compositor.zoom"); + assert_eq!(compositor.offset_x, -640, "compositor.offset_x"); + assert_eq!(compositor.offset_y, -360, "compositor.offset_y"); + assert_eq!(compositor.border, half_width, "compositor.border"); + assert_eq!(compositor.width, width, "compositor.width"); + assert_eq!(compositor.height, height, "compositor.height"); + } + + #[test] + fn test_sidebyside_bug_1() { + let mut compositor = Compositor { + mode: Mode::SideBySide, + zoom: 320, + offset_x: 315, + offset_y: -103, + ..Default::default() + }; + + let (pos0, pos1) = compositor.get_positions(); + + assert_eq!(compositor.zoom, 320, "compositor.zoom"); + assert_eq!(compositor.offset_x, 315, "compositor.offset_x"); + assert_eq!(compositor.offset_y, -103, "compositor.offset_y"); + assert_eq!(compositor.width, WIDTH, "compositor.width"); + assert_eq!(compositor.height, HEIGHT, "compositor.height"); + + assert_eq!(pos0.xpos, -389, "pos0.xpos"); + assert_eq!(pos0.ypos, -319, "pos0.ypos"); + assert_eq!(pos0.width, 1029, "pos0.width"); + assert_eq!(pos0.height, 1152, "pos0.height"); + assert_eq!(pos0.crop_right, 636, "pos0.crop_right"); + assert_eq!(pos0.crop_left, 0, "pos0.crop_left"); + + assert_eq!(pos1.xpos, 640, "pos1.xpos"); + assert_eq!(pos1.ypos, -319, "pos1.ypos"); + assert_eq!(pos1.width, 1659, "pos1.width"); + assert_eq!(pos1.height, 1152, "pos1.height"); + assert_eq!(pos1.crop_right, 0, "pos1.crop_right"); + assert_eq!(pos1.crop_left, 243, "pos1.crop_left"); + + compositor.move_pos(30, 0); + let (pos0, pos1) = compositor.get_positions(); + + assert_eq!(compositor.zoom, 320, "compositor.zoom"); + assert_eq!(compositor.offset_x, 345, "compositor.offset_x"); + assert_eq!(compositor.offset_y, -103, "compositor.offset_y"); + assert_eq!(compositor.width, WIDTH, "compositor.width"); + assert_eq!(compositor.height, HEIGHT, "compositor.height"); + + assert_eq!(pos0.xpos, -359, "pos0.xpos"); + assert_eq!(pos0.ypos, -319, "pos0.ypos"); + assert_eq!(pos0.width, 999, "pos0.width"); + assert_eq!(pos0.height, 1152, "pos0.height"); + assert_eq!(pos0.crop_right, 655, "pos0.crop_right"); + assert_eq!(pos0.crop_left, 0, "pos0.crop_left"); + + assert_eq!(pos1.xpos, 640, "pos1.xpos"); + assert_eq!(pos1.ypos, -319, "pos1.ypos"); + assert_eq!(pos1.width, 1689, "pos1.width"); + assert_eq!(pos1.height, 1152, "pos1.height"); + assert_eq!(pos1.crop_right, 0, "pos1.crop_right"); + assert_eq!(pos1.crop_left, 224, "pos1.crop_left"); + } +} diff --git a/video/stats/src/comparemixer/imp.rs b/video/stats/src/comparemixer/imp.rs index c1c7a65e1..a61f2348f 100644 --- a/video/stats/src/comparemixer/imp.rs +++ b/video/stats/src/comparemixer/imp.rs @@ -10,10 +10,14 @@ use gst::glib; use gst::prelude::*; use gst::subclass::prelude::*; +use gst_video::NavigationEvent; use crate::videoencoderstatsmeta::VideoEncoderStatsMeta; +use crate::comparemixer::compositor::Compositor; +use crate::comparemixer::compositor::Position; +use crate::comparemixer::compositor::Mode; -use std::sync::{LazyLock, Mutex}; +use std::sync::{LazyLock, Mutex, Arc}; use std::vec::Vec; static CAT: LazyLock = LazyLock::new(|| { @@ -24,6 +28,15 @@ static CAT: LazyLock = LazyLock::new(|| { ) }); +#[derive(Default)] +pub struct MouseState { + clicked: bool, + clicked_x: f64, + clicked_y: f64, + clicked_xpos: i32, + clicked_ypos: i32, +} + #[derive(Default, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Clone, Copy, glib::Enum)] #[enum_type(name = "GstVideoCompareMixerBackend")] #[repr(u32)] @@ -45,6 +58,17 @@ pub enum Backend { struct Settings { backend: Backend, split_screen: bool, + navigation_events: bool, +} + +impl Default for Settings { + fn default() -> Self { + Self { + backend: Backend::default(), + split_screen: false, + navigation_events: true, + } + } } pub struct VideoCompareMixer { @@ -56,15 +80,8 @@ pub struct VideoCompareMixer { overlay0: gst::Element, overlay1: gst::Element, settings: Mutex, -} - -impl Default for Settings { - fn default() -> Self { - Self { - backend: Backend::default(), - split_screen: false, - } - } + mouse_state: Arc>, + compositor_helper: Arc>>, } impl VideoCompareMixer { @@ -79,9 +96,247 @@ impl VideoCompareMixer { } } + pub fn add_navigation_events_probe( + &self, + ) { + let compositor_supports_crop: bool = self.settings.lock().unwrap().backend == Backend::GL; + + let mixer = self.obj().by_name("compositor").unwrap(); + let crop0 = self.obj().by_name("crop0").unwrap(); + let crop1 = self.obj().by_name("crop1").unwrap(); + let mixer_src_pad = mixer.static_pad("src").unwrap(); + let mixer_sink_0_pad = mixer.static_pad("sink_0").unwrap(); + let mixer_sink_1_pad = mixer.static_pad("sink_1").unwrap(); + + self.update_mixer( + &(*self.compositor_helper.lock().unwrap()).unwrap(), + &mixer_sink_0_pad, + &mixer_sink_1_pad, + &crop0, + &crop1, + compositor_supports_crop, + ); + + let compositor_helper_clone = self.compositor_helper.clone(); + let mouse_state = self.mouse_state.clone(); + let imp_weak = self.downgrade(); + // Probe added in the sink pad to get direct navigation events w/o transformation done by the zoom_mixer + mixer_src_pad.add_probe(gst::PadProbeType::EVENT_UPSTREAM, move |_, probe_info| { + let Some(ev) = probe_info.event() else { + return gst::PadProbeReturn::Ok; + }; + + if ev.type_() != gst::EventType::Navigation { + return gst::PadProbeReturn::Ok; + }; + + let Ok(nav_event) = NavigationEvent::parse(ev) else { + return gst::PadProbeReturn::Ok; + }; + + let compositor = &mut (*compositor_helper_clone.lock().unwrap()).unwrap(); + let original_compositor = *compositor; + let nav_event_clone = nav_event.clone(); + + match nav_event { + NavigationEvent::KeyPress { key, .. } => match key.as_str() { + "Left" | "Left arrow" => { + compositor.move_pos(-10, 0); + } + "Right" | "Right arrow" => { + compositor.move_pos(10, 0); + } + "Up" | "Up arrow" => { + compositor.move_pos(0, -10); + } + "Down" | "Down arrow" => { + compositor.move_pos(0, 10); + } + "plus" | "+" => { + compositor.zoom_in(); + } + "minus" | "-" => { + compositor.zoom_out(); + } + "r" => { + compositor.reset_position(); + } + "Shift_R" | "R" => { + compositor.reset(); + } + "1" => { + compositor.split_mode(); + let w = compositor.width; + compositor.move_border_to(w); + } + "2" => { + compositor.split_mode(); + compositor.move_border_to(0); + } + "3" => { + compositor.split_mode(); + compositor.reset_border(); + } + "4" => { + compositor.side_by_side_mode(); + } + "5" => { + compositor.split_mode(); + compositor.move_border(-10); + } + "6" => { + compositor.split_mode(); + compositor.move_border(10); + } + _ => { + gst::info!(CAT, "Unhandled key: {}", key); + }, + }, + NavigationEvent::MouseMove { x, y, .. } => { + let state = mouse_state.lock().unwrap(); + if state.clicked { + let new_xpos = (x - state.clicked_x) as i32 + state.clicked_xpos; + let new_ypos = (y - state.clicked_y) as i32 + state.clicked_ypos; + + compositor.move_pos_to(new_xpos, new_ypos); + } + } + NavigationEvent::MouseButtonPress { button, x, y, .. } => { + if button == 1 || button == 272 { + let mut state = mouse_state.lock().unwrap(); + state.clicked = true; + state.clicked_x = x; + state.clicked_y = y; + state.clicked_xpos = compositor.offset_x; + state.clicked_ypos = compositor.offset_y; + + if y >= 600.0 { + compositor.move_border_to(x as i32); + } + } else if button == 2 || button == 3 || button == 274 || button == 273 { + compositor.reset(); + } else if button == 4 { + compositor.zoom_in_center_at(x as i32, y as i32); + } else if button == 5 { + compositor.zoom_out_center_at(x as i32, y as i32); + } + } + NavigationEvent::MouseButtonRelease { button, .. } => { + if button == 1 || button == 272 { + let mut state = mouse_state.lock().unwrap(); + state.clicked = false; + } + } + // NavigationEvent::MouseScroll { x, y, delta_x, delta_y, ..} => { + // if delta_y > 0.0 { + // compositor.zoom_in_center_at(x as i32, y as i32); + // } else if delta_y < 0.0 { + // compositor.zoom_out_center_at(x as i32, y as i32); + // } + // } + _ => (), + } + + if original_compositor != *compositor { + gst::log!(CAT, "Compositor changed: {compositor:?}"); + let Some(imp) = imp_weak.upgrade() else { + return gst::PadProbeReturn::Ok; + }; + imp.update_mixer( + compositor, + &mixer_sink_0_pad, + &mixer_sink_1_pad, + &crop0, + &crop1, + compositor_supports_crop, + ); + *imp.compositor_helper.lock().unwrap() = Some(*compositor); + } + + gst::log!(CAT, "Navigation event: {nav_event_clone:?}"); + + gst::PadProbeReturn::Ok + }); + } + + fn fix_pos(pos: &mut Position, width: i32, compositor_supports_crop: bool) { + // workaround to handle gst issue when width==0 with any video mixers + // see `glvideomixer sink_0::width=0` in README.md + if pos.width == 0 { + pos.width = width; + pos.xpos = width; + } + + // workaround to handle gst issue when crop==total_width with compositor and vacompositor + // see `compositor and vacompositor video out of the box` in README.md + if !compositor_supports_crop { + if pos.crop_right == width { + pos.crop_right = width - 10; + } + + if pos.crop_left == width { + pos.crop_left = width - 10; + } + } + } + + fn update_mixer( + &self, + compositor_helper: &Compositor, + mixer_sink_0_pad: &gst::Pad, + mixer_sink_1_pad: &gst::Pad, + crop0: &gst::Element, + crop1: &gst::Element, + compositor_supports_crop: bool, + ) { + let (mut pos0, mut pos1) = compositor_helper.get_positions(); + + Self::fix_pos(&mut pos0, compositor_helper.width, compositor_supports_crop); + Self::fix_pos(&mut pos1, compositor_helper.width, compositor_supports_crop); + + gst::log!(CAT, "Position 0: {}x{}+{}+{}, crop_right: {}", pos0.width, pos0.height, pos0.xpos, pos0.ypos, pos0.crop_right); + gst::log!(CAT, "Position 1: {}x{}+{}+{}, crop_left: {}", pos1.width, pos1.height, pos1.xpos, pos1.ypos, pos1.crop_left); + + //TODO refactor avoid copy and paste + if compositor_supports_crop { + mixer_sink_0_pad.set_properties(&[ + ("width", &pos0.width), + ("height", &pos0.height), + ("xpos", &pos0.xpos), + ("ypos", &pos0.ypos), + ("crop-right", &pos0.crop_right), + ]); + + mixer_sink_1_pad.set_properties(&[ + ("width", &pos1.width), + ("height", &pos1.height), + ("xpos", &pos1.xpos), + ("ypos", &pos1.ypos), + ("crop-left", &pos1.crop_left), + ]); + } else { + mixer_sink_0_pad.set_properties(&[ + ("width", &pos0.width), + ("height", &pos0.height), + ("xpos", &pos0.xpos), + ("ypos", &pos0.ypos), + ]); + + mixer_sink_1_pad.set_properties(&[ + ("width", &pos1.width), + ("height", &pos1.height), + ("xpos", &pos1.xpos), + ("ypos", &pos1.ypos), + ]); + + gst::log!(CAT, "right crop: {}, left crop: {}", pos0.crop_right, pos1.crop_left); + crop0.set_property("right", pos0.crop_right); + crop1.set_property("left", pos1.crop_left); + } + } + fn prepare_pipeline(&self) -> Result<(), gst::ErrorMessage> { let settings = self.settings.lock().unwrap(); - let split_screen = settings.split_screen; let backend = settings.backend; drop(settings); @@ -90,22 +345,21 @@ impl VideoCompareMixer { .expect("Failed to create compositor element"); compositor.set_property("name", "compositor"); - if split_screen && backend != Backend::GL { - let crop0 = gst::ElementFactory::make("videocrop") - .build() - .expect("Failed to create crop0"); - crop0.set_property("name", "crop0"); - - let crop1 = gst::ElementFactory::make("videocrop") - .build() - .expect("Failed to create crop1"); - crop1.set_property("name", "crop1"); - - self.obj().add(&crop0).expect("Failed to add crop0 element"); - self.obj().add(&crop1).expect("Failed to add crop1 element"); - } + let crop0 = gst::ElementFactory::make("videocrop") + .build() + .expect("Failed to create crop0"); + crop0.set_property("name", "crop0"); - self.link_elements(&compositor, split_screen, backend)?; + let crop1 = gst::ElementFactory::make("videocrop") + .build() + .expect("Failed to create crop1"); + crop1.set_property("name", "crop1"); + + self.obj().add(&crop0).expect("Failed to add crop0 element"); + self.obj().add(&crop1).expect("Failed to add crop1 element"); + + // FIXME remove split_screen logic if not needed. It adds and links crops always + self.link_elements(&compositor, true, backend)?; self.add_overlay_probe(&self.overlay0); self.add_overlay_probe(&self.overlay1); @@ -178,6 +432,14 @@ impl VideoCompareMixer { .add(&self.overlay1) .expect("Failed to add overlay1 element"); + let caps_filter = gst::ElementFactory::make("capsfilter") + .name("capsfilter0") + .build() + .expect("Failed to create capsfilter0"); + self.obj() + .add(&caps_filter) + .expect("Failed to add capsfilter0 element"); + self.sinkpad0 .set_target(Some(&self.queue0.static_pad("sink").unwrap())) .expect("Failed to link sinkpad0 to queue0"); @@ -185,8 +447,10 @@ impl VideoCompareMixer { .set_target(Some(&self.queue1.static_pad("sink").unwrap())) .expect("Failed to link sinkpad1 to queue1"); + compositor.link(&caps_filter).expect("Failed to link compositor to capsfilter"); + self.srcpad - .set_target(Some(&compositor.static_pad("src").unwrap())) + .set_target(Some(&caps_filter.static_pad("src").unwrap())) .expect("Failed to link srcpad to compositor"); if split_screen && backend != Backend::GL { @@ -263,35 +527,35 @@ impl VideoCompareMixer { match event.view() { Caps(c) => { let caps = c.caps(); + gst::info!(CAT, "Received caps {caps:?}"); let s = caps.structure(0).unwrap(); let width = s.get::("width").unwrap(); - let half_width = width / 2; + let height = s.get::("height").unwrap(); let settings = self.settings.lock().unwrap(); let split_screen = settings.split_screen; - let backend = settings.backend; + let navigation_events = settings.navigation_events; drop(settings); - let compositor_sink1_pad = self.obj().by_name("compositor").unwrap().static_pad("sink_1").unwrap(); - if split_screen { - if backend != Backend::GL { - // Set crop properties for both crops - if let Some(crop0) = self.obj().by_name("crop0") { - crop0.set_property("right", half_width); - } - if let Some(crop1) = self.obj().by_name("crop1") { - crop1.set_property("left", half_width); - } - } else { - let compositor_sink0_pad = self.obj().by_name("compositor").unwrap().static_pad("sink_0").unwrap(); - compositor_sink0_pad.set_property("crop-right", half_width); - compositor_sink1_pad.set_property("crop-left", half_width); - } - compositor_sink1_pad.set_property("xpos", half_width); + let caps = format!("video/x-raw,width={},height={}", width, height); + self.obj().by_name("capsfilter0").unwrap().set_property_from_str("caps", &caps.as_str()); + + let compositor_mode = if split_screen { + Mode::Split } else { - compositor_sink1_pad.set_property("xpos", width); + Mode::SideBySide + }; + + let compositor_helper = Compositor::new( + compositor_mode, + width, + height, + ); + *self.compositor_helper.lock().unwrap() = Some(compositor_helper); + + if navigation_events { + self.add_navigation_events_probe(); } - gst::info!(CAT, "Received caps {caps:?}"); } _ => { gst::info!(CAT, "Other event"); @@ -347,14 +611,13 @@ impl ObjectSubclass for VideoCompareMixer { overlay0, overlay1, settings: Mutex::new(Settings::default()), + mouse_state: Arc::new(Mutex::new(MouseState::default())), + compositor_helper: Default::default(), } } } impl ObjectImpl for VideoCompareMixer { - // TODO - // navigation-evets = default true - fn properties() -> &'static [glib::ParamSpec] { static PROPERTIES: LazyLock> = LazyLock::new(|| { vec![ @@ -369,6 +632,12 @@ impl ObjectImpl for VideoCompareMixer { .default_value(false) .mutable_ready() .build(), + glib::ParamSpecBoolean::builder("navigation-events") + .nick("Navigation Events") + .blurb("Enable handling of navigation events for controlling the mixer") + .default_value(true) + .mutable_ready() + .build(), ] }); @@ -398,6 +667,16 @@ impl ObjectImpl for VideoCompareMixer { settings.split_screen ); } + "navigation-events" => { + settings.navigation_events = value.get().expect("type checked upstream"); + + gst::info!( + CAT, + imp = self, + "Set navigation-events to {:?}", + settings.navigation_events + ); + } _ => unimplemented!(), } } @@ -407,6 +686,7 @@ impl ObjectImpl for VideoCompareMixer { match pspec.name() { "backend" => settings.backend.to_value(), "split-screen" => settings.split_screen.to_value(), + "navigation-events" => settings.navigation_events.to_value(), _ => unimplemented!(), } } diff --git a/video/stats/src/comparemixer/mod.rs b/video/stats/src/comparemixer/mod.rs index 3b25d5b84..86cce9cc4 100644 --- a/video/stats/src/comparemixer/mod.rs +++ b/video/stats/src/comparemixer/mod.rs @@ -10,6 +10,7 @@ use gst::glib; use gst::prelude::*; +mod compositor; mod imp; glib::wrapper! { From 99f080915d5729853d42e7b5e06718cc7c73f0c8 Mon Sep 17 00:00:00 2001 From: Diego Nieto Date: Tue, 2 Sep 2025 13:46:23 +0200 Subject: [PATCH 18/46] video-encoder-stats: create vmaf property and handle optional behaviour * Detect in runtime whether vmaf element is available * Check whether property is enabled. Default: enabled * Expose decode_src pad only if vmaf pipeline branch is enabled * Add a new example not using vmaf --- video/stats/examples/video-stats-no-vmaf.rs | 34 +++++ video/stats/src/encoderstats/imp.rs | 159 ++++++++++++++------ 2 files changed, 144 insertions(+), 49 deletions(-) create mode 100644 video/stats/examples/video-stats-no-vmaf.rs diff --git a/video/stats/examples/video-stats-no-vmaf.rs b/video/stats/examples/video-stats-no-vmaf.rs new file mode 100644 index 000000000..35b17a2d5 --- /dev/null +++ b/video/stats/examples/video-stats-no-vmaf.rs @@ -0,0 +1,34 @@ +// Copyright (C) 2025, Fluendo S.A. +// Author: Diego Nieto +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +use anyhow::Error; +use gst::prelude::*; + +fn main() -> Result<(), Error> { + gst::init()?; + + let pipeline = gst::parse::launch("souphttpsrc location=\"https://ftp.nluug.nl/pub/graphics/blender/demo/movies/ToS/tears_of_steel_1080p.mov\" ! qtdemux name=demux demux.video_0 ! queue ! decodebin3 ! videoconvertscale ! capsfilter caps=\"video/x-raw,aspect-ratio=1/1\" ! tee name=tee ! queue name=encq0 ! video-encoder-stats encoder=\"x264enc bitrate=1024\" name=vs0 vmaf-stats=false tee. ! queue name=encq1 ! video-encoder-stats encoder=\"x264enc bitrate=512\" name=vs1 video-compare-mixer split-screen=true backend=CPU name=mixer vs0.src ! h264parse ! avdec_h264 ! mixer.sink_0 vs1.src ! h264parse ! avdec_h264 ! mixer.sink_1 mixer. ! autovideosink")?; + pipeline.set_state(gst::State::Playing)?; + + let bus = pipeline.bus().unwrap(); + while let Some(msg) = bus.timed_pop(gst::ClockTime::NONE) { + use gst::MessageView; + match msg.view() { + MessageView::Eos(..) => { + break; + } + MessageView::Error(..) => unreachable!(), + _ => (), + } + } + + pipeline.set_state(gst::State::Null)?; + + Ok(()) +} diff --git a/video/stats/src/encoderstats/imp.rs b/video/stats/src/encoderstats/imp.rs index c73ff6ad8..f0908bc08 100644 --- a/video/stats/src/encoderstats/imp.rs +++ b/video/stats/src/encoderstats/imp.rs @@ -34,6 +34,8 @@ pub struct EncoderStats { encoder: Mutex>, decoder: Mutex>, request_pad: Mutex>, + vmaf_stats: Mutex, + vmaf_available: bool, } impl EncoderStats { @@ -157,7 +159,18 @@ impl EncoderStats { let s = caps.structure(0).unwrap(); let fps = s.get::("framerate").ok(); self.stats.lock().unwrap().framerate = fps; - self.obj().by_name("vmaf0").unwrap().set_property("subsample", fps.unwrap().numer() as u32); + + // Only set vmaf subsample if vmaf is available and enabled + let vmaf_enabled = { + let vmaf_stats_guard = self.vmaf_stats.lock().unwrap(); + *vmaf_stats_guard && self.vmaf_available + }; + + if vmaf_enabled { + if let Some(vmaf) = self.obj().by_name("vmaf0") { + vmaf.set_property("subsample", fps.unwrap().numer() as u32); + } + } } _ => { gst::info!(CAT, "Other event"); @@ -172,10 +185,10 @@ impl EncoderStats { let encoder_guard = self.encoder.lock().unwrap(); encoder_guard.clone().expect("Encoder must be set") }; - - let decoder = { - let decoder_guard = self.decoder.lock().unwrap(); - decoder_guard.clone() + + let vmaf_enabled = { + let vmaf_stats_guard = self.vmaf_stats.lock().unwrap(); + *vmaf_stats_guard && self.vmaf_available }; let has_request_pad = { @@ -218,54 +231,62 @@ impl EncoderStats { .set_target(Some(&originalbuffersave.static_pad("sink").unwrap())) .expect("Failed to link sink pad to originalbuffersave element"); - let tee0_src_1 = tee0.request_pad_simple("src_%u").expect("tee0 src_1"); - let queue1 = gst::ElementFactory::make("queue") - .name("encintq1") - .build() - .expect("Failed to create queue encintq1"); - - // Use custom decoder if provided, otherwise use decodebin3 - let final_decoder = if let Some(custom_decoder) = decoder.clone() { - custom_decoder.set_property("name", "dec"); - self.obj().add(&custom_decoder).expect("Failed to add custom decoder element"); - custom_decoder - } else { - let decodebin3 = gst::ElementFactory::make("decodebin3") - .name("dec") - .build() - .expect("Failed to create decodebin3"); - self.obj().add(&decodebin3).expect("Failed to add decodebin3"); - decodebin3 - }; - - self.obj().add(&queue1).expect("Failed to add queue1"); - tee0_src_1.link(&queue1.static_pad("sink").unwrap()).expect("tee0.src_1 -> queue1"); - queue1.static_pad("src").unwrap().link(&final_decoder.static_pad("sink").unwrap()).expect("queue1.src -> decoder.sink"); + // Only create decoder branch if VMAF is enabled + if vmaf_enabled { + let decoder = { + let decoder_guard = self.decoder.lock().unwrap(); + decoder_guard.clone() + }; - // Conditionally add tee after decoder if request pad exists - if has_request_pad { - let decoder_tee = gst::ElementFactory::make("tee") - .name("decoder_tee") + let tee0_src_1 = tee0.request_pad_simple("src_%u").expect("tee0 src_1"); + let queue1 = gst::ElementFactory::make("queue") + .name("encintq1") .build() - .expect("Failed to create decoder_tee"); - self.obj().add(&decoder_tee).expect("Failed to add decoder_tee"); - - // Set up decoder -> decoder_tee connection - self.setup_decoder_to_tee_connection(final_decoder.clone(), decoder_tee.clone(), decoder.is_some()); - - // Connect decoder_tee src_0 to VMAF pipeline - let decoder_tee_src_0 = decoder_tee.request_pad_simple("src_%u").expect("decoder_tee src_0"); - self.setup_vmaf_pipeline(decoder_tee_src_0); + .expect("Failed to create queue encintq1"); + + // Use custom decoder if provided, otherwise use decodebin3 + let final_decoder = if let Some(custom_decoder) = decoder.clone() { + custom_decoder.set_property("name", "dec"); + self.obj().add(&custom_decoder).expect("Failed to add custom decoder element"); + custom_decoder + } else { + let decodebin3 = gst::ElementFactory::make("decodebin3") + .name("dec") + .build() + .expect("Failed to create decodebin3"); + self.obj().add(&decodebin3).expect("Failed to add decodebin3"); + decodebin3 + }; - // Connect decoder_tee src_1 to request pad - let decoder_tee_src_1 = decoder_tee.request_pad_simple("src_%u").expect("decoder_tee src_1"); - let request_pad_guard = self.request_pad.lock().unwrap(); - if let Some(ref request_pad) = *request_pad_guard { - request_pad.set_target(Some(&decoder_tee_src_1)).unwrap(); + self.obj().add(&queue1).expect("Failed to add queue1"); + tee0_src_1.link(&queue1.static_pad("sink").unwrap()).expect("tee0.src_1 -> queue1"); + queue1.static_pad("src").unwrap().link(&final_decoder.static_pad("sink").unwrap()).expect("queue1.src -> decoder.sink"); + + // Conditionally add tee after decoder if request pad exists + if has_request_pad { + let decoder_tee = gst::ElementFactory::make("tee") + .name("decoder_tee") + .build() + .expect("Failed to create decoder_tee"); + self.obj().add(&decoder_tee).expect("Failed to add decoder_tee"); + + // Set up decoder -> decoder_tee connection + self.setup_decoder_to_tee_connection(final_decoder.clone(), decoder_tee.clone(), decoder.is_some()); + + // Connect decoder_tee src_0 to VMAF pipeline + let decoder_tee_src_0 = decoder_tee.request_pad_simple("src_%u").expect("decoder_tee src_0"); + self.setup_vmaf_pipeline(decoder_tee_src_0); + + // Connect decoder_tee src_1 to request pad + let decoder_tee_src_1 = decoder_tee.request_pad_simple("src_%u").expect("decoder_tee src_1"); + let request_pad_guard = self.request_pad.lock().unwrap(); + if let Some(ref request_pad) = *request_pad_guard { + request_pad.set_target(Some(&decoder_tee_src_1)).unwrap(); + } + } else { + // No request pad - direct connection to VMAF pipeline + self.setup_decoder_to_vmaf_direct(final_decoder.clone(), decoder.is_some()); } - } else { - // No request pad - direct connection to VMAF pipeline - self.setup_decoder_to_vmaf_direct(final_decoder.clone(), decoder.is_some()); } unsafe @@ -459,6 +480,14 @@ impl ObjectSubclass for EncoderStats { .expect("Failed to create identity element"); identity.set_property("name", "identity"); + // Check if vmaf element is available + let vmaf_available = if gst::ElementFactory::find("vmaf").is_none() { + gst::warning!(CAT, "VMAF element not found, VMAF stats will be disabled"); + false + } else { + true + }; + Self { srcpad, sinkpad, @@ -467,6 +496,8 @@ impl ObjectSubclass for EncoderStats { encoder: Mutex::new(None), decoder: Mutex::new(None), request_pad: Mutex::new(None), + vmaf_stats: Mutex::new(true), // Default enabled + vmaf_available, } } } @@ -484,6 +515,11 @@ impl ObjectImpl for EncoderStats { .nick("The decoder element") .blurb("The decoder element to use for VMAF calculation (default: decodebin3)") .build(), + glib::ParamSpecBoolean::builder("vmaf-stats") + .nick("Enable VMAF stats") + .blurb("Enable VMAF statistics calculation (requires vmaf element)") + .default_value(true) + .build(), ] }); @@ -500,6 +536,10 @@ impl ObjectImpl for EncoderStats { let decoder_guard = self.decoder.lock().unwrap(); decoder_guard.clone().to_value() } + "vmaf-stats" => { + let vmaf_stats_guard = self.vmaf_stats.lock().unwrap(); + (*vmaf_stats_guard && self.vmaf_available).to_value() + } _ => unimplemented!(), } } @@ -528,6 +568,16 @@ impl ObjectImpl for EncoderStats { *decoder_guard = Some(dec_obj); } } + "vmaf-stats" => { + if let Ok(vmaf_stats) = value.get::() { + if vmaf_stats && !self.vmaf_available { + gst::warning!(CAT, imp = self, "Cannot enable VMAF stats: vmaf element not available"); + return; + } + let mut vmaf_stats_guard = self.vmaf_stats.lock().unwrap(); + *vmaf_stats_guard = vmaf_stats; + } + } _ => unimplemented!(), } } @@ -602,6 +652,17 @@ impl ElementImpl for EncoderStats { return None; } + // Only allow request pads if VMAF is enabled (since that's when we have decoder) + let vmaf_enabled = { + let vmaf_stats_guard = self.vmaf_stats.lock().unwrap(); + *vmaf_stats_guard && self.vmaf_available + }; + + if !vmaf_enabled { + gst::warning!(CAT, imp = self, "Cannot request decoder pad when VMAF stats are disabled"); + return None; + } + if templ.name() == "decoder_src" { let mut request_pad_guard = self.request_pad.lock().unwrap(); if request_pad_guard.is_some() { From ae5e025c75aaef6b39a27383eea005b263d8f3c4 Mon Sep 17 00:00:00 2001 From: Diego Nieto Date: Tue, 2 Sep 2025 15:34:28 +0200 Subject: [PATCH 19/46] video-encoder-stats: handle input queues internally --- video/stats/examples/video-stats-no-vmaf.rs | 2 +- .../examples/video-stats-split-screen.rs | 2 +- video/stats/examples/video-stats.rs | 2 +- video/stats/src/encoderstats/imp.rs | 22 +++++++++++++++++-- 4 files changed, 23 insertions(+), 5 deletions(-) diff --git a/video/stats/examples/video-stats-no-vmaf.rs b/video/stats/examples/video-stats-no-vmaf.rs index 35b17a2d5..f776205a6 100644 --- a/video/stats/examples/video-stats-no-vmaf.rs +++ b/video/stats/examples/video-stats-no-vmaf.rs @@ -13,7 +13,7 @@ use gst::prelude::*; fn main() -> Result<(), Error> { gst::init()?; - let pipeline = gst::parse::launch("souphttpsrc location=\"https://ftp.nluug.nl/pub/graphics/blender/demo/movies/ToS/tears_of_steel_1080p.mov\" ! qtdemux name=demux demux.video_0 ! queue ! decodebin3 ! videoconvertscale ! capsfilter caps=\"video/x-raw,aspect-ratio=1/1\" ! tee name=tee ! queue name=encq0 ! video-encoder-stats encoder=\"x264enc bitrate=1024\" name=vs0 vmaf-stats=false tee. ! queue name=encq1 ! video-encoder-stats encoder=\"x264enc bitrate=512\" name=vs1 video-compare-mixer split-screen=true backend=CPU name=mixer vs0.src ! h264parse ! avdec_h264 ! mixer.sink_0 vs1.src ! h264parse ! avdec_h264 ! mixer.sink_1 mixer. ! autovideosink")?; + let pipeline = gst::parse::launch("souphttpsrc location=\"https://ftp.nluug.nl/pub/graphics/blender/demo/movies/ToS/tears_of_steel_1080p.mov\" ! qtdemux name=demux demux.video_0 ! queue ! decodebin3 ! videoconvertscale ! capsfilter caps=\"video/x-raw,aspect-ratio=1/1\" ! tee name=tee ! video-encoder-stats encoder=\"x264enc bitrate=1024\" name=vs0 vmaf-stats=false tee. ! video-encoder-stats encoder=\"x264enc bitrate=512\" name=vs1 video-compare-mixer split-screen=true backend=CPU name=mixer vs0.src ! h264parse ! avdec_h264 ! mixer.sink_0 vs1.src ! h264parse ! avdec_h264 ! mixer.sink_1 mixer. ! autovideosink")?; pipeline.set_state(gst::State::Playing)?; let bus = pipeline.bus().unwrap(); diff --git a/video/stats/examples/video-stats-split-screen.rs b/video/stats/examples/video-stats-split-screen.rs index 486850283..c3882a716 100644 --- a/video/stats/examples/video-stats-split-screen.rs +++ b/video/stats/examples/video-stats-split-screen.rs @@ -13,7 +13,7 @@ use gst::prelude::*; fn main() -> Result<(), Error> { gst::init()?; - let pipeline = gst::parse::launch("souphttpsrc location=\"https://ftp.nluug.nl/pub/graphics/blender/demo/movies/ToS/tears_of_steel_1080p.mov\" ! qtdemux name=demux demux.video_0 ! queue ! decodebin3 ! videoconvertscale ! capsfilter caps=\"video/x-raw,aspect-ratio=1/1\" ! tee name=tee ! queue name=encq0 ! video-encoder-stats encoder=\"x264enc\" decoder=\"h264parse ! avdec_h264\" name=vs0 tee. ! queue name=encq1 ! video-encoder-stats encoder=\"x264enc bitrate=512\" name=vs1 video-compare-mixer split-screen=true backend=CPU name=mixer vs0.src ! decodebin3 ! mixer.sink_0 vs1.src ! decodebin3 ! mixer.sink_1 mixer. ! autovideosink")?; + let pipeline = gst::parse::launch("souphttpsrc location=\"https://ftp.nluug.nl/pub/graphics/blender/demo/movies/ToS/tears_of_steel_1080p.mov\" ! qtdemux name=demux demux.video_0 ! queue ! decodebin3 ! videoconvertscale ! capsfilter caps=\"video/x-raw,aspect-ratio=1/1\" ! tee name=tee ! video-encoder-stats encoder=\"x264enc\" decoder=\"h264parse ! avdec_h264\" name=vs0 tee. ! video-encoder-stats encoder=\"x264enc bitrate=512\" name=vs1 video-compare-mixer split-screen=true backend=CPU name=mixer vs0.src ! decodebin3 ! mixer.sink_0 vs1.src ! decodebin3 ! mixer.sink_1 mixer. ! autovideosink")?; pipeline.set_state(gst::State::Playing)?; let bus = pipeline.bus().unwrap(); diff --git a/video/stats/examples/video-stats.rs b/video/stats/examples/video-stats.rs index 137ce279d..e275cb8c5 100644 --- a/video/stats/examples/video-stats.rs +++ b/video/stats/examples/video-stats.rs @@ -13,7 +13,7 @@ use gst::prelude::*; fn main() -> Result<(), Error> { gst::init()?; - let pipeline = gst::parse::launch("souphttpsrc location=\"https://ftp.nluug.nl/pub/graphics/blender/demo/movies/ToS/tears_of_steel_1080p.mov\" ! qtdemux name=demux demux.video_0 ! queue ! decodebin3 ! videoconvertscale ! capsfilter caps=\"video/x-raw,aspect-ratio=1/1\" ! tee name=tee ! queue name=encq0 ! video-encoder-stats encoder=\"x264enc bitrate=1024\" decoder=\"h264parse ! avdec_h264\" name=vs0 ! fakesink tee. ! queue name=encq1 ! video-encoder-stats encoder=\"x264enc bitrate=512\" name=vs1 ! fakesink video-compare-mixer split-screen=false backend=CPU name=mixer vs0.decoder_src ! mixer.sink_0 vs1.decoder_src ! mixer.sink_1 mixer. ! autovideosink")?; + let pipeline = gst::parse::launch("souphttpsrc location=\"https://ftp.nluug.nl/pub/graphics/blender/demo/movies/ToS/tears_of_steel_1080p.mov\" ! qtdemux name=demux demux.video_0 ! queue ! decodebin3 ! videoconvertscale ! capsfilter caps=\"video/x-raw,aspect-ratio=1/1\" ! tee name=tee ! video-encoder-stats encoder=\"x264enc bitrate=1024\" decoder=\"h264parse ! avdec_h264\" name=vs0 ! fakesink tee. ! video-encoder-stats encoder=\"x264enc bitrate=512\" name=vs1 ! fakesink video-compare-mixer split-screen=false backend=CPU name=mixer vs0.decoder_src ! mixer.sink_0 vs1.decoder_src ! mixer.sink_1 mixer. ! autovideosink")?; pipeline.set_state(gst::State::Playing)?; let bus = pipeline.bus().unwrap(); diff --git a/video/stats/src/encoderstats/imp.rs b/video/stats/src/encoderstats/imp.rs index f0908bc08..aebb0b750 100644 --- a/video/stats/src/encoderstats/imp.rs +++ b/video/stats/src/encoderstats/imp.rs @@ -198,6 +198,20 @@ impl EncoderStats { encoder.set_property("name", "enc"); + // Add internal queue at the beginning + let obj_name = self.obj().name().to_string(); + let queue_name = if obj_name.contains("0") { + "encq0" + } else { + "encq1" + }; + + let input_queue = gst::ElementFactory::make("queue") + .name(queue_name) + .build() + .expect("Failed to create input queue"); + self.obj().add(&input_queue).expect("Failed to add input queue"); + let originalbuffersave = gst::ElementFactory::make("originalbuffersave") .build() .expect("Failed to create originalbuffersave element"); @@ -212,6 +226,9 @@ impl EncoderStats { self.obj().add(&tee0).unwrap(); self.obj().add(&encoder).expect("Failed to add encoder element"); + + // Link: input_queue -> originalbuffersave -> encoder -> identity -> tee0 + input_queue.link(&originalbuffersave).expect("Failed to link input queue to originalbuffersave"); originalbuffersave.link(&encoder).expect("Failed to link originalbuffersave to encoder"); encoder.link(&self.identity).expect("Failed to link encoder to identity"); self.identity.link(&tee0).expect("Failed to link identity to tee0"); @@ -227,9 +244,10 @@ impl EncoderStats { tee0_src_0.link(&queue0_sink_pad).expect("tee0.src_0 -> encintq0.sink"); self.srcpad.set_target(Some(&queue0_src_pad)).unwrap(); + // Connect sink ghostpad to input queue self.sinkpad - .set_target(Some(&originalbuffersave.static_pad("sink").unwrap())) - .expect("Failed to link sink pad to originalbuffersave element"); + .set_target(Some(&input_queue.static_pad("sink").unwrap())) + .expect("Failed to link sink pad to input queue"); // Only create decoder branch if VMAF is enabled if vmaf_enabled { From 8859ea12defb113ed3ecd78f18ab9251ad6b3e46 Mon Sep 17 00:00:00 2001 From: Diego Nieto Date: Tue, 2 Sep 2025 23:54:30 +0200 Subject: [PATCH 20/46] video-encoder-stats: add latency metric * get latency from SystemTime ** TODO: implement npt as webrtc-precise-sync-send.rs does * add timestamp before encoding as part of videoencoderstatsmeta * compare the input timestamp in video-compare-mixer with the current timestamp (after decoding the video stream) * add a new example video-stats-zero-latency to showcase the latency difference --- .../examples/video-stats-zero-latency.rs | 34 +++++ video/stats/src/encoderstats/imp.rs | 125 ++++++++++-------- video/stats/src/videoencoderstats.rs | 46 ++++++- 3 files changed, 149 insertions(+), 56 deletions(-) create mode 100644 video/stats/examples/video-stats-zero-latency.rs diff --git a/video/stats/examples/video-stats-zero-latency.rs b/video/stats/examples/video-stats-zero-latency.rs new file mode 100644 index 000000000..37872bae5 --- /dev/null +++ b/video/stats/examples/video-stats-zero-latency.rs @@ -0,0 +1,34 @@ +// Copyright (C) 2025, Fluendo S.A. +// Author: Diego Nieto +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +use anyhow::Error; +use gst::prelude::*; + +fn main() -> Result<(), Error> { + gst::init()?; + + let pipeline = gst::parse::launch("souphttpsrc location=\"https://ftp.nluug.nl/pub/graphics/blender/demo/movies/ToS/tears_of_steel_1080p.mov\" ! qtdemux name=demux demux.video_0 ! queue ! decodebin3 ! videoconvertscale ! capsfilter caps=\"video/x-raw,aspect-ratio=1/1\" ! tee name=tee ! video-encoder-stats encoder=\"x264enc bitrate=1024 tune=zerolatency\" name=vs0 vmaf-stats=false tee. ! video-encoder-stats encoder=\"x264enc bitrate=512 tune=zerolatency\" name=vs1 video-compare-mixer split-screen=true backend=CPU name=mixer vs0.src ! h264parse ! avdec_h264 ! mixer.sink_0 vs1.src ! h264parse ! avdec_h264 ! mixer.sink_1 mixer. ! autovideosink")?; + pipeline.set_state(gst::State::Playing)?; + + let bus = pipeline.bus().unwrap(); + while let Some(msg) = bus.timed_pop(gst::ClockTime::NONE) { + use gst::MessageView; + match msg.view() { + MessageView::Eos(..) => { + break; + } + MessageView::Error(..) => unreachable!(), + _ => (), + } + } + + pipeline.set_state(gst::State::Null)?; + + Ok(()) +} diff --git a/video/stats/src/encoderstats/imp.rs b/video/stats/src/encoderstats/imp.rs index aebb0b750..cf6ecc199 100644 --- a/video/stats/src/encoderstats/imp.rs +++ b/video/stats/src/encoderstats/imp.rs @@ -49,9 +49,8 @@ impl EncoderStats { let encoder_name = encoder_factory.name(); let stats = self.stats.clone(); - let obj_name = self.obj().name().to_string(); identity_src_pad.add_probe(gst::PadProbeType::BUFFER, move |_pad, probe_info| { - let Some(buffer) = probe_info.buffer_mut() else { + let Some(_) = probe_info.buffer() else { return gst::PadProbeReturn::Ok; }; @@ -60,64 +59,11 @@ impl EncoderStats { let num_buffers = identity_stats.get::("num-buffers").unwrap(); let mut stats = stats.lock().unwrap(); - let fps_n: i32; - if let Some(fps) = stats.framerate { - fps_n = fps.numer(); - } else { - return gst::PadProbeReturn::Ok; - } - - if num_buffers % (fps_n as u64) != 0 { - gst::log!(CAT, "Skipping probe for buffer {num_buffers} as it is not a multiple of framerate {fps_n}"); - return gst::PadProbeReturn::Ok; - } - - // FIXME: integrates queues internally to calculate the CPU usage - let thread_name = if obj_name.contains("0") { - "encq0:src" - } else { - "encq1:src" - }; - let (mut total_utime, mut total_stime) = get_cpu_usage(thread_name.to_string()); - - if encoder_name == "flulcevch264enc" { - // Fixme flulcevc uses multiple threads, so we need to get the CPU usage of all threads - let (utime, stime) = get_cpu_usage("lcevc".to_string()); - gst::log!(CAT, "flulcevc lcevc utime: {}, stime: {}", utime, stime); - // Add the CPU usage to the total CPU usage - total_utime += utime; - total_stime += stime; - - let (utime, stime) = get_cpu_usage("pool.".to_string()); - gst::log!(CAT, "flulcevc pool utime: {}, stime: {}", utime, stime); - // Add the CPU usage to the total CPU usage - total_utime += utime; - total_stime += stime; - } - if encoder_name == "lcevch264enc" { - // Fixme lcevc uses multiple threads, so we need to get the CPU usage of all threads - let (utime, stime) = get_cpu_usage("pool.".to_string()); - gst::log!(CAT, "lcevc pool utime: {}, stime: {}", utime, stime); - // Add the CPU usage to the total CPU usage - total_utime += utime; - total_stime += stime; - } - - stats.threads_utime = total_utime; - stats.threads_stime = total_stime; stats.num_bytes = num_bytes; stats.num_buffers = num_buffers; stats.name = encoder_name.to_string(); - let buffer = buffer.make_mut(); - - // Add the VideoEncoderStatsMeta to the buffer - VideoEncoderStatsMeta::add( - buffer, - stats.clone(), - ); - gst::PadProbeReturn::Ok }); } @@ -134,6 +80,9 @@ impl EncoderStats { }; stats.lock().unwrap().buffer_in(); gst::log!(CAT, "Buffer in encoder sink pad"); + stats.lock().unwrap().pre_encode_time = *gst::ClockTime::from_nseconds( + gst::SystemClock::obtain().upcast::().time().unwrap().nseconds() + ); gst::PadProbeReturn::Ok }); @@ -144,6 +93,9 @@ impl EncoderStats { }; stats.lock().unwrap().buffer_out(); gst::log!(CAT, "Buffer out encoder src pad"); + stats.lock().unwrap().post_encode_time = *gst::ClockTime::from_nseconds( + gst::SystemClock::obtain().upcast::().time().unwrap().nseconds() + ); gst::PadProbeReturn::Ok }); } @@ -212,6 +164,69 @@ impl EncoderStats { .expect("Failed to create input queue"); self.obj().add(&input_queue).expect("Failed to add input queue"); + // Add probe to input queue src pad to log buffer flow + let input_queue_src_pad = input_queue.static_pad("src").unwrap(); + let queue_name_clone = queue_name.to_string(); + let stats_clone = self.stats.clone(); + input_queue_src_pad.add_probe(gst::PadProbeType::BUFFER, move |_pad, probe_info| { + let Some(buffer) = probe_info.buffer_mut() else { + return gst::PadProbeReturn::Ok; + }; + gst::info!(CAT, "Buffer received in {} src pad, PTS: {:?}, DTS: {:?}, size: {}", + queue_name_clone, buffer.pts(), buffer.dts(), buffer.size()); + + let mut stats = stats_clone.lock().unwrap(); + + // Only update stats at framerate intervals + let fps_n: i32; + if let Some(fps) = stats.framerate { + fps_n = fps.numer(); + } else { + return gst::PadProbeReturn::Ok; + } + + let num_buffers = stats.num_buffers; + + if num_buffers % (fps_n as u64) != 0 { + gst::log!(CAT, "Skipping probe for buffer {num_buffers} as it is not a multiple of framerate {fps_n}"); + return gst::PadProbeReturn::Ok; + } + + let queue_name = if obj_name.contains("0") { "encq0:src" } else { "encq1:src" }; + let thread_patterns = match stats.name.as_str() { + "flulcevch264enc" => vec![queue_name, "lcevc", "pool."], + "lcevch264enc" => vec![queue_name, "pool."], + _ => vec![queue_name], + }; + + let (total_utime, total_stime) = thread_patterns.iter() + .map(|pattern| { + let (utime, stime) = get_cpu_usage(pattern.to_string()); + gst::log!(CAT, "Thread pattern '{}' - utime: {}, stime: {}", pattern, utime, stime); + (utime, stime) + }) + .fold((0u64, 0u64), |(acc_utime, acc_stime), (utime, stime)| { + (acc_utime + utime, acc_stime + stime) + }); + + stats.threads_utime = total_utime; + stats.threads_stime = total_stime; + + stats.input_time = *gst::ClockTime::from_nseconds( + gst::SystemClock::obtain().upcast::().time().unwrap().nseconds() + ); + + let buffer = buffer.make_mut(); + + // Add the VideoEncoderStatsMeta to the buffer + VideoEncoderStatsMeta::add( + buffer, + stats.clone(), + ); + + gst::PadProbeReturn::Ok + }); + let originalbuffersave = gst::ElementFactory::make("originalbuffersave") .build() .expect("Failed to create originalbuffersave element"); diff --git a/video/stats/src/videoencoderstats.rs b/video/stats/src/videoencoderstats.rs index 599ff5dc3..3be8159a3 100644 --- a/video/stats/src/videoencoderstats.rs +++ b/video/stats/src/videoencoderstats.rs @@ -13,6 +13,9 @@ use std::time::Duration; use std::fmt; use std::sync::LazyLock; +use gst::ffi::GstClockTime; +use gst::prelude::ClockExt; + use procfs::process::Process; static CAT: LazyLock = LazyLock::new(|| { @@ -23,7 +26,7 @@ static CAT: LazyLock = LazyLock::new(|| { ) }); -#[derive(Default, Clone, PartialEq, Debug)] +#[derive(Clone, PartialEq, Debug)] pub struct VideoEncoderStats { pub name: String, pub num_buffers: u64, @@ -35,6 +38,29 @@ pub struct VideoEncoderStats { pub threads_stime: u64, pub framerate: Option, pub vmaf_score: f64, + pub input_time: GstClockTime, + pub pre_encode_time: GstClockTime, + pub post_encode_time: GstClockTime, +} + +impl Default for VideoEncoderStats { + fn default() -> Self { + Self { + name: String::new(), + framerate: None, + num_bytes: 0, + num_buffers: 0, + time_last_buffers: VecDeque::::new(), + max_buffers_inside: 0, + total_processing_time: Duration::ZERO, + threads_utime: 0, + threads_stime: 0, + vmaf_score: 0.0, + input_time: 0, + pre_encode_time: 0, + post_encode_time: 0, + } + } } impl VideoEncoderStats { @@ -116,6 +142,24 @@ impl fmt::Display for VideoEncoderStats { f, "VMAF: {:.3}", vmaf_score + )?; + + let pre_encode_time = &self.pre_encode_time; + let post_encode_time = &self.post_encode_time; + let encode_latency = (*post_encode_time as f64 - *pre_encode_time as f64) / 1_000_000.0; + writeln!( + f, + "Encode latency: {:.3} ms", + encode_latency + )?; + + let current_time = gst::SystemClock::obtain(); + let latency = (current_time.time().unwrap().nseconds() as f64 - self.input_time as f64) / 1_000_000.0; + + writeln!( + f, + "End2End latency: {:.3} ms", + latency ) } } From f5a33a54b099c9f9288f67605f91385b1fa7bc86 Mon Sep 17 00:00:00 2001 From: Diego Nieto Date: Wed, 17 Sep 2025 18:55:18 +0200 Subject: [PATCH 21/46] video-encoder-stats: let vmaf score a N/A by default If vmaf metric is not available N/A will be shown in the metric instead of 0 value. --- video/stats/src/encoderstats/imp.rs | 3 ++- video/stats/src/videoencoderstats.rs | 15 +++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/video/stats/src/encoderstats/imp.rs b/video/stats/src/encoderstats/imp.rs index cf6ecc199..96ef42dfa 100644 --- a/video/stats/src/encoderstats/imp.rs +++ b/video/stats/src/encoderstats/imp.rs @@ -393,7 +393,8 @@ impl EncoderStats { glib::closure!( move |_vmaf: &gst::Element, score: f64| { let mut stats = stats.lock().unwrap(); - stats.vmaf_score = score; + stats.vmaf_score = Some(score); + println!("VMAF score: {:.3}", score); } ), ); diff --git a/video/stats/src/videoencoderstats.rs b/video/stats/src/videoencoderstats.rs index 3be8159a3..7d6ed3047 100644 --- a/video/stats/src/videoencoderstats.rs +++ b/video/stats/src/videoencoderstats.rs @@ -37,7 +37,7 @@ pub struct VideoEncoderStats { pub threads_utime: u64, pub threads_stime: u64, pub framerate: Option, - pub vmaf_score: f64, + pub vmaf_score: Option, pub input_time: GstClockTime, pub pre_encode_time: GstClockTime, pub post_encode_time: GstClockTime, @@ -55,7 +55,7 @@ impl Default for VideoEncoderStats { total_processing_time: Duration::ZERO, threads_utime: 0, threads_stime: 0, - vmaf_score: 0.0, + vmaf_score: None, input_time: 0, pre_encode_time: 0, post_encode_time: 0, @@ -137,12 +137,11 @@ impl fmt::Display for VideoEncoderStats { cpu_time_seconds )?; - let vmaf_score = self.vmaf_score; - writeln!( - f, - "VMAF: {:.3}", - vmaf_score - )?; + let vmaf_score_str = match self.vmaf_score { + Some(score) => format!("{:.3}", score), + None => "N/A".to_string(), + }; + writeln!(f, "VMAF: {}", vmaf_score_str)?; let pre_encode_time = &self.pre_encode_time; let post_encode_time = &self.post_encode_time; From 281ef4ffeebbdd97e53a48b99d1c786c1674f4cd Mon Sep 17 00:00:00 2001 From: Diego Nieto Date: Thu, 18 Sep 2025 00:28:48 +0200 Subject: [PATCH 22/46] video-encoder-stats: add silent property and post stats as messages This enables posting the stats as messages unless the new property silent is set true, by default is false --- video/stats/src/encoderstats/imp.rs | 51 +++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/video/stats/src/encoderstats/imp.rs b/video/stats/src/encoderstats/imp.rs index 96ef42dfa..e52b4c366 100644 --- a/video/stats/src/encoderstats/imp.rs +++ b/video/stats/src/encoderstats/imp.rs @@ -36,6 +36,8 @@ pub struct EncoderStats { request_pad: Mutex>, vmaf_stats: Mutex, vmaf_available: bool, + silent: Mutex, + last_message: Arc>>, } impl EncoderStats { @@ -168,6 +170,9 @@ impl EncoderStats { let input_queue_src_pad = input_queue.static_pad("src").unwrap(); let queue_name_clone = queue_name.to_string(); let stats_clone = self.stats.clone(); + let element_weak = self.obj().downgrade(); + let silent_arc = Arc::new(self.silent.lock().unwrap().clone()); + let last_message_arc = self.last_message.clone(); input_queue_src_pad.add_probe(gst::PadProbeType::BUFFER, move |_pad, probe_info| { let Some(buffer) = probe_info.buffer_mut() else { return gst::PadProbeReturn::Ok; @@ -216,6 +221,27 @@ impl EncoderStats { gst::SystemClock::obtain().upcast::().time().unwrap().nseconds() ); + if let Some(element) = element_weak.upgrade() { + let stats_message = format!("{}", stats.clone()); + + let structure = gst::Structure::builder("encoder-stats") + .field("message", &stats_message) + .build(); + + let message = gst::message::Application::new(structure); + let _ = element.post_message(message); + + let silent = *silent_arc; + + if !silent { + { + let mut last_message_guard = last_message_arc.lock().unwrap(); + *last_message_guard = Some(stats_message); + } + element.notify("last-message"); + } + } + let buffer = buffer.make_mut(); // Add the VideoEncoderStatsMeta to the buffer @@ -532,6 +558,8 @@ impl ObjectSubclass for EncoderStats { request_pad: Mutex::new(None), vmaf_stats: Mutex::new(true), // Default enabled vmaf_available, + silent: Mutex::new(false), // Default: not silent + last_message: Arc::new(Mutex::new(None)), } } } @@ -554,6 +582,15 @@ impl ObjectImpl for EncoderStats { .blurb("Enable VMAF statistics calculation (requires vmaf element)") .default_value(true) .build(), + glib::ParamSpecBoolean::builder("silent") + .nick("Silent") + .blurb("Enable silent mode (disable stdout output of stats)") + .default_value(false) + .build(), + glib::ParamSpecString::builder("last-message") + .nick("Last Message") + .blurb("The message describing current encoder statistics") + .build(), ] }); @@ -574,6 +611,14 @@ impl ObjectImpl for EncoderStats { let vmaf_stats_guard = self.vmaf_stats.lock().unwrap(); (*vmaf_stats_guard && self.vmaf_available).to_value() } + "silent" => { + let silent_guard = self.silent.lock().unwrap(); + (*silent_guard).to_value() + } + "last-message" => { + let last_message_guard = self.last_message.lock().unwrap(); + last_message_guard.clone().to_value() + } _ => unimplemented!(), } } @@ -612,6 +657,12 @@ impl ObjectImpl for EncoderStats { *vmaf_stats_guard = vmaf_stats; } } + "silent" => { + if let Ok(silent) = value.get::() { + let mut silent_guard = self.silent.lock().unwrap(); + *silent_guard = silent; + } + } _ => unimplemented!(), } } From 6290b221b1ab85b39efb2e011a2d221c6e028ef4 Mon Sep 17 00:00:00 2001 From: Diego Nieto Date: Thu, 18 Sep 2025 12:24:25 +0200 Subject: [PATCH 23/46] videoencoderstats: update metric output as average process time Remove throughput metric and leave avg. processing time instead --- video/stats/src/videoencoderstats.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/video/stats/src/videoencoderstats.rs b/video/stats/src/videoencoderstats.rs index 7d6ed3047..f15ababe5 100644 --- a/video/stats/src/videoencoderstats.rs +++ b/video/stats/src/videoencoderstats.rs @@ -118,11 +118,11 @@ impl fmt::Display for VideoEncoderStats { writeln!(f, "Bitrate: {:.3} kbps", bitrate_str)?; - let throughput = (1.0 + self.time_last_buffers.len() as f64)/self.avg_processing_time().as_secs_f64(); + let avg_processing_time = self.avg_processing_time().as_millis(); writeln!( f, - "Throughput: {:.2} fps", - throughput + "Processing time: {:.2} ms", + avg_processing_time )?; let cpu_time = self.threads_utime + self.threads_stime; From 0f8b586a82c9cf40d0b2e31d228231f9c02b095c Mon Sep 17 00:00:00 2001 From: Diego Nieto Date: Thu, 18 Sep 2025 16:02:15 +0200 Subject: [PATCH 24/46] video-encoder-stats: update meta timestamps of the current buffer instead of global values Relates to RDI-2963 --- video/stats/src/encoderstats/imp.rs | 182 ++++++++++++++--------- video/stats/src/videoencoderstatsmeta.rs | 4 + 2 files changed, 112 insertions(+), 74 deletions(-) diff --git a/video/stats/src/encoderstats/imp.rs b/video/stats/src/encoderstats/imp.rs index e52b4c366..a0c56e396 100644 --- a/video/stats/src/encoderstats/imp.rs +++ b/video/stats/src/encoderstats/imp.rs @@ -50,9 +50,12 @@ impl EncoderStats { let encoder_factory = encoder.factory().expect("encoder should have a factory"); let encoder_name = encoder_factory.name(); - let stats = self.stats.clone(); + let element_weak = self.obj().downgrade(); + let silent_arc = Arc::new(self.silent.lock().unwrap().clone()); + let last_message_arc = self.last_message.clone(); + identity_src_pad.add_probe(gst::PadProbeType::BUFFER, move |_pad, probe_info| { - let Some(_) = probe_info.buffer() else { + let Some(buffer) = probe_info.buffer_mut() else { return gst::PadProbeReturn::Ok; }; @@ -60,12 +63,43 @@ impl EncoderStats { let num_bytes = identity_stats.get::("num-bytes").unwrap(); let num_buffers = identity_stats.get::("num-buffers").unwrap(); - let mut stats = stats.lock().unwrap(); + let buffer = buffer.make_mut(); + if let Some(mut meta) = buffer.meta_mut::() { + let mut new_stats = meta.stats().clone(); + new_stats.num_bytes = num_bytes; + new_stats.num_buffers = num_buffers; + new_stats.name = encoder_name.to_string(); + + let stats_clone = new_stats.clone(); + meta.replace(new_stats); + + gst::log!(CAT, "Updated meta stats: encoder={}, buffers={}, bytes={}", + encoder_name, num_buffers, num_bytes); - stats.num_bytes = num_bytes; - stats.num_buffers = num_buffers; - stats.name = encoder_name.to_string(); + if let Some(element) = element_weak.upgrade() { + let stats_message = format!("{}", stats_clone.clone()); + let structure = gst::Structure::builder("encoder-stats") + .field("message", &stats_message) + .build(); + + let message = gst::message::Application::new(structure); + let _ = element.post_message(message); + + let silent = *silent_arc; + + if !silent { + { + let mut last_message_guard = last_message_arc.lock().unwrap(); + *last_message_guard = Some(stats_message); + } + element.notify("last-message"); + } + } + + } else { + gst::warning!(CAT, "No VideoEncoderStatsMeta found on buffer"); + } gst::PadProbeReturn::Ok }); } @@ -77,27 +111,55 @@ impl EncoderStats { let stats = self.stats.clone(); encoder_sink_pad.add_probe(gst::PadProbeType::BUFFER, move |_, probe_info| { - let Some(_) = probe_info.buffer() else { + let Some(buffer) = probe_info.buffer_mut() else { return gst::PadProbeReturn::Ok; }; stats.lock().unwrap().buffer_in(); gst::log!(CAT, "Buffer in encoder sink pad"); - stats.lock().unwrap().pre_encode_time = *gst::ClockTime::from_nseconds( + + let current_time = *gst::ClockTime::from_nseconds( gst::SystemClock::obtain().upcast::().time().unwrap().nseconds() ); + + let buffer = buffer.make_mut(); + if let Some(mut meta) = buffer.meta_mut::() { + let mut new_stats = meta.stats().clone(); + new_stats.pre_encode_time = current_time; + + meta.replace(new_stats); + + gst::log!(CAT, "Buffer in encoder sink pad, pre_encode_time: {}", current_time); + } else { + gst::warning!(CAT, "No VideoEncoderStatsMeta found on buffer in encoder sink probe"); + } + gst::PadProbeReturn::Ok }); let stats = self.stats.clone(); encoder_src_pad.add_probe(gst::PadProbeType::BUFFER, move |_, probe_info| { - let Some(_) = probe_info.buffer() else { + let Some(buffer) = probe_info.buffer_mut() else { return gst::PadProbeReturn::Ok; }; stats.lock().unwrap().buffer_out(); gst::log!(CAT, "Buffer out encoder src pad"); - stats.lock().unwrap().post_encode_time = *gst::ClockTime::from_nseconds( + + let current_time = *gst::ClockTime::from_nseconds( gst::SystemClock::obtain().upcast::().time().unwrap().nseconds() ); + + let buffer = buffer.make_mut(); + if let Some(mut meta) = buffer.meta_mut::() { + let mut new_stats = meta.stats().clone(); + new_stats.post_encode_time = current_time; + + meta.replace(new_stats); + + gst::log!(CAT, "Buffer out encoder src pad, post_encode_time: {}", current_time); + } else { + gst::warning!(CAT, "No VideoEncoderStatsMeta found on buffer in encoder src probe"); + } + gst::PadProbeReturn::Ok }); } @@ -113,13 +175,13 @@ impl EncoderStats { let s = caps.structure(0).unwrap(); let fps = s.get::("framerate").ok(); self.stats.lock().unwrap().framerate = fps; - + // Only set vmaf subsample if vmaf is available and enabled let vmaf_enabled = { let vmaf_stats_guard = self.vmaf_stats.lock().unwrap(); *vmaf_stats_guard && self.vmaf_available }; - + if vmaf_enabled { if let Some(vmaf) = self.obj().by_name("vmaf0") { vmaf.set_property("subsample", fps.unwrap().numer() as u32); @@ -170,19 +232,20 @@ impl EncoderStats { let input_queue_src_pad = input_queue.static_pad("src").unwrap(); let queue_name_clone = queue_name.to_string(); let stats_clone = self.stats.clone(); - let element_weak = self.obj().downgrade(); - let silent_arc = Arc::new(self.silent.lock().unwrap().clone()); - let last_message_arc = self.last_message.clone(); input_queue_src_pad.add_probe(gst::PadProbeType::BUFFER, move |_pad, probe_info| { let Some(buffer) = probe_info.buffer_mut() else { return gst::PadProbeReturn::Ok; }; - gst::info!(CAT, "Buffer received in {} src pad, PTS: {:?}, DTS: {:?}, size: {}", + gst::info!(CAT, "Buffer received in {} src pad, PTS: {:?}, DTS: {:?}, size: {}", queue_name_clone, buffer.pts(), buffer.dts(), buffer.size()); let mut stats = stats_clone.lock().unwrap(); - // Only update stats at framerate intervals + stats.input_time = *gst::ClockTime::from_nseconds( + gst::SystemClock::obtain().upcast::().time().unwrap().nseconds() + ); + + // Only update CPU stats at framerate intervals as it takes time let fps_n: i32; if let Some(fps) = stats.framerate { fps_n = fps.numer(); @@ -192,59 +255,30 @@ impl EncoderStats { let num_buffers = stats.num_buffers; - if num_buffers % (fps_n as u64) != 0 { - gst::log!(CAT, "Skipping probe for buffer {num_buffers} as it is not a multiple of framerate {fps_n}"); - return gst::PadProbeReturn::Ok; - } - - let queue_name = if obj_name.contains("0") { "encq0:src" } else { "encq1:src" }; - let thread_patterns = match stats.name.as_str() { - "flulcevch264enc" => vec![queue_name, "lcevc", "pool."], - "lcevch264enc" => vec![queue_name, "pool."], - _ => vec![queue_name], - }; - - let (total_utime, total_stime) = thread_patterns.iter() - .map(|pattern| { - let (utime, stime) = get_cpu_usage(pattern.to_string()); - gst::log!(CAT, "Thread pattern '{}' - utime: {}, stime: {}", pattern, utime, stime); - (utime, stime) - }) - .fold((0u64, 0u64), |(acc_utime, acc_stime), (utime, stime)| { - (acc_utime + utime, acc_stime + stime) - }); - - stats.threads_utime = total_utime; - stats.threads_stime = total_stime; - - stats.input_time = *gst::ClockTime::from_nseconds( - gst::SystemClock::obtain().upcast::().time().unwrap().nseconds() - ); - - if let Some(element) = element_weak.upgrade() { - let stats_message = format!("{}", stats.clone()); - - let structure = gst::Structure::builder("encoder-stats") - .field("message", &stats_message) - .build(); - - let message = gst::message::Application::new(structure); - let _ = element.post_message(message); + if num_buffers % (fps_n as u64) == 0 { + let queue_name = if obj_name.contains("0") { "encq0:src" } else { "encq1:src" }; + let thread_patterns = match stats.name.as_str() { + "flulcevch264enc" => vec![queue_name, "lcevc", "pool."], + "lcevch264enc" => vec![queue_name, "pool."], + _ => vec![queue_name], + }; - let silent = *silent_arc; - - if !silent { - { - let mut last_message_guard = last_message_arc.lock().unwrap(); - *last_message_guard = Some(stats_message); - } - element.notify("last-message"); - } + let (total_utime, total_stime) = thread_patterns.iter() + .map(|pattern| { + let (utime, stime) = get_cpu_usage(pattern.to_string()); + gst::log!(CAT, "Thread pattern '{}' - utime: {}, stime: {}", pattern, utime, stime); + (utime, stime) + }) + .fold((0u64, 0u64), |(acc_utime, acc_stime), (utime, stime)| { + (acc_utime + utime, acc_stime + stime) + }); + + stats.threads_utime = total_utime; + stats.threads_stime = total_stime; } let buffer = buffer.make_mut(); - // Add the VideoEncoderStatsMeta to the buffer VideoEncoderStatsMeta::add( buffer, stats.clone(), @@ -265,7 +299,7 @@ impl EncoderStats { .build() .expect("Failed to create tee0 element"); self.obj().add(&tee0).unwrap(); - + self.obj().add(&encoder).expect("Failed to add encoder element"); // Link: input_queue -> originalbuffersave -> encoder -> identity -> tee0 @@ -273,7 +307,7 @@ impl EncoderStats { originalbuffersave.link(&encoder).expect("Failed to link originalbuffersave to encoder"); encoder.link(&self.identity).expect("Failed to link encoder to identity"); self.identity.link(&tee0).expect("Failed to link identity to tee0"); - + let tee0_src_0 = tee0.request_pad_simple("src_%u").expect("tee0 src pad"); let queue0 = gst::ElementFactory::make("queue") .name("encintq0") @@ -302,7 +336,7 @@ impl EncoderStats { .name("encintq1") .build() .expect("Failed to create queue encintq1"); - + // Use custom decoder if provided, otherwise use decodebin3 let final_decoder = if let Some(custom_decoder) = decoder.clone() { custom_decoder.set_property("name", "dec"); @@ -433,7 +467,7 @@ impl EncoderStats { } fn setup_vmaf_pipeline(&self, input_pad: gst::Pad) { - let (videoconvert, capsfilter, tee1, originalbufferstore, queue_vmaf_0, queue_vmaf_1, vmaf, fakesink) = + let (videoconvert, capsfilter, tee1, originalbufferstore, queue_vmaf_0, queue_vmaf_1, vmaf, fakesink) = self.create_vmaf_pipeline_elements(); self.obj().add_many([ @@ -446,7 +480,7 @@ impl EncoderStats { input_pad.link(&videoconvert_sink).expect("input -> videoconvert"); videoconvert.link(&capsfilter).expect("videoconvert -> capsfilter"); capsfilter.link(&tee1).expect("capsfilter -> tee1"); - + let tee1_src_0 = tee1.request_pad_simple("src_%u").expect("tee1 src_0"); tee1_src_0.link(&originalbufferstore.static_pad("sink").unwrap()).expect("tee1.src_0 -> originalbufferstore"); originalbufferstore.link(&queue_vmaf_0).expect("originalbufferrestore -> queue_vmaf_0"); @@ -460,7 +494,7 @@ impl EncoderStats { } fn setup_decoder_to_vmaf_direct(&self, final_decoder: gst::Element, is_manual_decoder: bool) { - let (videoconvert, capsfilter, tee1, originalbufferstore, queue_vmaf_0, queue_vmaf_1, vmaf, fakesink) = + let (videoconvert, capsfilter, tee1, originalbufferstore, queue_vmaf_0, queue_vmaf_1, vmaf, fakesink) = self.create_vmaf_pipeline_elements(); self.obj().add_many([ @@ -473,7 +507,7 @@ impl EncoderStats { final_decoder.link(&videoconvert).expect("decoder -> videoconvert"); videoconvert.link(&capsfilter).expect("videoconvert -> capsfilter"); capsfilter.link(&tee1).expect("capsfilter -> tee1"); - + let tee1_src_0 = tee1.request_pad_simple("src_%u").expect("tee1 src_0"); tee1_src_0.link(&originalbufferstore.static_pad("sink").unwrap()).expect("tee1.src_0 -> originalbufferstore"); originalbufferstore.link(&queue_vmaf_0).expect("originalbufferrestore -> queue_vmaf_0"); @@ -636,7 +670,7 @@ impl ObjectImpl for EncoderStats { gst::error!(CAT, "The element is not a video encoder"); panic!("The element is not a video encoder"); } - + let mut encoder_guard = self.encoder.lock().unwrap(); *encoder_guard = Some(enc_obj); } @@ -765,7 +799,7 @@ impl ElementImpl for EncoderStats { request_pad.set_property("name", &pad_name); self.obj().add_pad(&request_pad).unwrap(); *request_pad_guard = Some(request_pad.clone()); - + gst::info!(CAT, imp = self, "Created request pad: {}", pad_name); Some(request_pad.upcast()) } else { diff --git a/video/stats/src/videoencoderstatsmeta.rs b/video/stats/src/videoencoderstatsmeta.rs index 632c429f6..4abb39045 100644 --- a/video/stats/src/videoencoderstatsmeta.rs +++ b/video/stats/src/videoencoderstatsmeta.rs @@ -41,6 +41,10 @@ impl VideoEncoderStatsMeta { pub fn stats(&self) -> &VideoEncoderStats { &self.0.stats } + + pub fn replace(&mut self, stats: VideoEncoderStats) { + self.0.stats = stats; + } } unsafe impl MetaAPI for VideoEncoderStatsMeta { From 25205b30188f6294236d04df71f028129d173be1 Mon Sep 17 00:00:00 2001 From: diegonieto Date: Fri, 19 Sep 2025 13:46:47 +0200 Subject: [PATCH 25/46] Update video/stats/src/videoencoderstats.rs Co-authored-by: Ruben Gonzalez <56379722+rgonzalezfluendo@users.noreply.github.com> --- video/stats/src/videoencoderstats.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/video/stats/src/videoencoderstats.rs b/video/stats/src/videoencoderstats.rs index f15ababe5..bd649f268 100644 --- a/video/stats/src/videoencoderstats.rs +++ b/video/stats/src/videoencoderstats.rs @@ -104,7 +104,7 @@ impl fmt::Display for VideoEncoderStats { writeln!( f, "Output size: {} KB", - self.num_bytes / 1024, // Convert to KB + self.num_bytes / 1000, // Convert to KB )?; let framerate = self.framerate.unwrap(); From abc43b2a24260f2316ba2cbbc75c80fd4542b5ba Mon Sep 17 00:00:00 2001 From: diegonieto Date: Fri, 19 Sep 2025 13:47:26 +0200 Subject: [PATCH 26/46] Update video/stats/src/videoencoderstats.rs Co-authored-by: Ruben Gonzalez <56379722+rgonzalezfluendo@users.noreply.github.com> --- video/stats/src/videoencoderstats.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/video/stats/src/videoencoderstats.rs b/video/stats/src/videoencoderstats.rs index bd649f268..d7d679079 100644 --- a/video/stats/src/videoencoderstats.rs +++ b/video/stats/src/videoencoderstats.rs @@ -114,7 +114,7 @@ impl fmt::Display for VideoEncoderStats { } else { 0.0 }; - let bitrate_str = bitrate/1024.0; // Convert to kbps + let bitrate_str = bitrate/1000.0; // Convert to kbps writeln!(f, "Bitrate: {:.3} kbps", bitrate_str)?; From 77a5145d4b2db508709f5d33ade15da8ba489803 Mon Sep 17 00:00:00 2001 From: Diego Nieto Date: Fri, 19 Sep 2025 14:16:38 +0200 Subject: [PATCH 27/46] videocomparestats: remove end to end latency It might not represent the actual end2end latency --- video/stats/src/videoencoderstats.rs | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/video/stats/src/videoencoderstats.rs b/video/stats/src/videoencoderstats.rs index d7d679079..87c47b0a6 100644 --- a/video/stats/src/videoencoderstats.rs +++ b/video/stats/src/videoencoderstats.rs @@ -14,7 +14,6 @@ use std::fmt; use std::sync::LazyLock; use gst::ffi::GstClockTime; -use gst::prelude::ClockExt; use procfs::process::Process; @@ -150,15 +149,6 @@ impl fmt::Display for VideoEncoderStats { f, "Encode latency: {:.3} ms", encode_latency - )?; - - let current_time = gst::SystemClock::obtain(); - let latency = (current_time.time().unwrap().nseconds() as f64 - self.input_time as f64) / 1_000_000.0; - - writeln!( - f, - "End2End latency: {:.3} ms", - latency ) } } From d75ac79215196e966a5475e685a600f8af24311d Mon Sep 17 00:00:00 2001 From: Diego Nieto Date: Fri, 19 Sep 2025 14:23:56 +0200 Subject: [PATCH 28/46] videostats: add plugin register in all the examples --- video/stats/examples/video-stats-no-vmaf.rs | 2 ++ video/stats/examples/video-stats-split-screen.rs | 2 ++ video/stats/examples/video-stats-zero-latency.rs | 4 +++- video/stats/examples/video-stats.rs | 2 ++ 4 files changed, 9 insertions(+), 1 deletion(-) diff --git a/video/stats/examples/video-stats-no-vmaf.rs b/video/stats/examples/video-stats-no-vmaf.rs index f776205a6..4e5d29c01 100644 --- a/video/stats/examples/video-stats-no-vmaf.rs +++ b/video/stats/examples/video-stats-no-vmaf.rs @@ -13,6 +13,8 @@ use gst::prelude::*; fn main() -> Result<(), Error> { gst::init()?; + gstvideostats::plugin_register_static().expect("Failed to register videostats plugin"); + let pipeline = gst::parse::launch("souphttpsrc location=\"https://ftp.nluug.nl/pub/graphics/blender/demo/movies/ToS/tears_of_steel_1080p.mov\" ! qtdemux name=demux demux.video_0 ! queue ! decodebin3 ! videoconvertscale ! capsfilter caps=\"video/x-raw,aspect-ratio=1/1\" ! tee name=tee ! video-encoder-stats encoder=\"x264enc bitrate=1024\" name=vs0 vmaf-stats=false tee. ! video-encoder-stats encoder=\"x264enc bitrate=512\" name=vs1 video-compare-mixer split-screen=true backend=CPU name=mixer vs0.src ! h264parse ! avdec_h264 ! mixer.sink_0 vs1.src ! h264parse ! avdec_h264 ! mixer.sink_1 mixer. ! autovideosink")?; pipeline.set_state(gst::State::Playing)?; diff --git a/video/stats/examples/video-stats-split-screen.rs b/video/stats/examples/video-stats-split-screen.rs index c3882a716..04cb4647e 100644 --- a/video/stats/examples/video-stats-split-screen.rs +++ b/video/stats/examples/video-stats-split-screen.rs @@ -13,6 +13,8 @@ use gst::prelude::*; fn main() -> Result<(), Error> { gst::init()?; + gstvideostats::plugin_register_static().expect("Failed to register videostats plugin"); + let pipeline = gst::parse::launch("souphttpsrc location=\"https://ftp.nluug.nl/pub/graphics/blender/demo/movies/ToS/tears_of_steel_1080p.mov\" ! qtdemux name=demux demux.video_0 ! queue ! decodebin3 ! videoconvertscale ! capsfilter caps=\"video/x-raw,aspect-ratio=1/1\" ! tee name=tee ! video-encoder-stats encoder=\"x264enc\" decoder=\"h264parse ! avdec_h264\" name=vs0 tee. ! video-encoder-stats encoder=\"x264enc bitrate=512\" name=vs1 video-compare-mixer split-screen=true backend=CPU name=mixer vs0.src ! decodebin3 ! mixer.sink_0 vs1.src ! decodebin3 ! mixer.sink_1 mixer. ! autovideosink")?; pipeline.set_state(gst::State::Playing)?; diff --git a/video/stats/examples/video-stats-zero-latency.rs b/video/stats/examples/video-stats-zero-latency.rs index 37872bae5..a5d916ae7 100644 --- a/video/stats/examples/video-stats-zero-latency.rs +++ b/video/stats/examples/video-stats-zero-latency.rs @@ -13,7 +13,9 @@ use gst::prelude::*; fn main() -> Result<(), Error> { gst::init()?; - let pipeline = gst::parse::launch("souphttpsrc location=\"https://ftp.nluug.nl/pub/graphics/blender/demo/movies/ToS/tears_of_steel_1080p.mov\" ! qtdemux name=demux demux.video_0 ! queue ! decodebin3 ! videoconvertscale ! capsfilter caps=\"video/x-raw,aspect-ratio=1/1\" ! tee name=tee ! video-encoder-stats encoder=\"x264enc bitrate=1024 tune=zerolatency\" name=vs0 vmaf-stats=false tee. ! video-encoder-stats encoder=\"x264enc bitrate=512 tune=zerolatency\" name=vs1 video-compare-mixer split-screen=true backend=CPU name=mixer vs0.src ! h264parse ! avdec_h264 ! mixer.sink_0 vs1.src ! h264parse ! avdec_h264 ! mixer.sink_1 mixer. ! autovideosink")?; + gstvideostats::plugin_register_static().expect("Failed to register videostats plugin"); + + let pipeline = gst::parse::launch("souphttpsrc location=\"https://ftp.nluug.nl/pub/graphics/blender/demo/movies/ToS/tears_of_steel_1080p.mov\" ! qtdemux name=demux demux.video_0 ! queue ! decodebin3 ! videoconvertscale ! capsfilter caps=\"video/x-raw,aspect-ratio=1/1\" ! tee name=tee ! video-encoder-stats encoder=\"x264enc bitrate=1024 tune=zerolatency speed-preset=ultrafast threads=4\" name=vs0 vmaf-stats=false tee. ! video-encoder-stats encoder=\"x264enc bitrate=512 tune=zerolatency speed-preset=ultrafast\" name=vs1 video-compare-mixer split-screen=true backend=CPU name=mixer vs0.src ! h264parse ! avdec_h264 ! mixer.sink_0 vs1.src ! h264parse ! avdec_h264 ! mixer.sink_1 mixer. ! autovideosink")?; pipeline.set_state(gst::State::Playing)?; let bus = pipeline.bus().unwrap(); diff --git a/video/stats/examples/video-stats.rs b/video/stats/examples/video-stats.rs index e275cb8c5..36895b34f 100644 --- a/video/stats/examples/video-stats.rs +++ b/video/stats/examples/video-stats.rs @@ -13,6 +13,8 @@ use gst::prelude::*; fn main() -> Result<(), Error> { gst::init()?; + gstvideostats::plugin_register_static().expect("Failed to register videostats plugin"); + let pipeline = gst::parse::launch("souphttpsrc location=\"https://ftp.nluug.nl/pub/graphics/blender/demo/movies/ToS/tears_of_steel_1080p.mov\" ! qtdemux name=demux demux.video_0 ! queue ! decodebin3 ! videoconvertscale ! capsfilter caps=\"video/x-raw,aspect-ratio=1/1\" ! tee name=tee ! video-encoder-stats encoder=\"x264enc bitrate=1024\" decoder=\"h264parse ! avdec_h264\" name=vs0 ! fakesink tee. ! video-encoder-stats encoder=\"x264enc bitrate=512\" name=vs1 ! fakesink video-compare-mixer split-screen=false backend=CPU name=mixer vs0.decoder_src ! mixer.sink_0 vs1.decoder_src ! mixer.sink_1 mixer. ! autovideosink")?; pipeline.set_state(gst::State::Playing)?; From 1242a6bf264df27bf40f8713bd06ca62bd170557 Mon Sep 17 00:00:00 2001 From: Diego Nieto Date: Tue, 23 Sep 2025 13:05:54 +0200 Subject: [PATCH 29/46] videostats: update to the latest gst-plugins-rs bindings Fix time object new methods --- Cargo.lock | 1130 ++++++++++++++++----------- video/stats/src/encoderstats/imp.rs | 6 +- 2 files changed, 686 insertions(+), 450 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7aaf980ec..f804274a0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -122,9 +122,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.99" +version = "1.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0674a1ddeecb70197781e945de4b3b8ffb61fa939a5597bcf48503737663100" +checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" [[package]] name = "arbitrary" @@ -897,9 +897,9 @@ dependencies = [ [[package]] name = "base64ct" -version = "1.6.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" +checksum = "55248b47b0caf0546f7988906588779981c43bb1bc9d0c44087278f80cdb44ba" [[package]] name = "bincode" @@ -993,9 +993,9 @@ dependencies = [ [[package]] name = "bitstream-io" -version = "4.6.0" +version = "4.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dd3ca0e6861b2a2d785a7f0c267193e8d423d8ae3475aad1842f725d6a380c9" +checksum = "93fd429eae40b3b4564d8cc4ec8aba032ab870d1eb80744f6d934d524c1bdc71" dependencies = [ "core2", ] @@ -1082,29 +1082,29 @@ dependencies = [ [[package]] name = "cairo-rs" version = "0.22.0" -source = "git+https://github.com/gtk-rs/gtk-rs-core?branch=main#bbe4d709b7337f1bfea7b2a347524689979c6756" +source = "git+https://github.com/gtk-rs/gtk-rs-core?branch=main#7b4ffff1879bc05f334f0d015302e65cf381e35d" dependencies = [ "bitflags 2.9.4", "cairo-sys-rs", - "glib", + "glib 0.22.0", "libc", ] [[package]] name = "cairo-sys-rs" version = "0.22.0" -source = "git+https://github.com/gtk-rs/gtk-rs-core?branch=main#bbe4d709b7337f1bfea7b2a347524689979c6756" +source = "git+https://github.com/gtk-rs/gtk-rs-core?branch=main#7b4ffff1879bc05f334f0d015302e65cf381e35d" dependencies = [ - "glib-sys", + "glib-sys 0.22.0", "libc", "system-deps", ] [[package]] name = "cc" -version = "1.2.37" +version = "1.2.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65193589c6404eb80b450d618eaf9a2cafaaafd57ecce47370519ef674a7bd44" +checksum = "80f41ae168f955c12fb8960b057d70d0ca153fb83182b57d86380443527be7e9" dependencies = [ "find-msvc-tools", "jobserver", @@ -1177,9 +1177,9 @@ dependencies = [ [[package]] name = "cfg-expr" -version = "0.18.0" +version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a2b34126159980f92da2a08bdec0694fd80fb5eb9e48aff25d20a0d8dfa710d" +checksum = "1a2c5f3bf25ec225351aa1c8e230d04d880d3bd89dea133537dafad4ae291e5c" dependencies = [ "smallvec", "target-lexicon", @@ -1234,9 +1234,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.47" +version = "4.5.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7eac00902d9d136acd712710d71823fb8ac8004ca445a89e73a41d45aa712931" +checksum = "e2134bb3ea021b78629caa971416385309e0131b351b25e01dc16fb54e1b5fae" dependencies = [ "clap_builder", "clap_derive", @@ -1244,9 +1244,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.47" +version = "4.5.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ad9bbf750e73b5884fb8a211a9424a1906c1e156724260fdae972f31d70e1d6" +checksum = "c2ba64afa3c0a6df7fa517765e31314e983f51dda798ffba27b988194fb65dc9" dependencies = [ "anstream", "anstyle", @@ -1594,8 +1594,18 @@ version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.20.11", + "darling_macro 0.20.11", +] + +[[package]] +name = "darling" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" +dependencies = [ + "darling_core 0.21.3", + "darling_macro 0.21.3", ] [[package]] @@ -1612,13 +1622,38 @@ dependencies = [ "syn 2.0.106", ] +[[package]] +name = "darling_core" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.106", +] + [[package]] name = "darling_macro" version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ - "darling_core", + "darling_core 0.20.11", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "darling_macro" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" +dependencies = [ + "darling_core 0.21.3", "quote", "syn 2.0.106", ] @@ -1855,7 +1890,7 @@ version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" dependencies = [ - "darling", + "darling 0.20.11", "proc-macro2", "quote", "syn 2.0.106", @@ -1937,9 +1972,9 @@ dependencies = [ [[package]] name = "dssim-core" -version = "3.2.11" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5baa7723d29512337a4694b2a9a06618ce86bac66c0bcf42c3149cb09791648" +checksum = "e3c601412450ff29a9258b2f85b18b38f658caf70fad1692f40ca863d86cb753" dependencies = [ "imgref", "itertools 0.14.0", @@ -2226,9 +2261,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fd99930f64d146689264c637b5af2f0233a933bef0d8570e2526bf9e083192d" +checksum = "1ced73b1dacfc750a6db6c0a0c3a3853c8b41997e2e2c563dc90804ae6867959" [[package]] name = "fixedbitset" @@ -2415,22 +2450,22 @@ dependencies = [ [[package]] name = "gdk-pixbuf" version = "0.22.0" -source = "git+https://github.com/gtk-rs/gtk-rs-core?branch=main#bbe4d709b7337f1bfea7b2a347524689979c6756" +source = "git+https://github.com/gtk-rs/gtk-rs-core?branch=main#7b4ffff1879bc05f334f0d015302e65cf381e35d" dependencies = [ "gdk-pixbuf-sys", "gio", - "glib", + "glib 0.22.0", "libc", ] [[package]] name = "gdk-pixbuf-sys" version = "0.22.0" -source = "git+https://github.com/gtk-rs/gtk-rs-core?branch=main#bbe4d709b7337f1bfea7b2a347524689979c6756" +source = "git+https://github.com/gtk-rs/gtk-rs-core?branch=main#7b4ffff1879bc05f334f0d015302e65cf381e35d" dependencies = [ - "gio-sys", - "glib-sys", - "gobject-sys", + "gio-sys 0.22.0", + "glib-sys 0.22.0", + "gobject-sys 0.22.0", "libc", "system-deps", ] @@ -2438,13 +2473,13 @@ dependencies = [ [[package]] name = "gdk4" version = "0.11.0" -source = "git+https://github.com/gtk-rs/gtk4-rs?branch=main#e8b597d2285cebf2c13373dc883e9a7e2bec2a00" +source = "git+https://github.com/gtk-rs/gtk4-rs?branch=main#d294c123a8e39d9717dc21c373e1af2c857dda30" dependencies = [ "cairo-rs", "gdk-pixbuf", "gdk4-sys", "gio", - "glib", + "glib 0.22.0", "libc", "pango", ] @@ -2452,13 +2487,13 @@ dependencies = [ [[package]] name = "gdk4-sys" version = "0.11.0" -source = "git+https://github.com/gtk-rs/gtk4-rs?branch=main#e8b597d2285cebf2c13373dc883e9a7e2bec2a00" +source = "git+https://github.com/gtk-rs/gtk4-rs?branch=main#d294c123a8e39d9717dc21c373e1af2c857dda30" dependencies = [ "cairo-sys-rs", "gdk-pixbuf-sys", - "gio-sys", - "glib-sys", - "gobject-sys", + "gio-sys 0.22.0", + "glib-sys 0.22.0", + "gobject-sys 0.22.0", "libc", "pango-sys", "pkg-config", @@ -2468,21 +2503,21 @@ dependencies = [ [[package]] name = "gdk4-wayland" version = "0.11.0" -source = "git+https://github.com/gtk-rs/gtk4-rs?branch=main#e8b597d2285cebf2c13373dc883e9a7e2bec2a00" +source = "git+https://github.com/gtk-rs/gtk4-rs?branch=main#d294c123a8e39d9717dc21c373e1af2c857dda30" dependencies = [ "gdk4", "gdk4-wayland-sys", "gio", - "glib", + "glib 0.22.0", "libc", ] [[package]] name = "gdk4-wayland-sys" version = "0.11.0" -source = "git+https://github.com/gtk-rs/gtk4-rs?branch=main#e8b597d2285cebf2c13373dc883e9a7e2bec2a00" +source = "git+https://github.com/gtk-rs/gtk4-rs?branch=main#d294c123a8e39d9717dc21c373e1af2c857dda30" dependencies = [ - "glib-sys", + "glib-sys 0.22.0", "libc", "system-deps", ] @@ -2490,12 +2525,12 @@ dependencies = [ [[package]] name = "gdk4-win32" version = "0.11.0" -source = "git+https://github.com/gtk-rs/gtk4-rs?branch=main#e8b597d2285cebf2c13373dc883e9a7e2bec2a00" +source = "git+https://github.com/gtk-rs/gtk4-rs?branch=main#d294c123a8e39d9717dc21c373e1af2c857dda30" dependencies = [ "gdk4", "gdk4-win32-sys", "gio", - "glib", + "glib 0.22.0", "khronos-egl", "libc", ] @@ -2503,10 +2538,10 @@ dependencies = [ [[package]] name = "gdk4-win32-sys" version = "0.11.0" -source = "git+https://github.com/gtk-rs/gtk4-rs?branch=main#e8b597d2285cebf2c13373dc883e9a7e2bec2a00" +source = "git+https://github.com/gtk-rs/gtk4-rs?branch=main#d294c123a8e39d9717dc21c373e1af2c857dda30" dependencies = [ "gdk4-sys", - "glib-sys", + "glib-sys 0.22.0", "libc", "system-deps", ] @@ -2514,22 +2549,22 @@ dependencies = [ [[package]] name = "gdk4-x11" version = "0.11.0" -source = "git+https://github.com/gtk-rs/gtk4-rs?branch=main#e8b597d2285cebf2c13373dc883e9a7e2bec2a00" +source = "git+https://github.com/gtk-rs/gtk4-rs?branch=main#d294c123a8e39d9717dc21c373e1af2c857dda30" dependencies = [ "gdk4", "gdk4-x11-sys", "gio", - "glib", + "glib 0.22.0", "libc", ] [[package]] name = "gdk4-x11-sys" version = "0.11.0" -source = "git+https://github.com/gtk-rs/gtk4-rs?branch=main#e8b597d2285cebf2c13373dc883e9a7e2bec2a00" +source = "git+https://github.com/gtk-rs/gtk4-rs?branch=main#d294c123a8e39d9717dc21c373e1af2c857dda30" dependencies = [ "gdk4-sys", - "glib-sys", + "glib-sys 0.22.0", "libc", "system-deps", ] @@ -2601,35 +2636,67 @@ checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "gio" version = "0.22.0" -source = "git+https://github.com/gtk-rs/gtk-rs-core?branch=main#bbe4d709b7337f1bfea7b2a347524689979c6756" +source = "git+https://github.com/gtk-rs/gtk-rs-core?branch=main#7b4ffff1879bc05f334f0d015302e65cf381e35d" dependencies = [ "futures-channel", "futures-core", "futures-io", "futures-util", - "gio-sys", - "glib", + "gio-sys 0.22.0", + "glib 0.22.0", "libc", "pin-project-lite", "smallvec", ] +[[package]] +name = "gio-sys" +version = "0.20.12" +source = "git+https://github.com/gtk-rs/gtk-rs-core?branch=0.20#337dde55bb1665a0a1286e689ec2819383d4e299" +dependencies = [ + "glib-sys 0.20.12", + "gobject-sys 0.20.12", + "libc", + "system-deps", + "windows-sys 0.59.0", +] + [[package]] name = "gio-sys" version = "0.22.0" -source = "git+https://github.com/gtk-rs/gtk-rs-core?branch=main#bbe4d709b7337f1bfea7b2a347524689979c6756" +source = "git+https://github.com/gtk-rs/gtk-rs-core?branch=main#7b4ffff1879bc05f334f0d015302e65cf381e35d" dependencies = [ - "glib-sys", - "gobject-sys", + "glib-sys 0.22.0", + "gobject-sys 0.22.0", "libc", "system-deps", "windows-sys 0.61.0", ] +[[package]] +name = "glib" +version = "0.20.12" +source = "git+https://github.com/gtk-rs/gtk-rs-core?branch=0.20#337dde55bb1665a0a1286e689ec2819383d4e299" +dependencies = [ + "bitflags 2.9.4", + "futures-channel", + "futures-core", + "futures-executor", + "futures-task", + "futures-util", + "gio-sys 0.20.12", + "glib-macros 0.20.12", + "glib-sys 0.20.12", + "gobject-sys 0.20.12", + "libc", + "memchr", + "smallvec", +] + [[package]] name = "glib" version = "0.22.0" -source = "git+https://github.com/gtk-rs/gtk-rs-core?branch=main#bbe4d709b7337f1bfea7b2a347524689979c6756" +source = "git+https://github.com/gtk-rs/gtk-rs-core?branch=main#7b4ffff1879bc05f334f0d015302e65cf381e35d" dependencies = [ "bitflags 2.9.4", "futures-channel", @@ -2637,19 +2704,31 @@ dependencies = [ "futures-executor", "futures-task", "futures-util", - "gio-sys", - "glib-macros", - "glib-sys", - "gobject-sys", + "gio-sys 0.22.0", + "glib-macros 0.22.0", + "glib-sys 0.22.0", + "gobject-sys 0.22.0", "libc", "memchr", "smallvec", ] +[[package]] +name = "glib-macros" +version = "0.20.12" +source = "git+https://github.com/gtk-rs/gtk-rs-core?branch=0.20#337dde55bb1665a0a1286e689ec2819383d4e299" +dependencies = [ + "heck 0.5.0", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.106", +] + [[package]] name = "glib-macros" version = "0.22.0" -source = "git+https://github.com/gtk-rs/gtk-rs-core?branch=main#bbe4d709b7337f1bfea7b2a347524689979c6756" +source = "git+https://github.com/gtk-rs/gtk-rs-core?branch=main#7b4ffff1879bc05f334f0d015302e65cf381e35d" dependencies = [ "heck 0.5.0", "proc-macro-crate", @@ -2658,10 +2737,19 @@ dependencies = [ "syn 2.0.106", ] +[[package]] +name = "glib-sys" +version = "0.20.12" +source = "git+https://github.com/gtk-rs/gtk-rs-core?branch=0.20#337dde55bb1665a0a1286e689ec2819383d4e299" +dependencies = [ + "libc", + "system-deps", +] + [[package]] name = "glib-sys" version = "0.22.0" -source = "git+https://github.com/gtk-rs/gtk-rs-core?branch=main#bbe4d709b7337f1bfea7b2a347524689979c6756" +source = "git+https://github.com/gtk-rs/gtk-rs-core?branch=main#7b4ffff1879bc05f334f0d015302e65cf381e35d" dependencies = [ "libc", "system-deps", @@ -2673,12 +2761,22 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +[[package]] +name = "gobject-sys" +version = "0.20.12" +source = "git+https://github.com/gtk-rs/gtk-rs-core?branch=0.20#337dde55bb1665a0a1286e689ec2819383d4e299" +dependencies = [ + "glib-sys 0.20.12", + "libc", + "system-deps", +] + [[package]] name = "gobject-sys" version = "0.22.0" -source = "git+https://github.com/gtk-rs/gtk-rs-core?branch=main#bbe4d709b7337f1bfea7b2a347524689979c6756" +source = "git+https://github.com/gtk-rs/gtk-rs-core?branch=main#7b4ffff1879bc05f334f0d015302e65cf381e35d" dependencies = [ - "glib-sys", + "glib-sys 0.22.0", "libc", "system-deps", ] @@ -2707,9 +2805,9 @@ dependencies = [ [[package]] name = "graphene-rs" version = "0.22.0" -source = "git+https://github.com/gtk-rs/gtk-rs-core?branch=main#bbe4d709b7337f1bfea7b2a347524689979c6756" +source = "git+https://github.com/gtk-rs/gtk-rs-core?branch=main#7b4ffff1879bc05f334f0d015302e65cf381e35d" dependencies = [ - "glib", + "glib 0.22.0", "graphene-sys", "libc", ] @@ -2717,9 +2815,9 @@ dependencies = [ [[package]] name = "graphene-sys" version = "0.22.0" -source = "git+https://github.com/gtk-rs/gtk-rs-core?branch=main#bbe4d709b7337f1bfea7b2a347524689979c6756" +source = "git+https://github.com/gtk-rs/gtk-rs-core?branch=main#7b4ffff1879bc05f334f0d015302e65cf381e35d" dependencies = [ - "glib-sys", + "glib-sys 0.22.0", "libc", "pkg-config", "system-deps", @@ -2739,11 +2837,11 @@ dependencies = [ [[package]] name = "gsk4" version = "0.11.0" -source = "git+https://github.com/gtk-rs/gtk4-rs?branch=main#e8b597d2285cebf2c13373dc883e9a7e2bec2a00" +source = "git+https://github.com/gtk-rs/gtk4-rs?branch=main#d294c123a8e39d9717dc21c373e1af2c857dda30" dependencies = [ "cairo-rs", "gdk4", - "glib", + "glib 0.22.0", "graphene-rs", "gsk4-sys", "libc", @@ -2753,12 +2851,12 @@ dependencies = [ [[package]] name = "gsk4-sys" version = "0.11.0" -source = "git+https://github.com/gtk-rs/gtk4-rs?branch=main#e8b597d2285cebf2c13373dc883e9a7e2bec2a00" +source = "git+https://github.com/gtk-rs/gtk4-rs?branch=main#d294c123a8e39d9717dc21c373e1af2c857dda30" dependencies = [ "cairo-sys-rs", "gdk4-sys", - "glib-sys", - "gobject-sys", + "glib-sys 0.22.0", + "gobject-sys 0.22.0", "graphene-sys", "libc", "pango-sys", @@ -2770,14 +2868,14 @@ name = "gst-plugin-analytics" version = "0.15.0-alpha.1" dependencies = [ "chrono", - "glib", - "gst-plugin-version-helper", - "gstreamer", + "glib 0.22.0", + "gst-plugin-version-helper 0.8.1", + "gstreamer 0.25.0", "gstreamer-analytics", - "gstreamer-base", + "gstreamer-base 0.25.0", "gstreamer-check", "gstreamer-rtp", - "gstreamer-video", + "gstreamer-video 0.25.0", "xmltree", ] @@ -2790,10 +2888,10 @@ dependencies = [ "byte-slice-cast", "ebur128", "gst-plugin-version-helper 0.8.1", - "gstreamer", + "gstreamer 0.25.0", "gstreamer-app", "gstreamer-audio", - "gstreamer-base", + "gstreamer-base 0.25.0", "gstreamer-check", "hrtf", "nnnoiseless", @@ -2806,8 +2904,8 @@ dependencies = [ name = "gst-plugin-audioparsers" version = "0.15.0-alpha.1" dependencies = [ - "gst-plugin-version-helper", - "gstreamer", + "gst-plugin-version-helper 0.8.1", + "gstreamer 0.25.0", ] [[package]] @@ -2832,11 +2930,11 @@ dependencies = [ "futures", "gio", "gst-plugin-version-helper 0.8.1", - "gstreamer", + "gstreamer 0.25.0", "gstreamer-audio", - "gstreamer-base", + "gstreamer-base 0.25.0", "gstreamer-check", - "gstreamer-video", + "gstreamer-video 0.25.0", "percent-encoding", "rand 0.9.2", "serde", @@ -2857,10 +2955,10 @@ dependencies = [ "cdg", "cdg_renderer", "gst-plugin-version-helper 0.8.1", - "gstreamer", + "gstreamer 0.25.0", "gstreamer-app", - "gstreamer-base", - "gstreamer-video", + "gstreamer-base 0.25.0", + "gstreamer-video 0.25.0", "image", ] @@ -2872,7 +2970,7 @@ dependencies = [ "byte-slice-cast", "claxon", "gst-plugin-version-helper 0.8.1", - "gstreamer", + "gstreamer 0.25.0", "gstreamer-audio", "gstreamer-check", ] @@ -2893,10 +2991,10 @@ dependencies = [ "clap", "either", "gst-plugin-version-helper 0.8.1", - "gstreamer", - "gstreamer-base", + "gstreamer 0.25.0", + "gstreamer-base 0.25.0", "gstreamer-check", - "gstreamer-video", + "gstreamer-video 0.25.0", "itertools 0.14.0", "pango", "pangocairo", @@ -2916,9 +3014,9 @@ dependencies = [ "byte-slice-cast", "csound", "gst-plugin-version-helper 0.8.1", - "gstreamer", + "gstreamer 0.25.0", "gstreamer-audio", - "gstreamer-base", + "gstreamer-base 0.25.0", "gstreamer-check", ] @@ -2928,9 +3026,9 @@ version = "0.15.0-alpha.1" dependencies = [ "dav1d", "gst-plugin-version-helper 0.8.1", - "gstreamer", - "gstreamer-base", - "gstreamer-video", + "gstreamer 0.25.0", + "gstreamer-base 0.25.0", + "gstreamer-video 0.25.0", "num_cpus", ] @@ -2941,12 +3039,12 @@ dependencies = [ "anyhow", "bytes", "futures", - "gst-plugin-version-helper", - "gstreamer", + "gst-plugin-version-helper 0.8.1", + "gstreamer 0.25.0", "gstreamer-audio", - "gstreamer-base", + "gstreamer-base 0.25.0", "reqwest 0.12.23", - "rustls 0.23.31", + "rustls 0.23.32", "serde", "serde_json", "signalsmith-stretch", @@ -2960,12 +3058,12 @@ dependencies = [ "gio", "gst-plugin-gtk4", "gst-plugin-version-helper 0.8.1", - "gstreamer", + "gstreamer 0.25.0", "gstreamer-app", "gstreamer-audio", - "gstreamer-base", + "gstreamer-base 0.25.0", "gstreamer-check", - "gstreamer-video", + "gstreamer-video 0.25.0", "gtk4", "parking_lot", "rand 0.9.2", @@ -2978,9 +3076,9 @@ dependencies = [ "byte-slice-cast", "ffv1", "gst-plugin-version-helper 0.8.1", - "gstreamer", + "gstreamer 0.25.0", "gstreamer-check", - "gstreamer-video", + "gstreamer-video 0.25.0", ] [[package]] @@ -2988,8 +3086,8 @@ name = "gst-plugin-file" version = "0.15.0-alpha.1" dependencies = [ "gst-plugin-version-helper 0.8.1", - "gstreamer", - "gstreamer-base", + "gstreamer 0.25.0", + "gstreamer-base 0.25.0", "url", ] @@ -3000,9 +3098,9 @@ dependencies = [ "byteorder", "flavors", "gst-plugin-version-helper 0.8.1", - "gstreamer", + "gstreamer 0.25.0", "gstreamer-audio", - "gstreamer-base", + "gstreamer-base 0.25.0", "nom 7.1.3", "num-rational", "smallvec", @@ -3017,14 +3115,14 @@ dependencies = [ "chrono", "dash-mpd", "gst-plugin-version-helper 0.8.1", - "gstreamer", + "gstreamer 0.25.0", "gstreamer-app", "gstreamer-audio", - "gstreamer-base", + "gstreamer-base 0.25.0", "gstreamer-check", "gstreamer-pbutils", "gstreamer-tag", - "gstreamer-video", + "gstreamer-video 0.25.0", "m3u8-rs", "quick-xml 0.38.3", "serde", @@ -3037,9 +3135,9 @@ dependencies = [ "atomic_refcell", "gif", "gst-plugin-version-helper 0.8.1", - "gstreamer", + "gstreamer 0.25.0", "gstreamer-check", - "gstreamer-video", + "gstreamer-video 0.25.0", ] [[package]] @@ -3048,10 +3146,10 @@ version = "0.15.0-alpha.1" dependencies = [ "anyhow", "gst-plugin-version-helper 0.8.1", - "gstreamer", + "gstreamer 0.25.0", "gstreamer-app", "gstreamer-check", - "gstreamer-video", + "gstreamer-video 0.25.0", ] [[package]] @@ -3063,14 +3161,14 @@ dependencies = [ "gdk4-win32", "gdk4-x11", "gst-plugin-version-helper 0.8.1", - "gstreamer", + "gstreamer 0.25.0", "gstreamer-allocators", - "gstreamer-base", + "gstreamer-base 0.25.0", "gstreamer-gl", "gstreamer-gl-egl", "gstreamer-gl-wayland", "gstreamer-gl-x11", - "gstreamer-video", + "gstreamer-video 0.25.0", "gtk4", "windows-sys 0.61.0", ] @@ -3089,12 +3187,12 @@ dependencies = [ "gst-plugin-fmp4", "gst-plugin-hlssink3", "gst-plugin-version-helper 0.8.1", - "gstreamer", + "gstreamer 0.25.0", "gstreamer-app", "gstreamer-audio", "gstreamer-check", "gstreamer-pbutils", - "gstreamer-video", + "gstreamer-video 0.25.0", "m3u8-rs", "serial_test", "thiserror 2.0.16", @@ -3107,13 +3205,14 @@ dependencies = [ "anyhow", "chrono", "gio", + "gst-plugin-fmp4", "gst-plugin-version-helper 0.8.1", - "gstreamer", + "gstreamer 0.25.0", "gstreamer-app", "gstreamer-audio", "gstreamer-check", "gstreamer-pbutils", - "gstreamer-video", + "gstreamer-video 0.25.0", "m3u8-rs", "sprintf", ] @@ -3124,11 +3223,11 @@ version = "0.15.0-alpha.1" dependencies = [ "byte-slice-cast", "gst-plugin-version-helper 0.8.1", - "gstreamer", + "gstreamer 0.25.0", "gstreamer-audio", - "gstreamer-base", + "gstreamer-base 0.25.0", "gstreamer-check", - "gstreamer-video", + "gstreamer-video 0.25.0", "num-traits", ] @@ -3139,7 +3238,7 @@ dependencies = [ "anyhow", "futures", "gst-plugin-version-helper 0.8.1", - "gstreamer", + "gstreamer 0.25.0", "gstreamer-app", "gstreamer-check", "gstreamer-utils", @@ -3154,7 +3253,7 @@ name = "gst-plugin-json" version = "0.15.0-alpha.1" dependencies = [ "gst-plugin-version-helper 0.8.1", - "gstreamer", + "gstreamer 0.25.0", "gstreamer-check", "serde", "serde_json", @@ -3167,7 +3266,7 @@ dependencies = [ "atomic_refcell", "byte-slice-cast", "gst-plugin-version-helper 0.8.1", - "gstreamer", + "gstreamer 0.25.0", "gstreamer-audio", "gstreamer-check", "lewton", @@ -3180,7 +3279,7 @@ dependencies = [ "gio", "gst-plugin-gtk4", "gst-plugin-version-helper 0.8.1", - "gstreamer", + "gstreamer 0.25.0", "gstreamer-audio", "gstreamer-check", "gtk4", @@ -3195,12 +3294,12 @@ dependencies = [ "anyhow", "bitstream-io", "gst-plugin-version-helper 0.8.1", - "gstreamer", + "gstreamer 0.25.0", "gstreamer-audio", - "gstreamer-base", + "gstreamer-base 0.25.0", "gstreamer-pbutils", "gstreamer-tag", - "gstreamer-video", + "gstreamer-video 0.25.0", "mp4-atom", "num-integer", "tempfile", @@ -3214,7 +3313,7 @@ dependencies = [ "anyhow", "bitstream-io", "gst-plugin-version-helper 0.8.1", - "gstreamer", + "gstreamer 0.25.0", "smallvec", ] @@ -3226,12 +3325,12 @@ dependencies = [ "byte-slice-cast", "byteorder", "data-encoding", - "glib", + "glib 0.22.0", "gst-plugin-version-helper 0.8.1", - "gstreamer", + "gstreamer 0.25.0", "gstreamer-audio", - "gstreamer-base", - "gstreamer-video", + "gstreamer-base 0.25.0", + "gstreamer-video 0.25.0", "libloading", "quick-xml 0.38.3", "smallvec", @@ -3245,25 +3344,38 @@ dependencies = [ "cairo-rs", "chrono", "gst-plugin-version-helper 0.8.1", - "gstreamer", - "gstreamer-base", + "gstreamer 0.25.0", + "gstreamer-base 0.25.0", "gstreamer-rtp", - "gstreamer-video", + "gstreamer-video 0.25.0", "pango", "pangocairo", "xmlparser", "xmltree", ] +[[package]] +name = "gst-plugin-originalbuffer" +version = "0.13.0" +source = "git+https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs?tag=0.13.0#fea2343968da83f3cce860f5a76fa2a2b0f22c16" +dependencies = [ + "atomic_refcell", + "glib 0.20.12", + "gst-plugin-version-helper 0.8.1 (git+https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs?tag=0.13.0)", + "gstreamer 0.23.7", + "gstreamer-video 0.23.7", + "once_cell", +] + [[package]] name = "gst-plugin-originalbuffer" version = "0.15.0-alpha.1" dependencies = [ "atomic_refcell", - "glib", + "glib 0.22.0", "gst-plugin-version-helper 0.8.1", - "gstreamer", - "gstreamer-video", + "gstreamer 0.25.0", + "gstreamer-video 0.25.0", ] [[package]] @@ -3271,9 +3383,9 @@ name = "gst-plugin-png" version = "0.15.0-alpha.1" dependencies = [ "gst-plugin-version-helper 0.8.1", - "gstreamer", + "gstreamer 0.25.0", "gstreamer-check", - "gstreamer-video", + "gstreamer-video 0.25.0", "parking_lot", "png", ] @@ -3288,16 +3400,16 @@ dependencies = [ "ctrlc", "env_logger", "futures", - "glib", + "glib 0.22.0", "gst-plugin-version-helper 0.8.1", - "gstreamer", - "gstreamer-base", + "gstreamer 0.25.0", + "gstreamer-base 0.25.0", "gstreamer-check", "itertools 0.14.0", "quinn", "quinn-proto", "rcgen", - "rustls 0.23.31", + "rustls 0.23.32", "rustls-pemfile 2.2.0", "rustls-pki-types", "serial_test", @@ -3312,8 +3424,8 @@ name = "gst-plugin-raptorq" version = "0.15.0-alpha.1" dependencies = [ "gst-plugin-version-helper 0.8.1", - "gstreamer", - "gstreamer-base", + "gstreamer 0.25.0", + "gstreamer-base 0.25.0", "gstreamer-check", "gstreamer-rtp", "rand 0.9.2", @@ -3326,9 +3438,9 @@ version = "0.15.0-alpha.1" dependencies = [ "atomic_refcell", "gst-plugin-version-helper 0.8.1", - "gstreamer", + "gstreamer 0.25.0", "gstreamer-check", - "gstreamer-video", + "gstreamer-video 0.25.0", "rav1e", ] @@ -3337,26 +3449,11 @@ name = "gst-plugin-regex" version = "0.15.0-alpha.1" dependencies = [ "gst-plugin-version-helper 0.8.1", - "gstreamer", + "gstreamer 0.25.0", "gstreamer-check", "regex", ] -[[package]] -name = "gst-plugin-relationmeta" -version = "0.14.0-alpha.1" -dependencies = [ - "chrono", - "glib", - "gst-plugin-version-helper 0.8.1", - "gstreamer", - "gstreamer-analytics", - "gstreamer-base", - "gstreamer-rtp", - "gstreamer-video", - "xmltree", -] - [[package]] name = "gst-plugin-reqwest" version = "0.15.0-alpha.1" @@ -3364,15 +3461,15 @@ dependencies = [ "bytes", "futures", "gst-plugin-version-helper 0.8.1", - "gstreamer", - "gstreamer-base", + "gstreamer 0.25.0", + "gstreamer-base 0.25.0", "headers", "http-body-util", "hyper 1.7.0", "mime", "pin-project-lite", "reqwest 0.12.23", - "rustls 0.23.31", + "rustls 0.23.32", "tokio", "url", ] @@ -3388,15 +3485,16 @@ dependencies = [ "chrono", "futures", "gio", + "glib 0.22.0", "gst-plugin-version-helper 0.8.1", - "gstreamer", + "gstreamer 0.25.0", "gstreamer-app", "gstreamer-audio", - "gstreamer-base", + "gstreamer-base 0.25.0", "gstreamer-check", "gstreamer-net", "gstreamer-rtp", - "gstreamer-video", + "gstreamer-video 0.25.0", "hex", "itertools 0.14.0", "log", @@ -3420,7 +3518,7 @@ dependencies = [ "data-encoding", "futures", "gst-plugin-version-helper 0.8.1", - "gstreamer", + "gstreamer 0.25.0", "gstreamer-app", "gstreamer-net", "gstreamer-pbutils", @@ -3438,11 +3536,11 @@ dependencies = [ name = "gst-plugin-skia" version = "0.15.0-alpha.1" dependencies = [ - "gst-plugin-version-helper", - "gstreamer", - "gstreamer-base", + "gst-plugin-version-helper 0.8.1", + "gstreamer 0.25.0", + "gstreamer-base 0.25.0", "gstreamer-check", - "gstreamer-video", + "gstreamer-video 0.25.0", "skia-safe", ] @@ -3452,9 +3550,9 @@ version = "0.15.0-alpha.1" dependencies = [ "clap", "gst-plugin-version-helper 0.8.1", - "gstreamer", + "gstreamer 0.25.0", "gstreamer-app", - "gstreamer-base", + "gstreamer-base 0.25.0", "gstreamer-check", "hex", "pretty_assertions", @@ -3473,11 +3571,11 @@ dependencies = [ "atomic_refcell", "futures", "gst-plugin-version-helper 0.8.1", - "gstreamer", + "gstreamer 0.25.0", "gstreamer-audio", - "gstreamer-base", + "gstreamer-base 0.25.0", "http 1.3.1", - "rustls 0.23.31", + "rustls 0.23.32", "serde", "serde_json", "tokio", @@ -3491,8 +3589,8 @@ dependencies = [ "anyhow", "futures", "gst-plugin-version-helper 0.8.1", - "gstreamer", - "gstreamer-base", + "gstreamer 0.25.0", + "gstreamer-base 0.25.0", "librespot-core", "librespot-metadata", "librespot-playback", @@ -3505,7 +3603,7 @@ name = "gst-plugin-streamgrouper" version = "0.15.0-alpha.1" dependencies = [ "gst-plugin-version-helper 0.8.1", - "gstreamer", + "gstreamer 0.25.0", "gstreamer-check", ] @@ -3514,8 +3612,8 @@ name = "gst-plugin-textaccumulate" version = "0.15.0-alpha.1" dependencies = [ "anyhow", - "gst-plugin-version-helper", - "gstreamer", + "gst-plugin-version-helper 0.8.1", + "gstreamer 0.25.0", "icu_locale", "icu_provider", "icu_segmenter", @@ -3528,7 +3626,7 @@ name = "gst-plugin-textahead" version = "0.15.0-alpha.1" dependencies = [ "gst-plugin-version-helper 0.8.1", - "gstreamer", + "gstreamer 0.25.0", ] [[package]] @@ -3536,7 +3634,7 @@ name = "gst-plugin-textwrap" version = "0.15.0-alpha.1" dependencies = [ "gst-plugin-version-helper 0.8.1", - "gstreamer", + "gstreamer 0.25.0", "gstreamer-check", "hyphenation", "textwrap", @@ -3556,8 +3654,9 @@ dependencies = [ "futures", "getifaddrs", "gio", + "gst-plugin-rtp", "gst-plugin-version-helper 0.8.1", - "gstreamer", + "gstreamer 0.25.0", "gstreamer-app", "gstreamer-audio", "gstreamer-check", @@ -3584,10 +3683,10 @@ dependencies = [ "gio", "gst-plugin-gtk4", "gst-plugin-version-helper 0.8.1", - "gstreamer", + "gstreamer 0.25.0", "gstreamer-audio", "gstreamer-check", - "gstreamer-video", + "gstreamer-video 0.25.0", "gtk4", "parking_lot", ] @@ -3603,7 +3702,7 @@ dependencies = [ "etherparse", "futures", "gst-plugin-version-helper 0.8.1", - "gstreamer", + "gstreamer 0.25.0", "pcap-file", "regex", "serde", @@ -3621,10 +3720,10 @@ version = "0.15.0-alpha.1" dependencies = [ "byte-slice-cast", "gst-plugin-version-helper 0.8.1", - "gstreamer", + "gstreamer 0.25.0", "gstreamer-audio", - "gstreamer-base", - "gstreamer-video", + "gstreamer-base 0.25.0", + "gstreamer-video 0.25.0", "num-traits", ] @@ -3635,7 +3734,7 @@ dependencies = [ "anyhow", "clap", "gst-plugin-version-helper 0.8.1", - "gstreamer", + "gstreamer 0.25.0", "gstreamer-app", "more-asserts", "reqwest 0.12.23", @@ -3649,16 +3748,25 @@ name = "gst-plugin-version-helper" version = "0.8.1" dependencies = [ "chrono", - "toml_edit 0.23.5", + "toml_edit 0.23.6", ] [[package]] name = "gst-plugin-version-helper" version = "0.8.1" -source = "git+https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs#c580400d5c96439b9451ccb669449859e36ebdc2" +source = "git+https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs?tag=0.13.0#fea2343968da83f3cce860f5a76fa2a2b0f22c16" dependencies = [ "chrono", - "toml_edit", + "toml_edit 0.22.27", +] + +[[package]] +name = "gst-plugin-version-helper" +version = "0.8.1" +source = "git+https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs#c553001f065c028b089ebe5817dca90a54fcf69d" +dependencies = [ + "chrono", + "toml_edit 0.23.6", ] [[package]] @@ -3671,10 +3779,10 @@ dependencies = [ "color-thief", "dssim-core", "gst-plugin-version-helper 0.8.1", - "gstreamer", - "gstreamer-base", + "gstreamer 0.25.0", + "gstreamer-base 0.25.0", "gstreamer-check", - "gstreamer-video", + "gstreamer-video 0.25.0", "image", "image_hasher", "rgb", @@ -3686,10 +3794,12 @@ version = "0.1.0" dependencies = [ "anyhow", "atomic_refcell", - "glib", + "glib 0.22.0", + "gst-plugin-originalbuffer 0.13.0", "gst-plugin-version-helper 0.8.1 (git+https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs)", - "gstreamer", - "gstreamer-video", + "gstreamer 0.25.0", + "gstreamer-check", + "gstreamer-video 0.25.0", "hound", "human_bytes", "procfs", @@ -3700,10 +3810,10 @@ name = "gst-plugin-vvdec" version = "0.15.0-alpha.1" dependencies = [ "gst-plugin-version-helper 0.8.1", - "gstreamer", + "gstreamer 0.25.0", "gstreamer-audio", "gstreamer-check", - "gstreamer-video", + "gstreamer-video 0.25.0", "vvdec", ] @@ -3712,9 +3822,9 @@ name = "gst-plugin-webp" version = "0.15.0-alpha.1" dependencies = [ "gst-plugin-version-helper 0.8.1", - "gstreamer", + "gstreamer 0.25.0", "gstreamer-check", - "gstreamer-video", + "gstreamer-video 0.25.0", "libwebp-sys2", "pretty_assertions", ] @@ -3747,15 +3857,15 @@ dependencies = [ "gst-plugin-version-helper 0.8.1", "gst-plugin-webrtc-signalling", "gst-plugin-webrtc-signalling-protocol", - "gstreamer", + "gstreamer 0.25.0", "gstreamer-app", "gstreamer-audio", - "gstreamer-base", + "gstreamer-base 0.25.0", "gstreamer-net", "gstreamer-rtp", "gstreamer-sdp", "gstreamer-utils", - "gstreamer-video", + "gstreamer-video 0.25.0", "gstreamer-webrtc", "http 1.3.1", "human_bytes", @@ -3767,7 +3877,7 @@ dependencies = [ "rand 0.9.2", "regex", "reqwest 0.12.23", - "rustls 0.23.31", + "rustls 0.23.32", "rustls-pemfile 2.2.0", "rustls-pki-types", "serde", @@ -3795,7 +3905,7 @@ dependencies = [ "futures", "gst-plugin-webrtc-signalling-protocol", "pin-project-lite", - "rustls 0.23.31", + "rustls 0.23.32", "rustls-pemfile 2.2.0", "rustls-pki-types", "serde", @@ -3826,33 +3936,57 @@ dependencies = [ "bytes", "futures", "gst-plugin-version-helper 0.8.1", - "gstreamer", + "gstreamer 0.25.0", "gstreamer-sdp", "gstreamer-webrtc", "parse_link_header", "reqwest 0.12.23", - "rustls 0.23.31", + "rustls 0.23.32", "tokio", ] +[[package]] +name = "gstreamer" +version = "0.23.7" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=0.23#0f9f71821ada8ec7c16c27e08181523cfb5d2a30" +dependencies = [ + "cfg-if", + "futures-channel", + "futures-core", + "futures-util", + "glib 0.20.12", + "gstreamer-sys 0.23.7", + "itertools 0.14.0", + "libc", + "muldiv", + "num-integer", + "num-rational", + "once_cell", + "option-operations 0.5.0", + "paste", + "pin-project-lite", + "smallvec", + "thiserror 2.0.16", +] + [[package]] name = "gstreamer" version = "0.25.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#8a41295b995e4154dee1c18cfea77efc25ec3817" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#c533fe960af057c55916ba3fb48c42837da98565" dependencies = [ "cfg-if", "futures-channel", "futures-core", "futures-util", - "glib", - "gstreamer-sys", + "glib 0.22.0", + "gstreamer-sys 0.25.0", "itertools 0.14.0", "kstring", "libc", "muldiv", "num-integer", "num-rational", - "option-operations", + "option-operations 0.6.0", "pastey", "pin-project-lite", "serde", @@ -3864,10 +3998,10 @@ dependencies = [ [[package]] name = "gstreamer-allocators" version = "0.25.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#8a41295b995e4154dee1c18cfea77efc25ec3817" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#c533fe960af057c55916ba3fb48c42837da98565" dependencies = [ - "glib", - "gstreamer", + "glib 0.22.0", + "gstreamer 0.25.0", "gstreamer-allocators-sys", "libc", ] @@ -3875,11 +4009,11 @@ dependencies = [ [[package]] name = "gstreamer-allocators-sys" version = "0.25.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#8a41295b995e4154dee1c18cfea77efc25ec3817" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#c533fe960af057c55916ba3fb48c42837da98565" dependencies = [ - "glib-sys", - "gobject-sys", - "gstreamer-sys", + "glib-sys 0.22.0", + "gobject-sys 0.22.0", + "gstreamer-sys 0.25.0", "libc", "system-deps", ] @@ -3887,10 +4021,10 @@ dependencies = [ [[package]] name = "gstreamer-analytics" version = "0.25.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#8a41295b995e4154dee1c18cfea77efc25ec3817" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#c533fe960af057c55916ba3fb48c42837da98565" dependencies = [ - "glib", - "gstreamer", + "glib 0.22.0", + "gstreamer 0.25.0", "gstreamer-analytics-sys", "libc", ] @@ -3898,10 +4032,10 @@ dependencies = [ [[package]] name = "gstreamer-analytics-sys" version = "0.25.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#8a41295b995e4154dee1c18cfea77efc25ec3817" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#c533fe960af057c55916ba3fb48c42837da98565" dependencies = [ - "glib-sys", - "gstreamer-sys", + "glib-sys 0.22.0", + "gstreamer-sys 0.25.0", "libc", "system-deps", ] @@ -3909,25 +4043,25 @@ dependencies = [ [[package]] name = "gstreamer-app" version = "0.25.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#8a41295b995e4154dee1c18cfea77efc25ec3817" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#c533fe960af057c55916ba3fb48c42837da98565" dependencies = [ "futures-core", "futures-sink", - "glib", - "gstreamer", + "glib 0.22.0", + "gstreamer 0.25.0", "gstreamer-app-sys", - "gstreamer-base", + "gstreamer-base 0.25.0", "libc", ] [[package]] name = "gstreamer-app-sys" version = "0.25.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#8a41295b995e4154dee1c18cfea77efc25ec3817" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#c533fe960af057c55916ba3fb48c42837da98565" dependencies = [ - "glib-sys", - "gstreamer-base-sys", - "gstreamer-sys", + "glib-sys 0.22.0", + "gstreamer-base-sys 0.25.0", + "gstreamer-sys 0.25.0", "libc", "system-deps", ] @@ -3935,13 +4069,13 @@ dependencies = [ [[package]] name = "gstreamer-audio" version = "0.25.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#8a41295b995e4154dee1c18cfea77efc25ec3817" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#c533fe960af057c55916ba3fb48c42837da98565" dependencies = [ "cfg-if", - "glib", - "gstreamer", + "glib 0.22.0", + "gstreamer 0.25.0", "gstreamer-audio-sys", - "gstreamer-base", + "gstreamer-base 0.25.0", "libc", "serde", "smallvec", @@ -3950,37 +4084,62 @@ dependencies = [ [[package]] name = "gstreamer-audio-sys" version = "0.25.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#8a41295b995e4154dee1c18cfea77efc25ec3817" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#c533fe960af057c55916ba3fb48c42837da98565" dependencies = [ - "glib-sys", - "gobject-sys", - "gstreamer-base-sys", - "gstreamer-sys", + "glib-sys 0.22.0", + "gobject-sys 0.22.0", + "gstreamer-base-sys 0.25.0", + "gstreamer-sys 0.25.0", "libc", "system-deps", ] +[[package]] +name = "gstreamer-base" +version = "0.23.7" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=0.23#0f9f71821ada8ec7c16c27e08181523cfb5d2a30" +dependencies = [ + "atomic_refcell", + "cfg-if", + "glib 0.20.12", + "gstreamer 0.23.7", + "gstreamer-base-sys 0.23.7", + "libc", +] + [[package]] name = "gstreamer-base" version = "0.25.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#8a41295b995e4154dee1c18cfea77efc25ec3817" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#c533fe960af057c55916ba3fb48c42837da98565" dependencies = [ "atomic_refcell", "cfg-if", - "glib", - "gstreamer", - "gstreamer-base-sys", + "glib 0.22.0", + "gstreamer 0.25.0", + "gstreamer-base-sys 0.25.0", + "libc", +] + +[[package]] +name = "gstreamer-base-sys" +version = "0.23.7" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=0.23#0f9f71821ada8ec7c16c27e08181523cfb5d2a30" +dependencies = [ + "glib-sys 0.20.12", + "gobject-sys 0.20.12", + "gstreamer-sys 0.23.7", "libc", + "system-deps", ] [[package]] name = "gstreamer-base-sys" version = "0.25.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#8a41295b995e4154dee1c18cfea77efc25ec3817" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#c533fe960af057c55916ba3fb48c42837da98565" dependencies = [ - "glib-sys", - "gobject-sys", - "gstreamer-sys", + "glib-sys 0.22.0", + "gobject-sys 0.22.0", + "gstreamer-sys 0.25.0", "libc", "system-deps", ] @@ -3988,21 +4147,21 @@ dependencies = [ [[package]] name = "gstreamer-check" version = "0.25.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#8a41295b995e4154dee1c18cfea77efc25ec3817" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#c533fe960af057c55916ba3fb48c42837da98565" dependencies = [ - "glib", - "gstreamer", + "glib 0.22.0", + "gstreamer 0.25.0", "gstreamer-check-sys", ] [[package]] name = "gstreamer-check-sys" version = "0.25.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#8a41295b995e4154dee1c18cfea77efc25ec3817" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#c533fe960af057c55916ba3fb48c42837da98565" dependencies = [ - "glib-sys", - "gobject-sys", - "gstreamer-sys", + "glib-sys 0.22.0", + "gobject-sys 0.22.0", + "gstreamer-sys 0.25.0", "libc", "system-deps", ] @@ -4010,23 +4169,23 @@ dependencies = [ [[package]] name = "gstreamer-gl" version = "0.25.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#8a41295b995e4154dee1c18cfea77efc25ec3817" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#c533fe960af057c55916ba3fb48c42837da98565" dependencies = [ - "glib", - "gstreamer", - "gstreamer-base", + "glib 0.22.0", + "gstreamer 0.25.0", + "gstreamer-base 0.25.0", "gstreamer-gl-sys", - "gstreamer-video", + "gstreamer-video 0.25.0", "libc", ] [[package]] name = "gstreamer-gl-egl" version = "0.25.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#8a41295b995e4154dee1c18cfea77efc25ec3817" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#c533fe960af057c55916ba3fb48c42837da98565" dependencies = [ - "glib", - "gstreamer", + "glib 0.22.0", + "gstreamer 0.25.0", "gstreamer-gl", "gstreamer-gl-egl-sys", "libc", @@ -4035,9 +4194,9 @@ dependencies = [ [[package]] name = "gstreamer-gl-egl-sys" version = "0.25.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#8a41295b995e4154dee1c18cfea77efc25ec3817" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#c533fe960af057c55916ba3fb48c42837da98565" dependencies = [ - "glib-sys", + "glib-sys 0.22.0", "gstreamer-gl-sys", "libc", "system-deps", @@ -4046,13 +4205,13 @@ dependencies = [ [[package]] name = "gstreamer-gl-sys" version = "0.25.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#8a41295b995e4154dee1c18cfea77efc25ec3817" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#c533fe960af057c55916ba3fb48c42837da98565" dependencies = [ - "glib-sys", - "gobject-sys", - "gstreamer-base-sys", - "gstreamer-sys", - "gstreamer-video-sys", + "glib-sys 0.22.0", + "gobject-sys 0.22.0", + "gstreamer-base-sys 0.25.0", + "gstreamer-sys 0.25.0", + "gstreamer-video-sys 0.25.0", "libc", "system-deps", ] @@ -4060,10 +4219,10 @@ dependencies = [ [[package]] name = "gstreamer-gl-wayland" version = "0.25.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#8a41295b995e4154dee1c18cfea77efc25ec3817" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#c533fe960af057c55916ba3fb48c42837da98565" dependencies = [ - "glib", - "gstreamer", + "glib 0.22.0", + "gstreamer 0.25.0", "gstreamer-gl", "gstreamer-gl-wayland-sys", "libc", @@ -4072,9 +4231,9 @@ dependencies = [ [[package]] name = "gstreamer-gl-wayland-sys" version = "0.25.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#8a41295b995e4154dee1c18cfea77efc25ec3817" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#c533fe960af057c55916ba3fb48c42837da98565" dependencies = [ - "glib-sys", + "glib-sys 0.22.0", "gstreamer-gl-sys", "libc", "system-deps", @@ -4083,10 +4242,10 @@ dependencies = [ [[package]] name = "gstreamer-gl-x11" version = "0.25.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#8a41295b995e4154dee1c18cfea77efc25ec3817" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#c533fe960af057c55916ba3fb48c42837da98565" dependencies = [ - "glib", - "gstreamer", + "glib 0.22.0", + "gstreamer 0.25.0", "gstreamer-gl", "gstreamer-gl-x11-sys", "libc", @@ -4095,9 +4254,9 @@ dependencies = [ [[package]] name = "gstreamer-gl-x11-sys" version = "0.25.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#8a41295b995e4154dee1c18cfea77efc25ec3817" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#c533fe960af057c55916ba3fb48c42837da98565" dependencies = [ - "glib-sys", + "glib-sys 0.22.0", "gstreamer-gl-sys", "libc", "system-deps", @@ -4106,22 +4265,22 @@ dependencies = [ [[package]] name = "gstreamer-net" version = "0.25.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#8a41295b995e4154dee1c18cfea77efc25ec3817" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#c533fe960af057c55916ba3fb48c42837da98565" dependencies = [ "gio", - "glib", - "gstreamer", + "glib 0.22.0", + "gstreamer 0.25.0", "gstreamer-net-sys", ] [[package]] name = "gstreamer-net-sys" version = "0.25.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#8a41295b995e4154dee1c18cfea77efc25ec3817" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#c533fe960af057c55916ba3fb48c42837da98565" dependencies = [ - "gio-sys", - "glib-sys", - "gstreamer-sys", + "gio-sys 0.22.0", + "glib-sys 0.22.0", + "gstreamer-sys 0.25.0", "libc", "system-deps", ] @@ -4129,13 +4288,13 @@ dependencies = [ [[package]] name = "gstreamer-pbutils" version = "0.25.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#8a41295b995e4154dee1c18cfea77efc25ec3817" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#c533fe960af057c55916ba3fb48c42837da98565" dependencies = [ - "glib", - "gstreamer", + "glib 0.22.0", + "gstreamer 0.25.0", "gstreamer-audio", "gstreamer-pbutils-sys", - "gstreamer-video", + "gstreamer-video 0.25.0", "libc", "thiserror 2.0.16", ] @@ -4143,13 +4302,13 @@ dependencies = [ [[package]] name = "gstreamer-pbutils-sys" version = "0.25.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#8a41295b995e4154dee1c18cfea77efc25ec3817" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#c533fe960af057c55916ba3fb48c42837da98565" dependencies = [ - "glib-sys", - "gobject-sys", + "glib-sys 0.22.0", + "gobject-sys 0.22.0", "gstreamer-audio-sys", - "gstreamer-sys", - "gstreamer-video-sys", + "gstreamer-sys 0.25.0", + "gstreamer-video-sys 0.25.0", "libc", "system-deps", ] @@ -4157,10 +4316,10 @@ dependencies = [ [[package]] name = "gstreamer-rtp" version = "0.25.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#8a41295b995e4154dee1c18cfea77efc25ec3817" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#c533fe960af057c55916ba3fb48c42837da98565" dependencies = [ - "glib", - "gstreamer", + "glib 0.22.0", + "gstreamer 0.25.0", "gstreamer-rtp-sys", "libc", ] @@ -4168,11 +4327,11 @@ dependencies = [ [[package]] name = "gstreamer-rtp-sys" version = "0.25.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#8a41295b995e4154dee1c18cfea77efc25ec3817" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#c533fe960af057c55916ba3fb48c42837da98565" dependencies = [ - "glib-sys", - "gstreamer-base-sys", - "gstreamer-sys", + "glib-sys 0.22.0", + "gstreamer-base-sys 0.25.0", + "gstreamer-sys 0.25.0", "libc", "system-deps", ] @@ -4180,20 +4339,31 @@ dependencies = [ [[package]] name = "gstreamer-sdp" version = "0.25.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#8a41295b995e4154dee1c18cfea77efc25ec3817" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#c533fe960af057c55916ba3fb48c42837da98565" dependencies = [ - "glib", - "gstreamer", + "glib 0.22.0", + "gstreamer 0.25.0", "gstreamer-sdp-sys", ] [[package]] name = "gstreamer-sdp-sys" version = "0.25.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#8a41295b995e4154dee1c18cfea77efc25ec3817" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#c533fe960af057c55916ba3fb48c42837da98565" dependencies = [ - "glib-sys", - "gstreamer-sys", + "glib-sys 0.22.0", + "gstreamer-sys 0.25.0", + "libc", + "system-deps", +] + +[[package]] +name = "gstreamer-sys" +version = "0.23.7" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=0.23#0f9f71821ada8ec7c16c27e08181523cfb5d2a30" +dependencies = [ + "glib-sys 0.20.12", + "gobject-sys 0.20.12", "libc", "system-deps", ] @@ -4201,11 +4371,11 @@ dependencies = [ [[package]] name = "gstreamer-sys" version = "0.25.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#8a41295b995e4154dee1c18cfea77efc25ec3817" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#c533fe960af057c55916ba3fb48c42837da98565" dependencies = [ "cfg-if", - "glib-sys", - "gobject-sys", + "glib-sys 0.22.0", + "gobject-sys 0.22.0", "libc", "system-deps", ] @@ -4213,10 +4383,10 @@ dependencies = [ [[package]] name = "gstreamer-tag" version = "0.25.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#8a41295b995e4154dee1c18cfea77efc25ec3817" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#c533fe960af057c55916ba3fb48c42837da98565" dependencies = [ - "glib", - "gstreamer", + "glib 0.22.0", + "gstreamer 0.25.0", "gstreamer-tag-sys", "libc", ] @@ -4224,11 +4394,11 @@ dependencies = [ [[package]] name = "gstreamer-tag-sys" version = "0.25.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#8a41295b995e4154dee1c18cfea77efc25ec3817" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#c533fe960af057c55916ba3fb48c42837da98565" dependencies = [ - "glib-sys", - "gobject-sys", - "gstreamer-sys", + "glib-sys 0.22.0", + "gobject-sys 0.22.0", + "gstreamer-sys 0.25.0", "libc", "system-deps", ] @@ -4236,39 +4406,68 @@ dependencies = [ [[package]] name = "gstreamer-utils" version = "0.25.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#8a41295b995e4154dee1c18cfea77efc25ec3817" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#c533fe960af057c55916ba3fb48c42837da98565" dependencies = [ - "gstreamer", + "gstreamer 0.25.0", "gstreamer-app", - "gstreamer-video", + "gstreamer-video 0.25.0", + "thiserror 2.0.16", +] + +[[package]] +name = "gstreamer-video" +version = "0.23.7" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=0.23#0f9f71821ada8ec7c16c27e08181523cfb5d2a30" +dependencies = [ + "cfg-if", + "futures-channel", + "glib 0.20.12", + "gstreamer 0.23.7", + "gstreamer-base 0.23.7", + "gstreamer-video-sys 0.23.7", + "libc", + "once_cell", "thiserror 2.0.16", ] [[package]] name = "gstreamer-video" version = "0.25.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#8a41295b995e4154dee1c18cfea77efc25ec3817" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#c533fe960af057c55916ba3fb48c42837da98565" dependencies = [ "cfg-if", "futures-channel", - "glib", - "gstreamer", - "gstreamer-base", - "gstreamer-video-sys", + "glib 0.22.0", + "gstreamer 0.25.0", + "gstreamer-base 0.25.0", + "gstreamer-video-sys 0.25.0", "libc", "serde", "thiserror 2.0.16", ] +[[package]] +name = "gstreamer-video-sys" +version = "0.23.7" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=0.23#0f9f71821ada8ec7c16c27e08181523cfb5d2a30" +dependencies = [ + "glib-sys 0.20.12", + "gobject-sys 0.20.12", + "gstreamer-base-sys 0.23.7", + "gstreamer-sys 0.23.7", + "libc", + "system-deps", +] + [[package]] name = "gstreamer-video-sys" version = "0.25.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#8a41295b995e4154dee1c18cfea77efc25ec3817" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#c533fe960af057c55916ba3fb48c42837da98565" dependencies = [ - "glib-sys", - "gobject-sys", - "gstreamer-base-sys", - "gstreamer-sys", + "glib-sys 0.22.0", + "gobject-sys 0.22.0", + "gstreamer-base-sys 0.25.0", + "gstreamer-sys 0.25.0", "libc", "system-deps", ] @@ -4276,10 +4475,10 @@ dependencies = [ [[package]] name = "gstreamer-webrtc" version = "0.25.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#8a41295b995e4154dee1c18cfea77efc25ec3817" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#c533fe960af057c55916ba3fb48c42837da98565" dependencies = [ - "glib", - "gstreamer", + "glib 0.22.0", + "gstreamer 0.25.0", "gstreamer-sdp", "gstreamer-webrtc-sys", "libc", @@ -4288,11 +4487,11 @@ dependencies = [ [[package]] name = "gstreamer-webrtc-sys" version = "0.25.0" -source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#8a41295b995e4154dee1c18cfea77efc25ec3817" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#c533fe960af057c55916ba3fb48c42837da98565" dependencies = [ - "glib-sys", + "glib-sys 0.22.0", "gstreamer-sdp-sys", - "gstreamer-sys", + "gstreamer-sys 0.25.0", "libc", "system-deps", ] @@ -4300,7 +4499,7 @@ dependencies = [ [[package]] name = "gtk4" version = "0.11.0" -source = "git+https://github.com/gtk-rs/gtk4-rs?branch=main#e8b597d2285cebf2c13373dc883e9a7e2bec2a00" +source = "git+https://github.com/gtk-rs/gtk4-rs?branch=main#d294c123a8e39d9717dc21c373e1af2c857dda30" dependencies = [ "cairo-rs", "field-offset", @@ -4308,7 +4507,7 @@ dependencies = [ "gdk-pixbuf", "gdk4", "gio", - "glib", + "glib 0.22.0", "graphene-rs", "gsk4", "gtk4-macros", @@ -4320,7 +4519,7 @@ dependencies = [ [[package]] name = "gtk4-macros" version = "0.11.0" -source = "git+https://github.com/gtk-rs/gtk4-rs?branch=main#e8b597d2285cebf2c13373dc883e9a7e2bec2a00" +source = "git+https://github.com/gtk-rs/gtk4-rs?branch=main#d294c123a8e39d9717dc21c373e1af2c857dda30" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -4331,14 +4530,14 @@ dependencies = [ [[package]] name = "gtk4-sys" version = "0.11.0" -source = "git+https://github.com/gtk-rs/gtk4-rs?branch=main#e8b597d2285cebf2c13373dc883e9a7e2bec2a00" +source = "git+https://github.com/gtk-rs/gtk4-rs?branch=main#d294c123a8e39d9717dc21c373e1af2c857dda30" dependencies = [ "cairo-sys-rs", "gdk-pixbuf-sys", "gdk4-sys", - "gio-sys", - "glib-sys", - "gobject-sys", + "gio-sys 0.22.0", + "glib-sys 0.22.0", + "gobject-sys 0.22.0", "graphene-sys", "gsk4-sys", "libc", @@ -4358,7 +4557,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap 2.11.3", + "indexmap 2.11.4", "slab", "tokio", "tokio-util", @@ -4377,7 +4576,7 @@ dependencies = [ "futures-core", "futures-sink", "http 1.3.1", - "indexmap 2.11.3", + "indexmap 2.11.4", "slab", "tokio", "tokio-util", @@ -4401,6 +4600,12 @@ dependencies = [ "foldhash", ] +[[package]] +name = "hashbrown" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" + [[package]] name = "headers" version = "0.4.1" @@ -4676,7 +4881,7 @@ dependencies = [ "http 1.3.1", "hyper 1.7.0", "hyper-util", - "rustls 0.23.31", + "rustls 0.23.32", "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", @@ -4932,12 +5137,13 @@ dependencies = [ [[package]] name = "image" -version = "0.25.6" +version = "0.25.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db35664ce6b9810857a38a906215e75a9c879f0696556a39f59c62829710251a" +checksum = "529feb3e6769d234375c4cf1ee2ce713682b8e76538cb13f9fc23e1400a591e7" dependencies = [ "bytemuck", "byteorder-lite", + "moxcms", "num-traits", ] @@ -4973,12 +5179,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.11.3" +version = "2.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92119844f513ffa41556430369ab02c295a3578af21cf945caa3e9e0c2481ac3" +checksum = "4b0f83760fb341a774ed326568e19f5a863af4a952def8c39f9ab92fd95b88e5" dependencies = [ "equivalent", - "hashbrown 0.15.5", + "hashbrown 0.16.0", "serde", "serde_core", ] @@ -5240,12 +5446,12 @@ dependencies = [ [[package]] name = "libloading" -version = "0.8.8" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" dependencies = [ "cfg-if", - "windows-targets 0.53.3", + "windows-link 0.2.0", ] [[package]] @@ -5640,6 +5846,16 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fafa6961cabd9c63bcd77a45d7e3b7f3b552b70417831fb0f56db717e72407e" +[[package]] +name = "moxcms" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd32fa8935aeadb8a8a6b6b351e40225570a37c43de67690383d87ef170cd08" +dependencies = [ + "num-traits", + "pxfm", +] + [[package]] name = "mp4-atom" version = "0.8.1" @@ -5676,11 +5892,12 @@ dependencies = [ [[package]] name = "nasm-rs" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12fcfa1bd49e0342ec1d07ed2be83b59963e7acbeb9310e1bb2c07b69dadd959" +checksum = "34f676553b60ccbb76f41f9ae8f2428dac3f259ff8f1c2468a174778d06a1af9" dependencies = [ "jobserver", + "log", ] [[package]] @@ -6029,6 +6246,15 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "option-operations" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c26d27bb1aeab65138e4bf7666045169d1717febcc9ff870166be8348b223d0" +dependencies = [ + "paste", +] + [[package]] name = "option-operations" version = "0.6.0" @@ -6058,10 +6284,10 @@ dependencies = [ [[package]] name = "pango" version = "0.22.0" -source = "git+https://github.com/gtk-rs/gtk-rs-core?branch=main#bbe4d709b7337f1bfea7b2a347524689979c6756" +source = "git+https://github.com/gtk-rs/gtk-rs-core?branch=main#7b4ffff1879bc05f334f0d015302e65cf381e35d" dependencies = [ "gio", - "glib", + "glib 0.22.0", "libc", "pango-sys", ] @@ -6069,10 +6295,10 @@ dependencies = [ [[package]] name = "pango-sys" version = "0.22.0" -source = "git+https://github.com/gtk-rs/gtk-rs-core?branch=main#bbe4d709b7337f1bfea7b2a347524689979c6756" +source = "git+https://github.com/gtk-rs/gtk-rs-core?branch=main#7b4ffff1879bc05f334f0d015302e65cf381e35d" dependencies = [ - "glib-sys", - "gobject-sys", + "glib-sys 0.22.0", + "gobject-sys 0.22.0", "libc", "system-deps", ] @@ -6080,10 +6306,10 @@ dependencies = [ [[package]] name = "pangocairo" version = "0.22.0" -source = "git+https://github.com/gtk-rs/gtk-rs-core?branch=main#bbe4d709b7337f1bfea7b2a347524689979c6756" +source = "git+https://github.com/gtk-rs/gtk-rs-core?branch=main#7b4ffff1879bc05f334f0d015302e65cf381e35d" dependencies = [ "cairo-rs", - "glib", + "glib 0.22.0", "libc", "pango", "pangocairo-sys", @@ -6092,10 +6318,10 @@ dependencies = [ [[package]] name = "pangocairo-sys" version = "0.22.0" -source = "git+https://github.com/gtk-rs/gtk-rs-core?branch=main#bbe4d709b7337f1bfea7b2a347524689979c6756" +source = "git+https://github.com/gtk-rs/gtk-rs-core?branch=main#7b4ffff1879bc05f334f0d015302e65cf381e35d" dependencies = [ "cairo-sys-rs", - "glib-sys", + "glib-sys 0.22.0", "libc", "pango-sys", "system-deps", @@ -6253,7 +6479,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset", - "indexmap 2.11.3", + "indexmap 2.11.4", ] [[package]] @@ -6434,7 +6660,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e7f4ffd8645efad783fc2844ac842367aa2e912d484950192564d57dc039a3a" dependencies = [ "equivalent", - "indexmap 2.11.3", + "indexmap 2.11.4", ] [[package]] @@ -6443,7 +6669,7 @@ version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "219cb19e96be00ab2e37d6e299658a0cfa83e52429179969b0f0121b4ac46983" dependencies = [ - "toml_edit 0.23.5", + "toml_edit 0.23.6", ] [[package]] @@ -6483,10 +6709,10 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc5b72d8145275d844d4b5f6d4e1eef00c8cd889edb6035c21675d1bb1f45c9f" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.4", "hex", "procfs-core", - "rustix", + "rustix 0.38.44", ] [[package]] @@ -6495,7 +6721,7 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "239df02d8349b06fc07398a3a1697b06418223b1c7725085e801e7c0fc6a12ec" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.4", "hex", ] @@ -6615,7 +6841,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4aeaa1f2460f1d348eeaeed86aea999ce98c1bded6f089ff8514c9d9dbdc973" dependencies = [ "anyhow", - "indexmap 2.11.3", + "indexmap 2.11.4", "log", "protobuf", "protobuf-support", @@ -6649,6 +6875,15 @@ dependencies = [ "psl-types", ] +[[package]] +name = "pxfm" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83f9b339b02259ada5c0f4a389b7fb472f933aa17ce176fd2ad98f28bb401fde" +dependencies = [ + "num-traits", +] + [[package]] name = "quick-xml" version = "0.37.5" @@ -6681,7 +6916,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash 2.1.1", - "rustls 0.23.31", + "rustls 0.23.32", "socket2 0.6.0", "thiserror 2.0.16", "tokio", @@ -6702,7 +6937,7 @@ dependencies = [ "rand 0.9.2", "ring", "rustc-hash 2.1.1", - "rustls 0.23.31", + "rustls 0.23.32", "rustls-pki-types", "rustls-platform-verifier", "slab", @@ -7026,7 +7261,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.31", + "rustls 0.23.32", "rustls-native-certs 0.8.1", "rustls-pki-types", "serde", @@ -7247,9 +7482,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.31" +version = "0.23.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0ebcbd2f03de0fc1122ad9bb24b127a5a6cd51d72604a3f3c50ac459762b6cc" +checksum = "cd3c25631629d034ce7cd9940adc9d45762d46de2b0f57193c4443b92c6d4d40" dependencies = [ "log", "once_cell", @@ -7294,7 +7529,7 @@ dependencies = [ "openssl-probe", "rustls-pki-types", "schannel", - "security-framework 3.4.0", + "security-framework 3.5.0", ] [[package]] @@ -7336,11 +7571,11 @@ dependencies = [ "jni", "log", "once_cell", - "rustls 0.23.31", + "rustls 0.23.32", "rustls-native-certs 0.8.1", "rustls-platform-verifier-android", "rustls-webpki 0.103.6", - "security-framework 3.4.0", + "security-framework 3.5.0", "security-framework-sys", "webpki-root-certs", "windows-sys 0.59.0", @@ -7514,9 +7749,9 @@ dependencies = [ [[package]] name = "security-framework" -version = "3.4.0" +version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60b369d18893388b345804dc0007963c99b7d665ae71d275812d828c6f089640" +checksum = "cc198e42d9b7510827939c9a15f5062a0c913f3371d765977e586d2fe6c16f4a" dependencies = [ "bitflags 2.9.4", "core-foundation 0.10.1", @@ -7543,9 +7778,9 @@ checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" [[package]] name = "serde" -version = "1.0.225" +version = "1.0.226" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd6c24dee235d0da097043389623fb913daddf92c76e9f5a1db88607a0bcbd1d" +checksum = "0dca6411025b24b60bfa7ec1fe1f8e710ac09782dca409ee8237ba74b51295fd" dependencies = [ "serde_core", "serde_derive", @@ -7563,18 +7798,18 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.225" +version = "1.0.226" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "659356f9a0cb1e529b24c01e43ad2bdf520ec4ceaf83047b83ddcc2251f96383" +checksum = "ba2ba63999edb9dac981fb34b3e5c0d111a69b0924e253ed29d83f7c99e966a4" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.225" +version = "1.0.226" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ea936adf78b1f766949a4977b91d2f5595825bd6ec079aa9543ad2685fc4516" +checksum = "8db53ae22f34573731bafa1db20f04027b2d25e02d8205921b569171699cdb33" dependencies = [ "proc-macro2", "quote", @@ -7616,9 +7851,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2789234a13a53fc4be1b51ea1bab45a3c338bdb884862a257d10e5a74ae009e6" +checksum = "5417783452c2be558477e104686f7de5dae53dba813c28435e0e70f82d9b04ee" dependencies = [ "serde_core", ] @@ -7637,15 +7872,15 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.14.0" +version = "3.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2c45cd61fefa9db6f254525d46e392b852e0e61d9a1fd36e5bd183450a556d5" +checksum = "c522100790450cf78eeac1507263d0a350d4d5b30df0c8e1fe051a10c22b376e" dependencies = [ "base64 0.22.1", "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.11.3", + "indexmap 2.11.4", "schemars 0.9.0", "schemars 1.0.4", "serde", @@ -7657,11 +7892,11 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.14.0" +version = "3.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de90945e6565ce0d9a25098082ed4ee4002e047cb59892c318d66821e14bb30f" +checksum = "327ada00f7d64abaac1e55a6911e90cf665aa051b9a561c7006c157f4633135e" dependencies = [ - "darling", + "darling 0.21.3", "proc-macro2", "quote", "syn 2.0.106", @@ -7765,9 +8000,9 @@ dependencies = [ [[package]] name = "signalsmith-stretch" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6b809d47705ed26a237cb032c4e9b781cd583ac5337d66a4501fca11c74c17f" +checksum = "51dae6f10b5532510f65c309c4d868babe3aecf6ce0782678081338311f176fd" dependencies = [ "bindgen 0.70.1", "cc", @@ -7829,7 +8064,7 @@ dependencies = [ "regex", "serde_json", "tar", - "toml 0.9.6", + "toml 0.9.7", ] [[package]] @@ -8189,9 +8424,9 @@ checksum = "e502f78cdbb8ba4718f566c418c52bc729126ffd16baee5baa718cf25dd5a69a" [[package]] name = "tempfile" -version = "3.22.0" +version = "3.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84fa4d11fadde498443cca10fd3ac23c951f0dc59e080e9f4b93d4df4e4eea53" +checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" dependencies = [ "fastrand", "getrandom 0.3.3", @@ -8223,9 +8458,9 @@ dependencies = [ [[package]] name = "test-with" -version = "0.15.3" +version = "0.15.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3c82f52ec88a4992bd0c46b8ef271c070c52689bf34f7effae7a26708cbf13d" +checksum = "e0f370b9efbfbbc5f057cbce9888373eaeb146a3095bb8cc869b199c94d15559" dependencies = [ "proc-macro-error2", "proc-macro2", @@ -8307,11 +8542,12 @@ dependencies = [ [[package]] name = "time" -version = "0.3.43" +version = "0.3.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83bde6f1ec10e72d583d91623c939f623002284ef622b87de38cfd546cbf2031" +checksum = "91e7d9e3bb61134e77bde20dd4825b97c010155709965fedf0f49bb138e52a9d" dependencies = [ "deranged", + "itoa", "libc", "num-conv", "num_threads", @@ -8430,7 +8666,7 @@ version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05f63835928ca123f1bef57abbcd23bb2ba0ac9ae1235f1e65bda0d06e7786bd" dependencies = [ - "rustls 0.23.31", + "rustls 0.23.32", "tokio", ] @@ -8467,7 +8703,7 @@ checksum = "489a59b6730eda1b0171fcfda8b121f4bee2b35cba8645ca35c5f7ba3eb736c1" dependencies = [ "futures-util", "log", - "rustls 0.23.31", + "rustls 0.23.32", "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", @@ -8502,14 +8738,14 @@ dependencies = [ [[package]] name = "toml" -version = "0.9.6" +version = "0.9.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae2a4cf385da23d1d53bc15cdfa5c2109e93d8d362393c801e87da2f72f0e201" +checksum = "00e5e5d9bf2475ac9d4f0d9edab68cc573dc2fd644b0dba36b0c30a92dd9eaa0" dependencies = [ - "indexmap 2.11.3", + "indexmap 2.11.4", "serde_core", - "serde_spanned 1.0.1", - "toml_datetime 0.7.1", + "serde_spanned 1.0.2", + "toml_datetime 0.7.2", "toml_parser", "toml_writer", "winnow", @@ -8526,9 +8762,9 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a197c0ec7d131bfc6f7e82c8442ba1595aeab35da7adbf05b6b73cd06a16b6be" +checksum = "32f1085dec27c2b6632b04c80b3bb1b4300d6495d1e129693bdda7d91e72eec1" dependencies = [ "serde_core", ] @@ -8539,7 +8775,7 @@ version = "0.22.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ - "indexmap 2.11.3", + "indexmap 2.11.4", "serde", "serde_spanned 0.6.9", "toml_datetime 0.6.11", @@ -8548,30 +8784,30 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.23.5" +version = "0.23.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2ad0b7ae9cfeef5605163839cb9221f453399f15cfb5c10be9885fcf56611f9" +checksum = "f3effe7c0e86fdff4f69cdd2ccc1b96f933e24811c5441d44904e8683e27184b" dependencies = [ - "indexmap 2.11.3", - "toml_datetime 0.7.1", + "indexmap 2.11.4", + "toml_datetime 0.7.2", "toml_parser", "winnow", ] [[package]] name = "toml_parser" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b551886f449aa90d4fe2bdaa9f4a2577ad2dde302c61ecf262d80b116db95c10" +checksum = "4cf893c33be71572e0e9aa6dd15e6677937abd686b066eac3f8cd3531688a627" dependencies = [ "winnow", ] [[package]] name = "toml_writer" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcc842091f2def52017664b53082ecbbeb5c7731092bad69d2c63050401dfd64" +checksum = "d163a63c116ce562a22cda521fcc4d79152e7aba014456fb5eb442f6d6a10109" [[package]] name = "tower" @@ -8728,7 +8964,7 @@ dependencies = [ "httparse", "log", "rand 0.9.2", - "rustls 0.23.31", + "rustls 0.23.32", "rustls-pki-types", "sha1", "thiserror 2.0.16", @@ -9127,7 +9363,7 @@ dependencies = [ "http 1.3.1", "log", "quinn", - "rustls 0.23.31", + "rustls 0.23.32", "rustls-native-certs 0.8.1", "thiserror 2.0.16", "tokio", @@ -9720,9 +9956,9 @@ checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" [[package]] name = "xattr" -version = "1.5.1" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af3a19837351dc82ba89f8a125e22a3c475f05aba604acc023d62b2739ae2909" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" dependencies = [ "libc", "rustix 1.1.2", diff --git a/video/stats/src/encoderstats/imp.rs b/video/stats/src/encoderstats/imp.rs index a0c56e396..d25442889 100644 --- a/video/stats/src/encoderstats/imp.rs +++ b/video/stats/src/encoderstats/imp.rs @@ -118,7 +118,7 @@ impl EncoderStats { gst::log!(CAT, "Buffer in encoder sink pad"); let current_time = *gst::ClockTime::from_nseconds( - gst::SystemClock::obtain().upcast::().time().unwrap().nseconds() + gst::SystemClock::obtain().upcast::().time().nseconds() ); let buffer = buffer.make_mut(); @@ -145,7 +145,7 @@ impl EncoderStats { gst::log!(CAT, "Buffer out encoder src pad"); let current_time = *gst::ClockTime::from_nseconds( - gst::SystemClock::obtain().upcast::().time().unwrap().nseconds() + gst::SystemClock::obtain().upcast::().time().nseconds() ); let buffer = buffer.make_mut(); @@ -242,7 +242,7 @@ impl EncoderStats { let mut stats = stats_clone.lock().unwrap(); stats.input_time = *gst::ClockTime::from_nseconds( - gst::SystemClock::obtain().upcast::().time().unwrap().nseconds() + gst::SystemClock::obtain().upcast::().time().nseconds() ); // Only update CPU stats at framerate intervals as it takes time From 8010eea9a9164d2e4adf1bed8fdeb5112aa0bef1 Mon Sep 17 00:00:00 2001 From: Diego Nieto Date: Wed, 24 Sep 2025 10:28:35 +0200 Subject: [PATCH 30/46] video-compare-mixer: unify overlay into a single element videoencoderstats meta is now read from the input queues. The overlay is not linked after the compositor and the values are updated once per second according to the given framerate of the pipeline A new fmt for video-compare-mixer has been created to compose both stats left and right into a single text for the overlay --- video/stats/src/comparemixer/imp.rs | 229 ++++++++++++++++++++-------- 1 file changed, 165 insertions(+), 64 deletions(-) diff --git a/video/stats/src/comparemixer/imp.rs b/video/stats/src/comparemixer/imp.rs index a61f2348f..ba74fb44c 100644 --- a/video/stats/src/comparemixer/imp.rs +++ b/video/stats/src/comparemixer/imp.rs @@ -19,6 +19,7 @@ use crate::comparemixer::compositor::Mode; use std::sync::{LazyLock, Mutex, Arc}; use std::vec::Vec; +use std::fmt; static CAT: LazyLock = LazyLock::new(|| { gst::DebugCategory::new( @@ -77,11 +78,99 @@ pub struct VideoCompareMixer { sinkpad1: gst::GhostPad, queue0: gst::Element, queue1: gst::Element, - overlay0: gst::Element, - overlay1: gst::Element, + overlay: gst::Element, settings: Mutex, mouse_state: Arc>, compositor_helper: Arc>>, + queue_stats: Arc>, +} + +#[derive(Default)] +pub struct QueueStats { + stats0: Option, + stats1: Option, + framerate: Option, +} + +impl fmt::Display for QueueStats { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let stats0_str = self.stats0.as_deref().unwrap_or("No stats available"); + let stats1_str = self.stats1.as_deref().unwrap_or("No stats available"); + + // Parse stats from both sides + let (encoder0, size0, buffers0, bitrate0, proc_time0, cpu0, vmaf0, latency0) = + parse_stats_string(stats0_str); + let (encoder1, size1, buffers1, bitrate1, proc_time1, cpu1, vmaf1, latency1) = + parse_stats_string(stats1_str); + + // Calculate dynamic spacing based on left side content length + let total_width: usize = 80; // Total desired width + + let encoder0_full = format!("Encoder: {}", encoder0); + let encoder_spacing = total_width.saturating_sub(encoder0_full.len()); + writeln!(f, "{}{:>width$}Encoder: {}", encoder0_full, "", encoder1, width = encoder_spacing)?; + + let size0_full = format!("Output size: {}", size0); + let size_spacing = total_width.saturating_sub(size0_full.len()); + writeln!(f, "{}{:>width$}Output size: {}", size0_full, "", size1, width = size_spacing)?; + + let buffers0_full = format!("Num buffers: {}", buffers0); + let buffers_spacing = total_width.saturating_sub(buffers0_full.len()); + writeln!(f, "{}{:>width$}Num buffers: {}", buffers0_full, "", buffers1, width = buffers_spacing)?; + + let bitrate0_full = format!("Bitrate: {}", bitrate0); + let bitrate_spacing = total_width.saturating_sub(bitrate0_full.len()); + writeln!(f, "{}{:>width$}Bitrate: {}", bitrate0_full, "", bitrate1, width = bitrate_spacing)?; + + let proc_time0_full = format!("Processing time: {}", proc_time0); + let proc_time_spacing = total_width.saturating_sub(proc_time0_full.len()); + writeln!(f, "{}{:>width$}Processing time: {}", proc_time0_full, "", proc_time1, width = proc_time_spacing)?; + + let cpu0_full = format!("CPU: {}", cpu0); + let cpu_spacing = total_width.saturating_sub(cpu0_full.len()); + writeln!(f, "{}{:>width$}CPU: {}", cpu0_full, "", cpu1, width = cpu_spacing)?; + + let vmaf0_full = format!("VMAF: {}", vmaf0); + let vmaf_spacing = total_width.saturating_sub(vmaf0_full.len()); + writeln!(f, "{}{:>width$}VMAF: {}", vmaf0_full, "", vmaf1, width = vmaf_spacing)?; + + let latency0_full = format!("Encode latency: {}", latency0); + let latency_spacing = total_width.saturating_sub(latency0_full.len()); + writeln!(f, "{}{:>width$}Encode latency: {}", latency0_full, "", latency1, width = latency_spacing) + } +} + +fn parse_stats_string(stats_str: &str) -> (String, String, String, String, String, String, String, String) { + let mut encoder = "Unknown".to_string(); + let mut size = "0 KB".to_string(); + let mut buffers = "0".to_string(); + let mut bitrate = "0.000 kbps".to_string(); + let mut proc_time = "0.00 ms".to_string(); + let mut cpu = "0 s".to_string(); + let mut vmaf = "N/A".to_string(); + let mut latency = "0.000 ms".to_string(); + + for line in stats_str.lines() { + if line.starts_with("Encoder: ") { + encoder = line.replace("Encoder: ", ""); + } else if line.starts_with("Output size: ") { + size = line.replace("Output size: ", ""); + } else if line.starts_with("Num buffers: ") { + buffers = line.replace("Num buffers: ", ""); + } else if line.starts_with("Bitrate: ") { + bitrate = line.replace("Bitrate: ", ""); + } else if line.starts_with("Processing time: ") { + proc_time = line.replace("Processing time: ", ""); + } else if line.starts_with("CPU: ") { + cpu = line.replace("CPU: ", ""); + } else if line.starts_with("VMAF: ") { + vmaf = line.replace("VMAF: ", ""); + } else if line.starts_with("Encode latency: ") { + latency = line.replace("Encode latency: ", ""); + } + } + + (encoder, size, buffers, bitrate, proc_time, cpu, vmaf, latency) } impl VideoCompareMixer { @@ -227,6 +316,7 @@ impl VideoCompareMixer { state.clicked = false; } } + // FIXME add support for scroll events // NavigationEvent::MouseScroll { x, y, delta_x, delta_y, ..} => { // if delta_y > 0.0 { // compositor.zoom_in_center_at(x as i32, y as i32); @@ -361,8 +451,8 @@ impl VideoCompareMixer { // FIXME remove split_screen logic if not needed. It adds and links crops always self.link_elements(&compositor, true, backend)?; - self.add_overlay_probe(&self.overlay0); - self.add_overlay_probe(&self.overlay1); + self.add_queue_probe(&self.queue0, 0); + self.add_queue_probe(&self.queue1, 1); unsafe { self.sinkpad0.set_event_full_function(|pad, parent, event| { @@ -378,37 +468,67 @@ impl VideoCompareMixer { Ok(()) } - fn add_overlay_probe(&self, overlay: &gst::Element) { - let overlay_src_pad = overlay.static_pad("video_sink").unwrap(); - let overlay_clone = overlay.clone(); - overlay_src_pad.add_probe(gst::PadProbeType::BUFFER, move |_: &gst::Pad, probe_info| { + fn add_queue_probe(&self, queue: &gst::Element, queue_index: u8) { + let queue_src_pad = queue.static_pad("src").unwrap(); + let queue_stats = self.queue_stats.clone(); + queue_src_pad.add_probe(gst::PadProbeType::BUFFER, move |_: &gst::Pad, probe_info| { let Some(buffer) = probe_info.buffer_mut() else { return gst::PadProbeReturn::Ok; }; + let mut stats = queue_stats.lock().unwrap(); + + // Only update at framerate intervals, similar to encoderstats + let fps_n: i32; + if let Some(fps) = stats.framerate { + fps_n = fps.numer(); + } else { + return gst::PadProbeReturn::Ok; + } + if let Some(statsmeta) = buffer.meta::() { - let stats = statsmeta.stats(); - let stats_string = format!("{stats}"); - overlay_clone.set_property("text", stats_string); + let num_buffers = statsmeta.stats().num_buffers; + + // Only update once per second based on framerate + if num_buffers % (fps_n as u64) == 0 { + let stats_string = format!("{}", statsmeta.stats()); + match queue_index { + 0 => stats.stats0 = Some(stats_string), + 1 => stats.stats1 = Some(stats_string), + _ => {} + } + } } gst::PadProbeReturn::Ok }); } + fn add_compositor_src_probe(&self, compositor: &gst::Element) { + let compositor_src_pad = compositor.static_pad("src").unwrap(); + let overlay = self.overlay.clone(); + let queue_stats = self.queue_stats.clone(); + + compositor_src_pad.add_probe(gst::PadProbeType::BUFFER, move |_: &gst::Pad, probe_info| { + let Some(_) = probe_info.buffer() else { + return gst::PadProbeReturn::Ok; + }; + + // Use formatted queue stats as overlay text + let stats = queue_stats.lock().unwrap(); + let stats_text = format!("{}", *stats); + overlay.set_property("text", stats_text); + + gst::PadProbeReturn::Ok + }); + } + fn link_elements( &self, compositor: &gst::Element, split_screen: bool, backend: Backend, ) -> Result<(), gst::ErrorMessage> { - self.overlay0.set_property_from_str("line-alignment", "left"); - self.overlay0.set_property_from_str("halignment", "left"); - self.overlay0.set_property_from_str("valignment", "top"); - self.overlay1.set_property_from_str("line-alignment", "right"); - self.overlay1.set_property_from_str("halignment", "right"); - self.overlay1.set_property_from_str("valignment", "top"); - let compositor_pad0 = compositor .request_pad_simple("sink_0") .expect("Failed to request pad sink_0"); @@ -426,11 +546,8 @@ impl VideoCompareMixer { .add(&self.queue1) .expect("Failed to add queue1 element"); self.obj() - .add(&self.overlay0) - .expect("Failed to add overlay0 element"); - self.obj() - .add(&self.overlay1) - .expect("Failed to add overlay1 element"); + .add(&self.overlay) + .expect("Failed to add overlay element"); let caps_filter = gst::ElementFactory::make("capsfilter") .name("capsfilter0") @@ -447,11 +564,12 @@ impl VideoCompareMixer { .set_target(Some(&self.queue1.static_pad("sink").unwrap())) .expect("Failed to link sinkpad1 to queue1"); - compositor.link(&caps_filter).expect("Failed to link compositor to capsfilter"); + compositor.link(&self.overlay).expect("Failed to link compositor to overlay"); + self.overlay.link(&caps_filter).expect("Failed to link overlay to capsfilter"); self.srcpad .set_target(Some(&caps_filter.static_pad("src").unwrap())) - .expect("Failed to link srcpad to compositor"); + .expect("Failed to link srcpad to capsfilter"); if split_screen && backend != Backend::GL { // Get crop elements by name since we can't store them in struct easily @@ -459,63 +577,44 @@ impl VideoCompareMixer { let crop1 = self.obj().by_name("crop1").expect("crop1 should exist"); self.queue0 - .static_pad("src") - .unwrap() - .link(&self.overlay0.static_pad("video_sink").unwrap()) - .expect("Failed to link queue0 to overlay0"); - self.overlay0 .static_pad("src") .unwrap() .link(&crop0.static_pad("sink").unwrap()) - .expect("Failed to link overlay0 to crop0"); + .expect("Failed to link queue0 to crop0"); crop0 .static_pad("src") .unwrap() .link(&compositor_pad0) - .expect("Failed to link crop0 to queue2"); + .expect("Failed to link crop0 to compositor"); self.queue1 - .static_pad("src") - .unwrap() - .link(&self.overlay1.static_pad("video_sink").unwrap()) - .expect("Failed to link queue1 to overlay1"); - self.overlay1 .static_pad("src") .unwrap() .link(&crop1.static_pad("sink").unwrap()) - .expect("Failed to link overlay1 to crop1"); + .expect("Failed to link queue1 to crop1"); crop1 .static_pad("src") .unwrap() .link(&compositor_pad1) - .expect("Failed to link crop1 to queue3"); + .expect("Failed to link crop1 to compositor"); } else { - // Direct connection without crops - overlay mode + // Direct connection without crops self.queue0 - .static_pad("src") - .unwrap() - .link(&self.overlay0.static_pad("video_sink").unwrap()) - .expect("Failed to link queue0 to overlay0"); - self.overlay0 .static_pad("src") .unwrap() .link(&compositor_pad0) - .expect("Failed to link overlay0 to queue2"); + .expect("Failed to link queue0 to compositor"); self.queue1 - .static_pad("src") - .unwrap() - .link(&self.overlay1.static_pad("video_sink").unwrap()) - .expect("Failed to link queue1 to overlay1"); - self.overlay1 .static_pad("src") .unwrap() .link(&compositor_pad1) - .expect("Failed to link overlay1 to queue3"); + .expect("Failed to link queue1 to compositor"); } + self.add_compositor_src_probe(compositor); + self.queue0.sync_state_with_parent().unwrap(); self.queue1.sync_state_with_parent().unwrap(); - self.overlay0.sync_state_with_parent().unwrap(); - self.overlay1.sync_state_with_parent().unwrap(); + self.overlay.sync_state_with_parent().unwrap(); self.obj().by_name("compositor").unwrap().sync_state_with_parent().unwrap(); Ok(()) } @@ -531,6 +630,13 @@ impl VideoCompareMixer { let s = caps.structure(0).unwrap(); let width = s.get::("width").unwrap(); let height = s.get::("height").unwrap(); + let fps = s.get::("framerate").ok(); + + // Update framerate in queue_stats + { + let mut stats = self.queue_stats.lock().unwrap(); + stats.framerate = fps; + } let settings = self.settings.lock().unwrap(); let split_screen = settings.split_screen; @@ -592,15 +698,10 @@ impl ObjectSubclass for VideoCompareMixer { .expect("Failed to create queue1"); queue1.set_property("name", "queue1"); - let overlay0 = gst::ElementFactory::make("textoverlay") - .build() - .expect("Failed to create overlay0"); - overlay0.set_property("name", "overlay0"); - - let overlay1 = gst::ElementFactory::make("textoverlay") + let overlay = gst::ElementFactory::make("textoverlay") .build() - .expect("Failed to create overlay1"); - overlay1.set_property("name", "overlay1"); + .expect("Failed to create overlay"); + overlay.set_property("name", "overlay"); Self { srcpad, @@ -608,11 +709,11 @@ impl ObjectSubclass for VideoCompareMixer { sinkpad1, queue0, queue1, - overlay0, - overlay1, + overlay, settings: Mutex::new(Settings::default()), mouse_state: Arc::new(Mutex::new(MouseState::default())), compositor_helper: Default::default(), + queue_stats: Arc::new(Mutex::new(QueueStats::default())), } } } From 4226fc6cf80a8e75d690733abe1bb9a9a997259d Mon Sep 17 00:00:00 2001 From: Diego Nieto Date: Thu, 25 Sep 2025 10:36:34 +0200 Subject: [PATCH 31/46] video-compare-mixer: add "overlay-stats" property to control overlay In this way it is possible to disable the overlay of the stats in the screen. It defaults to enabled. --- video/stats/src/comparemixer/imp.rs | 31 +++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/video/stats/src/comparemixer/imp.rs b/video/stats/src/comparemixer/imp.rs index ba74fb44c..0cca63320 100644 --- a/video/stats/src/comparemixer/imp.rs +++ b/video/stats/src/comparemixer/imp.rs @@ -60,6 +60,7 @@ struct Settings { backend: Backend, split_screen: bool, navigation_events: bool, + overlay_stats: bool, } impl Default for Settings { @@ -68,6 +69,7 @@ impl Default for Settings { backend: Backend::default(), split_screen: false, navigation_events: true, + overlay_stats: true, } } } @@ -508,17 +510,22 @@ impl VideoCompareMixer { let compositor_src_pad = compositor.static_pad("src").unwrap(); let overlay = self.overlay.clone(); let queue_stats = self.queue_stats.clone(); + let imp_weak = self.downgrade(); compositor_src_pad.add_probe(gst::PadProbeType::BUFFER, move |_: &gst::Pad, probe_info| { let Some(_) = probe_info.buffer() else { return gst::PadProbeReturn::Ok; }; - // Use formatted queue stats as overlay text - let stats = queue_stats.lock().unwrap(); - let stats_text = format!("{}", *stats); - overlay.set_property("text", stats_text); + let Some(imp) = imp_weak.upgrade() else { + return gst::PadProbeReturn::Ok; + }; + if imp.settings.lock().unwrap().overlay_stats { + let stats = queue_stats.lock().unwrap(); + let stats_text = format!("{}", *stats); + overlay.set_property("text", stats_text); + }; gst::PadProbeReturn::Ok }); } @@ -739,6 +746,11 @@ impl ObjectImpl for VideoCompareMixer { .default_value(true) .mutable_ready() .build(), + glib::ParamSpecBoolean::builder("overlay-stats") + .nick("Overlay Stats") + .blurb("Enable/disable the text overlay displaying encoder statistics") + .default_value(true) + .build(), ] }); @@ -778,6 +790,16 @@ impl ObjectImpl for VideoCompareMixer { settings.navigation_events ); } + "overlay-stats" => { + settings.overlay_stats = value.get().expect("type checked upstream"); + + gst::info!( + CAT, + imp = self, + "Set overlay-stats to {:?}", + settings.overlay_stats + ); + } _ => unimplemented!(), } } @@ -788,6 +810,7 @@ impl ObjectImpl for VideoCompareMixer { "backend" => settings.backend.to_value(), "split-screen" => settings.split_screen.to_value(), "navigation-events" => settings.navigation_events.to_value(), + "overlay-stats" => settings.overlay_stats.to_value(), _ => unimplemented!(), } } From d72a5afeb5837d5da4b948f539524d9c846e12d1 Mon Sep 17 00:00:00 2001 From: Diego Nieto Date: Fri, 26 Sep 2025 14:29:09 +0200 Subject: [PATCH 32/46] video-compare-mixer: remove deprected code --- video/stats/src/comparemixer/imp.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/video/stats/src/comparemixer/imp.rs b/video/stats/src/comparemixer/imp.rs index 0cca63320..04ad51fc4 100644 --- a/video/stats/src/comparemixer/imp.rs +++ b/video/stats/src/comparemixer/imp.rs @@ -450,8 +450,7 @@ impl VideoCompareMixer { self.obj().add(&crop0).expect("Failed to add crop0 element"); self.obj().add(&crop1).expect("Failed to add crop1 element"); - // FIXME remove split_screen logic if not needed. It adds and links crops always - self.link_elements(&compositor, true, backend)?; + self.link_elements(&compositor, backend)?; self.add_queue_probe(&self.queue0, 0); self.add_queue_probe(&self.queue1, 1); @@ -533,7 +532,6 @@ impl VideoCompareMixer { fn link_elements( &self, compositor: &gst::Element, - split_screen: bool, backend: Backend, ) -> Result<(), gst::ErrorMessage> { let compositor_pad0 = compositor @@ -578,7 +576,7 @@ impl VideoCompareMixer { .set_target(Some(&caps_filter.static_pad("src").unwrap())) .expect("Failed to link srcpad to capsfilter"); - if split_screen && backend != Backend::GL { + if backend != Backend::GL { // Get crop elements by name since we can't store them in struct easily let crop0 = self.obj().by_name("crop0").expect("crop0 should exist"); let crop1 = self.obj().by_name("crop1").expect("crop1 should exist"); From c14ec555198adb6f5c5d2923d35f8fa3d608be7a Mon Sep 17 00:00:00 2001 From: Diego Nieto Date: Fri, 26 Sep 2025 14:39:01 +0200 Subject: [PATCH 33/46] video-encoder-stats: remove println! --- video/stats/src/encoderstats/imp.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/video/stats/src/encoderstats/imp.rs b/video/stats/src/encoderstats/imp.rs index d25442889..e12cc3249 100644 --- a/video/stats/src/encoderstats/imp.rs +++ b/video/stats/src/encoderstats/imp.rs @@ -454,7 +454,6 @@ impl EncoderStats { move |_vmaf: &gst::Element, score: f64| { let mut stats = stats.lock().unwrap(); stats.vmaf_score = Some(score); - println!("VMAF score: {:.3}", score); } ), ); From d93cd6cea2996d2da0b9bc7b9ab4340025dca2e9 Mon Sep 17 00:00:00 2001 From: Diego Nieto Date: Sun, 28 Sep 2025 10:41:11 +0200 Subject: [PATCH 34/46] video-compare-mixer: improve layout alignment and polish code --- video/stats/src/comparemixer/imp.rs | 64 +++++++++++++++-------------- 1 file changed, 34 insertions(+), 30 deletions(-) diff --git a/video/stats/src/comparemixer/imp.rs b/video/stats/src/comparemixer/imp.rs index 04ad51fc4..ac1f3a83a 100644 --- a/video/stats/src/comparemixer/imp.rs +++ b/video/stats/src/comparemixer/imp.rs @@ -100,52 +100,58 @@ impl fmt::Display for QueueStats { let stats1_str = self.stats1.as_deref().unwrap_or("No stats available"); // Parse stats from both sides - let (encoder0, size0, buffers0, bitrate0, proc_time0, cpu0, vmaf0, latency0) = + let (encoder0, size0, bitrate0, proc_time0, cpu0, vmaf0, latency0) = parse_stats_string(stats0_str); - let (encoder1, size1, buffers1, bitrate1, proc_time1, cpu1, vmaf1, latency1) = + let (encoder1, size1, bitrate1, proc_time1, cpu1, vmaf1, latency1) = parse_stats_string(stats1_str); - // Calculate dynamic spacing based on left side content length - let total_width: usize = 80; // Total desired width - + let total_width: usize = 100; let encoder0_full = format!("Encoder: {}", encoder0); - let encoder_spacing = total_width.saturating_sub(encoder0_full.len()); - writeln!(f, "{}{:>width$}Encoder: {}", encoder0_full, "", encoder1, width = encoder_spacing)?; + let encoder1_full = format!("Encoder: {}", encoder1); + let encoder_spacing = total_width.saturating_sub(encoder0_full.len()+encoder1_full.len()); + writeln!(f, "{}{:>width$}{}", encoder0_full, "", encoder1_full, width = encoder_spacing)?; + let total_width: usize = 100; let size0_full = format!("Output size: {}", size0); - let size_spacing = total_width.saturating_sub(size0_full.len()); - writeln!(f, "{}{:>width$}Output size: {}", size0_full, "", size1, width = size_spacing)?; - - let buffers0_full = format!("Num buffers: {}", buffers0); - let buffers_spacing = total_width.saturating_sub(buffers0_full.len()); - writeln!(f, "{}{:>width$}Num buffers: {}", buffers0_full, "", buffers1, width = buffers_spacing)?; + let size1_full = format!("Output size: {}", size1); + let size_spacing = total_width.saturating_sub(size0_full.len()+size1_full.len()); + writeln!(f, "{}{:>width$}{}", size0_full, "", size1_full, width = size_spacing)?; + let total_width: usize = 100; let bitrate0_full = format!("Bitrate: {}", bitrate0); - let bitrate_spacing = total_width.saturating_sub(bitrate0_full.len()); - writeln!(f, "{}{:>width$}Bitrate: {}", bitrate0_full, "", bitrate1, width = bitrate_spacing)?; + let bitrate1_full = format!("Bitrate: {}", bitrate1); + let bitrate_spacing = total_width.saturating_sub(bitrate0_full.len()+bitrate1_full.len()); + writeln!(f, "{}{:>width$}{}", bitrate0_full, "", bitrate1_full, width = bitrate_spacing)?; + let total_width: usize = 95; let proc_time0_full = format!("Processing time: {}", proc_time0); - let proc_time_spacing = total_width.saturating_sub(proc_time0_full.len()); - writeln!(f, "{}{:>width$}Processing time: {}", proc_time0_full, "", proc_time1, width = proc_time_spacing)?; + let proc_time1_full = format!("Processing time: {}", proc_time1); + let proc_time_spacing = total_width.saturating_sub(proc_time0_full.len()+proc_time1_full.len()); + writeln!(f, "{}{:>width$}{}", proc_time0_full, "", proc_time1_full, width = proc_time_spacing)?; + let total_width: usize = 120; let cpu0_full = format!("CPU: {}", cpu0); - let cpu_spacing = total_width.saturating_sub(cpu0_full.len()); - writeln!(f, "{}{:>width$}CPU: {}", cpu0_full, "", cpu1, width = cpu_spacing)?; + let cpu1_full = format!("CPU: {}", cpu1); + let cpu_spacing = total_width.saturating_sub(cpu0_full.len()+cpu1_full.len()); + writeln!(f, "{}{:>width$}{}", cpu0_full, "", cpu1_full, width = cpu_spacing)?; + let total_width: usize = 112; let vmaf0_full = format!("VMAF: {}", vmaf0); - let vmaf_spacing = total_width.saturating_sub(vmaf0_full.len()); - writeln!(f, "{}{:>width$}VMAF: {}", vmaf0_full, "", vmaf1, width = vmaf_spacing)?; - - let latency0_full = format!("Encode latency: {}", latency0); - let latency_spacing = total_width.saturating_sub(latency0_full.len()); - writeln!(f, "{}{:>width$}Encode latency: {}", latency0_full, "", latency1, width = latency_spacing) + let vmaf1_full = format!("VMAF: {}", vmaf1); + let vmaf_spacing = total_width.saturating_sub(vmaf0_full.len()+vmaf1_full.len()); + writeln!(f, "{}{:>width$}{}", vmaf0_full, "", vmaf1_full, width = vmaf_spacing)?; + + let total_width: usize = 87; + let latency0_full = format!("Encoding latency: {}", latency0); + let latency1_full = format!("Encoding latency: {}", latency1); + let latency_spacing = total_width.saturating_sub(latency0_full.len()+latency1_full.len()); + writeln!(f, "{}{:>width$}{}", latency0_full, "", latency1_full, width = latency_spacing) } } -fn parse_stats_string(stats_str: &str) -> (String, String, String, String, String, String, String, String) { +fn parse_stats_string(stats_str: &str) -> (String, String, String, String, String, String, String) { let mut encoder = "Unknown".to_string(); let mut size = "0 KB".to_string(); - let mut buffers = "0".to_string(); let mut bitrate = "0.000 kbps".to_string(); let mut proc_time = "0.00 ms".to_string(); let mut cpu = "0 s".to_string(); @@ -157,8 +163,6 @@ fn parse_stats_string(stats_str: &str) -> (String, String, String, String, Strin encoder = line.replace("Encoder: ", ""); } else if line.starts_with("Output size: ") { size = line.replace("Output size: ", ""); - } else if line.starts_with("Num buffers: ") { - buffers = line.replace("Num buffers: ", ""); } else if line.starts_with("Bitrate: ") { bitrate = line.replace("Bitrate: ", ""); } else if line.starts_with("Processing time: ") { @@ -172,7 +176,7 @@ fn parse_stats_string(stats_str: &str) -> (String, String, String, String, Strin } } - (encoder, size, buffers, bitrate, proc_time, cpu, vmaf, latency) + (encoder, size, bitrate, proc_time, cpu, vmaf, latency) } impl VideoCompareMixer { From 4cdb1dcaa2bf72a7ed493abd9af5790441d1d81f Mon Sep 17 00:00:00 2001 From: Diego Nieto Date: Thu, 2 Oct 2025 13:20:05 +0200 Subject: [PATCH 35/46] videostats: update license and description --- video/stats/Cargo.toml | 2 +- video/stats/src/lib.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/video/stats/Cargo.toml b/video/stats/Cargo.toml index ca0d3116f..db01f4e30 100644 --- a/video/stats/Cargo.toml +++ b/video/stats/Cargo.toml @@ -5,7 +5,7 @@ authors = ["Diego Nieto "] repository = "https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs" license = "MPL-2.0" edition = "2021" -description = "videostats plugin" +description = "GStreamer Video Stats Plugin" [dependencies] hound = "3" diff --git a/video/stats/src/lib.rs b/video/stats/src/lib.rs index 0411810c5..57aee9064 100644 --- a/video/stats/src/lib.rs +++ b/video/stats/src/lib.rs @@ -25,7 +25,7 @@ gst::plugin_define!( env!("CARGO_PKG_DESCRIPTION"), plugin_init, concat!(env!("CARGO_PKG_VERSION"), "-", "commit-id"), - "MPL/X11", + "MPL", env!("CARGO_PKG_NAME"), env!("CARGO_PKG_NAME"), env!("CARGO_PKG_REPOSITORY"), From 5048d0bdf1210eb0a47d83bd4c13c592bf91c4da Mon Sep 17 00:00:00 2001 From: Diego Nieto Date: Sun, 12 Oct 2025 13:32:01 +0200 Subject: [PATCH 36/46] video-encoder-stats: isolate encoder GstObject between two queues Before the encoder was mixed with another objects that may introduce a little overhead. This way we ensure only encoder CPU is taken into account --- video/stats/src/encoderstats/imp.rs | 35 ++++++++++++++++++----------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/video/stats/src/encoderstats/imp.rs b/video/stats/src/encoderstats/imp.rs index e12cc3249..d18dcb57a 100644 --- a/video/stats/src/encoderstats/imp.rs +++ b/video/stats/src/encoderstats/imp.rs @@ -214,7 +214,13 @@ impl EncoderStats { encoder.set_property("name", "enc"); - // Add internal queue at the beginning + // Add originalbuffersave at the beginning + let originalbuffersave = gst::ElementFactory::make("originalbuffersave") + .build() + .expect("Failed to create originalbuffersave element"); + self.obj().add(&originalbuffersave).expect("Failed to add originalbuffersave element"); + + // Add internal queue after originalbuffersave let obj_name = self.obj().name().to_string(); let queue_name = if obj_name.contains("0") { "encq0" @@ -287,13 +293,15 @@ impl EncoderStats { gst::PadProbeReturn::Ok }); - let originalbuffersave = gst::ElementFactory::make("originalbuffersave") - .build() - .expect("Failed to create originalbuffersave element"); - self.obj().add(&originalbuffersave).expect("Failed to add originalbuffersave element"); - self.obj().add(&self.identity).unwrap(); + // Add new queue after encoder + let encoder_output_queue = gst::ElementFactory::make("queue") + .name("encoder_output_queue") + .build() + .expect("Failed to create encoder output queue"); + self.obj().add(&encoder_output_queue).expect("Failed to add encoder output queue"); + let tee0 = gst::ElementFactory::make("tee") .name("tee0") .build() @@ -302,10 +310,11 @@ impl EncoderStats { self.obj().add(&encoder).expect("Failed to add encoder element"); - // Link: input_queue -> originalbuffersave -> encoder -> identity -> tee0 - input_queue.link(&originalbuffersave).expect("Failed to link input queue to originalbuffersave"); - originalbuffersave.link(&encoder).expect("Failed to link originalbuffersave to encoder"); - encoder.link(&self.identity).expect("Failed to link encoder to identity"); + // Link: originalbuffersave -> input_queue -> encoder -> encoder_output_queue -> identity -> tee0 + originalbuffersave.link(&input_queue).expect("Failed to link originalbuffersave to input queue"); + input_queue.link(&encoder).expect("Failed to link input queue to encoder"); + encoder.link(&encoder_output_queue).expect("Failed to link encoder to encoder output queue"); + encoder_output_queue.link(&self.identity).expect("Failed to link encoder output queue to identity"); self.identity.link(&tee0).expect("Failed to link identity to tee0"); let tee0_src_0 = tee0.request_pad_simple("src_%u").expect("tee0 src pad"); @@ -319,10 +328,10 @@ impl EncoderStats { tee0_src_0.link(&queue0_sink_pad).expect("tee0.src_0 -> encintq0.sink"); self.srcpad.set_target(Some(&queue0_src_pad)).unwrap(); - // Connect sink ghostpad to input queue + // Connect sink ghostpad to originalbuffersave self.sinkpad - .set_target(Some(&input_queue.static_pad("sink").unwrap())) - .expect("Failed to link sink pad to input queue"); + .set_target(Some(&originalbuffersave.static_pad("sink").unwrap())) + .expect("Failed to link sink pad to originalbuffersave"); // Only create decoder branch if VMAF is enabled if vmaf_enabled { From 967e8828011476d8328d217ca7f7725cc46ebd72 Mon Sep 17 00:00:00 2001 From: Diego Nieto Date: Sun, 12 Oct 2025 13:39:54 +0200 Subject: [PATCH 37/46] videostats: merge examples into a single one --- video/stats/examples/video-stats-no-vmaf.rs | 36 ------------- .../examples/video-stats-split-screen.rs | 36 ------------- .../examples/video-stats-zero-latency.rs | 36 ------------- video/stats/examples/video-stats.rs | 53 ++++++++++++++++++- 4 files changed, 52 insertions(+), 109 deletions(-) delete mode 100644 video/stats/examples/video-stats-no-vmaf.rs delete mode 100644 video/stats/examples/video-stats-split-screen.rs delete mode 100644 video/stats/examples/video-stats-zero-latency.rs diff --git a/video/stats/examples/video-stats-no-vmaf.rs b/video/stats/examples/video-stats-no-vmaf.rs deleted file mode 100644 index 4e5d29c01..000000000 --- a/video/stats/examples/video-stats-no-vmaf.rs +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright (C) 2025, Fluendo S.A. -// Author: Diego Nieto -// -// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. -// If a copy of the MPL was not distributed with this file, You can obtain one at -// . -// -// SPDX-License-Identifier: MPL-2.0 - -use anyhow::Error; -use gst::prelude::*; - -fn main() -> Result<(), Error> { - gst::init()?; - - gstvideostats::plugin_register_static().expect("Failed to register videostats plugin"); - - let pipeline = gst::parse::launch("souphttpsrc location=\"https://ftp.nluug.nl/pub/graphics/blender/demo/movies/ToS/tears_of_steel_1080p.mov\" ! qtdemux name=demux demux.video_0 ! queue ! decodebin3 ! videoconvertscale ! capsfilter caps=\"video/x-raw,aspect-ratio=1/1\" ! tee name=tee ! video-encoder-stats encoder=\"x264enc bitrate=1024\" name=vs0 vmaf-stats=false tee. ! video-encoder-stats encoder=\"x264enc bitrate=512\" name=vs1 video-compare-mixer split-screen=true backend=CPU name=mixer vs0.src ! h264parse ! avdec_h264 ! mixer.sink_0 vs1.src ! h264parse ! avdec_h264 ! mixer.sink_1 mixer. ! autovideosink")?; - pipeline.set_state(gst::State::Playing)?; - - let bus = pipeline.bus().unwrap(); - while let Some(msg) = bus.timed_pop(gst::ClockTime::NONE) { - use gst::MessageView; - match msg.view() { - MessageView::Eos(..) => { - break; - } - MessageView::Error(..) => unreachable!(), - _ => (), - } - } - - pipeline.set_state(gst::State::Null)?; - - Ok(()) -} diff --git a/video/stats/examples/video-stats-split-screen.rs b/video/stats/examples/video-stats-split-screen.rs deleted file mode 100644 index 04cb4647e..000000000 --- a/video/stats/examples/video-stats-split-screen.rs +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright (C) 2025, Fluendo S.A. -// Author: Diego Nieto -// -// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. -// If a copy of the MPL was not distributed with this file, You can obtain one at -// . -// -// SPDX-License-Identifier: MPL-2.0 - -use anyhow::Error; -use gst::prelude::*; - -fn main() -> Result<(), Error> { - gst::init()?; - - gstvideostats::plugin_register_static().expect("Failed to register videostats plugin"); - - let pipeline = gst::parse::launch("souphttpsrc location=\"https://ftp.nluug.nl/pub/graphics/blender/demo/movies/ToS/tears_of_steel_1080p.mov\" ! qtdemux name=demux demux.video_0 ! queue ! decodebin3 ! videoconvertscale ! capsfilter caps=\"video/x-raw,aspect-ratio=1/1\" ! tee name=tee ! video-encoder-stats encoder=\"x264enc\" decoder=\"h264parse ! avdec_h264\" name=vs0 tee. ! video-encoder-stats encoder=\"x264enc bitrate=512\" name=vs1 video-compare-mixer split-screen=true backend=CPU name=mixer vs0.src ! decodebin3 ! mixer.sink_0 vs1.src ! decodebin3 ! mixer.sink_1 mixer. ! autovideosink")?; - pipeline.set_state(gst::State::Playing)?; - - let bus = pipeline.bus().unwrap(); - while let Some(msg) = bus.timed_pop(gst::ClockTime::NONE) { - use gst::MessageView; - match msg.view() { - MessageView::Eos(..) => { - break; - } - MessageView::Error(..) => unreachable!(), - _ => (), - } - } - - pipeline.set_state(gst::State::Null)?; - - Ok(()) -} diff --git a/video/stats/examples/video-stats-zero-latency.rs b/video/stats/examples/video-stats-zero-latency.rs deleted file mode 100644 index a5d916ae7..000000000 --- a/video/stats/examples/video-stats-zero-latency.rs +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright (C) 2025, Fluendo S.A. -// Author: Diego Nieto -// -// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. -// If a copy of the MPL was not distributed with this file, You can obtain one at -// . -// -// SPDX-License-Identifier: MPL-2.0 - -use anyhow::Error; -use gst::prelude::*; - -fn main() -> Result<(), Error> { - gst::init()?; - - gstvideostats::plugin_register_static().expect("Failed to register videostats plugin"); - - let pipeline = gst::parse::launch("souphttpsrc location=\"https://ftp.nluug.nl/pub/graphics/blender/demo/movies/ToS/tears_of_steel_1080p.mov\" ! qtdemux name=demux demux.video_0 ! queue ! decodebin3 ! videoconvertscale ! capsfilter caps=\"video/x-raw,aspect-ratio=1/1\" ! tee name=tee ! video-encoder-stats encoder=\"x264enc bitrate=1024 tune=zerolatency speed-preset=ultrafast threads=4\" name=vs0 vmaf-stats=false tee. ! video-encoder-stats encoder=\"x264enc bitrate=512 tune=zerolatency speed-preset=ultrafast\" name=vs1 video-compare-mixer split-screen=true backend=CPU name=mixer vs0.src ! h264parse ! avdec_h264 ! mixer.sink_0 vs1.src ! h264parse ! avdec_h264 ! mixer.sink_1 mixer. ! autovideosink")?; - pipeline.set_state(gst::State::Playing)?; - - let bus = pipeline.bus().unwrap(); - while let Some(msg) = bus.timed_pop(gst::ClockTime::NONE) { - use gst::MessageView; - match msg.view() { - MessageView::Eos(..) => { - break; - } - MessageView::Error(..) => unreachable!(), - _ => (), - } - } - - pipeline.set_state(gst::State::Null)?; - - Ok(()) -} diff --git a/video/stats/examples/video-stats.rs b/video/stats/examples/video-stats.rs index 36895b34f..c52205988 100644 --- a/video/stats/examples/video-stats.rs +++ b/video/stats/examples/video-stats.rs @@ -9,13 +9,64 @@ use anyhow::Error; use gst::prelude::*; +use std::env; + +fn print_usage() { + println!("Usage: video-stats [PIPELINE_TYPE]"); + println!(); + println!("Pipeline types:"); + println!(" default - Default pipeline with VMAF stats and decoder request pads"); + println!(" zero-latency - Zero latency pipeline with ultrafast encoding, no VMAF"); + println!(" split-screen - Split screen comparison using video-compare-mixer"); + println!(" no-vmaf - Pipeline without VMAF statistics"); + println!(); + println!("If no argument is provided, 'default' pipeline will be used."); +} + +fn get_pipeline_string(pipeline_type: &str) -> Result<&'static str, Error> { + match pipeline_type { + "default" => Ok("souphttpsrc location=\"https://ftp.nluug.nl/pub/graphics/blender/demo/movies/ToS/tears_of_steel_1080p.mov\" ! qtdemux name=demux demux.video_0 ! queue ! decodebin3 ! videoconvertscale ! capsfilter caps=\"video/x-raw,aspect-ratio=1/1\" ! tee name=tee ! video-encoder-stats encoder=\"x264enc bitrate=1024\" decoder=\"h264parse ! avdec_h264\" name=vs0 ! fakesink tee. ! video-encoder-stats encoder=\"x264enc bitrate=512\" name=vs1 ! fakesink video-compare-mixer split-screen=false backend=CPU name=mixer vs0.decoder_src ! mixer.sink_0 vs1.decoder_src ! mixer.sink_1 mixer. ! autovideosink"), + + "zero-latency" => Ok("souphttpsrc location=\"https://ftp.nluug.nl/pub/graphics/blender/demo/movies/ToS/tears_of_steel_1080p.mov\" ! qtdemux name=demux demux.video_0 ! queue ! decodebin3 ! videoconvertscale ! capsfilter caps=\"video/x-raw,aspect-ratio=1/1\" ! tee name=tee ! video-encoder-stats encoder=\"x264enc bitrate=1024 tune=zerolatency speed-preset=ultrafast threads=4\" name=vs0 vmaf-stats=false tee. ! video-encoder-stats encoder=\"x264enc bitrate=512 tune=zerolatency speed-preset=ultrafast\" name=vs1 video-compare-mixer split-screen=true backend=CPU name=mixer vs0.src ! h264parse ! avdec_h264 ! mixer.sink_0 vs1.src ! h264parse ! avdec_h264 ! mixer.sink_1 mixer. ! autovideosink"), + + "split-screen" => Ok("souphttpsrc location=\"https://ftp.nluug.nl/pub/graphics/blender/demo/movies/ToS/tears_of_steel_1080p.mov\" ! qtdemux name=demux demux.video_0 ! queue ! decodebin3 ! videoconvertscale ! capsfilter caps=\"video/x-raw,aspect-ratio=1/1\" ! tee name=tee ! video-encoder-stats encoder=\"x264enc\" decoder=\"h264parse ! avdec_h264\" name=vs0 tee. ! video-encoder-stats encoder=\"x264enc bitrate=512\" name=vs1 video-compare-mixer split-screen=true backend=CPU name=mixer vs0.src ! decodebin3 ! mixer.sink_0 vs1.src ! decodebin3 ! mixer.sink_1 mixer. ! autovideosink"), + + "no-vmaf" => Ok("souphttpsrc location=\"https://ftp.nluug.nl/pub/graphics/blender/demo/movies/ToS/tears_of_steel_1080p.mov\" ! qtdemux name=demux demux.video_0 ! queue ! decodebin3 ! videoconvertscale ! capsfilter caps=\"video/x-raw,aspect-ratio=1/1\" ! tee name=tee ! video-encoder-stats encoder=\"x264enc bitrate=1024\" name=vs0 vmaf-stats=false tee. ! video-encoder-stats encoder=\"x264enc bitrate=512\" name=vs1 video-compare-mixer split-screen=true backend=CPU name=mixer vs0.src ! h264parse ! avdec_h264 ! mixer.sink_0 vs1.src ! h264parse ! avdec_h264 ! mixer.sink_1 mixer. ! autovideosink"), + + _ => { + println!("Unknown pipeline type: {}", pipeline_type); + print_usage(); + Err(anyhow::anyhow!("Invalid pipeline type")) + } + } +} fn main() -> Result<(), Error> { gst::init()?; gstvideostats::plugin_register_static().expect("Failed to register videostats plugin"); - let pipeline = gst::parse::launch("souphttpsrc location=\"https://ftp.nluug.nl/pub/graphics/blender/demo/movies/ToS/tears_of_steel_1080p.mov\" ! qtdemux name=demux demux.video_0 ! queue ! decodebin3 ! videoconvertscale ! capsfilter caps=\"video/x-raw,aspect-ratio=1/1\" ! tee name=tee ! video-encoder-stats encoder=\"x264enc bitrate=1024\" decoder=\"h264parse ! avdec_h264\" name=vs0 ! fakesink tee. ! video-encoder-stats encoder=\"x264enc bitrate=512\" name=vs1 ! fakesink video-compare-mixer split-screen=false backend=CPU name=mixer vs0.decoder_src ! mixer.sink_0 vs1.decoder_src ! mixer.sink_1 mixer. ! autovideosink")?; + // Parse command line arguments + let args: Vec = env::args().collect(); + let pipeline_type = if args.len() > 1 { + if args[1] == "--help" || args[1] == "-h" { + print_usage(); + return Ok(()); + } + &args[1] + } else { + "default" + }; + + println!("Using pipeline type: {}", pipeline_type); + + let pipeline_string = get_pipeline_string(pipeline_type)?; + let pipeline = gst::parse::launch(pipeline_string)?; + + // Alternative test pipelines (commented out for reference) + // let pipeline = gst::parse::launch("videotestsrc is-live=true ! videoconvertscale ! capsfilter caps=\"video/x-raw,width=640,height=480,aspect-ratio=1/1,framerate=30/1\" ! tee name=tee ! video-encoder-stats encoder=\"x264enc bitrate=1024\" decoder=\"h264parse ! avdec_h264\" name=vs0 ! fakesink tee. ! video-encoder-stats encoder=\"x264enc bitrate=256\" name=vs1 ! fakesink video-compare-mixer split-screen=true backend=CPU name=mixer vs0.decoder_src ! mixer.sink_0 vs1.decoder_src ! mixer.sink_1 mixer. ! autovideosink")?; + // let pipeline = gst::parse::launch("gltestsrc is-live=true pattern=13 ! gldownload ! videoconvert ! capsfilter caps=\"video/x-raw,width=640,height=480,framerate=30/1\" ! tee name=tee tee.src_0 ! video-encoder-stats encoder=\"x264enc bitrate=1024\" decoder=\"h264parse ! avdec_h264\" name=vs0 ! fakesink tee.src_1 ! video-encoder-stats encoder=\"x264enc bitrate=512\" name=vs1 ! fakesink video-compare-mixer split-screen=true backend=CPU name=mixer vs0.decoder_src ! mixer.sink_0 vs1.decoder_src ! mixer.sink_1 mixer. ! autovideosink")?; + pipeline.set_state(gst::State::Playing)?; let bus = pipeline.bus().unwrap(); From 75e14471558c385e2a50d1a2090ff9a2a26e9194 Mon Sep 17 00:00:00 2001 From: Diego Nieto Date: Tue, 21 Oct 2025 16:27:13 +0200 Subject: [PATCH 38/46] video-compare-mixer: fix alignment issue --- video/stats/src/comparemixer/imp.rs | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/video/stats/src/comparemixer/imp.rs b/video/stats/src/comparemixer/imp.rs index ac1f3a83a..c81a1342a 100644 --- a/video/stats/src/comparemixer/imp.rs +++ b/video/stats/src/comparemixer/imp.rs @@ -105,43 +105,37 @@ impl fmt::Display for QueueStats { let (encoder1, size1, bitrate1, proc_time1, cpu1, vmaf1, latency1) = parse_stats_string(stats1_str); - let total_width: usize = 100; + let total_width: usize = 60; let encoder0_full = format!("Encoder: {}", encoder0); let encoder1_full = format!("Encoder: {}", encoder1); let encoder_spacing = total_width.saturating_sub(encoder0_full.len()+encoder1_full.len()); writeln!(f, "{}{:>width$}{}", encoder0_full, "", encoder1_full, width = encoder_spacing)?; - let total_width: usize = 100; let size0_full = format!("Output size: {}", size0); let size1_full = format!("Output size: {}", size1); let size_spacing = total_width.saturating_sub(size0_full.len()+size1_full.len()); writeln!(f, "{}{:>width$}{}", size0_full, "", size1_full, width = size_spacing)?; - let total_width: usize = 100; let bitrate0_full = format!("Bitrate: {}", bitrate0); let bitrate1_full = format!("Bitrate: {}", bitrate1); let bitrate_spacing = total_width.saturating_sub(bitrate0_full.len()+bitrate1_full.len()); writeln!(f, "{}{:>width$}{}", bitrate0_full, "", bitrate1_full, width = bitrate_spacing)?; - let total_width: usize = 95; let proc_time0_full = format!("Processing time: {}", proc_time0); let proc_time1_full = format!("Processing time: {}", proc_time1); let proc_time_spacing = total_width.saturating_sub(proc_time0_full.len()+proc_time1_full.len()); writeln!(f, "{}{:>width$}{}", proc_time0_full, "", proc_time1_full, width = proc_time_spacing)?; - let total_width: usize = 120; let cpu0_full = format!("CPU: {}", cpu0); let cpu1_full = format!("CPU: {}", cpu1); let cpu_spacing = total_width.saturating_sub(cpu0_full.len()+cpu1_full.len()); writeln!(f, "{}{:>width$}{}", cpu0_full, "", cpu1_full, width = cpu_spacing)?; - let total_width: usize = 112; let vmaf0_full = format!("VMAF: {}", vmaf0); let vmaf1_full = format!("VMAF: {}", vmaf1); let vmaf_spacing = total_width.saturating_sub(vmaf0_full.len()+vmaf1_full.len()); writeln!(f, "{}{:>width$}{}", vmaf0_full, "", vmaf1_full, width = vmaf_spacing)?; - let total_width: usize = 87; let latency0_full = format!("Encoding latency: {}", latency0); let latency1_full = format!("Encoding latency: {}", latency1); let latency_spacing = total_width.saturating_sub(latency0_full.len()+latency1_full.len()); @@ -711,6 +705,7 @@ impl ObjectSubclass for VideoCompareMixer { .build() .expect("Failed to create overlay"); overlay.set_property("name", "overlay"); + overlay.set_property_from_str("font-desc", "Consolas 10"); Self { srcpad, From 48c5ad1513ee9649d21578f432918f29f2805315 Mon Sep 17 00:00:00 2001 From: Diego Nieto Date: Tue, 28 Oct 2025 10:18:30 +0100 Subject: [PATCH 39/46] video-encoder-stats: provide a stats message when EOS Silent intermediate messages and signals based on property value. Provide always messages and signal with the final stats on EOS --- video/stats/src/encoderstats/imp.rs | 65 ++++++++++++++++++++++++----- 1 file changed, 54 insertions(+), 11 deletions(-) diff --git a/video/stats/src/encoderstats/imp.rs b/video/stats/src/encoderstats/imp.rs index d18dcb57a..4bb28e20b 100644 --- a/video/stats/src/encoderstats/imp.rs +++ b/video/stats/src/encoderstats/imp.rs @@ -49,11 +49,19 @@ impl EncoderStats { let encoder = self.obj().by_name("enc").expect("expected encoder"); let encoder_factory = encoder.factory().expect("encoder should have a factory"); let encoder_name = encoder_factory.name(); + let identity_clone = identity.clone(); + let stats_clone_eos = self.stats.clone(); + let element_weak_eos = self.obj().downgrade(); + let encoder_name_eos = encoder_name.clone(); let element_weak = self.obj().downgrade(); let silent_arc = Arc::new(self.silent.lock().unwrap().clone()); let last_message_arc = self.last_message.clone(); + // Clone Arc values for first probe + let silent_arc_buffer = silent_arc.clone(); + let last_message_arc_buffer = last_message_arc.clone(); + identity_src_pad.add_probe(gst::PadProbeType::BUFFER, move |_pad, probe_info| { let Some(buffer) = probe_info.buffer_mut() else { return gst::PadProbeReturn::Ok; @@ -76,21 +84,19 @@ impl EncoderStats { gst::log!(CAT, "Updated meta stats: encoder={}, buffers={}, bytes={}", encoder_name, num_buffers, num_bytes); - if let Some(element) = element_weak.upgrade() { - let stats_message = format!("{}", stats_clone.clone()); - - let structure = gst::Structure::builder("encoder-stats") - .field("message", &stats_message) - .build(); + if !*silent_arc_buffer { + if let Some(element) = element_weak.upgrade() { + let stats_message = format!("{}", stats_clone.clone()); - let message = gst::message::Application::new(structure); - let _ = element.post_message(message); + let structure = gst::Structure::builder("encoder-stats") + .field("message", &stats_message) + .build(); - let silent = *silent_arc; + let message = gst::message::Application::new(structure); - if !silent { + let _ = element.post_message(message); { - let mut last_message_guard = last_message_arc.lock().unwrap(); + let mut last_message_guard = last_message_arc_buffer.lock().unwrap(); *last_message_guard = Some(stats_message); } element.notify("last-message"); @@ -102,6 +108,43 @@ impl EncoderStats { } gst::PadProbeReturn::Ok }); + + identity_src_pad.add_probe(gst::PadProbeType::EVENT_DOWNSTREAM, move |_pad, probe_info| { + if let Some(event) = probe_info.event() { + if event.type_() == gst::EventType::Eos { + gst::info!(CAT, "EOS received, posting final stats"); + + // Update global stats with final values from identity + let identity_stats = identity_clone.property::("stats"); + let num_bytes = identity_stats.get::("num-bytes").unwrap_or(0); + let num_buffers = identity_stats.get::("num-buffers").unwrap_or(0); + + { + let mut stats = stats_clone_eos.lock().unwrap(); + stats.num_bytes = num_bytes; + stats.num_buffers = num_buffers; + stats.name = encoder_name_eos.to_string(); + } + + if let Some(element) = element_weak_eos.upgrade() { + let stats_message = format!("{}", stats_clone_eos.lock().unwrap()); + + let structure = gst::Structure::builder("encoder-stats") + .field("message", &stats_message) + .build(); + + let message = gst::message::Application::new(structure); + let _ = element.post_message(message); + { + let mut last_message_guard = last_message_arc.lock().unwrap(); + *last_message_guard = Some(stats_message); + } + element.notify("last-message"); + } + } + } + gst::PadProbeReturn::Ok + }); } fn add_encoder_probes(&self) { From b0ec05f892d8fc727b5292a2aaa4d9e71eb68b24 Mon Sep 17 00:00:00 2001 From: Diego Nieto Date: Tue, 28 Oct 2025 10:24:37 +0100 Subject: [PATCH 40/46] videoencoderstats: provide Max. buffers inside metric --- video/stats/src/videoencoderstats.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/video/stats/src/videoencoderstats.rs b/video/stats/src/videoencoderstats.rs index 87c47b0a6..fe0026dcd 100644 --- a/video/stats/src/videoencoderstats.rs +++ b/video/stats/src/videoencoderstats.rs @@ -105,6 +105,11 @@ impl fmt::Display for VideoEncoderStats { "Output size: {} KB", self.num_bytes / 1000, // Convert to KB )?; + writeln!( + f, + "Max. Buffers inside: {}", + self.max_buffers_inside + )?; let framerate = self.framerate.unwrap(); let total_time_secs = self.num_buffers as f64 / framerate.numer() as f64; From 70ac20b38e9c6c68ef275dd1cc25767247323f7b Mon Sep 17 00:00:00 2001 From: Diego Nieto Date: Tue, 28 Oct 2025 15:19:21 +0100 Subject: [PATCH 41/46] videoencoderstats: add num buffers needed for the first output metric --- video/stats/src/videoencoderstats.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/video/stats/src/videoencoderstats.rs b/video/stats/src/videoencoderstats.rs index fe0026dcd..818eb6632 100644 --- a/video/stats/src/videoencoderstats.rs +++ b/video/stats/src/videoencoderstats.rs @@ -32,6 +32,7 @@ pub struct VideoEncoderStats { pub num_bytes: u64, pub time_last_buffers: VecDeque, pub max_buffers_inside: usize, + pub num_buffers_first_output: i32, pub total_processing_time: Duration, pub threads_utime: u64, pub threads_stime: u64, @@ -51,6 +52,7 @@ impl Default for VideoEncoderStats { num_buffers: 0, time_last_buffers: VecDeque::::new(), max_buffers_inside: 0, + num_buffers_first_output: -1, total_processing_time: Duration::ZERO, threads_utime: 0, threads_stime: 0, @@ -72,6 +74,9 @@ impl VideoEncoderStats { } pub fn buffer_out(&mut self) { + if self.num_buffers_first_output == -1 { + self.num_buffers_first_output = self.time_last_buffers.len() as i32; + } if let Some(arrive) = self.time_last_buffers.pop_front() { let diff = arrive.elapsed(); self.total_processing_time += diff; @@ -110,6 +115,11 @@ impl fmt::Display for VideoEncoderStats { "Max. Buffers inside: {}", self.max_buffers_inside )?; + writeln!( + f, + "Num. Buffers first output: {}", + self.num_buffers_first_output + )?; let framerate = self.framerate.unwrap(); let total_time_secs = self.num_buffers as f64 / framerate.numer() as f64; From 83bf65f34cf00e74eaf3ccd57ec56e864ed3a4c3 Mon Sep 17 00:00:00 2001 From: Diego Nieto Date: Tue, 28 Oct 2025 16:56:38 +0100 Subject: [PATCH 42/46] videoencoderstats: add current buffers seens metric --- video/stats/src/videoencoderstats.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/video/stats/src/videoencoderstats.rs b/video/stats/src/videoencoderstats.rs index 818eb6632..1a0c920b5 100644 --- a/video/stats/src/videoencoderstats.rs +++ b/video/stats/src/videoencoderstats.rs @@ -32,6 +32,7 @@ pub struct VideoEncoderStats { pub num_bytes: u64, pub time_last_buffers: VecDeque, pub max_buffers_inside: usize, + pub current_buffers_inside_seen: usize, pub num_buffers_first_output: i32, pub total_processing_time: Duration, pub threads_utime: u64, @@ -52,6 +53,7 @@ impl Default for VideoEncoderStats { num_buffers: 0, time_last_buffers: VecDeque::::new(), max_buffers_inside: 0, + current_buffers_inside_seen: 0, num_buffers_first_output: -1, total_processing_time: Duration::ZERO, threads_utime: 0, @@ -70,7 +72,8 @@ impl VideoEncoderStats { if self.time_last_buffers.len() > self.max_buffers_inside { self.max_buffers_inside = self.time_last_buffers.len(); } - gst::log!(CAT, "Current buffers lenght {}", self.time_last_buffers.len()); + self.current_buffers_inside_seen = self.time_last_buffers.len(); + gst::log!(CAT, "Current buffers length {}", self.current_buffers_inside_seen); } pub fn buffer_out(&mut self) { @@ -115,6 +118,11 @@ impl fmt::Display for VideoEncoderStats { "Max. Buffers inside: {}", self.max_buffers_inside )?; + writeln!( + f, + "Current buffers inside: {}", + self.current_buffers_inside_seen + )?; writeln!( f, "Num. Buffers first output: {}", From 2f28375c72993f52d554234808f892d6a0eea278 Mon Sep 17 00:00:00 2001 From: msabiniok Date: Mon, 22 Dec 2025 15:09:41 +0100 Subject: [PATCH 43/46] video-encoder-stats: handle fractional framerate cases --- video/stats/src/videoencoderstats.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/video/stats/src/videoencoderstats.rs b/video/stats/src/videoencoderstats.rs index 1a0c920b5..857b87680 100644 --- a/video/stats/src/videoencoderstats.rs +++ b/video/stats/src/videoencoderstats.rs @@ -99,7 +99,7 @@ impl VideoEncoderStats { impl fmt::Display for VideoEncoderStats { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - if self.framerate.unwrap().denom() != 1 { + if self.framerate.unwrap().denom() != 1 && self.framerate.unwrap().denom() != 1001 { unimplemented!(); } @@ -130,7 +130,7 @@ impl fmt::Display for VideoEncoderStats { )?; let framerate = self.framerate.unwrap(); - let total_time_secs = self.num_buffers as f64 / framerate.numer() as f64; + let total_time_secs = self.num_buffers as f64 * framerate.denom() as f64 / framerate.numer() as f64; let bitrate = if total_time_secs > 0.0 { (self.num_bytes as f64 * 8.0) / total_time_secs } else { From c7cb30daa725c693a53ce2f496c3c1030a0fbfc2 Mon Sep 17 00:00:00 2001 From: Ruben Gonzalez Date: Wed, 21 Jan 2026 09:30:21 +0100 Subject: [PATCH 44/46] video-encoder-stats: Fixes UX for "Num. Buffers first output" From ``` Num. Buffers first output: -1 Num. Buffers first output: 1 Num. Buffers first output: 1 ``` to ``` Num. Buffers first output: No first output yet Num. Buffers first output: 1 Num. Buffers first output: 1 ``` Issue: RDI-3210 --- video/stats/src/videoencoderstats.rs | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/video/stats/src/videoencoderstats.rs b/video/stats/src/videoencoderstats.rs index 857b87680..d450d1c86 100644 --- a/video/stats/src/videoencoderstats.rs +++ b/video/stats/src/videoencoderstats.rs @@ -33,7 +33,7 @@ pub struct VideoEncoderStats { pub time_last_buffers: VecDeque, pub max_buffers_inside: usize, pub current_buffers_inside_seen: usize, - pub num_buffers_first_output: i32, + pub num_buffers_first_output: Option, pub total_processing_time: Duration, pub threads_utime: u64, pub threads_stime: u64, @@ -54,7 +54,7 @@ impl Default for VideoEncoderStats { time_last_buffers: VecDeque::::new(), max_buffers_inside: 0, current_buffers_inside_seen: 0, - num_buffers_first_output: -1, + num_buffers_first_output: None, total_processing_time: Duration::ZERO, threads_utime: 0, threads_stime: 0, @@ -77,8 +77,8 @@ impl VideoEncoderStats { } pub fn buffer_out(&mut self) { - if self.num_buffers_first_output == -1 { - self.num_buffers_first_output = self.time_last_buffers.len() as i32; + if self.num_buffers_first_output == None { + self.num_buffers_first_output = Some(self.time_last_buffers.len()); } if let Some(arrive) = self.time_last_buffers.pop_front() { let diff = arrive.elapsed(); @@ -123,11 +123,18 @@ impl fmt::Display for VideoEncoderStats { "Current buffers inside: {}", self.current_buffers_inside_seen )?; - writeln!( - f, - "Num. Buffers first output: {}", - self.num_buffers_first_output - )?; + if let Some(num_buffers_first_output) = self.num_buffers_first_output { + writeln!( + f, + "Num. Buffers first output: {}", + num_buffers_first_output + )?; + } else { + writeln!( + f, + "Num. Buffers first output: No first output yet" + )?; + } let framerate = self.framerate.unwrap(); let total_time_secs = self.num_buffers as f64 * framerate.denom() as f64 / framerate.numer() as f64; From 6fbeeec0ec19443a9fc1905981e0dc2153bf9939 Mon Sep 17 00:00:00 2001 From: Ruben Gonzalez Date: Wed, 21 Jan 2026 09:43:23 +0100 Subject: [PATCH 45/46] video-encoder-stats: Add max_processing_time This maximum value is only valid if there is no reordering of frames within the encoder (b-frames). If reordering occurs, it is necessary to mark the buffers to relate the input buffer to the output buffer. Similar to MR9622 MR9622: https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/9622 Issue: RDI-3209 --- video/stats/src/videoencoderstats.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/video/stats/src/videoencoderstats.rs b/video/stats/src/videoencoderstats.rs index d450d1c86..33f96c633 100644 --- a/video/stats/src/videoencoderstats.rs +++ b/video/stats/src/videoencoderstats.rs @@ -35,6 +35,7 @@ pub struct VideoEncoderStats { pub current_buffers_inside_seen: usize, pub num_buffers_first_output: Option, pub total_processing_time: Duration, + pub max_processing_time: Duration, pub threads_utime: u64, pub threads_stime: u64, pub framerate: Option, @@ -56,6 +57,7 @@ impl Default for VideoEncoderStats { current_buffers_inside_seen: 0, num_buffers_first_output: None, total_processing_time: Duration::ZERO, + max_processing_time: Duration::ZERO, threads_utime: 0, threads_stime: 0, vmaf_score: None, @@ -83,6 +85,9 @@ impl VideoEncoderStats { if let Some(arrive) = self.time_last_buffers.pop_front() { let diff = arrive.elapsed(); self.total_processing_time += diff; + if diff > self.max_processing_time { + self.max_processing_time = diff; + } } else { panic!("output buffer w/o input"); } @@ -153,6 +158,12 @@ impl fmt::Display for VideoEncoderStats { "Processing time: {:.2} ms", avg_processing_time )?; + let max_processing_time = self.max_processing_time.as_millis(); + writeln!( + f, + "Max processing time: {:.2} ms", + max_processing_time + )?; let cpu_time = self.threads_utime + self.threads_stime; #[cfg(target_os = "linux")] From 4e00969d31cb1ae76d971f0a54210322a8c16d1e Mon Sep 17 00:00:00 2001 From: Ruben Gonzalez Date: Wed, 21 Jan 2026 10:01:45 +0100 Subject: [PATCH 46/46] videoencoderstats: fmt --- video/stats/build.rs | 1 - video/stats/src/comparemixer/imp.rs | 242 ++++++++---- video/stats/src/encoderstats/imp.rs | 474 ++++++++++++++++------- video/stats/src/lib.rs | 4 +- video/stats/src/videoencoderstats.rs | 73 ++-- video/stats/src/videoencoderstatsmeta.rs | 11 +- 6 files changed, 532 insertions(+), 273 deletions(-) diff --git a/video/stats/build.rs b/video/stats/build.rs index 056534a8f..cda12e57e 100644 --- a/video/stats/build.rs +++ b/video/stats/build.rs @@ -1,4 +1,3 @@ - fn main() { gst_plugin_version_helper::info() } diff --git a/video/stats/src/comparemixer/imp.rs b/video/stats/src/comparemixer/imp.rs index c81a1342a..f594d1c5b 100644 --- a/video/stats/src/comparemixer/imp.rs +++ b/video/stats/src/comparemixer/imp.rs @@ -12,14 +12,14 @@ use gst::prelude::*; use gst::subclass::prelude::*; use gst_video::NavigationEvent; -use crate::videoencoderstatsmeta::VideoEncoderStatsMeta; use crate::comparemixer::compositor::Compositor; -use crate::comparemixer::compositor::Position; use crate::comparemixer::compositor::Mode; +use crate::comparemixer::compositor::Position; +use crate::videoencoderstatsmeta::VideoEncoderStatsMeta; -use std::sync::{LazyLock, Mutex, Arc}; -use std::vec::Vec; use std::fmt; +use std::sync::{Arc, LazyLock, Mutex}; +use std::vec::Vec; static CAT: LazyLock = LazyLock::new(|| { gst::DebugCategory::new( @@ -108,38 +108,88 @@ impl fmt::Display for QueueStats { let total_width: usize = 60; let encoder0_full = format!("Encoder: {}", encoder0); let encoder1_full = format!("Encoder: {}", encoder1); - let encoder_spacing = total_width.saturating_sub(encoder0_full.len()+encoder1_full.len()); - writeln!(f, "{}{:>width$}{}", encoder0_full, "", encoder1_full, width = encoder_spacing)?; + let encoder_spacing = total_width.saturating_sub(encoder0_full.len() + encoder1_full.len()); + writeln!( + f, + "{}{:>width$}{}", + encoder0_full, + "", + encoder1_full, + width = encoder_spacing + )?; let size0_full = format!("Output size: {}", size0); let size1_full = format!("Output size: {}", size1); - let size_spacing = total_width.saturating_sub(size0_full.len()+size1_full.len()); - writeln!(f, "{}{:>width$}{}", size0_full, "", size1_full, width = size_spacing)?; + let size_spacing = total_width.saturating_sub(size0_full.len() + size1_full.len()); + writeln!( + f, + "{}{:>width$}{}", + size0_full, + "", + size1_full, + width = size_spacing + )?; let bitrate0_full = format!("Bitrate: {}", bitrate0); let bitrate1_full = format!("Bitrate: {}", bitrate1); - let bitrate_spacing = total_width.saturating_sub(bitrate0_full.len()+bitrate1_full.len()); - writeln!(f, "{}{:>width$}{}", bitrate0_full, "", bitrate1_full, width = bitrate_spacing)?; + let bitrate_spacing = total_width.saturating_sub(bitrate0_full.len() + bitrate1_full.len()); + writeln!( + f, + "{}{:>width$}{}", + bitrate0_full, + "", + bitrate1_full, + width = bitrate_spacing + )?; let proc_time0_full = format!("Processing time: {}", proc_time0); let proc_time1_full = format!("Processing time: {}", proc_time1); - let proc_time_spacing = total_width.saturating_sub(proc_time0_full.len()+proc_time1_full.len()); - writeln!(f, "{}{:>width$}{}", proc_time0_full, "", proc_time1_full, width = proc_time_spacing)?; + let proc_time_spacing = + total_width.saturating_sub(proc_time0_full.len() + proc_time1_full.len()); + writeln!( + f, + "{}{:>width$}{}", + proc_time0_full, + "", + proc_time1_full, + width = proc_time_spacing + )?; let cpu0_full = format!("CPU: {}", cpu0); let cpu1_full = format!("CPU: {}", cpu1); - let cpu_spacing = total_width.saturating_sub(cpu0_full.len()+cpu1_full.len()); - writeln!(f, "{}{:>width$}{}", cpu0_full, "", cpu1_full, width = cpu_spacing)?; + let cpu_spacing = total_width.saturating_sub(cpu0_full.len() + cpu1_full.len()); + writeln!( + f, + "{}{:>width$}{}", + cpu0_full, + "", + cpu1_full, + width = cpu_spacing + )?; let vmaf0_full = format!("VMAF: {}", vmaf0); let vmaf1_full = format!("VMAF: {}", vmaf1); - let vmaf_spacing = total_width.saturating_sub(vmaf0_full.len()+vmaf1_full.len()); - writeln!(f, "{}{:>width$}{}", vmaf0_full, "", vmaf1_full, width = vmaf_spacing)?; + let vmaf_spacing = total_width.saturating_sub(vmaf0_full.len() + vmaf1_full.len()); + writeln!( + f, + "{}{:>width$}{}", + vmaf0_full, + "", + vmaf1_full, + width = vmaf_spacing + )?; let latency0_full = format!("Encoding latency: {}", latency0); let latency1_full = format!("Encoding latency: {}", latency1); - let latency_spacing = total_width.saturating_sub(latency0_full.len()+latency1_full.len()); - writeln!(f, "{}{:>width$}{}", latency0_full, "", latency1_full, width = latency_spacing) + let latency_spacing = total_width.saturating_sub(latency0_full.len() + latency1_full.len()); + writeln!( + f, + "{}{:>width$}{}", + latency0_full, + "", + latency1_full, + width = latency_spacing + ) } } @@ -185,9 +235,7 @@ impl VideoCompareMixer { } } - pub fn add_navigation_events_probe( - &self, - ) { + pub fn add_navigation_events_probe(&self) { let compositor_supports_crop: bool = self.settings.lock().unwrap().backend == Backend::GL; let mixer = self.obj().by_name("compositor").unwrap(); @@ -223,7 +271,7 @@ impl VideoCompareMixer { return gst::PadProbeReturn::Ok; }; - let compositor = &mut (*compositor_helper_clone.lock().unwrap()).unwrap(); + let compositor = &mut (*compositor_helper_clone.lock().unwrap()).unwrap(); let original_compositor = *compositor; let nav_event_clone = nav_event.clone(); @@ -279,7 +327,7 @@ impl VideoCompareMixer { } _ => { gst::info!(CAT, "Unhandled key: {}", key); - }, + } }, NavigationEvent::MouseMove { x, y, .. } => { let state = mouse_state.lock().unwrap(); @@ -384,8 +432,24 @@ impl VideoCompareMixer { Self::fix_pos(&mut pos0, compositor_helper.width, compositor_supports_crop); Self::fix_pos(&mut pos1, compositor_helper.width, compositor_supports_crop); - gst::log!(CAT, "Position 0: {}x{}+{}+{}, crop_right: {}", pos0.width, pos0.height, pos0.xpos, pos0.ypos, pos0.crop_right); - gst::log!(CAT, "Position 1: {}x{}+{}+{}, crop_left: {}", pos1.width, pos1.height, pos1.xpos, pos1.ypos, pos1.crop_left); + gst::log!( + CAT, + "Position 0: {}x{}+{}+{}, crop_right: {}", + pos0.width, + pos0.height, + pos0.xpos, + pos0.ypos, + pos0.crop_right + ); + gst::log!( + CAT, + "Position 1: {}x{}+{}+{}, crop_left: {}", + pos1.width, + pos1.height, + pos1.xpos, + pos1.ypos, + pos1.crop_left + ); //TODO refactor avoid copy and paste if compositor_supports_crop { @@ -419,7 +483,12 @@ impl VideoCompareMixer { ("ypos", &pos1.ypos), ]); - gst::log!(CAT, "right crop: {}, left crop: {}", pos0.crop_right, pos1.crop_left); + gst::log!( + CAT, + "right crop: {}, left crop: {}", + pos0.crop_right, + pos1.crop_left + ); crop0.set_property("right", pos0.crop_right); crop1.set_property("left", pos1.crop_left); } @@ -458,7 +527,9 @@ impl VideoCompareMixer { VideoCompareMixer::catch_panic_pad_function( parent, || false, - |video_compare_mixer| video_compare_mixer.sink_event(&pad.clone().upcast::(), event), + |video_compare_mixer| { + video_compare_mixer.sink_event(&pad.clone().upcast::(), event) + }, ); Ok(gst::FlowSuccess::Ok) }); @@ -470,37 +541,40 @@ impl VideoCompareMixer { fn add_queue_probe(&self, queue: &gst::Element, queue_index: u8) { let queue_src_pad = queue.static_pad("src").unwrap(); let queue_stats = self.queue_stats.clone(); - queue_src_pad.add_probe(gst::PadProbeType::BUFFER, move |_: &gst::Pad, probe_info| { - let Some(buffer) = probe_info.buffer_mut() else { - return gst::PadProbeReturn::Ok; - }; + queue_src_pad.add_probe( + gst::PadProbeType::BUFFER, + move |_: &gst::Pad, probe_info| { + let Some(buffer) = probe_info.buffer_mut() else { + return gst::PadProbeReturn::Ok; + }; - let mut stats = queue_stats.lock().unwrap(); + let mut stats = queue_stats.lock().unwrap(); - // Only update at framerate intervals, similar to encoderstats - let fps_n: i32; - if let Some(fps) = stats.framerate { - fps_n = fps.numer(); - } else { - return gst::PadProbeReturn::Ok; - } + // Only update at framerate intervals, similar to encoderstats + let fps_n: i32; + if let Some(fps) = stats.framerate { + fps_n = fps.numer(); + } else { + return gst::PadProbeReturn::Ok; + } - if let Some(statsmeta) = buffer.meta::() { - let num_buffers = statsmeta.stats().num_buffers; + if let Some(statsmeta) = buffer.meta::() { + let num_buffers = statsmeta.stats().num_buffers; - // Only update once per second based on framerate - if num_buffers % (fps_n as u64) == 0 { - let stats_string = format!("{}", statsmeta.stats()); - match queue_index { - 0 => stats.stats0 = Some(stats_string), - 1 => stats.stats1 = Some(stats_string), - _ => {} + // Only update once per second based on framerate + if num_buffers % (fps_n as u64) == 0 { + let stats_string = format!("{}", statsmeta.stats()); + match queue_index { + 0 => stats.stats0 = Some(stats_string), + 1 => stats.stats1 = Some(stats_string), + _ => {} + } } } - } - gst::PadProbeReturn::Ok - }); + gst::PadProbeReturn::Ok + }, + ); } fn add_compositor_src_probe(&self, compositor: &gst::Element) { @@ -509,22 +583,25 @@ impl VideoCompareMixer { let queue_stats = self.queue_stats.clone(); let imp_weak = self.downgrade(); - compositor_src_pad.add_probe(gst::PadProbeType::BUFFER, move |_: &gst::Pad, probe_info| { - let Some(_) = probe_info.buffer() else { - return gst::PadProbeReturn::Ok; - }; + compositor_src_pad.add_probe( + gst::PadProbeType::BUFFER, + move |_: &gst::Pad, probe_info| { + let Some(_) = probe_info.buffer() else { + return gst::PadProbeReturn::Ok; + }; - let Some(imp) = imp_weak.upgrade() else { - return gst::PadProbeReturn::Ok; - }; + let Some(imp) = imp_weak.upgrade() else { + return gst::PadProbeReturn::Ok; + }; - if imp.settings.lock().unwrap().overlay_stats { - let stats = queue_stats.lock().unwrap(); - let stats_text = format!("{}", *stats); - overlay.set_property("text", stats_text); - }; - gst::PadProbeReturn::Ok - }); + if imp.settings.lock().unwrap().overlay_stats { + let stats = queue_stats.lock().unwrap(); + let stats_text = format!("{}", *stats); + overlay.set_property("text", stats_text); + }; + gst::PadProbeReturn::Ok + }, + ); } fn link_elements( @@ -567,8 +644,12 @@ impl VideoCompareMixer { .set_target(Some(&self.queue1.static_pad("sink").unwrap())) .expect("Failed to link sinkpad1 to queue1"); - compositor.link(&self.overlay).expect("Failed to link compositor to overlay"); - self.overlay.link(&caps_filter).expect("Failed to link overlay to capsfilter"); + compositor + .link(&self.overlay) + .expect("Failed to link compositor to overlay"); + self.overlay + .link(&caps_filter) + .expect("Failed to link overlay to capsfilter"); self.srcpad .set_target(Some(&caps_filter.static_pad("src").unwrap())) @@ -618,7 +699,11 @@ impl VideoCompareMixer { self.queue0.sync_state_with_parent().unwrap(); self.queue1.sync_state_with_parent().unwrap(); self.overlay.sync_state_with_parent().unwrap(); - self.obj().by_name("compositor").unwrap().sync_state_with_parent().unwrap(); + self.obj() + .by_name("compositor") + .unwrap() + .sync_state_with_parent() + .unwrap(); Ok(()) } @@ -647,7 +732,10 @@ impl VideoCompareMixer { drop(settings); let caps = format!("video/x-raw,width={},height={}", width, height); - self.obj().by_name("capsfilter0").unwrap().set_property_from_str("caps", &caps.as_str()); + self.obj() + .by_name("capsfilter0") + .unwrap() + .set_property_from_str("caps", &caps.as_str()); let compositor_mode = if split_screen { Mode::Split @@ -655,11 +743,7 @@ impl VideoCompareMixer { Mode::SideBySide }; - let compositor_helper = Compositor::new( - compositor_mode, - width, - height, - ); + let compositor_helper = Compositor::new(compositor_mode, width, height); *self.compositor_helper.lock().unwrap() = Some(compositor_helper); if navigation_events { @@ -760,12 +844,7 @@ impl ObjectImpl for VideoCompareMixer { "backend" => { settings.backend = value.get().expect("type checked upstream"); - gst::info!( - CAT, - imp = self, - "Set backend to {:?}", - settings.backend - ); + gst::info!(CAT, imp = self, "Set backend to {:?}", settings.backend); } "split-screen" => { settings.split_screen = value.get().expect("type checked upstream"); @@ -896,5 +975,4 @@ impl ElementImpl for VideoCompareMixer { } } -impl BinImpl for VideoCompareMixer { -} +impl BinImpl for VideoCompareMixer {} diff --git a/video/stats/src/encoderstats/imp.rs b/video/stats/src/encoderstats/imp.rs index 4bb28e20b..71b48f997 100644 --- a/video/stats/src/encoderstats/imp.rs +++ b/video/stats/src/encoderstats/imp.rs @@ -14,9 +14,9 @@ use gst::subclass::prelude::*; use crate::videoencoderstats::*; use crate::videoencoderstatsmeta::VideoEncoderStatsMeta; +use std::sync::Arc; use std::sync::{LazyLock, Mutex}; use std::vec::Vec; -use std::sync::Arc; static CAT: LazyLock = LazyLock::new(|| { gst::DebugCategory::new( @@ -41,9 +41,7 @@ pub struct EncoderStats { } impl EncoderStats { - fn add_identity_probe( - &self, - ) { + fn add_identity_probe(&self) { let identity = self.obj().by_name("identity").expect("expected identity"); let identity_src_pad = identity.static_pad("src").unwrap(); let encoder = self.obj().by_name("enc").expect("expected encoder"); @@ -81,8 +79,13 @@ impl EncoderStats { let stats_clone = new_stats.clone(); meta.replace(new_stats); - gst::log!(CAT, "Updated meta stats: encoder={}, buffers={}, bytes={}", - encoder_name, num_buffers, num_bytes); + gst::log!( + CAT, + "Updated meta stats: encoder={}, buffers={}, bytes={}", + encoder_name, + num_buffers, + num_bytes + ); if !*silent_arc_buffer { if let Some(element) = element_weak.upgrade() { @@ -102,49 +105,51 @@ impl EncoderStats { element.notify("last-message"); } } - } else { gst::warning!(CAT, "No VideoEncoderStatsMeta found on buffer"); } gst::PadProbeReturn::Ok }); - identity_src_pad.add_probe(gst::PadProbeType::EVENT_DOWNSTREAM, move |_pad, probe_info| { - if let Some(event) = probe_info.event() { - if event.type_() == gst::EventType::Eos { - gst::info!(CAT, "EOS received, posting final stats"); - - // Update global stats with final values from identity - let identity_stats = identity_clone.property::("stats"); - let num_bytes = identity_stats.get::("num-bytes").unwrap_or(0); - let num_buffers = identity_stats.get::("num-buffers").unwrap_or(0); - - { - let mut stats = stats_clone_eos.lock().unwrap(); - stats.num_bytes = num_bytes; - stats.num_buffers = num_buffers; - stats.name = encoder_name_eos.to_string(); - } - - if let Some(element) = element_weak_eos.upgrade() { - let stats_message = format!("{}", stats_clone_eos.lock().unwrap()); + identity_src_pad.add_probe( + gst::PadProbeType::EVENT_DOWNSTREAM, + move |_pad, probe_info| { + if let Some(event) = probe_info.event() { + if event.type_() == gst::EventType::Eos { + gst::info!(CAT, "EOS received, posting final stats"); - let structure = gst::Structure::builder("encoder-stats") - .field("message", &stats_message) - .build(); + // Update global stats with final values from identity + let identity_stats = identity_clone.property::("stats"); + let num_bytes = identity_stats.get::("num-bytes").unwrap_or(0); + let num_buffers = identity_stats.get::("num-buffers").unwrap_or(0); - let message = gst::message::Application::new(structure); - let _ = element.post_message(message); { - let mut last_message_guard = last_message_arc.lock().unwrap(); - *last_message_guard = Some(stats_message); + let mut stats = stats_clone_eos.lock().unwrap(); + stats.num_bytes = num_bytes; + stats.num_buffers = num_buffers; + stats.name = encoder_name_eos.to_string(); + } + + if let Some(element) = element_weak_eos.upgrade() { + let stats_message = format!("{}", stats_clone_eos.lock().unwrap()); + + let structure = gst::Structure::builder("encoder-stats") + .field("message", &stats_message) + .build(); + + let message = gst::message::Application::new(structure); + let _ = element.post_message(message); + { + let mut last_message_guard = last_message_arc.lock().unwrap(); + *last_message_guard = Some(stats_message); + } + element.notify("last-message"); } - element.notify("last-message"); } } - } - gst::PadProbeReturn::Ok - }); + gst::PadProbeReturn::Ok + }, + ); } fn add_encoder_probes(&self) { @@ -161,7 +166,10 @@ impl EncoderStats { gst::log!(CAT, "Buffer in encoder sink pad"); let current_time = *gst::ClockTime::from_nseconds( - gst::SystemClock::obtain().upcast::().time().nseconds() + gst::SystemClock::obtain() + .upcast::() + .time() + .nseconds(), ); let buffer = buffer.make_mut(); @@ -171,9 +179,16 @@ impl EncoderStats { meta.replace(new_stats); - gst::log!(CAT, "Buffer in encoder sink pad, pre_encode_time: {}", current_time); + gst::log!( + CAT, + "Buffer in encoder sink pad, pre_encode_time: {}", + current_time + ); } else { - gst::warning!(CAT, "No VideoEncoderStatsMeta found on buffer in encoder sink probe"); + gst::warning!( + CAT, + "No VideoEncoderStatsMeta found on buffer in encoder sink probe" + ); } gst::PadProbeReturn::Ok @@ -188,7 +203,10 @@ impl EncoderStats { gst::log!(CAT, "Buffer out encoder src pad"); let current_time = *gst::ClockTime::from_nseconds( - gst::SystemClock::obtain().upcast::().time().nseconds() + gst::SystemClock::obtain() + .upcast::() + .time() + .nseconds(), ); let buffer = buffer.make_mut(); @@ -198,9 +216,16 @@ impl EncoderStats { meta.replace(new_stats); - gst::log!(CAT, "Buffer out encoder src pad, post_encode_time: {}", current_time); + gst::log!( + CAT, + "Buffer out encoder src pad, post_encode_time: {}", + current_time + ); } else { - gst::warning!(CAT, "No VideoEncoderStatsMeta found on buffer in encoder src probe"); + gst::warning!( + CAT, + "No VideoEncoderStatsMeta found on buffer in encoder src probe" + ); } gst::PadProbeReturn::Ok @@ -261,7 +286,9 @@ impl EncoderStats { let originalbuffersave = gst::ElementFactory::make("originalbuffersave") .build() .expect("Failed to create originalbuffersave element"); - self.obj().add(&originalbuffersave).expect("Failed to add originalbuffersave element"); + self.obj() + .add(&originalbuffersave) + .expect("Failed to add originalbuffersave element"); // Add internal queue after originalbuffersave let obj_name = self.obj().name().to_string(); @@ -275,7 +302,9 @@ impl EncoderStats { .name(queue_name) .build() .expect("Failed to create input queue"); - self.obj().add(&input_queue).expect("Failed to add input queue"); + self.obj() + .add(&input_queue) + .expect("Failed to add input queue"); // Add probe to input queue src pad to log buffer flow let input_queue_src_pad = input_queue.static_pad("src").unwrap(); @@ -285,13 +314,22 @@ impl EncoderStats { let Some(buffer) = probe_info.buffer_mut() else { return gst::PadProbeReturn::Ok; }; - gst::info!(CAT, "Buffer received in {} src pad, PTS: {:?}, DTS: {:?}, size: {}", - queue_name_clone, buffer.pts(), buffer.dts(), buffer.size()); + gst::info!( + CAT, + "Buffer received in {} src pad, PTS: {:?}, DTS: {:?}, size: {}", + queue_name_clone, + buffer.pts(), + buffer.dts(), + buffer.size() + ); let mut stats = stats_clone.lock().unwrap(); stats.input_time = *gst::ClockTime::from_nseconds( - gst::SystemClock::obtain().upcast::().time().nseconds() + gst::SystemClock::obtain() + .upcast::() + .time() + .nseconds(), ); // Only update CPU stats at framerate intervals as it takes time @@ -305,17 +343,28 @@ impl EncoderStats { let num_buffers = stats.num_buffers; if num_buffers % (fps_n as u64) == 0 { - let queue_name = if obj_name.contains("0") { "encq0:src" } else { "encq1:src" }; + let queue_name = if obj_name.contains("0") { + "encq0:src" + } else { + "encq1:src" + }; let thread_patterns = match stats.name.as_str() { "flulcevch264enc" => vec![queue_name, "lcevc", "pool."], "lcevch264enc" => vec![queue_name, "pool."], _ => vec![queue_name], }; - let (total_utime, total_stime) = thread_patterns.iter() + let (total_utime, total_stime) = thread_patterns + .iter() .map(|pattern| { let (utime, stime) = get_cpu_usage(pattern.to_string()); - gst::log!(CAT, "Thread pattern '{}' - utime: {}, stime: {}", pattern, utime, stime); + gst::log!( + CAT, + "Thread pattern '{}' - utime: {}, stime: {}", + pattern, + utime, + stime + ); (utime, stime) }) .fold((0u64, 0u64), |(acc_utime, acc_stime), (utime, stime)| { @@ -328,10 +377,7 @@ impl EncoderStats { let buffer = buffer.make_mut(); - VideoEncoderStatsMeta::add( - buffer, - stats.clone(), - ); + VideoEncoderStatsMeta::add(buffer, stats.clone()); gst::PadProbeReturn::Ok }); @@ -343,7 +389,9 @@ impl EncoderStats { .name("encoder_output_queue") .build() .expect("Failed to create encoder output queue"); - self.obj().add(&encoder_output_queue).expect("Failed to add encoder output queue"); + self.obj() + .add(&encoder_output_queue) + .expect("Failed to add encoder output queue"); let tee0 = gst::ElementFactory::make("tee") .name("tee0") @@ -351,24 +399,40 @@ impl EncoderStats { .expect("Failed to create tee0 element"); self.obj().add(&tee0).unwrap(); - self.obj().add(&encoder).expect("Failed to add encoder element"); + self.obj() + .add(&encoder) + .expect("Failed to add encoder element"); // Link: originalbuffersave -> input_queue -> encoder -> encoder_output_queue -> identity -> tee0 - originalbuffersave.link(&input_queue).expect("Failed to link originalbuffersave to input queue"); - input_queue.link(&encoder).expect("Failed to link input queue to encoder"); - encoder.link(&encoder_output_queue).expect("Failed to link encoder to encoder output queue"); - encoder_output_queue.link(&self.identity).expect("Failed to link encoder output queue to identity"); - self.identity.link(&tee0).expect("Failed to link identity to tee0"); + originalbuffersave + .link(&input_queue) + .expect("Failed to link originalbuffersave to input queue"); + input_queue + .link(&encoder) + .expect("Failed to link input queue to encoder"); + encoder + .link(&encoder_output_queue) + .expect("Failed to link encoder to encoder output queue"); + encoder_output_queue + .link(&self.identity) + .expect("Failed to link encoder output queue to identity"); + self.identity + .link(&tee0) + .expect("Failed to link identity to tee0"); let tee0_src_0 = tee0.request_pad_simple("src_%u").expect("tee0 src pad"); let queue0 = gst::ElementFactory::make("queue") - .name("encintq0") - .build() - .expect("Failed to create queue encintq0"); - self.obj().add(&queue0).expect("Failed to add queue encintq0"); + .name("encintq0") + .build() + .expect("Failed to create queue encintq0"); + self.obj() + .add(&queue0) + .expect("Failed to add queue encintq0"); let queue0_sink_pad = queue0.static_pad("sink").unwrap(); let queue0_src_pad = queue0.static_pad("src").unwrap(); - tee0_src_0.link(&queue0_sink_pad).expect("tee0.src_0 -> encintq0.sink"); + tee0_src_0 + .link(&queue0_sink_pad) + .expect("tee0.src_0 -> encintq0.sink"); self.srcpad.set_target(Some(&queue0_src_pad)).unwrap(); // Connect sink ghostpad to originalbuffersave @@ -392,20 +456,30 @@ impl EncoderStats { // Use custom decoder if provided, otherwise use decodebin3 let final_decoder = if let Some(custom_decoder) = decoder.clone() { custom_decoder.set_property("name", "dec"); - self.obj().add(&custom_decoder).expect("Failed to add custom decoder element"); + self.obj() + .add(&custom_decoder) + .expect("Failed to add custom decoder element"); custom_decoder } else { let decodebin3 = gst::ElementFactory::make("decodebin3") .name("dec") .build() .expect("Failed to create decodebin3"); - self.obj().add(&decodebin3).expect("Failed to add decodebin3"); + self.obj() + .add(&decodebin3) + .expect("Failed to add decodebin3"); decodebin3 }; self.obj().add(&queue1).expect("Failed to add queue1"); - tee0_src_1.link(&queue1.static_pad("sink").unwrap()).expect("tee0.src_1 -> queue1"); - queue1.static_pad("src").unwrap().link(&final_decoder.static_pad("sink").unwrap()).expect("queue1.src -> decoder.sink"); + tee0_src_1 + .link(&queue1.static_pad("sink").unwrap()) + .expect("tee0.src_1 -> queue1"); + queue1 + .static_pad("src") + .unwrap() + .link(&final_decoder.static_pad("sink").unwrap()) + .expect("queue1.src -> decoder.sink"); // Conditionally add tee after decoder if request pad exists if has_request_pad { @@ -413,17 +487,27 @@ impl EncoderStats { .name("decoder_tee") .build() .expect("Failed to create decoder_tee"); - self.obj().add(&decoder_tee).expect("Failed to add decoder_tee"); + self.obj() + .add(&decoder_tee) + .expect("Failed to add decoder_tee"); // Set up decoder -> decoder_tee connection - self.setup_decoder_to_tee_connection(final_decoder.clone(), decoder_tee.clone(), decoder.is_some()); + self.setup_decoder_to_tee_connection( + final_decoder.clone(), + decoder_tee.clone(), + decoder.is_some(), + ); // Connect decoder_tee src_0 to VMAF pipeline - let decoder_tee_src_0 = decoder_tee.request_pad_simple("src_%u").expect("decoder_tee src_0"); + let decoder_tee_src_0 = decoder_tee + .request_pad_simple("src_%u") + .expect("decoder_tee src_0"); self.setup_vmaf_pipeline(decoder_tee_src_0); // Connect decoder_tee src_1 to request pad - let decoder_tee_src_1 = decoder_tee.request_pad_simple("src_%u").expect("decoder_tee src_1"); + let decoder_tee_src_1 = decoder_tee + .request_pad_simple("src_%u") + .expect("decoder_tee src_1"); let request_pad_guard = self.request_pad.lock().unwrap(); if let Some(ref request_pad) = *request_pad_guard { request_pad.set_target(Some(&decoder_tee_src_1)).unwrap(); @@ -434,13 +518,14 @@ impl EncoderStats { } } - unsafe - { + unsafe { self.sinkpad.set_event_full_function(|pad, parent, event| { EncoderStats::catch_panic_pad_function( parent, || false, - |video_encoder_stats| video_encoder_stats.sink_event(&pad.clone().upcast::(), event), + |video_encoder_stats| { + video_encoder_stats.sink_event(&pad.clone().upcast::(), event) + }, ); Ok(gst::FlowSuccess::Ok) }); @@ -452,21 +537,41 @@ impl EncoderStats { Ok(()) } - fn setup_decoder_to_tee_connection(&self, final_decoder: gst::Element, decoder_tee: gst::Element, is_manual_decoder: bool) { + fn setup_decoder_to_tee_connection( + &self, + final_decoder: gst::Element, + decoder_tee: gst::Element, + is_manual_decoder: bool, + ) { if is_manual_decoder { // Manual decoder case: direct link - final_decoder.link(&decoder_tee).expect("decoder -> decoder_tee"); + final_decoder + .link(&decoder_tee) + .expect("decoder -> decoder_tee"); } else { // decodebin3 case: use connect_pad_added let decoder_tee_clone = decoder_tee.clone(); final_decoder.connect_pad_added(move |_dbin, src_pad| { let decoder_tee_sink = decoder_tee_clone.static_pad("sink").unwrap(); - src_pad.link(&decoder_tee_sink).expect("decodebin3.src -> decoder_tee.sink"); + src_pad + .link(&decoder_tee_sink) + .expect("decodebin3.src -> decoder_tee.sink"); }); } } - fn create_vmaf_pipeline_elements(&self) -> (gst::Element, gst::Element, gst::Element, gst::Element, gst::Element, gst::Element, gst::Element, gst::Element) { + fn create_vmaf_pipeline_elements( + &self, + ) -> ( + gst::Element, + gst::Element, + gst::Element, + gst::Element, + gst::Element, + gst::Element, + gst::Element, + gst::Element, + ) { let videoconvert = gst::ElementFactory::make("videoconvert") .build() .expect("Failed to create videoconvert"); @@ -502,73 +607,146 @@ impl EncoderStats { vmaf.connect_closure( "score", false, - glib::closure!( - move |_vmaf: &gst::Element, score: f64| { - let mut stats = stats.lock().unwrap(); - stats.vmaf_score = Some(score); - } - ), + glib::closure!(move |_vmaf: &gst::Element, score: f64| { + let mut stats = stats.lock().unwrap(); + stats.vmaf_score = Some(score); + }), ); } let fakesink = gst::ElementFactory::make("fakesink") .build() .expect("Failed to create fakesink"); - (videoconvert, capsfilter, tee1, originalbufferstore, queue_vmaf_0, queue_vmaf_1, vmaf, fakesink) + ( + videoconvert, + capsfilter, + tee1, + originalbufferstore, + queue_vmaf_0, + queue_vmaf_1, + vmaf, + fakesink, + ) } fn setup_vmaf_pipeline(&self, input_pad: gst::Pad) { - let (videoconvert, capsfilter, tee1, originalbufferstore, queue_vmaf_0, queue_vmaf_1, vmaf, fakesink) = - self.create_vmaf_pipeline_elements(); - - self.obj().add_many([ - &videoconvert, &capsfilter, &tee1, - &originalbufferstore, &queue_vmaf_0, &vmaf, &queue_vmaf_1, &fakesink, - ].as_ref()).expect("Failed to add vmaf branch elements"); + let ( + videoconvert, + capsfilter, + tee1, + originalbufferstore, + queue_vmaf_0, + queue_vmaf_1, + vmaf, + fakesink, + ) = self.create_vmaf_pipeline_elements(); + + self.obj() + .add_many( + [ + &videoconvert, + &capsfilter, + &tee1, + &originalbufferstore, + &queue_vmaf_0, + &vmaf, + &queue_vmaf_1, + &fakesink, + ] + .as_ref(), + ) + .expect("Failed to add vmaf branch elements"); // Link input_pad -> videoconvert -> capsfilter -> tee1 let videoconvert_sink = videoconvert.static_pad("sink").unwrap(); - input_pad.link(&videoconvert_sink).expect("input -> videoconvert"); - videoconvert.link(&capsfilter).expect("videoconvert -> capsfilter"); + input_pad + .link(&videoconvert_sink) + .expect("input -> videoconvert"); + videoconvert + .link(&capsfilter) + .expect("videoconvert -> capsfilter"); capsfilter.link(&tee1).expect("capsfilter -> tee1"); let tee1_src_0 = tee1.request_pad_simple("src_%u").expect("tee1 src_0"); - tee1_src_0.link(&originalbufferstore.static_pad("sink").unwrap()).expect("tee1.src_0 -> originalbufferstore"); - originalbufferstore.link(&queue_vmaf_0).expect("originalbufferrestore -> queue_vmaf_0"); + tee1_src_0 + .link(&originalbufferstore.static_pad("sink").unwrap()) + .expect("tee1.src_0 -> originalbufferstore"); + originalbufferstore + .link(&queue_vmaf_0) + .expect("originalbufferrestore -> queue_vmaf_0"); queue_vmaf_0.link(&vmaf).expect("queue_vmaf_0 -> vmaf"); vmaf.link(&fakesink).expect("vmaf -> fakesink"); let tee1_src_1 = tee1.request_pad_simple("src_%u").expect("tee1 src_1"); let vmaf_sink_1 = vmaf.request_pad_simple("sink_1").expect("vmaf sink_1"); - tee1_src_1.link(&queue_vmaf_1.static_pad("sink").unwrap()).expect("tee1.src_1 -> queue_vmaf_1"); - queue_vmaf_1.static_pad("src").unwrap().link(&vmaf_sink_1).expect("queue_vmaf_1.src -> vmaf.sink_1"); + tee1_src_1 + .link(&queue_vmaf_1.static_pad("sink").unwrap()) + .expect("tee1.src_1 -> queue_vmaf_1"); + queue_vmaf_1 + .static_pad("src") + .unwrap() + .link(&vmaf_sink_1) + .expect("queue_vmaf_1.src -> vmaf.sink_1"); } fn setup_decoder_to_vmaf_direct(&self, final_decoder: gst::Element, is_manual_decoder: bool) { - let (videoconvert, capsfilter, tee1, originalbufferstore, queue_vmaf_0, queue_vmaf_1, vmaf, fakesink) = - self.create_vmaf_pipeline_elements(); - - self.obj().add_many([ - &videoconvert, &capsfilter, &tee1, - &originalbufferstore, &queue_vmaf_0, &vmaf, &queue_vmaf_1, &fakesink, - ].as_ref()).expect("Failed to add vmaf branch elements"); + let ( + videoconvert, + capsfilter, + tee1, + originalbufferstore, + queue_vmaf_0, + queue_vmaf_1, + vmaf, + fakesink, + ) = self.create_vmaf_pipeline_elements(); + + self.obj() + .add_many( + [ + &videoconvert, + &capsfilter, + &tee1, + &originalbufferstore, + &queue_vmaf_0, + &vmaf, + &queue_vmaf_1, + &fakesink, + ] + .as_ref(), + ) + .expect("Failed to add vmaf branch elements"); if is_manual_decoder { // Manual decoder case: link decoder directly to videoconvert - final_decoder.link(&videoconvert).expect("decoder -> videoconvert"); - videoconvert.link(&capsfilter).expect("videoconvert -> capsfilter"); + final_decoder + .link(&videoconvert) + .expect("decoder -> videoconvert"); + videoconvert + .link(&capsfilter) + .expect("videoconvert -> capsfilter"); capsfilter.link(&tee1).expect("capsfilter -> tee1"); let tee1_src_0 = tee1.request_pad_simple("src_%u").expect("tee1 src_0"); - tee1_src_0.link(&originalbufferstore.static_pad("sink").unwrap()).expect("tee1.src_0 -> originalbufferstore"); - originalbufferstore.link(&queue_vmaf_0).expect("originalbufferrestore -> queue_vmaf_0"); + tee1_src_0 + .link(&originalbufferstore.static_pad("sink").unwrap()) + .expect("tee1.src_0 -> originalbufferstore"); + originalbufferstore + .link(&queue_vmaf_0) + .expect("originalbufferrestore -> queue_vmaf_0"); queue_vmaf_0.link(&vmaf).expect("queue_vmaf_0 -> vmaf"); vmaf.link(&fakesink).expect("vmaf -> fakesink"); let tee1_src_1 = tee1.request_pad_simple("src_%u").expect("tee1 src_1"); let vmaf_sink_1 = vmaf.request_pad_simple("sink_1").expect("vmaf sink_1"); - tee1_src_1.link(&queue_vmaf_1.static_pad("sink").unwrap()).expect("tee1.src_1 -> queue_vmaf_1"); - queue_vmaf_1.static_pad("src").unwrap().link(&vmaf_sink_1).expect("queue_vmaf_1.src -> vmaf.sink_1"); + tee1_src_1 + .link(&queue_vmaf_1.static_pad("sink").unwrap()) + .expect("tee1.src_1 -> queue_vmaf_1"); + queue_vmaf_1 + .static_pad("src") + .unwrap() + .link(&vmaf_sink_1) + .expect("queue_vmaf_1.src -> vmaf.sink_1"); } else { // decodebin3 case: use connect_pad_added for dynamic linking let tee1_clone = tee1.clone(); @@ -589,16 +767,32 @@ impl EncoderStats { let capsfilter_src = capsfilter_clone.static_pad("src").unwrap(); let tee1_sink = tee1_clone.static_pad("sink").unwrap(); if capsfilter_src.link(&tee1_sink).is_ok() { - let tee1_src_0 = tee1_clone.request_pad_simple("src_%u").expect("tee1 src_0"); - tee1_src_0.link(&originalbufferstore_clone.static_pad("sink").unwrap()).expect("tee1.src_0 -> originalbufferstore"); - originalbufferstore_clone.link(&queue_vmaf_0_clone).expect("originalbufferrestore -> queue_vmaf_0"); - queue_vmaf_0_clone.link(&vmaf_clone).expect("queue_vmaf_0 -> vmaf"); + let tee1_src_0 = + tee1_clone.request_pad_simple("src_%u").expect("tee1 src_0"); + tee1_src_0 + .link(&originalbufferstore_clone.static_pad("sink").unwrap()) + .expect("tee1.src_0 -> originalbufferstore"); + originalbufferstore_clone + .link(&queue_vmaf_0_clone) + .expect("originalbufferrestore -> queue_vmaf_0"); + queue_vmaf_0_clone + .link(&vmaf_clone) + .expect("queue_vmaf_0 -> vmaf"); vmaf_clone.link(&fakesink_clone).expect("vmaf -> fakesink"); - let tee1_src_1 = tee1_clone.request_pad_simple("src_%u").expect("tee1 src_1"); - let vmaf_sink_1 = vmaf_clone.request_pad_simple("sink_1").expect("vmaf sink_1"); - tee1_src_1.link(&queue_vmaf_1_clone.static_pad("sink").unwrap()).expect("tee1.src_1 -> queue_vmaf_1"); - queue_vmaf_1_clone.static_pad("src").unwrap().link(&vmaf_sink_1).expect("queue_vmaf_1.src -> vmaf.sink_1"); + let tee1_src_1 = + tee1_clone.request_pad_simple("src_%u").expect("tee1 src_1"); + let vmaf_sink_1 = vmaf_clone + .request_pad_simple("sink_1") + .expect("vmaf sink_1"); + tee1_src_1 + .link(&queue_vmaf_1_clone.static_pad("sink").unwrap()) + .expect("tee1.src_1 -> queue_vmaf_1"); + queue_vmaf_1_clone + .static_pad("src") + .unwrap() + .link(&vmaf_sink_1) + .expect("queue_vmaf_1.src -> vmaf.sink_1"); } } } @@ -712,12 +906,9 @@ impl ObjectImpl for EncoderStats { match pspec.name() { "encoder" => { if let Ok(Some(enc_obj)) = value.get::>() { - let factory = enc_obj - .factory() - .expect("Element should have a factory"); + let factory = enc_obj.factory().expect("Element should have a factory"); - if !factory.has_type(gst::ElementFactoryType::VIDEO_ENCODER) - { + if !factory.has_type(gst::ElementFactoryType::VIDEO_ENCODER) { gst::error!(CAT, "The element is not a video encoder"); panic!("The element is not a video encoder"); } @@ -735,7 +926,11 @@ impl ObjectImpl for EncoderStats { "vmaf-stats" => { if let Ok(vmaf_stats) = value.get::() { if vmaf_stats && !self.vmaf_available { - gst::warning!(CAT, imp = self, "Cannot enable VMAF stats: vmaf element not available"); + gst::warning!( + CAT, + imp = self, + "Cannot enable VMAF stats: vmaf element not available" + ); return; } let mut vmaf_stats_guard = self.vmaf_stats.lock().unwrap(); @@ -779,8 +974,7 @@ impl ElementImpl for EncoderStats { fn pad_templates() -> &'static [gst::PadTemplate] { static PAD_TEMPLATES: LazyLock> = LazyLock::new(|| { - let sink_caps = gst_video::VideoCapsBuilder::new() - .build(); + let sink_caps = gst_video::VideoCapsBuilder::new().build(); let src_caps = gst::Caps::new_any(); let video_src_pad_template = gst::PadTemplate::new( "src", @@ -804,7 +998,11 @@ impl ElementImpl for EncoderStats { ) .unwrap(); - vec![video_src_pad_template, video_sink_pad_template, request_src_pad_template] + vec![ + video_src_pad_template, + video_sink_pad_template, + request_src_pad_template, + ] }); PAD_TEMPLATES.as_ref() @@ -818,7 +1016,11 @@ impl ElementImpl for EncoderStats { ) -> Option { // Only allow request pads before ReadyToPaused transition if self.obj().current_state() >= gst::State::Paused { - gst::warning!(CAT, imp = self, "Cannot request pad after ReadyToPaused transition"); + gst::warning!( + CAT, + imp = self, + "Cannot request pad after ReadyToPaused transition" + ); return None; } @@ -829,7 +1031,11 @@ impl ElementImpl for EncoderStats { }; if !vmaf_enabled { - gst::warning!(CAT, imp = self, "Cannot request decoder pad when VMAF stats are disabled"); + gst::warning!( + CAT, + imp = self, + "Cannot request decoder pad when VMAF stats are disabled" + ); return None; } diff --git a/video/stats/src/lib.rs b/video/stats/src/lib.rs index 57aee9064..a66668f5e 100644 --- a/video/stats/src/lib.rs +++ b/video/stats/src/lib.rs @@ -9,10 +9,10 @@ use gst::glib; -mod videoencoderstats; -pub mod videoencoderstatsmeta; mod comparemixer; mod encoderstats; +mod videoencoderstats; +pub mod videoencoderstatsmeta; fn plugin_init(plugin: &gst::Plugin) -> Result<(), glib::BoolError> { comparemixer::register(plugin)?; diff --git a/video/stats/src/videoencoderstats.rs b/video/stats/src/videoencoderstats.rs index 33f96c633..d53b0b1c7 100644 --- a/video/stats/src/videoencoderstats.rs +++ b/video/stats/src/videoencoderstats.rs @@ -8,10 +8,10 @@ // SPDX-License-Identifier: MPL-2.0 use std::collections::VecDeque; -use std::time::Instant; -use std::time::Duration; use std::fmt; use std::sync::LazyLock; +use std::time::Duration; +use std::time::Instant; use gst::ffi::GstClockTime; @@ -75,7 +75,11 @@ impl VideoEncoderStats { self.max_buffers_inside = self.time_last_buffers.len(); } self.current_buffers_inside_seen = self.time_last_buffers.len(); - gst::log!(CAT, "Current buffers length {}", self.current_buffers_inside_seen); + gst::log!( + CAT, + "Current buffers length {}", + self.current_buffers_inside_seen + ); } pub fn buffer_out(&mut self) { @@ -104,66 +108,44 @@ impl VideoEncoderStats { impl fmt::Display for VideoEncoderStats { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - if self.framerate.unwrap().denom() != 1 && self.framerate.unwrap().denom() != 1001 { + if self.framerate.unwrap().denom() != 1 && self.framerate.unwrap().denom() != 1001 { unimplemented!(); } - writeln!( - f, - "Encoder: {}", - &self.name - )?; + writeln!(f, "Encoder: {}", &self.name)?; writeln!( f, "Output size: {} KB", self.num_bytes / 1000, // Convert to KB )?; - writeln!( - f, - "Max. Buffers inside: {}", - self.max_buffers_inside - )?; + writeln!(f, "Max. Buffers inside: {}", self.max_buffers_inside)?; writeln!( f, "Current buffers inside: {}", self.current_buffers_inside_seen )?; if let Some(num_buffers_first_output) = self.num_buffers_first_output { - writeln!( - f, - "Num. Buffers first output: {}", - num_buffers_first_output - )?; + writeln!(f, "Num. Buffers first output: {}", num_buffers_first_output)?; } else { - writeln!( - f, - "Num. Buffers first output: No first output yet" - )?; + writeln!(f, "Num. Buffers first output: No first output yet")?; } let framerate = self.framerate.unwrap(); - let total_time_secs = self.num_buffers as f64 * framerate.denom() as f64 / framerate.numer() as f64; + let total_time_secs = + self.num_buffers as f64 * framerate.denom() as f64 / framerate.numer() as f64; let bitrate = if total_time_secs > 0.0 { (self.num_bytes as f64 * 8.0) / total_time_secs } else { 0.0 }; - let bitrate_str = bitrate/1000.0; // Convert to kbps + let bitrate_str = bitrate / 1000.0; // Convert to kbps writeln!(f, "Bitrate: {:.3} kbps", bitrate_str)?; let avg_processing_time = self.avg_processing_time().as_millis(); - writeln!( - f, - "Processing time: {:.2} ms", - avg_processing_time - )?; + writeln!(f, "Processing time: {:.2} ms", avg_processing_time)?; let max_processing_time = self.max_processing_time.as_millis(); - writeln!( - f, - "Max processing time: {:.2} ms", - max_processing_time - )?; + writeln!(f, "Max processing time: {:.2} ms", max_processing_time)?; let cpu_time = self.threads_utime + self.threads_stime; #[cfg(target_os = "linux")] @@ -171,11 +153,7 @@ impl fmt::Display for VideoEncoderStats { let ticks_per_second = procfs::ticks_per_second() as u64; cpu_time as f64 / ticks_per_second as f64 }; - writeln!( - f, - "CPU: {} s", - cpu_time_seconds - )?; + writeln!(f, "CPU: {} s", cpu_time_seconds)?; let vmaf_score_str = match self.vmaf_score { Some(score) => format!("{:.3}", score), @@ -186,11 +164,7 @@ impl fmt::Display for VideoEncoderStats { let pre_encode_time = &self.pre_encode_time; let post_encode_time = &self.post_encode_time; let encode_latency = (*post_encode_time as f64 - *pre_encode_time as f64) / 1_000_000.0; - writeln!( - f, - "Encode latency: {:.3} ms", - encode_latency - ) + writeln!(f, "Encode latency: {:.3} ms", encode_latency) } } @@ -205,7 +179,14 @@ pub fn get_cpu_usage(name: String) -> (u64, u64) { for thread in process.tasks().unwrap().flatten() { let stat = thread.stat().unwrap(); if stat.comm.contains(&name) { - gst::log!(CAT, "Thread: {}, Comm: {}, Utime: {}, Stime: {}", thread.tid, stat.comm, stat.utime, stat.stime); + gst::log!( + CAT, + "Thread: {}, Comm: {}, Utime: {}, Stime: {}", + thread.tid, + stat.comm, + stat.utime, + stat.stime + ); total_utime += stat.utime; total_stime += stat.stime; } diff --git a/video/stats/src/videoencoderstatsmeta.rs b/video/stats/src/videoencoderstatsmeta.rs index 4abb39045..84a53e04e 100644 --- a/video/stats/src/videoencoderstatsmeta.rs +++ b/video/stats/src/videoencoderstatsmeta.rs @@ -25,8 +25,7 @@ impl VideoEncoderStatsMeta { stats: VideoEncoderStats, ) -> gst::MetaRefMut<'_, Self, gst::meta::Standalone> { unsafe { - let mut params = - mem::ManuallyDrop::new(imp::VideoEncoderStatsMetaParams { stats }); + let mut params = mem::ManuallyDrop::new(imp::VideoEncoderStatsMetaParams { stats }); let meta = gst::ffi::gst_buffer_add_meta( buffer.as_mut_ptr(), @@ -57,8 +56,7 @@ unsafe impl MetaAPI for VideoEncoderStatsMeta { impl fmt::Debug for VideoEncoderStatsMeta { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.debug_struct("VideoEncoderStatsMeta") - .finish() + f.debug_struct("VideoEncoderStatsMeta").finish() } } @@ -130,10 +128,7 @@ mod imp { if dest.meta::().is_some() { return true.into_glib(); } - super::VideoEncoderStatsMeta::add( - dest, - meta.stats.clone(), - ); + super::VideoEncoderStatsMeta::add(dest, meta.stats.clone()); true.into_glib() }