1 // Copyright (C) 2020 Julien Bardagi <julien.bardagi@gmail.com> 2 // 3 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or 4 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license 5 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your 6 // option. This file may not be copied, modified, or distributed 7 // except according to those terms. 8 9 use gst::glib; 10 use gst::prelude::*; 11 use gst::subclass::prelude::*; 12 use gst::{gst_debug, gst_info}; 13 use gst_base::subclass::prelude::*; 14 15 use atomic_refcell::AtomicRefCell; 16 use std::i32; 17 use std::sync::Mutex; 18 19 use once_cell::sync::Lazy; 20 use std::convert::TryInto; 21 22 use super::super::hsvutils; 23 24 // Default values of properties 25 const DEFAULT_HUE_SHIFT: f32 = 0.0; 26 const DEFAULT_SATURATION_MUL: f32 = 1.0; 27 const DEFAULT_SATURATION_OFF: f32 = 0.0; 28 const DEFAULT_VALUE_MUL: f32 = 1.0; 29 const DEFAULT_VALUE_OFF: f32 = 0.0; 30 31 // Property value storage 32 #[derive(Debug, Clone, Copy)] 33 struct Settings { 34 hue_shift: f32, 35 saturation_mul: f32, 36 saturation_off: f32, 37 value_mul: f32, 38 value_off: f32, 39 } 40 41 impl Default for Settings { default() -> Self42 fn default() -> Self { 43 Settings { 44 hue_shift: DEFAULT_HUE_SHIFT, 45 saturation_mul: DEFAULT_SATURATION_MUL, 46 saturation_off: DEFAULT_SATURATION_OFF, 47 value_mul: DEFAULT_VALUE_MUL, 48 value_off: DEFAULT_VALUE_OFF, 49 } 50 } 51 } 52 53 // Stream-specific state, i.e. video format configuration 54 struct State { 55 info: gst_video::VideoInfo, 56 } 57 58 // Struct containing all the element data 59 #[derive(Default)] 60 pub struct HsvFilter { 61 settings: Mutex<Settings>, 62 state: AtomicRefCell<Option<State>>, 63 } 64 65 static CAT: Lazy<gst::DebugCategory> = Lazy::new(|| { 66 gst::DebugCategory::new( 67 "hsvfilter", 68 gst::DebugColorFlags::empty(), 69 Some("Rust HSV transformation filter"), 70 ) 71 }); 72 73 #[glib::object_subclass] 74 impl ObjectSubclass for HsvFilter { 75 const NAME: &'static str = "HsvFilter"; 76 type Type = super::HsvFilter; 77 type ParentType = gst_base::BaseTransform; 78 } 79 80 impl HsvFilter { 81 #[inline] hsv_filter<CF, FF>( &self, frame: &mut gst_video::video_frame::VideoFrameRef<&mut gst::buffer::BufferRef>, to_hsv: CF, apply_filter: FF, ) where CF: Fn(&[u8]) -> [f32; 3], FF: Fn(&[f32; 3], &mut [u8]),82 fn hsv_filter<CF, FF>( 83 &self, 84 frame: &mut gst_video::video_frame::VideoFrameRef<&mut gst::buffer::BufferRef>, 85 to_hsv: CF, 86 apply_filter: FF, 87 ) where 88 CF: Fn(&[u8]) -> [f32; 3], 89 FF: Fn(&[f32; 3], &mut [u8]), 90 { 91 let settings = *self.settings.lock().unwrap(); 92 93 let width = frame.width() as usize; 94 let stride = frame.plane_stride()[0] as usize; 95 let nb_channels = frame.format_info().pixel_stride()[0] as usize; 96 let data = frame.plane_data_mut(0).unwrap(); 97 98 assert_eq!(data.len() % nb_channels, 0); 99 100 let line_bytes = width * nb_channels; 101 102 for line in data.chunks_exact_mut(stride) { 103 for p in line[..line_bytes].chunks_exact_mut(nb_channels) { 104 assert_eq!(p.len(), nb_channels); 105 106 let mut hsv = to_hsv(p); 107 108 hsv[0] = (hsv[0] + settings.hue_shift) % 360.0; 109 if hsv[0] < 0.0 { 110 hsv[0] += 360.0; 111 } 112 hsv[1] = hsvutils::Clamp::clamp( 113 settings.saturation_mul * hsv[1] + settings.saturation_off, 114 0.0, 115 1.0, 116 ); 117 hsv[2] = hsvutils::Clamp::clamp( 118 settings.value_mul * hsv[2] + settings.value_off, 119 0.0, 120 1.0, 121 ); 122 123 apply_filter(&hsv, p); 124 } 125 } 126 } 127 } 128 129 impl ObjectImpl for HsvFilter { properties() -> &'static [glib::ParamSpec]130 fn properties() -> &'static [glib::ParamSpec] { 131 static PROPERTIES: Lazy<Vec<glib::ParamSpec>> = Lazy::new(|| { 132 vec![ 133 glib::ParamSpec::new_float( 134 "hue-shift", 135 "Hue shift", 136 "Hue shifting in degrees", 137 f32::MIN, 138 f32::MAX, 139 DEFAULT_HUE_SHIFT, 140 glib::ParamFlags::READWRITE | gst::PARAM_FLAG_MUTABLE_PLAYING, 141 ), 142 glib::ParamSpec::new_float( 143 "saturation-mul", 144 "Saturation multiplier", 145 "Saturation multiplier to apply to the saturation value (before offset)", 146 f32::MIN, 147 f32::MAX, 148 DEFAULT_SATURATION_MUL, 149 glib::ParamFlags::READWRITE | gst::PARAM_FLAG_MUTABLE_PLAYING, 150 ), 151 glib::ParamSpec::new_float( 152 "saturation-off", 153 "Saturation offset", 154 "Saturation offset to add to the saturation value (after multiplier)", 155 f32::MIN, 156 f32::MAX, 157 DEFAULT_SATURATION_OFF, 158 glib::ParamFlags::READWRITE | gst::PARAM_FLAG_MUTABLE_PLAYING, 159 ), 160 glib::ParamSpec::new_float( 161 "value-mul", 162 "Value multiplier", 163 "Value multiplier to apply to the value (before offset)", 164 f32::MIN, 165 f32::MAX, 166 DEFAULT_VALUE_MUL, 167 glib::ParamFlags::READWRITE | gst::PARAM_FLAG_MUTABLE_PLAYING, 168 ), 169 glib::ParamSpec::new_float( 170 "value-off", 171 "Value offset", 172 "Value offset to add to the value (after multiplier)", 173 f32::MIN, 174 f32::MAX, 175 DEFAULT_VALUE_OFF, 176 glib::ParamFlags::READWRITE | gst::PARAM_FLAG_MUTABLE_PLAYING, 177 ), 178 ] 179 }); 180 181 PROPERTIES.as_ref() 182 } 183 set_property( &self, obj: &Self::Type, _id: usize, value: &glib::Value, pspec: &glib::ParamSpec, )184 fn set_property( 185 &self, 186 obj: &Self::Type, 187 _id: usize, 188 value: &glib::Value, 189 pspec: &glib::ParamSpec, 190 ) { 191 match pspec.name() { 192 "hue-shift" => { 193 let mut settings = self.settings.lock().unwrap(); 194 let hue_shift = value.get().expect("type checked upstream"); 195 gst_info!( 196 CAT, 197 obj: obj, 198 "Changing hue-shift from {} to {}", 199 settings.hue_shift, 200 hue_shift 201 ); 202 settings.hue_shift = hue_shift; 203 } 204 "saturation-mul" => { 205 let mut settings = self.settings.lock().unwrap(); 206 let saturation_mul = value.get().expect("type checked upstream"); 207 gst_info!( 208 CAT, 209 obj: obj, 210 "Changing saturation-mul from {} to {}", 211 settings.saturation_mul, 212 saturation_mul 213 ); 214 settings.saturation_mul = saturation_mul; 215 } 216 "saturation-off" => { 217 let mut settings = self.settings.lock().unwrap(); 218 let saturation_off = value.get().expect("type checked upstream"); 219 gst_info!( 220 CAT, 221 obj: obj, 222 "Changing saturation-off from {} to {}", 223 settings.saturation_off, 224 saturation_off 225 ); 226 settings.saturation_off = saturation_off; 227 } 228 "value-mul" => { 229 let mut settings = self.settings.lock().unwrap(); 230 let value_mul = value.get().expect("type checked upstream"); 231 gst_info!( 232 CAT, 233 obj: obj, 234 "Changing value-mul from {} to {}", 235 settings.value_mul, 236 value_mul 237 ); 238 settings.value_mul = value_mul; 239 } 240 "value-off" => { 241 let mut settings = self.settings.lock().unwrap(); 242 let value_off = value.get().expect("type checked upstream"); 243 gst_info!( 244 CAT, 245 obj: obj, 246 "Changing value-off from {} to {}", 247 settings.value_off, 248 value_off 249 ); 250 settings.value_off = value_off; 251 } 252 _ => unimplemented!(), 253 } 254 } 255 256 // Called whenever a value of a property is read. It can be called 257 // at any time from any thread. property(&self, _obj: &Self::Type, _id: usize, pspec: &glib::ParamSpec) -> glib::Value258 fn property(&self, _obj: &Self::Type, _id: usize, pspec: &glib::ParamSpec) -> glib::Value { 259 match pspec.name() { 260 "hue-shift" => { 261 let settings = self.settings.lock().unwrap(); 262 settings.hue_shift.to_value() 263 } 264 "saturation-mul" => { 265 let settings = self.settings.lock().unwrap(); 266 settings.saturation_mul.to_value() 267 } 268 "saturation-off" => { 269 let settings = self.settings.lock().unwrap(); 270 settings.saturation_off.to_value() 271 } 272 "value-mul" => { 273 let settings = self.settings.lock().unwrap(); 274 settings.value_mul.to_value() 275 } 276 "value-off" => { 277 let settings = self.settings.lock().unwrap(); 278 settings.value_off.to_value() 279 } 280 _ => unimplemented!(), 281 } 282 } 283 } 284 285 impl ElementImpl for HsvFilter { metadata() -> Option<&'static gst::subclass::ElementMetadata>286 fn metadata() -> Option<&'static gst::subclass::ElementMetadata> { 287 static ELEMENT_METADATA: Lazy<gst::subclass::ElementMetadata> = Lazy::new(|| { 288 gst::subclass::ElementMetadata::new( 289 "HSV filter", 290 "Filter/Effect/Converter/Video", 291 "Works within the HSV colorspace to apply tranformations to incoming frames", 292 "Julien Bardagi <julien.bardagi@gmail.com>", 293 ) 294 }); 295 296 Some(&*ELEMENT_METADATA) 297 } 298 pad_templates() -> &'static [gst::PadTemplate]299 fn pad_templates() -> &'static [gst::PadTemplate] { 300 static PAD_TEMPLATES: Lazy<Vec<gst::PadTemplate>> = Lazy::new(|| { 301 // src pad capabilities 302 let caps = gst::Caps::new_simple( 303 "video/x-raw", 304 &[ 305 ( 306 "format", 307 &gst::List::new(&[ 308 &gst_video::VideoFormat::Rgbx.to_str(), 309 &gst_video::VideoFormat::Xrgb.to_str(), 310 &gst_video::VideoFormat::Bgrx.to_str(), 311 &gst_video::VideoFormat::Xbgr.to_str(), 312 &gst_video::VideoFormat::Rgba.to_str(), 313 &gst_video::VideoFormat::Argb.to_str(), 314 &gst_video::VideoFormat::Bgra.to_str(), 315 &gst_video::VideoFormat::Abgr.to_str(), 316 &gst_video::VideoFormat::Rgb.to_str(), 317 &gst_video::VideoFormat::Bgr.to_str(), 318 ]), 319 ), 320 ("width", &gst::IntRange::<i32>::new(0, i32::MAX)), 321 ("height", &gst::IntRange::<i32>::new(0, i32::MAX)), 322 ( 323 "framerate", 324 &gst::FractionRange::new( 325 gst::Fraction::new(0, 1), 326 gst::Fraction::new(i32::MAX, 1), 327 ), 328 ), 329 ], 330 ); 331 332 let src_pad_template = gst::PadTemplate::new( 333 "src", 334 gst::PadDirection::Src, 335 gst::PadPresence::Always, 336 &caps, 337 ) 338 .unwrap(); 339 340 let sink_pad_template = gst::PadTemplate::new( 341 "sink", 342 gst::PadDirection::Sink, 343 gst::PadPresence::Always, 344 &caps, 345 ) 346 .unwrap(); 347 348 vec![src_pad_template, sink_pad_template] 349 }); 350 351 PAD_TEMPLATES.as_ref() 352 } 353 } 354 355 impl BaseTransformImpl for HsvFilter { 356 const MODE: gst_base::subclass::BaseTransformMode = 357 gst_base::subclass::BaseTransformMode::AlwaysInPlace; 358 const PASSTHROUGH_ON_SAME_CAPS: bool = false; 359 const TRANSFORM_IP_ON_PASSTHROUGH: bool = false; 360 unit_size(&self, _element: &Self::Type, caps: &gst::Caps) -> Option<usize>361 fn unit_size(&self, _element: &Self::Type, caps: &gst::Caps) -> Option<usize> { 362 gst_video::VideoInfo::from_caps(caps) 363 .map(|info| info.size()) 364 .ok() 365 } 366 set_caps( &self, element: &Self::Type, incaps: &gst::Caps, outcaps: &gst::Caps, ) -> Result<(), gst::LoggableError>367 fn set_caps( 368 &self, 369 element: &Self::Type, 370 incaps: &gst::Caps, 371 outcaps: &gst::Caps, 372 ) -> Result<(), gst::LoggableError> { 373 let _in_info = match gst_video::VideoInfo::from_caps(incaps) { 374 Err(_) => return Err(gst::loggable_error!(CAT, "Failed to parse input caps")), 375 Ok(info) => info, 376 }; 377 let out_info = match gst_video::VideoInfo::from_caps(outcaps) { 378 Err(_) => return Err(gst::loggable_error!(CAT, "Failed to parse output caps")), 379 Ok(info) => info, 380 }; 381 382 gst_debug!( 383 CAT, 384 obj: element, 385 "Configured for caps {} to {}", 386 incaps, 387 outcaps 388 ); 389 390 *self.state.borrow_mut() = Some(State { info: out_info }); 391 392 Ok(()) 393 } 394 stop(&self, element: &Self::Type) -> Result<(), gst::ErrorMessage>395 fn stop(&self, element: &Self::Type) -> Result<(), gst::ErrorMessage> { 396 // Drop state 397 *self.state.borrow_mut() = None; 398 399 gst_info!(CAT, obj: element, "Stopped"); 400 401 Ok(()) 402 } 403 transform_ip( &self, element: &Self::Type, buf: &mut gst::BufferRef, ) -> Result<gst::FlowSuccess, gst::FlowError>404 fn transform_ip( 405 &self, 406 element: &Self::Type, 407 buf: &mut gst::BufferRef, 408 ) -> Result<gst::FlowSuccess, gst::FlowError> { 409 let mut state_guard = self.state.borrow_mut(); 410 let state = state_guard.as_mut().ok_or(gst::FlowError::NotNegotiated)?; 411 412 let mut frame = gst_video::VideoFrameRef::from_buffer_ref_writable(buf, &state.info) 413 .map_err(|_| { 414 gst::element_error!( 415 element, 416 gst::CoreError::Failed, 417 ["Failed to map output buffer writable"] 418 ); 419 gst::FlowError::Error 420 })?; 421 422 match state.info.format() { 423 gst_video::VideoFormat::Rgbx 424 | gst_video::VideoFormat::Rgba 425 | gst_video::VideoFormat::Rgb => { 426 self.hsv_filter( 427 &mut frame, 428 |p| hsvutils::from_rgb(p[..3].try_into().expect("slice with incorrect length")), 429 |hsv, p| { 430 p[..3].copy_from_slice(&hsvutils::to_rgb(hsv)); 431 }, 432 ); 433 } 434 gst_video::VideoFormat::Xrgb | gst_video::VideoFormat::Argb => { 435 self.hsv_filter( 436 &mut frame, 437 |p| { 438 hsvutils::from_rgb(p[1..4].try_into().expect("slice with incorrect length")) 439 }, 440 |hsv, p| { 441 p[1..4].copy_from_slice(&hsvutils::to_rgb(hsv)); 442 }, 443 ); 444 } 445 gst_video::VideoFormat::Bgrx 446 | gst_video::VideoFormat::Bgra 447 | gst_video::VideoFormat::Bgr => { 448 self.hsv_filter( 449 &mut frame, 450 |p| hsvutils::from_bgr(p[..3].try_into().expect("slice with incorrect length")), 451 |hsv, p| { 452 p[..3].copy_from_slice(&hsvutils::to_bgr(hsv)); 453 }, 454 ); 455 } 456 gst_video::VideoFormat::Xbgr | gst_video::VideoFormat::Abgr => { 457 self.hsv_filter( 458 &mut frame, 459 |p| { 460 hsvutils::from_bgr(p[1..4].try_into().expect("slice with incorrect length")) 461 }, 462 |hsv, p| { 463 p[1..4].copy_from_slice(&hsvutils::to_bgr(hsv)); 464 }, 465 ); 466 } 467 _ => unreachable!(), 468 } 469 470 Ok(gst::FlowSuccess::Ok) 471 } 472 } 473