From f0e7c6ccd4ed86aff622d0b0f90b4ef197ad63b6 Mon Sep 17 00:00:00 2001 From: Craig McLure Date: Sat, 25 Jul 2026 08:35:31 +0100 Subject: [PATCH] feat: incrementally render encoder layouts and also render them for device plugin devices (#385) Co-authored-by: nekename <63245705+nekename@users.noreply.github.com> --- src-tauri/Cargo.lock | 2 +- src-tauri/Cargo.toml | 2 +- src-tauri/src/elgato.rs | 79 +------------- src-tauri/src/encoder_layouts.rs | 128 +++++++++++++++++++++++ src-tauri/src/events/inbound/states.rs | 105 +------------------ src-tauri/src/events/outbound/devices.rs | 22 ++++ src-tauri/src/main.rs | 1 + src-tauri/src/shared.rs | 14 +-- 8 files changed, 166 insertions(+), 187 deletions(-) create mode 100644 src-tauri/src/encoder_layouts.rs diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index 286bd95..1e3a06f 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -5394,7 +5394,7 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "streamdeck-strip-render" version = "0.1.0" -source = "git+https://github.com/FrostyCoolSlug/streamdeck-strip-render?rev=5a18406c#5a18406c3f7560f4f4220756e7da459d76559946" +source = "git+https://github.com/FrostyCoolSlug/streamdeck-strip-render?rev=23fa1399#23fa1399d110f500db295a37ea76c0eb75d067d5" dependencies = [ "ab_glyph", "anyhow", diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 349fd9a..d261931 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -23,7 +23,7 @@ tiny_http = "0.12" elgato-streamdeck = { version = "0.12", default-features = false, features = ["async"] } hidapi = "2.6" image = { version = "0.25", default-features = false, features = ["bmp", "jpeg"] } -streamdeck-strip-render = { git = "https://github.com/FrostyCoolSlug/streamdeck-strip-render", rev = "5a18406c" } +streamdeck-strip-render = { git = "https://github.com/FrostyCoolSlug/streamdeck-strip-render", rev = "23fa1399" } # Smaller utility libraries dashmap = { version = "6.1", features = ["serde"] } diff --git a/src-tauri/src/elgato.rs b/src-tauri/src/elgato.rs index b09de8c..a41d32d 100644 --- a/src-tauri/src/elgato.rs +++ b/src-tauri/src/elgato.rs @@ -1,6 +1,5 @@ +use crate::encoder_layouts::generate_encoder_image; use crate::events::inbound; -use crate::shared::{ActionInstance, Encoder, config_dir}; -use crate::store::profiles::{acquire_locks, get_slot}; use std::collections::HashMap; use std::sync::Arc; @@ -12,8 +11,7 @@ use elgato_streamdeck::{ images::{ImageRect, convert_image_with_format_async}, info::Kind, }; -use image::{DynamicImage, GenericImageView as _}; -use serde_json::Value; +use image::GenericImageView as _; use tokio::sync::RwLock; static ELGATO_DEVICES: LazyLock>> = LazyLock::new(|| RwLock::new(HashMap::new())); @@ -28,57 +26,6 @@ fn extract_average_colour(img: &image::DynamicImage) -> (u8, u8, u8) { ((r_sum / count) as u8, (g_sum / count) as u8, (b_sum / count) as u8) } -fn get_encoder_image(encoder: &Encoder, instance: &ActionInstance) -> Result { - // Clone the layout so we can mutate it for rendering without persisting - let mut layout = encoder.layout_parsed.clone(); - - if layout.is_null() { - // Something's gone horribly wrong here; we should have a layout. Render a blank image. - return Ok(DynamicImage::new_rgb8(200, 100)); - } - - let path = config_dir().join("plugins").join(&instance.action.plugin); - - // We need to validate whether title text and icon images are defined; if not, pull them from the state/action - if let Some(items_array) = layout.get_mut("items").and_then(Value::as_array_mut) { - // If the title is missing, provide it from the state/action - if let Some(title_item) = items_array.iter_mut().find(|item| item.get("key").and_then(Value::as_str) == Some("title")) { - let title_value = title_item.get("value").and_then(Value::as_str).unwrap_or("").trim(); - - if title_value.is_empty() { - // Try to pull the title from the state - let state_text = instance.states.get(instance.current_state as usize).and_then(|s| { - let t = s.text.trim(); - if t.is_empty() { None } else { Some(t) } - }); - - // If the state title is empty, fall back to the action name - let title = state_text.unwrap_or(instance.action.name.as_str()); - title_item["value"] = Value::String(title.to_string()); - } - } - - // If the icon is missing, provide it from the state/action - if let Some(icon_item) = items_array.iter_mut().find(|item| item.get("key").and_then(Value::as_str) == Some("icon")) { - let icon_empty = icon_item.get("value").and_then(Value::as_str).is_none_or(str::is_empty); - if icon_empty { - let icon = instance - .states - .get(instance.current_state as usize) - .map(|state| &state.image) - .filter(|image| !image.is_empty()) - .unwrap_or(&instance.action.icon); - - if !icon.is_empty() { - icon_item["value"] = Value::String(icon.clone()); - } - } - } - } - - streamdeck_strip_render::render_to_image(layout, &path, None) -} - pub async fn update_image(context: &crate::shared::Context, image: Option<&str>) -> Result<(), anyhow::Error> { if let Some(device) = ELGATO_DEVICES.read().await.get(&context.device) { let kind = device.kind(); @@ -92,26 +39,8 @@ pub async fn update_image(context: &crate::shared::Context, image: Option<&str>) let data = image.split_once(',').unwrap().1; let bytes = base64::engine::general_purpose::STANDARD.decode(data)?; if context.controller == "Encoder" { - let locks = acquire_locks().await; - let slot = get_slot(context, &locks).await?.clone(); - drop(locks); - if let Some(instance) = slot - && let Some(encoder) = &instance.action.encoder - { - let img = get_encoder_image(encoder, &instance)?; - device.write_lcd(context.position as u16 * 200, 0, &ImageRect::from_image_async(img.clone())?).await?; - } else { - // If we get here, this is either an Encoder action that doesn't have an Encoder config in the manifest, or we were - // unable to locate the instance for this action. This realistically shouldn't happen, but if it does, we'll fall back - // to rendering what was provided to this function call. - device - .write_lcd( - (context.position as u16 * 200) + 64, - 14, - &ImageRect::from_image_async(image::load_from_memory(&bytes)?.resize(72, 72, image::imageops::FilterType::Nearest))?, - ) - .await?; - } + let img = generate_encoder_image(context, &bytes).await?; + device.write_lcd(context.position as u16 * 200, 0, &ImageRect::from_image(img)?).await?; } else if context.controller == "Infobar" { let img = image::load_from_memory(&bytes)?; let Some(format) = device.kind().lcd_image_format() else { diff --git a/src-tauri/src/encoder_layouts.rs b/src-tauri/src/encoder_layouts.rs new file mode 100644 index 0000000..575998f --- /dev/null +++ b/src-tauri/src/encoder_layouts.rs @@ -0,0 +1,128 @@ +use crate::shared::{ActionInstance, Encoder, config_dir}; +use crate::store::profiles::{acquire_locks_mut, get_slot_mut}; + +use std::path::{Path, PathBuf}; + +use anyhow::Context; +use image::imageops::overlay; +use image::{DynamicImage, Rgba, RgbaImage}; +use log::{trace, warn}; +use serde_json::{Map, Value}; +use streamdeck_strip_render::layout::{LayoutItem, PixmapSource}; + +pub async fn generate_encoder_image(context: &crate::shared::Context, fallback: &[u8]) -> Result { + let mut locks = acquire_locks_mut().await; + let slot = get_slot_mut(context, &mut locks).await?; + + // We need to borrow the encoder instance to render the image, so we'll take it, then give it back when we're done. + let img = if let Some(instance) = slot { + if let Some(mut encoder) = instance.action.encoder.take() { + let result = get_encoder_image(&mut encoder, instance).context("Failed to render encoder image"); + instance.action.encoder = Some(encoder); + Some(result?) + } else { + None + } + } else { + None + }; + drop(locks); + + match img { + Some(img) => Ok(img), + None => { + // If we get here, this is either an Encoder action that doesn't have an Encoder config in the manifest, or we were + // unable to locate the instance for this action. This realistically shouldn't happen, but if it does, we'll fall back + // to rendering what was provided to this function call. + trace!("No encoder instance / config found for action; using fallback image"); + + let mut fallback_canvas = RgbaImage::from_pixel(200, 100, Rgba([0, 0, 0, 255])); + let fallback_img = image::load_from_memory(fallback) + .context("Failed to decode fallback image")? + .resize(72, 72, image::imageops::FilterType::Nearest); + + overlay(&mut fallback_canvas, &fallback_img.to_rgba8(), 64, 14); + + Ok(DynamicImage::ImageRgba8(fallback_canvas)) + } + } +} + +fn get_encoder_image(encoder: &mut Encoder, instance: &ActionInstance) -> Result { + let Some(ref mut renderer) = encoder.layout_parsed else { + // Something's gone horribly wrong here; we should have a layout. Render a blank image. + return Ok(DynamicImage::new_rgb8(200, 100)); + }; + + let path = config_dir().join("plugins").join(&instance.action.plugin); + let path_canonical = path.canonicalize()?; + + let state_idx = instance.current_state as usize; + + // Override the title if it's set in the state. + let override_title = { + let t = instance.states[state_idx].text.trim(); + (!t.is_empty()).then(|| t.to_string()) + }; + + // Similar to the title, except for the rendered icon (we check whether the state matches the default, and if it doesn't, we use it). + let state_image = &instance.states[state_idx].image.trim(); + + let override_icon = { + let default_image = &instance.action.states[state_idx].image.trim(); + (!state_image.is_empty() && state_image != default_image).then(|| state_image.to_string()) + }; + + // We need to do small item corrections here. + let mut feedback = Map::new(); + let layout = renderer.layout(); + + // If the layout doesn't have a title, fall back to the action title. + if let Some(title) = layout.item("title") + && let LayoutItem::Text(title) = title + && title.value.as_deref().is_none_or(str::is_empty) + { + feedback.insert("title".to_string(), Value::String(instance.action.name.clone())); + } + + // If the layout doesn't have an icon, fall back to the state image. + if let Some(icon) = layout.item("icon") + && let LayoutItem::Pixmap(icon) = icon + && icon.value == PixmapSource::None + { + feedback.insert("icon".to_string(), Value::String(state_image.to_string())); + } + + // Expand and sandbox every pixmap item's relative file path. + for item in &layout.items { + let LayoutItem::Pixmap(p) = item else { continue }; + let PixmapSource::File(v) = &p.value else { continue }; + if v.is_empty() { + continue; + } + + let resolved = { + // We need to make sure this path isn't already canonical. + let candidate = if Path::new(v).is_absolute() { PathBuf::from(v) } else { path.join(v) }; + match candidate.canonicalize() { + Ok(resolved) if resolved.starts_with(&path_canonical) => resolved.to_string_lossy().into_owned(), + Ok(resolved) => { + warn!("Attempted to load image outside of base path: {resolved:?}"); + String::new() + } + Err(_) => { + warn!("Unable to canonicalize path: {candidate:?}"); + String::new() + } + } + }; + + feedback.insert(item.key().to_string(), Value::String(resolved)); + } + + // Send changes to the renderer; note that the overrides are NOOP if they haven't changed. + renderer.set_icon_override(override_icon); + renderer.set_title_override(override_title); + renderer.set_feedback(Value::Object(feedback))?; + Ok(DynamicImage::ImageRgba8(renderer.get_image())) +} diff --git a/src-tauri/src/events/inbound/states.rs b/src-tauri/src/events/inbound/states.rs index 71281db..eee1feb 100644 --- a/src-tauri/src/events/inbound/states.rs +++ b/src-tauri/src/events/inbound/states.rs @@ -4,7 +4,6 @@ use crate::events::frontend::instances::update_state; use crate::store::profiles::{acquire_locks_mut, debounce_profile_save, get_instance_mut, save_profile}; use anyhow::bail; -use log::trace; use serde::Deserialize; use serde_json::Value; @@ -109,117 +108,17 @@ pub async fn set_image(mut event: ContextAndPayloadEvent) -> Re Ok(()) } -const COMMON_KEYS: &[&str] = &["enabled", "opacity", "background"]; -const BAR_KEYS: &[&str] = &["bar_bg_c", "bar_border_c", "bar_fill_c", "border_w", "range", "subtype", "value"]; pub async fn set_feedback(event: ContextAndPayloadEvent) -> Result<(), anyhow::Error> { let mut locks = acquire_locks_mut().await; if let Some(instance) = get_instance_mut(&event.context, &mut locks).await? && let Some(encoder) = &mut instance.action.encoder - && let Value::Object(map) = event.payload { - let layout = &mut encoder.layout_parsed; - if layout.is_null() { + let Some(layout) = &mut encoder.layout_parsed else { bail!("Layout is not loaded; cannot set feedback"); - } - - let Some(items_array) = layout.get_mut("items").and_then(Value::as_array_mut) else { - bail!("Layout has no items array"); }; - for (key, payload_value) in &map { - // Grab the item from the layout - let Some(item) = items_array.iter_mut().find(|item| { - matches!( - item.get("key").and_then(|v| v.as_str()), - Some(k) if k == key - ) - }) else { - trace!("setFeedback: no layout item found for key '{key}'"); - continue; - }; - - match payload_value { - // We have a direct value; find the key, and set it - Value::String(_) | Value::Number(_) => { - // Get the item type - let Some(item_type) = item.get("type").and_then(Value::as_str) else { - trace!("setFeedback: no type found for key '{key}'"); - continue; - }; - - match item_type { - "text" => { - // We need to map the value to a string - if let Value::Number(number) = payload_value { - item["value"] = Value::String(number.to_string()); - } else { - // Clone the string - item["value"] = payload_value.clone(); - } - } - - "bar" | "gbar" => { - if let Value::Number(value) = payload_value { - item["value"] = Value::Number(value.clone()); - } else if let Value::String(value) = payload_value { - if let Ok(value) = value.parse() { - item["value"] = Value::Number(value); - } else { - trace!("setFeedback: bar/gbar unexpected value for key '{key}' - {value}"); - } - } - } - - "pixmap" => { - // Update the pixmap value; this should already be a string - item["value"] = payload_value.clone(); - } - - // Ignore anything else - _ => { - trace!("setFeedback: unknown item type '{item_type}' for key '{key}'"); - } - } - } - - // We have an object, so we need to locate and map the change - Value::Object(obj) => { - // Get the item type - let Some(item_type) = item.get("type").and_then(Value::as_str) else { - trace!("setFeedback: missing or invalid 'type' field in item: {:?}", item); - continue; - }; - - // Get the valid keys for this item type - let type_keys: Vec<&str> = match item_type { - "text" => vec!["value", "color", "alignment", "font", "text-overflow"], - "pixmap" => vec!["value"], - "bar" => BAR_KEYS.to_vec(), - "gbar" => BAR_KEYS.iter().copied().chain(["bar_h"]).collect(), - unknown => { - trace!("setFeedback: unknown item type '{unknown}' for key '{key}'"); - continue; - } - }; - - // Add the common keys - let valid_keys: Vec<&str> = COMMON_KEYS.iter().copied().chain(type_keys).collect(); - - // Iterate over the values in the object - for (field, field_value) in obj { - if valid_keys.contains(&field.as_str()) { - item[field] = field_value.clone() - } - } - } - - _ => { - trace!("setFeedback: key '{key}' has unexpected payload type, ignoring"); - } - } - } - + layout.set_feedback(event.payload)?; update_state(crate::APP_HANDLE.get().unwrap(), instance.context.clone(), &mut locks).await?; } diff --git a/src-tauri/src/events/outbound/devices.rs b/src-tauri/src/events/outbound/devices.rs index 0172e0e..fe09883 100644 --- a/src-tauri/src/events/outbound/devices.rs +++ b/src-tauri/src/events/outbound/devices.rs @@ -1,8 +1,12 @@ use super::{send_to_all_plugins, send_to_plugin}; +use crate::encoder_layouts::generate_encoder_image; use crate::plugins::{DEVICE_NAMESPACES, info_param::DeviceInfo}; +use base64::Engine; +use image::ImageFormat; use serde::Serialize; +use std::io::Cursor; #[derive(Serialize)] #[allow(non_snake_case)] @@ -46,6 +50,11 @@ struct SetImageEvent { pub async fn update_image(context: crate::shared::Context, image: Option) -> Result<(), anyhow::Error> { if let Some(plugin) = DEVICE_NAMESPACES.read().await.get(&context.device[..2]) { + let image = match (context.controller.as_str(), image) { + ("Encoder", Some(img)) => Some(to_encoder_jpeg_data_uri(&context, &img).await?), + (_, img) => img, + }; + send_to_plugin( plugin, &SetImageEvent { @@ -64,6 +73,19 @@ pub async fn update_image(context: crate::shared::Context, image: Option Ok(()) } +async fn to_encoder_jpeg_data_uri(context: &crate::shared::Context, image: &str) -> Result { + let data = image.split_once(',').unwrap().1; + let bytes = base64::engine::general_purpose::STANDARD.decode(data)?; + + let img = generate_encoder_image(context, &bytes).await?; + + let mut buf = Vec::new(); + img.write_to(&mut Cursor::new(&mut buf), ImageFormat::Jpeg)?; + let encoded = base64::engine::general_purpose::STANDARD.encode(&buf); + + Ok(format!("data:image/jpeg;base64,{encoded}")) +} + pub async fn clear_screen(device: String) -> Result<(), anyhow::Error> { if let Some(plugin) = DEVICE_NAMESPACES.read().await.get(&device[..2]) { send_to_plugin( diff --git a/src-tauri/src/main.rs b/src-tauri/src/main.rs index 7e64140..e1bd314 100644 --- a/src-tauri/src/main.rs +++ b/src-tauri/src/main.rs @@ -4,6 +4,7 @@ mod application_watcher; mod device_sleep; mod elgato; +mod encoder_layouts; mod events; mod plugins; mod power_events; diff --git a/src-tauri/src/shared.rs b/src-tauri/src/shared.rs index c0603fd..f802d8a 100644 --- a/src-tauri/src/shared.rs +++ b/src-tauri/src/shared.rs @@ -8,6 +8,7 @@ use serde_inline_default::serde_inline_default; use anyhow::{Result, bail}; use dashmap::DashMap; +use streamdeck_strip_render::{get_incremental_renderer, strip_renderer::StripRenderer}; use tauri::Manager; use tokio::sync::RwLock; @@ -248,9 +249,8 @@ pub struct Encoder { pub layout: String, // Note: this is not a real manifest property; it is only used internally. - #[serde_inline_default(serde_json::Value::Null)] - #[serde(skip_serializing)] - pub layout_parsed: serde_json::Value, + #[serde(skip)] + pub layout_parsed: Option, } pub fn initialise_encoder_layout(action: &mut Action, layout: Option) -> Result<(), anyhow::Error> { @@ -274,20 +274,20 @@ pub fn initialise_encoder_layout(action: &mut Action, layout: Option) -> match layout_file.canonicalize() { Ok(resolved) if resolved.starts_with(&plugin_dir) => resolved.to_string_lossy().into_owned(), Ok(_) => { - encoder.layout_parsed = serde_json::Value::Null; + encoder.layout_parsed = None; bail!("Encoder layout path escapes plugin directory: {}", load_layout); } Err(error) => { - encoder.layout_parsed = serde_json::Value::Null; + encoder.layout_parsed = None; bail!("Failed to canonicalize encoder layout path {}: {}", load_layout, error); } } }; match load_encoder_layout(&layout) { - Ok(parsed) => encoder.layout_parsed = parsed, + Ok(parsed) => encoder.layout_parsed = Some(get_incremental_renderer(parsed, None)?), Err(error) => { - encoder.layout_parsed = serde_json::Value::Null; + encoder.layout_parsed = None; bail!("Failed to load encoder layout {}: {}", load_layout, error) } }