feat: incrementally render encoder layouts and also render them for device plugin devices (#385)

Co-authored-by: nekename <63245705+nekename@users.noreply.github.com>

src-tauri/Cargo.lock

@@ -5394,7 +5394,7 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596"
 [[package]]
 name = "streamdeck-strip-render"
 version = "0.1.0"
-source = "git+https://github.com/FrostyCoolSlug/streamdeck-strip-render?rev=5a18406c#5a18406c3f7560f4f4220756e7da459d76559946"
+source = "git+https://github.com/FrostyCoolSlug/streamdeck-strip-render?rev=23fa1399#23fa1399d110f500db295a37ea76c0eb75d067d5"
 dependencies = [
  "ab_glyph",
  "anyhow",

src-tauri/Cargo.toml

@@ -23,7 +23,7 @@ tiny_http = "0.12"
 elgato-streamdeck = { version = "0.12", default-features = false, features = ["async"] }
 hidapi = "2.6"
 image = { version = "0.25", default-features = false, features = ["bmp", "jpeg"] }
-streamdeck-strip-render = { git = "https://github.com/FrostyCoolSlug/streamdeck-strip-render", rev = "5a18406c" }
+streamdeck-strip-render = { git = "https://github.com/FrostyCoolSlug/streamdeck-strip-render", rev = "23fa1399" }
 
 # Smaller utility libraries
 dashmap = { version = "6.1", features = ["serde"] }

src-tauri/src/elgato.rs

@@ -1,6 +1,5 @@
+use crate::encoder_layouts::generate_encoder_image;
 use crate::events::inbound;
-use crate::shared::{ActionInstance, Encoder, config_dir};
-use crate::store::profiles::{acquire_locks, get_slot};
 
 use std::collections::HashMap;
 use std::sync::Arc;
@@ -12,8 +11,7 @@ use elgato_streamdeck::{
 	images::{ImageRect, convert_image_with_format_async},
 	info::Kind,
 };
-use image::{DynamicImage, GenericImageView as _};
-use serde_json::Value;
+use image::GenericImageView as _;
 use tokio::sync::RwLock;
 
 static ELGATO_DEVICES: LazyLock<RwLock<HashMap<String, AsyncStreamDeck>>> = LazyLock::new(|| RwLock::new(HashMap::new()));
@@ -28,57 +26,6 @@ fn extract_average_colour(img: &image::DynamicImage) -> (u8, u8, u8) {
 	((r_sum / count) as u8, (g_sum / count) as u8, (b_sum / count) as u8)
 }
 
-fn get_encoder_image(encoder: &Encoder, instance: &ActionInstance) -> Result<DynamicImage, anyhow::Error> {
-	// Clone the layout so we can mutate it for rendering without persisting
-	let mut layout = encoder.layout_parsed.clone();
-
-	if layout.is_null() {
-		// Something's gone horribly wrong here; we should have a layout. Render a blank image.
-		return Ok(DynamicImage::new_rgb8(200, 100));
-	}
-
-	let path = config_dir().join("plugins").join(&instance.action.plugin);
-
-	// We need to validate whether title text and icon images are defined; if not, pull them from the state/action
-	if let Some(items_array) = layout.get_mut("items").and_then(Value::as_array_mut) {
-		// If the title is missing, provide it from the state/action
-		if let Some(title_item) = items_array.iter_mut().find(|item| item.get("key").and_then(Value::as_str) == Some("title")) {
-			let title_value = title_item.get("value").and_then(Value::as_str).unwrap_or("").trim();
-
-			if title_value.is_empty() {
-				// Try to pull the title from the state
-				let state_text = instance.states.get(instance.current_state as usize).and_then(|s| {
-					let t = s.text.trim();
-					if t.is_empty() { None } else { Some(t) }
-				});
-
-				// If the state title is empty, fall back to the action name
-				let title = state_text.unwrap_or(instance.action.name.as_str());
-				title_item["value"] = Value::String(title.to_string());
-			}
-		}
-
-		// If the icon is missing, provide it from the state/action
-		if let Some(icon_item) = items_array.iter_mut().find(|item| item.get("key").and_then(Value::as_str) == Some("icon")) {
-			let icon_empty = icon_item.get("value").and_then(Value::as_str).is_none_or(str::is_empty);
-			if icon_empty {
-				let icon = instance
-					.states
-					.get(instance.current_state as usize)
-					.map(|state| &state.image)
-					.filter(|image| !image.is_empty())
-					.unwrap_or(&instance.action.icon);
-
-				if !icon.is_empty() {
-					icon_item["value"] = Value::String(icon.clone());
-				}
-			}
-		}
-	}
-
-	streamdeck_strip_render::render_to_image(layout, &path, None)
-}
-
 pub async fn update_image(context: &crate::shared::Context, image: Option<&str>) -> Result<(), anyhow::Error> {
 	if let Some(device) = ELGATO_DEVICES.read().await.get(&context.device) {
 		let kind = device.kind();
@@ -92,26 +39,8 @@ pub async fn update_image(context: &crate::shared::Context, image: Option<&str>)
 			let data = image.split_once(',').unwrap().1;
 			let bytes = base64::engine::general_purpose::STANDARD.decode(data)?;
 			if context.controller == "Encoder" {
-				let locks = acquire_locks().await;
-				let slot = get_slot(context, &locks).await?.clone();
-				drop(locks);
-				if let Some(instance) = slot
-					&& let Some(encoder) = &instance.action.encoder
-				{
-					let img = get_encoder_image(encoder, &instance)?;
-					device.write_lcd(context.position as u16 * 200, 0, &ImageRect::from_image_async(img.clone())?).await?;
-				} else {
-					// If we get here, this is either an Encoder action that doesn't have an Encoder config in the manifest, or we were
-					// unable to locate the instance for this action. This realistically shouldn't happen, but if it does, we'll fall back
-					// to rendering what was provided to this function call.
-					device
-						.write_lcd(
-							(context.position as u16 * 200) + 64,
-							14,
-							&ImageRect::from_image_async(image::load_from_memory(&bytes)?.resize(72, 72, image::imageops::FilterType::Nearest))?,
-						)
-						.await?;
-				}
+				let img = generate_encoder_image(context, &bytes).await?;
+				device.write_lcd(context.position as u16 * 200, 0, &ImageRect::from_image(img)?).await?;
 			} else if context.controller == "Infobar" {
 				let img = image::load_from_memory(&bytes)?;
 				let Some(format) = device.kind().lcd_image_format() else {

src-tauri/src/encoder_layouts.rs

@@ -0,0 +1,128 @@
+use crate::shared::{ActionInstance, Encoder, config_dir};
+use crate::store::profiles::{acquire_locks_mut, get_slot_mut};
+
+use std::path::{Path, PathBuf};
+
+use anyhow::Context;
+use image::imageops::overlay;
+use image::{DynamicImage, Rgba, RgbaImage};
+use log::{trace, warn};
+use serde_json::{Map, Value};
+use streamdeck_strip_render::layout::{LayoutItem, PixmapSource};
+
+pub async fn generate_encoder_image(context: &crate::shared::Context, fallback: &[u8]) -> Result<DynamicImage, anyhow::Error> {
+	let mut locks = acquire_locks_mut().await;
+	let slot = get_slot_mut(context, &mut locks).await?;
+
+	// We need to borrow the encoder instance to render the image, so we'll take it, then give it back when we're done.
+	let img = if let Some(instance) = slot {
+		if let Some(mut encoder) = instance.action.encoder.take() {
+			let result = get_encoder_image(&mut encoder, instance).context("Failed to render encoder image");
+			instance.action.encoder = Some(encoder);
+			Some(result?)
+		} else {
+			None
+		}
+	} else {
+		None
+	};
+	drop(locks);
+
+	match img {
+		Some(img) => Ok(img),
+		None => {
+			// If we get here, this is either an Encoder action that doesn't have an Encoder config in the manifest, or we were
+			// unable to locate the instance for this action. This realistically shouldn't happen, but if it does, we'll fall back
+			// to rendering what was provided to this function call.
+			trace!("No encoder instance / config found for action; using fallback image");
+
+			let mut fallback_canvas = RgbaImage::from_pixel(200, 100, Rgba([0, 0, 0, 255]));
+			let fallback_img = image::load_from_memory(fallback)
+				.context("Failed to decode fallback image")?
+				.resize(72, 72, image::imageops::FilterType::Nearest);
+
+			overlay(&mut fallback_canvas, &fallback_img.to_rgba8(), 64, 14);
+
+			Ok(DynamicImage::ImageRgba8(fallback_canvas))
+		}
+	}
+}
+
+fn get_encoder_image(encoder: &mut Encoder, instance: &ActionInstance) -> Result<DynamicImage, anyhow::Error> {
+	let Some(ref mut renderer) = encoder.layout_parsed else {
+		// Something's gone horribly wrong here; we should have a layout. Render a blank image.
+		return Ok(DynamicImage::new_rgb8(200, 100));
+	};
+
+	let path = config_dir().join("plugins").join(&instance.action.plugin);
+	let path_canonical = path.canonicalize()?;
+
+	let state_idx = instance.current_state as usize;
+
+	// Override the title if it's set in the state.
+	let override_title = {
+		let t = instance.states[state_idx].text.trim();
+		(!t.is_empty()).then(|| t.to_string())
+	};
+
+	// Similar to the title, except for the rendered icon (we check whether the state matches the default, and if it doesn't, we use it).
+	let state_image = &instance.states[state_idx].image.trim();
+
+	let override_icon = {
+		let default_image = &instance.action.states[state_idx].image.trim();
+		(!state_image.is_empty() && state_image != default_image).then(|| state_image.to_string())
+	};
+
+	// We need to do small item corrections here.
+	let mut feedback = Map::new();
+	let layout = renderer.layout();
+
+	// If the layout doesn't have a title, fall back to the action title.
+	if let Some(title) = layout.item("title")
+		&& let LayoutItem::Text(title) = title
+		&& title.value.as_deref().is_none_or(str::is_empty)
+	{
+		feedback.insert("title".to_string(), Value::String(instance.action.name.clone()));
+	}
+
+	// If the layout doesn't have an icon, fall back to the state image.
+	if let Some(icon) = layout.item("icon")
+		&& let LayoutItem::Pixmap(icon) = icon
+		&& icon.value == PixmapSource::None
+	{
+		feedback.insert("icon".to_string(), Value::String(state_image.to_string()));
+	}
+
+	// Expand and sandbox every pixmap item's relative file path.
+	for item in &layout.items {
+		let LayoutItem::Pixmap(p) = item else { continue };
+		let PixmapSource::File(v) = &p.value else { continue };
+		if v.is_empty() {
+			continue;
+		}
+
+		let resolved = {
+			// We need to make sure this path isn't already canonical.
+			let candidate = if Path::new(v).is_absolute() { PathBuf::from(v) } else { path.join(v) };
+			match candidate.canonicalize() {
+				Ok(resolved) if resolved.starts_with(&path_canonical) => resolved.to_string_lossy().into_owned(),
+				Ok(resolved) => {
+					warn!("Attempted to load image outside of base path: {resolved:?}");
+					String::new()
+				}
+				Err(_) => {
+					warn!("Unable to canonicalize path: {candidate:?}");
+					String::new()
+				}
+			}
+		};
+
+		feedback.insert(item.key().to_string(), Value::String(resolved));
+	}
+
+	// Send changes to the renderer; note that the overrides are NOOP if they haven't changed.
+	renderer.set_icon_override(override_icon);
+	renderer.set_title_override(override_title);
+	renderer.set_feedback(Value::Object(feedback))?;
+	Ok(DynamicImage::ImageRgba8(renderer.get_image()))
+}

src-tauri/src/events/inbound/states.rs

@@ -4,7 +4,6 @@ use crate::events::frontend::instances::update_state;
 use crate::store::profiles::{acquire_locks_mut, debounce_profile_save, get_instance_mut, save_profile};
 
 use anyhow::bail;
-use log::trace;
 use serde::Deserialize;
 use serde_json::Value;
 
@@ -109,117 +108,17 @@ pub async fn set_image(mut event: ContextAndPayloadEvent<SetImagePayload>) -> Re
 	Ok(())
 }
 
-const COMMON_KEYS: &[&str] = &["enabled", "opacity", "background"];
-const BAR_KEYS: &[&str] = &["bar_bg_c", "bar_border_c", "bar_fill_c", "border_w", "range", "subtype", "value"];
 pub async fn set_feedback(event: ContextAndPayloadEvent<Value>) -> Result<(), anyhow::Error> {
 	let mut locks = acquire_locks_mut().await;
 
 	if let Some(instance) = get_instance_mut(&event.context, &mut locks).await?
 		&& let Some(encoder) = &mut instance.action.encoder
-		&& let Value::Object(map) = event.payload
 	{
-		let layout = &mut encoder.layout_parsed;
-		if layout.is_null() {
+		let Some(layout) = &mut encoder.layout_parsed else {
 			bail!("Layout is not loaded; cannot set feedback");
-		}
-
-		let Some(items_array) = layout.get_mut("items").and_then(Value::as_array_mut) else {
-			bail!("Layout has no items array");
 		};
 
-		for (key, payload_value) in &map {
-			// Grab the item from the layout
-			let Some(item) = items_array.iter_mut().find(|item| {
-				matches!(
-					item.get("key").and_then(|v| v.as_str()),
-					Some(k) if k == key
-				)
-			}) else {
-				trace!("setFeedback: no layout item found for key '{key}'");
-				continue;
-			};
-
-			match payload_value {
-				// We have a direct value; find the key, and set it
-				Value::String(_) | Value::Number(_) => {
-					// Get the item type
-					let Some(item_type) = item.get("type").and_then(Value::as_str) else {
-						trace!("setFeedback: no type found for key '{key}'");
-						continue;
-					};
-
-					match item_type {
-						"text" => {
-							// We need to map the value to a string
-							if let Value::Number(number) = payload_value {
-								item["value"] = Value::String(number.to_string());
-							} else {
-								// Clone the string
-								item["value"] = payload_value.clone();
-							}
-						}
-
-						"bar" | "gbar" => {
-							if let Value::Number(value) = payload_value {
-								item["value"] = Value::Number(value.clone());
-							} else if let Value::String(value) = payload_value {
-								if let Ok(value) = value.parse() {
-									item["value"] = Value::Number(value);
-								} else {
-									trace!("setFeedback: bar/gbar unexpected value for key '{key}' - {value}");
-								}
-							}
-						}
-
-						"pixmap" => {
-							// Update the pixmap value; this should already be a string
-							item["value"] = payload_value.clone();
-						}
-
-						// Ignore anything else
-						_ => {
-							trace!("setFeedback: unknown item type '{item_type}' for key '{key}'");
-						}
-					}
-				}
-
-				// We have an object, so we need to locate and map the change
-				Value::Object(obj) => {
-					// Get the item type
-					let Some(item_type) = item.get("type").and_then(Value::as_str) else {
-						trace!("setFeedback: missing or invalid 'type' field in item: {:?}", item);
-						continue;
-					};
-
-					// Get the valid keys for this item type
-					let type_keys: Vec<&str> = match item_type {
-						"text" => vec!["value", "color", "alignment", "font", "text-overflow"],
-						"pixmap" => vec!["value"],
-						"bar" => BAR_KEYS.to_vec(),
-						"gbar" => BAR_KEYS.iter().copied().chain(["bar_h"]).collect(),
-						unknown => {
-							trace!("setFeedback: unknown item type '{unknown}' for key '{key}'");
-							continue;
-						}
-					};
-
-					// Add the common keys
-					let valid_keys: Vec<&str> = COMMON_KEYS.iter().copied().chain(type_keys).collect();
-
-					// Iterate over the values in the object
-					for (field, field_value) in obj {
-						if valid_keys.contains(&field.as_str()) {
-							item[field] = field_value.clone()
-						}
-					}
-				}
-
-				_ => {
-					trace!("setFeedback: key '{key}' has unexpected payload type, ignoring");
-				}
-			}
-		}
-
+		layout.set_feedback(event.payload)?;
 		update_state(crate::APP_HANDLE.get().unwrap(), instance.context.clone(), &mut locks).await?;
 	}
 

src-tauri/src/events/outbound/devices.rs

@@ -1,8 +1,12 @@
 use super::{send_to_all_plugins, send_to_plugin};
 
+use crate::encoder_layouts::generate_encoder_image;
 use crate::plugins::{DEVICE_NAMESPACES, info_param::DeviceInfo};
 
+use base64::Engine;
+use image::ImageFormat;
 use serde::Serialize;
+use std::io::Cursor;
 
 #[derive(Serialize)]
 #[allow(non_snake_case)]
@@ -46,6 +50,11 @@ struct SetImageEvent {
 
 pub async fn update_image(context: crate::shared::Context, image: Option<String>) -> Result<(), anyhow::Error> {
 	if let Some(plugin) = DEVICE_NAMESPACES.read().await.get(&context.device[..2]) {
+		let image = match (context.controller.as_str(), image) {
+			("Encoder", Some(img)) => Some(to_encoder_jpeg_data_uri(&context, &img).await?),
+			(_, img) => img,
+		};
+
 		send_to_plugin(
 			plugin,
 			&SetImageEvent {
@@ -64,6 +73,19 @@ pub async fn update_image(context: crate::shared::Context, image: Option<String>
 	Ok(())
 }
 
+async fn to_encoder_jpeg_data_uri(context: &crate::shared::Context, image: &str) -> Result<String, anyhow::Error> {
+	let data = image.split_once(',').unwrap().1;
+	let bytes = base64::engine::general_purpose::STANDARD.decode(data)?;
+
+	let img = generate_encoder_image(context, &bytes).await?;
+
+	let mut buf = Vec::new();
+	img.write_to(&mut Cursor::new(&mut buf), ImageFormat::Jpeg)?;
+	let encoded = base64::engine::general_purpose::STANDARD.encode(&buf);
+
+	Ok(format!("data:image/jpeg;base64,{encoded}"))
+}
+
 pub async fn clear_screen(device: String) -> Result<(), anyhow::Error> {
 	if let Some(plugin) = DEVICE_NAMESPACES.read().await.get(&device[..2]) {
 		send_to_plugin(

src-tauri/src/main.rs

@@ -4,6 +4,7 @@
 mod application_watcher;
 mod device_sleep;
 mod elgato;
+mod encoder_layouts;
 mod events;
 mod plugins;
 mod power_events;

src-tauri/src/shared.rs

@@ -8,6 +8,7 @@ use serde_inline_default::serde_inline_default;
 
 use anyhow::{Result, bail};
 use dashmap::DashMap;
+use streamdeck_strip_render::{get_incremental_renderer, strip_renderer::StripRenderer};
 use tauri::Manager;
 use tokio::sync::RwLock;
 
@@ -248,9 +249,8 @@ pub struct Encoder {
 	pub layout: String,
 
 	// Note: this is not a real manifest property; it is only used internally.
-	#[serde_inline_default(serde_json::Value::Null)]
-	#[serde(skip_serializing)]
-	pub layout_parsed: serde_json::Value,
+	#[serde(skip)]
+	pub layout_parsed: Option<StripRenderer>,
 }
 
 pub fn initialise_encoder_layout(action: &mut Action, layout: Option<String>) -> Result<(), anyhow::Error> {
@@ -274,20 +274,20 @@ pub fn initialise_encoder_layout(action: &mut Action, layout: Option<String>) ->
 		match layout_file.canonicalize() {
 			Ok(resolved) if resolved.starts_with(&plugin_dir) => resolved.to_string_lossy().into_owned(),
 			Ok(_) => {
-				encoder.layout_parsed = serde_json::Value::Null;
+				encoder.layout_parsed = None;
 				bail!("Encoder layout path escapes plugin directory: {}", load_layout);
 			}
 			Err(error) => {
-				encoder.layout_parsed = serde_json::Value::Null;
+				encoder.layout_parsed = None;
 				bail!("Failed to canonicalize encoder layout path {}: {}", load_layout, error);
 			}
 		}
 	};
 
 	match load_encoder_layout(&layout) {
-		Ok(parsed) => encoder.layout_parsed = parsed,
+		Ok(parsed) => encoder.layout_parsed = Some(get_incremental_renderer(parsed, None)?),
 		Err(error) => {
-			encoder.layout_parsed = serde_json::Value::Null;
+			encoder.layout_parsed = None;
 			bail!("Failed to load encoder layout {}: {}", load_layout, error)
 		}
 	}