Implement drawing images to Stream Deck screens

src-tauri/Cargo.lock

@@ -170,9 +170,9 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
 
 [[package]]
 name = "base64"
-version = "0.21.5"
+version = "0.21.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9"
+checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
 
 [[package]]
 name = "bitflags"
@@ -2033,9 +2033,11 @@ name = "opendeck"
 version = "2.0.0"
 dependencies = [
  "anyhow",
+ "base64 0.21.7",
  "elgato-streamdeck",
  "futures-util",
  "hidapi",
+ "image",
  "lazy_static",
  "log",
  "os_info",
@@ -2272,7 +2274,7 @@ version = "1.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e5699cc8a63d1aa2b1ee8e12b9ad70ac790d65788cd36101fa37f87ea46c4cef"
 dependencies = [
- "base64 0.21.5",
+ "base64 0.21.7",
  "indexmap 2.1.0",
  "line-wrap",
  "quick-xml",
@@ -2797,7 +2799,7 @@ version = "3.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "64cd236ccc1b7a29e7e2739f27c0b2dd199804abc4290e32f59f3b68d6405c23"
 dependencies = [
- "base64 0.21.5",
+ "base64 0.21.7",
  "chrono",
  "hex",
  "indexmap 1.9.3",
@@ -3255,7 +3257,7 @@ version = "1.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7b3475e55acec0b4a50fb96435f19631fb58cbcd31923e1a213de5c382536bbb"
 dependencies = [
- "base64 0.21.5",
+ "base64 0.21.7",
  "brotli",
  "ico",
  "json-patch",

src-tauri/Cargo.toml

@@ -18,11 +18,12 @@ tauri = { version = "1.5.2", features = [ "shell-execute", "shell-open", "system
 serde = { version = "1.0", features = [ "derive" ] }
 serde_json = "1.0"
 serialport = "4.2.2"
-elgato-streamdeck = { version = "0.5.0", default-features = false, features = [ "async" ] }
-hidapi = { version = "2.5.0", default-features = false, features = [ "linux-static-hidraw", "macos-shared-device", "windows-native" ] }
 tokio = { version = "1.34.0", features = [ "full" ] }
 tokio-tungstenite = "0.20.1"
 tiny_http = "0.12.0"
+elgato-streamdeck = { version = "0.5.0", default-features = false, features = [ "async" ] }
+hidapi = { version = "2.5.0", default-features = false, features = [ "linux-static-hidraw", "macos-shared-device", "windows-native" ] }
+image = { version = "0.24.6", default-features = false, features = [ "bmp", "jpeg" ] }
 # Smaller utility libraries
 futures-util = "0.3.29"
 anyhow = "1.0.75"
@@ -33,6 +34,7 @@ serde_with = "3.4.0"
 lazy_static = "1.4.0"
 os_info = "3.7.0"
 urlencoding = "2.1.3"
+base64 = "0.21.7"
 
 [features]
 custom-protocol = [ "tauri/custom-protocol" ]

src-tauri/src/devices/elgato.rs

@@ -1,8 +1,34 @@
 use crate::events::outbound::{keypad, encoder};
 
+use std::collections::HashMap;
+
 use elgato_streamdeck::{AsyncStreamDeck, DeviceStateUpdate, info};
 
-pub async fn init(device: AsyncStreamDeck) {
+use lazy_static::lazy_static;
+use tokio::sync::Mutex;
+use base64::Engine as _;
+
+lazy_static! {
+	static ref ELGATO_DEVICES: Mutex<HashMap<String, AsyncStreamDeck>> = Mutex::new(HashMap::new());
+}
+
+pub async fn update_image(context: &crate::shared::ActionContext, url: &str) -> Result<(), anyhow::Error> {
+	if let Some(device) = ELGATO_DEVICES.lock().await.get(&context.device) {
+		let data = url.split_once(',').unwrap().1;
+		let bytes = base64::engine::general_purpose::STANDARD.decode(data)?;
+		device.set_button_image(context.position, image::load_from_memory(&bytes)?).await?;
+	}
+	Ok(())
+}
+
+pub async fn clear_image(context: &crate::shared::ActionContext) -> Result<(), anyhow::Error> {
+	if let Some(device) = ELGATO_DEVICES.lock().await.get(&context.device) {
+		device.clear_button_image(context.position).await?;
+	}
+	Ok(())
+}
+
+pub(super) async fn init(device: AsyncStreamDeck) {
 	let kind = device.kind();
 	let device_type = match kind.product_id() {
 		info::PID_STREAMDECK_ORIGINAL | info::PID_STREAMDECK_ORIGINAL_V2 | info::PID_STREAMDECK_MK2 => 0,
@@ -23,6 +49,7 @@ pub async fn init(device: AsyncStreamDeck) {
 	});
 
 	let reader = device.get_reader();
+	ELGATO_DEVICES.lock().await.insert(device_id.clone(), device);
 	loop {
 		let updates = match reader.read(100.0).await {
 			Ok(updates) => updates,

src-tauri/src/devices/mod.rs

@@ -1,5 +1,5 @@
 mod prontokey;
-mod elgato;
+pub mod elgato;
 
 use std::collections::HashMap;
 

src-tauri/src/events/frontend.rs

@@ -207,6 +207,15 @@ pub async fn switch_property_inspector(old: Option<ActionContext>, new: Option<A
 	}
 }
 
+#[tauri::command]
+pub async fn update_image(context: ActionContext, image: String) {
+	if context.device.starts_with("sd-") {
+		if let Err(error) = crate::devices::elgato::update_image(&context, &image).await {
+			log::warn!("Failed to update device image at context {} with image {}: {}", context, image, error);
+		}
+	}
+}
+
 pub async fn update_state(app: &tauri::AppHandle, instance: &ActionInstance) -> Result<(), anyhow::Error> {
 	let window = app.get_window("main").unwrap();
 	window.emit("update_state", serde_json::to_string(instance).unwrap())?;

src-tauri/src/events/outbound/will_appear.rs

@@ -32,5 +32,11 @@ pub async fn will_disappear(instance: &ActionInstance) -> Result<(), anyhow::Err
 		payload: GenericInstancePayload::new(instance)
 	}).await?;
 
+	if instance.context.device.starts_with("sd-") {
+		if let Err(error) = crate::devices::elgato::clear_image(&instance.context).await {
+			log::warn!("Failed to clear device image at context {}: {}", instance.context, error);
+		}
+	}
+
 	Ok(())
 }

src-tauri/src/main.rs

@@ -30,7 +30,8 @@ async fn main() {
 			frontend::set_selected_profile,
 			frontend::delete_profile,
 			frontend::make_info,
-			frontend::switch_property_inspector
+			frontend::switch_property_inspector,
+			frontend::update_image
 		])
 		.plugin(tauri_plugin_log::Builder::default()
 			.targets([

src-tauri/tauri.conf.json

@@ -41,7 +41,7 @@
 			}
 		},
 		"security": {
-			"csp": "default-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data: https://appstore.elgato.com; frame-src http://localhost:57118; connect-src https://appstore.elgato.com;"
+			"csp": "default-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data: http://localhost:57118; https://appstore.elgato.com; frame-src http://localhost:57118; connect-src https://appstore.elgato.com;"
 		},
 		"allowlist": {
 			"shell": {

src/components/Key.svelte

@@ -2,7 +2,7 @@
 	import type { ActionInstance } from "$lib/ActionInstance";
 
 	import { inspectedInstance } from "$lib/propertyInspector";
-	import { getImage } from "$lib/rendererHelper";
+	import { getImage, renderImage } from "$lib/rendererHelper";
 
 	import { invoke } from "@tauri-apps/api";
 	import { listen } from "@tauri-apps/api/event";
@@ -59,6 +59,7 @@
 	$: {
 		image = getImage(state?.image, oldImage);
 		oldImage = image;
+		if (state) renderImage(context, state);
 	}
 </script>
 

src/lib/rendererHelper.ts

@@ -1,3 +1,5 @@
+import { invoke } from "@tauri-apps/api";
+
 import type { ActionState } from "./ActionState";
 
 export function getImage(image: string | undefined, fallback: string): string {
@@ -16,7 +18,7 @@ export function getImage(image: string | undefined, fallback: string): string {
 	return image;
 }
 
-export default async function renderImage(state: ActionState) {
+export async function renderImage(actionContext: string, state: ActionState) {
 	// Create canvas
 	let canvas = document.createElement("canvas");
 	canvas.width = 144;
@@ -24,10 +26,6 @@ export default async function renderImage(state: ActionState) {
 	let context = canvas.getContext("2d");
 	if (!context) return;
 
-	// White background
-	context.fillStyle = "white";
-	context.fillRect(0, 0, canvas.width, canvas.height);
-
 	// Load image
 	let image = document.createElement("img");
 	image.crossOrigin = "anonymous";
@@ -58,6 +56,6 @@ export default async function renderImage(state: ActionState) {
 		}
 	}
 
-	console.log(canvas.toDataURL("image/jpeg"));
+	await invoke("update_image", { context: actionContext, image: canvas.toDataURL("image/jpeg") });
 	canvas.remove();
 }