refactor: send bevy frames

demo/components/WebSocketExample.tsx

@@ -29,8 +29,8 @@ export default function WebSocketExample({port = 8080}:{port?:number}){
       try {
         const data = JSON.parse(event.data);
         console.log("WebSocket Message:", data);
-        if (data.images && data.images.length > 0) {
-          setImgUrl(data.images[0]);
+        if (data.image) {
+          setImgUrl(data.image);
         }
       } catch (e) {
         console.error("Error parsing the WebSocket response:", e);

new_media/Cargo.toml

@@ -9,14 +9,13 @@ default-run = "new_media"
 anyhow = "1.0.75"
 async-net = "2.0.0"
 base64 = "0.21.5"
-bevy = { version = "0.12.0", default-features = false, features = [
+bevy = { version = "0.12.1", default-features = false, features = [
   "bevy_asset",
   "bevy_core_pipeline",
   "bevy_render",
-  # "multi-threaded"
 ] }
 bevy_ws_server = { git = "https://github.com/cs50victor/bevy-ws-server.git", branch = "main"}
-bevy_gaussian_splatting = { git = "https://github.com/mosure/bevy_gaussian_splatting.git", branch="main", default-features = true}
+bevy_gaussian_splatting = { version = "2.0.2", default-features = true }
 bevy_panorbit_camera = "0.10.0"
 bevy_web_asset = "0.7.0"
 bytes = "1.5.0"
@@ -25,15 +24,15 @@ crossbeam-channel = "0.5.10"
 dotenvy = "0.15.7"
 futures = "0.3.29"
 futures-intrusive = "0.5.0"
-image = { version = "0.24.7", default-features = false }
+image = { version = "0.24.7", default-features = false, features = ["png"] }
 log = "0.4.20"
 pollster = "0.3.0"
 pretty_env_logger = "0.5.0"
 serde = { version = "1.0.192", features = ["derive"] }
 serde_json = "1.0.108"
 tokio = { version = "1.33.0", features = ["full"] }
 url = "2.5.0"
-wgpu = { version = "0.17.1" }
+wgpu = "0.17.1"
 tungstenite = "0.21.0"
 
 [target.x86_64-unknown-linux-gnu.dependencies]

new_media/src/frame_capture.rs

@@ -182,11 +182,16 @@ pub mod image_copy {
 }
 pub mod scene {
 
+    use std::io::Cursor;
+
+    use anyhow::Result;
+    use base64::{engine::general_purpose, Engine};
     use bevy::{
         prelude::*,
         render::{camera::RenderTarget, renderer::RenderDevice},
     };
 
+    use image::{DynamicImage, ImageBuffer, ImageOutputFormat, Rgba, RgbaImage};
     use wgpu::{Extent3d, TextureDescriptor, TextureDimension, TextureFormat, TextureUsages};
 
     use super::image_copy::ImageCopier;
@@ -197,6 +202,9 @@ pub mod scene {
     #[derive(Component, Deref, DerefMut)]
     struct ImageToSave(Handle<Image>);
 
+    #[derive(Resource)]
+    pub struct CurrImageBase64(pub String);
+
     pub struct CaptureFramePlugin;
     impl Plugin for CaptureFramePlugin {
         fn build(&self, app: &mut App) {
@@ -308,6 +316,7 @@ pub mod scene {
     fn update(
         mut images: ResMut<Assets<Image>>,
         images_to_save: Query<&ImageToSave>,
+        mut curr_base64_img: ResMut<CurrImageBase64>,
         single_frame_data: ResMut<crate::StreamingFrameData>,
         mut scene_controller: ResMut<SceneController>,
     ) {
@@ -323,16 +332,24 @@ pub mod scene {
                         Err(e) => panic!("Failed to create image buffer {e:?}"),
                     };
 
-                    let (_w, _h) = rgba_img.dimensions();
-
-                    // send image
+                    curr_base64_img.0 = image_to_browser_base64(&rgba_img).unwrap();
                 }
-                // if scene_controller.single_image {
-                //     app_exit_writer.send(AppExit);
-                // }
             } else {
                 scene_controller.state.decrement();
             }
         }
     }
+
+    fn image_to_browser_base64(img: &ImageBuffer<Rgba<u8>, Vec<u8>>)-> Result<String> {
+        let mut image_data: Vec<u8> = Vec::new();
+        img.write_to(&mut Cursor::new(&mut image_data), ImageOutputFormat::Png)?;
+        let res_base64 = general_purpose::STANDARD.encode(image_data);
+        Ok(format!("data:image/png;base64,{}", res_base64))
+    }
+
+    pub fn white_img_placeholder(w: u32, h: u32) -> String {
+        let mut img = RgbaImage::new(w,h);
+        // img.iter_mut().for_each(|pixel| *pixel = 255);
+        image_to_browser_base64(&img).unwrap()
+    }
 }

new_media/src/main.rs

@@ -1,4 +1,5 @@
 #![feature(ascii_char, async_closure, slice_pattern)]
+#[allow(clippy::type_complexity)]
 mod controls;
 mod frame_capture;
 mod server;
@@ -40,18 +41,18 @@ pub struct AudioSync {
 fn setup_gaussian_cloud(
     mut commands: Commands,
     asset_server: Res<AssetServer>,
-    _gaussian_assets: ResMut<Assets<GaussianCloud>>,
+    mut gaussian_assets: ResMut<Assets<GaussianCloud>>,
     mut scene_controller: ResMut<frame_capture::scene::SceneController>,
     mut images: ResMut<Assets<Image>>,
     render_device: Res<RenderDevice>,
 ) {
     // let remote_file = Some("https://huggingface.co/datasets/cs50victor/splats/resolve/main/train/point_cloud/iteration_7000/point_cloud.gcloud");
     // TODO: figure out how to load remote files later
-    let splat_file = "splats/bonsai/point_cloud/iteration_7000/point_cloud.gcloud";
-    log::info!("loading {}", splat_file);
-    let cloud = asset_server.load(splat_file.to_string());
+    // let splat_file = "splats/bonsai/point_cloud/iteration_7000/point_cloud.gcloud";
+    // log::info!("loading {}", splat_file);
+    // let cloud = asset_server.load(splat_file.to_string());
 
-    // let cloud = gaussian_assets.add(GaussianCloud::test_model());
+    let cloud = gaussian_assets.add(GaussianCloud::test_model());
 
     let render_target = frame_capture::scene::setup_render_target(
         &mut commands,
@@ -106,6 +107,7 @@ fn main() {
 
     App::new()
     .insert_resource(frame_capture::scene::SceneController::new(config.width, config.height))
+    .insert_resource(frame_capture::scene::CurrImageBase64(frame_capture::scene::white_img_placeholder(config.width, config.height)))
     .insert_resource(ClearColor(Color::rgb_u8(0, 0, 0)))
     .add_plugins((
         bevy_web_asset::WebAssetPlugin,
@@ -134,6 +136,7 @@ fn main() {
         receive_message
     ))
     // .add_systems(OnEnter(AppState::Active), setup_gaussian_cloud)
+    .add_systems(Startup, setup_gaussian_cloud)
     .run();
 }