feat: add image websocket + demo

cs50victor · Jan 22, 2024 · 5a466b2 · 5a466b2
1 parent cc4715f
commit 5a466b2
Show file tree

Hide file tree

Showing 9 changed files with 993 additions and 1,404 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/demo/app/page.tsx b/demo/app/page.tsx
@@ -1,9 +1,11 @@
-export default function Page() {
+import WebSocketExample from "~/components/WebSocketExample";
 
+export default function Page() {
   return (
     <div className="min-h-dvh prose w-full mx-auto">
       <div className="prose flex flex-col items-center justify-center">
-        <h1>Homepage</h1>
+        <h1 className="text-center">Curr Image from New Media</h1>
+        <WebSocketExample/>
       </div>
     </div>
   )

diff --git a/demo/components/WebSocketExample.tsx b/demo/components/WebSocketExample.tsx
@@ -0,0 +1,64 @@
+"use client"
+
+import { useEffect, useState } from 'react';
+import Image from 'next/image'
+
+export default function WebSocketExample({port = 8080}:{port?:number}){
+  const [imgUrl, setImgUrl] = useState<string|null>(null);
+  const [socket, setSocket] = useState<WebSocket|null>(null)
+
+  useEffect(() => {
+    const socket = new WebSocket(`ws://localhost:${port}`)
+
+    setSocket(socket);
+
+    socket.onopen = () => {
+      console.log('WebSocket Open');
+      socket.send("hello");
+    };
+
+    socket.onclose = () => {
+      console.log('WebSocket Close');
+    };
+
+    socket.onerror = (error) => {
+      console.error("WebSocket Error:", error);
+    };
+
+    socket.onmessage = (event) => {
+      try {
+        const data = JSON.parse(event.data);
+        console.log("WebSocket Message:", data);
+        if (data.images && data.images.length > 0) {
+          setImgUrl(data.images[0]);
+        }
+      } catch (e) {
+        console.error("Error parsing the WebSocket response:", e);
+      }
+    };
+
+    return () => {
+      socket.close();
+    };
+  }, []);
+
+  return (
+    <div>
+      {imgUrl ? (
+        <div className="h-[80vh] w-[80vw] relative">
+            <Image
+              className="rounded"
+              src={imgUrl}
+              alt="Streamed image"
+              objectFit="cover"
+              fill
+              />
+          </div>
+      ): (
+        <p>loading ...</p>
+        )}
+    </div>
+  )
+};
+
+// {/* <img style={{border: "1px solid black"}} src={imgUrl} alt="Received from server" /> */}
diff --git a/new_media/Cargo.toml b/new_media/Cargo.toml
@@ -6,16 +6,17 @@ rust-version.workspace = true
 default-run = "new_media"
 
 [dependencies]
-actix-web = "4.4.0"
 anyhow = "1.0.75"
-async-openai = "0.18.0"
+async-net = "2.0.0"
 base64 = "0.21.5"
 bevy = { version = "0.12.0", default-features = false, features = [
   "bevy_asset",
   "bevy_core_pipeline",
   "bevy_render",
+  # "multi-threaded"
 ] }
-bevy_gaussian_splatting = { version = "2.0.2", default-features = true }
+bevy_ws_server = { git = "https://github.com/cs50victor/bevy-ws-server.git", branch = "main"}
+bevy_gaussian_splatting = { git = "https://github.com/mosure/bevy_gaussian_splatting.git", branch="main", default-features = true}
 bevy_panorbit_camera = "0.10.0"
 bevy_web_asset = "0.7.0"
 bytes = "1.5.0"
@@ -24,20 +25,19 @@ crossbeam-channel = "0.5.10"
 dotenvy = "0.15.7"
 futures = "0.3.29"
 futures-intrusive = "0.5.0"
-image = { version = "0.24.7", default-features = false, features = ["png"] }
+image = { version = "0.24.7", default-features = false }
 log = "0.4.20"
-parking_lot = "0.12.1"
 pollster = "0.3.0"
 pretty_env_logger = "0.5.0"
 serde = { version = "1.0.192", features = ["derive"] }
 serde_json = "1.0.108"
 tokio = { version = "1.33.0", features = ["full"] }
 url = "2.5.0"
 wgpu = { version = "0.17.1" }
+tungstenite = "0.21.0"
 
 [target.x86_64-unknown-linux-gnu.dependencies]
 winit = { version = "0.29.10", default-features = false, features = ["x11"]}
 
 # [target.i686-unknown-linux-gnu.dependencies]
 # winit = { version = "0.29.10", default-features = false, features = ["x11"]}
-
diff --git a/new_media/src/frame_capture.rs b/new_media/src/frame_capture.rs
@@ -205,7 +205,6 @@ pub mod scene {
                 PostUpdate,
                 update
                     .run_if(resource_exists::<crate::StreamingFrameData>())
-                    .run_if(resource_exists::<crate::AsyncRuntime>()),
             );
         }
     }
@@ -311,7 +310,6 @@ pub mod scene {
     fn update(
         mut images: ResMut<Assets<Image>>,
         images_to_save: Query<&ImageToSave>,
-        async_runtime: Res<crate::AsyncRuntime>,
         single_frame_data: ResMut<crate::StreamingFrameData>,
         mut scene_controller: ResMut<SceneController>,
     ) {

diff --git a/new_media/src/llm.rs b/new_media/src/llm.rs