Skip to content

Commit

Permalink
refactor: minor design update
Browse files Browse the repository at this point in the history
  • Loading branch information
@cs50victor
cs50victor committed Jan 14, 2024
1 parent db9ddec commit 90d1d9b
Show file tree
Hide file tree
Showing 5 changed files with 86 additions and 49 deletions.
4 changes: 3 additions & 1 deletion lkgpt/src/frame_capture.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -318,7 +318,6 @@ pub mod scene {
if let SceneState::Render(n) = scene_controller.state {
if n < 1 {
let single_frame_data = single_frame_data.into_inner();
let (w, h) = scene_controller.dimensions();
let pixel_size = single_frame_data.pixel_size;
for image in images_to_save.iter() {
let img_bytes = images.get_mut(image.id()).unwrap();
Expand All @@ -328,6 +327,8 @@ pub mod scene {
Err(e) => panic!("Failed to create image buffer {e:?}"),
};

let (w, h) = rgba_img.dimensions();

if let Err(e) = async_runtime
.rt
.spawn_blocking({
Expand All @@ -339,6 +340,7 @@ pub mod scene {
let (stride_y, stride_u, stride_v) = video_frame.buffer.strides();
let (data_y, data_u, data_v) = video_frame.buffer.data_mut();

// convert captured rgba image to i420
livekit::webrtc::native::yuv_helper::abgr_to_i420(
rgba_img.as_raw(),
w * pixel_size,
Expand Down
90 changes: 50 additions & 40 deletions lkgpt/src/main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ use std::sync::{

use chrono::Utc;
use frame_capture::scene::SceneController;
use image::{ImageBuffer, Rgb, RgbaImage};
use livekit::{publication::LocalTrackPublication, webrtc::video_frame::VideoBuffer, Room};
// use actix_web::{middleware, web::Data, App, HttpServer};
use log::info;
Expand All @@ -37,6 +38,7 @@ use bevy_panorbit_camera::{PanOrbitCamera, PanOrbitCameraPlugin};

use bevy_gaussian_splatting::{GaussianCloud, GaussianSplattingBundle, GaussianSplattingPlugin};

use pollster::FutureExt;
use stt::{receive_and_process_audio, STT};

use futures::StreamExt;
Expand All @@ -47,6 +49,7 @@ use livekit::{
};
use log::{error, warn};
use serde::{Deserialize, Serialize};
use video::ReceivedVideoFrame;

use crate::{
controls::WorldControlChannel, llm::LLMChannel, server::RoomData, stt::AudioInputChannel,
Expand Down Expand Up @@ -146,8 +149,10 @@ pub fn handle_room_events(
_video_channel: Res<video::VideoChannel>,
audio_syncer: ResMut<AudioSync>,
mut room_events: ResMut<LivekitRoom>,
single_frame_data: ResMut<crate::StreamingFrameData>,
) {
while let Ok(event) = room_events.room_events.try_recv() {
println!("\n\n🤡received room event {:?}", event);
match event {
RoomEvent::TrackSubscribed { track, publication: _, participant: _user } => {
match track {
Expand All @@ -160,62 +165,67 @@ pub fn handle_room_events(
let mut start_time = Utc::now().time();
let mut ms_audio_buffer: Vec<i16> = Vec::new();
while let Some(frame) = audio_stream.next().await {
if audio_should_stop.load(Ordering::Relaxed) {
continue;
}
ms_audio_buffer.extend_from_slice(&frame.data);
let elapsed = (Utc::now().time() - start_time).num_milliseconds();
// 5ms of audio
if elapsed >= 5 {
// 10ms of audio
if elapsed >= 10 {
if let Err(e) = audio_channel_tx.send(frame.data.to_vec()) {
log::error!("Couldn't send audio frame to stt {e}");
};
start_time = Utc::now().time();
}
if audio_should_stop.load(Ordering::Relaxed) {
break;
}
}
});
},
RemoteTrack::Video(video_track) => {
let video_rtc_track = video_track.rtc_track();
let pixel_size = 4;
let mut video_stream = NativeVideoStream::new(video_rtc_track);

async_runtime.rt.spawn(async move {
// every 10 video frames
let mut i = 0;
while let Some(frame) = video_stream.next().await {
// frame.buffer.to_argb(format, dst, dst_stride, dst_width, dst_height)
let _video_frame_buffer = frame.buffer.to_i420();
// let width: u32 = video_frame_buffer.width();
// let height: u32 = video_frame_buffer.height();
// let rgba_stride = video_frame_buffer.width() * 4;

// let (stride_y, stride_u, stride_v) = video_frame_buffer.strides();
// let (data_y, data_u, data_v) = video_frame_buffer.data();

// livekit::webrtc::native::yuv_helper::i420_to_rgba(
// data_y,
// stride_y,
// data_u,
// stride_u,
// data_v,
// stride_v,
// rgba_ptr,
// rgba_stride,
// video_frame_buffer.width() as i32,
// video_frame_buffer.height() as i32,
// );

// if let Err(e)= audio_channel_tx.send(frame.data.to_vec()){
// log::error!("Couldn't send audio frame to stt {e}");
// };

/*
image: image::ImageBuffer<image::Rgba<u8>, Vec<u8>>,
framebuffer: std::sync::Arc<parking_lot::Mutex<Vec<u8>>>,
video_frame: std::sync::Arc<
parking_lot::Mutex<
livekit::webrtc::video_frame::VideoFrame<livekit::webrtc::video_frame::I420Buffer>,
>,
>,
*/
log::error!("🤡received video frame | {:#?}", frame);
// VIDEO FRAME BUFFER (i420_buffer)
let video_frame_buffer = frame.buffer.to_i420();
let width = video_frame_buffer.width();
let height = video_frame_buffer.height();
let rgba_stride = video_frame_buffer.width() * pixel_size;

let (stride_y, stride_u, stride_v) = video_frame_buffer.strides();
let (data_y, data_u, data_v) = video_frame_buffer.data();

let rgba_buffer = RgbaImage::new(width, height);
let rgba_raw = unsafe {
std::slice::from_raw_parts_mut(
rgba_buffer.as_raw().as_ptr() as *mut u8,
rgba_buffer.len(),
)
};

livekit::webrtc::native::yuv_helper::i420_to_rgba(
data_y,
stride_y,
data_u,
stride_u,
data_v,
stride_v,
rgba_raw,
rgba_stride,
video_frame_buffer.width() as i32,
video_frame_buffer.height() as i32,
);

if let Err(e) = rgba_buffer.save(format!("camera/{i}.png")) {
log::error!("Couldn't save video frame {e}");
};
i += 1;
}
info!("🤡ended video thread");
});
},
};
Expand Down
27 changes: 21 additions & 6 deletions lkgpt/src/server.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -189,27 +189,42 @@ mod lsdk_webhook {
Err(e) => return Resp::InternalServerError().json(ServerMsg::error(e.to_string())),
};

if event.room.is_some() && event.event == "room_started" {
// room_finished
if event.room.is_some() {
let livekit_protocol::Room {
name: participant_room_name,
max_participants,
num_participants,
..
} = event.room.unwrap();
let event = event.event;
if event == "room_started" {
if num_participants < max_participants {
info!("...connecting to room");

if num_participants < max_participants {
info!("...connecting to room");
let server_data = server_data.lock();

log::info!("app state {:#?}", *server_data.app_state);

*server_data.app_state.lock() =
crate::ParticipantRoomName(participant_room_name);

log::info!("app state {:?}", *server_data.app_state);

info!("\nSERVER FINISHED PROCESSING ROOM_STARTED WEBHOOK");
};
} else if event == "room_finished" {
let server_data = server_data.lock();

log::info!("app state {:#?}", *server_data.app_state);

*server_data.app_state.lock() = crate::ParticipantRoomName(participant_room_name);
*server_data.app_state.lock() =
crate::ParticipantRoomName(format!("reset:{participant_room_name}"));

log::info!("app state {:?}", *server_data.app_state);

info!("\nSERVER FINISHED PROCESSING ROOM_STARTED WEBHOOK");
};
info!("\nSERVER FINISHED PROCESSING ROOM_FINISHED WEBHOOK");
}
} else {
info!("received event {}", event.event);
}
Expand Down
5 changes: 5 additions & 0 deletions lkgpt/src/video.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,11 @@ use bevy::ecs::{
use futures::StreamExt;
use livekit::webrtc::video_stream::native::NativeVideoStream;

pub struct ReceivedVideoFrame {
pub image_buffer: Vec<u8>,
pub timestamp: i64, // When the frame was captured in microseconds
}

#[derive(Resource)]
pub struct VideoChannel {
pub tx: crossbeam_channel::Sender<Vec<i16>>,
Expand Down
9 changes: 7 additions & 2 deletions meet/app/r/[name]/page.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,13 +37,18 @@ export default function LivekitRoom({ params }: { params: { name: string } }) {
<ActiveRoom roomName={roomName} userChoices={preJoinChoices} />
) : (
<div className="flex flex-col justify-center items-center h-full">
<h1 className="text-4xl font-semibold font-display text-brand mb-4">Kitt2</h1>
<h1 className="text-4xl font-semibold font-display text-brand">Kitt2</h1>
<PreJoinNoSSR
className="p-4 rounded-xl
className="p-4 rounded-xl w-full max-w-sm
[&*.lk-camera-off-note]:bg-cyan-950
[&*.lk-camera-off-note]:flex
[&*.lk-camera-off-note]:items-center
[&*.lk-camera-off-note]:justify-center
[&*.lk-camera-off-note>svg]:stroke-black
[&*.lk-camera-off-note>svg>path]:fill-gray-300
[&*.lk-camera-off-note]:rounded-full
[&*.lk-camera-off-note]:aspect-w-4
[&*.lk-camera-off-note]:aspect-h-4
[&*.lk-camera-off-note]:overflow-hidden
[&*.lk-camera-off-note]:border
Expand Down

0 comments on commit 90d1d9b

Please sign in to comment.