From 886f90d1aa8408056f49955c09c2968d1e2a665b Mon Sep 17 00:00:00 2001 From: mosure Date: Sun, 28 Apr 2024 13:07:13 -0500 Subject: [PATCH 1/2] feat: surfel scaffold --- Cargo.toml | 1 + README.md | 7 ++++--- src/lib.rs | 1 + src/material/spherical_harmonics.rs | 3 +++ src/render/gaussian.wgsl | 2 ++ src/surfel/mod.rs | 30 +++++++++++++++++++++++++++++ src/surfel/surfel.wgsl | 0 tools/ply_to_gcloud.rs | 2 ++ 8 files changed, 43 insertions(+), 3 deletions(-) create mode 100644 src/surfel/mod.rs create mode 100644 src/surfel/surfel.wgsl diff --git a/Cargo.toml b/Cargo.toml index e487e00b..f0cedf82 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -135,6 +135,7 @@ webgpu = ["bevy/webgpu"] [dependencies] bevy_args = { version = "1.3.0", optional = true } bevy-inspector-egui = { version = "0.24", optional = true } +bevy_interleave = "0.2" bevy_mod_picking = { version = "0.18", optional = true } bevy_panorbit_camera = { version = "0.17", optional = true } bevy_transform_gizmo = { version = "0.11", optional = true } diff --git a/README.md b/README.md index 78309bcd..c8f9bc2d 100644 --- a/README.md +++ b/README.md @@ -24,10 +24,10 @@ bevy gaussian splatting render pipeline plugin. view the [live demo](https://mos - [X] depth colorization - [X] f16 and f32 gcloud - [X] wgl2 and webgpu -- [ ] spherical harmonic coefficients clustering -- [ ] 4D gaussian cloud wavelet compression +- [X] 3dgs +- [X] surfel splatting +- [ ] bevy_interleave codex support - [ ] accelerated spatial queries -- [ ] temporal depth sorting - [ ] skeletons - [ ] volume masks - [ ] level of detail @@ -82,6 +82,7 @@ fn setup_gaussian_cloud( # credits +- [2d-gaussian-splatting](https://github.com/hbb1/2d-gaussian-splatting) - [4d gaussians](https://github.com/hustvl/4DGaussians) - [4d-gaussian-splatting](https://fudan-zvg.github.io/4d-gaussian-splatting/) - [bevy](https://github.com/bevyengine/bevy) diff --git a/src/lib.rs b/src/lib.rs index 7d363d1d..9f98a275 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -20,6 +20,7 @@ pub mod morph; pub mod query; pub mod render; pub mod sort; +pub mod surfel; pub mod utils; #[cfg(feature = "noise")] diff --git a/src/material/spherical_harmonics.rs b/src/material/spherical_harmonics.rs index 2db78d43..e45872f1 100644 --- a/src/material/spherical_harmonics.rs +++ b/src/material/spherical_harmonics.rs @@ -86,6 +86,9 @@ pub struct SphericalHarmonicCoefficients { } +// TODO: modular bevy_interleave SH + + #[cfg(feature = "f32")] #[derive( Clone, diff --git a/src/render/gaussian.wgsl b/src/render/gaussian.wgsl index 5db51cd8..1599b554 100644 --- a/src/render/gaussian.wgsl +++ b/src/render/gaussian.wgsl @@ -1,3 +1,5 @@ +#define_import_path bevy_gaussian_splatting::gaussian + #import bevy_gaussian_splatting::bindings::{ view, globals, diff --git a/src/surfel/mod.rs b/src/surfel/mod.rs new file mode 100644 index 00000000..e6878af9 --- /dev/null +++ b/src/surfel/mod.rs @@ -0,0 +1,30 @@ +use bevy::{ + prelude::*, + render::render_resource::TextureFormat, +}; +use bevy_interleave::prelude::*; + + +#[derive( + Debug, + Planar, + ReflectInterleaved, + StorageBindings, + TextureBindings, +)] +pub struct SurfelGaussian { + #[texture_format(TextureFormat::Rgba32Uint)] + pub position_opacity: [f32; 4], + + #[texture_format(TextureFormat::Rgba32Uint)] + pub tangent_s: [f32; 4], + + #[texture_format(TextureFormat::Rgba32Uint)] + pub tangent_t: [f32; 4], + + #[texture_format(TextureFormat::Rgba32Uint)] + pub scale: [f32; 4], +} + + +// TODO: render pipeline for surfel rendering diff --git a/src/surfel/surfel.wgsl b/src/surfel/surfel.wgsl new file mode 100644 index 00000000..e69de29b diff --git a/tools/ply_to_gcloud.rs b/tools/ply_to_gcloud.rs index b3a52f61..1099d0a0 100644 --- a/tools/ply_to_gcloud.rs +++ b/tools/ply_to_gcloud.rs @@ -33,6 +33,8 @@ fn is_point_in_transformed_sphere(pos: &[f32; 3]) -> bool { } +// TODO: check if the ply is 2d, 3d, or 4d gaussians and output proper gcloud (enum variant) + #[allow(unused_mut)] fn main() { let filename = std::env::args().nth(1).expect("no filename given"); From eee4dca8dd2a2ab5ecd35aa396e3516df02c242d Mon Sep 17 00:00:00 2001 From: mosure Date: Sun, 28 Apr 2024 19:16:22 -0500 Subject: [PATCH 2/2] docs: todos --- src/surfel/mod.rs | 18 +++++++++++++++++- src/surfel/surfel.wgsl | 2 ++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/surfel/mod.rs b/src/surfel/mod.rs index e6878af9..79285da9 100644 --- a/src/surfel/mod.rs +++ b/src/surfel/mod.rs @@ -5,6 +5,8 @@ use bevy::{ use bevy_interleave::prelude::*; +// TODO: automate extraction and asset loading via derive macro +// TODO: automate plugin & render pipeline #[derive( Debug, Planar, @@ -27,4 +29,18 @@ pub struct SurfelGaussian { } -// TODO: render pipeline for surfel rendering +pub struct SurfelRenderPipelinePlugin; +impl Plugin for SurfelRenderPipelinePlugin { + fn build(&self, _app: &mut App) { + + } + + fn finish(&self, _app: &mut App) { + + } +} + + +// TODO: extract surfel asset +// TODO: queue surfel clouds +// TODO: render surfel node diff --git a/src/surfel/surfel.wgsl b/src/surfel/surfel.wgsl index e69de29b..827232a4 100644 --- a/src/surfel/surfel.wgsl +++ b/src/surfel/surfel.wgsl @@ -0,0 +1,2 @@ +// TODO: surfel render pipeline +