feat: radix depth sort

.cargo/config.toml

@@ -2,6 +2,7 @@
 
 [target.wasm32-unknown-unknown]
 runner = "wasm-server-runner"
+rustflags = ["--cfg=web_sys_unstable_apis"]
 
 
 # fix spurious network error on windows

.gitignore

@@ -4,6 +4,7 @@ Cargo.lock
 **/*.rs.bk
 *.pdb
 
+*.py
 
 *.ply
 *.gcloud

.vscode/launch.json

@@ -29,15 +29,15 @@
         {
             "type": "lldb",
             "request": "launch",
-            "name": "Debug executable 'bevy_gaussian_splatting'",
+            "name": "Debug executable 'viewer'",
             "cargo": {
                 "args": [
                     "build",
-                    "--bin=bevy_gaussian_splatting",
+                    "--bin=viewer",
                     "--package=bevy_gaussian_splatting"
                 ],
                 "filter": {
-                    "name": "bevy_gaussian_splatting",
+                    "name": "viewer",
                     "kind": "bin"
                 }
             },
@@ -50,16 +50,16 @@
         {
             "type": "lldb",
             "request": "launch",
-            "name": "Debug unit tests in executable 'bevy_gaussian_splatting'",
+            "name": "Debug unit tests in executable 'viewer'",
             "cargo": {
                 "args": [
                     "test",
                     "--no-run",
-                    "--bin=bevy_gaussian_splatting",
+                    "--bin=viewer",
                     "--package=bevy_gaussian_splatting"
                 ],
                 "filter": {
-                    "name": "bevy_gaussian_splatting",
+                    "name": "viewer",
                     "kind": "bin"
                 }
             },

Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "bevy_gaussian_splatting"
 description = "bevy gaussian splatting render pipeline plugin"
-version = "0.1.0"
+version = "0.2.0"
 edition = "2021"
 authors = ["mosure <[email protected]>"]
 license = "MIT"
@@ -10,26 +10,32 @@ categories = ["computer-vision", "graphics", "rendering", "rendering::data-forma
 homepage = "https://github.com/mosure/bevy_gaussian_splatting"
 repository = "https://github.com/mosure/bevy_gaussian_splatting"
 readme = "README.md"
-include = ["tools"]
 exclude = [".devcontainer", ".github", "docs", "dist", "build", "assets", "credits"]
-default-run = "bevy_gaussian_splatting"
+default-run = "viewer"
 
-# TODO: use minimal bevy features
 [dependencies]
-bevy = "0.11.3"
 bevy-inspector-egui = "0.20.0"
 bevy_panorbit_camera = "0.8.0"
 bincode2 = "2.0.1"
 bytemuck = "1.14.0"
 flate2 = "1.0.28"
 ply-rs = "0.1.3"
+rand = "0.8.5"
 serde = "1.0.189"
+wgpu = "0.16.0"
 
 
 [target.'cfg(target_arch = "wasm32")'.dependencies]
 console_error_panic_hook = "0.1.7"
 wasm-bindgen = "0.2.87"
 
+
+# TODO: use minimal bevy features
+[dependencies.bevy]
+version = "0.11.3"
+default-features = true
+
+
 [dependencies.web-sys]
 version = "0.3.4"
 features = [
@@ -63,8 +69,8 @@ codegen-units = 1
 path = "src/lib.rs"
 
 [[bin]]
-name = "bevy_gaussian_splatting"
-path = "src/main.rs"
+name = "viewer"
+path = "viewer/viewer.rs"
 
 [[bin]]
 name = "ply_to_gcloud"

README.md

@@ -12,13 +12,15 @@
 
 bevy gaussian splatting render pipeline plugin
 
-`cargo run -- {path to ply or gcloud.gz file}`
+`cargo run -- {path to ply or gcloud file}`
 
 ## capabilities
 
 - [X] ply to gcloud converter
 - [X] gcloud and ply asset loaders
 - [X] bevy gaussian cloud render pipeline
+- [ ] wasm support /w [live demo](https://mosure.github.io/bevy_gaussian_splatting)
+- [ ] temporal depth sorting
 - [ ] f16 and f32 gcloud support
 - [ ] 4D gaussian clouds via morph targets
 - [ ] bevy_openxr support
@@ -43,14 +45,25 @@ fn setup_gaussian_cloud(
     asset_server: Res<AssetServer>,
 ) {
     commands.spawn(GaussianSplattingBundle {
-        cloud: asset_server.load("scenes/icecream.ply"),
+        cloud: asset_server.load("scenes/icecream.gcloud"),
         ..Default::default()
     });
 
     commands.spawn(Camera3dBundle::default());
 }
 ```
 
+## tools
+
+- [ply to gcloud converter](tools/README.md#ply-to-gcloud-converter)
+- [] gaussian cloud training tool
+
+## wasm support
+
+to build wasm run:
+- `cargo build --target wasm32-unknown-unknown --release`
+- `wasm-bindgen --out-dir ./out/ --target web ./target/`
+
 
 ## compatible bevy versions
 
@@ -64,11 +77,13 @@ fn setup_gaussian_cloud(
 - [4d gaussians](https://github.com/hustvl/4DGaussians)
 - [bevy](https://github.com/bevyengine/bevy)
 - [bevy-hanabi](https://github.com/djeedai/bevy_hanabi)
+- [deformable-3d-gaussians](https://github.com/ingra14m/Deformable-3D-Gaussians)
 - [diff-gaussian-rasterization](https://github.com/graphdeco-inria/diff-gaussian-rasterization)
 - [dreamgaussian](https://github.com/dreamgaussian/dreamgaussian)
 - [dynamic-3d-gaussians](https://github.com/JonathonLuiten/Dynamic3DGaussians)
 - [ewa splatting](https://www.cs.umd.edu/~zwicker/publications/EWASplatting-TVCG02.pdf)
 - [gaussian-splatting](https://github.com/graphdeco-inria/gaussian-splatting)
+- [gaussian-splatting-viewer](https://github.com/limacv/GaussianSplattingViewer/tree/main)
 - [gaussian-splatting-web](https://github.com/cvlab-epfl/gaussian-splatting-web)
 - [making gaussian splats smaller](https://aras-p.info/blog/2023/09/13/Making-Gaussian-Splats-smaller/)
 - [onesweep](https://arxiv.org/ftp/arxiv/papers/2206/2206.01784.pdf)

src/gaussian.rs

@@ -1,3 +1,4 @@
+use rand::seq::SliceRandom;
 use std::{
     io::{
         BufReader,
@@ -41,7 +42,9 @@ const fn num_sh_coefficients(degree: usize) -> usize {
     }
 }
 const SH_DEGREE: usize = 3;
-pub const MAX_SH_COEFF_COUNT: usize = num_sh_coefficients(SH_DEGREE) * 3;
+pub const SH_CHANNELS: usize = 3;
+pub const MAX_SH_COEFF_COUNT_PER_CHANNEL: usize = num_sh_coefficients(SH_DEGREE);
+pub const MAX_SH_COEFF_COUNT: usize = MAX_SH_COEFF_COUNT_PER_CHANNEL * SH_CHANNELS;
 #[derive(
     Clone,
     Copy,
@@ -95,8 +98,9 @@ where
             A: serde::de::SeqAccess<'de>,
         {
             let mut coefficients = [0.0; MAX_SH_COEFF_COUNT];
-            for i in 0..MAX_SH_COEFF_COUNT {
-                coefficients[i] = seq
+
+            for (i, coefficient) in coefficients.iter_mut().enumerate().take(MAX_SH_COEFF_COUNT) {
+                *coefficient = seq
                     .next_element()?
                     .ok_or_else(|| serde::de::Error::invalid_length(i, &self))?;
             }
@@ -126,11 +130,9 @@ pub const MAX_SIZE_VARIANCE: f32 = 5.0;
 // TODO: support f16 gaussian clouds (shader and asset loader)
 pub struct Gaussian {
     pub rotation: [f32; 4],
-    pub position: Vec3,
-    pub scale: Vec3,
-    pub opacity: f32,
+    pub position: [f32; 4],
+    pub scale_opacity: [f32; 4],
     pub spherical_harmonic: SphericalHarmonicCoefficients,
-    padding: f32,
 }
 
 #[derive(
@@ -153,9 +155,18 @@ impl GaussianCloud {
                 0.0,
                 0.0,
             ],
-            position: Vec3::new(0.0, 0.0, 0.0),
-            scale: Vec3::new(0.5, 0.5, 0.5),
-            opacity: 0.8,
+            position: [
+                0.0,
+                0.0,
+                0.0,
+                1.0,
+            ],
+            scale_opacity: [
+                0.5,
+                0.5,
+                0.5,
+                0.5,
+            ],
             spherical_harmonic: SphericalHarmonicCoefficients{
                 coefficients: [
                     1.0, 0.0, 1.0,
@@ -176,16 +187,18 @@ impl GaussianCloud {
                     0.6, 0.1, 0.2,
                 ],
             },
-            padding: 0.0,
         };
         let mut cloud = GaussianCloud(Vec::new());
 
         for &x in [-0.5, 0.5].iter() {
             for &y in [-0.5, 0.5].iter() {
                 for &z in [-0.5, 0.5].iter() {
-                    let mut g = origin.clone();
-                    g.position = Vec3::new(x, y, z);
+                    let mut g = origin;
+                    g.position = [x, y, z, 1.0];
                     cloud.0.push(g);
+
+                    let mut rng = rand::thread_rng();
+                    cloud.0.last_mut().unwrap().spherical_harmonic.coefficients.shuffle(&mut rng);
                 }
             }
         }
@@ -235,14 +248,16 @@ impl AssetLoader for GaussianCloudLoader {
                     let cloud = GaussianCloud(ply_cloud);
 
                     load_context.set_default_asset(LoadedAsset::new(cloud));
-                    return Ok(());
+
+                    Ok(())
                 },
                 Some(ext) if ext == "gcloud" => {
                     let decompressed = GzDecoder::new(bytes);
                     let cloud: GaussianCloud = deserialize_from(decompressed).expect("failed to decode cloud");
 
                     load_context.set_default_asset(LoadedAsset::new(cloud));
-                    return Ok(());
+
+                    Ok(())
                 },
                 _ => Ok(()),
             }

src/lib.rs

@@ -32,6 +32,7 @@ pub struct GaussianSplattingPlugin;
 impl Plugin for GaussianSplattingPlugin {
     fn build(&self, app: &mut App) {
         // TODO: allow hot reloading of GaussianCloud handle through inspector UI
+        app.register_type::<SphericalHarmonicCoefficients>();
         app.register_type::<GaussianCloud>();
         app.add_asset::<GaussianCloud>();
         app.register_asset_reflect::<GaussianCloud>();

src/ply.rs

@@ -1,9 +1,6 @@
 use std::io::BufRead;
 
-use bevy::{
-    asset::Error,
-    math::Vec3,
-};
+use bevy::asset::Error;
 use ply_rs::{
     ply::{
         Property,
@@ -14,7 +11,9 @@ use ply_rs::{
 
 use crate::gaussian::{
     Gaussian,
+    MAX_SH_COEFF_COUNT_PER_CHANNEL,
     MAX_SIZE_VARIANCE,
+    SH_CHANNELS,
 };
 
 
@@ -25,19 +24,16 @@ impl PropertyAccess for Gaussian {
 
     fn set_property(&mut self, key: String, property: Property) {
         match (key.as_ref(), property) {
-            ("x", Property::Float(v))           => self.position.x = v,
-            ("y", Property::Float(v))           => self.position.y = v,
-            ("z", Property::Float(v))           => self.position.z = v,
-            // ("nx", Property::Float(v))          => self.normal.x = v,
-            // ("ny", Property::Float(v))          => self.normal.y = v,
-            // ("nz", Property::Float(v))          => self.normal.z = v,
+            ("x", Property::Float(v))           => self.position[0] = v,
+            ("y", Property::Float(v))           => self.position[1] = v,
+            ("z", Property::Float(v))           => self.position[2] = v,
             ("f_dc_0", Property::Float(v))      => self.spherical_harmonic.coefficients[0] = v,
             ("f_dc_1", Property::Float(v))      => self.spherical_harmonic.coefficients[1] = v,
             ("f_dc_2", Property::Float(v))      => self.spherical_harmonic.coefficients[2] = v,
-            ("opacity", Property::Float(v))     => self.opacity = 1.0 / (1.0 + (-v).exp()),
-            ("scale_0", Property::Float(v))     => self.scale.x = v,
-            ("scale_1", Property::Float(v))     => self.scale.y = v,
-            ("scale_2", Property::Float(v))     => self.scale.z = v,
+            ("scale_0", Property::Float(v))     => self.scale_opacity[0] = v,
+            ("scale_1", Property::Float(v))     => self.scale_opacity[1] = v,
+            ("scale_2", Property::Float(v))     => self.scale_opacity[2] = v,
+            ("opacity", Property::Float(v))     => self.scale_opacity[3] = 1.0 / (1.0 + (-v).exp()),
             ("rot_0", Property::Float(v))       => self.rotation[0] = v,
             ("rot_1", Property::Float(v))       => self.rotation[1] = v,
             ("rot_2", Property::Float(v))       => self.rotation[2] = v,
@@ -47,7 +43,6 @@ impl PropertyAccess for Gaussian {
 
                 match i {
                     _ if i + 3 < self.spherical_harmonic.coefficients.len() => {
-                        // TODO: verify this is the correct sh order (packed not planar)
                         self.spherical_harmonic.coefficients[i + 3] = v;
                     },
                     _ => { },
@@ -65,18 +60,36 @@ pub fn parse_ply(mut reader: &mut dyn BufRead) -> Result<Vec<Gaussian>, Error> {
     let mut cloud = Vec::new();
 
     for (_ignore_key, element) in &header.elements {
-        match element.name.as_ref() {
-            "vertex" => { cloud = gaussian_parser.read_payload_for_element(&mut reader, &element, &header)?; },
-            _ => {},
+        if element.name == "vertex" {
+            cloud = gaussian_parser.read_payload_for_element(&mut reader, element, &header)?;
         }
     }
 
     for gaussian in &mut cloud {
-        let mean_scale = (gaussian.scale.x + gaussian.scale.y + gaussian.scale.z) / 3.0;
-        gaussian.scale = gaussian.scale
-            .max(Vec3::splat(mean_scale - MAX_SIZE_VARIANCE))
-            .min(Vec3::splat(mean_scale + MAX_SIZE_VARIANCE))
-            .exp();
+        gaussian.position[3] = 1.0;
+
+        let mean_scale = (gaussian.scale_opacity[0] + gaussian.scale_opacity[1] + gaussian.scale_opacity[2]) / 3.0;
+        for i in 0..3 {
+            gaussian.scale_opacity[i] = gaussian.scale_opacity[i]
+                .max(mean_scale - MAX_SIZE_VARIANCE)
+                .min(mean_scale + MAX_SIZE_VARIANCE)
+                .exp();
+        }
+
+        let sh_src = gaussian.spherical_harmonic.coefficients;
+        let sh = &mut gaussian.spherical_harmonic.coefficients;
+
+        for (i, sh_src) in sh_src.iter().enumerate().skip(SH_CHANNELS) {
+            let j = i - SH_CHANNELS;
+
+            let channel = j / (MAX_SH_COEFF_COUNT_PER_CHANNEL - 1);
+            let coefficient = (j % (MAX_SH_COEFF_COUNT_PER_CHANNEL - 1)) + 1;
+
+            let interleaved_idx = coefficient * SH_CHANNELS + channel;
+            assert!(interleaved_idx >= SH_CHANNELS);
+
+            sh[interleaved_idx] = *sh_src;
+        }
     }
 
     Ok(cloud)