feat: render pipeline (#2)

* feat: render pipeline (scaffold) * refactor: minimize vertex buffer copies * feat: render pipeline end-to-end scaffold * feat: bind groups and vertex buffer layout * asset: icecream.ply * docs: credit dreamgaussian * fix: render device query & more correct bind groups * refactor: extracting comments and code shuffle * docs: flair * docs: clarity * chore: add bench/docs/linting * fix: proper uniform usage (buffer size errors remain) * refactor: separate gaussian uniforms and gaussian cloud buffer * fix: pipeline validation errors * tri vertex output * chore: launch config and ignore .DS_Store * chore: don't modify binary ply/splats * chore: fix bevy assets in vscode debug * feat: test gaussian cloud * feat: ellipsoid projection, triangle strip, simplify fragment shader * fix: gaussian struct padding & ellipse UV (needs camera fix) * fix: camera controls * feat: test cube * fix: cube
mosure · Oct 13, 2023 · bbc56c9 · bbc56c9
1 parent 9a93b6d
commit bbc56c9
Show file tree

Hide file tree

Showing 14 changed files with 1,165 additions and 201 deletions.
diff --git a/.gitattributes b/.gitattributes
@@ -0,0 +1,8 @@
+* text=auto eol=lf
+*.{cmd,[cC][mM][dD]} text eol=crlf
+*.{bat,[bB][aA][tT]} text eol=crlf
+*.sh text eol=lf
+*.conf text eol=lf
+
+*.ply binary
+*.splat binary
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -19,7 +19,7 @@ jobs:
         os: [windows-latest, macos-latest]
 
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 60
+    timeout-minutes: 120
 
     steps:
     - uses: actions/checkout@v3
@@ -36,5 +36,8 @@ jobs:
     - name: build
       run: cargo build
 
-    - name: run tests
+    - name: lint
+      run: cargo clippy
+
+    - name: test
       run: cargo test
diff --git a/.gitignore b/.gitignore
@@ -6,3 +6,5 @@ Cargo.lock
 
 
 *.ply
+
+.DS_Store
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -0,0 +1,73 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "type": "lldb",
+            "request": "launch",
+            "name": "Debug unit tests in library 'bevy_gaussian_splatting'",
+            "cargo": {
+                "args": [
+                    "test",
+                    "--no-run",
+                    "--lib",
+                    "--package=bevy_gaussian_splatting"
+                ],
+                "filter": {
+                    "name": "bevy_gaussian_splatting",
+                    "kind": "lib"
+                }
+            },
+            "args": [],
+            "cwd": "${workspaceFolder}",
+            "env": {
+                "CARGO_MANIFEST_DIR": "${workspaceFolder}",
+            }
+        },
+        {
+            "type": "lldb",
+            "request": "launch",
+            "name": "Debug executable 'bevy_gaussian_splatting'",
+            "cargo": {
+                "args": [
+                    "build",
+                    "--bin=bevy_gaussian_splatting",
+                    "--package=bevy_gaussian_splatting"
+                ],
+                "filter": {
+                    "name": "bevy_gaussian_splatting",
+                    "kind": "bin"
+                }
+            },
+            "args": [],
+            "cwd": "${workspaceFolder}",
+            "env": {
+                "CARGO_MANIFEST_DIR": "${workspaceFolder}",
+            }
+        },
+        {
+            "type": "lldb",
+            "request": "launch",
+            "name": "Debug unit tests in executable 'bevy_gaussian_splatting'",
+            "cargo": {
+                "args": [
+                    "test",
+                    "--no-run",
+                    "--bin=bevy_gaussian_splatting",
+                    "--package=bevy_gaussian_splatting"
+                ],
+                "filter": {
+                    "name": "bevy_gaussian_splatting",
+                    "kind": "bin"
+                }
+            },
+            "args": [],
+            "cwd": "${workspaceFolder}",
+            "env": {
+                "CARGO_MANIFEST_DIR": "${workspaceFolder}",
+            }
+        }
+    ]
+}
diff --git a/Cargo.toml b/Cargo.toml
@@ -15,6 +15,7 @@ exclude = [".devcontainer", ".github", "docs", "dist", "build", "assets", "credi
 [dependencies]
 bevy = "0.11.2"
 bevy_panorbit_camera = "0.8.0"
+bytemuck = "1.14.0"
 ply-rs = "0.1.3"
 
 

diff --git a/README.md b/README.md
@@ -1,4 +1,4 @@
-# bevy_gaussian_splatting
+# bevy_gaussian_splatting 🌌
 
 [![test](https://github.com/mosure/bevy_gaussian_splatting/workflows/test/badge.svg)](https://github.com/Mosure/bevy_gaussian_splatting/actions?query=workflow%3Atest)
 [![GitHub License](https://img.shields.io/github/license/mosure/bevy_gaussian_splatting)](https://raw.githubusercontent.com/mosure/bevy_gaussian_splatting/main/LICENSE)
@@ -15,8 +15,8 @@ bevy gaussian splatting render pipeline plugin
 ## capabilities
 
 - [ ] bevy gaussian cloud render pipeline
+- [ ] 4D gaussian clouds via morph targets
 - [ ] bevy 3D camera to gaussian cloud pipeline
-- [ ] 4D gaussian clouds
 
 ## usage
 
@@ -37,7 +37,7 @@ fn setup_gaussian_cloud(
     asset_server: Res<AssetServer>,
 ) {
     commands.spawn(GaussianSplattingBundle {
-        verticies: asset_server.load("scenes/test.ply"),
+        cloud: asset_server.load("scenes/icecream.ply"),
         ..Default::default()
     });
 
@@ -56,10 +56,17 @@ fn setup_gaussian_cloud(
 # credits
 
 - [bevy](https://github.com/bevyengine/bevy)
+- [bevy-hanabi](https://github.com/djeedai/bevy_hanabi)
 - [diff-gaussian-rasterization](https://github.com/graphdeco-inria/diff-gaussian-rasterization)
+- [dreamgaussian](https://github.com/dreamgaussian/dreamgaussian)
 - [dynamic-3d-gaussians](https://github.com/JonathonLuiten/Dynamic3DGaussians)
 - [gaussian-splatting](https://github.com/graphdeco-inria/gaussian-splatting)
 - [gaussian-splatting-web](https://github.com/cvlab-epfl/gaussian-splatting-web)
+- [making gaussian splats smaller](https://aras-p.info/blog/2023/09/13/Making-Gaussian-Splats-smaller/)
+- [onesweep](https://arxiv.org/ftp/arxiv/papers/2206/2206.01784.pdf)
 - [point-visualizer](https://github.com/mosure/point-visualizer)
 - [rusty-automata](https://github.com/mosure/rusty-automata)
+- [splat](https://github.com/antimatter15/splat)
+- [splatter](https://github.com/Lichtso/splatter)
 - [sturdy-dollop](https://github.com/mosure/sturdy-dollop)
+- [taichi_3d_gaussian_splatting](https://github.com/wanmeihuali/taichi_3d_gaussian_splatting)
diff --git a/assets/scenes/icecream.ply b/assets/scenes/icecream.ply
diff --git a/src/gaussian.rs b/src/gaussian.rs
@@ -1,6 +1,9 @@
-use std::io::{
-    BufReader,
-    Cursor,
+use std::{
+    io::{
+        BufReader,
+        Cursor,
+    },
+    marker::Copy,
 };
 
 use bevy::{
@@ -10,19 +13,21 @@ use bevy::{
         LoadContext,
         LoadedAsset,
     },
-    reflect::TypeUuid,
+    reflect::{
+        TypePath,
+        TypeUuid,
+    },
+    render::render_resource::ShaderType,
     utils::BoxedFuture,
 };
+use bytemuck::{
+    Pod,
+    Zeroable,
+};
 
 use crate::ply::parse_ply;
 
 
-#[derive(Clone, Debug, Default, Reflect)]
-pub struct AnisotropicCovariance {
-    pub mean: Vec3,
-    pub covariance: Mat3,
-}
-
 const fn num_sh_coefficients(degree: usize) -> usize {
     if degree == 0 {
         1
@@ -32,30 +37,101 @@ const fn num_sh_coefficients(degree: usize) -> usize {
 }
 const SH_DEGREE: usize = 3;
 pub const MAX_SH_COEFF_COUNT: usize = num_sh_coefficients(SH_DEGREE) * 3;
-#[derive(Clone, Debug, Reflect)]
+#[derive(Clone, Copy, ShaderType, Pod, Zeroable)]
+#[repr(C)]
 pub struct SphericalHarmonicCoefficients {
-    pub coefficients: [Vec3; MAX_SH_COEFF_COUNT],
+    pub coefficients: [f32; MAX_SH_COEFF_COUNT],
 }
 impl Default for SphericalHarmonicCoefficients {
     fn default() -> Self {
         Self {
-            coefficients: [Vec3::ZERO; MAX_SH_COEFF_COUNT],
+            coefficients: [0.0; MAX_SH_COEFF_COUNT],
         }
     }
 }
 
-#[derive(Clone, Debug, Default, Reflect)]
+#[derive(Clone, Default, Copy, ShaderType, Pod, Zeroable)]
+#[repr(C)]
 pub struct Gaussian {
-    pub normal: Vec3,
+    //pub anisotropic_covariance: AnisotropicCovariance,
+    //pub normal: Vec3,
+    pub rotation: [f32; 4],
+    pub position: Vec3,
+    pub scale: Vec3,
     pub opacity: f32,
-    pub transform: Transform,
-    pub anisotropic_covariance: AnisotropicCovariance,
     pub spherical_harmonic: SphericalHarmonicCoefficients,
+    padding: f32,
 }
 
-#[derive(Clone, Debug, Reflect, TypeUuid)]
+#[derive(Clone, TypeUuid, TypePath)]
 #[uuid = "ac2f08eb-bc32-aabb-ff21-51571ea332d5"]
-pub struct GaussianCloud(Vec<Gaussian>);
+pub struct GaussianCloud(pub Vec<Gaussian>);
+
+impl GaussianCloud {
+    pub fn test_model() -> Self {
+        let origin = Gaussian {
+            rotation: [
+                1.0,
+                0.0,
+                0.0,
+                0.0,
+            ],
+            position: Vec3::new(0.0, 0.0, 0.0),
+            scale: Vec3::new(0.5, 0.5, 0.5),
+            opacity: 0.8,
+            spherical_harmonic: SphericalHarmonicCoefficients{
+                coefficients: [
+                    1.0, 0.0, 1.0,
+                    0.0, 0.5, 0.0,
+                    0.3, 0.2, 0.0,
+                    0.4, 0.0, 0.2,
+                    0.1, 0.0, 0.0,
+                    0.0, 0.3, 0.3,
+                    0.0, 1.0, 1.0,
+                    0.3, 0.0, 0.0,
+                    0.0, 0.0, 0.0,
+                    0.0, 0.3, 1.0,
+                    0.5, 0.3, 0.0,
+                    0.2, 0.3, 0.1,
+                    0.6, 0.3, 0.1,
+                    0.0, 0.3, 0.2,
+                    0.0, 0.5, 0.3,
+                    0.6, 0.1, 0.2,
+                ],
+            },
+            padding: 0.0,
+        };
+        let mut cloud = GaussianCloud(Vec::new());
+
+        for &x in [-1.0, 1.0].iter() {
+            for &y in [-1.0, 1.0].iter() {
+                for &z in [-1.0, 1.0].iter() {
+                    let mut g = origin.clone();
+                    g.position = Vec3::new(x, y, z);
+                    cloud.0.push(g);
+                }
+            }
+        }
+
+        cloud
+    }
+}
+
+
+#[derive(Component, Reflect, Clone)]
+pub struct GaussianCloudSettings {
+    pub global_scale: f32,
+    pub global_transform: GlobalTransform,
+}
+
+impl Default for GaussianCloudSettings {
+    fn default() -> Self {
+        Self {
+            global_scale: 1.0,
+            global_transform: Transform::IDENTITY.into(),
+        }
+    }
+}
 
 
 #[derive(Default)]

diff --git a/src/lib.rs b/src/lib.rs
@@ -1,8 +1,11 @@
 use bevy::prelude::*;
 
-use gaussian::{
+pub use gaussian::{
+    Gaussian,
     GaussianCloud,
     GaussianCloudLoader,
+    GaussianCloudSettings,
+    SphericalHarmonicCoefficients,
 };
 
 use render::RenderPipelinePlugin;
@@ -13,22 +16,28 @@ pub mod render;
 pub mod utils;
 
 
-#[derive(Component, Default)]
+#[derive(Bundle, Default, Reflect)]
 pub struct GaussianSplattingBundle {
-    pub transform: Transform,
-    pub verticies: Handle<GaussianCloud>,
+    pub settings: GaussianCloudSettings, // TODO: implement global transform
+    pub cloud: Handle<GaussianCloud>,
 }
 
-// TODO: add render pipeline config
+
+#[derive(Component, Default)]
+struct GaussianSplattingCamera;
+// TODO: filter camera 3D entities
+
 pub struct GaussianSplattingPlugin;
 
 impl Plugin for GaussianSplattingPlugin {
     fn build(&self, app: &mut App) {
         app.add_asset::<GaussianCloud>();
         app.init_asset_loader::<GaussianCloudLoader>();
 
-        app.add_plugins(RenderPipelinePlugin);
+        app.register_type::<GaussianSplattingBundle>();
 
-        // TODO: add GaussianSplattingBundle system
+        app.add_plugins((
+            RenderPipelinePlugin,
+        ));
     }
 }
diff --git a/src/main.rs b/src/main.rs
@@ -12,6 +12,7 @@ use bevy_panorbit_camera::{
 };
 
 use bevy_gaussian_splatting::{
+    GaussianCloud,
     GaussianSplattingBundle,
     GaussianSplattingPlugin,
     utils::setup_hooks,
@@ -41,10 +42,13 @@ impl Default for GaussianSplattingViewer {
 
 fn setup_gaussian_cloud(
     mut commands: Commands,
-    asset_server: Res<AssetServer>,
+    _asset_server: Res<AssetServer>,
+    mut gaussian_assets: ResMut<Assets<GaussianCloud>>,
 ) {
+    let cloud = gaussian_assets.add(GaussianCloud::test_model());
     commands.spawn(GaussianSplattingBundle {
-        verticies: asset_server.load("scenes/test.ply"),
+        cloud,
+        // cloud: _asset_server.load("scenes/icecream.ply"),
         ..Default::default()
     });