Skip to content

Commit 2cf00d0

Browse files
mvaligurskyMartin ValigurskyCopilot
authored
Initial implementation of unified gsplat rendering (#7825)
* Initial implementation of unified gsplat rendering * lint * types * comments * optimized path for splat resources without lod + cheaper shader without lookup * lint * types * handle splats that change transform, by copying them to work buffer * renamed shader chunks * unused variable * cleanup * cleanup * cleanup * update to allow work buffers to be updated while waiting for the sorting results. Preserves smooth splat movement in the world. * generalized LOD to handle different number than 3 hardcoded levels * updated centers management - removed frequent large allocation (reuse buffer now), removed copy when passing to worker * fixed to first frame handling * adjustment * moving splat updates its lod * SH handling support * example lint * correct global sorting of splats with scales * lint * handling adding / removing splats at runtime * lint * cleanup * handles adding / removing splats * lint * renderer * refactor * cleanup, fixes * lint * example lint * Update src/scene/gsplat/unified/gsplat-centers-buffer.js Co-authored-by: Copilot <[email protected]> * Update src/scene/gsplat/unified/gsplat-work-buffer.js Co-authored-by: Copilot <[email protected]> * Apply suggestions from code review Co-authored-by: Copilot <[email protected]> --------- Co-authored-by: Martin Valigursky <[email protected]> Co-authored-by: Copilot <[email protected]>
1 parent 6d9e442 commit 2cf00d0

34 files changed

+2651
-159
lines changed

‎examples/src/examples/gaussian-splatting/lod.example.mjs

Lines changed: 187 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,187 @@
1+
// @config HIDDEN
2+
import { deviceType, rootPath } from 'examples/utils';
3+
import * as pc from 'playcanvas';
4+
5+
const canvas = /** @type {HTMLCanvasElement} */ (document.getElementById('application-canvas'));
6+
window.focus();
7+
8+
const gfxOptions = {
9+
deviceTypes: [deviceType],
10+
11+
// disable antialiasing as gaussian splats do not benefit from it and it's expensive
12+
antialias: false
13+
};
14+
15+
const device = await pc.createGraphicsDevice(canvas, gfxOptions);
16+
device.maxPixelRatio = Math.min(window.devicePixelRatio, 2);
17+
18+
const createOptions = new pc.AppOptions();
19+
createOptions.graphicsDevice = device;
20+
createOptions.mouse = new pc.Mouse(document.body);
21+
createOptions.touch = new pc.TouchDevice(document.body);
22+
createOptions.keyboard = new pc.Keyboard(document.body);
23+
24+
createOptions.componentSystems = [
25+
pc.RenderComponentSystem,
26+
pc.CameraComponentSystem,
27+
pc.LightComponentSystem,
28+
pc.ScriptComponentSystem,
29+
pc.GSplatComponentSystem
30+
];
31+
createOptions.resourceHandlers = [pc.TextureHandler, pc.ContainerHandler, pc.ScriptHandler, pc.GSplatHandler];
32+
33+
const app = new pc.AppBase(canvas);
34+
app.init(createOptions);
35+
36+
// Set the canvas to fill the window and automatically change resolution to be the same as the canvas size
37+
app.setCanvasFillMode(pc.FILLMODE_FILL_WINDOW);
38+
app.setCanvasResolution(pc.RESOLUTION_AUTO);
39+
40+
// Ensure canvas is resized when window changes size
41+
const resize = () => app.resizeCanvas();
42+
window.addEventListener('resize', resize);
43+
app.on('destroy', () => {
44+
window.removeEventListener('resize', resize);
45+
});
46+
47+
pc.Tracing.set(pc.TRACEID_SHADER_ALLOC, true);
48+
49+
const assets = {
50+
biker: new pc.Asset('gsplat', 'gsplat', { url: `${rootPath}/static/assets/splats/biker.compressed.ply` }),
51+
// church: new pc.Asset('gsplat', 'gsplat', { url: `${rootPath}/static/assets/splats/church.ply` }),
52+
// church: new pc.Asset('gsplat', 'gsplat', { url: `${rootPath}/static/assets/splats/onsen.ply` }),
53+
// church: new pc.Asset('gsplat', 'gsplat', { url: `${rootPath}/static/assets/splats/uzumasa.ply` }),
54+
church: new pc.Asset('gsplat', 'gsplat', { url: `${rootPath}/static/assets/splats/morocco.ply` }),
55+
56+
// logo: new pc.Asset('gsplat', 'gsplat', { url: `${rootPath}/static/assets/splats/playcanvas-logo/meta.json` }),
57+
logo: new pc.Asset('gsplat', 'gsplat', { url: `${rootPath}/static/assets/splats/pclogo.ply` }),
58+
// logo: new pc.Asset('gsplat', 'gsplat', { url: `${rootPath}/static/assets/splats/anneli.ply` }),
59+
// logo: new pc.Asset('gsplat', 'gsplat', { url: `${rootPath}/static/assets/splats/museum.ply` }),
60+
// logo: new pc.Asset('gsplat', 'gsplat', { url: `${rootPath}/static/assets/splats/uzumasa.ply` }),
61+
guitar: new pc.Asset('gsplat', 'gsplat', { url: `${rootPath}/static/assets/splats/guitar.compressed.ply` }),
62+
63+
shoe: new pc.Asset('gsplat', 'gsplat', { url: `${rootPath}/static/assets/splats/shoe-with-sh.ply` }),
64+
shoeNoSh: new pc.Asset('gsplat', 'gsplat', { url: `${rootPath}/static/assets/splats/shoe-without-sh.ply` }),
65+
66+
// pokemon: new pc.Asset('gsplat', 'gsplat', { url: `${rootPath}/static/assets/splats/pokemon.ply` }),
67+
skull: new pc.Asset('gsplat', 'gsplat', { url: `${rootPath}/static/assets/splats/skull.ply` }),
68+
69+
fly: new pc.Asset('fly', 'script', { url: `${rootPath}/static/scripts/camera/fly-camera.js` }),
70+
orbit: new pc.Asset('script', 'script', { url: `${rootPath}/static/scripts/camera/orbit-camera.js` })
71+
};
72+
73+
const assetListLoader = new pc.AssetListLoader(Object.values(assets), app.assets);
74+
assetListLoader.load(() => {
75+
app.start();
76+
77+
// create a splat entity and place it in the world
78+
const biker = new pc.Entity();
79+
biker.setLocalPosition(2.5, 1, 1);
80+
biker.setLocalEulerAngles(180, 90, 0);
81+
// biker.setLocalScale(0.7, 0.7, 0.7);
82+
// biker.setLocalScale(7, 7, 7);
83+
84+
const biker2 = new pc.Entity();
85+
biker2.setLocalPosition(2.5, 1, 0);
86+
biker2.setLocalEulerAngles(180, 90, 0);
87+
// biker2.setLocalScale(0.7, 0.7, 0.7);
88+
biker2.setLocalScale(7, 7, 7);
89+
90+
91+
const logo = new pc.Entity();
92+
logo.setLocalPosition(0, 1.5, 1);
93+
logo.setLocalEulerAngles(180, 0, 0);
94+
logo.setLocalScale(0.5, 0.5, 0.5);
95+
96+
const church = new pc.Entity();
97+
church.setLocalEulerAngles(180, 90, 0);
98+
99+
const guitar = new pc.Entity();
100+
guitar.setLocalPosition(0, 0.6, 4);
101+
guitar.setLocalEulerAngles(180, 0, 0);
102+
guitar.setLocalScale(0.5, 0.5, 0.5);
103+
104+
// Create an Entity with a camera component
105+
const camera = new pc.Entity();
106+
camera.addComponent('camera', {
107+
clearColor: new pc.Color(0.2, 0.2, 0.2),
108+
fov: 75,
109+
toneMapping: pc.TONEMAP_ACES
110+
});
111+
camera.setLocalPosition(-0.8, 2, 3);
112+
camera.addComponent('script');
113+
camera.script.create('orbitCameraInputMouse');
114+
camera.script.create('orbitCameraInputTouch');
115+
camera.script.create('flyCamera', {
116+
attributes: {
117+
speed: 2
118+
}
119+
});
120+
121+
app.root.addChild(camera);
122+
123+
// temporary API
124+
const manager = new pc.GSplatManager(app.graphicsDevice, camera);
125+
126+
const worldLayer = app.scene.layers.getLayerByName('World');
127+
worldLayer.addMeshInstances([manager.renderer.meshInstance]);
128+
129+
manager.add(assets.church.resource, church);
130+
manager.add(assets.guitar.resource, guitar);
131+
manager.add(assets.logo.resource, logo);
132+
133+
134+
let timeToChange = 1;
135+
let time = 0;
136+
let guitarTime = 0;
137+
let added = false;
138+
app.on('update', (/** @type {number} */ dt) => {
139+
time += dt;
140+
timeToChange -= dt;
141+
142+
logo.rotateLocal(0, 100 * dt, 0);
143+
144+
// each even second, update the guitar as well
145+
// if (Math.floor(time) % 2 === 0) {
146+
guitarTime += dt;
147+
148+
// orbit guitar around
149+
guitar.setLocalPosition(0.5 * Math.sin(guitarTime), 2, 0.5 * Math.cos(guitarTime) + 1);
150+
// }
151+
152+
// ping pong logo between two positions along x-axies
153+
logo.setLocalPosition(5.5 + 5 * Math.sin(time), 1.5, -2);
154+
155+
156+
if (timeToChange <= 0) {
157+
158+
if (!added) {
159+
console.log('adding shoe');
160+
added = true;
161+
timeToChange = 1;
162+
163+
// worldLayer.removeMeshInstances([manager.meshInstance]);
164+
manager.add(assets.skull.resource, biker);
165+
// manager.add(assets.shoe.resource, biker2);
166+
// worldLayer.addMeshInstances([manager.meshInstance]);
167+
168+
} else {
169+
console.log('removing shoe');
170+
added = false;
171+
timeToChange = 1;
172+
173+
// worldLayer.removeMeshInstances([manager.meshInstance]);
174+
175+
176+
manager.remove(biker);
177+
// manager.remove(biker2);
178+
// worldLayer.addMeshInstances([manager.meshInstance]);
179+
}
180+
}
181+
182+
183+
manager.update();
184+
});
185+
});
186+
187+
export { app };

‎src/index.js

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -235,6 +235,7 @@ export { GSplatData } from './scene/gsplat/gsplat-data.js';
235235
export { GSplatResourceBase } from './scene/gsplat/gsplat-resource-base.js';
236236
export { GSplatResource } from './scene/gsplat/gsplat-resource.js';
237237
export { GSplatInstance } from './scene/gsplat/gsplat-instance.js';
238+
export { GSplatManager } from './scene/gsplat/unified/gsplat-manager.js'; // temporary till proper API is exposed
238239
export { GSplatSogsData } from './scene/gsplat/gsplat-sogs-data.js';
239240
export { GSplatSogsResource } from './scene/gsplat/gsplat-sogs-resource.js';
240241

‎src/scene/gsplat/gsplat-instance.js

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,12 @@ import { MeshInstance } from '../mesh-instance.js';
55
import { GSplatResolveSH } from './gsplat-resolve-sh.js';
66
import { GSplatSorter } from './gsplat-sorter.js';
77
import { GSplatSogsData } from './gsplat-sogs-data.js';
8+
import { GSplatResourceBase } from './gsplat-resource-base.js';
89
import { ShaderMaterial } from '../materials/shader-material.js';
910
import { BLEND_NONE, BLEND_PREMULTIPLIED } from '../constants.js';
1011

1112
/**
1213
* @import { Camera } from '../camera.js'
13-
* @import { GSplatResourceBase } from './gsplat-resource-base.js'
1414
* @import { GraphNode } from '../graph-node.js'
1515
* @import { Texture } from '../../platform/graphics/texture.js'
1616
*/
@@ -113,7 +113,7 @@ class GSplatInstance {
113113
this.sorter.init(this.orderTexture, centers, chunks);
114114
this.sorter.on('updated', (count) => {
115115
// limit splat render count to exclude those behind the camera
116-
this.meshInstance.instancingCount = Math.ceil(count / resource.instanceSize);
116+
this.meshInstance.instancingCount = Math.ceil(count / GSplatResourceBase.instanceSize);
117117

118118
// update splat count on the material
119119
this.material.setParameter('numSplats', count);

‎src/scene/gsplat/gsplat-resource-base.js

Lines changed: 87 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,10 @@ import { Mesh } from '../mesh.js';
99

1010
/**
1111
* @import { GraphicsDevice } from '../../platform/graphics/graphics-device.js'
12-
* @import { GSplatData } from './gsplat-data.js';
13-
* @import { GSplatCompressedData } from './gsplat-compressed-data.js';
14-
* @import { GSplatSogsData } from './gsplat-sogs-data.js';
12+
* @import { GSplatData } from './gsplat-data.js'
13+
* @import { GSplatCompressedData } from './gsplat-compressed-data.js'
14+
* @import { GSplatSogsData } from './gsplat-sogs-data.js'
15+
* @import { GSplatLodBlocks } from './unified/gsplat-lod-blocks.js'
1516
*/
1617

1718
/**
@@ -26,6 +27,9 @@ class GSplatResourceBase {
2627
/** @type {GSplatData | GSplatCompressedData | GSplatSogsData} */
2728
gsplatData;
2829

30+
/** @type {GSplatLodBlocks|null} */
31+
lodBlocks = null;
32+
2933
/** @type {Float32Array} */
3034
centers;
3135

@@ -38,6 +42,9 @@ class GSplatResourceBase {
3842
/** @type {VertexBuffer} */
3943
instanceIndices;
4044

45+
/** @type {boolean} */
46+
hasLod = false;
47+
4148
constructor(device, gsplatData) {
4249
this.device = device;
4350
this.gsplatData = gsplatData;
@@ -49,22 +56,24 @@ class GSplatResourceBase {
4956
gsplatData.calcAabb(this.aabb);
5057

5158
// construct the mesh
59+
this.mesh = GSplatResourceBase.createMesh(device);
60+
this.instanceIndices = GSplatResourceBase.createInstanceIndices(device, gsplatData.numSplats);
5261

53-
// number of quads to combine into a single instance. this is to increase occupancy
54-
// in the vertex shader.
55-
const splatInstanceSize = 128;
56-
const numSplats = Math.ceil(gsplatData.numSplats / splatInstanceSize) * splatInstanceSize;
57-
const numSplatInstances = numSplats / splatInstanceSize;
62+
// keep extra reference since mesh is shared between instances
63+
this.mesh.incRefCount();
5864

59-
// specify the base splat index per instance
60-
const indexData = new Uint32Array(numSplatInstances);
61-
for (let i = 0; i < numSplatInstances; ++i) {
62-
indexData[i] = i * splatInstanceSize;
63-
}
65+
this.mesh.aabb.copy(this.aabb);
66+
}
6467

65-
const vertexFormat = new VertexFormat(device, [
66-
{ semantic: SEMANTIC_ATTR13, components: 1, type: TYPE_UINT32, asInt: true }
67-
]);
68+
destroy() {
69+
this.mesh?.destroy();
70+
this.instanceIndices?.destroy();
71+
}
72+
73+
static createMesh(device) {
74+
// number of quads to combine into a single instance. this is to increase occupancy
75+
// in the vertex shader.
76+
const splatInstanceSize = GSplatResourceBase.instanceSize;
6877

6978
// build the instance mesh
7079
const meshPositions = new Float32Array(12 * splatInstanceSize);
@@ -83,28 +92,37 @@ class GSplatResourceBase {
8392
], i * 6);
8493
}
8594

86-
this.mesh = new Mesh(device);
87-
this.mesh.setPositions(meshPositions, 3);
88-
this.mesh.setIndices(meshIndices);
89-
this.mesh.update();
95+
const mesh = new Mesh(device);
96+
mesh.setPositions(meshPositions, 3);
97+
mesh.setIndices(meshIndices);
98+
mesh.update();
9099

91-
// keep extra reference since mesh is shared between instances
92-
this.mesh.incRefCount();
100+
return mesh;
101+
}
93102

94-
this.mesh.aabb.copy(this.aabb);
103+
static createInstanceIndices(device, splatCount) {
104+
const splatInstanceSize = GSplatResourceBase.instanceSize;
105+
const numSplats = Math.ceil(splatCount / splatInstanceSize) * splatInstanceSize;
106+
const numSplatInstances = numSplats / splatInstanceSize;
107+
108+
const indexData = new Uint32Array(numSplatInstances);
109+
for (let i = 0; i < numSplatInstances; ++i) {
110+
indexData[i] = i * splatInstanceSize;
111+
}
95112

96-
this.instanceIndices = new VertexBuffer(device, vertexFormat, numSplatInstances, {
113+
const vertexFormat = new VertexFormat(device, [
114+
{ semantic: SEMANTIC_ATTR13, components: 1, type: TYPE_UINT32, asInt: true }
115+
]);
116+
117+
const instanceIndices = new VertexBuffer(device, vertexFormat, numSplatInstances, {
97118
usage: BUFFER_STATIC,
98119
data: indexData.buffer
99120
});
100-
}
101121

102-
destroy() {
103-
this.mesh?.destroy();
104-
this.instanceIndices?.destroy();
122+
return instanceIndices;
105123
}
106124

107-
get instanceSize() {
125+
static get instanceSize() {
108126
return 128; // number of splats per instance
109127
}
110128

@@ -150,6 +168,46 @@ class GSplatResourceBase {
150168
});
151169
}
152170

171+
/**
172+
* Calculate block centers by averaging splat centers within each block
173+
*
174+
* @param {number} numSplats - Total number of splats
175+
* @param {number} blockSize - Size of each block
176+
* @param {number} numBlocks - Number of blocks (avoids recalculation)
177+
* @param {Float32Array} blocksCenter - Output array for block centers (3 floats per block)
178+
* @protected
179+
*/
180+
calculateBlockCenters(numSplats, blockSize, numBlocks, blocksCenter) {
181+
for (let blockIdx = 0; blockIdx < numBlocks; blockIdx++) {
182+
const startIdx = blockIdx * blockSize;
183+
const endIdx = Math.min(startIdx + blockSize, numSplats);
184+
const blockSplatCount = endIdx - startIdx;
185+
186+
// Calculate block center by averaging all splat centers in this block
187+
let centerX = 0, centerY = 0, centerZ = 0;
188+
for (let i = startIdx; i < endIdx; i++) {
189+
const centerBase = i * 3;
190+
centerX += this.centers[centerBase];
191+
centerY += this.centers[centerBase + 1];
192+
centerZ += this.centers[centerBase + 2];
193+
}
194+
195+
// Store average center in blocksCenter
196+
const blockCenterBase = blockIdx * 3;
197+
blocksCenter[blockCenterBase] = centerX / blockSplatCount;
198+
blocksCenter[blockCenterBase + 1] = centerY / blockSplatCount;
199+
blocksCenter[blockCenterBase + 2] = centerZ / blockSplatCount;
200+
}
201+
}
202+
203+
/**
204+
* Generate LODs if supported. To be implemented by derived classes.
205+
*
206+
* @protected
207+
*/
208+
generateLods() {
209+
}
210+
153211
instantiate() {
154212
Debug.removed('GSplatResource.instantiate is removed. Use gsplat component instead');
155213
}

0 commit comments

Comments
 (0)