Skip to content

WebXRManager: Add Raw Camera Access module. #31487

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 13 commits into from
Jul 30, 2025
1 change: 1 addition & 0 deletions examples/files.json
Original file line number Diff line number Diff line change
Expand Up @@ -475,6 +475,7 @@
"webaudio_visualizer"
],
"webxr": [
"webxr_ar_camera_access",
"webxr_ar_cones",
"webxr_ar_hittest",
"webxr_ar_lighting",
Expand Down
Binary file added examples/screenshots/webxr_ar_camera_access.jpg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
138 changes: 138 additions & 0 deletions examples/webxr_ar_camera_access.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,138 @@
<!DOCTYPE html>
<html lang="en">
<head>
<title>three.js ar - camera access</title>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=no">
<link type="text/css" rel="stylesheet" href="main.css">
</head>
<body>

<div id="info">
<a href="https://threejs.org" target="_blank" rel="noopener">three.js</a> ar - camera access<br/>
</div>

<script type="importmap">
{
"imports": {
"three": "../build/three.module.js",
"three/addons/": "./jsm/"
}
}
</script>

<script type="module">

import * as THREE from 'three';
import { ARButton } from 'three/addons/webxr/ARButton.js';

const clock = new THREE.Clock();

let camera, scene, renderer;
let cube;

init();

async function init() {

const container = document.createElement( 'div' );
document.body.appendChild( container );

scene = new THREE.Scene();

camera = new THREE.PerspectiveCamera( 70, window.innerWidth / window.innerHeight, 0.01, 20 );

const defaultLight = new THREE.HemisphereLight( 0xffffff, 0xbbbbff, 3 );
defaultLight.position.set( 0.5, 1, 0.25 );
scene.add( defaultLight );

//

renderer = new THREE.WebGLRenderer( { antialias: true, alpha: true } );
renderer.setPixelRatio( window.devicePixelRatio );
renderer.setSize( window.innerWidth, window.innerHeight );
renderer.setAnimationLoop( animate );
renderer.xr.enabled = true;
container.appendChild( renderer.domElement );

//

document.body.appendChild( ARButton.createButton( renderer, { requiredFeatures: [ 'camera-access' ] } ) );

//

const boxGeometry = new THREE.BoxGeometry( 1, 1, 1 );
const boxMaterial = new THREE.MeshStandardMaterial();
cube = new THREE.Mesh( boxGeometry, boxMaterial );
cube.position.z = -3;

scene.add( cube );

//

window.addEventListener( 'resize', onWindowResize );

}

function onWindowResize() {

camera.aspect = window.innerWidth / window.innerHeight;
camera.updateProjectionMatrix();

renderer.setSize( window.innerWidth, window.innerHeight );

}

//

function setCameraTexture() {

if ( ! renderer.xr.getSession() ) {

if ( cube.material.map ) {

cube.material.map = null;
cube.material.needsUpdate = true;

}

return;

}

const frame = renderer.xr.getFrame();
const referenceSpace = renderer.xr.getReferenceSpace();

if ( !frame || !referenceSpace ) return;

const viewerPose = frame.getViewerPose( referenceSpace );

if ( ! viewerPose ) return;

const view = viewerPose.views.find( view => view.camera );

const cameraTexture = renderer.xr.getCameraTexture( view.camera );

if ( cube.material.map === cameraTexture ) return;

cube.material.map = cameraTexture;
cube.material.needsUpdate = true;

}

function animate() {

const delta = clock.getDelta();

setCameraTexture();

cube.rotation.x += delta;
cube.rotation.y += delta;

renderer.render( scene, camera );

}

</script>
</body>
</html>
6 changes: 5 additions & 1 deletion src/renderers/webgl/WebGLTextures.js
Original file line number Diff line number Diff line change
Expand Up @@ -515,7 +515,7 @@ function WebGLTextures( _gl, extensions, state, properties, capabilities, utils,

if ( texture.isVideoTexture ) updateVideoTexture( texture );

if ( texture.isRenderTargetTexture === false && texture.version > 0 && textureProperties.__version !== texture.version ) {
if ( texture.isRenderTargetTexture === false && texture.isRawTexture !== true && texture.version > 0 && textureProperties.__version !== texture.version ) {

const image = texture.image;

Expand All @@ -534,6 +534,10 @@ function WebGLTextures( _gl, extensions, state, properties, capabilities, utils,

}

} else if ( texture.isRawTexture ) {

textureProperties.__webglTexture = texture.sourceTexture ? texture.sourceTexture : null;

}

state.bindTexture( _gl.TEXTURE_2D, textureProperties.__webglTexture, _gl.TEXTURE0 + slot );
Expand Down
16 changes: 6 additions & 10 deletions src/renderers/webxr/WebXRDepthSensing.js
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { PlaneGeometry } from '../../geometries/PlaneGeometry.js';
import { ShaderMaterial } from '../../materials/ShaderMaterial.js';
import { Mesh } from '../../objects/Mesh.js';
import { Texture } from '../../textures/Texture.js';
import { RawTexture } from '../../textures/RawTexture.js';

const _occlusion_vertex = `
void main() {
Expand Down Expand Up @@ -42,9 +42,9 @@ class WebXRDepthSensing {
constructor() {

/**
* A texture representing the depth of the user's environment.
* An opaque texture representing the depth of the user's environment.
*
* @type {?Texture}
* @type {?RawTexture}
*/
this.texture = null;

Expand Down Expand Up @@ -74,18 +74,14 @@ class WebXRDepthSensing {
/**
* Inits the depth sensing module
*
* @param {WebGLRenderer} renderer - The renderer.
* @param {XRWebGLDepthInformation} depthData - The XR depth data.
* @param {XRRenderState} renderState - The XR render state.
*/
init( renderer, depthData, renderState ) {
init( depthData, renderState ) {

if ( this.texture === null ) {

const texture = new Texture();

const texProps = renderer.properties.get( texture );
texProps.__webglTexture = depthData.texture;
const texture = new RawTexture( depthData.depthTexture );

if ( ( depthData.depthNear !== renderState.depthNear ) || ( depthData.depthFar !== renderState.depthFar ) ) {

Expand Down Expand Up @@ -146,7 +142,7 @@ class WebXRDepthSensing {
/**
* Returns a texture representing the depth of the user's environment.
*
* @return {?Texture} The depth texture.
* @return {?RawTexture} The depth texture.
*/
getDepthTexture() {

Expand Down
57 changes: 56 additions & 1 deletion src/renderers/webxr/WebXRManager.js
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import { WebGLAnimation } from '../webgl/WebGLAnimation.js';
import { WebGLRenderTarget } from '../WebGLRenderTarget.js';
import { WebXRController } from './WebXRController.js';
import { DepthTexture } from '../../textures/DepthTexture.js';
import { RawTexture } from '../../textures/RawTexture.js';
import { DepthFormat, DepthStencilFormat, RGBAFormat, UnsignedByteType, UnsignedIntType, UnsignedInt248Type } from '../../constants.js';
import { WebXRDepthSensing } from './WebXRDepthSensing.js';

Expand Down Expand Up @@ -52,6 +53,7 @@ class WebXRManager extends EventDispatcher {
let xrFrame = null;

const depthSensing = new WebXRDepthSensing();
const cameraAccessTextures = {};
const attributes = gl.getContextAttributes();

let initialRenderTarget = null;
Expand Down Expand Up @@ -232,6 +234,11 @@ class WebXRManager extends EventDispatcher {
_currentDepthFar = null;

depthSensing.reset();
for ( const key in cameraAccessTextures ) {

delete cameraAccessTextures[ key ];

}

// restore framebuffer/rendering state

Expand Down Expand Up @@ -873,6 +880,19 @@ class WebXRManager extends EventDispatcher {

};

/**
* Retrieves an opaque texture from the view-aligned {@link XRCamera}.
* Only available during the current animation loop.
*
* @param {XRCamera} xrCamera - The camera to query.
* @return {?Texture} An opaque texture representing the current raw camera frame.
*/
this.getCameraTexture = function ( xrCamera ) {

return cameraAccessTextures[ xrCamera ];

};

// Animation Loop

let onAnimationFrameCallback = null;
Expand Down Expand Up @@ -978,7 +998,42 @@ class WebXRManager extends EventDispatcher {

if ( depthData && depthData.isValid && depthData.texture ) {

depthSensing.init( renderer, depthData, session.renderState );
depthSensing.init( depthData, session.renderState );

}

}

const cameraAccessEnabled = enabledFeatures &&
enabledFeatures.includes( 'camera-access' );

if ( cameraAccessEnabled ) {

renderer.state.unbindTexture();

if ( glBinding ) {

for ( let i = 0; i < views.length; i ++ ) {

const camera = views[ i ].camera;

if ( camera ) {

let cameraTex = cameraAccessTextures[ camera ];

if ( ! cameraTex ) {

cameraTex = new RawTexture();
cameraAccessTextures[ camera ] = cameraTex;

}

const glTexture = glBinding.getCameraImage( camera );
cameraTex.sourceTexture = glTexture;

}

}

}

Expand Down
45 changes: 45 additions & 0 deletions src/textures/RawTexture.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
import { Texture } from './Texture.js';

/**
* Represents a texture created externally from the renderer context.
*
* This may be a texture from a protected media stream, device camera feed,
* or other data feeds like a depth sensor.
*
* Note that this class is only supported in {@link WebGLRenderer} right now.
*
* @augments Texture
*/
class RawTexture extends Texture {
Copy link
Owner

@mrdoob mrdoob Jul 31, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe ExternalTexture could be a better name?
Are there other uses for it? If not, may be WebXRCameraTexture?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We also use it for webxr depth textures, but there was some debate about naming in the linked issues and RawTexture was settled on. I could see it being useful for anything that uses raw texture handles, possibly protected content?


/**
* Creates a new raw texture.
*
* @param {?WebGLTexture} [sourceTexture=null] - The external texture.
*/
constructor( sourceTexture = null ) {

super();

/**
* The external source texture.
*
* @type {?WebGLTexture}
* @default null
*/
this.sourceTexture = sourceTexture;

/**
* This flag can be used for type testing.
*
* @type {boolean}
* @readonly
* @default true
*/
this.isRawTexture = true;

}

}

export { RawTexture };