immersive-web · cabanier · Jan 24, 2024 · Feb 9, 2024 · Aug 28, 2024 · Sep 9, 2024
diff --git a/proposals/body-tracking.html b/proposals/body-tracking.html
@@ -0,0 +1,271 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+  <meta charset='utf-8'>
+  <meta name='viewport' content='width=device-width, initial-scale=1, user-scalable=no'>
+  <meta name='mobile-web-app-capable' content='yes'>
+  <meta name='apple-mobile-web-app-capable' content='yes'>
+
+  <meta http-equiv="origin-trial"
+    content="Ahfj+MLeL6bh+LNmpnSdepftxoDHHwjUG2KWZ4jjCb1WoZxtBlzF3cDHuJNVqnhr3HXJwQ+kLaw57NO15S0mRwwAAABkeyJvcmlnaW4iOiJodHRwczovL2ltbWVyc2l2ZS13ZWIuZ2l0aHViLmlvOjQ0MyIsImZlYXR1cmUiOiJXZWJYUlBsYW5lRGV0ZWN0aW9uIiwiZXhwaXJ5IjoxNjI5ODQ5NTk5fQ==">
+
+  <title>WebXR body tracking</title>
+
+  <link href='../css/common.css' rel='stylesheet'>
+  </link>
+
+</head>
+
+<body>
+  <header>
+    <details open>
+      <summary>Simple body tracking</summary>
+      This sample demonstrates using the WebXR body tracking API
+      to show your body's joints.
+      <p>
+
+        <a class="back" href="./index.html">Back</a>
+      </p>
+    </details>
+  </header>
+
+  <script type="importmap">
+      {
+        "imports": {
+          "three": "https://cdn.jsdelivr.net/npm/three@0.161.0/build/three.module.js",
+          "three/addons/": "https://cdn.jsdelivr.net/npm/three@0.161.0/examples/jsm/"
+        }
+      }
+    </script>
+
+  <script type="module">
+    // Code adapted from three.js' WebXR hit test sample.
+    // three.js is covered by MIT license which can be found at:
+    // https://github.com/mrdoob/three.js/blob/master/LICENSE
+
+
+    import * as THREE from 'three';
+    import { BoxLineGeometry } from 'three/addons/geometries/BoxLineGeometry.js';
+    import { WebXRButton } from '../js/util/webxr-button.js';
+    import { hitTest, filterHitTestResults } from '../js/hit-test.js';
+    import { mat4 } from '../js/render/math/gl-matrix.js';
+
+    let arButton = null;
+    let vrButton = null;
+    let camera, scene, renderer;
+    let room, spheres, skeleton;
+    const scalehand = new THREE.Matrix4().makeScale(3, 3, 3);
+    let positions = new Float32Array(16 * 83);
+
+    const jointHierarchy = {
+      'spine-lower': 'spine-middle',
+      'spine-middle': 'spine-upper',
+      'spine-upper': 'chest',
+
+      'left-upper-leg': 'hips',
+      'left-lower-leg': 'left-upper-leg',
+
+      'right-upper-leg': 'hips',
+      'right-lower-leg': 'left-upper-leg',
+
+      'left-arm-lower': 'left-arm-upper',
+      'left-arm-upper': 'left-shoulder',
+      'left-scapula': 'chest',
+
+      'right-arm-lower': 'right-arm-upper',
+      'right-arm-upper': 'right-shoulder',
+      'right-scapula': 'chest'
+    };
+
+    init();
+
+    function dist(p) {
+      return Math.sqrt(p.x * p.x + p.y * p.y + p.z * p.z);
+    }
+
+    function init() {
+      scene = new THREE.Scene();
+      scene.background = new THREE.Color(0x505050);
+
+      camera = new THREE.PerspectiveCamera(50, window.innerWidth / window.innerHeight, 0.1, 50);
+
+      const light = new THREE.HemisphereLight(0xffffff, 0xbbbbff, 1);
+      light.position.set(0.5, 1, 0.25);
+      scene.add(light);
+
+      renderer = new THREE.WebGLRenderer({ antialias: true });
+      renderer.setPixelRatio(window.devicePixelRatio);
+      renderer.setSize(window.innerWidth, window.innerHeight);
+      renderer.setAnimationLoop(render);
+      renderer.xr.enabled = true;
+      renderer.autoClear = false;
+      document.body.appendChild(renderer.domElement);
+
+      vrButton = new WebXRButton({
+        onRequestSession: onRequestVRSession,
+        onEndSession: onEndSession,
+        textEnterXRTitle: "START VR",
+        textXRNotFoundTitle: "VR NOT FOUND",
+        textExitXRTitle: "EXIT VR",
+      });
+
+      arButton = new WebXRButton({
+        onRequestSession: onRequestARSession,
+        onEndSession: onEndSession,
+        textEnterXRTitle: "START AR",
+        textXRNotFoundTitle: "AR NOT FOUND",
+        textExitXRTitle: "EXIT AR",
+      });
+
+      document.querySelector('header').appendChild(vrButton.domElement);
+      document.querySelector('header').appendChild(document.createElement("br"));
+      document.querySelector('header').appendChild(arButton.domElement);
+
+      if (navigator.xr) {
+        navigator.xr.isSessionSupported('immersive-ar')
+          .then((supported) => {
+            arButton.enabled = supported;
+          });
+        navigator.xr.isSessionSupported('immersive-vr')
+          .then((supported) => {
+            vrButton.enabled = supported;
+          });
+      }
+
+      window.addEventListener('resize', onWindowResize);
+
+    }
+
+    function onRequestSession(isAR) {
+      let sessionInit = {
+        requiredFeatures: ['body-tracking'],
+        optionalFeatures: ['local-floor', 'bounded-floor'],
+      };
+      navigator.xr.requestSession(isAR ? 'immersive-ar' : 'immersive-vr', sessionInit).then((session) => {
+        session.mode = isAR ? 'immersive-ar' : 'immersive-vr';
+        isAR ? arButton.setSession(session) : vrButton.setSession(session);
+        onSessionStarted(session);
+      });
+    }
+
+    function onRequestVRSession() {
+      onRequestSession(false);
+    }
+
+    function onRequestARSession() {
+      onRequestSession(true);
+    }
+
+    function onSessionStarted(session) {
+      session.addEventListener('end', onSessionEnded);
+
+      renderer.xr.setSession(session);
+
+      renderer.setAnimationLoop(render);
+    }
+
+    function onEndSession(session) {
+      session.end();
+    }
+
+    function onSessionEnded(event) {
+      arButton.setSession(null);
+      vrButton.setSession(null);
+
+      renderer.setAnimationLoop(null);
+    }
+
+    function onWindowResize() {
+      camera.aspect = window.innerWidth / window.innerHeight;
+      camera.updateProjectionMatrix();
+
+      renderer.setSize(window.innerWidth, window.innerHeight);
+    }
+
+    function render(timestamp, frame) {
+      if (frame) {
+
+        if (frame.body) {
+
+          let body = frame.body;
+
+          if (spheres == undefined) {
+            const geometry = new THREE.IcosahedronGeometry(0.01, 3);
+            const material = new THREE.MeshLambertMaterial();
+
+            spheres = new THREE.InstancedMesh(geometry, material, body.size);
+            spheres.translateZ(-1).setRotationFromMatrix(new THREE.Matrix4().makeRotationY(Math.PI));
+            spheres.instanceMatrix.setUsage(THREE.DynamicDrawUsage); // will be updated every frame
+            scene.add(spheres);
+
+            const color = new THREE.Color();
+
+            for (let i = 0; i < spheres.count; i++) {
+              spheres.setColorAt(i, color.setHex(0xffffff * Math.random()));
+            }
+
+            const boxgeometry = new THREE.BoxGeometry(1, 1, 1);
+            boxgeometry.translate(-.5, 0, 0);
+            const boxmaterial = new THREE.MeshBasicMaterial({ color: 0xffffff });
+            skeleton = new THREE.InstancedMesh(boxgeometry, boxmaterial, Object.keys(jointHierarchy).length);
+            skeleton.translateZ(-1).setRotationFromMatrix(new THREE.Matrix4().makeRotationY(Math.PI));
+            skeleton.instanceMatrix.setUsage(THREE.DynamicDrawUsage); // will be updated every frame
+            scene.add(skeleton);
+
+            spheres.instanceMatrix.needsUpdate = true;
+            skeleton.instanceMatrix.needsUpdate = true;
+          }
+
+          let i = 0;
+          let position = [];
+          const matrix = new THREE.Matrix4();
+          const scaledMatrix = new THREE.Matrix4();
+          let space = renderer.xr.getReferenceSpace();
+          frame.fillPoses(body.values(), space, positions);
+          body.forEach(part => {
+            matrix.fromArray(positions.slice(i * 16, (i + 1) * 16));
+
+            if (!part.jointName.includes("hand")) {
+              scaledMatrix.makeScale(3, 3, 3);
+            } else {
+              -             scaledMatrix.identity();
+            }
+            scaledMatrix.copyPosition(matrix);
+
+            spheres.setMatrixAt(i++, scaledMatrix);
+          });
+
+          i = 0;
+          for (const [key, value] of Object.entries(jointHierarchy)) {
+            let distance = dist(frame.getPose(body.get(key), body.get(value)).transform.position);
+            if (key.includes('right')) {
+              distance = -distance;
+            }
+            if (key.includes('scapula')) {
+              distance = -distance;
+            }
+            const pose = frame.getPose(body.get(key), space);
+            const position = pose.transform.position;
+            const orientation = pose.transform.orientation;
+
+            matrix.compose(
+              new THREE.Vector3(position.x, position.y, position.z),
+              new THREE.Quaternion(orientation.x, orientation.y, orientation.z, orientation.w),
+              new THREE.Vector3(distance, 0.02, 0.02)
+            );
+
+            skeleton.setMatrixAt(i++, matrix);
+          }
+
+          spheres.instanceMatrix.needsUpdate = true;
+          skeleton.instanceMatrix.needsUpdate = true;
+        }
+        renderer.render(scene, camera);
+      }
+    }
+
+  </script>
+</body>
+
+</html>
diff --git a/proposals/index.html b/proposals/index.html
@@ -133,6 +133,9 @@ <h2 class='tagline'>Proposals</h2>
               description: 'Demonstrates use of the mesh detection API in an immersive-ar session. ' +
                            'Implements JavaScript-level hit-test on the meshes and leverages the Anchors API.' },
 
+            { title: 'Body tracking', category: 'XR',
+              path: 'body-tracking.html',
+              description: 'Demonstrates use of the body tracking in an immersive session. ' },
           ];
 
           let mainElement = document.getElementById("main");