Scene view component (#2947)

* added scene_view component

* fixed timing issue with uninizialized app

* added tests

* Added website demo

* code review

* fixed pytests

* code cleanup

* Added additional camera parameters

* removed text renderer

* some cleanup

* remove non-functional text renderers

* clean up tests

* add note about current limitation

* simplify waiting for parent scene

* simplify demo layout

* remove duplicate camera functions

* undo unnecessary change

* try to fix flaky test

---------

Co-authored-by: Falko Schindler <falko@zauberzeug.com>

nicegui/elements/scene_view.js

@@ -0,0 +1,154 @@
+import * as THREE from "three";
+
+export default {
+  template: `
+    <div style="position:relative">
+      <canvas style="position:relative"></canvas>
+    </div>`,
+
+  async mounted() {
+    await this.$nextTick();
+    this.scene = getElement(this.scene_id).scene;
+
+    if (this.camera_type === "perspective") {
+      this.camera = new THREE.PerspectiveCamera(
+        this.camera_params.fov,
+        this.width / this.height,
+        this.camera_params.near,
+        this.camera_params.far
+      );
+    } else {
+      this.camera = new THREE.OrthographicCamera(
+        (-this.camera_params.size / 2) * (this.width / this.height),
+        (this.camera_params.size / 2) * (this.width / this.height),
+        this.camera_params.size / 2,
+        -this.camera_params.size / 2,
+        this.camera_params.near,
+        this.camera_params.far
+      );
+    }
+    this.look_at = new THREE.Vector3(0, 0, 0);
+    this.camera.lookAt(this.look_at);
+    this.camera.up = new THREE.Vector3(0, 0, 1);
+    this.camera.position.set(0, -3, 5);
+
+    this.renderer = undefined;
+    try {
+      this.renderer = new THREE.WebGLRenderer({
+        antialias: true,
+        alpha: true,
+        canvas: this.$el.children[0],
+      });
+    } catch {
+      this.$el.innerHTML = "Could not create WebGL renderer.";
+      this.$el.style.width = this.width + "px";
+      this.$el.style.height = this.height + "px";
+      this.$el.style.padding = "10px";
+      this.$el.style.border = "1px solid silver";
+      return;
+    }
+    this.renderer.setClearColor("#eee");
+    this.renderer.setSize(this.width, this.height);
+
+    this.$nextTick(() => this.resize());
+    window.addEventListener("resize", this.resize, false);
+
+    const render = () => {
+      requestAnimationFrame(() => setTimeout(() => render(), 1000 / 20));
+      TWEEN.update();
+      this.renderer.render(this.scene, this.camera);
+    };
+    render();
+
+    const raycaster = new THREE.Raycaster();
+    const click_handler = (mouseEvent) => {
+      let x = (mouseEvent.offsetX / this.renderer.domElement.width) * 2 - 1;
+      let y = -(mouseEvent.offsetY / this.renderer.domElement.height) * 2 + 1;
+      raycaster.setFromCamera({ x: x, y: y }, this.camera);
+      this.$emit("click3d", {
+        hits: raycaster
+          .intersectObjects(this.scene.children, true)
+          .filter((o) => o.object.object_id)
+          .map((o) => ({
+            object_id: o.object.object_id,
+            object_name: o.object.name,
+            point: o.point,
+          })),
+        click_type: mouseEvent.type,
+        button: mouseEvent.button,
+        alt_key: mouseEvent.altKey,
+        ctrl_key: mouseEvent.ctrlKey,
+        meta_key: mouseEvent.metaKey,
+        shift_key: mouseEvent.shiftKey,
+      });
+    };
+    this.$el.onclick = click_handler;
+    this.$el.ondblclick = click_handler;
+
+    const connectInterval = setInterval(() => {
+      if (window.socket.id === undefined) return;
+      this.$emit("init", { socket_id: window.socket.id });
+      clearInterval(connectInterval);
+    }, 100);
+  },
+
+  beforeDestroy() {
+    window.removeEventListener("resize", this.resize);
+  },
+
+  methods: {
+    move_camera(x, y, z, look_at_x, look_at_y, look_at_z, up_x, up_y, up_z, duration) {
+      if (this.camera_tween) this.camera_tween.stop();
+      this.camera_tween = new TWEEN.Tween([
+        this.camera.position.x,
+        this.camera.position.y,
+        this.camera.position.z,
+        this.camera.up.x,
+        this.camera.up.y,
+        this.camera.up.z,
+        this.look_at.x,
+        this.look_at.y,
+        this.look_at.z,
+      ])
+        .to(
+          [
+            x === null ? this.camera.position.x : x,
+            y === null ? this.camera.position.y : y,
+            z === null ? this.camera.position.z : z,
+            up_x === null ? this.camera.up.x : up_x,
+            up_y === null ? this.camera.up.y : up_y,
+            up_z === null ? this.camera.up.z : up_z,
+            look_at_x === null ? this.look_at.x : look_at_x,
+            look_at_y === null ? this.look_at.y : look_at_y,
+            look_at_z === null ? this.look_at.z : look_at_z,
+          ],
+          duration * 1000
+        )
+        .onUpdate((p) => {
+          this.camera.position.set(p[0], p[1], p[2]);
+          this.camera.up.set(p[3], p[4], p[5]); // NOTE: before calling lookAt
+          this.look_at.set(p[6], p[7], p[8]);
+          this.camera.lookAt(p[6], p[7], p[8]);
+        })
+        .start();
+    },
+    resize() {
+      const { clientWidth, clientHeight } = this.$el;
+      this.renderer.setSize(clientWidth, clientHeight);
+      this.camera.aspect = clientWidth / clientHeight;
+      if (this.camera_type === "orthographic") {
+        this.camera.left = (-this.camera.aspect * this.camera_params.size) / 2;
+        this.camera.right = (this.camera.aspect * this.camera_params.size) / 2;
+      }
+      this.camera.updateProjectionMatrix();
+    },
+  },
+
+  props: {
+    width: Number,
+    height: Number,
+    camera_type: String,
+    camera_params: Object,
+    scene_id: String,
+  },
+};

nicegui/elements/scene_view.py

@@ -0,0 +1,129 @@
+import asyncio
+from typing import Any, Callable, Optional
+
+from typing_extensions import Self
+
+from ..awaitable_response import AwaitableResponse, NullResponse
+from ..element import Element
+from ..events import GenericEventArguments, SceneClickEventArguments, SceneClickHit, handle_event
+from .scene import Scene, SceneCamera
+
+
+class SceneView(Element,
+                component='scene_view.js',
+                libraries=['lib/tween/tween.umd.js'],
+                exposed_libraries=['lib/three/three.module.js']):
+
+    def __init__(self,
+                 scene: Scene,
+                 width: int = 400,
+                 height: int = 300,
+                 camera: Optional[SceneCamera] = None,
+                 on_click: Optional[Callable[..., Any]] = None,
+                 ) -> None:
+        """Scene View
+
+        Display an additional view of a 3D scene using `three.js <https://threejs.org/>`_.
+        This component can only show a scene and not modify it.
+        You can, however, independently move the camera.
+
+        Current limitation: 2D and 3D text objects are not supported and will not be displayed in the scene view.
+
+        :param scene: the scene which will be shown on the canvas
+        :param width: width of the canvas
+        :param height: height of the canvas
+        :param camera: camera definition, either instance of ``ui.scene.perspective_camera`` (default) or ``ui.scene.orthographic_camera``
+        :param on_click: callback to execute when a 3D object is clicked
+        """
+        super().__init__()
+        self._props['width'] = width
+        self._props['height'] = height
+        self._props['scene_id'] = scene.id
+        self.camera = camera or Scene.perspective_camera()
+        self._props['camera_type'] = self.camera.type
+        self._props['camera_params'] = self.camera.params
+        self._click_handlers = [on_click] if on_click else []
+        self.is_initialized = False
+        self.on('init', self._handle_init)
+        self.on('click3d', self._handle_click)
+
+    def on_click(self, callback: Callable[..., Any]) -> Self:
+        """Add a callback to be invoked when a 3D object is clicked."""
+        self._click_handlers.append(callback)
+        return self
+
+    def _handle_init(self, e: GenericEventArguments) -> None:
+        self.is_initialized = True
+        with self.client.individual_target(e.args['socket_id']):
+            self.move_camera(duration=0)
+
+    async def initialized(self) -> None:
+        """Wait until the scene is initialized."""
+        event = asyncio.Event()
+        self.on('init', event.set, [])
+        await self.client.connected()
+        await event.wait()
+
+    def run_method(self, name: str, *args: Any, timeout: float = 1, check_interval: float = 0.01) -> AwaitableResponse:
+        if not self.is_initialized:
+            return NullResponse()
+        return super().run_method(name, *args, timeout=timeout, check_interval=check_interval)
+
+    def _handle_click(self, e: GenericEventArguments) -> None:
+        arguments = SceneClickEventArguments(
+            sender=self,
+            client=self.client,
+            click_type=e.args['click_type'],
+            button=e.args['button'],
+            alt=e.args['alt_key'],
+            ctrl=e.args['ctrl_key'],
+            meta=e.args['meta_key'],
+            shift=e.args['shift_key'],
+            hits=[SceneClickHit(
+                object_id=hit['object_id'],
+                object_name=hit['object_name'],
+                x=hit['point']['x'],
+                y=hit['point']['y'],
+                z=hit['point']['z'],
+            ) for hit in e.args['hits']],
+        )
+        for handler in self._click_handlers:
+            handle_event(handler, arguments)
+
+    def move_camera(self,
+                    x: Optional[float] = None,
+                    y: Optional[float] = None,
+                    z: Optional[float] = None,
+                    look_at_x: Optional[float] = None,
+                    look_at_y: Optional[float] = None,
+                    look_at_z: Optional[float] = None,
+                    up_x: Optional[float] = None,
+                    up_y: Optional[float] = None,
+                    up_z: Optional[float] = None,
+                    duration: float = 0.5) -> None:
+        """Move the camera to a new position.
+
+        :param x: camera x position
+        :param y: camera y position
+        :param z: camera z position
+        :param look_at_x: camera look-at x position
+        :param look_at_y: camera look-at y position
+        :param look_at_z: camera look-at z position
+        :param up_x: x component of the camera up vector
+        :param up_y: y component of the camera up vector
+        :param up_z: z component of the camera up vector
+        :param duration: duration of the movement in seconds (default: `0.5`)
+        """
+        self.camera.x = self.camera.x if x is None else x
+        self.camera.y = self.camera.y if y is None else y
+        self.camera.z = self.camera.z if z is None else z
+        self.camera.look_at_x = self.camera.look_at_x if look_at_x is None else look_at_x
+        self.camera.look_at_y = self.camera.look_at_y if look_at_y is None else look_at_y
+        self.camera.look_at_z = self.camera.look_at_z if look_at_z is None else look_at_z
+        self.camera.up_x = self.camera.up_x if up_x is None else up_x
+        self.camera.up_y = self.camera.up_y if up_y is None else up_y
+        self.camera.up_z = self.camera.up_z if up_z is None else up_z
+        self.run_method('move_camera',
+                        self.camera.x, self.camera.y, self.camera.z,
+                        self.camera.look_at_x, self.camera.look_at_y, self.camera.look_at_z,
+                        self.camera.up_x, self.camera.up_y, self.camera.up_z, duration)

nicegui/ui.py

@@ -71,6 +71,7 @@ __all__ = [
     'restructured_text',
     'row',
     'scene',
+    'scene_view'
     'scroll_area',
     'select',
     'separator',
@@ -194,6 +195,7 @@ from .elements.range import Range as range  # pylint: disable=redefined-builtin
 from .elements.restructured_text import ReStructuredText as restructured_text
 from .elements.row import Row as row
 from .elements.scene import Scene as scene
+from .elements.scene_view import SceneView as scene_view
 from .elements.scroll_area import ScrollArea as scroll_area
 from .elements.select import Select as select
 from .elements.separator import Separator as separator

tests/test_scene_view.py

@@ -0,0 +1,49 @@
+from typing import Optional
+
+import numpy as np
+
+from nicegui import ui
+from nicegui.testing import Screen
+
+
+def test_create_dynamically(screen: Screen):
+    scene = ui.scene()
+    scene_view: Optional[ui.scene_view] = None
+
+    def create():
+        nonlocal scene_view
+        scene_view = ui.scene_view(scene)
+    ui.button('Create', on_click=create)
+
+    screen.open('/')
+    screen.click('Create')
+    screen.wait(0.5)
+    assert scene_view is not None
+    assert screen.selenium.execute_script(f'return getElement({scene_view.id}).scene == getElement({scene.id}).scene')
+
+
+def test_object_creation_via_context(screen: Screen):
+    with ui.scene() as scene:
+        scene.box()
+
+    scene_view = ui.scene_view(scene)
+
+    screen.open('/')
+    screen.wait(1)
+    assert screen.selenium.execute_script(f'return getElement({scene_view.id}).scene == getElement({scene.id}).scene')
+
+
+def test_camera_move(screen: Screen):
+    with ui.scene() as scene:
+        scene.box()
+
+    scene_view = ui.scene_view(scene)
+
+    screen.open('/')
+
+    screen.wait(0.5)
+    scene_view.move_camera(x=1, y=2, z=3, look_at_x=4, look_at_y=5, look_at_z=6, up_x=7, up_y=8, up_z=9, duration=0.0)
+
+    screen.wait(1)
+    position = screen.selenium.execute_script(f'return getElement({scene_view.id}).camera_tween._object')
+    assert np.allclose(position, [1, 2, 3, 7, 8, 9, 4, 5, 6])

website/documentation/content/scene_documentation.py

@@ -113,6 +113,20 @@ async def wait_for_init() -> None:
         scene.move_camera(x=1, y=-1, z=1.5, duration=2)
 
 
+@doc.demo(ui.scene_view)
+def scene_views():
+    with ui.grid(columns=2).classes('w-full'):
+        with ui.scene().classes('h-64 col-span-2') as scene:
+            teapot = 'https://upload.wikimedia.org/wikipedia/commons/9/93/Utah_teapot_(solid).stl'
+            scene.stl(teapot).scale(0.3)
+
+        with ui.scene_view(scene).classes('h-32') as scene_view1:
+            scene_view1.move_camera(x=1, y=-3, z=5)
+
+        with ui.scene_view(scene).classes('h-32') as scene_view2:
+            scene_view2.move_camera(x=0, y=4, z=3)
+
+
 @doc.demo('Camera Parameters', '''
     You can use the `camera` argument to `ui.scene` to use a custom camera.
     This allows you to set the field of view of a perspective camera or the size of an orthographic camera.