Merge pull request #1201 from zauberzeug/draggable-scene-objects

Support drag and drop for 3D objects in `ui.scene`

nicegui/elements/lib/three/modules/DragControls.js

@@ -0,0 +1,220 @@
+import {
+	EventDispatcher,
+	Matrix4,
+	Plane,
+	Raycaster,
+	Vector2,
+	Vector3
+} from 'three';
+
+const _plane = new Plane();
+const _raycaster = new Raycaster();
+
+const _pointer = new Vector2();
+const _offset = new Vector3();
+const _intersection = new Vector3();
+const _worldPosition = new Vector3();
+const _inverseMatrix = new Matrix4();
+
+class DragControls extends EventDispatcher {
+
+	constructor( _objects, _camera, _domElement ) {
+
+		super();
+
+		_domElement.style.touchAction = 'none'; // disable touch scroll
+
+		let _selected = null, _hovered = null;
+
+		const _intersections = [];
+
+		//
+
+		const scope = this;
+
+		function activate() {
+
+			_domElement.addEventListener( 'pointermove', onPointerMove );
+			_domElement.addEventListener( 'pointerdown', onPointerDown );
+			_domElement.addEventListener( 'pointerup', onPointerCancel );
+			_domElement.addEventListener( 'pointerleave', onPointerCancel );
+
+		}
+
+		function deactivate() {
+
+			_domElement.removeEventListener( 'pointermove', onPointerMove );
+			_domElement.removeEventListener( 'pointerdown', onPointerDown );
+			_domElement.removeEventListener( 'pointerup', onPointerCancel );
+			_domElement.removeEventListener( 'pointerleave', onPointerCancel );
+
+			_domElement.style.cursor = '';
+
+		}
+
+		function dispose() {
+
+			deactivate();
+
+		}
+
+		function getObjects() {
+
+			return _objects;
+
+		}
+
+		function getRaycaster() {
+
+			return _raycaster;
+
+		}
+
+		function onPointerMove( event ) {
+
+			if ( scope.enabled === false ) return;
+
+			updatePointer( event );
+
+			_raycaster.setFromCamera( _pointer, _camera );
+
+			if ( _selected ) {
+
+				if ( _raycaster.ray.intersectPlane( _plane, _intersection ) ) {
+
+					_selected.position.copy( _intersection.sub( _offset ).applyMatrix4( _inverseMatrix ) );
+
+				}
+
+				scope.dispatchEvent( { type: 'drag', object: _selected } );
+
+				return;
+
+			}
+
+			// hover support
+
+			if ( event.pointerType === 'mouse' || event.pointerType === 'pen' ) {
+
+				_intersections.length = 0;
+
+				_raycaster.setFromCamera( _pointer, _camera );
+				_raycaster.intersectObjects( _objects, true, _intersections );
+
+				if ( _intersections.length > 0 ) {
+
+					const object = _intersections[ 0 ].object;
+
+					_plane.setFromNormalAndCoplanarPoint( _camera.getWorldDirection( _plane.normal ), _worldPosition.setFromMatrixPosition( object.matrixWorld ) );
+
+					if ( _hovered !== object && _hovered !== null ) {
+
+						scope.dispatchEvent( { type: 'hoveroff', object: _hovered } );
+
+						_domElement.style.cursor = 'auto';
+						_hovered = null;
+
+					}
+
+					if ( _hovered !== object ) {
+
+						scope.dispatchEvent( { type: 'hoveron', object: object } );
+
+						_domElement.style.cursor = 'pointer';
+						_hovered = object;
+
+					}
+
+				} else {
+
+					if ( _hovered !== null ) {
+
+						scope.dispatchEvent( { type: 'hoveroff', object: _hovered } );
+
+						_domElement.style.cursor = 'auto';
+						_hovered = null;
+
+					}
+
+				}
+
+			}
+
+		}
+
+		function onPointerDown( event ) {
+
+			if ( scope.enabled === false ) return;
+
+			updatePointer( event );
+
+			_intersections.length = 0;
+
+			_raycaster.setFromCamera( _pointer, _camera );
+			_raycaster.intersectObjects( _objects, true, _intersections );
+
+			if ( _intersections.length > 0 ) {
+
+				_selected = ( scope.transformGroup === true ) ? _objects[ 0 ] : _intersections[ 0 ].object;
+
+				_plane.setFromNormalAndCoplanarPoint( _camera.getWorldDirection( _plane.normal ), _worldPosition.setFromMatrixPosition( _selected.matrixWorld ) );
+
+				if ( _raycaster.ray.intersectPlane( _plane, _intersection ) ) {
+
+					_inverseMatrix.copy( _selected.parent.matrixWorld ).invert();
+					_offset.copy( _intersection ).sub( _worldPosition.setFromMatrixPosition( _selected.matrixWorld ) );
+
+				}
+
+				_domElement.style.cursor = 'move';
+
+				scope.dispatchEvent( { type: 'dragstart', object: _selected } );
+
+			}
+
+
+		}
+
+		function onPointerCancel() {
+
+			if ( scope.enabled === false ) return;
+
+			if ( _selected ) {
+
+				scope.dispatchEvent( { type: 'dragend', object: _selected } );
+
+				_selected = null;
+
+			}
+
+			_domElement.style.cursor = _hovered ? 'pointer' : 'auto';
+
+		}
+
+		function updatePointer( event ) {
+
+			const rect = _domElement.getBoundingClientRect();
+
+			_pointer.x = ( event.clientX - rect.left ) / rect.width * 2 - 1;
+			_pointer.y = - ( event.clientY - rect.top ) / rect.height * 2 + 1;
+
+		}
+
+		activate();
+
+		// API
+
+		this.enabled = true;
+		this.transformGroup = false;
+
+		this.activate = activate;
+		this.deactivate = deactivate;
+		this.dispose = dispose;
+		this.getObjects = getObjects;
+		this.getRaycaster = getRaycaster;
+
+	}
+
+}
+
+export { DragControls };

nicegui/elements/scene.js

@@ -1,6 +1,7 @@
 import * as THREE from "three";
 import { CSS2DRenderer, CSS2DObject } from "CSS2DRenderer";
 import { CSS3DRenderer, CSS3DObject } from "CSS3DRenderer";
+import { DragControls } from "DragControls";
 import { OrbitControls } from "OrbitControls";
 import { STLLoader } from "STLLoader";
 
@@ -59,6 +60,7 @@ export default {
     this.scene = new THREE.Scene();
     this.objects = new Map();
     this.objects.set("scene", this.scene);
+    this.draggable_objects = [];
 
     window["scene_" + this.$el.id] = this.scene; // NOTE: for selenium tests only
 
@@ -117,6 +119,28 @@ export default {
       this.scene.add(grid);
     }
     this.controls = new OrbitControls(this.camera, this.renderer.domElement);
+    this.drag_controls = new DragControls(this.draggable_objects, this.camera, this.renderer.domElement);
+    const applyConstraint = (constraint, position) => {
+      if (!constraint) return;
+      const [variable, expression] = constraint.split("=").map((s) => s.trim());
+      position[variable] = eval(expression.replace(/x|y|z/g, (match) => `(${position[match]})`));
+    };
+    const handleDrag = (event) => {
+      this.drag_constraints.split(",").forEach((constraint) => applyConstraint(constraint, event.object.position));
+      if (event.type === "drag") return;
+      this.$emit(event.type, {
+        type: event.type,
+        object_id: event.object.object_id,
+        object_name: event.object.name,
+        x: event.object.position.x,
+        y: event.object.position.y,
+        z: event.object.position.z,
+      });
+      this.controls.enabled = event.type == "dragend";
+    };
+    this.drag_controls.addEventListener("dragstart", handleDrag);
+    this.drag_controls.addEventListener("drag", handleDrag);
+    this.drag_controls.addEventListener("dragend", handleDrag);
 
     const render = () => {
       requestAnimationFrame(() => setTimeout(() => render(), 1000 / 20));
@@ -300,10 +324,16 @@ export default {
       if (!this.objects.has(object_id)) return;
       this.objects.get(object_id).visible = value;
     },
+    draggable(object_id, value) {
+      if (!this.objects.has(object_id)) return;
+      if (value) this.draggable_objects.push(this.objects.get(object_id));
+      else this.draggable_objects.pop(this.objects.get(object_id));
+    },
     delete(object_id) {
       if (!this.objects.has(object_id)) return;
       this.objects.get(object_id).removeFromParent();
       this.objects.delete(object_id);
+      this.draggable_objects.pop(this.objects.get(object_id));
     },
     set_texture_url(object_id, url) {
       if (!this.objects.has(object_id)) return;
@@ -371,5 +401,6 @@ export default {
     width: Number,
     height: Number,
     grid: Boolean,
+    drag_constraints: String,
   },
 };

nicegui/elements/scene.py

@@ -3,7 +3,8 @@ from typing import Any, Callable, Dict, List, Optional, Union
 
 from .. import binding, globals
 from ..element import Element
-from ..events import GenericEventArguments, SceneClickEventArguments, SceneClickHit, handle_event
+from ..events import (GenericEventArguments, SceneClickEventArguments, SceneClickHit, SceneDragEventArguments,
+                      handle_event)
 from ..helpers import KWONLY_SLOTS
 from .scene_object3d import Object3D
 
@@ -33,6 +34,7 @@ class Scene(Element,
                 'lib/three/three.module.js',
                 'lib/three/modules/CSS2DRenderer.js',
                 'lib/three/modules/CSS3DRenderer.js',
+                'lib/three/modules/DragControls.js',
                 'lib/three/modules/OrbitControls.js',
                 'lib/three/modules/STLLoader.js',
             ]):
@@ -57,10 +59,13 @@ class Scene(Element,
                  height: int = 300,
                  grid: bool = True,
                  on_click: Optional[Callable[..., Any]] = None,
+                 on_drag_start: Optional[Callable[..., Any]] = None,
+                 on_drag_end: Optional[Callable[..., Any]] = None,
+                 drag_constraints: str = '',
                  ) -> None:
         """3D Scene
 
-        Display a 3d scene using `three.js <https://threejs.org/>`_.
+        Display a 3D scene using `three.js <https://threejs.org/>`_.
         Currently NiceGUI supports boxes, spheres, cylinders/cones, extrusions, straight lines, curves and textured meshes.
         Objects can be translated, rotated and displayed with different color, opacity or as wireframes.
         They can also be grouped to apply joint movements.
@@ -68,7 +73,10 @@ class Scene(Element,
         :param width: width of the canvas
         :param height: height of the canvas
         :param grid: whether to display a grid
-        :param on_click: callback to execute when a 3d object is clicked
+        :param on_click: callback to execute when a 3D object is clicked
+        :param on_drag_start: callback to execute when a 3D object is dragged
+        :param on_drag_end: callback to execute when a 3D object is dropped
+        :param drag_constraints: comma-separated JavaScript expression for constraining positions of dragged objects (e.g. ``'x = 0, z = y / 2'``)
         """
         super().__init__()
         self._props['width'] = width
@@ -78,9 +86,14 @@ class Scene(Element,
         self.stack: List[Union[Object3D, SceneObject]] = [SceneObject()]
         self.camera: SceneCamera = SceneCamera()
         self.on_click = on_click
+        self.on_drag_start = on_drag_start
+        self.on_drag_end = on_drag_end
         self.is_initialized = False
         self.on('init', self.handle_init)
         self.on('click3d', self.handle_click)
+        self.on('dragstart', self.handle_drag)
+        self.on('dragend', self.handle_drag)
+        self._props['drag_constraints'] = drag_constraints
 
     def __enter__(self) -> 'Scene':
         Object3D.current_scene = self
@@ -124,6 +137,21 @@ class Scene(Element,
         )
         handle_event(self.on_click, arguments)
 
+    def handle_drag(self, e: GenericEventArguments) -> None:
+        arguments = SceneDragEventArguments(
+            sender=self,
+            client=self.client,
+            type=e.args['type'],
+            object_id=e.args['object_id'],
+            object_name=e.args['object_name'],
+            x=e.args['x'],
+            y=e.args['y'],
+            z=e.args['z'],
+        )
+        if arguments.type == 'dragend':
+            self.objects[arguments.object_id].move(arguments.x, arguments.y, arguments.z)
+        handle_event(self.on_drag_start if arguments.type == 'dragstart' else self.on_drag_end, arguments)
+
     def __len__(self) -> int:
         return len(self.objects)
 

nicegui/elements/scene_object3d.py

@@ -24,6 +24,7 @@ class Object3D:
         self.opacity: float = 1.0
         self.side_: str = 'front'
         self.visible_: bool = True
+        self.draggable_: bool = False
         self.x: float = 0
         self.y: float = 0
         self.z: float = 0
@@ -46,6 +47,7 @@ class Object3D:
         self._rotate()
         self._scale()
         self._visible()
+        self._draggable()
 
     def __enter__(self):
         self.scene.stack.append(self)
@@ -75,6 +77,9 @@ class Object3D:
     def _visible(self) -> None:
         self.scene.run_method('visible', self.id, self.visible_)
 
+    def _draggable(self) -> None:
+        self.scene.run_method('draggable', self.id, self.draggable_)
+
     def _delete(self) -> None:
         self.scene.run_method('delete', self.id)
 
@@ -132,6 +137,12 @@ class Object3D:
             self._visible()
         return self
 
+    def draggable(self, value: bool = True):
+        if self.draggable_ != value:
+            self.draggable_ = value
+            self._draggable()
+        return self
+
     def delete(self) -> None:
         children = [object for object in self.scene.objects.values() if object.parent == self]
         for child in children:

nicegui/events.py

@@ -1,6 +1,6 @@
 from dataclasses import dataclass
 from inspect import Parameter, signature
-from typing import TYPE_CHECKING, Any, Awaitable, BinaryIO, Callable, Dict, List, Optional
+from typing import TYPE_CHECKING, Any, Awaitable, BinaryIO, Callable, Dict, List, Literal, Optional
 
 from . import background_tasks, globals
 from .helpers import KWONLY_SLOTS
@@ -53,6 +53,16 @@ class SceneClickEventArguments(ClickEventArguments):
     hits: List[SceneClickHit]
 
 
+@dataclass(**KWONLY_SLOTS)
+class SceneDragEventArguments(ClickEventArguments):
+    type: Literal['dragstart', 'dragend']
+    object_id: str
+    object_name: str
+    x: float
+    y: float
+    z: float
+
+
 @dataclass(**KWONLY_SLOTS)
 class ColorPickEventArguments(EventArguments):
     color: str

npm.json

@@ -86,6 +86,7 @@
       "package/examples/jsm/renderers/CSS2DRenderer(\\.min)?\\.js",
       "package/examples/jsm/renderers/CSS3DRenderer(\\.min)?\\.js",
       "package/examples/jsm/controls/OrbitControls(\\.min)?\\.js",
+      "package/examples/jsm/controls/DragControls(\\.min)?\\.js",
       "package/examples/jsm/loaders/STLLoader(\\.min)?\\.js",
       "package/examples/jsm/libs/tween\\.module\\.min\\.js"
     ],

website/more_documentation/scene_documentation.py

@@ -56,6 +56,33 @@ def more() -> None:
             scene.sphere().move(x=-1, z=1).with_name('sphere')
             scene.box().move(x=1, z=1).with_name('box')
 
+    @text_demo('Draggable objects', '''
+        You can make objects draggable using the `.draggable` method.
+        There is an optional `on_drag_start` and `on_drag_end` argument to `ui.scene` to handle drag events.
+        The callbacks receive a `SceneDragEventArguments` object with the following attributes:
+        
+        - `type`: the type of drag event ("dragstart" or "dragend").
+        - `object_id`: the id of the object that was dragged.
+        - `object_name`: the name of the object that was dragged.
+        - `x`, `y`, `z`: the x, y and z coordinates of the dragged object.
+               
+        You can also use the `drag_constraints` argument to set comma-separated JavaScript expressions
+        for constraining positions of dragged objects.
+    ''')
+    def draggable_objects() -> None:
+        from nicegui import events
+
+        def handle_drag(e: events.SceneDragEventArguments):
+            ui.notify(f'You dropped the sphere at ({e.x:.2f}, {e.y:.2f}, {e.z:.2f})')
+
+        with ui.scene(width=285, height=220,
+                      drag_constraints='z = 1', on_drag_end=handle_drag) as scene:
+            sphere = scene.sphere().move(z=1).draggable()
+
+        ui.switch('draggable sphere',
+                  value=sphere.draggable_,
+                  on_change=lambda e: sphere.draggable(e.value))
+
     @text_demo('Rendering point clouds', '''
         You can render point clouds using the `point_cloud` method.
         The `points` argument is a list of point coordinates, and the `colors` argument is a list of RGB colors (0..1).