Frontend Decouple | Expose Data Layer (#1757)

* template for Fred

* manage Data update requested by the frontend
We still need to manage the case when data update is pushed by the back-end
The data needs to be encapsulated in _TaipyData()

* manage Data update requested by the frontend
We still need to manage the case when data update is pushed by the back-end
The data needs to be encapsulated in _TaipyData()

* dependency

* describing data payload

* use an object as indicator for data refresh

* block every update for variables that do not have _TPMDL_ in their name

* update api

* per Fred

* encapsulate pandas.* into _TaipyData
do not push Module or Function to frontend

* get correct payload for datakey + dont filter for function

* remove taipydata encapsulation + treat pd dataframe as taipydata in custom page

* remove unnecessary check

* update onChange param requirement

* per Fred

* per Fabien + test fixes

* resolve conflic

---------

Co-authored-by: Fred Lefévère-Laoide <90181748+FredLL-Avaiga@users.noreply.github.com>
Co-authored-by: Fred Lefévère-Laoide <Fred.Lefevere-Laoide@Taipy.io>

frontend/taipy-gui/base/src/app.ts

@@ -3,14 +3,15 @@ import { sendWsMessage, TAIPY_CLIENT_ID } from "../../src/context/wsUtils";
 import { uploadFile } from "../../src/workers/fileupload";
 
 import { Socket, io } from "socket.io-client";
-import { DataManager, ModuleData } from "./dataManager";
+import { nanoid } from 'nanoid';
+import { DataManager, ModuleData, RequestDataOptions } from "./dataManager";
 import { initSocket } from "./socket";
 import { TaipyWsAdapter, WsAdapter } from "./wsAdapter";
 import { WsMessageType } from "../../src/context/wsUtils";
 import { getBase } from "./utils";
 
 export type OnInitHandler = (taipyApp: TaipyApp) => void;
-export type OnChangeHandler = (taipyApp: TaipyApp, encodedName: string, value: unknown) => void;
+export type OnChangeHandler = (taipyApp: TaipyApp, encodedName: string, value: unknown, dataEventKey?: string) => void;
 export type OnNotifyHandler = (taipyApp: TaipyApp, type: string, message: string) => void;
 export type OnReloadHandler = (taipyApp: TaipyApp, removedChanges: ModuleData) => void;
 export type OnWsMessage = (taipyApp: TaipyApp, event: string, payload: unknown) => void;
@@ -23,6 +24,8 @@ export type OnEvent =
     | OnWsMessage
     | OnWsStatusUpdate;
 type Route = [string, string];
+type RequestDataCallback = (taipyApp: TaipyApp, encodedName: string, dataEventKey: string, value: unknown) => void;
+
 
 export class TaipyApp {
     socket: Socket;
@@ -33,6 +36,7 @@ export class TaipyApp {
     _onWsMessage: OnWsMessage | undefined;
     _onWsStatusUpdate: OnWsStatusUpdate | undefined;
     _ackList: string[];
+    _rdc: Record<string, Record<string, RequestDataCallback>>;
     variableData: DataManager | undefined;
     functionData: DataManager | undefined;
     appId: string;
@@ -47,7 +51,7 @@ export class TaipyApp {
         onInit: OnInitHandler | undefined = undefined,
         onChange: OnChangeHandler | undefined = undefined,
         path: string | undefined = undefined,
-        socket: Socket | undefined = undefined,
+        socket: Socket | undefined = undefined
     ) {
         socket = socket || io("/", { autoConnect: false, path: `${this.getBaseUrl()}socket.io` });
         this.onInit = onInit;
@@ -63,6 +67,7 @@ export class TaipyApp {
         this.socket = socket;
         this.wsAdapters = [new TaipyWsAdapter()];
         this._ackList = [];
+        this._rdc = {};
         // Init socket io connection
         initSocket(socket, this);
     }
@@ -88,14 +93,14 @@ export class TaipyApp {
     }
 
     set onChange(handler: OnChangeHandler | undefined) {
-        if (handler !== undefined && handler.length !== 3) {
-            throw new Error("onChange() requires three parameters");
+        if (handler !== undefined && handler.length !== 3 && handler.length !== 4) {
+            throw new Error("onChange() requires three or four parameters");
         }
         this._onChange = handler;
     }
 
-    onChangeEvent(encodedName: string, value: unknown) {
-        this.onChange && this.onChange(this, encodedName, value);
+    onChangeEvent(encodedName: string, value: unknown, dataEventKey?: string) {
+        this.onChange && this.onChange(this, encodedName, value, dataEventKey);
     }
 
     get onNotify() {
@@ -195,8 +200,8 @@ export class TaipyApp {
         return this.variableData?.getName(encodedName);
     }
 
-    get(encodedName: string) {
-        return this.variableData?.get(encodedName);
+    get(encodedName: string, dataEventKey?: string) {
+        return this.variableData?.get(encodedName, dataEventKey);
     }
 
     getInfo(encodedName: string) {
@@ -220,19 +225,45 @@ export class TaipyApp {
         return this.routes;
     }
 
+    deleteRequestedData(encodedName: string, dataEventKey: string) {
+        this.variableData?.deleteRequestedData(encodedName, dataEventKey);
+    }
+
     // This update will only send the request to Taipy Gui backend
     // the actual update will be handled when the backend responds
     update(encodedName: string, value: unknown) {
         this.sendWsMessage("U", encodedName, { value: value });
     }
 
+
+    // Request Data from taipy backend
+    // This will trigger the backend to send the data to the frontend
+    requestData(encodedName: string, cb: RequestDataCallback, options?: RequestDataOptions) {
+        const varInfo = this.getInfo(encodedName);
+        if (!varInfo?.data_update) {
+            throw new Error(`Cannot request data for ${encodedName}. Not supported for type of ${varInfo?.type}`);
+        }
+        // Populate pagekey if there is no pagekey
+        if (!options) {
+            options = { pagekey: nanoid(10) };
+        }
+        options.pagekey = options?.pagekey || nanoid(10);
+        const dataKey = options.pagekey;
+        // preserve options for this data key so it can be called during refresh
+        this.variableData?.addRequestDataOptions(encodedName, dataKey, options);
+        // preserve callback so it can be called later
+        this._rdc[encodedName] = { ...this._rdc[encodedName], [dataKey]: cb };
+        // call the ws to request data
+        this.sendWsMessage("DU", encodedName, options);
+    }
+
     getContext() {
         return this.context;
     }
 
     updateContext(path: string | undefined = "") {
         if (!path || path === "") {
-            path = window.location.pathname.replace(this.getBaseUrl(), "") || "/"
+            path = window.location.pathname.replace(this.getBaseUrl(), "") || "/";
         }
         this.sendWsMessage("GMC", "get_module_context", { path: path || "/" });
     }

frontend/taipy-gui/base/src/dataManager.ts

@@ -6,18 +6,51 @@ export interface VarData {
     type: string;
     value: unknown;
     encoded_name: string;
+    data_update: boolean;
 }
 
+type ColumnName = string;
+
+export type RequestDataOptions = {
+    columns?: Array<ColumnName>;
+    pagekey?: string;
+    alldata?: boolean;
+    start?: number;
+    end?: number;
+    filters?: Array<{ col: ColumnName; value: string | boolean | number; action: string }>;
+    aggregates?: Array<ColumnName>;
+    applies?: { [key: ColumnName]: string };
+    infinite?: boolean;
+    reverse?: boolean;
+    orderby?: ColumnName;
+    sort?: "asc" | "desc";
+    styles?: { [key: ColumnName]: string };
+    tooltips?: { [key: ColumnName]: string };
+    handlenan?: boolean;
+    compare_datas?: string;
+};
+
+type RequestDataEntry = {
+    options: RequestDataOptions;
+    receivedData: unknown;
+}
+
+export const getRequestedDataKey = (payload?: unknown) =>
+    (!!payload && typeof payload == "object" && "pagekey" in payload && (payload["pagekey"] as string)) || undefined;
+
 // This class hold the information of variables and real-time value of variables
 export class DataManager {
     // key: encoded name, value: real-time value
     _data: Record<string, unknown>;
     // Initial data fetched from taipy-gui backend
     _init_data: ModuleData;
+    // key: encodedName -> dataEventKey -> requeste data
+    _requested_data: Record<string, Record<string, RequestDataEntry>>;
 
     constructor(variableModuleData: ModuleData) {
         this._data = {};
         this._init_data = {};
+        this._requested_data = {};
         this.init(variableModuleData);
     }
 
@@ -73,13 +106,32 @@ export class DataManager {
         return undefined;
     }
 
-    get(encodedName: string): unknown {
+    get(encodedName: string, dataEventKey?: string): unknown {
+        // handle requested data
+        if (dataEventKey) {
+            if (!(encodedName in this._requested_data)) {
+                throw new Error(`Encoded name '${encodedName}' is not available in Taipy GUI`);
+            }
+            if (!(dataEventKey in this._requested_data[encodedName])) {
+                throw new Error(`Event key '${dataEventKey}' is not available for encoded name '${encodedName}' in Taipy GUI`);
+            }
+            return this._requested_data[encodedName][dataEventKey].receivedData;
+        }
+        // handle normal data
         if (!(encodedName in this._data)) {
-            throw new Error(`${encodedName} is not available in Taipy Gui`);
+            throw new Error(`${encodedName} is not available in Taipy GUI`);
         }
         return this._data[encodedName];
     }
 
+    addRequestDataOptions(encodedName: string, dataEventKey: string, options: RequestDataOptions) {
+        if (!(encodedName in this._requested_data)) {
+            this._requested_data[encodedName] = {};
+        }
+        // This would overrides object with the same key
+        this._requested_data[encodedName][dataEventKey] = { options: options, receivedData: undefined };
+    }
+
     getInfo(encodedName: string): VarData | undefined {
         for (const context in this._init_data) {
             for (const variable in this._init_data[context]) {
@@ -100,10 +152,28 @@ export class DataManager {
         return this._data;
     }
 
-    update(encodedName: string, value: unknown) {
+    update(encodedName: string, value: unknown, dataEventKey?: string) {
+        // handle requested data
+        if (dataEventKey) {
+            if (!(encodedName in this._requested_data)) {
+                throw new Error(`Encoded name '${encodedName}' is not available in Taipy GUI`);
+            }
+            if (!(dataEventKey in this._requested_data[encodedName])) {
+                throw new Error(`Event key '${dataEventKey}' is not available for encoded name '${encodedName}' in Taipy GUI`);
+            }
+            this._requested_data[encodedName][dataEventKey].receivedData = value;
+            return;
+        }
+        // handle normal data
         if (!(encodedName in this._data)) {
             throw new Error(`${encodedName} is not available in Taipy Gui`);
         }
         this._data[encodedName] = value;
     }
+
+    deleteRequestedData(encodedName: string, dataEventKey: string) {
+        if (encodedName in this._requested_data && dataEventKey in this._requested_data[encodedName]) {
+            delete this._requested_data[encodedName][dataEventKey];
+        }
+    }
 }

frontend/taipy-gui/base/src/wsAdapter.ts

@@ -2,7 +2,7 @@ import merge from "lodash/merge";
 import { TaipyApp } from "./app";
 import { IdMessage, storeClientId } from "../../src/context/utils";
 import { WsMessage } from "../../src/context/wsUtils";
-import { DataManager, ModuleData } from "./dataManager";
+import { DataManager, getRequestedDataKey, ModuleData } from "./dataManager";
 
 export abstract class WsAdapter {
     abstract supportedMessageTypes: string[];
@@ -34,8 +34,25 @@ export class TaipyWsAdapter extends WsAdapter {
                 for (const muPayload of message.payload as [MultipleUpdatePayload]) {
                     const encodedName = muPayload.name;
                     const { value } = muPayload.payload;
-                    taipyApp.variableData?.update(encodedName, value);
-                    taipyApp.onChangeEvent(encodedName, value);
+                    if (value && typeof (value as any).__taipy_refresh === "boolean") {
+                        // refresh all requested data for this encodedName var
+                        const requestDataOptions = taipyApp.variableData?._requested_data[encodedName];
+                        for (const dataKey in requestDataOptions) {
+                            const requestDataEntry = requestDataOptions[dataKey];
+                            const { options } = requestDataEntry;
+                            taipyApp.sendWsMessage("DU", encodedName, options);
+                        }
+                        return true;
+                    }
+                    const dataKey = getRequestedDataKey(muPayload.payload);
+                    taipyApp.variableData?.update(encodedName, value, dataKey);
+                    // call the callback if it exists for request data
+                    if (dataKey && (encodedName in taipyApp._rdc && dataKey in taipyApp._rdc[encodedName])) {
+                        const cb = taipyApp._rdc[encodedName]?.[dataKey];
+                        cb(taipyApp, encodedName, dataKey, value);
+                        delete taipyApp._rdc[encodedName][dataKey];
+                    }
+                    taipyApp.onChangeEvent(encodedName, value, dataKey);
                 }
             } else if (message.type === "ID") {
                 const { id } = message as unknown as IdMessage;

frontend/taipy-gui/package-lock.json

@@ -19,6 +19,7 @@
         "date-fns": "^3.6.0",
         "date-fns-tz": "^3.1.3",
         "lodash": "^4.17.21",
+        "nanoid": "^5.0.7",
         "notistack": "^3.0.0",
         "plotly.js": "^2.33.0",
         "react": "^18.2.0",
@@ -12588,9 +12589,9 @@
       "integrity": "sha512-TvmkNhkv8yct0SVBSy+o8wYzXjE4Zz3PCesbfs8HiCXXdcTuocApFv11UWlNFWKYsP2okqrhb7JNlSm9InBhIw=="
     },
     "node_modules/nanoid": {
-      "version": "3.3.7",
-      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.7.tgz",
-      "integrity": "sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g==",
+      "version": "5.0.7",
+      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.0.7.tgz",
+      "integrity": "sha512-oLxFY2gd2IqnjcYyOXD8XGCftpGtZP2AbHbOkthDkvRywH5ayNtPVy9YlOPcHckXzbLTCHpkb7FB+yuxKV13pQ==",
       "funding": [
         {
           "type": "github",
@@ -12598,10 +12599,10 @@
         }
       ],
       "bin": {
-        "nanoid": "bin/nanoid.cjs"
+        "nanoid": "bin/nanoid.js"
       },
       "engines": {
-        "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1"
+        "node": "^18 || >=20"
       }
     },
     "node_modules/native-promise-only": {
@@ -13409,6 +13410,23 @@
       "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-4.2.0.tgz",
       "integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ=="
     },
+    "node_modules/postcss/node_modules/nanoid": {
+      "version": "3.3.7",
+      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.7.tgz",
+      "integrity": "sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/ai"
+        }
+      ],
+      "bin": {
+        "nanoid": "bin/nanoid.cjs"
+      },
+      "engines": {
+        "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1"
+      }
+    },
     "node_modules/potpack": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/potpack/-/potpack-1.0.2.tgz",

frontend/taipy-gui/package.json

@@ -14,6 +14,7 @@
     "date-fns": "^3.6.0",
     "date-fns-tz": "^3.1.3",
     "lodash": "^4.17.21",
+    "nanoid": "^5.0.7",
     "notistack": "^3.0.0",
     "plotly.js": "^2.33.0",
     "react": "^18.2.0",

frontend/taipy-gui/src/components/Taipy/AutoLoadingTable.tsx

@@ -216,7 +216,7 @@ const AutoLoadingTable = (props: TaipyTableProps) => {
     const hover = useDynamicProperty(props.hoverText, props.defaultHoverText, undefined);
     const baseColumns = useDynamicJsonProperty(props.columns, props.defaultColumns, defaultColumns);
 
-    const refresh = typeof props.data === "number";
+    const refresh = props.data && typeof props.data.__taipy_refresh === "boolean";
 
     useEffect(() => {
         if (!refresh && props.data && page.current.key && props.data[page.current.key] !== undefined) {

frontend/taipy-gui/src/components/Taipy/Chart.tsx

@@ -293,7 +293,7 @@ const Chart = (props: ChartProp) => {
     const theme = useTheme();
     const module = useModule();
 
-    const refresh = typeof data === "number" ? data : 0;
+    const refresh = typeof data.__taipy_refresh === "boolean";
     const className = useClassNames(props.libClassName, props.dynamicClassName, props.className);
     const active = useDynamicProperty(props.active, props.defaultActive, true);
     const render = useDynamicProperty(props.render, props.defaultRender, true);

frontend/taipy-gui/src/components/Taipy/Chat.tsx

@@ -298,7 +298,8 @@ const Chat = (props: ChatProps) => {
         setShowMessage(false);
     }, []);
 
-    const refresh = typeof props.messages === "number";
+    // const refresh = typeof props.messages === "number";
+    const refresh = props.messages && typeof props.messages.__taipy_refresh === "boolean";
 
     useEffect(() => {
         if (!refresh && props.messages && page.current.key && props.messages[page.current.key] !== undefined) {

frontend/taipy-gui/src/components/Taipy/PaginatedTable.tsx

@@ -124,7 +124,7 @@ const PaginatedTable = (props: TaipyPaginatedTableProps) => {
     const formatConfig = useFormatConfig();
     const module = useModule();
 
-    const refresh = typeof props.data === "number";
+    const refresh = props.data && typeof props.data.__taipy_refresh === "boolean";
     const className = useClassNames(props.libClassName, props.dynamicClassName, props.className);
     const active = useDynamicProperty(props.active, props.defaultActive, true);
     const editable = useDynamicProperty(props.editable, props.defaultEditable, false);

frontend/taipy-gui/src/context/taipyReducers.ts

@@ -323,6 +323,7 @@ export const taipyReducer = (state: TaipyState, baseAction: TaipyBaseAction): Ta
         case Types.Update:
             const newValue = action.payload.value as Record<string, unknown>;
             const oldValue = (state.data[action.name] as Record<string, unknown>) || {};
+            delete oldValue.__taipy_refresh;
             if (typeof action.payload.infinite === "boolean" && action.payload.infinite) {
                 const start = newValue.start;
                 if (typeof start === "number") {

taipy/gui/custom/_page.py

@@ -14,7 +14,10 @@ from __future__ import annotations
 import typing as t
 from abc import ABC, abstractmethod
 
+import pandas as pd
+
 from ..page import Page as BasePage
+from ..utils import _TaipyData
 from ..utils.singleton import _Singleton
 
 
@@ -48,6 +51,8 @@ class ResourceHandler(ABC):
 
     rh_id: str = ""
 
+    data_layer_supported_types: t.Tuple[t.Type, ...] = (_TaipyData, pd.DataFrame, pd.Series)
+
     def __init__(self) -> None:
         _ExternalResourceHandlerManager().register(self)
 

taipy/gui/custom/utils.py

@@ -0,0 +1,37 @@
+# Copyright 2021-2024 Avaiga Private Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+# an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+
+import contextlib
+import typing as t
+
+from flask import has_request_context
+from flask.globals import request
+
+from ..server import _Server
+from ._page import ResourceHandler, _ExternalResourceHandlerManager
+
+
+def is_in_custom_page_context() -> bool:
+    "NOT DOCUMENTED"
+    resource_handler_id = None
+    with contextlib.suppress(Exception):
+        if has_request_context():
+            resource_handler_id = request.cookies.get(_Server._RESOURCE_HANDLER_ARG, None)
+    return resource_handler_id is not None
+
+
+def get_current_resource_handler() -> t.Optional[ResourceHandler]:
+    "NOT DOCUMENTED"
+    resource_handler_id = None
+    with contextlib.suppress(Exception):
+        if has_request_context():
+            resource_handler_id = request.cookies.get(_Server._RESOURCE_HANDLER_ARG, None)
+    return _ExternalResourceHandlerManager().get(resource_handler_id) if resource_handler_id else None

taipy/gui/data/pandas_data_accessor.py

@@ -296,16 +296,16 @@ class _PandasDataAccessor(_DataAccessor):
             # real number of rows is needed to calculate the number of pages
             rowcount = len(df)
             # here we'll deal with start and end values from payload if present
-            if isinstance(payload["start"], int):
-                start = int(payload["start"])
+            if isinstance(payload.get("start", 0), int):
+                start = int(payload.get("start", 0))
             else:
                 try:
                     start = int(str(payload["start"]), base=10)
                 except Exception:
                     _warn(f'start should be an int value {payload["start"]}.')
                     start = 0
-            if isinstance(payload["end"], int):
-                end = int(payload["end"])
+            if isinstance(payload.get("end", -1), int):
+                end = int(payload.get("end", -1))
             else:
                 try:
                     end = int(str(payload["end"]), base=10)

taipy/gui/gui.py

@@ -37,7 +37,6 @@ from flask import (
     Flask,
     g,
     has_app_context,
-    has_request_context,
     jsonify,
     request,
     send_file,
@@ -63,6 +62,7 @@ from ._warnings import TaipyGuiWarning, _warn
 from .builder import _ElementApiGenerator
 from .config import Config, ConfigParameter, _Config
 from .custom import Page as CustomPage
+from .custom.utils import get_current_resource_handler, is_in_custom_page_context
 from .data.content_accessor import _ContentAccessor
 from .data.data_accessor import _DataAccessors
 from .data.data_format import _DataFormat
@@ -106,7 +106,7 @@ from .utils import (
 from .utils._adapter import _Adapter
 from .utils._bindings import _Bindings
 from .utils._evaluator import _Evaluator
-from .utils._variable_directory import _MODULE_ID, _VariableDirectory
+from .utils._variable_directory import _is_moduled_variable, _VariableDirectory
 from .utils.chart_config_builder import _build_chart_config
 from .utils.table_col_builder import _enhance_columns
 
@@ -1033,15 +1033,15 @@ class Gui:
                         setattr(self._bindings(), var_name, newvalue)
         return ("", 200)
 
-    _data_request_counter = 1
-
     def __send_var_list_update(  # noqa C901
         self,
         modified_vars: t.List[str],
         front_var: t.Optional[str] = None,
     ):
         ws_dict = {}
-        values = {v: _getscopeattr_drill(self, v) for v in modified_vars}
+        values = {v: _getscopeattr_drill(self, v) for v in modified_vars if _is_moduled_variable(v)}
+        if not values:
+            return
         for k, v in values.items():
             if isinstance(v, (_TaipyData, _TaipyContentHtml)) and v.get_name() in modified_vars:
                 modified_vars.remove(v.get_name())
@@ -1049,10 +1049,10 @@ class Gui:
                 modified_vars.remove(k)
         for _var in modified_vars:
             newvalue = values.get(_var)
-            if isinstance(newvalue, _TaipyData):
-                # A changing integer that triggers a data request
-                newvalue = Gui._data_request_counter
-                Gui._data_request_counter = (Gui._data_request_counter % 100) + 1
+            resource_handler = get_current_resource_handler()
+            custom_page_filtered_types = resource_handler.data_layer_supported_types if resource_handler else ()
+            if isinstance(newvalue, (_TaipyData)) or isinstance(newvalue, custom_page_filtered_types):
+                newvalue = {"__taipy_refresh": True}
             else:
                 if isinstance(newvalue, (_TaipyContent, _TaipyContentImage)):
                     ret_value = self.__get_content_accessor().get_info(
@@ -1072,14 +1072,9 @@ class Gui:
                     )
                 elif isinstance(newvalue, _TaipyBase):
                     newvalue = newvalue.get()
-                if isinstance(newvalue, (dict, _MapDict)):
-                    # Skip in taipy-gui, available in custom frontend
-                    resource_handler_id = None
-                    with contextlib.suppress(Exception):
-                        if has_request_context():
-                            resource_handler_id = request.cookies.get(_Server._RESOURCE_HANDLER_ARG, None)
-                    if resource_handler_id is None:
-                        continue  # this var has no transformer
+                # Skip in taipy-gui, available in custom frontend
+                if isinstance(newvalue, (dict, _MapDict)) and not is_in_custom_page_context():
+                    continue
                 if isinstance(newvalue, float) and math.isnan(newvalue):
                     # do not let NaN go through json, it is not handle well (dies silently through websocket)
                     newvalue = None
@@ -1115,6 +1110,10 @@ class Gui:
     def __request_data_update(self, var_name: str, payload: t.Any) -> None:
         # Use custom attrgetter function to allow value binding for _MapDict
         newvalue = _getscopeattr_drill(self, var_name)
+        resource_handler = get_current_resource_handler()
+        custom_page_filtered_types = resource_handler.data_layer_supported_types if resource_handler else ()
+        if not isinstance(newvalue, _TaipyData) and isinstance(newvalue, custom_page_filtered_types):
+            newvalue = _TaipyData(newvalue, "")
         if isinstance(newvalue, _TaipyData):
             ret_payload = None
             if isinstance(payload, dict):
@@ -1178,6 +1177,9 @@ class Gui:
     def __get_variable_tree(self, data: t.Dict[str, t.Any]):
         # Module Context -> Variable -> Variable data (name, type, initial_value)
         variable_tree: t.Dict[str, t.Dict[str, t.Dict[str, t.Any]]] = {}
+        # Types of data to be handled by the data layer and filtered out here
+        resource_handler = get_current_resource_handler()
+        filtered_value_types = resource_handler.data_layer_supported_types if resource_handler else ()
         for k, v in data.items():
             if isinstance(v, _TaipyBase):
                 data[k] = v.get()
@@ -1186,11 +1188,15 @@ class Gui:
                 var_module_name = "__main__"
             if var_module_name not in variable_tree:
                 variable_tree[var_module_name] = {}
-            variable_tree[var_module_name][var_name] = {
-                "type": type(v).__name__,
-                "value": data[k],
-                "encoded_name": k,
-            }
+            data_update = isinstance(v, filtered_value_types)
+            value = None if data_update else data[k]
+            if _is_moduled_variable(k):
+                variable_tree[var_module_name][var_name] = {
+                    "type": type(v).__name__,
+                    "value": value,
+                    "encoded_name": k,
+                    "data_update": data_update,
+                }
         return variable_tree
 
     def __handle_ws_get_data_tree(self):
@@ -2116,7 +2122,7 @@ class Gui:
         return encoded_var_name
 
     def _bind_var_val(self, var_name: str, value: t.Any) -> bool:
-        if _MODULE_ID not in var_name:
+        if not _is_moduled_variable(var_name):
             var_name = self.__var_dir.add_var(var_name, self._get_locals_context())
         if not hasattr(self._bindings(), var_name):
             self._bind(var_name, value)
@@ -2331,8 +2337,12 @@ class Gui:
         with self.get_flask_app().app_context() if has_app_context() else contextlib.nullcontext():  # type: ignore[attr-defined]
             self.__set_client_id_in_context(client_id)
             with self._set_locals_context(page._get_module_name()):
-                for k in self._get_locals_bind().keys():
-                    if (not page._binding_variables or k in page._binding_variables) and not k.startswith("_"):
+                for k, v in self._get_locals_bind().items():
+                    if (
+                        (not page._binding_variables or k in page._binding_variables)
+                        and not k.startswith("_")
+                        and not isinstance(v, ModuleType)
+                    ):
                         self._bind_var(k)
 
     def __render_page(self, page_name: str) -> t.Any:

taipy/gui/utils/_variable_directory.py

@@ -119,6 +119,8 @@ _MODULE_NAME_MAP: t.List[str] = []
 _MODULE_ID = "_TPMDL_"
 _RE_TPMDL_DECODE = re.compile(r"(.*?)" + _MODULE_ID + r"(\d+)$")
 
+def _is_moduled_variable(var_name: str):
+    return _MODULE_ID in var_name
 
 def _variable_encode(var_name: str, module_name: t.Optional[str]):
     if module_name is None:

tests/gui/helpers.py

@@ -70,10 +70,14 @@ class Helpers:
         args = received_message["args"]
         assert "type" in args and args["type"] == type
         assert "payload" in args
-        payload = args["payload"][0]
-        assert "name" in payload and varname in payload["name"]
-        assert "payload" in payload and "value" in payload["payload"] and payload["payload"]["value"] == value
-        logging.getLogger().debug(payload["payload"]["value"])
+        payload_arr = args["payload"]
+        found_payload = False
+        for payload in payload_arr:
+            if "name" in payload and varname in payload["name"]:
+                assert "payload" in payload and "value" in payload["payload"] and payload["payload"]["value"] == value
+                found_payload = True
+                logging.getLogger().debug(payload["payload"]["value"])
+        assert found_payload
 
     @staticmethod
     def assert_outward_simple_ws_message(received_message, type, varname, value):

tests/gui/server/ws/test_a.py

@@ -41,5 +41,5 @@ def test_a_button_pressed(gui: Gui, helpers):
     assert gui._bindings()._get_all_scopes()[sid].x == 20  # type: ignore
     # assert for received message (message that would be sent to the front-end client)
     received_messages = ws_client.get_received()
-    helpers.assert_outward_ws_message(received_messages[0], "MU", "x", 20)
-    helpers.assert_outward_ws_message(received_messages[1], "MU", "text", "a random text")
+    helpers.assert_outward_ws_message(received_messages[0], "MU", "tpec_TpExPr_x_TPMDL_0", 20)
+    helpers.assert_outward_ws_message(received_messages[1], "MU", "tpec_TpExPr_text_TPMDL_0", "a random text")

tests/gui/server/ws/test_ru.py

@@ -33,8 +33,13 @@ def test_ru_selector(gui: Gui, helpers, csvdata):
     sid = helpers.create_scope_and_get_sid(gui)
     # Get the jsx once so that the page will be evaluated -> variable will be registered
     flask_client.get(f"/taipy-jsx/test?client_id={sid}")
-    ws_client.emit("message", {"client_id": sid, "type": "RU", "name": "", "payload": {"names": ["selected_val"]}})
+    ws_client.emit(
+        "message",
+        {"client_id": sid, "type": "RU", "name": "", "payload": {"names": ["_TpLv_tpec_TpExPr_selected_val_TPMDL_0"]}},
+    )
     # assert for received message (message that would be sent to the front-end client)
     received_messages = ws_client.get_received()
     assert len(received_messages)
-    helpers.assert_outward_ws_message(received_messages[0], "MU", "selected_val", ["value1", "value2"])
+    helpers.assert_outward_ws_message(
+        received_messages[0], "MU", "_TpLv_tpec_TpExPr_selected_val_TPMDL_0", ["value1", "value2"]
+    )