multiple traces with new decimator api (#1062) (#1731)

* multiple traces with new decimator api

* only assign the decimated data if it has been processd

* revert changes for chart mode

* fix test

* add additional chart types for relayout + fix single trace error

* Update taipy/gui/data/decimator/base.py

Co-authored-by: Fabien Lelaquais <86590727+FabienLelaquais@users.noreply.github.com>

* Update taipy/gui/data/decimator/__init__.py

Co-authored-by: Fabien Lelaquais <86590727+FabienLelaquais@users.noreply.github.com>

* Update taipy/gui/data/decimator/base.py

Co-authored-by: Fabien Lelaquais <86590727+FabienLelaquais@users.noreply.github.com>

* Update taipy/gui/data/decimator/base.py

Co-authored-by: Fabien Lelaquais <86590727+FabienLelaquais@users.noreply.github.com>

* Update frontend/taipy-gui/src/components/Taipy/Chart.tsx

Co-authored-by: Fabien Lelaquais <86590727+FabienLelaquais@users.noreply.github.com>

* fix 1

* per Fabien

* fix tests

---------

Co-authored-by: Fabien Lelaquais <86590727+FabienLelaquais@users.noreply.github.com>

frontend/taipy-gui/src/components/Taipy/Chart.tsx

@@ -112,7 +112,7 @@ export const getValue = <T,>(
     values: TraceValueType | undefined,
     arr: T[],
     idx: number,
-    returnUndefined = false
+    returnUndefined = false,
 ): (string | number)[] | undefined => {
     const value = getValueFromCol(values, getArrayValue(arr, idx) as unknown as string);
     if (!returnUndefined || value.length) {
@@ -150,7 +150,7 @@ const getDecimatorsPayload = (
     modes: string[],
     columns: Record<string, ColumnDesc>,
     traces: string[][],
-    relayoutData?: PlotRelayoutEvent
+    relayoutData?: PlotRelayoutEvent,
 ) => {
     return decimators
         ? {
@@ -165,7 +165,12 @@ const getDecimatorsPayload = (
                             zAxis: getAxis(traces, i, columns, 2),
                             chartMode: modes[i],
                         }
-                      : undefined
+                      : {
+                            xAxis: getAxis(traces, i, columns, 0),
+                            yAxis: getAxis(traces, i, columns, 1),
+                            zAxis: getAxis(traces, i, columns, 2),
+                            chartMode: modes[i],
+                        },
               ),
               relayoutData: relayoutData,
           }
@@ -357,9 +362,9 @@ const Chart = (props: ChartProp) => {
                             plotRef.current,
                             config.modes,
                             config.columns,
-                            config.traces
-                        )
-                    )
+                            config.traces,
+                        ),
+                    ),
                 );
             }
         }
@@ -431,7 +436,7 @@ const Chart = (props: ChartProp) => {
             height === undefined
                 ? ({ ...defaultStyle, width: width } as CSSProperties)
                 : ({ ...defaultStyle, width: width, height: height } as CSSProperties),
-        [width, height]
+        [width, height],
     );
     const skelStyle = useMemo(() => ({ ...style, minHeight: "7em" }), [style]);
 
@@ -576,9 +581,9 @@ const Chart = (props: ChartProp) => {
                             config.modes,
                             config.columns,
                             config.traces,
-                            eventData
-                        )
-                    )
+                            eventData,
+                        ),
+                    ),
                 );
             }
         },
@@ -593,7 +598,7 @@ const Chart = (props: ChartProp) => {
             config.decimators,
             updateVarName,
             module,
-        ]
+        ],
     );
 
     const clickHandler = useCallback(
@@ -622,18 +627,18 @@ const Chart = (props: ChartProp) => {
                         y: map ? undefined : transform(yaxis, "top")(evt?.clientY),
                         lon: map ? xaxis.p2c() : undefined,
                         x: map ? undefined : transform(xaxis, "left")(evt?.clientX),
-                    })
-                )
+                    }),
+                ),
             );
         },
-        [dispatch, module, id, onClick]
+        [dispatch, module, id, onClick],
     );
 
     const onInitialized = useCallback(
         (figure: Readonly<Figure>, graphDiv: Readonly<HTMLElement>) => {
             onClick && graphDiv.addEventListener("click", clickHandler);
         },
-        [onClick, clickHandler]
+        [onClick, clickHandler],
     );
 
     const getRealIndex = useCallback(
@@ -642,10 +647,10 @@ const Chart = (props: ChartProp) => {
                 ? props.figure
                     ? index
                     : data[dataKey].tp_index
-                    ? (data[dataKey].tp_index[index] as number)
-                    : index
+                      ? (data[dataKey].tp_index[index] as number)
+                      : index
                 : 0,
-        [data, dataKey, props.figure]
+        [data, dataKey, props.figure],
     );
 
     const onSelect = useCallback(
@@ -656,7 +661,8 @@ const Chart = (props: ChartProp) => {
                     tr[pt.curveNumber].push(getRealIndex(getPlotIndex(pt)));
                     return tr;
                 }, [] as number[][]);
-                if (config.traces.length === 0) { // figure
+                if (config.traces.length === 0) {
+                    // figure
                     const theVar = getUpdateVar(updateVars, "selected");
                     theVar && dispatch(createSendUpdateAction(theVar, traces, module, props.onChange, propagate));
                     return;
@@ -665,7 +671,15 @@ const Chart = (props: ChartProp) => {
                     const upvars = traces.map((_, idx) => getUpdateVar(updateVars, `selected${idx}`));
                     const setVars = new Set(upvars.filter((v) => v));
                     if (traces.length > 1 && setVars.size === 1) {
-                        dispatch(createSendUpdateAction(setVars.values().next().value, traces, module, props.onChange, propagate));
+                        dispatch(
+                            createSendUpdateAction(
+                                setVars.values().next().value,
+                                traces,
+                                module,
+                                props.onChange,
+                                propagate,
+                            ),
+                        );
                         return;
                     }
                     traces.forEach((tr, idx) => {
@@ -681,7 +695,7 @@ const Chart = (props: ChartProp) => {
                 }
             }
         },
-        [getRealIndex, dispatch, updateVars, propagate, props.onChange, config.traces.length, module]
+        [getRealIndex, dispatch, updateVars, propagate, props.onChange, config.traces.length, module],
     );
 
     return render ? (

taipy/gui/data/__init__.py

@@ -10,5 +10,4 @@
 # specific language governing permissions and limitations under the License.
 
 from .data_accessor import _DataAccessor
-from .decimator import LTTB, RDP, MinMaxDecimator, ScatterDecimator
-from .utils import Decimator
+from .decimator import *

taipy/gui/data/decimator/__init__.py

@@ -9,7 +9,11 @@
 # an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
 # specific language governing permissions and limitations under the License.
 
+from .base import Decimator
 from .lttb import LTTB
 from .minmax import MinMaxDecimator
 from .rdp import RDP
 from .scatter_decimator import ScatterDecimator
+
+# Export the following classes
+__all__ = ["LTTB", "MinMaxDecimator", "RDP", "ScatterDecimator", "Decimator"]

taipy/gui/data/decimator/base.py

@@ -0,0 +1,286 @@
+# Copyright 2021-2024 Avaiga Private Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+# an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+
+from __future__ import annotations
+
+import typing as t
+from abc import ABC, abstractmethod
+
+import numpy as np
+import pandas as pd
+
+from ..._warnings import _warn
+
+
+class Decimator(ABC):
+    """Base class for decimating chart data.
+
+    *Decimating* is the term used to name the process of reducing the number of
+    data points displayed in charts while retaining the overall shape of the traces.
+    `Decimator` is a base class that does decimation on data sets.
+
+    Taipy GUI comes out-of-the-box with several implementation of this class for
+    different use cases.
+    """
+
+    _CHART_MODES: t.List[str] = []
+
+    def __init__(
+        self,
+        threshold: t.Optional[int],
+        zoom: t.Optional[bool],
+        # apply_decimator: t.Optional[t.Callable] = None,
+        # on_decimate: t.Optional[t.Callable] = None,
+    ) -> None:  # noqa: E501
+        """Initialize a new `Decimator`.
+
+        Arguments:
+            threshold (Optional[int]): The minimum amount of data points before the
+                decimator class is applied.
+            zoom (Optional[bool]): set to True to reapply the decimation
+                when zoom or re-layout events are triggered.
+        """
+        # on_decimate (Optional[Callable]): an user-defined function that is executed when the decimator
+        #     is found during runtime. This function can be used to provide custom decimation logic.
+        # apply_decimator (Optional[Callable]): an user-defined function that is executed when the decimator
+        #     is applied to modify the data.
+        super().__init__()
+        self.threshold = threshold
+        self._zoom = zoom if zoom is not None else True
+        self.__user_defined_on_decimate = None
+        self.__user_defined_apply_decimator = None
+
+    def _is_applicable(self, data: t.Any, nb_rows_max: int, chart_mode: str):
+        if chart_mode not in self._CHART_MODES:
+            _warn(
+                f"Decimator '{type(self).__name__}' is not optimized for chart mode '{chart_mode}'. Consider using other chart mode such as '{f'{chr(39)}, {chr(39)}'.join(self._CHART_MODES)}.'"  # noqa: E501
+            )
+        if self.threshold is None:
+            if nb_rows_max < len(data):
+                return True
+        elif self.threshold < len(data):
+            return True
+        return False
+
+    def __get_indexed_df_col(self, df):
+        index = 0
+        while f"tAiPy_index_{index}" in df.columns:
+            index += 1
+        result = f"tAiPy_index_{index}"
+        df[result] = df.index
+        return result
+
+    def _df_relayout(
+        self,
+        dataframe: pd.DataFrame,
+        x_column: t.Optional[str],
+        y_column: str,
+        chart_mode: str,
+        x0: t.Optional[float],
+        x1: t.Optional[float],
+        y0: t.Optional[float],
+        y1: t.Optional[float],
+        is_copied: bool,
+    ):
+        if chart_mode not in ["lines+markers", "lines", "markers"]:
+            _warn(
+                f"Decimator zoom feature is not applicable for '{chart_mode}' chart_mode. It is only applicable for 'lines+markers', 'lines', and 'markers' chart modes."  # noqa: E501
+            )
+            return dataframe, is_copied
+        # if chart data is invalid
+        if x0 is None and x1 is None and y0 is None and y1 is None:
+            return dataframe, is_copied
+        df = dataframe if is_copied else dataframe.copy()
+        is_copied = True
+        has_x_col = True
+
+        if not x_column:
+            x_column = self.__get_indexed_df_col(df)
+            has_x_col = False
+
+        df_filter_conditions = []
+        # filter by x column by default
+        if x0 is not None:
+            df_filter_conditions.append(df[x_column] > x0)
+        if x1 is not None:
+            df_filter_conditions.append(df[x_column] < x1)
+        # y column will be filtered only if chart_mode is not lines+markers (eg. markers)
+        if chart_mode not in ["lines+markers", "lines"]:
+            if y0 is not None:
+                df_filter_conditions.append(df[y_column] > y0)
+            if y1 is not None:
+                df_filter_conditions.append(df[y_column] < y1)
+        if df_filter_conditions:
+            df = df.loc[np.bitwise_and.reduce(df_filter_conditions)]
+        if not has_x_col:
+            df.drop(x_column, axis=1, inplace=True)
+        return df, is_copied
+
+    def _df_apply_decimator(
+        self,
+        dataframe: pd.DataFrame,
+        x_column_name: t.Optional[str],
+        y_column_name: str,
+        z_column_name: str,
+        payload: t.Dict[str, t.Any],
+        is_copied: bool,
+    ):
+        df = dataframe if is_copied else dataframe.copy()
+        if not x_column_name:
+            x_column_name = self.__get_indexed_df_col(df)
+        column_list = [x_column_name, y_column_name, z_column_name] if z_column_name else [x_column_name, y_column_name]
+        points = df[column_list].to_numpy()
+        mask = self.decimate(points, payload)
+        return df[mask], is_copied
+
+    def _on_decimate_df(
+        self,
+        df: pd.DataFrame,
+        decimator_instance_payload: t.Dict[str, t.Any],
+        decimator_payload: t.Dict[str, t.Any],
+        is_copied: bool = False,
+        filter_unused_columns: bool = True,
+    ):
+        decimator_var_name = decimator_instance_payload.get("decimator")
+        x_column, y_column, z_column = (
+            decimator_instance_payload.get("xAxis", ""),
+            decimator_instance_payload.get("yAxis", ""),
+            decimator_instance_payload.get("zAxis", ""),
+        )
+        chart_mode = decimator_instance_payload.get("chartMode", "")
+        if self._zoom and "relayoutData" in decimator_payload is not None and not z_column:
+            relayout_data = decimator_payload.get("relayoutData", {})
+            x0 = relayout_data.get("xaxis.range[0]")
+            x1 = relayout_data.get("xaxis.range[1]")
+            y0 = relayout_data.get("yaxis.range[0]")
+            y1 = relayout_data.get("yaxis.range[1]")
+
+            df, is_copied = self._df_relayout(
+                t.cast(pd.DataFrame, df), x_column, y_column, chart_mode, x0, x1, y0, y1, is_copied
+            )
+
+        nb_rows_max = decimator_payload.get("width")
+        is_decimator_applied = False
+        if nb_rows_max and self._is_applicable(df, nb_rows_max, chart_mode):
+            try:
+                df, is_copied = self.apply_decimator(
+                    t.cast(pd.DataFrame, df),
+                    x_column,
+                    y_column,
+                    z_column,
+                    payload=decimator_payload,
+                    is_copied=is_copied,
+                )
+                is_decimator_applied = True
+            except Exception as e:
+                _warn(f"Limit rows error with {decimator_var_name} for Dataframe", e)
+        # only include columns involving the decimator
+        if filter_unused_columns:
+            filterd_columns = [x_column, y_column, z_column] if z_column else [x_column, y_column]
+            df = df.filter(filterd_columns, axis=1)
+        return df, is_decimator_applied, is_copied
+
+    def on_decimate(
+        self,
+        df: pd.DataFrame,
+        decimator_instance_payload: t.Dict[str, t.Any],
+        decimator_payload: t.Dict[str, t.Any],
+        is_copied: bool = False,
+        filter_unused_columns: bool = True,
+    ):
+        """NOT DOCUMENTED
+
+        This function is executed whenever a decimator is found during runtime.
+
+        Users can override this function by providing an alternate implementation inside the constructor
+        to provide custom decimation logic.
+
+        Arguments:
+            df (pandas.DataFrame): The DataFrame that will be decimated.
+            decimator_instance_payload (Dict[str, any]): The payload for the current instance of decimator. Each
+                decimation request might contain multiple decimators to handle multiple traces.
+            decimator_payload (Dict[str, any]): The full decimator payload including the current decimator.
+            is_copied (bool): A flag to indicate if the DataFrame is copied.
+            filter_unused_columns (bool): A flag to indicate if the DataFrame columns should be filtered to only
+                include the columns that are involved with the decimator.
+
+        Returns:
+            A tuple containing the decimated DataFrame, a flag indicating if the decimator is applied,
+            and a flag indicating if the DataFrame was copied.
+        """
+        if self.__user_defined_on_decimate and callable(self.__user_defined_on_decimate):
+            try:
+                return self.__user_defined_on_decimate(
+                    df, decimator_instance_payload, decimator_payload, is_copied, filter_unused_columns
+                )
+            except Exception as e:
+                _warn("Error executing user defined on_decimate function: ", e)
+        return self._on_decimate_df(df, decimator_instance_payload, decimator_payload, filter_unused_columns)
+
+    def apply_decimator(
+        self,
+        dataframe: pd.DataFrame,
+        x_column_name: t.Optional[str],
+        y_column_name: str,
+        z_column_name: str,
+        payload: t.Dict[str, t.Any],
+        is_copied: bool,
+    ):
+        """NOT DOCUMENTED
+        This function is executed whenever a decimator is applied to the data.
+
+        This function is used by default the `on_decimate` function.
+        Users can override this function by providing an alternate function inside the constructor
+        to provide custom handling only when the decimator is applied. This avoids the need to override
+        the default `on_decimate` handling logic.
+
+        Arguments:
+            dataframe (pandas.DataFrame): The DataFrame that will be decimated.
+            x_column_name (Optional[str]): The name of the x-axis column.
+            y_column_name (str): The name of the y-axis column.
+            z_column_name (str): The name of the z-axis column.
+            payload (Dict[str, any]): The payload for the current decimator.
+            is_copied (bool): A flag to indicate if the DataFrame is copied.
+
+        Returns:
+            A tuple containing the decimated DataFrame and a flag indicating if the DataFrame was copied
+        """
+        if self.__user_defined_apply_decimator and callable(self.__user_defined_apply_decimator):
+            try:
+                return self.__user_defined_apply_decimator(
+                    dataframe, x_column_name, y_column_name, z_column_name, payload, is_copied
+                )
+            except Exception as e:
+                _warn("Error executing user defined apply_decimator function: ", e)
+        return self._df_apply_decimator(dataframe, x_column_name, y_column_name, z_column_name, payload, is_copied)
+
+    @abstractmethod
+    def decimate(self, data: np.ndarray, payload: t.Dict[str, t.Any]) -> np.ndarray:
+        """NOT DOCUMENTED
+        Decimate the dataset.
+
+        This method is executed when the appropriate conditions specified in the
+        constructor are met. This function implements the algorithm that determines
+        which data points are kept or dropped.
+
+        Arguments:
+            data (numpy.array): An array containing all the data points represented as
+                tuples.
+            payload (Dict[str, any]): additional information on charts that is provided
+                at runtime.
+
+        Returns:
+            An array of Boolean mask values. The array should set True or False for each
+                of its indexes where True indicates that the corresponding data point
+                from *data* should be preserved, or False requires that this
+                data point be dropped.
+        """
+        raise NotImplementedError

taipy/gui/data/decimator/lttb.py

@@ -13,7 +13,7 @@ import typing as t
 
 import numpy as np
 
-from ..utils import Decimator
+from .base import Decimator
 
 
 class LTTB(Decimator):
@@ -28,7 +28,14 @@ class LTTB(Decimator):
 
     _CHART_MODES = ["lines+markers", "lines", "markers"]
 
-    def __init__(self, n_out: int, threshold: t.Optional[int] = None, zoom: t.Optional[bool] = True) -> None:
+    def __init__(
+        self,
+        n_out: int,
+        threshold: t.Optional[int] = None,
+        zoom: t.Optional[bool] = True,
+        # on_decimate: t.Optional[t.Callable] = None,
+        # apply_decimator: t.Optional[t.Callable] = None,
+    ) -> None:
         """Initialize a new `LTTB`.
 
         Arguments:
@@ -38,6 +45,10 @@ class LTTB(Decimator):
             zoom (Optional[bool]): set to True to reapply the decimation
                 when zoom or re-layout events are triggered.
         """
+        # on_decimate (Optional[Callable]): an user-defined function that is executed when the decimator
+        #     is found during runtime. This function can be used to provide custom decimation logic.
+        # apply_decimator (Optional[Callable]): an user-defined function that is executed when the decimator
+        #     is applied to modify the data.
         super().__init__(threshold, zoom)
         self._n_out = n_out
 

taipy/gui/data/decimator/minmax.py

@@ -13,7 +13,7 @@ import typing as t
 
 import numpy as np
 
-from ..utils import Decimator
+from .base import Decimator
 
 
 class MinMaxDecimator(Decimator):
@@ -27,7 +27,14 @@ class MinMaxDecimator(Decimator):
 
     _CHART_MODES = ["lines+markers", "lines", "markers"]
 
-    def __init__(self, n_out: int, threshold: t.Optional[int] = None, zoom: t.Optional[bool] = True):
+    def __init__(
+        self,
+        n_out: int,
+        threshold: t.Optional[int] = None,
+        zoom: t.Optional[bool] = True,
+        # on_decimate: t.Optional[t.Callable] = None,
+        # apply_decimator: t.Optional[t.Callable] = None,
+    ):
         """Initialize a new `MinMaxDecimator`.
 
         Arguments:
@@ -37,6 +44,10 @@ class MinMaxDecimator(Decimator):
             zoom (Optional[bool]): set to True to reapply the decimation
                 when zoom or re-layout events are triggered.
         """
+        # on_decimate (Optional[Callable]): an user-defined function that is executed when the decimator
+        #     is found during runtime. This function can be used to provide custom decimation logic.
+        # apply_decimator (Optional[Callable]): an user-defined function that is executed when the decimator
+        #     is applied to modify the data.
         super().__init__(threshold, zoom)
         self._n_out = n_out // 2
 

taipy/gui/data/decimator/rdp.py

@@ -13,7 +13,7 @@ import typing as t
 
 import numpy as np
 
-from ..utils import Decimator
+from .base import Decimator
 
 
 class RDP(Decimator):
@@ -34,6 +34,8 @@ class RDP(Decimator):
         n_out: t.Optional[int] = None,
         threshold: t.Optional[int] = None,
         zoom: t.Optional[bool] = True,
+        # on_decimate: t.Optional[t.Callable] = None,
+        # apply_decimator: t.Optional[t.Callable] = None,
     ):
         """Initialize a new `RDP`.
 
@@ -49,6 +51,10 @@ class RDP(Decimator):
             zoom (Optional[bool]): set to True to reapply the decimation
                 when zoom or re-layout events are triggered.
         """
+        # on_decimate (Optional[Callable]): an user-defined function that is executed when the decimator
+        #     is found during runtime. This function can be used to provide custom decimation logic.
+        # apply_decimator (Optional[Callable]): an user-defined function that is executed when the decimator
+        #     is applied to modify the data.
         super().__init__(threshold, zoom)
         self._epsilon = epsilon
         self._n_out = n_out

taipy/gui/data/decimator/scatter_decimator.py

@@ -13,7 +13,7 @@ import typing as t
 
 import numpy as np
 
-from ..utils import Decimator
+from .base import Decimator
 
 
 class ScatterDecimator(Decimator):
@@ -34,6 +34,8 @@ class ScatterDecimator(Decimator):
         max_overlap_points: t.Optional[int] = None,
         threshold: t.Optional[int] = None,
         zoom: t.Optional[bool] = True,
+        # on_decimate: t.Optional[t.Callable] = None,
+        # apply_decimator: t.Optional[t.Callable] = None,
     ):
         """Initialize a new `ScatterDecimator`.
 
@@ -48,6 +50,10 @@ class ScatterDecimator(Decimator):
             zoom (Optional[bool]): set to True to reapply the decimation
                 when zoom or re-layout events are triggered.
         """
+        # on_decimate (Optional[Callable]): an user-defined function that is executed when the decimator
+        #     is found during runtime. This function can be used to provide custom decimation logic.
+        # apply_decimator (Optional[Callable]): an user-defined function that is executed when the decimator
+        #     is applied to modify the data.
         super().__init__(threshold, zoom)
         binning_ratio = binning_ratio if binning_ratio is not None else 1
         self._binning_ratio = binning_ratio if binning_ratio > 0 else 1

taipy/gui/data/pandas_data_accessor.py

@@ -26,7 +26,6 @@ from ..utils import _RE_PD_TYPE, _get_date_col_str_name
 from .comparison import _compare_function
 from .data_accessor import _DataAccessor
 from .data_format import _DataFormat
-from .utils import _df_data_filter, _df_relayout
 
 _has_arrow_module = False
 if util.find_spec("pyarrow"):
@@ -391,46 +390,40 @@ class _PandasDataAccessor(_DataAccessor):
             ret_payload["alldata"] = True
             decimator_payload: t.Dict[str, t.Any] = payload.get("decimatorPayload", {})
             decimators = decimator_payload.get("decimators", [])
-            nb_rows_max = decimator_payload.get("width")
+            decimated_dfs: t.List[pd.DataFrame] = []
             for decimator_pl in decimators:
+                if decimator_pl is None:
+                    continue
                 decimator = decimator_pl.get("decimator")
+                if decimator is None:
+                    x_column = decimator_pl.get("xAxis", "")
+                    y_column = decimator_pl.get("yAxis", "")
+                    z_column = decimator_pl.get("zAxis", "")
+                    filterd_columns = [x_column, y_column, z_column] if z_column else [x_column, y_column]
+                    decimated_df = df.copy().filter(filterd_columns, axis=1)
+                    decimated_dfs.append(decimated_df)
+                    continue
                 decimator_instance = (
                     self._gui._get_user_instance(decimator, PropertyType.decimator.value)
                     if decimator is not None
                     else None
                 )
                 if isinstance(decimator_instance, PropertyType.decimator.value):
-                    x_column, y_column, z_column = (
-                        decimator_pl.get("xAxis", ""),
-                        decimator_pl.get("yAxis", ""),
-                        decimator_pl.get("zAxis", ""),
+                    # Run the on_decimate method -> check if the decimator should be applied
+                    # -> apply the decimator
+                    decimated_df, is_decimator_applied, is_copied = decimator_instance.on_decimate(
+                        df, decimator_pl, decimator_payload, is_copied
                     )
-                    chart_mode = decimator_pl.get("chartMode", "")
-                    if decimator_instance._zoom and "relayoutData" in decimator_payload and not z_column:
-                        relayoutData = decimator_payload.get("relayoutData", {})
-                        x0 = relayoutData.get("xaxis.range[0]")
-                        x1 = relayoutData.get("xaxis.range[1]")
-                        y0 = relayoutData.get("yaxis.range[0]")
-                        y1 = relayoutData.get("yaxis.range[1]")
-
-                        df, is_copied = _df_relayout(
-                            t.cast(pd.DataFrame, df), x_column, y_column, chart_mode, x0, x1, y0, y1, is_copied
-                        )
-
-                    if nb_rows_max and decimator_instance._is_applicable(df, nb_rows_max, chart_mode):
-                        try:
-                            df, is_copied = _df_data_filter(
-                                t.cast(pd.DataFrame, df),
-                                x_column,
-                                y_column,
-                                z_column,
-                                decimator=decimator_instance,
-                                payload=decimator_payload,
-                                is_copied=is_copied,
-                            )
-                            self._gui._call_on_change(f"{var_name}.{decimator}.nb_rows", len(df))
-                        except Exception as e:
-                            _warn(f"Limit rows error with {decimator} for Dataframe", e)
+                    # add decimated dataframe to the list of decimated
+                    decimated_dfs.append(decimated_df)
+                    if is_decimator_applied:
+                        self._gui._call_on_change(f"{var_name}.{decimator}.nb_rows", len(decimated_df))
+            # merge the decimated dataframes
+            if len(decimated_dfs) > 1:
+                df = pd.merge(*decimated_dfs, how="outer", left_index=True, right_index=True)
+            elif len(decimated_dfs) == 1:
+                df = decimated_dfs[0]
+            df = self.__build_transferred_cols(columns, t.cast(pd.DataFrame, df), is_copied=is_copied)
             if data_format is _DataFormat.CSV:
                 df = self.__build_transferred_cols(
                     columns,

taipy/gui/data/utils.py

@@ -1,150 +0,0 @@
-# Copyright 2021-2024 Avaiga Private Limited
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
-# an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations under the License.
-
-from __future__ import annotations
-
-import typing as t
-from abc import ABC, abstractmethod
-
-import numpy as np
-
-from .._warnings import _warn
-
-if t.TYPE_CHECKING:
-    import pandas as pd
-
-
-class Decimator(ABC):
-    """Base class for decimating chart data.
-
-    *Decimating* is the term used to name the process of reducing the number of
-    data points displayed in charts while retaining the overall shape of the traces.
-    `Decimator` is a base class that does decimation on data sets.
-
-    Taipy GUI comes out-of-the-box with several implementation of this class for
-    different use cases.
-    """
-
-    _CHART_MODES: t.List[str] = []
-
-    def __init__(self, threshold: t.Optional[int], zoom: t.Optional[bool]) -> None:
-        """Initialize a new `Decimator`.
-
-        Arguments:
-            threshold (Optional[int]): The minimum amount of data points before the
-                decimator class is applied.
-            zoom (Optional[bool]): set to True to reapply the decimation
-                when zoom or re-layout events are triggered.
-        """
-        super().__init__()
-        self.threshold = threshold
-        self._zoom = zoom if zoom is not None else True
-
-    def _is_applicable(self, data: t.Any, nb_rows_max: int, chart_mode: str):
-        if chart_mode not in self._CHART_MODES:
-            _warn(f"Decimator '{type(self).__name__}' is not optimized for chart mode '{chart_mode}'. Consider using other chart mode such as '{f'{chr(39)}, {chr(39)}'.join(self._CHART_MODES)}.'")  # noqa: E501
-        if self.threshold is None:
-            if nb_rows_max < len(data):
-                return True
-        elif self.threshold < len(data):
-            return True
-        return False
-
-    @abstractmethod
-    def decimate(self, data: np.ndarray, payload: t.Dict[str, t.Any]) -> np.ndarray:
-        """Decimate function.
-
-        This method is executed when the appropriate conditions specified in the
-        constructor are met. This function implements the algorithm that determines
-        which data points are kept or dropped.
-
-        Arguments:
-            data (numpy.array): An array containing all the data points represented as
-                tuples.
-            payload (Dict[str, any]): additional information on charts that is provided
-                at runtime.
-
-        Returns:
-            An array of Boolean mask values. The array should set True or False for each
-                of its indexes where True indicates that the corresponding data point
-                from *data* should be preserved, or False requires that this
-                data point be dropped.
-        """
-        raise NotImplementedError
-
-
-def _df_data_filter(
-    dataframe: pd.DataFrame,
-    x_column_name: t.Optional[str],
-    y_column_name: str,
-    z_column_name: str,
-    decimator: Decimator,
-    payload: t.Dict[str, t.Any],
-    is_copied: bool,
-):
-    df = dataframe.copy() if not is_copied else dataframe
-    if not x_column_name:
-        index = 0
-        while f"tAiPy_index_{index}" in df.columns:
-            index += 1
-        x_column_name = f"tAiPy_index_{index}"
-        df[x_column_name] = df.index
-    column_list = [x_column_name, y_column_name, z_column_name] if z_column_name else [x_column_name, y_column_name]
-    points = df[column_list].to_numpy()
-    mask = decimator.decimate(points, payload)
-    return df[mask], is_copied
-
-
-def _df_relayout(
-    dataframe: pd.DataFrame,
-    x_column: t.Optional[str],
-    y_column: str,
-    chart_mode: str,
-    x0: t.Optional[float],
-    x1: t.Optional[float],
-    y0: t.Optional[float],
-    y1: t.Optional[float],
-    is_copied: bool,
-):
-    if chart_mode not in ["lines+markers", "markers"]:
-        return dataframe, is_copied
-    # if chart data is invalid
-    if x0 is None and x1 is None and y0 is None and y1 is None:
-        return dataframe, is_copied
-    df = dataframe.copy() if not is_copied else dataframe
-    is_copied = True
-    has_x_col = True
-
-    if not x_column:
-        index = 0
-        while f"tAiPy_index_{index}" in df.columns:
-            index += 1
-        x_column = f"tAiPy_index_{index}"
-        df[x_column] = df.index
-        has_x_col = False
-
-    df_filter_conditions = []
-    # filter by x column by default
-    if x0 is not None:
-        df_filter_conditions.append(df[x_column] > x0)
-    if x1 is not None:
-        df_filter_conditions.append(df[x_column] < x1)
-    # y column will be filtered only if chart_mode is not lines+markers (eg. markers)
-    if chart_mode != "lines+markers":
-        if y0 is not None:
-            df_filter_conditions.append(df[y_column] > y0)
-        if y1 is not None:
-            df_filter_conditions.append(df[y_column] < y1)
-    if df_filter_conditions:
-        df = df.loc[np.bitwise_and.reduce(df_filter_conditions)]
-    if not has_x_col:
-        df.drop(x_column, axis=1, inplace=True)
-    return df, is_copied

tests/gui/data/test_pandas_data_accessor.py

@@ -257,7 +257,7 @@ def test_filter_by_date(gui: Gui, helpers, small_dataframe):
 
 
 def test_decimator(gui: Gui, helpers, small_dataframe):
-    a_decimator = ScatterDecimator()  # noqa: F841
+    a_decimator = ScatterDecimator(threshold=1)  # noqa: F841
 
     accessor = _PandasDataAccessor(gui)
     pd = pandas.DataFrame(data=small_dataframe)
@@ -265,7 +265,7 @@ def test_decimator(gui: Gui, helpers, small_dataframe):
     # set gui frame
     gui._set_frame(inspect.currentframe())
 
-    gui.add_page("test", "<|Hello {a_decimator}|button|id={btn_id}|>")
+    gui.add_page("test", "<|Hello {a_decimator}|button|>")
     gui.run(run_server=False)
     flask_client = gui._server.test_client()
 
@@ -283,7 +283,9 @@ def test_decimator(gui: Gui, helpers, small_dataframe):
                 "end": -1,
                 "alldata": True,
                 "decimatorPayload": {
-                    "decimators": [{"decimator": "a_decimator", "chartMode": "markers"}],
+                    "decimators": [
+                        {"decimator": "a_decimator", "chartMode": "markers", "xAxis": "name", "yAxis": "value"}
+                    ],
                     "width": 100,
                 },
             },

tests/gui/gui_specific/test_df_filter.py

@@ -13,31 +13,30 @@ from taipy.gui.data.decimator.lttb import LTTB
 from taipy.gui.data.decimator.minmax import MinMaxDecimator
 from taipy.gui.data.decimator.rdp import RDP
 from taipy.gui.data.decimator.scatter_decimator import ScatterDecimator
-from taipy.gui.data.utils import _df_data_filter
 
 
 def test_data_filter_1(csvdata):
-    df, _ = _df_data_filter(csvdata[:1500], None, "Daily hospital occupancy", "", MinMaxDecimator(100), {}, False)
+    df, _ = MinMaxDecimator(100)._df_apply_decimator(csvdata[:1500], None, "Daily hospital occupancy", "", {}, False)
     assert df.shape[0] == 100
 
 
 def test_data_filter_2(csvdata):
-    df, _ = _df_data_filter(csvdata[:1500], None, "Daily hospital occupancy", "", LTTB(100), {}, False)
+    df, _ = LTTB(100)._df_apply_decimator(csvdata[:1500], None, "Daily hospital occupancy", "", {}, False)
     assert df.shape[0] == 100
 
 
 def test_data_filter_3(csvdata):
-    df, _ = _df_data_filter(csvdata[:1500], None, "Daily hospital occupancy", "", RDP(n_out=100), {}, False)
+    df, _ = RDP(n_out=100)._df_apply_decimator(csvdata[:1500], None, "Daily hospital occupancy", "", {}, False)
     assert df.shape[0] == 100
 
 
 def test_data_filter_4(csvdata):
-    df, _ = _df_data_filter(csvdata[:1500], None, "Daily hospital occupancy", "", RDP(epsilon=100), {}, False)
+    df, _ = RDP(epsilon=100)._df_apply_decimator(csvdata[:1500], None, "Daily hospital occupancy", "", {}, False)
     assert df.shape[0] == 18
 
 
 def test_data_filter_5(csvdata):
-    df, _ = _df_data_filter(
-        csvdata[:1500], None, "Daily hospital occupancy", "", ScatterDecimator(), {"width": 200, "height": 100}, False
+    df, _ = ScatterDecimator()._df_apply_decimator(
+        csvdata[:1500], None, "Daily hospital occupancy", "", {"width": 200, "height": 100}, False
     )
     assert df.shape[0] == 1150