#426 case insensitive filtering (#2087)

* Add case-insensitive filtering for string columns

* Refactor string column filtering for case-insensitive matching

* feat: Add case-insensitive filtering to TableFilter component

- Implemented case-insensitive string filtering functionality for the TableFilter component.
- Added a toggle for case sensitivity in the UI.
- Updated tests to validate case-insensitive filtering.
- Improved existing logic for filtering based on the `matchCase` state.

* feat: Implement case-sensitive filtering, backend support, and backend tests

* Added case sensitivty SVG icon

* Refactor code: Update case sensitivity toggle switch in TableFilter component as per requested by maintainer

* Fix: Implement proper case-insensitive and date filtering logic in PandasDataAccessor

- Updated filtering logic to handle case-insensitive comparisons and string operations.
- Fixed issue with date filtering by correctly handling datetime conversion.
- Resolved test failures for contains and equals operations in case-insensitive string filtering.
- All test cases now pass successfully.

* Refactor date column handling and string operations in pandas_data_accessor.py

* Refactor pandas_data_accessor.py

* Refactor TableFilter component and related files

* Refactor TableFilter component to include match case option

* Fixed linter issues using ruff

---------

Co-authored-by: Fred Lefévère-Laoide <90181748+FredLL-Avaiga@users.noreply.github.com>

frontend/taipy-gui/src/components/Taipy/TableFilter.spec.tsx

@@ -12,7 +12,7 @@
  */
 
 import React from "react";
-import { render } from "@testing-library/react";
+import { render, screen } from "@testing-library/react";
 import "@testing-library/jest-dom";
 import userEvent from "@testing-library/user-event";
 
@@ -48,7 +48,8 @@ beforeEach(() => {
 });
 
 afterEach(() => {
-    // @ts-ignore
+    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+    // @ts-expect-error
     delete window.matchMedia;
 });
 
@@ -115,7 +116,7 @@ describe("Table Filter Component", () => {
         expect(validate).not.toBeDisabled();
     });
     it("behaves on boolean column", async () => {
-        const { getByTestId, getAllByTestId, findByRole, getByText, getAllByText } = render(
+        const { getByTestId, getAllByTestId, findByRole, getByText } = render(
             <TableFilter columns={tableColumns} colsOrder={colsOrder} onValidate={jest.fn()} filteredCount={0} />
         );
         const elt = getByTestId("FilterListIcon");
@@ -245,3 +246,45 @@ describe("Table Filter Component", () => {
         expect(ddElts2).toHaveLength(2);
     });
 });
+describe("Table Filter Component - Case Insensitive Test", () => {
+    it("renders the case sensitivity toggle switch", async () => {
+        const { getByTestId, getAllByTestId, findByRole, getByText, getAllByText } = render(
+            <TableFilter columns={tableColumns} colsOrder={colsOrder} onValidate={jest.fn()} filteredCount={0} />
+        );
+
+        // Open filter popover
+        const filterIcon = getByTestId("FilterListIcon");
+        await userEvent.click(filterIcon);
+
+        // Select string column from dropdown
+        const dropdownIcons = getAllByTestId("ArrowDropDownIcon");
+        await userEvent.click(dropdownIcons[0].parentElement?.firstElementChild || dropdownIcons[0]);
+        await findByRole("listbox");
+        await userEvent.click(getByText("StringCol"));
+
+        // Select 'contains' filter action
+        await userEvent.click(dropdownIcons[1].parentElement?.firstElementChild || dropdownIcons[1]);
+        await findByRole("listbox");
+        await userEvent.click(getByText("contains"));
+
+        // Check for the case-sensitive toggle and interact with it
+        const caseSensitiveToggle = screen.getByRole("checkbox", { name: /case sensitive toggle/i });
+        expect(caseSensitiveToggle).toBeInTheDocument(); // Ensure the toggle is rendered
+        await userEvent.click(caseSensitiveToggle); // Toggle case sensitivity off
+
+        // Input some test text and validate case insensitivity
+        const inputs = getAllByText("Empty String");
+        const inputField = inputs[0].nextElementSibling?.firstElementChild || inputs[0];
+        await userEvent.click(inputField);
+        await userEvent.type(inputField, "CASETEST");
+
+        // Ensure the validate button is enabled
+        const validateButton = getByTestId("CheckIcon").parentElement;
+        expect(validateButton).not.toBeDisabled();
+
+        // Test case-insensitivity by changing input case
+        await userEvent.clear(inputField);
+        await userEvent.type(inputField, "casetest");
+        expect(validateButton).not.toBeDisabled();
+    });
+});

frontend/taipy-gui/src/components/Taipy/TableFilter.tsx

@@ -27,12 +27,14 @@ import Popover, { PopoverOrigin } from "@mui/material/Popover";
 import Select, { SelectChangeEvent } from "@mui/material/Select";
 import TextField from "@mui/material/TextField";
 import Tooltip from "@mui/material/Tooltip";
+import Switch from "@mui/material/Switch";
 import { DateField, LocalizationProvider } from "@mui/x-date-pickers";
 import { AdapterDateFns } from "@mui/x-date-pickers/AdapterDateFnsV3";
 
 import { ColumnDesc, defaultDateFormat, getSortByIndex, iconInRowSx, FilterDesc } from "./tableUtils";
 import { getDateTime, getTypeFromDf } from "../../utils";
 import { getSuffixedClassNames } from "./utils";
+import { MatchCase } from "../icons/MatchCase";
 
 interface TableFilterProps {
     columns: Record<string, ColumnDesc>;
@@ -92,7 +94,13 @@ const getActionsByType = (colType?: string) =>
     (colType && colType in actionsByType && actionsByType[colType]) ||
     (colType === "any" ? { ...actionsByType.string, ...actionsByType.number } : actionsByType.string);
 
-const getFilterDesc = (columns: Record<string, ColumnDesc>, colId?: string, act?: string, val?: string) => {
+const getFilterDesc = (
+    columns: Record<string, ColumnDesc>,
+    colId?: string,
+    act?: string,
+    val?: string,
+    matchCase?: boolean
+) => {
     if (colId && act && val !== undefined) {
         const colType = getTypeFromDf(columns[colId].type);
         if (val === "" && (colType === "date" || colType === "number" || colType === "boolean")) {
@@ -113,6 +121,7 @@ const getFilterDesc = (columns: Record<string, ColumnDesc>, colId?: string, act?
                             : val
                         : val,
                 type: colType,
+                matchCase: !!matchCase,
             } as FilterDesc;
         } catch (e) {
             console.info("could not parse value ", val, e);
@@ -126,9 +135,15 @@ const FilterRow = (props: FilterRowProps) => {
     const [colId, setColId] = useState<string>("");
     const [action, setAction] = useState<string>("");
     const [val, setVal] = useState<string>("");
+    const [matchCase, setMatchCase] = useState<boolean>(false);
     const [enableCheck, setEnableCheck] = useState(false);
     const [enableDel, setEnableDel] = useState(false);
 
+    // Function to handle case-sensitivity toggle
+    const toggleMatchCase = useCallback(() => {
+        setMatchCase((prev) => !prev);
+    }, []);
+
     const onColSelect = useCallback(
         (e: SelectChangeEvent<string>) => {
             setColId(e.target.value);
@@ -136,6 +151,7 @@ const FilterRow = (props: FilterRowProps) => {
         },
         [columns, action, val]
     );
+
     const onActSelect = useCallback(
         (e: SelectChangeEvent<string>) => {
             setAction(e.target.value);
@@ -143,6 +159,7 @@ const FilterRow = (props: FilterRowProps) => {
         },
         [columns, colId, val]
     );
+
     const onValueChange = useCallback(
         (e: ChangeEvent<HTMLInputElement>) => {
             setVal(e.target.value);
@@ -150,13 +167,16 @@ const FilterRow = (props: FilterRowProps) => {
         },
         [columns, colId, action]
     );
+
     const onValueAutoComp = useCallback(
         (e: SyntheticEvent, value: string | null) => {
-            setVal(value || "");
-            setEnableCheck(!!getFilterDesc(columns, colId, action, value || ""));
+            const inputValue = value || "";
+            setVal(inputValue);
+            setEnableCheck(!!getFilterDesc(columns, colId, action, inputValue));
         },
         [columns, colId, action]
     );
+
     const onValueSelect = useCallback(
         (e: SelectChangeEvent<string>) => {
             setVal(e.target.value);
@@ -164,6 +184,7 @@ const FilterRow = (props: FilterRowProps) => {
         },
         [columns, colId, action]
     );
+
     const onDateChange = useCallback(
         (v: Date | null) => {
             const dv = !(v instanceof Date) || isNaN(v.valueOf()) ? "" : v.toISOString();
@@ -174,10 +195,11 @@ const FilterRow = (props: FilterRowProps) => {
     );
 
     const onDeleteClick = useCallback(() => setFilter(idx, undefined as unknown as FilterDesc, true), [idx, setFilter]);
+
     const onCheckClick = useCallback(() => {
-        const fd = getFilterDesc(columns, colId, action, val);
+        const fd = getFilterDesc(columns, colId, action, val, matchCase);
         fd && setFilter(idx, fd);
-    }, [idx, setFilter, columns, colId, action, val]);
+    }, [idx, setFilter, columns, colId, action, val, matchCase]);
 
     useEffect(() => {
         if (filter && idx > -1) {
@@ -280,9 +302,24 @@ const FilterRow = (props: FilterRowProps) => {
                         onChange={onValueChange}
                         label={`${val ? "" : "Empty "}String`}
                         margin="dense"
+                        slotProps={{
+                            input: {
+                                endAdornment: (
+                                    <Switch
+                                        onChange={toggleMatchCase}
+                                        checked={matchCase}
+                                        size="small"
+                                        checkedIcon={<MatchCase />}
+                                        icon={<MatchCase color="disabled" />}
+                                        inputProps={{ "aria-label": "Case Sensitive Toggle" }}
+                                    />
+                                ),
+                            },
+                        }}
                     />
                 )}
             </Grid>
+
             <Grid size={1}>
                 <Tooltip title="Validate">
                     <span>

frontend/taipy-gui/src/components/Taipy/tableUtils.tsx

@@ -222,6 +222,7 @@ export interface FilterDesc {
     action: string;
     value: string | number | boolean | Date;
     type: string;
+    matchcase?: boolean;
 }
 
 export const defaultColumns = {} as Record<string, ColumnDesc>;

frontend/taipy-gui/src/components/icons/MatchCase.tsx

@@ -0,0 +1,23 @@
+/*
+ * Copyright 2021-2024 Avaiga Private Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+import React from "react";
+import SvgIcon, { SvgIconProps } from "@mui/material/SvgIcon";
+
+export const MatchCase = (props: SvgIconProps) => (
+    <SvgIcon {...props} viewBox="0 0 16 16">
+        <g stroke="currentColor">
+            <path d="M20 14c0-1.5-.5-2-2-2h-2v-1c0-1 0-1-2-1v9h4c1.5 0 2-.53 2-2zm-8-2c0-1.5-.53-2-2-2H6c-1.5 0-2 .5-2 2v7h2v-3h4v3h2zm-2-5h4V5h-4zm12 2v11c0 1.11-.89 2-2 2H4a2 2 0 0 1-2-2V9c0-1.11.89-2 2-2h4V5l2-2h4l2 2v2h4a2 2 0 0 1 2 2m-6 8h2v-3h-2zM6 12h4v2H6z" />
+        </g>
+    </SvgIcon>
+);

taipy/gui/data/pandas_data_accessor.py

@@ -267,18 +267,37 @@ class _PandasDataAccessor(_DataAccessor):
                 col = fd.get("col")
                 val = fd.get("value")
                 action = fd.get("action")
+                match_case = fd.get("matchCase", False) is not False  # Ensure it's a boolean
+                right = None
+                col_expr = f"`{col}`"
+
                 if isinstance(val, str):
                     if self.__is_date_column(t.cast(pd.DataFrame, df), col):
                         val = datetime.fromisoformat(val[:-1])
+                    elif not match_case:
+                        if action != "contains":
+                            col_expr = f"{col_expr}.str.lower()"
+                        val = val.lower()
+                    vars.append(val)
+                    val_var = f"@vars[{len(vars) - 1}]"
+                    if action == "contains":
+                        right = f".str.contains({val_var}{'' if match_case else ', case=False'})"
+                else:
                     vars.append(val)
-                val = f"@vars[{len(vars) - 1}]" if isinstance(val, (str, datetime)) else val
-                right = f".str.contains({val})" if action == "contains" else f" {action} {val}"
+                    val_var = f"@vars[{len(vars) - 1}]"
+
+                if right is None:
+                    right = f" {action} {val_var}"
+
                 if query:
                     query += " and "
-                query += f"`{col}`{right}"
+                query += f"{col_expr}{right}"
+
+            # Apply filters using df.query()
             try:
-                df = df.query(query)
-                is_copied = True
+                if query:
+                    df = df.query(query)
+                    is_copied = True
             except Exception as e:
                 _warn(f"Dataframe filtering: invalid query '{query}' on {df.head()}", e)
 

tests/gui/data/test_pandas_data_accessor.py

@@ -13,8 +13,11 @@ import inspect
 import os
 from datetime import datetime
 from importlib import util
+from unittest.mock import Mock
 
 import pandas
+import pandas as pd
+import pytest
 from flask import g
 
 from taipy.gui import Gui
@@ -23,6 +26,26 @@ from taipy.gui.data.decimator import ScatterDecimator
 from taipy.gui.data.pandas_data_accessor import _PandasDataAccessor
 
 
+# Define a mock to simulate _DataFormat behavior with a 'value' attribute
+class MockDataFormat:
+    LIST = Mock(value="list")
+    CSV = Mock(value="csv")
+
+@pytest.fixture
+def pandas_accessor():
+    gui = Mock()
+    return _PandasDataAccessor(gui=gui)
+
+@pytest.fixture
+def sample_df():
+    data = {
+        "StringCol": ["Apple", "Banana", "Cherry", "apple"],
+        "NumberCol": [10, 20, 30, 40],
+        "BoolCol": [True, False, True, False],
+        "DateCol": pd.to_datetime(["2020-01-01", "2021-06-15", "2022-08-22", "2023-03-05"])
+    }
+    return pd.DataFrame(data)
+
 def test_simple_data(gui: Gui, helpers, small_dataframe):
     accessor = _PandasDataAccessor(gui)
     pd = pandas.DataFrame(data=small_dataframe)
@@ -255,6 +278,58 @@ def test_filter_by_date(gui: Gui, helpers, small_dataframe):
     value = accessor.get_data("x", pd, query, _DataFormat.JSON)
     assert len(value["value"]["data"]) == 1
 
+def test_contains_case_sensitive(pandas_accessor, sample_df):
+    payload = {
+        "filters": [{"col": "StringCol", "value": "Apple", "action": "contains", "matchCase": True}]
+    }
+    result = pandas_accessor.get_data("test_var", sample_df, payload, MockDataFormat.LIST)
+    filtered_data = pd.DataFrame(result['value']['data'])
+
+    assert len(filtered_data) == 1
+    assert filtered_data.iloc[0]['StringCol'] == 'Apple'
+
+def test_contains_case_insensitive(pandas_accessor, sample_df):
+    payload = {
+        "filters": [{"col": "StringCol", "value": "apple", "action": "contains", "matchCase": False}]
+    }
+    result = pandas_accessor.get_data("test_var", sample_df, payload, MockDataFormat.LIST)
+    filtered_data = pd.DataFrame(result['value']['data'])
+
+    assert len(filtered_data) == 2
+    assert 'Apple' in filtered_data['StringCol'].values
+    assert 'apple' in filtered_data['StringCol'].values
+
+def test_equals_case_sensitive(pandas_accessor, sample_df):
+    payload = {
+        "filters": [{"col": "StringCol", "value": "Apple", "action": "==", "matchCase": True}]
+    }
+    result = pandas_accessor.get_data("test_var", sample_df, payload, MockDataFormat.LIST)
+    filtered_data = pd.DataFrame(result['value']['data'])
+
+    assert len(filtered_data) == 1
+    assert filtered_data.iloc[0]['StringCol'] == 'Apple'
+
+def test_equals_case_insensitive(pandas_accessor, sample_df):
+    payload = {
+        "filters": [{"col": "StringCol", "value": "apple", "action": "==", "matchCase": False}]
+    }
+    result = pandas_accessor.get_data("test_var", sample_df, payload, MockDataFormat.LIST)
+    filtered_data = pd.DataFrame(result['value']['data'])
+
+    assert len(filtered_data) == 2
+    assert 'Apple' in filtered_data['StringCol'].values
+    assert 'apple' in filtered_data['StringCol'].values
+
+def test_not_equals_case_insensitive(pandas_accessor, sample_df):
+    payload = {
+        "filters": [{"col": "StringCol", "value": "apple", "action": "!=", "matchCase": False}]
+    }
+    result = pandas_accessor.get_data("test_var", sample_df, payload, MockDataFormat.LIST)
+    filtered_data = pd.DataFrame(result['value']['data'])
+
+    assert len(filtered_data) == 2
+    assert 'Banana' in filtered_data['StringCol'].values
+    assert 'Cherry' in filtered_data['StringCol'].values
 
 def test_decimator(gui: Gui, helpers, small_dataframe):
     a_decimator = ScatterDecimator(threshold=1)  # noqa: F841