Merge pull request #806 from Avaiga/feature/support-for-older-pandas

feat: support for pandas 1.3

Pipfile

@@ -21,7 +21,7 @@ markdown = "==3.4.4"
 marshmallow = "==3.20.1"
 networkx = "==2.6"
 openpyxl = "==3.1.2"
-pandas = ">=2.0.2,<2.3.0"
+pandas = ">=1.3.5,<2.3.0"
 pyarrow = "==14.0.2"
 pymongo = {extras = ["srv"], version = "==4.2.0"}
 python-dotenv = "==1.0.0"

taipy/core/Pipfile

@@ -4,7 +4,7 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
-pandas = ">=2.0.2,<2.3.0"
+pandas = ">=1.3.5,<2.3.0"
 networkx = "==2.6"
 openpyxl = "==3.1.2"
 pyarrow = "==14.0.2"

taipy/core/data/_abstract_sql.py

@@ -118,7 +118,7 @@ class _AbstractSQLDataNode(DataNode, _AbstractTabularDataNode):
             **properties,
         )
         self._engine = None
-        if not self._last_edit_date:
+        if not self._last_edit_date:  # type: ignore
             self._last_edit_date = datetime.now()
 
         self._TAIPY_PROPERTIES.update(
@@ -226,9 +226,14 @@ class _AbstractSQLDataNode(DataNode, _AbstractTabularDataNode):
         join_operator=JoinOperator.AND,
     ):
         with self._get_engine().connect() as conn:
+            result = conn.execute(text(self._get_read_query(operators, join_operator)))
+
+            # On pandas 1.3.5 there's a bug that makes that the dataframe from sqlalchemy query is
+            # created without headers
+            keys = [col for col in result.keys()]
             if columns:
-                return pd.DataFrame(conn.execute(text(self._get_read_query(operators, join_operator))))[columns]
-            return pd.DataFrame(conn.execute(text(self._get_read_query(operators, join_operator))))
+                return pd.DataFrame(result, columns=keys)[columns]
+            return pd.DataFrame(result, columns=keys)
 
     @abstractmethod
     def _get_read_query(self, operators: Optional[Union[List, Tuple]] = None, join_operator=JoinOperator.AND):

taipy/core/data/excel.py

@@ -305,6 +305,11 @@ class ExcelDataNode(DataNode, _AbstractFileDataNode, _AbstractTabularDataNode):
                 )
 
     def _append(self, data: Any):
+        from importlib.metadata import version
+
+        if version("pandas") < "1.4":
+            raise ImportError("The append method is only available for pandas version 1.4 or higher.")
+
         if isinstance(data, Dict) and all(isinstance(x, (pd.DataFrame, np.ndarray)) for x in data.values()):
             self.__append_excel_with_multiple_sheets(data)
         elif isinstance(data, pd.DataFrame):

taipy/core/data/parquet.py

@@ -248,8 +248,10 @@ class ParquetDataNode(DataNode, _AbstractFileDataNode, _AbstractTabularDataNode)
         if isinstance(data, pd.DataFrame):
             data.to_parquet(self._path, **kwargs)
         else:
-            pd.DataFrame(data).to_parquet(self._path, **kwargs)
-
+            _df = pd.DataFrame(data)
+            # Ensure that the columns are strings, otherwise writing will fail with pandas 1.3.5
+            _df.columns = _df.columns.astype(str)
+            _df.to_parquet(self._path, **kwargs)
         self.track_edit(timestamp=datetime.now(), job_id=job_id)
 
     def read_with_kwargs(self, **read_kwargs):

taipy/core/setup.py

@@ -31,7 +31,7 @@ requirements = [
     "pyarrow>=14.0.2,<15.0",
     "networkx>=2.6,<3.0",
     "openpyxl>=3.1.2,<3.2",
-    "pandas>=2.0.0,<3.0",
+    "pandas>=1.3.5,<3.0",
     "pymongo[srv]>=4.2.0,<5.0",
     "sqlalchemy>=2.0.16,<2.1",
     "toml>=0.10,<0.11",

taipy/gui/Pipfile

@@ -12,7 +12,7 @@ gevent = "==23.7.0"
 gevent-websocket = "==0.10.1"
 kthread = "==0.2.3"
 markdown = "==3.4.4"
-pandas = ">=2.0.2,<2.3.0"
+pandas = ">=1.3.5,<2.3.0"
 pyarrow = "==14.0.2"
 pyngrok = "==5.1"
 python-dotenv = "==1.0.0"

taipy/gui/setup.py

@@ -34,7 +34,7 @@ requirements = [
     "flask-cors>=4.0.0,<5.0",
     "flask-socketio>=5.3.6,<6.0",
     "markdown>=3.4.4,<4.0",
-    "pandas>=2.0.0,<3.0",
+    "pandas>=1.3.5,<3.0",
     "python-dotenv>=1.0.0,<1.1",
     "pytz>=2021.3,<2022.2",
     "tzlocal>=3.0,<5.0",

tools/packages/taipy-core/setup.requirements.txt

@@ -1,7 +1,7 @@
 boto3>=1.29.4,<=1.34.34
 networkx>=2.6,<=3.2.1
 openpyxl>=3.1.2,<=3.1.2
-pandas>=2.0.0,<=2.2.0
+pandas>=1.3.5,<=2.2.0
 pyarrow>=14.0.2,<=15.0.0
 pymongo[srv]>=4.2.0,<=4.6.1
 sqlalchemy>=2.0.16,<=2.0.25

tools/packages/taipy-gui/setup.requirements.txt

@@ -7,7 +7,7 @@ gevent-websocket>=0.10.1,<=0.10.1
 gitignore-parser>=0.1,<=0.1.11
 kthread>=0.2.3,<=0.2.3
 markdown>=3.4.4,<=3.5.2
-pandas>=2.0.0,<=2.2.0
+pandas>=1.3.5,<=2.2.0
 python-dotenv>=1.0.0,<=1.0.1
 pytz>=2021.3,<=2023.3.post1
 simple-websocket>=0.10.1,<=1.0.0