Merge branch 'develop' into test/fileSelector

.gitignore

@@ -86,6 +86,7 @@ dist/
 .taipy/
 user_data/
 .my_data/
+Data
 
 # demo files
 demo-*

taipy/core/data/_abstract_sql.py

@@ -154,24 +154,25 @@ class _AbstractSQLDataNode(DataNode, _TabularDataNodeMixin):
         return self._engine
 
     def _conn_string(self) -> str:
-        engine = self.properties.get(self.__DB_ENGINE_KEY)
+        properties = self.properties
+        engine = properties.get(self.__DB_ENGINE_KEY)
 
         if self.__DB_USERNAME_KEY in self._ENGINE_REQUIRED_PROPERTIES[engine]:
-            username = self.properties.get(self.__DB_USERNAME_KEY)
+            username = properties.get(self.__DB_USERNAME_KEY)
             username = urllib.parse.quote_plus(username)
 
         if self.__DB_PASSWORD_KEY in self._ENGINE_REQUIRED_PROPERTIES[engine]:
-            password = self.properties.get(self.__DB_PASSWORD_KEY)
+            password = properties.get(self.__DB_PASSWORD_KEY)
             password = urllib.parse.quote_plus(password)
 
         if self.__DB_NAME_KEY in self._ENGINE_REQUIRED_PROPERTIES[engine]:
-            db_name = self.properties.get(self.__DB_NAME_KEY)
+            db_name = properties.get(self.__DB_NAME_KEY)
             db_name = urllib.parse.quote_plus(db_name)
 
-        host = self.properties.get(self.__DB_HOST_KEY, self.__DB_HOST_DEFAULT)
-        port = self.properties.get(self.__DB_PORT_KEY, self.__DB_PORT_DEFAULT)
-        driver = self.properties.get(self.__DB_DRIVER_KEY, self.__DB_DRIVER_DEFAULT)
-        extra_args = self.properties.get(self.__DB_EXTRA_ARGS_KEY, {})
+        host = properties.get(self.__DB_HOST_KEY, self.__DB_HOST_DEFAULT)
+        port = properties.get(self.__DB_PORT_KEY, self.__DB_PORT_DEFAULT)
+        driver = properties.get(self.__DB_DRIVER_KEY, self.__DB_DRIVER_DEFAULT)
+        extra_args = properties.get(self.__DB_EXTRA_ARGS_KEY, {})
 
         if driver:
             extra_args = {**extra_args, "driver": driver}
@@ -186,23 +187,24 @@ class _AbstractSQLDataNode(DataNode, _TabularDataNodeMixin):
         elif engine == self.__ENGINE_POSTGRESQL:
             return f"postgresql+psycopg2://{username}:{password}@{host}:{port}/{db_name}?{extra_args_str}"
         elif engine == self.__ENGINE_SQLITE:
-            folder_path = self.properties.get(self.__SQLITE_FOLDER_PATH, self.__SQLITE_FOLDER_PATH_DEFAULT)
-            file_extension = self.properties.get(self.__SQLITE_FILE_EXTENSION, self.__SQLITE_FILE_EXTENSION_DEFAULT)
+            folder_path = properties.get(self.__SQLITE_FOLDER_PATH, self.__SQLITE_FOLDER_PATH_DEFAULT)
+            file_extension = properties.get(self.__SQLITE_FILE_EXTENSION, self.__SQLITE_FILE_EXTENSION_DEFAULT)
             return "sqlite:///" + os.path.join(folder_path, f"{db_name}{file_extension}")
-
         raise UnknownDatabaseEngine(f"Unknown engine: {engine}")
 
     def filter(self, operators: Optional[Union[List, Tuple]] = None, join_operator=JoinOperator.AND):
-        if self.properties[self._EXPOSED_TYPE_PROPERTY] == self._EXPOSED_TYPE_PANDAS:
+        properties = self.properties
+        if properties[self._EXPOSED_TYPE_PROPERTY] == self._EXPOSED_TYPE_PANDAS:
             return self._read_as_pandas_dataframe(operators=operators, join_operator=join_operator)
-        if self.properties[self._EXPOSED_TYPE_PROPERTY] == self._EXPOSED_TYPE_NUMPY:
+        if properties[self._EXPOSED_TYPE_PROPERTY] == self._EXPOSED_TYPE_NUMPY:
             return self._read_as_numpy(operators=operators, join_operator=join_operator)
         return self._read_as(operators=operators, join_operator=join_operator)
 
     def _read(self):
-        if self.properties[self._EXPOSED_TYPE_PROPERTY] == self._EXPOSED_TYPE_PANDAS:
+        properties = self.properties
+        if properties[self._EXPOSED_TYPE_PROPERTY] == self._EXPOSED_TYPE_PANDAS:
             return self._read_as_pandas_dataframe()
-        if self.properties[self._EXPOSED_TYPE_PROPERTY] == self._EXPOSED_TYPE_NUMPY:
+        if properties[self._EXPOSED_TYPE_PROPERTY] == self._EXPOSED_TYPE_NUMPY:
             return self._read_as_numpy()
         return self._read_as()
 

taipy/core/data/_data_manager.py

@@ -32,11 +32,10 @@ from .data_node_id import DataNodeId
 
 
 class _DataManager(_Manager[DataNode], _VersionMixin):
-    __DATA_NODE_CLASS_MAP = DataNode._class_map()  # type: ignore
+    _DATA_NODE_CLASS_MAP = DataNode._class_map()  # type: ignore
     _ENTITY_NAME = DataNode.__name__
     _EVENT_ENTITY_TYPE = EventEntityType.DATA_NODE
     _repository: _DataFSRepository
-    __NAME_KEY = "name"
 
     @classmethod
     def _bulk_get_or_create(
@@ -102,7 +101,7 @@ class _DataManager(_Manager[DataNode], _VersionMixin):
             else:
                 storage_type = Config.sections[DataNodeConfig.name][_Config.DEFAULT_KEY].storage_type
 
-            return cls.__DATA_NODE_CLASS_MAP[storage_type](
+            return cls._DATA_NODE_CLASS_MAP[storage_type](
                 config_id=data_node_config.id,
                 scope=data_node_config.scope or DataNodeConfig._DEFAULT_SCOPE,
                 validity_period=data_node_config.validity_period,

taipy/core/data/_file_datanode_mixin.py

@@ -14,11 +14,13 @@ import pathlib
 import shutil
 from datetime import datetime
 from os.path import isfile
-from typing import Any, Dict, Optional
+from typing import Any, Callable, Dict, Optional
 
 from taipy.config.config import Config
+from taipy.logger._taipy_logger import _TaipyLogger
 
 from .._entity._reload import _self_reload
+from ..reason import InvalidUploadFile, ReasonCollection, UploadFileCanNotBeRead
 from .data_node import DataNode
 from .data_node_id import Edit
 
@@ -34,6 +36,8 @@ class _FileDataNodeMixin(object):
     _DEFAULT_PATH_KEY = "default_path"
     _IS_GENERATED_KEY = "is_generated"
 
+    __logger = _TaipyLogger._get_logger()
+
     def __init__(self, properties: Dict) -> None:
         self._path: str = properties.get(self._PATH_KEY, properties.get(self._DEFAULT_PATH_KEY))
         self._is_generated: bool = properties.get(self._IS_GENERATED_KEY, self._path is None)
@@ -92,3 +96,56 @@ class _FileDataNodeMixin(object):
         if os.path.exists(old_path):
             shutil.move(old_path, new_path)
         return new_path
+
+    def _get_downloadable_path(self) -> str:
+        """Get the downloadable path of the file data of the data node.
+
+        Returns:
+            The downloadable path of the file data of the data node if it exists, otherwise an empty string.
+        """
+        if os.path.exists(self.path) and isfile(self._path):
+            return self.path
+
+        return ""
+
+    def _upload(self, path: str, upload_checker: Optional[Callable[[str, Any], bool]] = None) -> ReasonCollection:
+        """Upload a file data to the data node.
+
+        Parameters:
+            path (str): The path of the file to upload to the data node.
+            upload_checker (Optional[Callable[[str, Any], bool]]): A function to check if the upload is allowed.
+                The function takes the title of the upload data and the data itself as arguments and returns
+                True if the upload is allowed, otherwise False.
+
+        Returns:
+            True if the upload was successful, otherwise False.
+        """
+        from ._data_manager_factory import _DataManagerFactory
+
+        reason_collection = ReasonCollection()
+
+        upload_path = pathlib.Path(path)
+
+        try:
+            upload_data = self._read_from_path(str(upload_path))
+        except Exception as err:
+            self.__logger.error(f"Error while uploading {upload_path.name} to data node {self.id}:")  # type: ignore[attr-defined]
+            self.__logger.error(f"Error: {err}")
+            reason_collection._add_reason(self.id, UploadFileCanNotBeRead(upload_path.name, self.id))  # type: ignore[attr-defined]
+            return reason_collection
+
+        if upload_checker is not None:
+            if not upload_checker(upload_path.name, upload_data):
+                reason_collection._add_reason(self.id, InvalidUploadFile(upload_path.name, self.id))  # type: ignore[attr-defined]
+                return reason_collection
+
+        shutil.copy(upload_path, self.path)
+
+        self.track_edit(timestamp=datetime.now())  # type: ignore[attr-defined]
+        self.unlock_edit()  # type: ignore[attr-defined]
+        _DataManagerFactory._build_manager()._set(self)  # type: ignore[arg-type]
+
+        return reason_collection
+
+    def _read_from_path(self, path: Optional[str] = None, **read_kwargs) -> Any:
+        raise NotImplementedError

taipy/core/data/_tabular_datanode_mixin.py

@@ -26,18 +26,19 @@ class _TabularDataNodeMixin(object):
     _EXPOSED_TYPE_NUMPY = "numpy"
     _EXPOSED_TYPE_PANDAS = "pandas"
     _EXPOSED_TYPE_MODIN = "modin"  # Deprecated in favor of pandas since 3.1.0
-    __VALID_STRING_EXPOSED_TYPES = [_EXPOSED_TYPE_PANDAS, _EXPOSED_TYPE_NUMPY]
+    _VALID_STRING_EXPOSED_TYPES = [_EXPOSED_TYPE_PANDAS, _EXPOSED_TYPE_NUMPY]
 
     def __init__(self, **kwargs) -> None:
-        self._decoder: Union[Callable[[List[Any]], Any], Callable[[Dict[Any, Any]], Any]]
+        self._decoder: Union[Callable, Any]
         self.custom_document = kwargs.get(self._EXPOSED_TYPE_PROPERTY)
-        if kwargs.get(self._HAS_HEADER_PROPERTY, True):
-            self._decoder = self._default_decoder_with_header
-        else:
-            self._decoder = self._default_decoder_without_header
+
         custom_decoder = getattr(self.custom_document, "decode", None)
         if callable(custom_decoder):
             self._decoder = custom_decoder
+        elif kwargs.get(self._HAS_HEADER_PROPERTY, True):
+            self._decoder = self._default_decoder_with_header
+        else:
+            self._decoder = self._default_decoder_without_header
 
         self._encoder = self._default_encoder
         custom_encoder = getattr(self.custom_document, "encode", None)
@@ -66,7 +67,7 @@ class _TabularDataNodeMixin(object):
 
     @classmethod
     def _check_exposed_type(cls, exposed_type):
-        valid_string_exposed_types = cls.__VALID_STRING_EXPOSED_TYPES
+        valid_string_exposed_types = cls._VALID_STRING_EXPOSED_TYPES
         if isinstance(exposed_type, str) and exposed_type not in valid_string_exposed_types:
             raise InvalidExposedType(
                 f"Invalid string exposed type {exposed_type}. Supported values are "

taipy/core/data/aws_s3.py

@@ -147,15 +147,17 @@ class S3ObjectDataNode(DataNode):
         return cls.__STORAGE_TYPE
 
     def _read(self):
+        properties = self.properties
         aws_s3_object = self._s3_client.get_object(
-            Bucket=self.properties[self.__AWS_STORAGE_BUCKET_NAME],
-            Key=self.properties[self.__AWS_S3_OBJECT_KEY],
+            Bucket=properties[self.__AWS_STORAGE_BUCKET_NAME],
+            Key=properties[self.__AWS_S3_OBJECT_KEY],
         )
         return aws_s3_object["Body"].read().decode("utf-8")
 
     def _write(self, data: Any):
+        properties = self.properties
         self._s3_client.put_object(
-            Bucket=self.properties[self.__AWS_STORAGE_BUCKET_NAME],
-            Key=self.properties[self.__AWS_S3_OBJECT_KEY],
+            Bucket=properties[self.__AWS_STORAGE_BUCKET_NAME],
+            Key=properties[self.__AWS_S3_OBJECT_KEY],
             Body=data,
         )

taipy/core/data/csv.py

@@ -137,59 +137,71 @@ class CSVDataNode(DataNode, _FileDataNodeMixin, _TabularDataNodeMixin):
         return cls.__STORAGE_TYPE
 
     def _read(self):
-        if self.properties[self._EXPOSED_TYPE_PROPERTY] == self._EXPOSED_TYPE_PANDAS:
-            return self._read_as_pandas_dataframe()
-        if self.properties[self._EXPOSED_TYPE_PROPERTY] == self._EXPOSED_TYPE_NUMPY:
-            return self._read_as_numpy()
-        return self._read_as()
-
-    def _read_as(self):
-        with open(self._path, encoding=self.properties[self.__ENCODING_KEY]) as csvFile:
-            if self.properties[self._HAS_HEADER_PROPERTY]:
-                reader = csv.DictReader(csvFile)
-            else:
-                reader = csv.reader(csvFile)
+        return self._read_from_path()
+
+    def _read_from_path(self, path: Optional[str] = None, **read_kwargs) -> Any:
+        if path is None:
+            path = self._path
+
+        properties = self.properties
+        if properties[self._EXPOSED_TYPE_PROPERTY] == self._EXPOSED_TYPE_PANDAS:
+            return self._read_as_pandas_dataframe(path=path)
+        if properties[self._EXPOSED_TYPE_PROPERTY] == self._EXPOSED_TYPE_NUMPY:
+            return self._read_as_numpy(path=path)
+        return self._read_as(path=path)
 
-            return [self._decoder(line) for line in reader]
+    def _read_as(self, path: str):
+        properties = self.properties
+        with open(path, encoding=properties[self.__ENCODING_KEY]) as csvFile:
+            if properties[self._HAS_HEADER_PROPERTY]:
+                reader_with_header = csv.DictReader(csvFile)
+                return [self._decoder(line) for line in reader_with_header]
 
-    def _read_as_numpy(self) -> np.ndarray:
-        return self._read_as_pandas_dataframe().to_numpy()
+            reader_without_header = csv.reader(csvFile)
+            return [self._decoder(line) for line in reader_without_header]
+
+    def _read_as_numpy(self, path: str) -> np.ndarray:
+        return self._read_as_pandas_dataframe(path=path).to_numpy()
 
     def _read_as_pandas_dataframe(
-        self, usecols: Optional[List[int]] = None, column_names: Optional[List[str]] = None
+        self,
+        path: str,
+        usecols: Optional[List[int]] = None,
+        column_names: Optional[List[str]] = None,
     ) -> pd.DataFrame:
         try:
-            if self.properties[self._HAS_HEADER_PROPERTY]:
+            properties = self.properties
+            if properties[self._HAS_HEADER_PROPERTY]:
                 if column_names:
-                    return pd.read_csv(self._path, encoding=self.properties[self.__ENCODING_KEY])[column_names]
-                return pd.read_csv(self._path, encoding=self.properties[self.__ENCODING_KEY])
+                    return pd.read_csv(path, encoding=properties[self.__ENCODING_KEY])[column_names]
+                return pd.read_csv(path, encoding=properties[self.__ENCODING_KEY])
             else:
                 if usecols:
-                    return pd.read_csv(
-                        self._path, encoding=self.properties[self.__ENCODING_KEY], header=None, usecols=usecols
-                    )
-                return pd.read_csv(self._path, encoding=self.properties[self.__ENCODING_KEY], header=None)
+                    return pd.read_csv(path, encoding=properties[self.__ENCODING_KEY], header=None, usecols=usecols)
+                return pd.read_csv(path, encoding=properties[self.__ENCODING_KEY], header=None)
         except pd.errors.EmptyDataError:
             return pd.DataFrame()
 
     def _append(self, data: Any):
-        if isinstance(data, pd.DataFrame):
-            data.to_csv(self._path, mode="a", index=False, encoding=self.properties[self.__ENCODING_KEY], header=False)
-        else:
-            pd.DataFrame(data).to_csv(
-                self._path, mode="a", index=False, encoding=self.properties[self.__ENCODING_KEY], header=False
-            )
-
-    def _write(self, data: Any):
-        exposed_type = self.properties[self._EXPOSED_TYPE_PROPERTY]
-        if self.properties[self._HAS_HEADER_PROPERTY]:
-            self._convert_data_to_dataframe(exposed_type, data).to_csv(
-                self._path, index=False, encoding=self.properties[self.__ENCODING_KEY]
-            )
-        else:
-            self._convert_data_to_dataframe(exposed_type, data).to_csv(
-                self._path, index=False, encoding=self.properties[self.__ENCODING_KEY], header=False
-            )
+        properties = self.properties
+        exposed_type = properties[self._EXPOSED_TYPE_PROPERTY]
+        data = self._convert_data_to_dataframe(exposed_type, data)
+        data.to_csv(self._path, mode="a", index=False, encoding=properties[self.__ENCODING_KEY], header=False)
+
+    def _write(self, data: Any, columns: Optional[List[str]] = None):
+        properties = self.properties
+        exposed_type = properties[self._EXPOSED_TYPE_PROPERTY]
+        data = self._convert_data_to_dataframe(exposed_type, data)
+
+        if columns and isinstance(data, pd.DataFrame):
+            data.columns = pd.Index(columns, dtype="object")
+
+        data.to_csv(
+            self._path,
+            index=False,
+            encoding=properties[self.__ENCODING_KEY],
+            header=properties[self._HAS_HEADER_PROPERTY],
+        )
 
     def write_with_column_names(self, data: Any, columns: Optional[List[str]] = None, job_id: Optional[JobId] = None):
         """Write a selection of columns.
@@ -199,8 +211,5 @@ class CSVDataNode(DataNode, _FileDataNodeMixin, _TabularDataNodeMixin):
             columns (Optional[List[str]]): The list of column names to write.
             job_id (JobId^): An optional identifier of the writer.
         """
-        df = self._convert_data_to_dataframe(self.properties[self._EXPOSED_TYPE_PROPERTY], data)
-        if columns and isinstance(df, pd.DataFrame):
-            df.columns = pd.Index(columns, dtype="object")
-        df.to_csv(self._path, index=False, encoding=self.properties[self.__ENCODING_KEY])
+        self._write(data, columns)
         self.track_edit(timestamp=datetime.now(), job_id=job_id)

taipy/core/data/excel.py

@@ -10,7 +10,7 @@
 # specific language governing permissions and limitations under the License.
 
 from datetime import datetime, timedelta
-from typing import Any, Dict, List, Optional, Set, Tuple, Union
+from typing import Any, Dict, List, Optional, Set, Union
 
 import numpy as np
 import pandas as pd
@@ -150,34 +150,50 @@ class ExcelDataNode(DataNode, _FileDataNodeMixin, _TabularDataNodeMixin):
                 _TabularDataNodeMixin._check_exposed_type(t)
 
     def _read(self):
-        if self.properties[self._EXPOSED_TYPE_PROPERTY] == self._EXPOSED_TYPE_PANDAS:
-            return self._read_as_pandas_dataframe()
-        if self.properties[self._EXPOSED_TYPE_PROPERTY] == self._EXPOSED_TYPE_NUMPY:
-            return self._read_as_numpy()
-        return self._read_as()
-
-    def _read_as(self):
+        return self._read_from_path()
+
+    def _read_from_path(self, path: Optional[str] = None, **read_kwargs) -> Any:
+        if path is None:
+            path = self._path
+
+        exposed_type = self.properties[self._EXPOSED_TYPE_PROPERTY]
+        if exposed_type == self._EXPOSED_TYPE_PANDAS:
+            return self._read_as_pandas_dataframe(path=path)
+        if exposed_type == self._EXPOSED_TYPE_NUMPY:
+            return self._read_as_numpy(path=path)
+        return self._read_as(path=path)
+
+    def _read_sheet_with_exposed_type(
+        self, path: str, sheet_exposed_type: str, sheet_name: str
+    ) -> Optional[Union[np.ndarray, pd.DataFrame]]:
+        if sheet_exposed_type == self._EXPOSED_TYPE_NUMPY:
+            return self._read_as_numpy(path, sheet_name)
+        elif sheet_exposed_type == self._EXPOSED_TYPE_PANDAS:
+            return self._read_as_pandas_dataframe(path, sheet_name)
+        return None
+
+    def _read_as(self, path: str):
         try:
-            excel_file = load_workbook(self._path)
-            exposed_type = self.properties[self._EXPOSED_TYPE_PROPERTY]
+            properties = self.properties
+            excel_file = load_workbook(path)
+            exposed_type = properties[self._EXPOSED_TYPE_PROPERTY]
             work_books = {}
             sheet_names = excel_file.sheetnames
 
-            user_provided_sheet_names = self.properties.get(self.__SHEET_NAME_PROPERTY) or []
-            if not isinstance(user_provided_sheet_names, (List, Set, Tuple)):
+            user_provided_sheet_names = properties.get(self.__SHEET_NAME_PROPERTY) or []
+            if not isinstance(user_provided_sheet_names, (list, set, tuple)):
                 user_provided_sheet_names = [user_provided_sheet_names]
 
             provided_sheet_names = user_provided_sheet_names or sheet_names
 
             for sheet_name in provided_sheet_names:
                 if sheet_name not in sheet_names:
-                    raise NonExistingExcelSheet(sheet_name, self._path)
+                    raise NonExistingExcelSheet(sheet_name, path)
 
             if isinstance(exposed_type, List):
-                if len(provided_sheet_names) != len(self.properties[self._EXPOSED_TYPE_PROPERTY]):
+                if len(provided_sheet_names) != len(exposed_type):
                     raise ExposedTypeLengthMismatch(
-                        f"Expected {len(provided_sheet_names)} exposed types, got "
-                        f"{len(self.properties[self._EXPOSED_TYPE_PROPERTY])}"
+                        f"Expected {len(provided_sheet_names)} exposed types, got " f"{len(exposed_type)}"
                     )
 
             for i, sheet_name in enumerate(provided_sheet_names):
@@ -191,14 +207,13 @@ class ExcelDataNode(DataNode, _FileDataNodeMixin, _TabularDataNodeMixin):
                         sheet_exposed_type = exposed_type[i]
 
                     if isinstance(sheet_exposed_type, str):
-                        if sheet_exposed_type == self._EXPOSED_TYPE_NUMPY:
-                            work_books[sheet_name] = self._read_as_pandas_dataframe(sheet_name).to_numpy()
-                        elif sheet_exposed_type == self._EXPOSED_TYPE_PANDAS:
-                            work_books[sheet_name] = self._read_as_pandas_dataframe(sheet_name)
+                        sheet_data = self._read_sheet_with_exposed_type(path, sheet_exposed_type, sheet_name)
+                        if sheet_data is not None:
+                            work_books[sheet_name] = sheet_data
                         continue
 
                 res = [[col.value for col in row] for row in work_sheet.rows]
-                if self.properties[self._HAS_HEADER_PROPERTY] and res:
+                if properties[self._HAS_HEADER_PROPERTY] and res:
                     header = res.pop(0)
                     for i, row in enumerate(res):
                         res[i] = sheet_exposed_type(**dict([[h, r] for h, r in zip(header, row)]))
@@ -214,31 +229,36 @@ class ExcelDataNode(DataNode, _FileDataNodeMixin, _TabularDataNodeMixin):
 
         return work_books
 
-    def _read_as_numpy(self):
-        sheets = self._read_as_pandas_dataframe()
+    def _read_as_numpy(self, path: str, sheet_names=None):
+        sheets = self._read_as_pandas_dataframe(path=path, sheet_names=sheet_names)
         if isinstance(sheets, dict):
             return {sheet_name: df.to_numpy() for sheet_name, df in sheets.items()}
         return sheets.to_numpy()
 
-    def _do_read_excel(self, sheet_names, kwargs) -> Union[Dict[Union[int, str], pd.DataFrame], pd.DataFrame]:
-        return pd.read_excel(self._path, sheet_name=sheet_names, **kwargs)
+    def _do_read_excel(
+        self, path: str, sheet_names, kwargs
+    ) -> Union[Dict[Union[int, str], pd.DataFrame], pd.DataFrame]:
+        return pd.read_excel(path, sheet_name=sheet_names, **kwargs)
 
     def __get_sheet_names_and_header(self, sheet_names):
         kwargs = {}
+        properties = self.properties
         if sheet_names is None:
-            sheet_names = self.properties[self.__SHEET_NAME_PROPERTY]
-        if not self.properties[self._HAS_HEADER_PROPERTY]:
+            sheet_names = properties[self.__SHEET_NAME_PROPERTY]
+        if not properties[self._HAS_HEADER_PROPERTY]:
             kwargs["header"] = None
         return sheet_names, kwargs
 
-    def _read_as_pandas_dataframe(self, sheet_names=None) -> Union[Dict[Union[int, str], pd.DataFrame], pd.DataFrame]:
+    def _read_as_pandas_dataframe(
+        self, path: str, sheet_names=None
+    ) -> Union[Dict[Union[int, str], pd.DataFrame], pd.DataFrame]:
         sheet_names, kwargs = self.__get_sheet_names_and_header(sheet_names)
         try:
-            return self._do_read_excel(sheet_names, kwargs)
+            return self._do_read_excel(path, sheet_names, kwargs)
         except pd.errors.EmptyDataError:
             return pd.DataFrame()
 
-    def __append_excel_with_single_sheet(self, append_excel_fct, *args, **kwargs):
+    def _append_excel_with_single_sheet(self, append_excel_fct, *args, **kwargs):
         sheet_name = self.properties.get(self.__SHEET_NAME_PROPERTY)
 
         with pd.ExcelWriter(self._path, mode="a", engine="openpyxl", if_sheet_exists="overlay") as writer:
@@ -252,7 +272,12 @@ class ExcelDataNode(DataNode, _FileDataNodeMixin, _TabularDataNodeMixin):
                 sheet_name = list(writer.sheets.keys())[0]
                 append_excel_fct(writer, *args, **kwargs, startrow=writer.sheets[sheet_name].max_row)
 
-    def __append_excel_with_multiple_sheets(self, data: Any, columns: List[str] = None):
+    def _set_column_if_dataframe(self, data: Any, columns) -> Union[pd.DataFrame, Any]:
+        if isinstance(data, pd.DataFrame):
+            data.columns = pd.Index(columns, dtype="object")
+        return data
+
+    def _append_excel_with_multiple_sheets(self, data: Any, columns: List[str] = None):
         with pd.ExcelWriter(self._path, mode="a", engine="openpyxl", if_sheet_exists="overlay") as writer:
             # Each key stands for a sheet name
             for sheet_name in data.keys():
@@ -262,7 +287,7 @@ class ExcelDataNode(DataNode, _FileDataNodeMixin, _TabularDataNodeMixin):
                     df = data[sheet_name]
 
                 if columns:
-                    data[sheet_name].columns = columns
+                    df = self._set_column_if_dataframe(df, columns)
 
                 df.to_excel(
                     writer, sheet_name=sheet_name, index=False, header=False, startrow=writer.sheets[sheet_name].max_row
@@ -275,13 +300,13 @@ class ExcelDataNode(DataNode, _FileDataNodeMixin, _TabularDataNodeMixin):
             raise ImportError("The append method is only available for pandas version 1.4 or higher.")
 
         if isinstance(data, Dict) and all(isinstance(x, (pd.DataFrame, np.ndarray)) for x in data.values()):
-            self.__append_excel_with_multiple_sheets(data)
+            self._append_excel_with_multiple_sheets(data)
         elif isinstance(data, pd.DataFrame):
-            self.__append_excel_with_single_sheet(data.to_excel, index=False, header=False)
+            self._append_excel_with_single_sheet(data.to_excel, index=False, header=False)
         else:
-            self.__append_excel_with_single_sheet(pd.DataFrame(data).to_excel, index=False, header=False)
+            self._append_excel_with_single_sheet(pd.DataFrame(data).to_excel, index=False, header=False)
 
-    def __write_excel_with_single_sheet(self, write_excel_fct, *args, **kwargs):
+    def _write_excel_with_single_sheet(self, write_excel_fct, *args, **kwargs):
         if sheet_name := self.properties.get(self.__SHEET_NAME_PROPERTY):
             if not isinstance(sheet_name, str):
                 if len(sheet_name) > 1:
@@ -292,24 +317,26 @@ class ExcelDataNode(DataNode, _FileDataNodeMixin, _TabularDataNodeMixin):
         else:
             write_excel_fct(*args, **kwargs)
 
-    def __write_excel_with_multiple_sheets(self, data: Any, columns: List[str] = None):
+    def _write_excel_with_multiple_sheets(self, data: Any, columns: List[str] = None):
         with pd.ExcelWriter(self._path) as writer:
             # Each key stands for a sheet name
+            properties = self.properties
             for key in data.keys():
-                df = self._convert_data_to_dataframe(self.properties[self._EXPOSED_TYPE_PROPERTY], data[key])
+                df = self._convert_data_to_dataframe(properties[self._EXPOSED_TYPE_PROPERTY], data[key])
 
                 if columns:
-                    data[key].columns = columns
+                    df = self._set_column_if_dataframe(df, columns)
 
-                df.to_excel(writer, key, index=False, header=self.properties[self._HAS_HEADER_PROPERTY] or False)
+                df.to_excel(writer, key, index=False, header=properties[self._HAS_HEADER_PROPERTY] or False)
 
     def _write(self, data: Any):
         if isinstance(data, Dict):
-            return self.__write_excel_with_multiple_sheets(data)
+            return self._write_excel_with_multiple_sheets(data)
         else:
-            data = self._convert_data_to_dataframe(self.properties[self._EXPOSED_TYPE_PROPERTY], data)
-            self.__write_excel_with_single_sheet(
-                data.to_excel, self._path, index=False, header=self.properties[self._HAS_HEADER_PROPERTY] or None
+            properties = self.properties
+            data = self._convert_data_to_dataframe(properties[self._EXPOSED_TYPE_PROPERTY], data)
+            self._write_excel_with_single_sheet(
+                data.to_excel, self._path, index=False, header=properties[self._HAS_HEADER_PROPERTY] or None
             )
 
     def write_with_column_names(self, data: Any, columns: List[str] = None, job_id: Optional[JobId] = None):
@@ -321,10 +348,10 @@ class ExcelDataNode(DataNode, _FileDataNodeMixin, _TabularDataNodeMixin):
             job_id (JobId^): An optional identifier of the writer.
         """
         if isinstance(data, Dict) and all(isinstance(x, (pd.DataFrame, np.ndarray)) for x in data.values()):
-            self.__write_excel_with_multiple_sheets(data, columns=columns)
+            self._write_excel_with_multiple_sheets(data, columns=columns)
         else:
             df = pd.DataFrame(data)
             if columns:
-                df.columns = pd.Index(columns, dtype="object")
-            self.__write_excel_with_single_sheet(df.to_excel, self.path, index=False)
+                df = self._set_column_if_dataframe(df, columns)
+            self._write_excel_with_single_sheet(df.to_excel, self.path, index=False)
         self.track_edit(timestamp=datetime.now(), job_id=job_id)

taipy/core/data/generic.py

@@ -108,7 +108,7 @@ class GenericDataNode(DataNode):
             editor_expiration_date,
             **properties,
         )
-        if not self._last_edit_date:
+        if not self._last_edit_date:  # type: ignore
             self._last_edit_date = datetime.now()
 
         self._TAIPY_PROPERTIES.update(
@@ -125,8 +125,9 @@ class GenericDataNode(DataNode):
         return cls.__STORAGE_TYPE
 
     def _read(self):
-        if read_fct := self.properties[self._OPTIONAL_READ_FUNCTION_PROPERTY]:
-            if read_fct_args := self.properties.get(self.__READ_FUNCTION_ARGS_PROPERTY, None):
+        properties = self.properties
+        if read_fct := properties[self._OPTIONAL_READ_FUNCTION_PROPERTY]:
+            if read_fct_args := properties.get(self.__READ_FUNCTION_ARGS_PROPERTY, None):
                 if not isinstance(read_fct_args, list):
                     return read_fct(*[read_fct_args])
                 return read_fct(*read_fct_args)
@@ -134,8 +135,9 @@ class GenericDataNode(DataNode):
         raise MissingReadFunction(f"The read function is not defined in data node config {self.config_id}.")
 
     def _write(self, data: Any):
-        if write_fct := self.properties[self._OPTIONAL_WRITE_FUNCTION_PROPERTY]:
-            if write_fct_args := self.properties.get(self.__WRITE_FUNCTION_ARGS_PROPERTY, None):
+        properties = self.properties
+        if write_fct := properties[self._OPTIONAL_WRITE_FUNCTION_PROPERTY]:
+            if write_fct_args := properties.get(self.__WRITE_FUNCTION_ARGS_PROPERTY, None):
                 if not isinstance(write_fct_args, list):
                     return write_fct(data, *[write_fct_args])
                 return write_fct(data, *write_fct_args)

taipy/core/data/json.py

@@ -150,7 +150,13 @@ class JSONDataNode(DataNode, _FileDataNodeMixin):
         self.properties[self._DECODER_KEY] = decoder
 
     def _read(self):
-        with open(self._path, "r", encoding=self.properties[self.__ENCODING_KEY]) as f:
+        return self._read_from_path()
+
+    def _read_from_path(self, path: Optional[str] = None, **read_kwargs) -> Any:
+        if path is None:
+            path = self._path
+
+        with open(path, "r", encoding=self.properties[self.__ENCODING_KEY]) as f:
             return json.load(f, cls=self._decoder)
 
     def _append(self, data: Any):

taipy/core/data/parquet.py

@@ -157,7 +157,10 @@ class ParquetDataNode(DataNode, _FileDataNodeMixin, _TabularDataNodeMixin):
         with _Reloader():
             self._write_default_data(default_value)
 
-        if not self._last_edit_date and (isfile(self._path) or isdir(self._path)):
+        if (
+            not self._last_edit_date  # type: ignore
+            and (isfile(self._path) or isdir(self._path[:-1] if self._path.endswith("*") else self._path))
+        ):
             self._last_edit_date = datetime.now()
         self._TAIPY_PROPERTIES.update(
             {
@@ -178,18 +181,43 @@ class ParquetDataNode(DataNode, _FileDataNodeMixin, _TabularDataNodeMixin):
         return cls.__STORAGE_TYPE
 
     def _read(self):
-        return self.read_with_kwargs()
+        return self._read_from_path()
+
+    def _read_from_path(self, path: Optional[str] = None, **read_kwargs) -> Any:
+        if path is None:
+            path = self._path
+
+        # return None if data was never written
+        if not self.last_edit_date:
+            self._DataNode__logger.warning(
+                f"Data node {self.id} from config {self.config_id} is being read but has never been written."
+            )
+            return None
+
+        kwargs = self.properties[self.__READ_KWARGS_PROPERTY]
+        kwargs.update(
+            {
+                self.__ENGINE_PROPERTY: self.properties[self.__ENGINE_PROPERTY],
+            }
+        )
+        kwargs.update(read_kwargs)
+
+        if self.properties[self._EXPOSED_TYPE_PROPERTY] == self._EXPOSED_TYPE_PANDAS:
+            return self._read_as_pandas_dataframe(path, kwargs)
+        if self.properties[self._EXPOSED_TYPE_PROPERTY] == self._EXPOSED_TYPE_NUMPY:
+            return self._read_as_numpy(path, kwargs)
+        return self._read_as(path, kwargs)
 
-    def _read_as(self, read_kwargs: Dict):
+    def _read_as(self, path: str, read_kwargs: Dict):
         custom_class = self.properties[self._EXPOSED_TYPE_PROPERTY]
-        list_of_dicts = self._read_as_pandas_dataframe(read_kwargs).to_dict(orient="records")
+        list_of_dicts = self._read_as_pandas_dataframe(path, read_kwargs).to_dict(orient="records")
         return [custom_class(**dct) for dct in list_of_dicts]
 
-    def _read_as_numpy(self, read_kwargs: Dict) -> np.ndarray:
-        return self._read_as_pandas_dataframe(read_kwargs).to_numpy()
+    def _read_as_numpy(self, path: str, read_kwargs: Dict) -> np.ndarray:
+        return self._read_as_pandas_dataframe(path, read_kwargs).to_numpy()
 
-    def _read_as_pandas_dataframe(self, read_kwargs: Dict) -> pd.DataFrame:
-        return pd.read_parquet(self._path, **read_kwargs)
+    def _read_as_pandas_dataframe(self, path: str, read_kwargs: Dict) -> pd.DataFrame:
+        return pd.read_parquet(path, **read_kwargs)
 
     def _append(self, data: Any):
         self.write_with_kwargs(data, engine="fastparquet", append=True)
@@ -208,14 +236,15 @@ class ParquetDataNode(DataNode, _FileDataNodeMixin, _TabularDataNodeMixin):
             **write_kwargs (dict[str, any]): The keyword arguments passed to the function
                 `pandas.DataFrame.to_parquet()`.
         """
+        properties = self.properties
         kwargs = {
-            self.__ENGINE_PROPERTY: self.properties[self.__ENGINE_PROPERTY],
-            self.__COMPRESSION_PROPERTY: self.properties[self.__COMPRESSION_PROPERTY],
+            self.__ENGINE_PROPERTY: properties[self.__ENGINE_PROPERTY],
+            self.__COMPRESSION_PROPERTY: properties[self.__COMPRESSION_PROPERTY],
         }
-        kwargs.update(self.properties[self.__WRITE_KWARGS_PROPERTY])
+        kwargs.update(properties[self.__WRITE_KWARGS_PROPERTY])
         kwargs.update(write_kwargs)
 
-        df = self._convert_data_to_dataframe(self.properties[self._EXPOSED_TYPE_PROPERTY], data)
+        df = self._convert_data_to_dataframe(properties[self._EXPOSED_TYPE_PROPERTY], data)
         if isinstance(df, pd.Series):
             df = pd.DataFrame(df)
 
@@ -233,23 +262,4 @@ class ParquetDataNode(DataNode, _FileDataNodeMixin, _TabularDataNodeMixin):
             **read_kwargs (dict[str, any]): The keyword arguments passed to the function
                 `pandas.read_parquet()`.
         """
-        # return None if data was never written
-        if not self.last_edit_date:
-            self._DataNode__logger.warning(
-                f"Data node {self.id} from config {self.config_id} is being read but has never been written."
-            )
-            return None
-
-        kwargs = self.properties[self.__READ_KWARGS_PROPERTY]
-        kwargs.update(
-            {
-                self.__ENGINE_PROPERTY: self.properties[self.__ENGINE_PROPERTY],
-            }
-        )
-        kwargs.update(read_kwargs)
-
-        if self.properties[self._EXPOSED_TYPE_PROPERTY] == self._EXPOSED_TYPE_PANDAS:
-            return self._read_as_pandas_dataframe(kwargs)
-        if self.properties[self._EXPOSED_TYPE_PROPERTY] == self._EXPOSED_TYPE_NUMPY:
-            return self._read_as_numpy(kwargs)
-        return self._read_as(kwargs)
+        return self._read_from_path(**read_kwargs)

taipy/core/data/pickle.py

@@ -11,7 +11,7 @@
 
 import pickle
 from datetime import datetime, timedelta
-from typing import List, Optional, Set
+from typing import Any, List, Optional, Set
 
 from taipy.config.common.scope import Scope
 
@@ -116,7 +116,13 @@ class PickleDataNode(DataNode, _FileDataNodeMixin):
         return cls.__STORAGE_TYPE
 
     def _read(self):
-        with open(self._path, "rb") as pf:
+        return self._read_from_path()
+
+    def _read_from_path(self, path: Optional[str] = None, **read_kwargs) -> Any:
+        if path is None:
+            path = self._path
+
+        with open(path, "rb") as pf:
             return pickle.load(pf)
 
     def _write(self, data):

taipy/core/data/sql.py

@@ -131,10 +131,11 @@ class SQLDataNode(_AbstractSQLDataNode):
         return self.properties.get(self.__READ_QUERY_KEY)
 
     def _do_append(self, data, engine, connection) -> None:
-        if not self.properties.get(self._APPEND_QUERY_BUILDER_KEY):
+        append_query_builder_fct = self.properties.get(self._APPEND_QUERY_BUILDER_KEY)
+        if not append_query_builder_fct:
             raise MissingAppendQueryBuilder
 
-        queries = self.properties.get(self._APPEND_QUERY_BUILDER_KEY)(data)
+        queries = append_query_builder_fct(data)
         self.__execute_queries(queries, connection)
 
     def _do_write(self, data, engine, connection) -> None:

taipy/core/data/sql_table.py

@@ -122,7 +122,7 @@ class SQLTableDataNode(_AbstractSQLDataNode):
 
     def __insert_data(self, data, engine, connection, delete_table: bool = False) -> None:
         table = self._create_table(engine)
-        self.__insert_dataframe(
+        self._insert_dataframe(
             self._convert_data_to_dataframe(self.properties[self._EXPOSED_TYPE_PROPERTY], data),
             table,
             connection,
@@ -137,7 +137,7 @@ class SQLTableDataNode(_AbstractSQLDataNode):
         )
 
     @classmethod
-    def __insert_dicts(cls, data: List[Dict], table: Any, connection: Any, delete_table: bool) -> None:
+    def _insert_dicts(cls, data: List[Dict], table: Any, connection: Any, delete_table: bool) -> None:
         """
         This method will insert the data contained in a list of dictionaries into a table. The query itself is handled
         by SQLAlchemy, so it's only needed to pass the correct data type.
@@ -146,14 +146,14 @@ class SQLTableDataNode(_AbstractSQLDataNode):
         connection.execute(table.insert(), data)
 
     @classmethod
-    def __insert_dataframe(
+    def _insert_dataframe(
         cls, df: Union[pd.DataFrame, pd.Series], table: Any, connection: Any, delete_table: bool
-    ) -> None:
+        ) -> None:
         if isinstance(df, pd.Series):
             data = [df.to_dict()]
         elif isinstance(df, pd.DataFrame):
             data = df.to_dict(orient="records")
-        cls.__insert_dicts(data, table, connection, delete_table)
+        cls._insert_dicts(data, table, connection, delete_table)
 
     @classmethod
     def __delete_all_rows(cls, table: Any, connection: Any, delete_table: bool) -> None:

taipy/core/reason/__init__.py

@@ -13,8 +13,10 @@ from .reason import (
     DataNodeEditInProgress,
     DataNodeIsNotWritten,
     EntityIsNotSubmittableEntity,
+    InvalidUploadFile,
     NotGlobalScope,
     Reason,
+    UploadFileCanNotBeRead,
     WrongConfigType,
 )
 from .reason_collection import ReasonCollection

taipy/core/reason/reason.py

@@ -123,3 +123,35 @@ class NotGlobalScope(Reason):
 
     def __init__(self, config_id: str):
         Reason.__init__(self, f'Data node config "{config_id}" does not have GLOBAL scope')
+
+
+class UploadFileCanNotBeRead(Reason, _DataNodeReasonMixin):
+    """
+    The uploaded file can not be read, therefore is not a valid data file for the data node.
+
+    Attributes:
+        file_name (str): The name of the file that was uploaded.
+        datanode_id (str): The datanode id that the file is intended to upload to.
+    """
+
+    def __init__(self, file_name: str, datanode_id: str):
+        Reason.__init__(
+            self,
+            f"The uploaded file {file_name} can not be read, "
+            f'therefore is not a valid data file for data node "{datanode_id}"',
+        )
+        _DataNodeReasonMixin.__init__(self, datanode_id)
+
+
+class InvalidUploadFile(Reason, _DataNodeReasonMixin):
+    """
+    The uploaded file has invalid data, therefore is not a valid data file for the data node.
+
+    Attributes:
+        file_name (str): The name of the file that was uploaded.
+        datanode_id (str): The datanode id that the file is intended to upload to.
+    """
+
+    def __init__(self, file_name: str, datanode_id: str):
+        Reason.__init__(self, f'The uploaded file {file_name} has invalid data for data node "{datanode_id}"')
+        _DataNodeReasonMixin.__init__(self, datanode_id)

tests/core/data/test_aws_s3_data_node.py

@@ -13,7 +13,9 @@ import boto3
 import pytest
 from moto import mock_s3
 
+from taipy.config import Config
 from taipy.config.common.scope import Scope
+from taipy.core.data._data_manager_factory import _DataManagerFactory
 from taipy.core.data.aws_s3 import S3ObjectDataNode
 
 
@@ -29,14 +31,10 @@ class TestS3ObjectDataNode:
         }
     ]
 
-    @mock_s3
     @pytest.mark.parametrize("properties", __properties)
     def test_create(self, properties):
-        aws_s3_object_dn = S3ObjectDataNode(
-            "foo_bar_aws_s3",
-            Scope.SCENARIO,
-            properties=properties,
-        )
+        s3_object_dn_config = Config.configure_s3_object_data_node(id="foo_bar_aws_s3", **properties)
+        aws_s3_object_dn = _DataManagerFactory._build_manager()._create_and_set(s3_object_dn_config, None, None)
         assert isinstance(aws_s3_object_dn, S3ObjectDataNode)
         assert aws_s3_object_dn.storage_type() == "s3_object"
         assert aws_s3_object_dn.config_id == "foo_bar_aws_s3"

tests/core/data/test_csv_data_node.py

@@ -9,19 +9,24 @@
 # an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
 # specific language governing permissions and limitations under the License.
 
+import dataclasses
 import os
 import pathlib
 import uuid
-from datetime import datetime
+from datetime import datetime, timedelta
 from time import sleep
 
+import freezegun
+import numpy as np
 import pandas as pd
 import pytest
+from pandas.testing import assert_frame_equal
 
 from taipy.config.common.scope import Scope
 from taipy.config.config import Config
 from taipy.config.exceptions.exceptions import InvalidConfigurationId
 from taipy.core.data._data_manager import _DataManager
+from taipy.core.data._data_manager_factory import _DataManagerFactory
 from taipy.core.data.csv import CSVDataNode
 from taipy.core.data.data_node_id import DataNodeId
 from taipy.core.exceptions.exceptions import InvalidExposedType
@@ -35,12 +40,20 @@ def cleanup():
         os.remove(path)
 
 
+@dataclasses.dataclass
+class MyCustomObject:
+    id: int
+    integer: int
+    text: str
+
+
 class TestCSVDataNode:
     def test_create(self):
-        path = "data/node/path"
-        dn = CSVDataNode(
-            "foo_bar", Scope.SCENARIO, properties={"path": path, "has_header": False, "name": "super name"}
+        default_path = "data/node/path"
+        csv_dn_config = Config.configure_csv_data_node(
+            id="foo_bar", default_path=default_path, has_header=False, name="super name"
         )
+        dn = _DataManagerFactory._build_manager()._create_and_set(csv_dn_config, None, None)
         assert isinstance(dn, CSVDataNode)
         assert dn.storage_type() == "csv"
         assert dn.config_id == "foo_bar"
@@ -51,12 +64,23 @@ class TestCSVDataNode:
         assert dn.last_edit_date is None
         assert dn.job_ids == []
         assert not dn.is_ready_for_reading
-        assert dn.path == path
+        assert dn.path == default_path
         assert dn.has_header is False
         assert dn.exposed_type == "pandas"
 
+        csv_dn_config = Config.configure_csv_data_node(
+            id="foo", default_path=default_path, has_header=True, exposed_type=MyCustomObject
+        )
+        dn = _DataManagerFactory._build_manager()._create_and_set(csv_dn_config, None, None)
+        assert dn.storage_type() == "csv"
+        assert dn.config_id == "foo"
+        assert dn.has_header is True
+        assert dn.exposed_type == MyCustomObject
+
         with pytest.raises(InvalidConfigurationId):
-            CSVDataNode("foo bar", Scope.SCENARIO, properties={"path": path, "has_header": False, "name": "super name"})
+            CSVDataNode(
+                "foo bar", Scope.SCENARIO, properties={"path": default_path, "has_header": False, "name": "super name"}
+            )
 
     def test_modin_deprecated_in_favor_of_pandas(self):
         path = os.path.join(pathlib.Path(__file__).parent.resolve(), "data_sample/example.csv")
@@ -169,3 +193,125 @@ class TestCSVDataNode:
 
         assert ".data" not in dn.path
         assert os.path.exists(dn.path)
+
+    def test_get_downloadable_path(self):
+        path = os.path.join(pathlib.Path(__file__).parent.resolve(), "data_sample/example.csv")
+        dn = CSVDataNode("foo", Scope.SCENARIO, properties={"path": path, "exposed_type": "pandas"})
+        assert dn._get_downloadable_path() == path
+
+    def test_get_downloadable_path_with_not_existing_file(self):
+        dn = CSVDataNode("foo", Scope.SCENARIO, properties={"path": "NOT_EXISTING.csv", "exposed_type": "pandas"})
+        assert dn._get_downloadable_path() == ""
+
+    def test_upload(self, csv_file, tmpdir_factory):
+        old_csv_path = tmpdir_factory.mktemp("data").join("df.csv").strpath
+        old_data = pd.DataFrame([{"a": 0, "b": 1, "c": 2}, {"a": 3, "b": 4, "c": 5}])
+
+        dn = CSVDataNode("foo", Scope.SCENARIO, properties={"path": old_csv_path, "exposed_type": "pandas"})
+        dn.write(old_data)
+        old_last_edit_date = dn.last_edit_date
+
+        upload_content = pd.read_csv(csv_file)
+
+        with freezegun.freeze_time(old_last_edit_date + timedelta(seconds=1)):
+            dn._upload(csv_file)
+
+        assert_frame_equal(dn.read(), upload_content)  # The content of the dn should change to the uploaded content
+        assert dn.last_edit_date > old_last_edit_date
+        assert dn.path == old_csv_path  # The path of the dn should not change
+
+    def test_upload_with_upload_check_pandas(self, csv_file, tmpdir_factory):
+        old_csv_path = tmpdir_factory.mktemp("data").join("df.csv").strpath
+        old_data = pd.DataFrame([{"a": 0, "b": 1, "c": 2}, {"a": 3, "b": 4, "c": 5}])
+
+        dn = CSVDataNode("foo", Scope.SCENARIO, properties={"path": old_csv_path, "exposed_type": "pandas"})
+        dn.write(old_data)
+        old_last_edit_date = dn.last_edit_date
+
+        def check_data_column(upload_path, upload_data):
+            return upload_path.endswith(".csv") and upload_data.columns.tolist() == ["a", "b", "c"]
+
+        not_exists_csv_path = tmpdir_factory.mktemp("data").join("not_exists.csv").strpath
+        reasons = dn._upload(not_exists_csv_path, upload_checker=check_data_column)
+        assert bool(reasons) is False
+        assert (
+            str(list(reasons._reasons[dn.id])[0]) == "The uploaded file not_exists.csv can not be read,"
+            f' therefore is not a valid data file for data node "{dn.id}"'
+        )
+
+        not_csv_path = tmpdir_factory.mktemp("data").join("wrong_format_df.not_csv").strpath
+        old_data.to_csv(not_csv_path, index=False)
+        # The upload should fail when the file is not a csv
+        reasons = dn._upload(not_csv_path, upload_checker=check_data_column)
+        assert bool(reasons) is False
+        assert (
+            str(list(reasons._reasons[dn.id])[0])
+            == f'The uploaded file wrong_format_df.not_csv has invalid data for data node "{dn.id}"'
+        )
+
+        wrong_format_csv_path = tmpdir_factory.mktemp("data").join("wrong_format_df.csv").strpath
+        pd.DataFrame([{"a": 1, "b": 2, "d": 3}, {"a": 4, "b": 5, "d": 6}]).to_csv(wrong_format_csv_path, index=False)
+        # The upload should fail when check_data_column() return False
+        reasons = dn._upload(wrong_format_csv_path, upload_checker=check_data_column)
+        assert bool(reasons) is False
+        assert (
+            str(list(reasons._reasons[dn.id])[0])
+            == f'The uploaded file wrong_format_df.csv has invalid data for data node "{dn.id}"'
+        )
+
+        assert_frame_equal(dn.read(), old_data)  # The content of the dn should not change when upload fails
+        assert dn.last_edit_date == old_last_edit_date  # The last edit date should not change when upload fails
+        assert dn.path == old_csv_path  # The path of the dn should not change
+
+        # The upload should succeed when check_data_column() return True
+        assert dn._upload(csv_file, upload_checker=check_data_column)
+
+    def test_upload_with_upload_check_numpy(self, tmpdir_factory):
+        old_csv_path = tmpdir_factory.mktemp("data").join("df.csv").strpath
+        old_data = np.array([[1, 2, 3], [4, 5, 6]])
+
+        new_csv_path = tmpdir_factory.mktemp("data").join("new_upload_data.csv").strpath
+        new_data = np.array([[1, 2, 3], [4, 5, 6]])
+        pd.DataFrame(new_data).to_csv(new_csv_path, index=False)
+
+        dn = CSVDataNode("foo", Scope.SCENARIO, properties={"path": old_csv_path, "exposed_type": "numpy"})
+        dn.write(old_data)
+        old_last_edit_date = dn.last_edit_date
+
+        def check_data_is_positive(upload_path, upload_data):
+            return upload_path.endswith(".csv") and np.all(upload_data > 0)
+
+        not_exists_csv_path = tmpdir_factory.mktemp("data").join("not_exists.csv").strpath
+        reasons = dn._upload(not_exists_csv_path, upload_checker=check_data_is_positive)
+        assert bool(reasons) is False
+        assert (
+            str(list(reasons._reasons[dn.id])[0]) == "The uploaded file not_exists.csv can not be read"
+            f', therefore is not a valid data file for data node "{dn.id}"'
+        )
+
+        not_csv_path = tmpdir_factory.mktemp("data").join("wrong_format_df.not_csv").strpath
+        pd.DataFrame(old_data).to_csv(not_csv_path, index=False)
+        # The upload should fail when the file is not a csv
+        reasons = dn._upload(not_csv_path, upload_checker=check_data_is_positive)
+        assert bool(reasons) is False
+        assert (
+            str(list(reasons._reasons[dn.id])[0])
+            == f'The uploaded file wrong_format_df.not_csv has invalid data for data node "{dn.id}"'
+        )
+
+        wrong_format_csv_path = tmpdir_factory.mktemp("data").join("wrong_format_df.csv").strpath
+        pd.DataFrame(np.array([[-1, 2, 3], [-4, -5, -6]])).to_csv(wrong_format_csv_path, index=False)
+        # The upload should fail when check_data_is_positive() return False
+        reasons = dn._upload(wrong_format_csv_path, upload_checker=check_data_is_positive)
+        assert bool(reasons) is False
+        assert (
+            str(list(reasons._reasons[dn.id])[0])
+            == f'The uploaded file wrong_format_df.csv has invalid data for data node "{dn.id}"'
+        )
+
+        np.array_equal(dn.read(), old_data)  # The content of the dn should not change when upload fails
+        assert dn.last_edit_date == old_last_edit_date  # The last edit date should not change when upload fails
+        assert dn.path == old_csv_path  # The path of the dn should not change
+
+        # The upload should succeed when check_data_is_positive() return True
+        assert dn._upload(new_csv_path, upload_checker=check_data_is_positive)

tests/core/data/test_excel_data_node.py

@@ -12,17 +12,20 @@
 import os
 import pathlib
 import uuid
-from datetime import datetime
+from datetime import datetime, timedelta
 from time import sleep
 from typing import Dict
 
+import freezegun
 import numpy as np
 import pandas as pd
 import pytest
+from pandas.testing import assert_frame_equal
 
 from taipy.config.common.scope import Scope
 from taipy.config.config import Config
 from taipy.core.data._data_manager import _DataManager
+from taipy.core.data._data_manager_factory import _DataManagerFactory
 from taipy.core.data.data_node_id import DataNodeId
 from taipy.core.data.excel import ExcelDataNode
 from taipy.core.exceptions.exceptions import (
@@ -75,11 +78,10 @@ class TestExcelDataNode:
     def test_create(self):
         path = "data/node/path"
         sheet_names = ["sheet_name_1", "sheet_name_2"]
-        dn = ExcelDataNode(
-            "foo_bar",
-            Scope.SCENARIO,
-            properties={"path": path, "has_header": False, "sheet_name": sheet_names, "name": "super name"},
+        excel_dn_config = Config.configure_excel_data_node(
+            id="foo_bar", default_path=path, has_header=False, sheet_name="Sheet1", name="super name"
         )
+        dn = _DataManagerFactory._build_manager()._create_and_set(excel_dn_config, None, None)
         assert isinstance(dn, ExcelDataNode)
         assert dn.storage_type() == "excel"
         assert dn.config_id == "foo_bar"
@@ -93,7 +95,48 @@ class TestExcelDataNode:
         assert not dn.is_ready_for_reading
         assert dn.path == path
         assert dn.has_header is False
-        assert dn.sheet_name == sheet_names
+        assert dn.sheet_name == "Sheet1"
+
+        excel_dn_config_1 = Config.configure_excel_data_node(
+            id="baz", default_path=path, has_header=True, sheet_name="Sheet1", exposed_type=MyCustomObject
+        )
+        dn_1 = _DataManagerFactory._build_manager()._create_and_set(excel_dn_config_1, None, None)
+        assert isinstance(dn_1, ExcelDataNode)
+        assert dn_1.has_header is True
+        assert dn_1.sheet_name == "Sheet1"
+        assert dn_1.exposed_type == MyCustomObject
+
+        excel_dn_config_2 = Config.configure_excel_data_node(
+            id="baz",
+            default_path=path,
+            has_header=True,
+            sheet_name=sheet_names,
+            exposed_type={"Sheet1": "pandas", "Sheet2": "numpy"},
+        )
+        dn_2 = _DataManagerFactory._build_manager()._create_and_set(excel_dn_config_2, None, None)
+        assert isinstance(dn_2, ExcelDataNode)
+        assert dn_2.sheet_name == sheet_names
+        assert dn_2.exposed_type == {"Sheet1": "pandas", "Sheet2": "numpy"}
+
+        excel_dn_config_3 = Config.configure_excel_data_node(
+            id="baz", default_path=path, has_header=True, sheet_name=sheet_names, exposed_type=MyCustomObject
+        )
+        dn_3 = _DataManagerFactory._build_manager()._create_and_set(excel_dn_config_3, None, None)
+        assert isinstance(dn_3, ExcelDataNode)
+        assert dn_3.sheet_name == sheet_names
+        assert dn_3.exposed_type == MyCustomObject
+
+        excel_dn_config_4 = Config.configure_excel_data_node(
+            id="baz",
+            default_path=path,
+            has_header=True,
+            sheet_name=sheet_names,
+            exposed_type={"Sheet1": MyCustomObject, "Sheet2": MyCustomObject2},
+        )
+        dn_4 = _DataManagerFactory._build_manager()._create_and_set(excel_dn_config_4, None, None)
+        assert isinstance(dn_4, ExcelDataNode)
+        assert dn_4.sheet_name == sheet_names
+        assert dn_4.exposed_type == {"Sheet1": MyCustomObject, "Sheet2": MyCustomObject2}
 
     def test_get_user_properties(self, excel_file):
         dn_1 = ExcelDataNode("dn_1", Scope.SCENARIO, properties={"path": "data/node/path"})
@@ -365,3 +408,127 @@ class TestExcelDataNode:
 
         assert ".data" not in dn.path
         assert os.path.exists(dn.path)
+
+    def test_get_download_path(self):
+        path = os.path.join(pathlib.Path(__file__).parent.resolve(), "data_sample/example.xlsx")
+        dn = ExcelDataNode("foo", Scope.SCENARIO, properties={"path": path, "exposed_type": "pandas"})
+        assert dn._get_downloadable_path() == path
+
+    def test_get_downloadable_path_with_not_existing_file(self):
+        dn = ExcelDataNode("foo", Scope.SCENARIO, properties={"path": "NOT_EXISTING.xlsx", "exposed_type": "pandas"})
+        assert dn._get_downloadable_path() == ""
+
+    def test_upload(self, excel_file, tmpdir_factory):
+        old_xlsx_path = tmpdir_factory.mktemp("data").join("df.xlsx").strpath
+        old_data = pd.DataFrame([{"a": 0, "b": 1, "c": 2}, {"a": 3, "b": 4, "c": 5}])
+
+        dn = ExcelDataNode("foo", Scope.SCENARIO, properties={"path": old_xlsx_path, "exposed_type": "pandas"})
+        dn.write(old_data)
+        old_last_edit_date = dn.last_edit_date
+
+        upload_content = pd.read_excel(excel_file)
+
+        with freezegun.freeze_time(old_last_edit_date + timedelta(seconds=1)):
+            dn._upload(excel_file)
+
+        assert_frame_equal(dn.read()["Sheet1"], upload_content)  # The data of dn should change to the uploaded content
+        assert dn.last_edit_date > old_last_edit_date
+        assert dn.path == old_xlsx_path  # The path of the dn should not change
+
+    def test_upload_with_upload_check_pandas(self, excel_file, tmpdir_factory):
+        old_xlsx_path = tmpdir_factory.mktemp("data").join("df.xlsx").strpath
+        old_data = pd.DataFrame([{"a": 0, "b": 1, "c": 2}, {"a": 3, "b": 4, "c": 5}])
+
+        dn = ExcelDataNode("foo", Scope.SCENARIO, properties={"path": old_xlsx_path, "exposed_type": "pandas"})
+        dn.write(old_data)
+        old_last_edit_date = dn.last_edit_date
+
+        def check_data_column(upload_path, upload_data):
+            """Check if the uploaded data has the correct file format and
+            the sheet named "Sheet1" has the correct columns.
+            """
+            return upload_path.endswith(".xlsx") and upload_data["Sheet1"].columns.tolist() == ["a", "b", "c"]
+
+        not_exists_xlsx_path = tmpdir_factory.mktemp("data").join("not_exists.xlsx").strpath
+        reasons = dn._upload(not_exists_xlsx_path, upload_checker=check_data_column)
+        assert bool(reasons) is False
+        assert (
+            str(list(reasons._reasons[dn.id])[0]) == "The uploaded file not_exists.xlsx can not be read,"
+            f' therefore is not a valid data file for data node "{dn.id}"'
+        )
+
+        not_xlsx_path = tmpdir_factory.mktemp("data").join("wrong_format_df.xlsm").strpath
+        old_data.to_excel(not_xlsx_path, index=False)
+        # The upload should fail when the file is not a xlsx
+        reasons = dn._upload(not_xlsx_path, upload_checker=check_data_column)
+        assert bool(reasons) is False
+        assert (
+            str(list(reasons._reasons[dn.id])[0])
+            == f'The uploaded file wrong_format_df.xlsm has invalid data for data node "{dn.id}"'
+        )
+
+        wrong_format_xlsx_path = tmpdir_factory.mktemp("data").join("wrong_format_df.xlsx").strpath
+        pd.DataFrame([{"a": 1, "b": 2, "d": 3}, {"a": 4, "b": 5, "d": 6}]).to_excel(wrong_format_xlsx_path, index=False)
+        # The upload should fail when check_data_column() return False
+        reasons = dn._upload(wrong_format_xlsx_path, upload_checker=check_data_column)
+        assert bool(reasons) is False
+        assert (
+            str(list(reasons._reasons[dn.id])[0])
+            == f'The uploaded file wrong_format_df.xlsx has invalid data for data node "{dn.id}"'
+        )
+
+        assert_frame_equal(dn.read()["Sheet1"], old_data)  # The content of the dn should not change when upload fails
+        assert dn.last_edit_date == old_last_edit_date  # The last edit date should not change when upload fails
+        assert dn.path == old_xlsx_path  # The path of the dn should not change
+
+        # The upload should succeed when check_data_column() return True
+        assert dn._upload(excel_file, upload_checker=check_data_column)
+
+    def test_upload_with_upload_check_numpy(self, tmpdir_factory):
+        old_excel_path = tmpdir_factory.mktemp("data").join("df.xlsx").strpath
+        old_data = np.array([[1, 2, 3], [4, 5, 6]])
+
+        new_excel_path = tmpdir_factory.mktemp("data").join("new_upload_data.xlsx").strpath
+        new_data = np.array([[1, 2, 3], [4, 5, 6]])
+        pd.DataFrame(new_data).to_excel(new_excel_path, index=False)
+
+        dn = ExcelDataNode("foo", Scope.SCENARIO, properties={"path": old_excel_path, "exposed_type": "numpy"})
+        dn.write(old_data)
+        old_last_edit_date = dn.last_edit_date
+
+        def check_data_is_positive(upload_path, upload_data):
+            return upload_path.endswith(".xlsx") and np.all(upload_data["Sheet1"] > 0)
+
+        not_exists_xlsx_path = tmpdir_factory.mktemp("data").join("not_exists.xlsx").strpath
+        reasons = dn._upload(not_exists_xlsx_path, upload_checker=check_data_is_positive)
+        assert bool(reasons) is False
+        assert (
+            str(list(reasons._reasons[dn.id])[0]) == "The uploaded file not_exists.xlsx can not be read,"
+            f' therefore is not a valid data file for data node "{dn.id}"'
+        )
+
+        wrong_format_not_excel_path = tmpdir_factory.mktemp("data").join("wrong_format_df.xlsm").strpath
+        pd.DataFrame(old_data).to_excel(wrong_format_not_excel_path, index=False)
+        # The upload should fail when the file is not a excel
+        reasons = dn._upload(wrong_format_not_excel_path, upload_checker=check_data_is_positive)
+        assert bool(reasons) is False
+        assert (
+            str(list(reasons._reasons[dn.id])[0])
+            == f'The uploaded file wrong_format_df.xlsm has invalid data for data node "{dn.id}"'
+        )
+
+        not_xlsx_path = tmpdir_factory.mktemp("data").join("wrong_format_df.xlsx").strpath
+        pd.DataFrame(np.array([[-1, 2, 3], [-4, -5, -6]])).to_excel(not_xlsx_path, index=False)
+        # The upload should fail when check_data_is_positive() return False
+        reasons = dn._upload(not_xlsx_path, upload_checker=check_data_is_positive)
+        assert (
+            str(list(reasons._reasons[dn.id])[0])
+            == f'The uploaded file wrong_format_df.xlsx has invalid data for data node "{dn.id}"'
+        )
+
+        np.array_equal(dn.read()["Sheet1"], old_data)  # The content of the dn should not change when upload fails
+        assert dn.last_edit_date == old_last_edit_date  # The last edit date should not change when upload fails
+        assert dn.path == old_excel_path  # The path of the dn should not change
+
+        # The upload should succeed when check_data_is_positive() return True
+        assert dn._upload(new_excel_path, upload_checker=check_data_is_positive)

tests/core/data/test_filter_sql_table_data_node.py

@@ -9,7 +9,6 @@
 # an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
 # specific language governing permissions and limitations under the License.
 
-from importlib import util
 from unittest.mock import patch
 
 import numpy as np
@@ -30,60 +29,6 @@ class MyCustomObject:
 
 
 class TestFilterSQLTableDataNode:
-    __pandas_properties = [
-        {
-            "db_name": "taipy",
-            "db_engine": "sqlite",
-            "table_name": "example",
-            "db_extra_args": {
-                "TrustServerCertificate": "yes",
-                "other": "value",
-            },
-        },
-    ]
-
-    if util.find_spec("pyodbc"):
-        __pandas_properties.append(
-            {
-                "db_username": "sa",
-                "db_password": "Passw0rd",
-                "db_name": "taipy",
-                "db_engine": "mssql",
-                "table_name": "example",
-                "db_extra_args": {
-                    "TrustServerCertificate": "yes",
-                },
-            },
-        )
-
-    if util.find_spec("pymysql"):
-        __pandas_properties.append(
-            {
-                "db_username": "sa",
-                "db_password": "Passw0rd",
-                "db_name": "taipy",
-                "db_engine": "mysql",
-                "table_name": "example",
-                "db_extra_args": {
-                    "TrustServerCertificate": "yes",
-                },
-            },
-        )
-
-    if util.find_spec("psycopg2"):
-        __pandas_properties.append(
-            {
-                "db_username": "sa",
-                "db_password": "Passw0rd",
-                "db_name": "taipy",
-                "db_engine": "postgresql",
-                "table_name": "example",
-                "db_extra_args": {
-                    "TrustServerCertificate": "yes",
-                },
-            },
-        )
-
     def test_filter_pandas_exposed_type(self, tmp_sqlite_sqlite3_file_path):
         folder_path, db_name, file_extension = tmp_sqlite_sqlite3_file_path
         properties = {

tests/core/data/test_generic_data_node.py

@@ -11,8 +11,10 @@
 
 import pytest
 
+from taipy.config import Config
 from taipy.config.common.scope import Scope
 from taipy.config.exceptions.exceptions import InvalidConfigurationId
+from taipy.core.data._data_manager_factory import _DataManagerFactory
 from taipy.core.data.data_node import DataNode
 from taipy.core.data.data_node_id import DataNodeId
 from taipy.core.data.generic import GenericDataNode
@@ -52,10 +54,12 @@ def reset_data():
 class TestGenericDataNode:
     data = list(range(10))
 
-    def test_create(self):
-        dn = GenericDataNode(
-            "foo_bar", Scope.SCENARIO, properties={"read_fct": read_fct, "write_fct": write_fct, "name": "super name"}
+    def test_create_with_both_read_fct_and_write_fct(self):
+        data_manager = _DataManagerFactory._build_manager()
+        generic_dn_config = Config.configure_generic_data_node(
+            id="foo_bar", read_fct=read_fct, write_fct=write_fct, name="super name"
         )
+        dn = data_manager._create_and_set(generic_dn_config, None, None)
         assert isinstance(dn, GenericDataNode)
         assert dn.storage_type() == "generic"
         assert dn.config_id == "foo_bar"
@@ -69,68 +73,76 @@ class TestGenericDataNode:
         assert dn.properties["read_fct"] == read_fct
         assert dn.properties["write_fct"] == write_fct
 
-        dn_1 = GenericDataNode(
-            "foo", Scope.SCENARIO, properties={"read_fct": read_fct, "write_fct": None, "name": "foo"}
-        )
-        assert isinstance(dn, GenericDataNode)
-        assert dn_1.storage_type() == "generic"
-        assert dn_1.config_id == "foo"
-        assert dn_1.name == "foo"
-        assert dn_1.scope == Scope.SCENARIO
-        assert dn_1.id is not None
-        assert dn_1.owner_id is None
-        assert dn_1.last_edit_date is not None
-        assert dn_1.job_ids == []
-        assert dn_1.is_ready_for_reading
-        assert dn_1.properties["read_fct"] == read_fct
-        assert dn_1.properties["write_fct"] is None
-
-        dn_2 = GenericDataNode(
-            "xyz", Scope.SCENARIO, properties={"read_fct": None, "write_fct": write_fct, "name": "xyz"}
-        )
+        with pytest.raises(InvalidConfigurationId):
+            GenericDataNode("foo bar", Scope.SCENARIO, properties={"read_fct": read_fct, "write_fct": write_fct})
+
+    def test_create_with_read_fct_and_none_write_fct(self):
+        data_manager = _DataManagerFactory._build_manager()
+        generic_dn_config = Config.configure_generic_data_node(id="foo", read_fct=read_fct, write_fct=None, name="foo")
+        dn = data_manager._create_and_set(generic_dn_config, None, None)
         assert isinstance(dn, GenericDataNode)
-        assert dn_2.storage_type() == "generic"
-        assert dn_2.config_id == "xyz"
-        assert dn_2.name == "xyz"
-        assert dn_2.scope == Scope.SCENARIO
-        assert dn_2.id is not None
-        assert dn_2.owner_id is None
-        assert dn_2.last_edit_date is not None
-        assert dn_2.job_ids == []
-        assert dn_2.is_ready_for_reading
-        assert dn_2.properties["read_fct"] is None
-        assert dn_2.properties["write_fct"] == write_fct
-
-        dn_3 = GenericDataNode("xyz", Scope.SCENARIO, properties={"read_fct": read_fct, "name": "xyz"})
+        assert dn.storage_type() == "generic"
+        assert dn.config_id == "foo"
+        assert dn.name == "foo"
+        assert dn.scope == Scope.SCENARIO
+        assert dn.id is not None
+        assert dn.owner_id is None
+        assert dn.last_edit_date is not None
+        assert dn.job_ids == []
+        assert dn.is_ready_for_reading
+        assert dn.properties["read_fct"] == read_fct
+        assert dn.properties["write_fct"] is None
+
+    def test_create_with_write_fct_and_none_read_fct(self):
+        data_manager = _DataManagerFactory._build_manager()
+        generic_dn_config = Config.configure_generic_data_node(id="xyz", read_fct=None, write_fct=write_fct, name="xyz")
+        dn = data_manager._create_and_set(generic_dn_config, None, None)
         assert isinstance(dn, GenericDataNode)
-        assert dn_3.storage_type() == "generic"
-        assert dn_3.config_id == "xyz"
-        assert dn_3.name == "xyz"
-        assert dn_3.scope == Scope.SCENARIO
-        assert dn_3.id is not None
-        assert dn_3.owner_id is None
-        assert dn_3.last_edit_date is not None
-        assert dn_3.job_ids == []
-        assert dn_3.is_ready_for_reading
-        assert dn_3.properties["read_fct"] == read_fct
-        assert dn_3.properties["write_fct"] is None
-
-        dn_4 = GenericDataNode("xyz", Scope.SCENARIO, properties={"write_fct": write_fct, "name": "xyz"})
+        assert dn.storage_type() == "generic"
+        assert dn.config_id == "xyz"
+        assert dn.name == "xyz"
+        assert dn.scope == Scope.SCENARIO
+        assert dn.id is not None
+        assert dn.owner_id is None
+        assert dn.last_edit_date is not None
+        assert dn.job_ids == []
+        assert dn.is_ready_for_reading
+        assert dn.properties["read_fct"] is None
+        assert dn.properties["write_fct"] == write_fct
+
+    def test_create_with_read_fct(self):
+        data_manager = _DataManagerFactory._build_manager()
+        generic_dn_config = Config.configure_generic_data_node(id="acb", read_fct=read_fct, name="acb")
+        dn = data_manager._create_and_set(generic_dn_config, None, None)
         assert isinstance(dn, GenericDataNode)
-        assert dn_4.storage_type() == "generic"
-        assert dn_4.config_id == "xyz"
-        assert dn_4.name == "xyz"
-        assert dn_4.scope == Scope.SCENARIO
-        assert dn_4.id is not None
-        assert dn_4.owner_id is None
-        assert dn_4.last_edit_date is not None
-        assert dn_4.job_ids == []
-        assert dn_4.is_ready_for_reading
-        assert dn_4.properties["read_fct"] is None
-        assert dn_4.properties["write_fct"] == write_fct
+        assert dn.storage_type() == "generic"
+        assert dn.config_id == "acb"
+        assert dn.name == "acb"
+        assert dn.scope == Scope.SCENARIO
+        assert dn.id is not None
+        assert dn.owner_id is None
+        assert dn.last_edit_date is not None
+        assert dn.job_ids == []
+        assert dn.is_ready_for_reading
+        assert dn.properties["read_fct"] == read_fct
+        assert dn.properties["write_fct"] is None
 
-        with pytest.raises(InvalidConfigurationId):
-            GenericDataNode("foo bar", Scope.SCENARIO, properties={"read_fct": read_fct, "write_fct": write_fct})
+    def test_create_with_write_fct(self):
+        data_manager = _DataManagerFactory._build_manager()
+        generic_dn_config = Config.configure_generic_data_node(id="mno", write_fct=write_fct, name="mno")
+        dn = data_manager._create_and_set(generic_dn_config, None, None)
+        assert isinstance(dn, GenericDataNode)
+        assert dn.storage_type() == "generic"
+        assert dn.config_id == "mno"
+        assert dn.name == "mno"
+        assert dn.scope == Scope.SCENARIO
+        assert dn.id is not None
+        assert dn.owner_id is None
+        assert dn.last_edit_date is not None
+        assert dn.job_ids == []
+        assert dn.is_ready_for_reading
+        assert dn.properties["read_fct"] is None
+        assert dn.properties["write_fct"] == write_fct
 
     def test_get_user_properties(self):
         dn_1 = GenericDataNode(

tests/core/data/test_in_memory_data_node.py

@@ -11,8 +11,10 @@
 
 import pytest
 
+from taipy.config import Config
 from taipy.config.common.scope import Scope
 from taipy.config.exceptions.exceptions import InvalidConfigurationId
+from taipy.core.data._data_manager_factory import _DataManagerFactory
 from taipy.core.data.data_node_id import DataNodeId
 from taipy.core.data.in_memory import InMemoryDataNode
 from taipy.core.exceptions.exceptions import NoData
@@ -20,18 +22,14 @@ from taipy.core.exceptions.exceptions import NoData
 
 class TestInMemoryDataNodeEntity:
     def test_create(self):
-        dn = InMemoryDataNode(
-            "foobar_bazy",
-            Scope.SCENARIO,
-            DataNodeId("id_uio"),
-            "owner_id",
-            properties={"default_data": "In memory Data Node", "name": "my name"},
+        in_memory_dn_config = Config.configure_in_memory_data_node(
+            id="foobar_bazy", default_data="In memory Data Node", name="my name"
         )
+        dn = _DataManagerFactory._build_manager()._create_and_set(in_memory_dn_config, "owner_id", None)
         assert isinstance(dn, InMemoryDataNode)
         assert dn.storage_type() == "in_memory"
         assert dn.config_id == "foobar_bazy"
         assert dn.scope == Scope.SCENARIO
-        assert dn.id == "id_uio"
         assert dn.name == "my name"
         assert dn.owner_id == "owner_id"
         assert dn.last_edit_date is not None
@@ -39,7 +37,8 @@ class TestInMemoryDataNodeEntity:
         assert dn.is_ready_for_reading
         assert dn.read() == "In memory Data Node"
 
-        dn_2 = InMemoryDataNode("foo", Scope.SCENARIO)
+        in_memory_dn_config_2 = Config.configure_in_memory_data_node(id="foo")
+        dn_2 = _DataManagerFactory._build_manager()._create_and_set(in_memory_dn_config_2, None, None)
         assert dn_2.last_edit_date is None
         assert not dn_2.is_ready_for_reading
 

tests/core/data/test_json_data_node.py

@@ -18,6 +18,7 @@ from dataclasses import dataclass
 from enum import Enum
 from time import sleep
 
+import freezegun
 import numpy as np
 import pandas as pd
 import pytest
@@ -26,6 +27,7 @@ from taipy.config.common.scope import Scope
 from taipy.config.config import Config
 from taipy.config.exceptions.exceptions import InvalidConfigurationId
 from taipy.core.data._data_manager import _DataManager
+from taipy.core.data._data_manager_factory import _DataManagerFactory
 from taipy.core.data.data_node_id import DataNodeId
 from taipy.core.data.json import JSONDataNode
 from taipy.core.data.operator import JoinOperator, Operator
@@ -87,21 +89,40 @@ class MyCustomDecoder(json.JSONDecoder):
 class TestJSONDataNode:
     def test_create(self):
         path = "data/node/path"
-        dn = JSONDataNode("foo_bar", Scope.SCENARIO, properties={"default_path": path, "name": "super name"})
-        assert isinstance(dn, JSONDataNode)
-        assert dn.storage_type() == "json"
-        assert dn.config_id == "foo_bar"
-        assert dn.name == "super name"
-        assert dn.scope == Scope.SCENARIO
-        assert dn.id is not None
-        assert dn.owner_id is None
-        assert dn.last_edit_date is None
-        assert dn.job_ids == []
-        assert not dn.is_ready_for_reading
-        assert dn.path == path
+        json_dn_config = Config.configure_json_data_node(id="foo_bar", default_path=path, name="super name")
+        dn_1 = _DataManagerFactory._build_manager()._create_and_set(json_dn_config, None, None)
+        assert isinstance(dn_1, JSONDataNode)
+        assert dn_1.storage_type() == "json"
+        assert dn_1.config_id == "foo_bar"
+        assert dn_1.name == "super name"
+        assert dn_1.scope == Scope.SCENARIO
+        assert dn_1.id is not None
+        assert dn_1.owner_id is None
+        assert dn_1.last_edit_date is None
+        assert dn_1.job_ids == []
+        assert not dn_1.is_ready_for_reading
+        assert dn_1.path == path
+
+        json_dn_config_2 = Config.configure_json_data_node(id="foo", default_path=path, encoding="utf-16")
+        dn_2 = _DataManagerFactory._build_manager()._create_and_set(json_dn_config_2, None, None)
+        assert isinstance(dn_2, JSONDataNode)
+        assert dn_2.storage_type() == "json"
+        assert dn_2.properties["encoding"] == "utf-16"
+        assert dn_2.encoding == "utf-16"
+
+        json_dn_config_3 = Config.configure_json_data_node(
+            id="foo", default_path=path, encoder=MyCustomEncoder, decoder=MyCustomDecoder
+        )
+        dn_3 = _DataManagerFactory._build_manager()._create_and_set(json_dn_config_3, None, None)
+        assert isinstance(dn_3, JSONDataNode)
+        assert dn_3.storage_type() == "json"
+        assert dn_3.properties["encoder"] == MyCustomEncoder
+        assert dn_3.encoder == MyCustomEncoder
+        assert dn_3.properties["decoder"] == MyCustomDecoder
+        assert dn_3.decoder == MyCustomDecoder
 
         with pytest.raises(InvalidConfigurationId):
-            dn = JSONDataNode(
+            _ = JSONDataNode(
                 "foo bar", Scope.SCENARIO, properties={"default_path": path, "has_header": False, "name": "super name"}
             )
 
@@ -370,3 +391,79 @@ class TestJSONDataNode:
 
         assert ".data" not in dn.path
         assert os.path.exists(dn.path)
+
+    def test_get_download_path(self):
+        path = os.path.join(pathlib.Path(__file__).parent.resolve(), "data_sample/json/example_dict.json")
+        dn = JSONDataNode("foo", Scope.SCENARIO, properties={"path": path})
+        assert dn._get_downloadable_path() == path
+
+    def test_get_download_path_with_not_existed_file(self):
+        dn = JSONDataNode("foo", Scope.SCENARIO, properties={"path": "NOT_EXISTED.json"})
+        assert dn._get_downloadable_path() == ""
+
+    def test_upload(self, json_file, tmpdir_factory):
+        old_json_path = tmpdir_factory.mktemp("data").join("df.json").strpath
+        old_data = [{"a": 0, "b": 1, "c": 2}, {"a": 3, "b": 4, "c": 5}]
+
+        dn = JSONDataNode("foo", Scope.SCENARIO, properties={"path": old_json_path})
+        dn.write(old_data)
+        old_last_edit_date = dn.last_edit_date
+
+        with open(json_file, "r") as f:
+            upload_content = json.load(f)
+
+        with freezegun.freeze_time(old_last_edit_date + datetime.timedelta(seconds=1)):
+            dn._upload(json_file)
+
+        assert dn.read() == upload_content  # The content of the dn should change to the uploaded content
+        assert dn.last_edit_date > old_last_edit_date
+        assert dn.path == old_json_path  # The path of the dn should not change
+
+    def test_upload_with_upload_check(self, json_file, tmpdir_factory):
+        old_json_path = tmpdir_factory.mktemp("data").join("df.json").strpath
+        old_data = [{"a": 0, "b": 1, "c": 2}, {"a": 3, "b": 4, "c": 5}]
+
+        dn = JSONDataNode("foo", Scope.SCENARIO, properties={"path": old_json_path})
+        dn.write(old_data)
+        old_last_edit_date = dn.last_edit_date
+
+        def check_data_keys(upload_path, upload_data):
+            all_column_is_abc = all(data.keys() == {"a", "b", "c"} for data in upload_data)
+            return upload_path.endswith(".json") and all_column_is_abc
+
+        not_exists_json_path = tmpdir_factory.mktemp("data").join("not_exists.json").strpath
+        reasons = dn._upload(not_exists_json_path, upload_checker=check_data_keys)
+        assert bool(reasons) is False
+        assert (
+            str(list(reasons._reasons[dn.id])[0]) == "The uploaded file not_exists.json can not be read,"
+            f' therefore is not a valid data file for data node "{dn.id}"'
+        )
+
+        not_json_path = tmpdir_factory.mktemp("data").join("wrong_format_df.not_json").strpath
+        with open(not_json_path, "w") as f:
+            json.dump([{"a": 1, "b": 2, "d": 3}, {"a": 4, "b": 5, "d": 6}], f)
+        # The upload should fail when the file is not a json
+        reasons = dn._upload(not_json_path, upload_checker=check_data_keys)
+        assert bool(reasons) is False
+        assert (
+            str(list(reasons._reasons[dn.id])[0])
+            == f'The uploaded file wrong_format_df.not_json has invalid data for data node "{dn.id}"'
+        )
+
+        wrong_format_json_path = tmpdir_factory.mktemp("data").join("wrong_format_df.json").strpath
+        with open(wrong_format_json_path, "w") as f:
+            json.dump([{"a": 1, "b": 2, "d": 3}, {"a": 4, "b": 5, "d": 6}], f)
+        # The upload should fail when check_data_keys() return False
+        reasons = dn._upload(wrong_format_json_path, upload_checker=check_data_keys)
+        assert bool(reasons) is False
+        assert (
+            str(list(reasons._reasons[dn.id])[0])
+            == f'The uploaded file wrong_format_df.json has invalid data for data node "{dn.id}"'
+        )
+
+        assert dn.read() == old_data  # The content of the dn should not change when upload fails
+        assert dn.last_edit_date == old_last_edit_date  # The last edit date should not change when upload fails
+        assert dn.path == old_json_path  # The path of the dn should not change
+
+        # The upload should succeed when check_data_keys() return True
+        assert dn._upload(json_file, upload_checker=check_data_keys)

tests/core/data/test_mongo_data_node.py

@@ -20,9 +20,11 @@ import pytest
 from bson import ObjectId
 from bson.errors import InvalidDocument
 
+from taipy.config import Config
 from taipy.config.common.scope import Scope
 from taipy.core import MongoDefaultDocument
 from taipy.core.common._mongo_connector import _connect_mongodb
+from taipy.core.data._data_manager_factory import _DataManagerFactory
 from taipy.core.data.data_node_id import DataNodeId
 from taipy.core.data.mongo import MongoCollectionDataNode
 from taipy.core.data.operator import JoinOperator, Operator
@@ -76,11 +78,8 @@ class TestMongoCollectionDataNode:
 
     @pytest.mark.parametrize("properties", __properties)
     def test_create(self, properties):
-        mongo_dn = MongoCollectionDataNode(
-            "foo_bar",
-            Scope.SCENARIO,
-            properties=properties,
-        )
+        mongo_dn_config = Config.configure_mongo_collection_data_node("foo_bar", **properties)
+        mongo_dn = _DataManagerFactory._build_manager()._create_and_set(mongo_dn_config, None, None)
         assert isinstance(mongo_dn, MongoCollectionDataNode)
         assert mongo_dn.storage_type() == "mongo_collection"
         assert mongo_dn.config_id == "foo_bar"

tests/core/data/test_parquet_data_node.py

@@ -12,17 +12,21 @@
 import os
 import pathlib
 import uuid
-from datetime import datetime
+from datetime import datetime, timedelta
 from importlib import util
 from time import sleep
 
+import freezegun
+import numpy as np
 import pandas as pd
 import pytest
+from pandas.testing import assert_frame_equal
 
 from taipy.config.common.scope import Scope
 from taipy.config.config import Config
 from taipy.config.exceptions.exceptions import InvalidConfigurationId
 from taipy.core.data._data_manager import _DataManager
+from taipy.core.data._data_manager_factory import _DataManagerFactory
 from taipy.core.data.data_node_id import DataNodeId
 from taipy.core.data.parquet import ParquetDataNode
 from taipy.core.exceptions.exceptions import (
@@ -65,9 +69,10 @@ class TestParquetDataNode:
     def test_create(self):
         path = "data/node/path"
         compression = "snappy"
-        dn = ParquetDataNode(
-            "foo_bar", Scope.SCENARIO, properties={"path": path, "compression": compression, "name": "super name"}
+        parquet_dn_config = Config.configure_parquet_data_node(
+            id="foo_bar", default_path=path, compression=compression, name="super name"
         )
+        dn = _DataManagerFactory._build_manager()._create_and_set(parquet_dn_config, None, None)
         assert isinstance(dn, ParquetDataNode)
         assert dn.storage_type() == "parquet"
         assert dn.config_id == "foo_bar"
@@ -83,6 +88,13 @@ class TestParquetDataNode:
         assert dn.compression == "snappy"
         assert dn.engine == "pyarrow"
 
+        parquet_dn_config_1 = Config.configure_parquet_data_node(
+            id="bar", default_path=path, compression=compression, exposed_type=MyCustomObject
+        )
+        dn_1 = _DataManagerFactory._build_manager()._create_and_set(parquet_dn_config_1, None, None)
+        assert isinstance(dn_1, ParquetDataNode)
+        assert dn_1.exposed_type == MyCustomObject
+
         with pytest.raises(InvalidConfigurationId):
             dn = ParquetDataNode("foo bar", Scope.SCENARIO, properties={"path": path, "name": "super name"})
 
@@ -221,3 +233,132 @@ class TestParquetDataNode:
 
         assert ".data" not in dn.path
         assert os.path.exists(dn.path)
+
+    def test_get_downloadable_path(self):
+        path = os.path.join(pathlib.Path(__file__).parent.resolve(), "data_sample/example.parquet")
+        dn = ParquetDataNode("foo", Scope.SCENARIO, properties={"path": path, "exposed_type": "pandas"})
+        assert dn._get_downloadable_path() == path
+
+    def test_get_downloadable_path_with_not_existing_file(self):
+        dn = ParquetDataNode("foo", Scope.SCENARIO, properties={"path": "NOT_EXISTING.parquet"})
+        assert dn._get_downloadable_path() == ""
+
+    def test_get_downloadable_path_as_directory_should_return_nothing(self):
+        path = os.path.join(pathlib.Path(__file__).parent.resolve(), "data_sample/parquet_example")
+        dn = ParquetDataNode("foo", Scope.SCENARIO, properties={"path": path})
+        assert dn._get_downloadable_path() == ""
+
+    def test_upload(self, parquet_file_path, tmpdir_factory):
+        old_parquet_path = tmpdir_factory.mktemp("data").join("df.parquet").strpath
+        old_data = pd.DataFrame([{"a": 0, "b": 1, "c": 2}, {"a": 3, "b": 4, "c": 5}])
+
+        dn = ParquetDataNode("foo", Scope.SCENARIO, properties={"path": old_parquet_path, "exposed_type": "pandas"})
+        dn.write(old_data)
+        old_last_edit_date = dn.last_edit_date
+
+        upload_content = pd.read_parquet(parquet_file_path)
+
+        with freezegun.freeze_time(old_last_edit_date + timedelta(seconds=1)):
+            dn._upload(parquet_file_path)
+
+        assert_frame_equal(dn.read(), upload_content)  # The content of the dn should change to the uploaded content
+        assert dn.last_edit_date > old_last_edit_date
+        assert dn.path == old_parquet_path  # The path of the dn should not change
+
+    def test_upload_with_upload_check_pandas(self, parquet_file_path, tmpdir_factory):
+        old_parquet_path = tmpdir_factory.mktemp("data").join("df.parquet").strpath
+        old_data = pd.DataFrame([{"a": 0, "b": 1, "c": 2}, {"a": 3, "b": 4, "c": 5}])
+
+        dn = ParquetDataNode("foo", Scope.SCENARIO, properties={"path": old_parquet_path, "exposed_type": "pandas"})
+        dn.write(old_data)
+        old_last_edit_date = dn.last_edit_date
+
+        def check_data_column(upload_path, upload_data):
+            return upload_path.endswith(".parquet") and upload_data.columns.tolist() == ["a", "b", "c"]
+
+        not_exists_parquet_path = tmpdir_factory.mktemp("data").join("not_exists.parquet").strpath
+        reasons = dn._upload(not_exists_parquet_path, upload_checker=check_data_column)
+        assert bool(reasons) is False
+        assert (
+            str(list(reasons._reasons[dn.id])[0]) == "The uploaded file not_exists.parquet can not be read,"
+            f' therefore is not a valid data file for data node "{dn.id}"'
+        )
+
+        not_parquet_path = tmpdir_factory.mktemp("data").join("wrong_format_df.not_parquet").strpath
+        old_data.to_parquet(not_parquet_path, index=False)
+        # The upload should fail when the file is not a parquet
+        reasons = dn._upload(not_parquet_path, upload_checker=check_data_column)
+        assert bool(reasons) is False
+        assert (
+            str(list(reasons._reasons[dn.id])[0])
+            == f'The uploaded file wrong_format_df.not_parquet has invalid data for data node "{dn.id}"'
+        )
+
+        wrong_format_parquet_path = tmpdir_factory.mktemp("data").join("wrong_format_df.parquet").strpath
+        pd.DataFrame([{"a": 1, "b": 2, "d": 3}, {"a": 4, "b": 5, "d": 6}]).to_parquet(
+            wrong_format_parquet_path, index=False
+        )
+        # The upload should fail when check_data_column() return False
+        reasons = dn._upload(wrong_format_parquet_path, upload_checker=check_data_column)
+        assert bool(reasons) is False
+        assert (
+            str(list(reasons._reasons[dn.id])[0])
+            == f'The uploaded file wrong_format_df.parquet has invalid data for data node "{dn.id}"'
+        )
+
+        assert_frame_equal(dn.read(), old_data)  # The content of the dn should not change when upload fails
+        assert dn.last_edit_date == old_last_edit_date  # The last edit date should not change when upload fails
+        assert dn.path == old_parquet_path  # The path of the dn should not change
+
+        # The upload should succeed when check_data_column() return True
+        assert dn._upload(parquet_file_path, upload_checker=check_data_column)
+
+    def test_upload_with_upload_check_numpy(self, tmpdir_factory):
+        old_parquet_path = tmpdir_factory.mktemp("data").join("df.parquet").strpath
+        old_data = np.array([[1, 2, 3], [4, 5, 6]])
+
+        new_parquet_path = tmpdir_factory.mktemp("data").join("new_upload_data.parquet").strpath
+        new_data = np.array([[1, 2, 3], [4, 5, 6]])
+        pd.DataFrame(new_data, columns=["a", "b", "c"]).to_parquet(new_parquet_path, index=False)
+
+        dn = ParquetDataNode("foo", Scope.SCENARIO, properties={"path": old_parquet_path, "exposed_type": "numpy"})
+        dn.write(old_data)
+        old_last_edit_date = dn.last_edit_date
+
+        def check_data_is_positive(upload_path, upload_data):
+            return upload_path.endswith(".parquet") and np.all(upload_data > 0)
+
+        not_exists_parquet_path = tmpdir_factory.mktemp("data").join("not_exists.parquet").strpath
+        reasons = dn._upload(not_exists_parquet_path, upload_checker=check_data_is_positive)
+        assert bool(reasons) is False
+        assert (
+            str(list(reasons._reasons[dn.id])[0]) == "The uploaded file not_exists.parquet can not be read,"
+            f' therefore is not a valid data file for data node "{dn.id}"'
+        )
+
+        not_parquet_path = tmpdir_factory.mktemp("data").join("wrong_format_df.not_parquet").strpath
+        pd.DataFrame(old_data, columns=["a", "b", "c"]).to_parquet(not_parquet_path, index=False)
+        # The upload should fail when the file is not a parquet
+        reasons = dn._upload(not_parquet_path, upload_checker=check_data_is_positive)
+        assert (
+            str(list(reasons._reasons[dn.id])[0])
+            == f'The uploaded file wrong_format_df.not_parquet has invalid data for data node "{dn.id}"'
+        )
+
+        wrong_format_parquet_path = tmpdir_factory.mktemp("data").join("wrong_format_df.parquet").strpath
+        pd.DataFrame(np.array([[-1, 2, 3], [-4, -5, -6]]), columns=["a", "b", "c"]).to_parquet(
+            wrong_format_parquet_path, index=False
+        )
+        # The upload should fail when check_data_is_positive() return False
+        reasons = dn._upload(wrong_format_parquet_path, upload_checker=check_data_is_positive)
+        assert (
+            str(list(reasons._reasons[dn.id])[0])
+            == f'The uploaded file wrong_format_df.parquet has invalid data for data node "{dn.id}"'
+        )
+
+        np.array_equal(dn.read(), old_data)  # The content of the dn should not change when upload fails
+        assert dn.last_edit_date == old_last_edit_date  # The last edit date should not change when upload fails
+        assert dn.path == old_parquet_path  # The path of the dn should not change
+
+        # The upload should succeed when check_data_is_positive() return True
+        assert dn._upload(new_parquet_path, upload_checker=check_data_is_positive)

tests/core/data/test_pickle_data_node.py

@@ -11,16 +11,20 @@
 
 import os
 import pathlib
-from datetime import datetime
+import pickle
+from datetime import datetime, timedelta
 from time import sleep
 
+import freezegun
 import pandas as pd
 import pytest
+from pandas.testing import assert_frame_equal
 
 from taipy.config.common.scope import Scope
 from taipy.config.config import Config
 from taipy.config.exceptions.exceptions import InvalidConfigurationId
 from taipy.core.data._data_manager import _DataManager
+from taipy.core.data._data_manager_factory import _DataManagerFactory
 from taipy.core.data.pickle import PickleDataNode
 from taipy.core.exceptions.exceptions import NoData
 
@@ -42,9 +46,17 @@ class TestPickleDataNodeEntity:
         for f in glob.glob("*.p"):
             os.remove(f)
 
+    def test_create_with_manager(self, pickle_file_path):
+        parquet_dn_config = Config.configure_pickle_data_node(id="baz", default_path=pickle_file_path)
+        parquet_dn = _DataManagerFactory._build_manager()._create_and_set(parquet_dn_config, None, None)
+        assert isinstance(parquet_dn, PickleDataNode)
+
     def test_create(self):
-        dn = PickleDataNode("foobar_bazxyxea", Scope.SCENARIO, properties={"default_data": "Data"})
-        assert os.path.isfile(os.path.join(Config.core.storage_folder.strip("/"), "pickles", dn.id + ".p"))
+        pickle_dn_config = Config.configure_pickle_data_node(
+            id="foobar_bazxyxea", default_path="Data", default_data="Data"
+        )
+        dn = _DataManagerFactory._build_manager()._create_and_set(pickle_dn_config, None, None)
+
         assert isinstance(dn, PickleDataNode)
         assert dn.storage_type() == "pickle"
         assert dn.config_id == "foobar_bazxyxea"
@@ -192,3 +204,77 @@ class TestPickleDataNodeEntity:
 
         assert ".data" not in dn.path
         assert os.path.exists(dn.path)
+
+    def test_get_download_path(self):
+        path = os.path.join(pathlib.Path(__file__).parent.resolve(), "data_sample/example.p")
+        dn = PickleDataNode("foo", Scope.SCENARIO, properties={"path": path})
+        assert dn._get_downloadable_path() == path
+
+    def test_get_download_path_with_not_existed_file(self):
+        dn = PickleDataNode("foo", Scope.SCENARIO, properties={"path": "NOT_EXISTED.p"})
+        assert dn._get_downloadable_path() == ""
+
+    def test_upload(self, pickle_file_path, tmpdir_factory):
+        old_pickle_path = tmpdir_factory.mktemp("data").join("df.p").strpath
+        old_data = pd.DataFrame([{"a": 0, "b": 1, "c": 2}, {"a": 3, "b": 4, "c": 5}])
+
+        dn = PickleDataNode("foo", Scope.SCENARIO, properties={"path": old_pickle_path})
+        dn.write(old_data)
+        old_last_edit_date = dn.last_edit_date
+
+        upload_content = pd.read_pickle(pickle_file_path)
+
+        with freezegun.freeze_time(old_last_edit_date + timedelta(seconds=1)):
+            dn._upload(pickle_file_path)
+
+        assert_frame_equal(dn.read(), upload_content)  # The content of the dn should change to the uploaded content
+        assert dn.last_edit_date > old_last_edit_date
+        assert dn.path == old_pickle_path  # The path of the dn should not change
+
+    def test_upload_with_upload_check(self, pickle_file_path, tmpdir_factory):
+        old_pickle_path = tmpdir_factory.mktemp("data").join("df.p").strpath
+        old_data = pd.DataFrame([{"a": 0, "b": 1, "c": 2}, {"a": 3, "b": 4, "c": 5}])
+
+        dn = PickleDataNode("foo", Scope.SCENARIO, properties={"path": old_pickle_path})
+        dn.write(old_data)
+        old_last_edit_date = dn.last_edit_date
+
+        def check_data_column(upload_path, upload_data):
+            return upload_path.endswith(".p") and upload_data.columns.tolist() == ["a", "b", "c"]
+
+        not_exists_json_path = tmpdir_factory.mktemp("data").join("not_exists.json").strpath
+        reasons = dn._upload(not_exists_json_path, upload_checker=check_data_column)
+        assert bool(reasons) is False
+        assert (
+            str(list(reasons._reasons[dn.id])[0]) == "The uploaded file not_exists.json can not be read,"
+            f' therefore is not a valid data file for data node "{dn.id}"'
+        )
+
+        not_pickle_path = tmpdir_factory.mktemp("data").join("wrong_format_df.not_pickle").strpath
+        with open(str(not_pickle_path), "wb") as f:
+            pickle.dump(pd.DataFrame([{"a": 1, "b": 2, "d": 3}, {"a": 4, "b": 5, "d": 6}]), f)
+        # The upload should fail when the file is not a pickle
+        reasons = dn._upload(not_pickle_path, upload_checker=check_data_column)
+        assert bool(reasons) is False
+        assert (
+            str(list(reasons._reasons[dn.id])[0])
+            == f'The uploaded file wrong_format_df.not_pickle has invalid data for data node "{dn.id}"'
+        )
+
+        wrong_format_pickle_path = tmpdir_factory.mktemp("data").join("wrong_format_df.p").strpath
+        with open(str(wrong_format_pickle_path), "wb") as f:
+            pickle.dump(pd.DataFrame([{"a": 1, "b": 2, "d": 3}, {"a": 4, "b": 5, "d": 6}]), f)
+        # The upload should fail when check_data_column() return False
+        reasons = dn._upload(wrong_format_pickle_path, upload_checker=check_data_column)
+        assert bool(reasons) is False
+        assert (
+            str(list(reasons._reasons[dn.id])[0])
+            == f'The uploaded file wrong_format_df.p has invalid data for data node "{dn.id}"'
+        )
+
+        assert_frame_equal(dn.read(), old_data)  # The content of the dn should not change when upload fails
+        assert dn.last_edit_date == old_last_edit_date  # The last edit date should not change when upload fails
+        assert dn.path == old_pickle_path  # The path of the dn should not change
+
+        # The upload should succeed when check_data_column() return True
+        assert dn._upload(pickle_file_path, upload_checker=check_data_column)

tests/core/data/test_read_excel_data_node.py

@@ -58,6 +58,7 @@ class MyCustomObject2:
 excel_file_path = os.path.join(pathlib.Path(__file__).parent.resolve(), "data_sample/example.xlsx")
 sheet_names = ["Sheet1", "Sheet2"]
 custom_class_dict = {"Sheet1": MyCustomObject1, "Sheet2": MyCustomObject2}
+custom_pandas_numpy_exposed_type_dict = {"Sheet1": "pandas", "Sheet2": "numpy"}
 
 
 def test_raise_no_data_with_header():
@@ -300,7 +301,7 @@ def test_read_multi_sheet_with_header_pandas():
     assert isinstance(data_pandas, Dict)
     assert len(data_pandas) == 2
     assert all(
-        len(data_pandas[sheet_name] == 5) and isinstance(data_pandas[sheet_name], pd.DataFrame)
+        len(data_pandas[sheet_name]) == 5 and isinstance(data_pandas[sheet_name], pd.DataFrame)
         for sheet_name in sheet_names
     )
     assert list(data_pandas.keys()) == sheet_names
@@ -331,7 +332,7 @@ def test_read_multi_sheet_with_header_numpy():
     assert isinstance(data_numpy, Dict)
     assert len(data_numpy) == 2
     assert all(
-        len(data_numpy[sheet_name] == 5) and isinstance(data_numpy[sheet_name], np.ndarray)
+        len(data_numpy[sheet_name]) == 5 and isinstance(data_numpy[sheet_name], np.ndarray)
         for sheet_name in sheet_names
     )
     assert list(data_numpy.keys()) == sheet_names
@@ -400,7 +401,7 @@ def test_read_multi_sheet_with_header_single_custom_exposed_type():
             assert row_custom_no_sheet_name.text == row_custom.text
 
 
-def test_read_multi_sheet_with_header_multiple_custom_exposed_type():
+def test_read_multi_sheet_with_header_multiple_custom_object_exposed_type():
     data_pandas = pd.read_excel(excel_file_path, sheet_name=sheet_names)
 
     # With sheet name
@@ -461,6 +462,48 @@ def test_read_multi_sheet_with_header_multiple_custom_exposed_type():
             assert row_custom_no_sheet_name.text == row_custom.text
 
 
+def test_read_multi_sheet_with_header_multiple_custom_pandas_numpy_exposed_type():
+    # With sheet name
+    excel_dn_as_pandas_numpy = ExcelDataNode(
+        "bar",
+        Scope.SCENARIO,
+        properties={
+            "path": excel_file_path,
+            "sheet_name": sheet_names,
+            "exposed_type": custom_pandas_numpy_exposed_type_dict,
+        },
+    )
+    assert excel_dn_as_pandas_numpy.properties["exposed_type"] == custom_pandas_numpy_exposed_type_dict
+    multi_data_custom = excel_dn_as_pandas_numpy.read()
+    assert isinstance(multi_data_custom["Sheet1"], pd.DataFrame)
+    assert isinstance(multi_data_custom["Sheet2"], np.ndarray)
+
+    excel_dn_as_pandas_numpy = ExcelDataNode(
+        "bar",
+        Scope.SCENARIO,
+        properties={
+            "path": excel_file_path,
+            "sheet_name": sheet_names,
+            "exposed_type": ["pandas", "numpy"],
+        },
+    )
+    assert excel_dn_as_pandas_numpy.properties["exposed_type"] == ["pandas", "numpy"]
+    multi_data_custom = excel_dn_as_pandas_numpy.read()
+    assert isinstance(multi_data_custom["Sheet1"], pd.DataFrame)
+    assert isinstance(multi_data_custom["Sheet2"], np.ndarray)
+
+    # Without sheet name
+    excel_dn_as_pandas_numpy_no_sheet_name = ExcelDataNode(
+        "bar",
+        Scope.SCENARIO,
+        properties={"path": excel_file_path, "exposed_type": custom_pandas_numpy_exposed_type_dict},
+    )
+    assert excel_dn_as_pandas_numpy_no_sheet_name.properties["exposed_type"] == custom_pandas_numpy_exposed_type_dict
+    multi_data_custom_no_sheet_name = excel_dn_as_pandas_numpy_no_sheet_name.read()
+    assert isinstance(multi_data_custom_no_sheet_name["Sheet1"], pd.DataFrame)
+    assert isinstance(multi_data_custom_no_sheet_name["Sheet2"], np.ndarray)
+
+
 def test_read_multi_sheet_without_header_pandas():
     # With sheet name
     excel_data_node_as_pandas = ExcelDataNode(
@@ -525,7 +568,7 @@ def test_read_multi_sheet_without_header_numpy():
         assert np.array_equal(data_numpy[key], data_numpy_no_sheet_name[key])
 
 
-def test_read_multi_sheet_without_header_single_custom_exposed_type():
+def test_read_multi_sheet_without_header_single_custom_object_exposed_type():
     data_pandas = pd.read_excel(excel_file_path, header=None, sheet_name=sheet_names)
 
     # With sheet name
@@ -579,7 +622,7 @@ def test_read_multi_sheet_without_header_single_custom_exposed_type():
             assert row_custom_no_sheet_name.text == row_custom.text
 
 
-def test_read_multi_sheet_without_header_multiple_custom_exposed_type():
+def test_read_multi_sheet_without_header_multiple_custom_object_exposed_type():
     data_pandas = pd.read_excel(excel_file_path, header=None, sheet_name=sheet_names)
 
     # With sheet names
@@ -643,3 +686,51 @@ def test_read_multi_sheet_without_header_multiple_custom_exposed_type():
             assert row_custom_no_sheet_name.id == row_custom.id
             assert row_custom_no_sheet_name.integer == row_custom.integer
             assert row_custom_no_sheet_name.text == row_custom.text
+
+
+def test_read_multi_sheet_without_header_multiple_custom_pandas_numpy_exposed_type():
+    # With sheet names
+    excel_dn_as_pandas_numpy = ExcelDataNode(
+        "bar",
+        Scope.SCENARIO,
+        properties={
+            "path": excel_file_path,
+            "sheet_name": sheet_names,
+            "exposed_type": custom_pandas_numpy_exposed_type_dict,
+            "has_header": False,
+        },
+    )
+    assert excel_dn_as_pandas_numpy.properties["exposed_type"] == custom_pandas_numpy_exposed_type_dict
+    multi_data_custom = excel_dn_as_pandas_numpy.read()
+    assert isinstance(multi_data_custom["Sheet1"], pd.DataFrame)
+    assert isinstance(multi_data_custom["Sheet2"], np.ndarray)
+
+    excel_dn_as_pandas_numpy = ExcelDataNode(
+        "bar",
+        Scope.SCENARIO,
+        properties={
+            "path": excel_file_path,
+            "sheet_name": sheet_names,
+            "exposed_type": ["pandas", "numpy"],
+            "has_header": False,
+        },
+    )
+    assert excel_dn_as_pandas_numpy.properties["exposed_type"] == ["pandas", "numpy"]
+    multi_data_custom = excel_dn_as_pandas_numpy.read()
+    assert isinstance(multi_data_custom["Sheet1"], pd.DataFrame)
+    assert isinstance(multi_data_custom["Sheet2"], np.ndarray)
+
+    # Without sheet names
+    excel_dn_as_pandas_numpy_no_sheet_name = ExcelDataNode(
+        "bar",
+        Scope.SCENARIO,
+        properties={
+            "path": excel_file_path,
+            "has_header": False,
+            "exposed_type": custom_pandas_numpy_exposed_type_dict,
+        },
+    )
+    multi_data_custom_no_sheet_name = excel_dn_as_pandas_numpy_no_sheet_name.read()
+    multi_data_custom_no_sheet_name = excel_dn_as_pandas_numpy.read()
+    assert isinstance(multi_data_custom_no_sheet_name["Sheet1"], pd.DataFrame)
+    assert isinstance(multi_data_custom_no_sheet_name["Sheet2"], np.ndarray)

tests/core/data/test_read_sql_table_data_node.py

@@ -17,6 +17,7 @@ import pandas as pd
 import pytest
 
 from taipy.config.common.scope import Scope
+from taipy.core.data.operator import JoinOperator, Operator
 from taipy.core.data.sql_table import SQLTableDataNode
 
 
@@ -29,7 +30,7 @@ class MyCustomObject:
 
 
 class TestReadSQLTableDataNode:
-    __pandas_properties = [
+    __sql_properties = [
         {
             "db_name": "taipy",
             "db_engine": "sqlite",
@@ -42,7 +43,7 @@ class TestReadSQLTableDataNode:
     ]
 
     if util.find_spec("pyodbc"):
-        __pandas_properties.append(
+        __sql_properties.append(
             {
                 "db_username": "sa",
                 "db_password": "Passw0rd",
@@ -56,7 +57,7 @@ class TestReadSQLTableDataNode:
         )
 
     if util.find_spec("pymysql"):
-        __pandas_properties.append(
+        __sql_properties.append(
             {
                 "db_username": "sa",
                 "db_password": "Passw0rd",
@@ -70,7 +71,7 @@ class TestReadSQLTableDataNode:
         )
 
     if util.find_spec("psycopg2"):
-        __pandas_properties.append(
+        __sql_properties.append(
             {
                 "db_username": "sa",
                 "db_password": "Passw0rd",
@@ -87,9 +88,9 @@ class TestReadSQLTableDataNode:
     def mock_read_value():
         return {"foo": ["baz", "quux", "corge"], "bar": ["quux", "quuz", None]}
 
-    @pytest.mark.parametrize("pandas_properties", __pandas_properties)
-    def test_read_pandas(self, pandas_properties):
-        custom_properties = pandas_properties.copy()
+    @pytest.mark.parametrize("sql_properties", __sql_properties)
+    def test_read_pandas(self, sql_properties):
+        custom_properties = sql_properties.copy()
 
         sql_data_node_as_pandas = SQLTableDataNode(
             "foo",
@@ -105,9 +106,106 @@ class TestReadSQLTableDataNode:
             assert isinstance(pandas_data, pd.DataFrame)
             assert pandas_data.equals(pd.DataFrame(self.mock_read_value()))
 
-    @pytest.mark.parametrize("pandas_properties", __pandas_properties)
-    def test_read_numpy(self, pandas_properties):
-        custom_properties = pandas_properties.copy()
+    def test_build_connection_string(self):
+        sql_properties = {
+            "db_username": "sa",
+            "db_password": "Passw0rd",
+            "db_name": "taipy",
+            "db_engine": "mssql",
+            "table_name": "example",
+            "db_driver": "default server",
+            "db_extra_args": {
+                "TrustServerCertificate": "yes",
+                "other": "value",
+            },
+        }
+        custom_properties = sql_properties.copy()
+        mssql_sql_data_node = SQLTableDataNode(
+            "foo",
+            Scope.SCENARIO,
+            properties=custom_properties,
+        )
+        assert (
+            mssql_sql_data_node._conn_string()
+            == "mssql+pyodbc://sa:Passw0rd@localhost:1433/taipy?TrustServerCertificate=yes&other=value&driver=default+server"
+        )
+
+        custom_properties["db_engine"] = "mysql"
+        mysql_sql_data_node = SQLTableDataNode(
+            "foo",
+            Scope.SCENARIO,
+            properties=custom_properties,
+        )
+        assert (
+            mysql_sql_data_node._conn_string()
+            == "mysql+pymysql://sa:Passw0rd@localhost:1433/taipy?TrustServerCertificate=yes&other=value&driver=default+server"
+        )
+
+        custom_properties["db_engine"] = "postgresql"
+        postgresql_sql_data_node = SQLTableDataNode(
+            "foo",
+            Scope.SCENARIO,
+            properties=custom_properties,
+        )
+        assert (
+            postgresql_sql_data_node._conn_string()
+            == "postgresql+psycopg2://sa:Passw0rd@localhost:1433/taipy?TrustServerCertificate=yes&other=value&driver=default+server"
+        )
+
+        custom_properties["db_engine"] = "sqlite"
+        sqlite_sql_data_node = SQLTableDataNode(
+            "foo",
+            Scope.SCENARIO,
+            properties=custom_properties,
+        )
+        assert sqlite_sql_data_node._conn_string() == "sqlite:///taipy.db"
+
+    @pytest.mark.parametrize("sql_properties", __sql_properties)
+    def test_get_read_query(self, sql_properties):
+        custom_properties = sql_properties.copy()
+
+        sql_data_node = SQLTableDataNode(
+            "foo",
+            Scope.SCENARIO,
+            properties=custom_properties,
+        )
+
+        assert sql_data_node._get_read_query(("key", 1, Operator.EQUAL)) == "SELECT * FROM example WHERE key = '1'"
+        assert sql_data_node._get_read_query(("key", 1, Operator.NOT_EQUAL)) == "SELECT * FROM example WHERE key <> '1'"
+        assert (
+            sql_data_node._get_read_query(("key", 1, Operator.GREATER_THAN)) == "SELECT * FROM example WHERE key > '1'"
+        )
+        assert (
+            sql_data_node._get_read_query(("key", 1, Operator.GREATER_OR_EQUAL))
+            == "SELECT * FROM example WHERE key >= '1'"
+        )
+        assert sql_data_node._get_read_query(("key", 1, Operator.LESS_THAN)) == "SELECT * FROM example WHERE key < '1'"
+        assert (
+            sql_data_node._get_read_query(("key", 1, Operator.LESS_OR_EQUAL))
+            == "SELECT * FROM example WHERE key <= '1'"
+        )
+
+        with pytest.raises(NotImplementedError):
+            sql_data_node._get_read_query(
+                [("key", 1, Operator.EQUAL), ("key2", 2, Operator.GREATER_THAN)], "SOME JoinOperator"
+            )
+
+        assert (
+            sql_data_node._get_read_query(
+                [("key", 1, Operator.EQUAL), ("key2", 2, Operator.GREATER_THAN)], JoinOperator.AND
+            )
+            == "SELECT * FROM example WHERE key = '1' AND key2 > '2'"
+        )
+        assert (
+            sql_data_node._get_read_query(
+                [("key", 1, Operator.EQUAL), ("key2", 2, Operator.GREATER_THAN)], JoinOperator.OR
+            )
+            == "SELECT * FROM example WHERE key = '1' OR key2 > '2'"
+        )
+
+    @pytest.mark.parametrize("sql_properties", __sql_properties)
+    def test_read_numpy(self, sql_properties):
+        custom_properties = sql_properties.copy()
         custom_properties["exposed_type"] = "numpy"
 
         sql_data_node_as_pandas = SQLTableDataNode(
@@ -124,9 +222,9 @@ class TestReadSQLTableDataNode:
             assert isinstance(numpy_data, np.ndarray)
             assert np.array_equal(numpy_data, pd.DataFrame(self.mock_read_value()).to_numpy())
 
-    @pytest.mark.parametrize("pandas_properties", __pandas_properties)
-    def test_read_custom_exposed_type(self, pandas_properties):
-        custom_properties = pandas_properties.copy()
+    @pytest.mark.parametrize("sql_properties", __sql_properties)
+    def test_read_custom_exposed_type(self, sql_properties):
+        custom_properties = sql_properties.copy()
 
         custom_properties.pop("db_extra_args")
         custom_properties["exposed_type"] = MyCustomObject

tests/core/data/test_sql_data_node.py

@@ -17,11 +17,13 @@ import pandas as pd
 import pytest
 from pandas.testing import assert_frame_equal
 
+from taipy.config import Config
 from taipy.config.common.scope import Scope
+from taipy.core.data._data_manager_factory import _DataManagerFactory
 from taipy.core.data.data_node_id import DataNodeId
 from taipy.core.data.operator import JoinOperator, Operator
 from taipy.core.data.sql import SQLDataNode
-from taipy.core.exceptions.exceptions import MissingAppendQueryBuilder, MissingRequiredProperty
+from taipy.core.exceptions.exceptions import MissingAppendQueryBuilder, MissingRequiredProperty, UnknownDatabaseEngine
 
 
 class MyCustomObject:
@@ -36,6 +38,7 @@ def my_write_query_builder_with_pandas(data: pd.DataFrame):
     insert_data = data.to_dict("records")
     return ["DELETE FROM example", ("INSERT INTO example VALUES (:foo, :bar)", insert_data)]
 
+
 def my_append_query_builder_with_pandas(data: pd.DataFrame):
     insert_data = data.to_dict("records")
     return [("INSERT INTO example VALUES (:foo, :bar)", insert_data)]
@@ -46,7 +49,7 @@ def single_write_query_builder(data):
 
 
 class TestSQLDataNode:
-    __pandas_properties = [
+    __sql_properties = [
         {
             "db_name": "taipy.sqlite3",
             "db_engine": "sqlite",
@@ -60,7 +63,7 @@ class TestSQLDataNode:
     ]
 
     if util.find_spec("pyodbc"):
-        __pandas_properties.append(
+        __sql_properties.append(
             {
                 "db_username": "sa",
                 "db_password": "Passw0rd",
@@ -74,9 +77,8 @@ class TestSQLDataNode:
             },
         )
 
-
     if util.find_spec("pymysql"):
-        __pandas_properties.append(
+        __sql_properties.append(
             {
                 "db_username": "sa",
                 "db_password": "Passw0rd",
@@ -90,9 +92,8 @@ class TestSQLDataNode:
             },
         )
 
-
     if util.find_spec("psycopg2"):
-        __pandas_properties.append(
+        __sql_properties.append(
             {
                 "db_username": "sa",
                 "db_password": "Passw0rd",
@@ -106,14 +107,10 @@ class TestSQLDataNode:
             },
         )
 
-
-    @pytest.mark.parametrize("pandas_properties", __pandas_properties)
-    def test_create(self, pandas_properties):
-        dn = SQLDataNode(
-            "foo_bar",
-            Scope.SCENARIO,
-            properties=pandas_properties,
-        )
+    @pytest.mark.parametrize("properties", __sql_properties)
+    def test_create(self, properties):
+        sql_dn_config = Config.configure_sql_data_node(id="foo_bar", **properties)
+        dn = _DataManagerFactory._build_manager()._create_and_set(sql_dn_config, None, None)
         assert isinstance(dn, SQLDataNode)
         assert dn.storage_type() == "sql"
         assert dn.config_id == "foo_bar"
@@ -126,8 +123,18 @@ class TestSQLDataNode:
         assert dn.read_query == "SELECT * FROM example"
         assert dn.write_query_builder == my_write_query_builder_with_pandas
 
+        sql_dn_config_1 = Config.configure_sql_data_node(
+            id="foo",
+            **properties,
+            append_query_builder=my_append_query_builder_with_pandas,
+            exposed_type=MyCustomObject,
+        )
+        dn_1 = _DataManagerFactory._build_manager()._create_and_set(sql_dn_config_1, None, None)
+        assert isinstance(dn, SQLDataNode)
+        assert dn_1.exposed_type == MyCustomObject
+        assert dn_1.append_query_builder == my_append_query_builder_with_pandas
 
-    @pytest.mark.parametrize("properties", __pandas_properties)
+    @pytest.mark.parametrize("properties", __sql_properties)
     def test_get_user_properties(self, properties):
         custom_properties = properties.copy()
         custom_properties["foo"] = "bar"
@@ -142,24 +149,54 @@ class TestSQLDataNode:
         "properties",
         [
             {},
-            {"db_username": "foo"},
-            {"db_username": "foo", "db_password": "foo"},
-            {"db_username": "foo", "db_password": "foo", "db_name": "foo"},
-            {"engine": "sqlite"},
-            {"engine": "mssql", "db_name": "foo"},
-            {"engine": "mysql", "db_username": "foo"},
-            {"engine": "postgresql", "db_username": "foo", "db_password": "foo"},
+            {"read_query": "ready query"},
+            {"read_query": "ready query", "write_query_builder": "write query"},
+            {"read_query": "ready query", "write_query_builder": "write query", "db_username": "foo"},
+            {
+                "read_query": "ready query",
+                "write_query_builder": "write query",
+                "db_username": "foo",
+                "db_password": "foo",
+            },
+            {
+                "read_query": "ready query",
+                "write_query_builder": "write query",
+                "db_username": "foo",
+                "db_password": "foo",
+                "db_name": "foo",
+            },
+            {"read_query": "ready query", "write_query_builder": "write query", "db_engine": "some engine"},
+            {"read_query": "ready query", "write_query_builder": "write query", "db_engine": "sqlite"},
+            {"read_query": "ready query", "write_query_builder": "write query", "db_engine": "mssql", "db_name": "foo"},
+            {
+                "read_query": "ready query",
+                "write_query_builder": "write query",
+                "db_engine": "mysql",
+                "db_username": "foo",
+            },
+            {
+                "read_query": "ready query",
+                "write_query_builder": "write query",
+                "db_engine": "postgresql",
+                "db_username": "foo",
+                "db_password": "foo",
+            },
         ],
     )
     def test_create_with_missing_parameters(self, properties):
         with pytest.raises(MissingRequiredProperty):
             SQLDataNode("foo", Scope.SCENARIO, DataNodeId("dn_id"))
-        with pytest.raises(MissingRequiredProperty):
-            SQLDataNode("foo", Scope.SCENARIO, DataNodeId("dn_id"), properties=properties)
-
-    @pytest.mark.parametrize("pandas_properties", __pandas_properties)
-    def test_write_query_builder(self, pandas_properties):
-        custom_properties = pandas_properties.copy()
+        engine = properties.get("db_engine")
+        if engine is not None and engine not in ["sqlite", "mssql", "mysql", "postgresql"]:
+            with pytest.raises(UnknownDatabaseEngine):
+                SQLDataNode("foo", Scope.SCENARIO, DataNodeId("dn_id"), properties=properties)
+        else:
+            with pytest.raises(MissingRequiredProperty):
+                SQLDataNode("foo", Scope.SCENARIO, DataNodeId("dn_id"), properties=properties)
+
+    @pytest.mark.parametrize("properties", __sql_properties)
+    def test_write_query_builder(self, properties):
+        custom_properties = properties.copy()
         custom_properties.pop("db_extra_args")
         dn = SQLDataNode("foo_bar", Scope.SCENARIO, properties=custom_properties)
         with patch("sqlalchemy.engine.Engine.connect") as engine_mock:
@@ -184,7 +221,6 @@ class TestSQLDataNode:
             assert len(engine_mock.mock_calls[4].args) == 1
             assert engine_mock.mock_calls[4].args[0].text == "DELETE FROM example"
 
-
     @pytest.mark.parametrize(
         "tmp_sqlite_path",
         [

tests/core/data/test_sql_table_data_node.py

@@ -14,7 +14,9 @@ from unittest.mock import patch
 
 import pytest
 
+from taipy.config import Config
 from taipy.config.common.scope import Scope
+from taipy.core.data._data_manager_factory import _DataManagerFactory
 from taipy.core.data.data_node_id import DataNodeId
 from taipy.core.data.sql_table import SQLTableDataNode
 from taipy.core.exceptions.exceptions import InvalidExposedType, MissingRequiredProperty
@@ -29,7 +31,7 @@ class MyCustomObject:
 
 
 class TestSQLTableDataNode:
-    __pandas_properties = [
+    __sql_properties = [
         {
             "db_name": "taipy",
             "db_engine": "sqlite",
@@ -42,7 +44,7 @@ class TestSQLTableDataNode:
     ]
 
     if util.find_spec("pyodbc"):
-        __pandas_properties.append(
+        __sql_properties.append(
             {
                 "db_username": "sa",
                 "db_password": "Passw0rd",
@@ -56,7 +58,7 @@ class TestSQLTableDataNode:
         )
 
     if util.find_spec("pymysql"):
-        __pandas_properties.append(
+        __sql_properties.append(
             {
                 "db_username": "sa",
                 "db_password": "Passw0rd",
@@ -70,7 +72,7 @@ class TestSQLTableDataNode:
         )
 
     if util.find_spec("psycopg2"):
-        __pandas_properties.append(
+        __sql_properties.append(
             {
                 "db_username": "sa",
                 "db_password": "Passw0rd",
@@ -83,13 +85,10 @@ class TestSQLTableDataNode:
             },
         )
 
-    @pytest.mark.parametrize("pandas_properties", __pandas_properties)
-    def test_create(self, pandas_properties):
-        dn = SQLTableDataNode(
-            "foo_bar",
-            Scope.SCENARIO,
-            properties=pandas_properties,
-        )
+    @pytest.mark.parametrize("properties", __sql_properties)
+    def test_create(self, properties):
+        sql_table_dn_config = Config.configure_sql_table_data_node("foo_bar", **properties)
+        dn = _DataManagerFactory._build_manager()._create_and_set(sql_table_dn_config, None, None)
         assert isinstance(dn, SQLTableDataNode)
         assert dn.storage_type() == "sql_table"
         assert dn.config_id == "foo_bar"
@@ -102,7 +101,14 @@ class TestSQLTableDataNode:
         assert dn.table_name == "example"
         assert dn._get_base_read_query() == "SELECT * FROM example"
 
-    @pytest.mark.parametrize("properties", __pandas_properties)
+        sql_table_dn_config_1 = Config.configure_sql_table_data_node(
+            "foo_bar", **properties, exposed_type=MyCustomObject
+        )
+        dn_1 = _DataManagerFactory._build_manager()._create_and_set(sql_table_dn_config_1, None, None)
+        assert isinstance(dn_1, SQLTableDataNode)
+        assert dn_1.exposed_type == MyCustomObject
+
+    @pytest.mark.parametrize("properties", __sql_properties)
     def test_get_user_properties(self, properties):
         custom_properties = properties.copy()
         custom_properties["foo"] = "bar"
@@ -129,28 +135,28 @@ class TestSQLTableDataNode:
             SQLTableDataNode("foo", Scope.SCENARIO, DataNodeId("dn_id"), properties=properties)
 
     @patch("taipy.core.data.sql_table.SQLTableDataNode._read_as_pandas_dataframe", return_value="pandas")
-    @pytest.mark.parametrize("pandas_properties", __pandas_properties)
-    def test_modin_deprecated_in_favor_of_pandas(self, mock_read_as_pandas_dataframe, pandas_properties):
-        pandas_properties["exposed_type"] = "modin"
-        sql_data_node_as_modin = SQLTableDataNode("foo", Scope.SCENARIO, properties=pandas_properties)
+    @pytest.mark.parametrize("properties", __sql_properties)
+    def test_modin_deprecated_in_favor_of_pandas(self, mock_read_as_pandas_dataframe, properties):
+        properties["exposed_type"] = "modin"
+        sql_data_node_as_modin = SQLTableDataNode("foo", Scope.SCENARIO, properties=properties)
         assert sql_data_node_as_modin.properties["exposed_type"] == "pandas"
         assert sql_data_node_as_modin.read() == "pandas"
 
-    @pytest.mark.parametrize("pandas_properties", __pandas_properties)
-    def test_raise_error_invalid_exposed_type(self, pandas_properties):
-        custom_properties = pandas_properties.copy()
+    @pytest.mark.parametrize("properties", __sql_properties)
+    def test_raise_error_invalid_exposed_type(self, properties):
+        custom_properties = properties.copy()
         custom_properties.pop("db_extra_args")
         custom_properties["exposed_type"] = "foo"
         with pytest.raises(InvalidExposedType):
             SQLTableDataNode("foo", Scope.SCENARIO, properties=custom_properties)
 
-    @pytest.mark.parametrize("pandas_properties", __pandas_properties)
+    @pytest.mark.parametrize("properties", __sql_properties)
     @patch("pandas.read_sql_query")
-    def test_engine_cache(self, _, pandas_properties):
+    def test_engine_cache(self, _, properties):
         dn = SQLTableDataNode(
             "foo",
             Scope.SCENARIO,
-            properties=pandas_properties,
+            properties=properties,
         )
 
         assert dn._engine is None

tests/core/data/test_write_sql_table_data_node.py

@@ -28,7 +28,7 @@ class MyCustomObject:
 
 
 class TestWriteSQLTableDataNode:
-    __pandas_properties = [
+    __sql_properties = [
         {
             "db_name": "taipy",
             "db_engine": "sqlite",
@@ -41,7 +41,7 @@ class TestWriteSQLTableDataNode:
     ]
 
     if util.find_spec("pyodbc"):
-        __pandas_properties.append(
+        __sql_properties.append(
             {
                 "db_username": "sa",
                 "db_password": "Passw0rd",
@@ -55,7 +55,7 @@ class TestWriteSQLTableDataNode:
         )
 
     if util.find_spec("pymysql"):
-        __pandas_properties.append(
+        __sql_properties.append(
             {
                 "db_username": "sa",
                 "db_password": "Passw0rd",
@@ -69,7 +69,7 @@ class TestWriteSQLTableDataNode:
         )
 
     if util.find_spec("psycopg2"):
-        __pandas_properties.append(
+        __sql_properties.append(
             {
                 "db_username": "sa",
                 "db_password": "Passw0rd",
@@ -82,9 +82,9 @@ class TestWriteSQLTableDataNode:
             },
         )
 
-    @pytest.mark.parametrize("pandas_properties", __pandas_properties)
-    def test_write_pandas(self, pandas_properties):
-        custom_properties = pandas_properties.copy()
+    @pytest.mark.parametrize("properties", __sql_properties)
+    def test_write_pandas(self, properties):
+        custom_properties = properties.copy()
         custom_properties.pop("db_extra_args")
         sql_table_dn = SQLTableDataNode("foo", Scope.SCENARIO, properties=custom_properties)
 
@@ -94,7 +94,7 @@ class TestWriteSQLTableDataNode:
             cursor_mock = engine_mock.return_value.__enter__.return_value
             cursor_mock.execute.side_effect = None
 
-            with patch("taipy.core.data.sql_table.SQLTableDataNode._SQLTableDataNode__insert_dataframe") as mck:
+            with patch("taipy.core.data.sql_table.SQLTableDataNode._insert_dataframe") as mck:
                 df = pd.DataFrame([{"a": 11, "b": 22, "c": 33}, {"a": 44, "b": 55, "c": 66}])
                 sql_table_dn.write(df)
                 assert mck.call_count == 1
@@ -112,9 +112,9 @@ class TestWriteSQLTableDataNode:
                 sql_table_dn.write(None)
                 assert mck.call_count == 5
 
-    @pytest.mark.parametrize("pandas_properties", __pandas_properties)
-    def test_write_numpy(self, pandas_properties):
-        custom_properties = pandas_properties.copy()
+    @pytest.mark.parametrize("properties", __sql_properties)
+    def test_write_numpy(self, properties):
+        custom_properties = properties.copy()
         custom_properties["exposed_type"] = "numpy"
         custom_properties.pop("db_extra_args")
         sql_table_dn = SQLTableDataNode("foo", Scope.SCENARIO, properties=custom_properties)
@@ -125,7 +125,7 @@ class TestWriteSQLTableDataNode:
             cursor_mock = engine_mock.return_value.__enter__.return_value
             cursor_mock.execute.side_effect = None
 
-            with patch("taipy.core.data.sql_table.SQLTableDataNode._SQLTableDataNode__insert_dataframe") as mck:
+            with patch("taipy.core.data.sql_table.SQLTableDataNode._insert_dataframe") as mck:
                 arr = np.array([[1], [2], [3], [4], [5]])
                 sql_table_dn.write(arr)
                 assert mck.call_count == 1
@@ -139,9 +139,9 @@ class TestWriteSQLTableDataNode:
                 sql_table_dn.write(None)
                 assert mck.call_count == 4
 
-    @pytest.mark.parametrize("pandas_properties", __pandas_properties)
-    def test_write_custom_exposed_type(self, pandas_properties):
-        custom_properties = pandas_properties.copy()
+    @pytest.mark.parametrize("properties", __sql_properties)
+    def test_write_custom_exposed_type(self, properties):
+        custom_properties = properties.copy()
         custom_properties["exposed_type"] = MyCustomObject
         custom_properties.pop("db_extra_args")
         sql_table_dn = SQLTableDataNode("foo", Scope.SCENARIO, properties=custom_properties)
@@ -152,7 +152,7 @@ class TestWriteSQLTableDataNode:
             cursor_mock = engine_mock.return_value.__enter__.return_value
             cursor_mock.execute.side_effect = None
 
-            with patch("taipy.core.data.sql_table.SQLTableDataNode._SQLTableDataNode__insert_dataframe") as mck:
+            with patch("taipy.core.data.sql_table.SQLTableDataNode._insert_dataframe") as mck:
                 custom_data = [
                     MyCustomObject(1, 2),
                     MyCustomObject(3, 4),