test_data_node.py 37 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964
  1. # Copyright 2021-2025 Avaiga Private Limited
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
  4. # the License. You may obtain a copy of the License at
  5. #
  6. # http://www.apache.org/licenses/LICENSE-2.0
  7. #
  8. # Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
  9. # an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
  10. # specific language governing permissions and limitations under the License.
  11. import os
  12. from datetime import datetime, timedelta
  13. from time import sleep
  14. from unittest import mock
  15. import freezegun
  16. import pandas as pd
  17. import pytest
  18. import taipy.core as tp
  19. from taipy import Scope
  20. from taipy.common.config import Config
  21. from taipy.common.config.exceptions.exceptions import InvalidConfigurationId
  22. from taipy.core.data._data_manager import _DataManager
  23. from taipy.core.data._data_manager_factory import _DataManagerFactory
  24. from taipy.core.data.data_node import DataNode
  25. from taipy.core.data.data_node_id import (
  26. EDIT_COMMENT_KEY,
  27. EDIT_EDITOR_ID_KEY,
  28. EDIT_JOB_ID_KEY,
  29. EDIT_TIMESTAMP_KEY,
  30. DataNodeId,
  31. )
  32. from taipy.core.data.in_memory import InMemoryDataNode
  33. from taipy.core.exceptions.exceptions import DataNodeIsBeingEdited, NoData
  34. from taipy.core.job.job_id import JobId
  35. from taipy.core.task.task import Task
  36. from .utils import FakeDataNode
  37. def funct_a_b(input: str):
  38. print("task_a_b") # noqa: T201
  39. return "B"
  40. def funct_b_c(input: str):
  41. print("task_b_c") # noqa: T201
  42. return "C"
  43. def funct_b_d(input: str):
  44. print("task_b_d") # noqa: T201
  45. return "D"
  46. class TestDataNode:
  47. def test_dn_equals(self, data_node):
  48. data_manager = _DataManagerFactory()._build_manager()
  49. dn_id = data_node.id
  50. data_manager._repository._save(data_node)
  51. # # To test if instance is same type
  52. task = Task("task", {}, print, [], [], dn_id)
  53. dn_2 = data_manager._get(dn_id)
  54. assert data_node == dn_2
  55. assert data_node != dn_id
  56. assert data_node != task
  57. def test_create_with_default_values(self):
  58. dn = DataNode("foo_bar")
  59. assert dn.config_id == "foo_bar"
  60. assert dn.scope == Scope.SCENARIO
  61. assert dn.id is not None
  62. assert dn.name is None
  63. assert dn.owner_id is None
  64. assert dn.parent_ids == set()
  65. assert dn.last_edit_date is None
  66. assert dn.job_ids == []
  67. assert not dn.is_ready_for_reading
  68. assert len(dn.properties) == 0
  69. def test_create_with_ranks(self):
  70. # Test _rank is propagated from the config
  71. cfg = Config.configure_data_node("foo_bar")
  72. cfg._ranks = {"A": 1, "B": 2, "C": 0}
  73. dn = DataNode("foo_bar")
  74. assert dn.config_id == "foo_bar"
  75. assert dn.scope == Scope.SCENARIO
  76. assert dn.id is not None
  77. assert dn.name is None
  78. assert dn.owner_id is None
  79. assert dn.parent_ids == set()
  80. assert dn.last_edit_date is None
  81. assert dn.job_ids == []
  82. assert not dn.is_ready_for_reading
  83. assert len(dn.properties) == 0
  84. assert dn._get_rank("A") == 1
  85. assert dn._get_rank("B") == 2
  86. assert dn._get_rank("C") == 0
  87. def test_is_up_to_date_when_not_written(self):
  88. dn_confg_1 = Config.configure_in_memory_data_node("dn_1", default_data="a")
  89. dn_confg_2 = Config.configure_in_memory_data_node("dn_2")
  90. task_config_1 = Config.configure_task("t1", funct_a_b, [dn_confg_1], [dn_confg_2])
  91. scenario_config = Config.configure_scenario("sc", [task_config_1])
  92. scenario = tp.create_scenario(scenario_config)
  93. assert scenario.dn_1.is_up_to_date is True
  94. assert scenario.dn_2.is_up_to_date is False
  95. tp.submit(scenario)
  96. assert scenario.dn_1.is_up_to_date is True
  97. assert scenario.dn_2.is_up_to_date is True
  98. def test_create(self):
  99. a_date = datetime.now()
  100. dn = DataNode(
  101. "foo_bar",
  102. Scope.SCENARIO,
  103. DataNodeId("an_id"),
  104. "a_scenario_id",
  105. {"a_parent_id"},
  106. a_date,
  107. [{"job_id": "a_job_id"}],
  108. edit_in_progress=False,
  109. prop="erty",
  110. name="a name",
  111. )
  112. assert dn.config_id == "foo_bar"
  113. assert dn.scope == Scope.SCENARIO
  114. assert dn.id == "an_id"
  115. assert dn.name == "a name"
  116. assert dn.owner_id == "a_scenario_id"
  117. assert dn.parent_ids == {"a_parent_id"}
  118. assert dn.last_edit_date == a_date
  119. assert dn.job_ids == ["a_job_id"]
  120. assert dn.is_ready_for_reading
  121. assert len(dn.properties) == 2
  122. assert dn.properties == {"prop": "erty", "name": "a name"}
  123. with pytest.raises(InvalidConfigurationId):
  124. DataNode("foo bar")
  125. def test_read_write(self):
  126. dn = FakeDataNode("foo_bar")
  127. _DataManagerFactory._build_manager()._repository._save(dn)
  128. with pytest.raises(NoData):
  129. assert dn.read() is None
  130. dn.read_or_raise()
  131. assert dn.write_has_been_called == 0
  132. assert dn.read_has_been_called == 0
  133. assert not dn.is_ready_for_reading
  134. assert dn.last_edit_date is None
  135. assert dn.job_ids == []
  136. assert dn.edits == []
  137. dn.write("Any data")
  138. assert dn.write_has_been_called == 1
  139. assert dn.read_has_been_called == 0
  140. assert dn.last_edit_date is not None
  141. first_edition = dn.last_edit_date
  142. assert dn.is_ready_for_reading
  143. assert dn.job_ids == []
  144. assert len(dn.edits) == 1
  145. assert dn.get_last_edit()["timestamp"] == dn.last_edit_date
  146. sleep(0.1)
  147. dn.write("Any other data", job_id := JobId("a_job_id"))
  148. assert dn.write_has_been_called == 2
  149. assert dn.read_has_been_called == 0
  150. second_edition = dn.last_edit_date
  151. assert first_edition < second_edition
  152. assert dn.is_ready_for_reading
  153. assert dn.job_ids == [job_id]
  154. assert len(dn.edits) == 2
  155. assert dn.get_last_edit()["timestamp"] == dn.last_edit_date
  156. dn.read()
  157. assert dn.write_has_been_called == 2
  158. assert dn.read_has_been_called == 1
  159. second_edition = dn.last_edit_date
  160. assert first_edition < second_edition
  161. assert dn.is_ready_for_reading
  162. assert dn.job_ids == [job_id]
  163. def test_lock_initialization(self):
  164. dn = InMemoryDataNode("dn", Scope.SCENARIO)
  165. assert not dn.edit_in_progress
  166. assert dn._editor_id is None
  167. assert dn._editor_expiration_date is None
  168. def test_locked_dn_unlockable_only_by_same_editor(self):
  169. dn = InMemoryDataNode("dn", Scope.SCENARIO)
  170. _DataManagerFactory._build_manager()._repository._save(dn)
  171. dn.lock_edit("user_1")
  172. assert dn.edit_in_progress
  173. assert dn._editor_id == "user_1"
  174. assert dn._editor_expiration_date is not None
  175. with pytest.raises(DataNodeIsBeingEdited):
  176. dn.lock_edit("user_2")
  177. with pytest.raises(DataNodeIsBeingEdited):
  178. dn.unlock_edit("user_2")
  179. dn.unlock_edit("user_1")
  180. assert not dn.edit_in_progress
  181. assert dn._editor_id is None
  182. assert dn._editor_expiration_date is None
  183. def test_none_editor_can_lock_a_locked_dn(self):
  184. dn = InMemoryDataNode("dn", Scope.SCENARIO)
  185. _DataManagerFactory._build_manager()._repository._save(dn)
  186. dn.lock_edit("user")
  187. assert dn.edit_in_progress
  188. assert dn._editor_id == "user"
  189. assert dn._editor_expiration_date is not None
  190. dn.lock_edit()
  191. assert dn.edit_in_progress
  192. assert dn._editor_id is None
  193. assert dn._editor_expiration_date is None
  194. def test_none_editor_can_unlock_a_locked_dn(self):
  195. dn = InMemoryDataNode("dn", Scope.SCENARIO)
  196. _DataManagerFactory._build_manager()._repository._save(dn)
  197. dn.lock_edit("user")
  198. assert dn.edit_in_progress
  199. assert dn._editor_id == "user"
  200. assert dn._editor_expiration_date is not None
  201. dn.unlock_edit()
  202. assert not dn.edit_in_progress
  203. assert dn._editor_id is None
  204. assert dn._editor_expiration_date is None
  205. dn.lock_edit()
  206. assert dn.edit_in_progress
  207. assert dn._editor_id is None
  208. assert dn._editor_expiration_date is None
  209. dn.unlock_edit()
  210. assert not dn.edit_in_progress
  211. assert dn._editor_id is None
  212. assert dn._editor_expiration_date is None
  213. def test_ready_for_reading(self):
  214. dn = InMemoryDataNode("foo_bar", Scope.CYCLE)
  215. _DataManagerFactory._build_manager()._repository._save(dn)
  216. assert dn.last_edit_date is None
  217. assert not dn.is_ready_for_reading
  218. assert dn.job_ids == []
  219. dn.lock_edit()
  220. assert dn.last_edit_date is None
  221. assert not dn.is_ready_for_reading
  222. assert dn.job_ids == []
  223. dn.unlock_edit()
  224. assert dn.last_edit_date is None
  225. assert not dn.is_ready_for_reading
  226. assert dn.job_ids == []
  227. dn.lock_edit()
  228. assert dn.last_edit_date is None
  229. assert not dn.is_ready_for_reading
  230. assert dn.job_ids == []
  231. dn.write("toto", job_id := JobId("a_job_id"))
  232. assert dn.last_edit_date is not None
  233. assert dn.is_ready_for_reading
  234. assert dn.job_ids == [job_id]
  235. def test_is_valid_no_validity_period(self):
  236. dn = InMemoryDataNode("foo", Scope.SCENARIO, DataNodeId("id"), "name", "owner_id")
  237. _DataManagerFactory._build_manager()._repository._save(dn)
  238. # Test Never been written
  239. assert not dn.is_valid
  240. # test has been written
  241. dn.write("My data")
  242. assert dn.is_valid
  243. def test_is_valid_with_30_min_validity_period(self):
  244. dn = InMemoryDataNode(
  245. "foo", Scope.SCENARIO, DataNodeId("id"), "name", "owner_id", validity_period=timedelta(minutes=30)
  246. )
  247. _DataManagerFactory._build_manager()._repository._save(dn)
  248. # Test Never been written
  249. assert dn.is_valid is False
  250. # Has been written less than 30 minutes ago
  251. dn.write("My data")
  252. assert dn.is_valid is True
  253. # Has been written more than 30 minutes ago
  254. dn.last_edit_date = datetime.now() + timedelta(days=-1)
  255. assert dn.is_valid is False
  256. def test_is_valid_with_5_days_validity_period(self):
  257. dn = InMemoryDataNode("foo", Scope.SCENARIO, validity_period=timedelta(days=5))
  258. _DataManagerFactory._build_manager()._repository._save(dn)
  259. # Test Never been written
  260. assert dn.is_valid is False
  261. # Has been written less than 30 minutes ago
  262. dn.write("My data")
  263. assert dn.is_valid is True
  264. # Has been written more than 30 minutes ago
  265. dn._last_edit_date = datetime.now() - timedelta(days=6)
  266. _DataManager()._repository._save(dn)
  267. assert dn.is_valid is False
  268. def test_is_up_to_date(self, current_datetime):
  269. dn_confg_1 = Config.configure_in_memory_data_node("dn_1")
  270. dn_confg_2 = Config.configure_in_memory_data_node("dn_2")
  271. dn_confg_3 = Config.configure_in_memory_data_node("dn_3", scope=Scope.GLOBAL)
  272. task_config_1 = Config.configure_task("t1", print, [dn_confg_1], [dn_confg_2])
  273. task_config_2 = Config.configure_task("t2", print, [dn_confg_2], [dn_confg_3])
  274. scenario_config = Config.configure_scenario("sc", [task_config_1, task_config_2])
  275. scenario_1 = tp.create_scenario(scenario_config)
  276. assert len(_DataManager._get_all()) == 3
  277. dn_1_1 = scenario_1.data_nodes["dn_1"]
  278. dn_2_1 = scenario_1.data_nodes["dn_2"]
  279. dn_3_1 = scenario_1.data_nodes["dn_3"]
  280. assert dn_1_1.last_edit_date is None
  281. assert dn_2_1.last_edit_date is None
  282. assert dn_3_1.last_edit_date is None
  283. dn_1_1.last_edit_date = current_datetime + timedelta(1)
  284. dn_2_1.last_edit_date = current_datetime + timedelta(2)
  285. dn_3_1.last_edit_date = current_datetime + timedelta(3)
  286. assert dn_1_1.is_up_to_date
  287. assert dn_2_1.is_up_to_date
  288. assert dn_3_1.is_up_to_date
  289. dn_2_1.last_edit_date = current_datetime + timedelta(4)
  290. assert dn_1_1.is_up_to_date
  291. assert dn_2_1.is_up_to_date
  292. assert not dn_3_1.is_up_to_date
  293. dn_1_1.last_edit_date = current_datetime + timedelta(5)
  294. assert dn_1_1.is_up_to_date
  295. assert not dn_2_1.is_up_to_date
  296. assert not dn_3_1.is_up_to_date
  297. dn_1_1.last_edit_date = current_datetime + timedelta(1)
  298. dn_2_1.last_edit_date = current_datetime + timedelta(2)
  299. dn_3_1.last_edit_date = current_datetime + timedelta(3)
  300. def test_is_up_to_date_across_scenarios(self, current_datetime):
  301. dn_confg_1 = Config.configure_in_memory_data_node("dn_1", scope=Scope.SCENARIO)
  302. dn_confg_2 = Config.configure_in_memory_data_node("dn_2", scope=Scope.SCENARIO)
  303. dn_confg_3 = Config.configure_in_memory_data_node("dn_3", scope=Scope.GLOBAL)
  304. task_config_1 = Config.configure_task("t1", print, [dn_confg_1], [dn_confg_2])
  305. task_config_2 = Config.configure_task("t2", print, [dn_confg_2], [dn_confg_3])
  306. scenario_config = Config.configure_scenario("sc", [task_config_1, task_config_2])
  307. scenario_1 = tp.create_scenario(scenario_config)
  308. scenario_2 = tp.create_scenario(scenario_config)
  309. assert len(_DataManager._get_all()) == 5
  310. dn_1_1 = scenario_1.data_nodes["dn_1"]
  311. dn_2_1 = scenario_1.data_nodes["dn_2"]
  312. dn_1_2 = scenario_2.data_nodes["dn_1"]
  313. dn_2_2 = scenario_2.data_nodes["dn_2"]
  314. dn_3 = scenario_1.data_nodes["dn_3"]
  315. assert dn_3 == scenario_2.data_nodes["dn_3"]
  316. assert dn_1_1.last_edit_date is None
  317. assert dn_2_1.last_edit_date is None
  318. assert dn_1_2.last_edit_date is None
  319. assert dn_2_2.last_edit_date is None
  320. assert dn_3.last_edit_date is None
  321. dn_1_1.last_edit_date = current_datetime + timedelta(1)
  322. dn_2_1.last_edit_date = current_datetime + timedelta(2)
  323. dn_1_2.last_edit_date = current_datetime + timedelta(3)
  324. dn_2_2.last_edit_date = current_datetime + timedelta(4)
  325. dn_3.last_edit_date = current_datetime + timedelta(5)
  326. assert dn_1_1.is_up_to_date
  327. assert dn_2_1.is_up_to_date
  328. assert dn_1_2.is_up_to_date
  329. assert dn_2_2.is_up_to_date
  330. assert dn_3.is_up_to_date
  331. dn_2_1.last_edit_date = current_datetime + timedelta(6)
  332. assert dn_1_1.is_up_to_date
  333. assert dn_2_1.is_up_to_date
  334. assert dn_1_2.is_up_to_date
  335. assert dn_2_2.is_up_to_date
  336. assert not dn_3.is_up_to_date
  337. dn_2_1.last_edit_date = current_datetime + timedelta(2)
  338. dn_2_2.last_edit_date = current_datetime + timedelta(6)
  339. assert dn_1_1.is_up_to_date
  340. assert dn_2_1.is_up_to_date
  341. assert dn_1_2.is_up_to_date
  342. assert dn_2_2.is_up_to_date
  343. assert not dn_3.is_up_to_date
  344. dn_2_2.last_edit_date = current_datetime + timedelta(4)
  345. dn_1_1.last_edit_date = current_datetime + timedelta(6)
  346. assert dn_1_1.is_up_to_date
  347. assert not dn_2_1.is_up_to_date
  348. assert dn_1_2.is_up_to_date
  349. assert dn_2_2.is_up_to_date
  350. assert not dn_3.is_up_to_date
  351. dn_1_2.last_edit_date = current_datetime + timedelta(6)
  352. assert dn_1_1.is_up_to_date
  353. assert not dn_2_1.is_up_to_date
  354. assert dn_1_2.is_up_to_date
  355. assert not dn_2_2.is_up_to_date
  356. assert not dn_3.is_up_to_date
  357. def test_do_not_recompute_data_node_valid_but_continue_sequence_execution(self):
  358. a = Config.configure_data_node("A", "pickle", default_data="A")
  359. b = Config.configure_data_node("B", "pickle")
  360. c = Config.configure_data_node("C", "pickle")
  361. d = Config.configure_data_node("D", "pickle")
  362. task_a_b = Config.configure_task("task_a_b", funct_a_b, input=a, output=b, skippable=True)
  363. task_b_c = Config.configure_task("task_b_c", funct_b_c, input=b, output=c)
  364. task_b_d = Config.configure_task("task_b_d", funct_b_d, input=b, output=d)
  365. scenario_cfg = Config.configure_scenario("scenario", [task_a_b, task_b_c, task_b_d])
  366. scenario = tp.create_scenario(scenario_cfg)
  367. scenario.submit()
  368. assert scenario.A.read() == "A"
  369. assert scenario.B.read() == "B"
  370. assert scenario.C.read() == "C"
  371. assert scenario.D.read() == "D"
  372. scenario.submit()
  373. assert len(tp.get_jobs()) == 6
  374. jobs_and_status = [(job.task.config_id, job.status) for job in tp.get_jobs()]
  375. assert ("task_a_b", tp.Status.COMPLETED) in jobs_and_status
  376. assert ("task_a_b", tp.Status.SKIPPED) in jobs_and_status
  377. assert ("task_b_c", tp.Status.COMPLETED) in jobs_and_status
  378. assert ("task_b_d", tp.Status.COMPLETED) in jobs_and_status
  379. def test_data_node_update_after_writing(self):
  380. dn = FakeDataNode("foo")
  381. _DataManager._repository._save(dn)
  382. assert not _DataManager._get(dn.id).is_ready_for_reading
  383. dn.write("Any data")
  384. assert dn.is_ready_for_reading
  385. assert _DataManager._get(dn.id).is_ready_for_reading
  386. def test_expiration_date_raise_if_never_write(self):
  387. dn = FakeDataNode("foo")
  388. with pytest.raises(NoData):
  389. _ = dn.expiration_date
  390. def test_validity_null_if_never_write(self):
  391. dn = FakeDataNode("foo")
  392. assert dn.validity_period is None
  393. def test_auto_update_and_reload(self, current_datetime):
  394. dn_1 = InMemoryDataNode(
  395. "foo",
  396. scope=Scope.GLOBAL,
  397. id=DataNodeId("an_id"),
  398. owner_id=None,
  399. parent_ids=None,
  400. last_edit_date=current_datetime,
  401. edits=[{"job_id": "a_job_id"}],
  402. edit_in_progress=False,
  403. validity_period=None,
  404. properties={
  405. "name": "foo",
  406. },
  407. )
  408. dm = _DataManager()
  409. dm._repository._save(dn_1)
  410. dn_2 = dm._get(dn_1)
  411. # auto set & reload on scope attribute
  412. assert dn_1.scope == Scope.GLOBAL
  413. assert dn_2.scope == Scope.GLOBAL
  414. dn_1.scope = Scope.CYCLE
  415. assert dn_1.scope == Scope.CYCLE
  416. assert dn_2.scope == Scope.CYCLE
  417. dn_2.scope = Scope.SCENARIO
  418. assert dn_1.scope == Scope.SCENARIO
  419. assert dn_2.scope == Scope.SCENARIO
  420. new_datetime = current_datetime + timedelta(1)
  421. new_datetime_1 = current_datetime + timedelta(3)
  422. # auto set & reload on last_edit_date attribute
  423. assert dn_1.last_edit_date == current_datetime
  424. assert dn_2.last_edit_date == current_datetime
  425. dn_1.last_edit_date = new_datetime_1
  426. assert dn_1.last_edit_date == new_datetime_1
  427. assert dn_2.last_edit_date == new_datetime_1
  428. dn_2.last_edit_date = new_datetime
  429. assert dn_1.last_edit_date == new_datetime
  430. assert dn_2.last_edit_date == new_datetime
  431. # auto set & reload on name attribute
  432. assert dn_1.name == "foo"
  433. assert dn_2.name == "foo"
  434. dn_1.name = "fed"
  435. assert dn_1.name == "fed"
  436. assert dn_2.name == "fed"
  437. dn_2.name = "def"
  438. assert dn_1.name == "def"
  439. assert dn_2.name == "def"
  440. # auto set & reload on parent_ids attribute (set() object does not have auto set yet)
  441. assert dn_1.parent_ids == set()
  442. assert dn_2.parent_ids == set()
  443. dn_1._parent_ids.update(["sc2"])
  444. _DataManager._update(dn_1)
  445. assert dn_1.parent_ids == {"sc2"}
  446. assert dn_2.parent_ids == {"sc2"}
  447. dn_2._parent_ids.clear()
  448. dn_2._parent_ids.update(["sc1"])
  449. _DataManager._update(dn_2)
  450. assert dn_1.parent_ids == {"sc1"}
  451. assert dn_2.parent_ids == {"sc1"}
  452. dn_2._parent_ids.clear()
  453. _DataManager._update(dn_2)
  454. # auto set & reload on edit_in_progress attribute
  455. assert not dn_2.edit_in_progress
  456. assert not dn_1.edit_in_progress
  457. dn_1.edit_in_progress = True
  458. assert dn_1.edit_in_progress
  459. assert dn_2.edit_in_progress
  460. dn_2.unlock_edit()
  461. assert not dn_1.edit_in_progress
  462. assert not dn_2.edit_in_progress
  463. dn_1.lock_edit()
  464. assert dn_1.edit_in_progress
  465. assert dn_2.edit_in_progress
  466. # auto set & reload on validity_period attribute
  467. time_period_1 = timedelta(1)
  468. time_period_2 = timedelta(5)
  469. assert dn_1.validity_period is None
  470. assert dn_2.validity_period is None
  471. dn_1.validity_period = time_period_1
  472. assert dn_1.validity_period == time_period_1
  473. assert dn_2.validity_period == time_period_1
  474. dn_2.validity_period = time_period_2
  475. assert dn_1.validity_period == time_period_2
  476. assert dn_2.validity_period == time_period_2
  477. dn_1.last_edit_date = new_datetime
  478. assert len(dn_1.job_ids) == 1
  479. assert len(dn_2.job_ids) == 1
  480. with dn_1 as dn:
  481. assert dn.config_id == "foo"
  482. assert dn.owner_id is None
  483. assert dn.scope == Scope.SCENARIO
  484. assert dn.last_edit_date == new_datetime
  485. assert dn.name == "def"
  486. assert dn.edit_in_progress
  487. assert dn.validity_period == time_period_2
  488. assert len(dn.job_ids) == 1
  489. assert dn._is_in_context
  490. new_datetime_2 = new_datetime + timedelta(5)
  491. dn.scope = Scope.CYCLE
  492. dn.last_edit_date = new_datetime_2
  493. dn.name = "abc"
  494. dn.edit_in_progress = False
  495. dn.validity_period = None
  496. assert dn.config_id == "foo"
  497. assert dn.owner_id is None
  498. assert dn.scope == Scope.SCENARIO
  499. assert dn.last_edit_date == new_datetime
  500. assert dn.name == "def"
  501. assert dn.edit_in_progress
  502. assert dn.validity_period == time_period_2
  503. assert len(dn.job_ids) == 1
  504. assert dn_1.config_id == "foo"
  505. assert dn_1.owner_id is None
  506. assert dn_1.scope == Scope.CYCLE
  507. assert dn_1.last_edit_date == new_datetime_2
  508. assert dn_1.name == "abc"
  509. assert not dn_1.edit_in_progress
  510. assert dn_1.validity_period is None
  511. assert not dn_1._is_in_context
  512. assert len(dn_1.job_ids) == 1
  513. def test_auto_update_and_reload_properties(self):
  514. dn_1 = InMemoryDataNode("foo", scope=Scope.GLOBAL, properties={"name": "def"})
  515. dm = _DataManager()
  516. dm._repository._save(dn_1)
  517. dn_2 = dm._get(dn_1)
  518. # auto set & reload on properties attribute
  519. assert dn_1.properties == {"name": "def"}
  520. assert dn_2.properties == {"name": "def"}
  521. dn_1._properties["qux"] = 4
  522. assert dn_1.properties["qux"] == 4
  523. assert dn_2.properties["qux"] == 4
  524. assert dn_1.properties == {"qux": 4, "name": "def"}
  525. assert dn_2.properties == {"qux": 4, "name": "def"}
  526. dn_2._properties["qux"] = 5
  527. assert dn_1.properties["qux"] == 5
  528. assert dn_2.properties["qux"] == 5
  529. dn_1.properties["temp_key_1"] = "temp_value_1"
  530. dn_1.properties["temp_key_2"] = "temp_value_2"
  531. assert dn_1.properties == {
  532. "name": "def",
  533. "qux": 5,
  534. "temp_key_1": "temp_value_1",
  535. "temp_key_2": "temp_value_2",
  536. }
  537. assert dn_2.properties == {"name": "def", "qux": 5, "temp_key_1": "temp_value_1", "temp_key_2": "temp_value_2"}
  538. dn_1.properties.pop("temp_key_1")
  539. assert "temp_key_1" not in dn_1.properties.keys()
  540. assert "temp_key_1" not in dn_1.properties.keys()
  541. assert dn_1.properties == {"name": "def", "qux": 5, "temp_key_2": "temp_value_2"}
  542. assert dn_2.properties == {"name": "def", "qux": 5, "temp_key_2": "temp_value_2"}
  543. dn_2.properties.pop("temp_key_2")
  544. assert dn_1.properties == {"qux": 5, "name": "def"}
  545. assert dn_2.properties == {"qux": 5, "name": "def"}
  546. assert "temp_key_2" not in dn_1.properties.keys()
  547. assert "temp_key_2" not in dn_2.properties.keys()
  548. dn_1.properties["temp_key_3"] = 0
  549. assert dn_1.properties == {"qux": 5, "temp_key_3": 0, "name": "def"}
  550. assert dn_2.properties == {"qux": 5, "temp_key_3": 0, "name": "def"}
  551. dn_1.properties.update({"temp_key_3": 1})
  552. assert dn_1.properties == {"qux": 5, "temp_key_3": 1, "name": "def"}
  553. assert dn_2.properties == {"qux": 5, "temp_key_3": 1, "name": "def"}
  554. dn_1.properties.update({})
  555. assert dn_1.properties == {"qux": 5, "temp_key_3": 1, "name": "def"}
  556. assert dn_2.properties == {"qux": 5, "temp_key_3": 1, "name": "def"}
  557. dn_1.properties["temp_key_4"] = 0
  558. dn_1.properties["temp_key_5"] = 0
  559. with dn_1 as dn:
  560. assert dn._is_in_context
  561. assert dn.properties["qux"] == 5
  562. assert dn.properties["temp_key_3"] == 1
  563. assert dn.properties["temp_key_4"] == 0
  564. assert dn.properties["temp_key_5"] == 0
  565. dn.properties["qux"] = 9
  566. dn.properties.pop("temp_key_3")
  567. dn.properties.pop("temp_key_4")
  568. dn.properties.update({"temp_key_4": 1})
  569. dn.properties.update({"temp_key_5": 2})
  570. dn.properties.pop("temp_key_5")
  571. dn.properties.update({})
  572. assert dn.properties["qux"] == 5
  573. assert dn.properties["temp_key_3"] == 1
  574. assert dn.properties["temp_key_4"] == 0
  575. assert dn.properties["temp_key_5"] == 0
  576. assert not dn_1._is_in_context
  577. assert dn_1.properties["qux"] == 9
  578. assert "temp_key_3" not in dn_1.properties.keys()
  579. assert dn_1.properties["temp_key_4"] == 1
  580. assert "temp_key_5" not in dn_1.properties.keys()
  581. def test_get_parents(self, data_node):
  582. with mock.patch("taipy.core.get_parents") as mck:
  583. data_node.get_parents()
  584. mck.assert_called_once_with(data_node)
  585. def test_data_node_with_env_variable_value_not_stored(self):
  586. dn_config = Config.configure_data_node("A", prop="ENV[FOO]")
  587. with mock.patch.dict(os.environ, {"FOO": "bar"}):
  588. dn = _DataManager._bulk_get_or_create([dn_config])[dn_config]
  589. assert dn._properties.data["prop"] == "ENV[FOO]"
  590. assert dn.properties["prop"] == "bar"
  591. def test_path_populated_with_config_default_path(self):
  592. dn_config = Config.configure_data_node("data_node", "pickle", default_path="foo.p")
  593. assert dn_config.default_path == "foo.p"
  594. data_node = _DataManager._bulk_get_or_create([dn_config])[dn_config]
  595. assert data_node.path == "foo.p"
  596. data_node.path = "baz.p"
  597. assert data_node.path == "baz.p"
  598. def test_edit_edit_tracking(self):
  599. dn_config = Config.configure_data_node("A")
  600. data_node = _DataManager._bulk_get_or_create([dn_config])[dn_config]
  601. data_node.write(data="1", job_id="job_1")
  602. data_node.write(data="2", job_id="job_1")
  603. data_node.write(data="3", job_id="job_1")
  604. assert len(data_node.edits) == 3
  605. assert len(data_node.job_ids) == 3
  606. assert data_node.edits[-1] == data_node.get_last_edit()
  607. assert data_node.last_edit_date == data_node.get_last_edit().get("timestamp")
  608. date = datetime(2050, 1, 1, 12, 12)
  609. data_node.write(data="4", timestamp=date, message="This is a comment on this edit", env="staging")
  610. assert len(data_node.edits) == 4
  611. assert len(data_node.job_ids) == 3
  612. assert data_node.edits[-1] == data_node.get_last_edit()
  613. last_edit = data_node.get_last_edit()
  614. assert last_edit["message"] == "This is a comment on this edit"
  615. assert last_edit["env"] == "staging"
  616. assert last_edit["timestamp"] == date
  617. def test_label(self):
  618. a_date = datetime.now()
  619. dn = DataNode(
  620. "foo_bar",
  621. Scope.SCENARIO,
  622. DataNodeId("an_id"),
  623. "a_scenario_id",
  624. {"a_parent_id"},
  625. a_date,
  626. [{"job_id": "a_job_id"}],
  627. edit_in_progress=False,
  628. prop="erty",
  629. name="a name",
  630. )
  631. with mock.patch("taipy.core.get") as get_mck:
  632. class MockOwner:
  633. label = "owner_label"
  634. def get_label(self):
  635. return self.label
  636. get_mck.return_value = MockOwner()
  637. assert dn.get_label() == "owner_label > " + dn.name
  638. assert dn.get_simple_label() == dn.name
  639. def test_explicit_label(self):
  640. a_date = datetime.now()
  641. dn = DataNode(
  642. "foo_bar",
  643. Scope.SCENARIO,
  644. DataNodeId("an_id"),
  645. "a_scenario_id",
  646. {"a_parent_id"},
  647. a_date,
  648. [{"job_id": "a_job_id"}],
  649. edit_in_progress=False,
  650. label="a label",
  651. name="a name",
  652. )
  653. assert dn.get_label() == "a label"
  654. assert dn.get_simple_label() == "a label"
  655. def test_change_data_node_name(self):
  656. cgf = Config.configure_data_node("foo", scope=Scope.GLOBAL)
  657. dn = tp.create_global_data_node(cgf)
  658. dn.name = "bar"
  659. assert dn.name == "bar"
  660. # This new syntax will be the only one allowed: https://github.com/Avaiga/taipy-core/issues/806
  661. dn.properties["name"] = "baz"
  662. assert dn.name == "baz"
  663. def test_locked_data_node_write_should_fail_with_wrong_editor(self):
  664. dn_config = Config.configure_data_node("A")
  665. dn = _DataManager._bulk_get_or_create([dn_config])[dn_config]
  666. dn.lock_edit("editor_1")
  667. # Should raise exception for wrong editor
  668. with pytest.raises(DataNodeIsBeingEdited):
  669. dn.write("data", editor_id="editor_2")
  670. # Should succeed with correct editor
  671. dn.write("data", editor_id="editor_1")
  672. assert dn.read() == "data"
  673. def test_locked_data_node_write_should_fail_before_expiration_date_and_succeed_after(self):
  674. dn_config = Config.configure_data_node("A")
  675. dn = _DataManager._bulk_get_or_create([dn_config])[dn_config]
  676. lock_time = datetime.now()
  677. with freezegun.freeze_time(lock_time):
  678. dn.lock_edit("editor_1")
  679. with freezegun.freeze_time(lock_time + timedelta(minutes=29)):
  680. # Should raise exception for wrong editor and expiration date NOT passed
  681. with pytest.raises(DataNodeIsBeingEdited):
  682. dn.write("data", editor_id="editor_2")
  683. with freezegun.freeze_time(lock_time + timedelta(minutes=31)):
  684. # Should succeed with wrong editor but expiration date passed
  685. dn.write("data", editor_id="editor_2")
  686. assert dn.read() == "data"
  687. def test_locked_data_node_append_should_fail_with_wrong_editor(self):
  688. dn_config = Config.configure_csv_data_node("A")
  689. dn = _DataManager._bulk_get_or_create([dn_config])[dn_config]
  690. first_line = pd.DataFrame(data={"col1": [1], "col2": [3]})
  691. second_line = pd.DataFrame(data={"col1": [2], "col2": [4]})
  692. data = pd.DataFrame(data={"col1": [1, 2], "col2": [3, 4]})
  693. dn.write(first_line)
  694. assert first_line.equals(dn.read())
  695. dn.lock_edit("editor_1")
  696. with pytest.raises(DataNodeIsBeingEdited):
  697. dn.append(second_line, editor_id="editor_2")
  698. dn.append(second_line, editor_id="editor_1")
  699. assert dn.read().equals(data)
  700. def test_locked_data_node_append_should_fail_before_expiration_date_and_succeed_after(self):
  701. dn_config = Config.configure_csv_data_node("A")
  702. dn = _DataManager._bulk_get_or_create([dn_config])[dn_config]
  703. first_line = pd.DataFrame(data={"col1": [1], "col2": [3]})
  704. second_line = pd.DataFrame(data={"col1": [2], "col2": [4]})
  705. data = pd.DataFrame(data={"col1": [1, 2], "col2": [3, 4]})
  706. dn.write(first_line)
  707. assert first_line.equals(dn.read())
  708. lock_time = datetime.now()
  709. with freezegun.freeze_time(lock_time):
  710. dn.lock_edit("editor_1")
  711. with freezegun.freeze_time(lock_time + timedelta(minutes=29)):
  712. # Should raise exception for wrong editor and expiration date NOT passed
  713. with pytest.raises(DataNodeIsBeingEdited):
  714. dn.append(second_line, editor_id="editor_2")
  715. with freezegun.freeze_time(lock_time + timedelta(minutes=31)):
  716. # Should succeed with wrong editor but expiration date passed
  717. dn.append(second_line, editor_id="editor_2")
  718. assert dn.read().equals(data)
  719. def test_orchestrator_write_without_editor_id(self):
  720. dn_config = Config.configure_data_node("A")
  721. dn = _DataManager._bulk_get_or_create([dn_config])[dn_config]
  722. dn.lock_edit("editor_1")
  723. # Orchestrator write without editor_id should succeed
  724. dn.write("orchestrator_data")
  725. assert dn.read() == "orchestrator_data"
  726. def test_editor_fails_writing_a_data_node_locked_by_orchestrator(self):
  727. dn_config = Config.configure_data_node("A")
  728. dn = _DataManager._bulk_get_or_create([dn_config])[dn_config]
  729. dn.lock_edit() # Locked by orchestrator
  730. with pytest.raises(DataNodeIsBeingEdited):
  731. dn.write("data", editor_id="editor_1")
  732. # Orchestrator write without editor_id should succeed
  733. dn.write("orchestrator_data", job_id=JobId("job_1"))
  734. assert dn.read() == "orchestrator_data"
  735. def test_editor_fails_appending_a_data_node_locked_by_orchestrator(self):
  736. dn_config = Config.configure_csv_data_node("A")
  737. dn = _DataManager._bulk_get_or_create([dn_config])[dn_config]
  738. first_line = pd.DataFrame(data={"col1": [1], "col2": [3]})
  739. second_line = pd.DataFrame(data={"col1": [2], "col2": [4]})
  740. data = pd.DataFrame(data={"col1": [1, 2], "col2": [3, 4]})
  741. dn.write(first_line)
  742. assert first_line.equals(dn.read())
  743. dn = _DataManager._bulk_get_or_create([dn_config])[dn_config]
  744. dn.lock_edit() # Locked by orchestrator
  745. with pytest.raises(DataNodeIsBeingEdited):
  746. dn.append(second_line, editor_id="editor_1")
  747. assert dn.read().equals(first_line)
  748. dn.append(second_line, job_id=JobId("job_1"))
  749. assert dn.read().equals(data)
  750. def test_track_edit(self):
  751. dn_config = Config.configure_data_node("A")
  752. data_node = _DataManager._bulk_get_or_create([dn_config])[dn_config]
  753. before = datetime.now()
  754. data_node.track_edit(job_id="job_1")
  755. data_node.track_edit(editor_id="editor_1")
  756. data_node.track_edit(comment="This is a comment on this edit")
  757. data_node.track_edit(editor_id="editor_2", comment="This is another comment on this edit")
  758. data_node.track_edit(editor_id="editor_3", foo="bar")
  759. after = datetime.now()
  760. timestamp = datetime.now()
  761. data_node.track_edit(timestamp=timestamp)
  762. _DataManagerFactory._build_manager()._update(data_node)
  763. # To save the edits because track edit does not save the data node
  764. assert len(data_node.edits) == 6
  765. assert data_node.edits[-1] == data_node.get_last_edit()
  766. assert data_node.last_edit_date == data_node.get_last_edit().get(EDIT_TIMESTAMP_KEY)
  767. edit_0 = data_node.edits[0]
  768. assert len(edit_0) == 2
  769. assert edit_0[EDIT_JOB_ID_KEY] == "job_1"
  770. assert edit_0[EDIT_TIMESTAMP_KEY] >= before
  771. assert edit_0[EDIT_TIMESTAMP_KEY] <= after
  772. edit_1 = data_node.edits[1]
  773. assert len(edit_1) == 2
  774. assert edit_1[EDIT_EDITOR_ID_KEY] == "editor_1"
  775. assert edit_1[EDIT_TIMESTAMP_KEY] >= before
  776. assert edit_1[EDIT_TIMESTAMP_KEY] <= after
  777. edit_2 = data_node.edits[2]
  778. assert len(edit_2) == 2
  779. assert edit_2[EDIT_COMMENT_KEY] == "This is a comment on this edit"
  780. assert edit_2[EDIT_TIMESTAMP_KEY] >= before
  781. assert edit_2[EDIT_TIMESTAMP_KEY] <= after
  782. edit_3 = data_node.edits[3]
  783. assert len(edit_3) == 3
  784. assert edit_3[EDIT_EDITOR_ID_KEY] == "editor_2"
  785. assert edit_3[EDIT_COMMENT_KEY] == "This is another comment on this edit"
  786. assert edit_3[EDIT_TIMESTAMP_KEY] >= before
  787. assert edit_3[EDIT_TIMESTAMP_KEY] <= after
  788. edit_4 = data_node.edits[4]
  789. assert len(edit_4) == 3
  790. assert edit_4[EDIT_EDITOR_ID_KEY] == "editor_3"
  791. assert edit_4["foo"] == "bar"
  792. assert edit_4[EDIT_TIMESTAMP_KEY] >= before
  793. assert edit_4[EDIT_TIMESTAMP_KEY] <= after
  794. edit_5 = data_node.edits[5]
  795. assert len(edit_5) == 1
  796. assert edit_5[EDIT_TIMESTAMP_KEY] == timestamp
  797. def test_normalize_path(self):
  798. dn = DataNode(
  799. config_id="foo_bar",
  800. scope=Scope.SCENARIO,
  801. id=DataNodeId("an_id"),
  802. path=r"data\foo\bar.csv",
  803. )
  804. assert dn.config_id == "foo_bar"
  805. assert dn.scope == Scope.SCENARIO
  806. assert dn.id == "an_id"
  807. assert dn.properties["path"] == "data/foo/bar.csv"