test_submission.py 37 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944
  1. # Copyright 2021-2025 Avaiga Private Limited
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
  4. # the License. You may obtain a copy of the License at
  5. #
  6. # http://www.apache.org/licenses/LICENSE-2.0
  7. #
  8. # Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
  9. # an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
  10. # specific language governing permissions and limitations under the License.
  11. from datetime import datetime
  12. import freezegun
  13. import pytest
  14. from taipy.core import TaskId
  15. from taipy.core.job._job_manager_factory import _JobManagerFactory
  16. from taipy.core.job.job import Job
  17. from taipy.core.job.status import Status
  18. from taipy.core.submission._submission_manager_factory import _SubmissionManagerFactory
  19. from taipy.core.submission.submission import Submission
  20. from taipy.core.submission.submission_status import SubmissionStatus
  21. from taipy.core.task._task_manager_factory import _TaskManagerFactory
  22. from taipy.core.task.task import Task
  23. def test_submission_equals(submission):
  24. submission_manager = _SubmissionManagerFactory()._build_manager()
  25. submission_id = submission.id
  26. submission_manager._repository._save(submission)
  27. # To test if instance is same type
  28. task = Task("task", {}, print, [], [], submission_id)
  29. submission_2 = submission_manager._get(submission_id)
  30. assert submission == submission_2
  31. assert submission != submission_id
  32. assert submission != task
  33. def test_create_submission(scenario, job, current_datetime):
  34. submission_1 = Submission(scenario.id, scenario._ID_PREFIX, scenario.config_id)
  35. assert submission_1.id is not None
  36. assert submission_1.entity_id == scenario.id
  37. assert submission_1.entity_type == scenario._ID_PREFIX
  38. assert submission_1.entity_config_id == scenario.config_id
  39. assert submission_1.jobs == []
  40. assert isinstance(submission_1.creation_date, datetime)
  41. assert submission_1._submission_status == SubmissionStatus.SUBMITTED
  42. assert submission_1._version is not None
  43. submission_2 = Submission(
  44. scenario.id,
  45. scenario._ID_PREFIX,
  46. scenario.config_id,
  47. "submission_id",
  48. [job],
  49. {"debug": True, "log": "log_file", "retry_note": 5},
  50. current_datetime,
  51. SubmissionStatus.COMPLETED,
  52. "version_id",
  53. )
  54. assert submission_2.id == "submission_id"
  55. assert submission_2.entity_id == scenario.id
  56. assert submission_2.entity_type == scenario._ID_PREFIX
  57. assert submission_2.entity_config_id == scenario.config_id
  58. assert submission_2._jobs == [job]
  59. assert submission_2._properties == {"debug": True, "log": "log_file", "retry_note": 5}
  60. assert submission_2.creation_date == current_datetime
  61. assert submission_2._submission_status == SubmissionStatus.COMPLETED
  62. assert submission_2._version == "version_id"
  63. class MockJob:
  64. def __init__(self, id: str, status):
  65. self.status = status
  66. self.id = id
  67. def is_failed(self):
  68. return self.status == Status.FAILED
  69. def is_canceled(self):
  70. return self.status == Status.CANCELED
  71. def is_blocked(self):
  72. return self.status == Status.BLOCKED
  73. def is_pending(self):
  74. return self.status == Status.PENDING
  75. def is_running(self):
  76. return self.status == Status.RUNNING
  77. def is_completed(self):
  78. return self.status == Status.COMPLETED
  79. def is_skipped(self):
  80. return self.status == Status.SKIPPED
  81. def is_abandoned(self):
  82. return self.status == Status.ABANDONED
  83. def is_submitted(self):
  84. return self.status == Status.SUBMITTED
  85. def __test_update_submission_status(job_ids, expected_submission_status):
  86. jobs = {
  87. "job0_submitted": MockJob("job0_submitted", Status.SUBMITTED),
  88. "job1_failed": MockJob("job1_failed", Status.FAILED),
  89. "job2_canceled": MockJob("job2_canceled", Status.CANCELED),
  90. "job3_blocked": MockJob("job3_blocked", Status.BLOCKED),
  91. "job4_pending": MockJob("job4_pending", Status.PENDING),
  92. "job5_running": MockJob("job5_running", Status.RUNNING),
  93. "job6_completed": MockJob("job6_completed", Status.COMPLETED),
  94. "job7_skipped": MockJob("job7_skipped", Status.SKIPPED),
  95. "job8_abandoned": MockJob("job8_abandoned", Status.ABANDONED),
  96. }
  97. submission = Submission("submission_id", "ENTITY_TYPE", "entity_config_id")
  98. _SubmissionManagerFactory._build_manager()._repository._save(submission)
  99. submission.jobs = [jobs[job_id] for job_id in job_ids]
  100. for job_id in job_ids:
  101. job = jobs[job_id]
  102. _SubmissionManagerFactory._build_manager()._update_submission_status(submission, job)
  103. assert submission.submission_status == expected_submission_status
  104. @pytest.mark.parametrize(
  105. "job_ids, expected_submission_status",
  106. [
  107. (["job1_failed"], SubmissionStatus.FAILED),
  108. (["job2_canceled"], SubmissionStatus.CANCELED),
  109. (["job3_blocked"], SubmissionStatus.BLOCKED),
  110. (["job4_pending"], SubmissionStatus.PENDING),
  111. (["job5_running"], SubmissionStatus.RUNNING),
  112. (["job6_completed"], SubmissionStatus.COMPLETED),
  113. (["job7_skipped"], SubmissionStatus.COMPLETED),
  114. (["job8_abandoned"], SubmissionStatus.UNDEFINED),
  115. ],
  116. )
  117. def test_update_single_submission_status(job_ids, expected_submission_status):
  118. __test_update_submission_status(job_ids, expected_submission_status)
  119. @pytest.mark.parametrize(
  120. "job_ids, expected_submission_status",
  121. [
  122. (["job1_failed", "job1_failed"], SubmissionStatus.FAILED),
  123. (["job1_failed", "job2_canceled"], SubmissionStatus.FAILED),
  124. (["job1_failed", "job3_blocked"], SubmissionStatus.FAILED),
  125. (["job1_failed", "job4_pending"], SubmissionStatus.FAILED),
  126. (["job1_failed", "job5_running"], SubmissionStatus.FAILED),
  127. (["job1_failed", "job6_completed"], SubmissionStatus.FAILED),
  128. (["job1_failed", "job7_skipped"], SubmissionStatus.FAILED),
  129. (["job1_failed", "job8_abandoned"], SubmissionStatus.FAILED),
  130. (["job2_canceled", "job1_failed"], SubmissionStatus.FAILED),
  131. (["job3_blocked", "job1_failed"], SubmissionStatus.FAILED),
  132. (["job4_pending", "job1_failed"], SubmissionStatus.FAILED),
  133. (["job5_running", "job1_failed"], SubmissionStatus.FAILED),
  134. (["job6_completed", "job1_failed"], SubmissionStatus.FAILED),
  135. (["job7_skipped", "job1_failed"], SubmissionStatus.FAILED),
  136. (["job8_abandoned", "job1_failed"], SubmissionStatus.FAILED),
  137. ],
  138. )
  139. def test_update_submission_status_with_one_failed_job_in_jobs(job_ids, expected_submission_status):
  140. __test_update_submission_status(job_ids, expected_submission_status)
  141. @pytest.mark.parametrize(
  142. "job_ids, expected_submission_status",
  143. [
  144. (["job2_canceled", "job2_canceled"], SubmissionStatus.CANCELED),
  145. (["job2_canceled", "job3_blocked"], SubmissionStatus.CANCELED),
  146. (["job2_canceled", "job4_pending"], SubmissionStatus.CANCELED),
  147. (["job2_canceled", "job5_running"], SubmissionStatus.CANCELED),
  148. (["job2_canceled", "job6_completed"], SubmissionStatus.CANCELED),
  149. (["job2_canceled", "job7_skipped"], SubmissionStatus.CANCELED),
  150. (["job2_canceled", "job8_abandoned"], SubmissionStatus.CANCELED),
  151. (["job3_blocked", "job2_canceled"], SubmissionStatus.CANCELED),
  152. (["job4_pending", "job2_canceled"], SubmissionStatus.CANCELED),
  153. (["job5_running", "job2_canceled"], SubmissionStatus.CANCELED),
  154. (["job6_completed", "job2_canceled"], SubmissionStatus.CANCELED),
  155. (["job7_skipped", "job2_canceled"], SubmissionStatus.CANCELED),
  156. (["job8_abandoned", "job2_canceled"], SubmissionStatus.CANCELED),
  157. ],
  158. )
  159. def test_update_submission_status_with_one_canceled_job_in_jobs(job_ids, expected_submission_status):
  160. __test_update_submission_status(job_ids, expected_submission_status)
  161. @pytest.mark.parametrize(
  162. "job_ids, expected_submission_status",
  163. [
  164. (["job4_pending", "job3_blocked"], SubmissionStatus.PENDING),
  165. (["job4_pending", "job4_pending"], SubmissionStatus.PENDING),
  166. (["job4_pending", "job6_completed"], SubmissionStatus.PENDING),
  167. (["job4_pending", "job7_skipped"], SubmissionStatus.PENDING),
  168. (["job3_blocked", "job4_pending"], SubmissionStatus.PENDING),
  169. (["job6_completed", "job4_pending"], SubmissionStatus.PENDING),
  170. (["job7_skipped", "job4_pending"], SubmissionStatus.PENDING),
  171. ],
  172. )
  173. def test_update_submission_status_with_no_failed_or_cancel_one_pending_in_jobs(job_ids, expected_submission_status):
  174. __test_update_submission_status(job_ids, expected_submission_status)
  175. @pytest.mark.parametrize(
  176. "job_ids, expected_submission_status",
  177. [
  178. (["job5_running", "job3_blocked"], SubmissionStatus.RUNNING),
  179. (["job5_running", "job4_pending"], SubmissionStatus.RUNNING),
  180. (["job5_running", "job5_running"], SubmissionStatus.RUNNING),
  181. (["job5_running", "job6_completed"], SubmissionStatus.RUNNING),
  182. (["job5_running", "job7_skipped"], SubmissionStatus.RUNNING),
  183. (["job3_blocked", "job5_running"], SubmissionStatus.RUNNING),
  184. (["job4_pending", "job5_running"], SubmissionStatus.RUNNING),
  185. (["job6_completed", "job5_running"], SubmissionStatus.RUNNING),
  186. (["job7_skipped", "job5_running"], SubmissionStatus.RUNNING),
  187. ],
  188. )
  189. def test_update_submission_status_with_no_failed_cancel_nor_pending_one_running_in_jobs(
  190. job_ids, expected_submission_status
  191. ):
  192. __test_update_submission_status(job_ids, expected_submission_status)
  193. @pytest.mark.parametrize(
  194. "job_ids, expected_submission_status",
  195. [
  196. (["job3_blocked", "job3_blocked"], SubmissionStatus.BLOCKED),
  197. (["job3_blocked", "job6_completed"], SubmissionStatus.BLOCKED),
  198. (["job3_blocked", "job7_skipped"], SubmissionStatus.BLOCKED),
  199. (["job6_completed", "job3_blocked"], SubmissionStatus.BLOCKED),
  200. (["job7_skipped", "job3_blocked"], SubmissionStatus.BLOCKED),
  201. ],
  202. )
  203. def test_update_submission_status_with_no_failed_cancel_pending_nor_running_one_blocked_in_jobs(
  204. job_ids, expected_submission_status
  205. ):
  206. __test_update_submission_status(job_ids, expected_submission_status)
  207. @pytest.mark.parametrize(
  208. "job_ids, expected_submission_status",
  209. [
  210. (["job6_completed", "job6_completed"], SubmissionStatus.COMPLETED),
  211. (["job6_completed", "job7_skipped"], SubmissionStatus.COMPLETED),
  212. (["job7_skipped", "job6_completed"], SubmissionStatus.COMPLETED),
  213. (["job7_skipped", "job7_skipped"], SubmissionStatus.COMPLETED),
  214. ],
  215. )
  216. def test_update_submission_status_with_only_completed_or_skipped_in_jobs(job_ids, expected_submission_status):
  217. __test_update_submission_status(job_ids, expected_submission_status)
  218. @pytest.mark.parametrize(
  219. "job_ids, expected_submission_status",
  220. [
  221. (["job3_blocked", "job8_abandoned"], SubmissionStatus.UNDEFINED),
  222. (["job4_pending", "job8_abandoned"], SubmissionStatus.UNDEFINED),
  223. (["job5_running", "job8_abandoned"], SubmissionStatus.UNDEFINED),
  224. (["job6_completed", "job8_abandoned"], SubmissionStatus.UNDEFINED),
  225. (["job7_skipped", "job8_abandoned"], SubmissionStatus.UNDEFINED),
  226. (["job8_abandoned", "job8_abandoned"], SubmissionStatus.UNDEFINED),
  227. (["job8_abandoned", "job3_blocked"], SubmissionStatus.UNDEFINED),
  228. (["job8_abandoned", "job4_pending"], SubmissionStatus.UNDEFINED),
  229. (["job8_abandoned", "job5_running"], SubmissionStatus.UNDEFINED),
  230. (["job8_abandoned", "job6_completed"], SubmissionStatus.UNDEFINED),
  231. (["job8_abandoned", "job7_skipped"], SubmissionStatus.UNDEFINED),
  232. ],
  233. )
  234. def test_update_submission_status_with_wrong_case_abandoned_without_cancel_or_failed_in_jobs(
  235. job_ids, expected_submission_status
  236. ):
  237. __test_update_submission_status(job_ids, expected_submission_status)
  238. def test_auto_update_and_reload():
  239. task = Task(config_id="name_1", properties={}, function=print, id=TaskId("task_1"))
  240. submission_1 = Submission(task.id, task._ID_PREFIX, task.config_id, properties={})
  241. job_1 = Job("job_1", task, submission_1.id, submission_1.entity_id)
  242. job_2 = Job("job_2", task, submission_1.id, submission_1.entity_id)
  243. _TaskManagerFactory._build_manager()._repository._save(task)
  244. _SubmissionManagerFactory._build_manager()._repository._save(submission_1)
  245. _JobManagerFactory._build_manager()._repository._save(job_1)
  246. _JobManagerFactory._build_manager()._repository._save(job_2)
  247. submission_2 = _SubmissionManagerFactory._build_manager()._get(submission_1)
  248. assert submission_1.id == submission_2.id
  249. assert submission_1.entity_id == submission_2.entity_id
  250. assert submission_1.creation_date == submission_2.creation_date
  251. assert submission_1.submission_status == submission_2.submission_status
  252. # auto set & reload on jobs attribute
  253. assert submission_1.jobs == []
  254. assert submission_2.jobs == []
  255. submission_1.jobs = [job_1]
  256. assert submission_1.jobs == [job_1]
  257. assert submission_2.jobs == [job_1]
  258. submission_2.jobs = [job_2]
  259. assert submission_1.jobs == [job_2]
  260. assert submission_2.jobs == [job_2]
  261. submission_1.jobs = [job_1, job_2]
  262. assert submission_1.jobs == [job_1, job_2]
  263. assert submission_2.jobs == [job_1, job_2]
  264. submission_2.jobs = [job_2, job_1]
  265. assert submission_1.jobs == [job_2, job_1]
  266. assert submission_2.jobs == [job_2, job_1]
  267. # auto set & reload on is_canceled attribute
  268. assert not submission_1.is_canceled
  269. assert not submission_2.is_canceled
  270. submission_1.is_canceled = True
  271. assert submission_1.is_canceled
  272. assert submission_2.is_canceled
  273. submission_2.is_canceled = False
  274. assert not submission_1.is_canceled
  275. assert not submission_2.is_canceled
  276. # auto set & reload on is_completed attribute
  277. assert not submission_1.is_completed
  278. assert not submission_2.is_completed
  279. submission_1.is_completed = True
  280. assert submission_1.is_completed
  281. assert submission_2.is_completed
  282. submission_2.is_completed = False
  283. assert not submission_1.is_completed
  284. assert not submission_2.is_completed
  285. # auto set & reload on is_abandoned attribute
  286. assert not submission_1.is_abandoned
  287. assert not submission_2.is_abandoned
  288. submission_1.is_abandoned = True
  289. assert submission_1.is_abandoned
  290. assert submission_2.is_abandoned
  291. submission_2.is_abandoned = False
  292. assert not submission_1.is_abandoned
  293. assert not submission_2.is_abandoned
  294. # auto set & reload on submission_status attribute
  295. assert submission_1.submission_status == SubmissionStatus.SUBMITTED
  296. assert submission_2.submission_status == SubmissionStatus.SUBMITTED
  297. submission_1.submission_status = SubmissionStatus.BLOCKED
  298. assert submission_1.submission_status == SubmissionStatus.BLOCKED
  299. assert submission_2.submission_status == SubmissionStatus.BLOCKED
  300. submission_2.submission_status = SubmissionStatus.COMPLETED
  301. assert submission_1.submission_status == SubmissionStatus.COMPLETED
  302. assert submission_2.submission_status == SubmissionStatus.COMPLETED
  303. with submission_1 as submission:
  304. assert submission.jobs == [job_2, job_1]
  305. assert submission.submission_status == SubmissionStatus.COMPLETED
  306. submission.jobs = [job_1]
  307. submission.submission_status = SubmissionStatus.PENDING
  308. assert submission.jobs == [job_2, job_1]
  309. assert submission.submission_status == SubmissionStatus.COMPLETED
  310. assert submission_1.jobs == [job_1]
  311. assert submission_1.submission_status == SubmissionStatus.PENDING
  312. assert submission_2.jobs == [job_1]
  313. assert submission_2.submission_status == SubmissionStatus.PENDING
  314. def test_auto_update_and_reload_properties():
  315. task = Task(config_id="name_1", properties={}, function=print, id=TaskId("task_1"))
  316. submission_1 = Submission(task.id, task._ID_PREFIX, task.config_id, properties={})
  317. _TaskManagerFactory._build_manager()._repository._save(task)
  318. _SubmissionManagerFactory._build_manager()._repository._save(submission_1)
  319. submission_2 = _SubmissionManagerFactory._build_manager()._get(submission_1)
  320. # auto set & reload on properties attribute
  321. assert submission_1.properties == {}
  322. assert submission_2.properties == {}
  323. submission_1._properties["qux"] = 4
  324. assert submission_1.properties["qux"] == 4
  325. assert submission_2.properties["qux"] == 4
  326. assert submission_1.properties == {"qux": 4}
  327. assert submission_2.properties == {"qux": 4}
  328. submission_2._properties["qux"] = 5
  329. assert submission_1.properties["qux"] == 5
  330. assert submission_2.properties["qux"] == 5
  331. submission_1.properties["temp_key_1"] = "temp_value_1"
  332. submission_1.properties["temp_key_2"] = "temp_value_2"
  333. assert submission_1.properties == {"qux": 5, "temp_key_1": "temp_value_1", "temp_key_2": "temp_value_2"}
  334. assert submission_2.properties == {"qux": 5, "temp_key_1": "temp_value_1", "temp_key_2": "temp_value_2"}
  335. submission_1.properties.pop("temp_key_1")
  336. assert "temp_key_1" not in submission_1.properties.keys()
  337. assert "temp_key_1" not in submission_1.properties.keys()
  338. assert submission_1.properties == {"qux": 5, "temp_key_2": "temp_value_2"}
  339. assert submission_2.properties == {"qux": 5, "temp_key_2": "temp_value_2"}
  340. submission_2.properties.pop("temp_key_2")
  341. assert submission_1.properties == {"qux": 5}
  342. assert submission_2.properties == {"qux": 5}
  343. assert "temp_key_2" not in submission_1.properties.keys()
  344. assert "temp_key_2" not in submission_2.properties.keys()
  345. submission_1.properties["temp_key_3"] = 0
  346. assert submission_1.properties == {"qux": 5, "temp_key_3": 0}
  347. assert submission_2.properties == {"qux": 5, "temp_key_3": 0}
  348. submission_1.properties.update({"temp_key_3": 1})
  349. assert submission_1.properties == {"qux": 5, "temp_key_3": 1}
  350. assert submission_2.properties == {"qux": 5, "temp_key_3": 1}
  351. submission_1.properties.update({})
  352. assert submission_1.properties == {"qux": 5, "temp_key_3": 1}
  353. assert submission_2.properties == {"qux": 5, "temp_key_3": 1}
  354. submission_1.properties["temp_key_4"] = 0
  355. submission_1.properties["temp_key_5"] = 0
  356. with submission_1 as submission:
  357. assert submission.properties["qux"] == 5
  358. assert submission.properties["temp_key_3"] == 1
  359. assert submission.properties["temp_key_4"] == 0
  360. assert submission.properties["temp_key_5"] == 0
  361. submission.properties["qux"] = 9
  362. submission.properties.pop("temp_key_3")
  363. submission.properties.pop("temp_key_4")
  364. submission.properties.update({"temp_key_4": 1})
  365. submission.properties.update({"temp_key_5": 2})
  366. submission.properties.pop("temp_key_5")
  367. submission.properties.update({})
  368. assert submission.properties["qux"] == 5
  369. assert submission.properties["temp_key_3"] == 1
  370. assert submission.properties["temp_key_4"] == 0
  371. assert submission.properties["temp_key_5"] == 0
  372. assert submission_1.properties["qux"] == 9
  373. assert "temp_key_3" not in submission_1.properties.keys()
  374. assert submission_1.properties["temp_key_4"] == 1
  375. assert "temp_key_5" not in submission_1.properties.keys()
  376. @pytest.mark.parametrize(
  377. "job_statuses, expected_submission_statuses",
  378. [
  379. (
  380. [Status.SUBMITTED, Status.PENDING, Status.RUNNING, Status.COMPLETED],
  381. [SubmissionStatus.PENDING, SubmissionStatus.PENDING, SubmissionStatus.RUNNING, SubmissionStatus.COMPLETED],
  382. ),
  383. (
  384. [Status.SUBMITTED, Status.PENDING, Status.RUNNING, Status.SKIPPED],
  385. [SubmissionStatus.PENDING, SubmissionStatus.PENDING, SubmissionStatus.RUNNING, SubmissionStatus.COMPLETED],
  386. ),
  387. (
  388. [Status.SUBMITTED, Status.PENDING, Status.RUNNING, Status.FAILED],
  389. [SubmissionStatus.PENDING, SubmissionStatus.PENDING, SubmissionStatus.RUNNING, SubmissionStatus.FAILED],
  390. ),
  391. (
  392. [Status.SUBMITTED, Status.PENDING, Status.CANCELED],
  393. [SubmissionStatus.PENDING, SubmissionStatus.PENDING, SubmissionStatus.CANCELED],
  394. ),
  395. (
  396. [Status.SUBMITTED, Status.PENDING, Status.RUNNING, Status.CANCELED],
  397. [SubmissionStatus.PENDING, SubmissionStatus.PENDING, SubmissionStatus.RUNNING, SubmissionStatus.CANCELED],
  398. ),
  399. ([Status.SUBMITTED, Status.BLOCKED], [SubmissionStatus.PENDING, SubmissionStatus.BLOCKED]),
  400. ([Status.SUBMITTED, Status.SKIPPED], [SubmissionStatus.PENDING, SubmissionStatus.COMPLETED]),
  401. ],
  402. )
  403. def test_update_submission_status_with_single_job_completed(job_statuses, expected_submission_statuses):
  404. submission_manager = _SubmissionManagerFactory._build_manager()
  405. job = MockJob("job_id", Status.SUBMITTED)
  406. submission = Submission("submission_id", "ENTITY_TYPE", "entity_config_id")
  407. submission_manager._repository._save(submission)
  408. assert submission.submission_status == SubmissionStatus.SUBMITTED
  409. for job_status, submission_status in zip(job_statuses, expected_submission_statuses):
  410. job.status = job_status
  411. submission_manager._update_submission_status(submission, job)
  412. assert submission.submission_status == submission_status
  413. def __test_update_submission_status_with_two_jobs(job_ids, job_statuses, expected_submission_statuses):
  414. submission_manager = _SubmissionManagerFactory._build_manager()
  415. jobs = {job_id: MockJob(job_id, Status.SUBMITTED) for job_id in job_ids}
  416. submission = Submission("submission_id", "ENTITY_TYPE", "entity_config_id")
  417. submission_manager._repository._save(submission)
  418. assert submission.submission_status == SubmissionStatus.SUBMITTED
  419. for (job_id, job_status), submission_status in zip(job_statuses, expected_submission_statuses):
  420. job = jobs[job_id]
  421. job.status = job_status
  422. submission_manager._update_submission_status(submission, job)
  423. assert submission.submission_status == submission_status
  424. @pytest.mark.parametrize(
  425. "job_ids, job_statuses, expected_submission_statuses",
  426. [
  427. (
  428. ["job_1", "job_2"],
  429. [
  430. ("job_1", Status.SUBMITTED),
  431. ("job_2", Status.SUBMITTED),
  432. ("job_1", Status.PENDING),
  433. ("job_2", Status.PENDING),
  434. ("job_1", Status.RUNNING),
  435. ("job_2", Status.RUNNING),
  436. ("job_1", Status.COMPLETED),
  437. ("job_2", Status.COMPLETED),
  438. ],
  439. [
  440. SubmissionStatus.PENDING,
  441. SubmissionStatus.PENDING,
  442. SubmissionStatus.PENDING,
  443. SubmissionStatus.PENDING,
  444. SubmissionStatus.RUNNING,
  445. SubmissionStatus.RUNNING,
  446. SubmissionStatus.RUNNING,
  447. SubmissionStatus.COMPLETED,
  448. ],
  449. ),
  450. (
  451. ["job_1", "job_2"],
  452. [
  453. ("job_1", Status.SUBMITTED),
  454. ("job_2", Status.SUBMITTED),
  455. ("job_1", Status.PENDING),
  456. ("job_1", Status.RUNNING),
  457. ("job_2", Status.PENDING),
  458. ("job_2", Status.RUNNING),
  459. ("job_1", Status.COMPLETED),
  460. ("job_2", Status.COMPLETED),
  461. ],
  462. [
  463. SubmissionStatus.PENDING,
  464. SubmissionStatus.PENDING,
  465. SubmissionStatus.PENDING,
  466. SubmissionStatus.RUNNING,
  467. SubmissionStatus.RUNNING,
  468. SubmissionStatus.RUNNING,
  469. SubmissionStatus.RUNNING,
  470. SubmissionStatus.COMPLETED,
  471. ],
  472. ),
  473. (
  474. ["job_1", "job_2"],
  475. [
  476. ("job_1", Status.SUBMITTED),
  477. ("job_2", Status.SUBMITTED),
  478. ("job_1", Status.BLOCKED),
  479. ("job_2", Status.PENDING),
  480. ("job_2", Status.RUNNING),
  481. ("job_2", Status.COMPLETED),
  482. ("job_1", Status.PENDING),
  483. ("job_1", Status.RUNNING),
  484. ("job_1", Status.COMPLETED),
  485. ],
  486. [
  487. SubmissionStatus.PENDING,
  488. SubmissionStatus.PENDING,
  489. SubmissionStatus.PENDING,
  490. SubmissionStatus.PENDING,
  491. SubmissionStatus.RUNNING,
  492. SubmissionStatus.BLOCKED,
  493. SubmissionStatus.PENDING,
  494. SubmissionStatus.RUNNING,
  495. SubmissionStatus.COMPLETED,
  496. ],
  497. ),
  498. ],
  499. )
  500. def test_update_submission_status_with_two_jobs_completed(job_ids, job_statuses, expected_submission_statuses):
  501. __test_update_submission_status_with_two_jobs(job_ids, job_statuses, expected_submission_statuses)
  502. @pytest.mark.parametrize(
  503. "job_ids, job_statuses, expected_submission_statuses",
  504. [
  505. (
  506. ["job_1", "job_2"],
  507. [
  508. ("job_1", Status.SUBMITTED),
  509. ("job_2", Status.SUBMITTED),
  510. ("job_1", Status.PENDING),
  511. ("job_2", Status.PENDING),
  512. ("job_1", Status.RUNNING),
  513. ("job_2", Status.SKIPPED),
  514. ("job_1", Status.COMPLETED),
  515. ],
  516. [
  517. SubmissionStatus.PENDING,
  518. SubmissionStatus.PENDING,
  519. SubmissionStatus.PENDING,
  520. SubmissionStatus.PENDING,
  521. SubmissionStatus.RUNNING,
  522. SubmissionStatus.RUNNING,
  523. SubmissionStatus.COMPLETED,
  524. ],
  525. ),
  526. (
  527. ["job_1", "job_2"],
  528. [
  529. ("job_1", Status.SUBMITTED),
  530. ("job_2", Status.SUBMITTED),
  531. ("job_1", Status.PENDING),
  532. ("job_1", Status.RUNNING),
  533. ("job_2", Status.PENDING),
  534. ("job_2", Status.SKIPPED),
  535. ("job_1", Status.COMPLETED),
  536. ],
  537. [
  538. SubmissionStatus.PENDING,
  539. SubmissionStatus.PENDING,
  540. SubmissionStatus.PENDING,
  541. SubmissionStatus.RUNNING,
  542. SubmissionStatus.RUNNING,
  543. SubmissionStatus.RUNNING,
  544. SubmissionStatus.COMPLETED,
  545. ],
  546. ),
  547. (
  548. ["job_1", "job_2"],
  549. [
  550. ("job_1", Status.SUBMITTED),
  551. ("job_2", Status.SUBMITTED),
  552. ("job_1", Status.BLOCKED),
  553. ("job_2", Status.PENDING),
  554. ("job_2", Status.RUNNING),
  555. ("job_2", Status.COMPLETED),
  556. ("job_1", Status.PENDING),
  557. ("job_1", Status.SKIPPED),
  558. ],
  559. [
  560. SubmissionStatus.PENDING,
  561. SubmissionStatus.PENDING,
  562. SubmissionStatus.PENDING,
  563. SubmissionStatus.PENDING,
  564. SubmissionStatus.RUNNING,
  565. SubmissionStatus.BLOCKED,
  566. SubmissionStatus.PENDING,
  567. SubmissionStatus.COMPLETED,
  568. ],
  569. ),
  570. (
  571. ["job_1", "job_2"],
  572. [
  573. ("job_1", Status.SUBMITTED),
  574. ("job_2", Status.SUBMITTED),
  575. ("job_1", Status.PENDING),
  576. ("job_2", Status.PENDING),
  577. ("job_1", Status.SKIPPED),
  578. ("job_2", Status.SKIPPED),
  579. ],
  580. [
  581. SubmissionStatus.PENDING,
  582. SubmissionStatus.PENDING,
  583. SubmissionStatus.PENDING,
  584. SubmissionStatus.PENDING,
  585. SubmissionStatus.PENDING,
  586. SubmissionStatus.COMPLETED,
  587. ],
  588. ),
  589. (
  590. ["job_1", "job_2"],
  591. [
  592. ("job_1", Status.SUBMITTED),
  593. ("job_2", Status.SUBMITTED),
  594. ("job_1", Status.PENDING),
  595. ("job_1", Status.SKIPPED),
  596. ("job_2", Status.PENDING),
  597. ("job_2", Status.SKIPPED),
  598. ],
  599. [
  600. SubmissionStatus.PENDING,
  601. SubmissionStatus.PENDING,
  602. SubmissionStatus.PENDING,
  603. SubmissionStatus.PENDING,
  604. SubmissionStatus.PENDING,
  605. SubmissionStatus.COMPLETED,
  606. ],
  607. ),
  608. (
  609. ["job_1", "job_2"],
  610. [
  611. ("job_1", Status.SUBMITTED),
  612. ("job_2", Status.SUBMITTED),
  613. ("job_1", Status.BLOCKED),
  614. ("job_2", Status.PENDING),
  615. ("job_2", Status.SKIPPED),
  616. ("job_1", Status.PENDING),
  617. ("job_1", Status.SKIPPED),
  618. ],
  619. [
  620. SubmissionStatus.PENDING,
  621. SubmissionStatus.PENDING,
  622. SubmissionStatus.PENDING,
  623. SubmissionStatus.PENDING,
  624. SubmissionStatus.BLOCKED,
  625. SubmissionStatus.PENDING,
  626. SubmissionStatus.COMPLETED,
  627. ],
  628. ),
  629. ],
  630. )
  631. def test_update_submission_status_with_two_jobs_skipped(job_ids, job_statuses, expected_submission_statuses):
  632. __test_update_submission_status_with_two_jobs(job_ids, job_statuses, expected_submission_statuses)
  633. @pytest.mark.parametrize(
  634. "job_ids, job_statuses, expected_submission_statuses",
  635. [
  636. (
  637. ["job_1", "job_2"],
  638. [
  639. ("job_1", Status.SUBMITTED),
  640. ("job_2", Status.SUBMITTED),
  641. ("job_1", Status.PENDING),
  642. ("job_2", Status.PENDING),
  643. ("job_1", Status.RUNNING),
  644. ("job_2", Status.RUNNING),
  645. ("job_1", Status.FAILED),
  646. ("job_2", Status.COMPLETED),
  647. ],
  648. [
  649. SubmissionStatus.PENDING,
  650. SubmissionStatus.PENDING,
  651. SubmissionStatus.PENDING,
  652. SubmissionStatus.PENDING,
  653. SubmissionStatus.RUNNING,
  654. SubmissionStatus.RUNNING,
  655. SubmissionStatus.FAILED,
  656. SubmissionStatus.FAILED,
  657. ],
  658. ),
  659. (
  660. ["job_1", "job_2"],
  661. [
  662. ("job_1", Status.SUBMITTED),
  663. ("job_2", Status.SUBMITTED),
  664. ("job_1", Status.PENDING),
  665. ("job_1", Status.RUNNING),
  666. ("job_2", Status.PENDING),
  667. ("job_2", Status.RUNNING),
  668. ("job_1", Status.COMPLETED),
  669. ("job_2", Status.FAILED),
  670. ],
  671. [
  672. SubmissionStatus.PENDING,
  673. SubmissionStatus.PENDING,
  674. SubmissionStatus.PENDING,
  675. SubmissionStatus.RUNNING,
  676. SubmissionStatus.RUNNING,
  677. SubmissionStatus.RUNNING,
  678. SubmissionStatus.RUNNING,
  679. SubmissionStatus.FAILED,
  680. ],
  681. ),
  682. (
  683. ["job_1", "job_2"],
  684. [
  685. ("job_1", Status.SUBMITTED),
  686. ("job_2", Status.SUBMITTED),
  687. ("job_1", Status.BLOCKED),
  688. ("job_2", Status.PENDING),
  689. ("job_2", Status.RUNNING),
  690. ("job_2", Status.FAILED),
  691. ("job_1", Status.ABANDONED),
  692. ],
  693. [
  694. SubmissionStatus.PENDING,
  695. SubmissionStatus.PENDING,
  696. SubmissionStatus.PENDING,
  697. SubmissionStatus.PENDING,
  698. SubmissionStatus.RUNNING,
  699. SubmissionStatus.FAILED,
  700. SubmissionStatus.FAILED,
  701. ],
  702. ),
  703. ],
  704. )
  705. def test_update_submission_status_with_two_jobs_failed(job_ids, job_statuses, expected_submission_statuses):
  706. __test_update_submission_status_with_two_jobs(job_ids, job_statuses, expected_submission_statuses)
  707. @pytest.mark.parametrize(
  708. "job_ids, job_statuses, expected_submission_statuses",
  709. [
  710. (
  711. ["job_1", "job_2"],
  712. [
  713. ("job_1", Status.SUBMITTED),
  714. ("job_2", Status.SUBMITTED),
  715. ("job_1", Status.PENDING),
  716. ("job_2", Status.PENDING),
  717. ("job_1", Status.RUNNING),
  718. ("job_2", Status.RUNNING),
  719. ("job_1", Status.CANCELED),
  720. ("job_2", Status.COMPLETED),
  721. ],
  722. [
  723. SubmissionStatus.PENDING,
  724. SubmissionStatus.PENDING,
  725. SubmissionStatus.PENDING,
  726. SubmissionStatus.PENDING,
  727. SubmissionStatus.RUNNING,
  728. SubmissionStatus.RUNNING,
  729. SubmissionStatus.CANCELED,
  730. SubmissionStatus.CANCELED,
  731. ],
  732. ),
  733. (
  734. ["job_1", "job_2"],
  735. [
  736. ("job_1", Status.SUBMITTED),
  737. ("job_2", Status.SUBMITTED),
  738. ("job_1", Status.PENDING),
  739. ("job_1", Status.RUNNING),
  740. ("job_2", Status.PENDING),
  741. ("job_2", Status.RUNNING),
  742. ("job_1", Status.COMPLETED),
  743. ("job_2", Status.CANCELED),
  744. ],
  745. [
  746. SubmissionStatus.PENDING,
  747. SubmissionStatus.PENDING,
  748. SubmissionStatus.PENDING,
  749. SubmissionStatus.RUNNING,
  750. SubmissionStatus.RUNNING,
  751. SubmissionStatus.RUNNING,
  752. SubmissionStatus.RUNNING,
  753. SubmissionStatus.CANCELED,
  754. ],
  755. ),
  756. (
  757. ["job_1", "job_2"],
  758. [
  759. ("job_1", Status.SUBMITTED),
  760. ("job_2", Status.SUBMITTED),
  761. ("job_1", Status.BLOCKED),
  762. ("job_2", Status.PENDING),
  763. ("job_2", Status.RUNNING),
  764. ("job_2", Status.CANCELED),
  765. ("job_1", Status.ABANDONED),
  766. ],
  767. [
  768. SubmissionStatus.PENDING,
  769. SubmissionStatus.PENDING,
  770. SubmissionStatus.PENDING,
  771. SubmissionStatus.PENDING,
  772. SubmissionStatus.RUNNING,
  773. SubmissionStatus.CANCELED,
  774. SubmissionStatus.CANCELED,
  775. ],
  776. ),
  777. ],
  778. )
  779. def test_update_submission_status_with_two_jobs_canceled(job_ids, job_statuses, expected_submission_statuses):
  780. __test_update_submission_status_with_two_jobs(job_ids, job_statuses, expected_submission_statuses)
  781. def test_is_finished():
  782. submission_manager = _SubmissionManagerFactory._build_manager()
  783. submission = Submission("entity_id", "entity_type", "entity_config_id", "submission_id")
  784. submission_manager._repository._save(submission)
  785. assert len(submission_manager._get_all()) == 1
  786. assert submission._submission_status == SubmissionStatus.SUBMITTED
  787. assert not submission.is_finished()
  788. submission.submission_status = SubmissionStatus.UNDEFINED
  789. assert submission.submission_status == SubmissionStatus.UNDEFINED
  790. assert not submission.is_finished()
  791. submission.submission_status = SubmissionStatus.CANCELED
  792. assert submission.submission_status == SubmissionStatus.CANCELED
  793. assert submission.is_finished()
  794. submission.submission_status = SubmissionStatus.FAILED
  795. assert submission.submission_status == SubmissionStatus.FAILED
  796. assert submission.is_finished()
  797. submission.submission_status = SubmissionStatus.BLOCKED
  798. assert submission.submission_status == SubmissionStatus.BLOCKED
  799. assert not submission.is_finished()
  800. submission.submission_status = SubmissionStatus.RUNNING
  801. assert submission.submission_status == SubmissionStatus.RUNNING
  802. assert not submission.is_finished()
  803. submission.submission_status = SubmissionStatus.PENDING
  804. assert submission.submission_status == SubmissionStatus.PENDING
  805. assert not submission.is_finished()
  806. submission.submission_status = SubmissionStatus.COMPLETED
  807. assert submission.submission_status == SubmissionStatus.COMPLETED
  808. assert submission.is_finished()
  809. def test_execution_duration():
  810. task = Task(config_id="task_1", properties={}, function=print, id=TaskId("task_1"))
  811. submission = Submission(task.id, task._ID_PREFIX, task.config_id, properties={})
  812. job_1 = Job("job_1", task, submission.id, submission.entity_id)
  813. job_2 = Job("job_2", task, submission.id, submission.entity_id)
  814. _TaskManagerFactory._build_manager()._repository._save(task)
  815. _SubmissionManagerFactory._build_manager()._repository._save(submission)
  816. _JobManagerFactory._build_manager()._repository._save(job_1)
  817. _JobManagerFactory._build_manager()._repository._save(job_2)
  818. submission.jobs = [job_1, job_2]
  819. _SubmissionManagerFactory._build_manager()._update(submission)
  820. with freezegun.freeze_time("2024-09-25 13:30:35"):
  821. job_1.running()
  822. job_2.pending()
  823. assert submission.run_at == datetime(2024, 9, 25, 13, 30, 35)
  824. assert submission.execution_duration > 0
  825. with freezegun.freeze_time("2024-09-25 13:33:45"):
  826. job_1.completed()
  827. job_2.running()
  828. assert submission.execution_duration == 190 # = 13:33:45 - 13:30:35
  829. assert submission.run_at == datetime(2024, 9, 25, 13, 30, 35)
  830. # Job 2 is not completed, so the submission is not completed
  831. assert submission.finished_at is None
  832. with freezegun.freeze_time("2024-09-25 13:35:50"):
  833. job_2.completed()
  834. assert submission.finished_at == datetime(2024, 9, 25, 13, 35, 50)
  835. assert submission.execution_duration == 315 # = 13:35:50 - 13:30:35