pyi_generator.py 30 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937
  1. """The pyi generator module."""
  2. from __future__ import annotations
  3. import ast
  4. import contextlib
  5. import importlib
  6. import inspect
  7. import logging
  8. import re
  9. import subprocess
  10. import textwrap
  11. import typing
  12. from inspect import getfullargspec
  13. from multiprocessing import Pool, cpu_count
  14. from pathlib import Path
  15. from types import ModuleType, SimpleNamespace
  16. from typing import Any, Callable, Iterable, Type, get_args
  17. try:
  18. import black
  19. import black.mode
  20. except ImportError:
  21. black = None
  22. from reflex.components.component import Component
  23. from reflex.utils import types as rx_types
  24. from reflex.vars import Var
  25. logger = logging.getLogger("pyi_generator")
  26. INIT_FILE = Path("reflex/__init__.pyi").resolve()
  27. PWD = Path(".").resolve()
  28. EXCLUDED_FILES = [
  29. "__init__.py",
  30. # "app.py",
  31. "component.py",
  32. "bare.py",
  33. "foreach.py",
  34. "cond.py",
  35. "match.py",
  36. "multiselect.py",
  37. "literals.py",
  38. ]
  39. # These props exist on the base component, but should not be exposed in create methods.
  40. EXCLUDED_PROPS = [
  41. "alias",
  42. "children",
  43. "event_triggers",
  44. "library",
  45. "lib_dependencies",
  46. "tag",
  47. "is_default",
  48. "special_props",
  49. "_invalid_children",
  50. "_memoization_mode",
  51. "_rename_props",
  52. "_valid_children",
  53. "_valid_parents",
  54. "State",
  55. ]
  56. DEFAULT_TYPING_IMPORTS = {
  57. "overload",
  58. "Any",
  59. "Dict",
  60. # "List",
  61. "Literal",
  62. "Optional",
  63. "Union",
  64. }
  65. def _walk_files(path):
  66. """Walk all files in a path.
  67. This can be replaced with Path.walk() in python3.12.
  68. Args:
  69. path: The path to walk.
  70. Yields:
  71. The next file in the path.
  72. """
  73. for p in Path(path).iterdir():
  74. if p.is_dir():
  75. yield from _walk_files(p)
  76. continue
  77. yield p.resolve()
  78. def _relative_to_pwd(path: Path) -> Path:
  79. """Get the relative path of a path to the current working directory.
  80. Args:
  81. path: The path to get the relative path for.
  82. Returns:
  83. The relative path.
  84. """
  85. if path.is_absolute():
  86. return path.relative_to(PWD)
  87. return path
  88. def _get_type_hint(value, type_hint_globals, is_optional=True) -> str:
  89. """Resolve the type hint for value.
  90. Args:
  91. value: The type annotation as a str or actual types/aliases.
  92. type_hint_globals: The globals to use to resolving a type hint str.
  93. is_optional: Whether the type hint should be wrapped in Optional.
  94. Returns:
  95. The resolved type hint as a str.
  96. """
  97. res = ""
  98. args = get_args(value)
  99. if value is type(None):
  100. return "None"
  101. if rx_types.is_union(value):
  102. if type(None) in value.__args__:
  103. res_args = [
  104. _get_type_hint(arg, type_hint_globals, rx_types.is_optional(arg))
  105. for arg in value.__args__
  106. if arg is not type(None)
  107. ]
  108. if len(res_args) == 1:
  109. return f"Optional[{res_args[0]}]"
  110. else:
  111. res = f"Union[{', '.join(res_args)}]"
  112. return f"Optional[{res}]"
  113. res_args = [
  114. _get_type_hint(arg, type_hint_globals, rx_types.is_optional(arg))
  115. for arg in value.__args__
  116. ]
  117. return f"Union[{', '.join(res_args)}]"
  118. if args:
  119. inner_container_type_args = (
  120. [repr(arg) for arg in args]
  121. if rx_types.is_literal(value)
  122. else [
  123. _get_type_hint(arg, type_hint_globals, is_optional=False)
  124. for arg in args
  125. if arg is not type(None)
  126. ]
  127. )
  128. res = f"{value.__name__}[{', '.join(inner_container_type_args)}]"
  129. if value.__name__ == "Var":
  130. # For Var types, Union with the inner args so they can be passed directly.
  131. types = [res] + [
  132. _get_type_hint(arg, type_hint_globals, is_optional=False)
  133. for arg in args
  134. if arg is not type(None)
  135. ]
  136. if len(types) > 1:
  137. res = ", ".join(types)
  138. res = f"Union[{res}]"
  139. elif isinstance(value, str):
  140. ev = eval(value, type_hint_globals)
  141. if rx_types.is_optional(ev):
  142. # hints = {
  143. # _get_type_hint(arg, type_hint_globals, is_optional=False)
  144. # for arg in ev.__args__
  145. # }
  146. return _get_type_hint(ev, type_hint_globals, is_optional=False)
  147. # return f"Optional[{', '.join(hints)}]"
  148. if rx_types.is_union(ev):
  149. res = [
  150. _get_type_hint(arg, type_hint_globals, rx_types.is_optional(arg))
  151. for arg in ev.__args__
  152. ]
  153. return f"Union[{', '.join(res)}]"
  154. res = (
  155. _get_type_hint(ev, type_hint_globals, is_optional=False)
  156. if ev.__name__ == "Var"
  157. else value
  158. )
  159. else:
  160. res = value.__name__
  161. if is_optional and not res.startswith("Optional"):
  162. res = f"Optional[{res}]"
  163. return res
  164. def _generate_imports(typing_imports: Iterable[str]) -> list[ast.ImportFrom]:
  165. """Generate the import statements for the stub file.
  166. Args:
  167. typing_imports: The typing imports to include.
  168. Returns:
  169. The list of import statements.
  170. """
  171. return [
  172. ast.ImportFrom(
  173. module="typing",
  174. names=[ast.alias(name=imp) for imp in sorted(typing_imports)],
  175. ),
  176. *ast.parse( # type: ignore
  177. textwrap.dedent(
  178. """
  179. from reflex.vars import Var, BaseVar, ComputedVar
  180. from reflex.event import EventChain, EventHandler, EventSpec
  181. from reflex.style import Style"""
  182. )
  183. ).body,
  184. # *[
  185. # ast.ImportFrom(module=name, names=[ast.alias(name=val) for val in values])
  186. # for name, values in EXTRA_IMPORTS.items()
  187. # ],
  188. ]
  189. def _generate_docstrings(clzs: list[Type[Component]], props: list[str]) -> str:
  190. """Generate the docstrings for the create method.
  191. Args:
  192. clzs: The classes to generate docstrings for.
  193. props: The props to generate docstrings for.
  194. Returns:
  195. The docstring for the create method.
  196. """
  197. props_comments = {}
  198. comments = []
  199. for clz in clzs:
  200. for line in inspect.getsource(clz).splitlines():
  201. reached_functions = re.search("def ", line)
  202. if reached_functions:
  203. # We've reached the functions, so stop.
  204. break
  205. # Get comments for prop
  206. if line.strip().startswith("#"):
  207. comments.append(line)
  208. continue
  209. # Check if this line has a prop.
  210. match = re.search("\\w+:", line)
  211. if match is None:
  212. # This line doesn't have a var, so continue.
  213. continue
  214. # Get the prop.
  215. prop = match.group(0).strip(":")
  216. if prop in props:
  217. if not comments: # do not include undocumented props
  218. continue
  219. props_comments[prop] = [
  220. comment.strip().strip("#") for comment in comments
  221. ]
  222. comments.clear()
  223. clz = clzs[0]
  224. new_docstring = []
  225. for line in (clz.create.__doc__ or "").splitlines():
  226. if "**" in line:
  227. indent = line.split("**")[0]
  228. for nline in [
  229. f"{indent}{n}:{' '.join(c)}" for n, c in props_comments.items()
  230. ]:
  231. new_docstring.append(nline)
  232. new_docstring.append(line)
  233. return "\n".join(new_docstring)
  234. def _extract_func_kwargs_as_ast_nodes(
  235. func: Callable,
  236. type_hint_globals: dict[str, Any],
  237. ) -> list[tuple[ast.arg, ast.Constant | None]]:
  238. """Get the kwargs already defined on the function.
  239. Args:
  240. func: The function to extract kwargs from.
  241. type_hint_globals: The globals to use to resolving a type hint str.
  242. Returns:
  243. The list of kwargs as ast arg nodes.
  244. """
  245. spec = getfullargspec(func)
  246. kwargs = []
  247. for kwarg in spec.kwonlyargs:
  248. arg = ast.arg(arg=kwarg)
  249. if kwarg in spec.annotations:
  250. arg.annotation = ast.Name(
  251. id=_get_type_hint(spec.annotations[kwarg], type_hint_globals)
  252. )
  253. default = None
  254. if spec.kwonlydefaults is not None and kwarg in spec.kwonlydefaults:
  255. default = ast.Constant(value=spec.kwonlydefaults[kwarg])
  256. kwargs.append((arg, default))
  257. return kwargs
  258. def _extract_class_props_as_ast_nodes(
  259. func: Callable,
  260. clzs: list[Type],
  261. type_hint_globals: dict[str, Any],
  262. extract_real_default: bool = False,
  263. ) -> list[tuple[ast.arg, ast.Constant | None]]:
  264. """Get the props defined on the class and all parents.
  265. Args:
  266. func: The function that kwargs will be added to.
  267. clzs: The classes to extract props from.
  268. type_hint_globals: The globals to use to resolving a type hint str.
  269. extract_real_default: Whether to extract the real default value from the
  270. pydantic field definition.
  271. Returns:
  272. The list of props as ast arg nodes
  273. """
  274. spec = getfullargspec(func)
  275. all_props = []
  276. kwargs = []
  277. for target_class in clzs:
  278. # Import from the target class to ensure type hints are resolvable.
  279. exec(f"from {target_class.__module__} import *", type_hint_globals)
  280. for name, value in target_class.__annotations__.items():
  281. if (
  282. name in spec.kwonlyargs
  283. or name in EXCLUDED_PROPS
  284. or name in all_props
  285. or (isinstance(value, str) and "ClassVar" in value)
  286. ):
  287. continue
  288. all_props.append(name)
  289. default = None
  290. if extract_real_default:
  291. # TODO: This is not currently working since the default is not type compatible
  292. # with the annotation in some cases.
  293. with contextlib.suppress(AttributeError, KeyError):
  294. # Try to get default from pydantic field definition.
  295. default = target_class.__fields__[name].default
  296. if isinstance(default, Var):
  297. default = default._decode() # type: ignore
  298. kwargs.append(
  299. (
  300. ast.arg(
  301. arg=name,
  302. annotation=ast.Name(
  303. id=_get_type_hint(value, type_hint_globals)
  304. ),
  305. ),
  306. ast.Constant(value=default),
  307. )
  308. )
  309. return kwargs
  310. def _get_parent_imports(func):
  311. _imports = {"reflex.vars": ["Var"]}
  312. for type_hint in inspect.get_annotations(func).values():
  313. try:
  314. match = re.match(r"\w+\[([\w\d]+)\]", type_hint)
  315. except TypeError:
  316. continue
  317. if match:
  318. type_hint = match.group(1)
  319. if type_hint in importlib.import_module(func.__module__).__dir__():
  320. _imports.setdefault(func.__module__, []).append(type_hint)
  321. return _imports
  322. def _generate_component_create_functiondef(
  323. node: ast.FunctionDef | None,
  324. clz: type[Component] | type[SimpleNamespace],
  325. type_hint_globals: dict[str, Any],
  326. ) -> ast.FunctionDef:
  327. """Generate the create function definition for a Component.
  328. Args:
  329. node: The existing create functiondef node from the ast
  330. clz: The Component class to generate the create functiondef for.
  331. type_hint_globals: The globals to use to resolving a type hint str.
  332. Returns:
  333. The create functiondef node for the ast.
  334. Raises:
  335. TypeError: If clz is not a subclass of Component.
  336. """
  337. if not issubclass(clz, Component):
  338. raise TypeError(f"clz must be a subclass of Component, not {clz!r}")
  339. # add the imports needed by get_type_hint later
  340. type_hint_globals.update(
  341. {name: getattr(typing, name) for name in DEFAULT_TYPING_IMPORTS}
  342. )
  343. if clz.__module__ != clz.create.__module__:
  344. _imports = _get_parent_imports(clz.create)
  345. for name, values in _imports.items():
  346. exec(f"from {name} import {','.join(values)}", type_hint_globals)
  347. kwargs = _extract_func_kwargs_as_ast_nodes(clz.create, type_hint_globals)
  348. # kwargs associated with props defined in the class and its parents
  349. all_classes = [c for c in clz.__mro__ if issubclass(c, Component)]
  350. prop_kwargs = _extract_class_props_as_ast_nodes(
  351. clz.create, all_classes, type_hint_globals
  352. )
  353. all_props = [arg[0].arg for arg in prop_kwargs]
  354. kwargs.extend(prop_kwargs)
  355. # event handler kwargs
  356. kwargs.extend(
  357. (
  358. ast.arg(
  359. arg=trigger,
  360. annotation=ast.Name(
  361. id="Optional[Union[EventHandler, EventSpec, list, function, BaseVar]]"
  362. ),
  363. ),
  364. ast.Constant(value=None),
  365. )
  366. for trigger in sorted(clz().get_event_triggers().keys())
  367. )
  368. logger.debug(f"Generated {clz.__name__}.create method with {len(kwargs)} kwargs")
  369. create_args = ast.arguments(
  370. args=[ast.arg(arg="cls")],
  371. posonlyargs=[],
  372. vararg=ast.arg(arg="children"),
  373. kwonlyargs=[arg[0] for arg in kwargs],
  374. kw_defaults=[arg[1] for arg in kwargs],
  375. kwarg=ast.arg(arg="props"),
  376. defaults=[],
  377. )
  378. definition = ast.FunctionDef(
  379. name="create",
  380. args=create_args,
  381. body=[
  382. ast.Expr(
  383. value=ast.Constant(value=_generate_docstrings(all_classes, all_props))
  384. ),
  385. ast.Expr(
  386. value=ast.Ellipsis(),
  387. ),
  388. ],
  389. decorator_list=[
  390. ast.Name(id="overload"),
  391. *(
  392. node.decorator_list
  393. if node is not None
  394. else [ast.Name(id="classmethod")]
  395. ),
  396. ],
  397. lineno=node.lineno if node is not None else None,
  398. returns=ast.Constant(value=clz.__name__),
  399. )
  400. return definition
  401. def _generate_staticmethod_call_functiondef(
  402. node: ast.FunctionDef | None,
  403. clz: type[Component] | type[SimpleNamespace],
  404. type_hint_globals: dict[str, Any],
  405. ) -> ast.FunctionDef | None:
  406. ...
  407. fullspec = getfullargspec(clz.__call__)
  408. call_args = ast.arguments(
  409. args=[
  410. ast.arg(
  411. name,
  412. annotation=ast.Name(
  413. id=_get_type_hint(
  414. anno := fullspec.annotations[name],
  415. type_hint_globals,
  416. is_optional=rx_types.is_optional(anno),
  417. )
  418. ),
  419. )
  420. for name in fullspec.args
  421. ],
  422. posonlyargs=[],
  423. kwonlyargs=[],
  424. kw_defaults=[],
  425. kwarg=ast.arg(arg="props"),
  426. defaults=[],
  427. )
  428. definition = ast.FunctionDef(
  429. name="__call__",
  430. args=call_args,
  431. body=[
  432. ast.Expr(value=ast.Constant(value=clz.__call__.__doc__)),
  433. ast.Expr(
  434. value=ast.Constant(...),
  435. ),
  436. ],
  437. decorator_list=[ast.Name(id="staticmethod")],
  438. lineno=node.lineno if node is not None else None,
  439. returns=ast.Constant(
  440. value=_get_type_hint(
  441. typing.get_type_hints(clz.__call__).get("return", None),
  442. type_hint_globals,
  443. )
  444. ),
  445. )
  446. return definition
  447. def _generate_namespace_call_functiondef(
  448. node: ast.ClassDef | None,
  449. clz_name: str,
  450. classes: dict[str, type[Component] | type[SimpleNamespace]],
  451. type_hint_globals: dict[str, Any],
  452. ) -> ast.FunctionDef | None:
  453. """Generate the __call__ function definition for a SimpleNamespace.
  454. Args:
  455. node: The existing __call__ classdef parent node from the ast
  456. clz_name: The name of the SimpleNamespace class to generate the __call__ functiondef for.
  457. classes: Map name to actual class definition.
  458. type_hint_globals: The globals to use to resolving a type hint str.
  459. Returns:
  460. The create functiondef node for the ast.
  461. """
  462. # add the imports needed by get_type_hint later
  463. type_hint_globals.update(
  464. {name: getattr(typing, name) for name in DEFAULT_TYPING_IMPORTS}
  465. )
  466. clz = classes[clz_name]
  467. if not hasattr(clz.__call__, "__self__"):
  468. return _generate_staticmethod_call_functiondef(node, clz, type_hint_globals) # type: ignore
  469. # Determine which class is wrapped by the namespace __call__ method
  470. component_clz = clz.__call__.__self__
  471. if clz.__call__.__func__.__name__ != "create":
  472. return None
  473. definition = _generate_component_create_functiondef(
  474. node=None,
  475. clz=component_clz, # type: ignore
  476. type_hint_globals=type_hint_globals,
  477. )
  478. definition.name = "__call__"
  479. # Turn the definition into a staticmethod
  480. del definition.args.args[0] # remove `cls` arg
  481. definition.decorator_list = [ast.Name(id="staticmethod")]
  482. return definition
  483. class StubGenerator(ast.NodeTransformer):
  484. """A node transformer that will generate the stubs for a given module."""
  485. def __init__(
  486. self, module: ModuleType, classes: dict[str, Type[Component | SimpleNamespace]]
  487. ):
  488. """Initialize the stub generator.
  489. Args:
  490. module: The actual module object module to generate stubs for.
  491. classes: The actual Component class objects to generate stubs for.
  492. """
  493. super().__init__()
  494. # Dict mapping class name to actual class object.
  495. self.classes = classes
  496. # Track the last class node that was visited.
  497. self.current_class = None
  498. # These imports will be included in the AST of stub files.
  499. self.typing_imports = DEFAULT_TYPING_IMPORTS
  500. # Whether those typing imports have been inserted yet.
  501. self.inserted_imports = False
  502. # Collected import statements from the module.
  503. self.import_statements: list[str] = []
  504. # This dict is used when evaluating type hints.
  505. self.type_hint_globals = module.__dict__.copy()
  506. @staticmethod
  507. def _remove_docstring(
  508. node: ast.Module | ast.ClassDef | ast.FunctionDef,
  509. ) -> ast.Module | ast.ClassDef | ast.FunctionDef:
  510. """Removes any docstring in place.
  511. Args:
  512. node: The node to remove the docstring from.
  513. Returns:
  514. The modified node.
  515. """
  516. if (
  517. node.body
  518. and isinstance(node.body[0], ast.Expr)
  519. and isinstance(node.body[0].value, ast.Constant)
  520. ):
  521. node.body.pop(0)
  522. return node
  523. def _current_class_is_component(self) -> bool:
  524. """Check if the current class is a Component.
  525. Returns:
  526. Whether the current class is a Component.
  527. """
  528. return (
  529. self.current_class is not None
  530. and self.current_class in self.classes
  531. and issubclass(self.classes[self.current_class], Component)
  532. )
  533. def visit_Module(self, node: ast.Module) -> ast.Module:
  534. """Visit a Module node and remove docstring from body.
  535. Args:
  536. node: The Module node to visit.
  537. Returns:
  538. The modified Module node.
  539. """
  540. self.generic_visit(node)
  541. return self._remove_docstring(node) # type: ignore
  542. def visit_Import(
  543. self, node: ast.Import | ast.ImportFrom
  544. ) -> ast.Import | ast.ImportFrom | list[ast.Import | ast.ImportFrom]:
  545. """Collect import statements from the module.
  546. If this is the first import statement, insert the typing imports before it.
  547. Args:
  548. node: The import node to visit.
  549. Returns:
  550. The modified import node(s).
  551. """
  552. self.import_statements.append(ast.unparse(node))
  553. if not self.inserted_imports:
  554. self.inserted_imports = True
  555. return _generate_imports(self.typing_imports) + [node]
  556. return node
  557. def visit_ImportFrom(
  558. self, node: ast.ImportFrom
  559. ) -> ast.Import | ast.ImportFrom | list[ast.Import | ast.ImportFrom] | None:
  560. """Visit an ImportFrom node.
  561. Remove any `from __future__ import *` statements, and hand off to visit_Import.
  562. Args:
  563. node: The ImportFrom node to visit.
  564. Returns:
  565. The modified ImportFrom node.
  566. """
  567. if node.module == "__future__":
  568. return None # ignore __future__ imports
  569. return self.visit_Import(node)
  570. def visit_ClassDef(self, node: ast.ClassDef) -> ast.ClassDef:
  571. """Visit a ClassDef node.
  572. Remove all assignments in the class body, and add a create functiondef
  573. if one does not exist.
  574. Args:
  575. node: The ClassDef node to visit.
  576. Returns:
  577. The modified ClassDef node.
  578. """
  579. exec("\n".join(self.import_statements), self.type_hint_globals)
  580. self.current_class = node.name
  581. self._remove_docstring(node)
  582. # Define `__call__` as a real function so the docstring appears in the stub.
  583. call_definition = None
  584. for child in node.body[:]:
  585. found_call = False
  586. if isinstance(child, ast.Assign):
  587. for target in child.targets[:]:
  588. if isinstance(target, ast.Name) and target.id == "__call__":
  589. child.targets.remove(target)
  590. found_call = True
  591. if not found_call:
  592. continue
  593. if not child.targets[:]:
  594. node.body.remove(child)
  595. call_definition = _generate_namespace_call_functiondef(
  596. node,
  597. self.current_class,
  598. self.classes,
  599. type_hint_globals=self.type_hint_globals,
  600. )
  601. break
  602. self.generic_visit(node) # Visit child nodes.
  603. if (
  604. not any(
  605. isinstance(child, ast.FunctionDef) and child.name == "create"
  606. for child in node.body
  607. )
  608. and self._current_class_is_component()
  609. ):
  610. # Add a new .create FunctionDef since one does not exist.
  611. node.body.append(
  612. _generate_component_create_functiondef(
  613. node=None,
  614. clz=self.classes[self.current_class],
  615. type_hint_globals=self.type_hint_globals,
  616. )
  617. )
  618. if call_definition is not None:
  619. node.body.append(call_definition)
  620. if not node.body:
  621. # We should never return an empty body.
  622. node.body.append(ast.Expr(value=ast.Ellipsis()))
  623. self.current_class = None
  624. return node
  625. def visit_FunctionDef(self, node: ast.FunctionDef) -> Any:
  626. """Visit a FunctionDef node.
  627. Special handling for `.create` functions to add type hints for all props
  628. defined on the component class.
  629. Remove all private functions and blank out the function body of the
  630. remaining public functions.
  631. Args:
  632. node: The FunctionDef node to visit.
  633. Returns:
  634. The modified FunctionDef node (or None).
  635. """
  636. if node.name == "create" and self.current_class in self.classes:
  637. node = _generate_component_create_functiondef(
  638. node, self.classes[self.current_class], self.type_hint_globals
  639. )
  640. else:
  641. if node.name.startswith("_") and node.name != "__call__":
  642. return None # remove private methods
  643. if node.body[-1] != ast.Expr(value=ast.Ellipsis()):
  644. # Blank out the function body for public functions.
  645. node.body = [ast.Expr(value=ast.Ellipsis())]
  646. return node
  647. def visit_Assign(self, node: ast.Assign) -> ast.Assign | None:
  648. """Remove non-annotated assignment statements.
  649. Args:
  650. node: The Assign node to visit.
  651. Returns:
  652. The modified Assign node (or None).
  653. """
  654. # Special case for assignments to `typing.Any` as fallback.
  655. if (
  656. node.value is not None
  657. and isinstance(node.value, ast.Name)
  658. and node.value.id == "Any"
  659. ):
  660. return node
  661. if self._current_class_is_component():
  662. # Remove annotated assignments in Component classes (props)
  663. return None
  664. return node
  665. def visit_AnnAssign(self, node: ast.AnnAssign) -> ast.AnnAssign | None:
  666. """Visit an AnnAssign node (Annotated assignment).
  667. Remove private target and remove the assignment value in the stub.
  668. Args:
  669. node: The AnnAssign node to visit.
  670. Returns:
  671. The modified AnnAssign node (or None).
  672. """
  673. # skip ClassVars
  674. if (
  675. isinstance(node.annotation, ast.Subscript)
  676. and isinstance(node.annotation.value, ast.Name)
  677. and node.annotation.value.id == "ClassVar"
  678. ):
  679. return node
  680. if isinstance(node.target, ast.Name) and node.target.id.startswith("_"):
  681. return None
  682. if self.current_class in self.classes:
  683. # Remove annotated assignments in Component classes (props)
  684. return None
  685. # Blank out assignments in type stubs.
  686. node.value = None
  687. return node
  688. class PyiGenerator:
  689. """A .pyi file generator that will scan all defined Component in Reflex and
  690. generate the approriate stub.
  691. """
  692. modules: list = []
  693. root: str = ""
  694. current_module: Any = {}
  695. def _write_pyi_file(self, module_path: Path, source: str):
  696. relpath = str(_relative_to_pwd(module_path)).replace("\\", "/")
  697. pyi_content = [
  698. f'"""Stub file for {relpath}"""',
  699. "# ------------------- DO NOT EDIT ----------------------",
  700. "# This file was generated by `reflex/utils/pyi_generator.py`!",
  701. "# ------------------------------------------------------",
  702. "",
  703. ]
  704. if black is not None:
  705. for formatted_line in black.format_file_contents(
  706. src_contents=source,
  707. fast=True,
  708. mode=black.mode.Mode(is_pyi=True),
  709. ).splitlines():
  710. # Bit of a hack here, since the AST cannot represent comments.
  711. if "def create(" in formatted_line or "Figure" in formatted_line:
  712. pyi_content.append(formatted_line + " # type: ignore")
  713. else:
  714. pyi_content.append(formatted_line)
  715. pyi_content.append("") # add empty line at the end for formatting
  716. else:
  717. pyi_content = source.splitlines()
  718. pyi_path = module_path.with_suffix(".pyi")
  719. pyi_path.write_text("\n".join(pyi_content))
  720. logger.info(f"Wrote {relpath}")
  721. def _scan_file(self, module_path: Path):
  722. module_import = (
  723. _relative_to_pwd(module_path)
  724. .with_suffix("")
  725. .as_posix()
  726. .replace("/", ".")
  727. .replace("\\", ".")
  728. )
  729. module = importlib.import_module(module_import)
  730. logger.debug(f"Read {module_path}")
  731. class_names = {
  732. name: obj
  733. for name, obj in vars(module).items()
  734. if inspect.isclass(obj)
  735. and (issubclass(obj, Component) or issubclass(obj, SimpleNamespace))
  736. and obj != Component
  737. and inspect.getmodule(obj) == module
  738. }
  739. if not class_names:
  740. return
  741. new_tree = StubGenerator(module, class_names).visit(
  742. ast.parse(inspect.getsource(module))
  743. )
  744. self._write_pyi_file(module_path, ast.unparse(new_tree))
  745. def _scan_files_multiprocess(self, files: list[Path]):
  746. with Pool(processes=cpu_count()) as pool:
  747. pool.map(self._scan_file, files)
  748. def _scan_files(self, files: list[Path]):
  749. for file in files:
  750. self._scan_file(file)
  751. def scan_all(self, targets, changed_files: list[Path] | None = None):
  752. """Scan all targets for class inheriting Component and generate the .pyi files.
  753. Args:
  754. targets: the list of file/folders to scan.
  755. changed_files (optional): the list of changed files since the last run.
  756. """
  757. file_targets = []
  758. for target in targets:
  759. target_path = Path(target)
  760. if (
  761. target_path.is_file()
  762. and target_path.suffix == ".py"
  763. and target_path.name not in EXCLUDED_FILES
  764. ):
  765. file_targets.append(target_path)
  766. continue
  767. if not target_path.is_dir():
  768. continue
  769. for file_path in _walk_files(target_path):
  770. relative = _relative_to_pwd(file_path)
  771. if relative.name in EXCLUDED_FILES or file_path.suffix != ".py":
  772. continue
  773. if (
  774. changed_files is not None
  775. and _relative_to_pwd(file_path) not in changed_files
  776. ):
  777. continue
  778. file_targets.append(file_path)
  779. # check if pyi changed but not the source
  780. if changed_files is not None:
  781. for changed_file in changed_files:
  782. if changed_file.suffix != ".pyi":
  783. continue
  784. py_file_path = changed_file.with_suffix(".py")
  785. if not py_file_path.exists() and changed_file.exists():
  786. changed_file.unlink()
  787. if py_file_path in file_targets:
  788. continue
  789. subprocess.run(["git", "checkout", changed_file])
  790. if cpu_count() == 1 or len(file_targets) < 5:
  791. self._scan_files(file_targets)
  792. else:
  793. self._scan_files_multiprocess(file_targets)
  794. def generate_init():
  795. """Generate a pyi file for the main __init__.py."""
  796. from reflex import _MAPPING # type: ignore
  797. imports = [
  798. f"from {path if mod != path.rsplit('.')[-1] or mod == 'page' else '.'.join(path.rsplit('.')[:-1])} import {mod} as {mod}"
  799. for mod, path in _MAPPING.items()
  800. ]
  801. imports.append("")
  802. with contextlib.suppress(Exception):
  803. INIT_FILE.write_text("\n".join(imports))