Sitewide index for AI (#4688)

This PR fix #3412 by serving another 2 JSON files in addition to the
existing `search_index.json`.

- /static/sitewide_index.json: Compared to `search_index.json`, it has
the code of the documentation pages, but it does not have the examples

- /static/examples_index.json: Just the examples as we would otherwise
see in `search_index.json`

This greatly benefits AI as they can learn about the syntax of NiceGUI,
and no need to worry about 2 mixed items in the `search_index.json`
muddying the waters and making AI difficult to learn from the pattern.

---

There should not be any impact on the existing NiceGUI website.

---------

Co-authored-by: Falko Schindler <falko@zauberzeug.com>

website/documentation/code_extraction.py

@@ -0,0 +1,33 @@
+import inspect
+import re
+from typing import Callable
+
+import isort
+
+UNCOMMENT_PATTERN = re.compile(r'^(\s*)# ?')
+
+
+def _uncomment(text: str) -> str:
+    return UNCOMMENT_PATTERN.sub(r'\1', text)  # NOTE: non-executed lines should be shown in the code examples
+
+
+def get_full_code(f: Callable) -> str:
+    """Get the full code of a function as a string."""
+    code = inspect.getsource(f).split('# END OF DEMO', 1)[0].strip().splitlines()
+    code = [line for line in code if not line.endswith('# HIDE')]
+    while not code[0].strip().startswith(('def', 'async def')):
+        del code[0]
+    del code[0]
+    if code[0].strip().startswith('"""'):
+        while code[0].strip() != '"""':
+            del code[0]
+        del code[0]
+    indentation = len(code[0]) - len(code[0].lstrip())
+    code = [line[indentation:] for line in code]
+    code = ['from nicegui import ui'] + [_uncomment(line) for line in code]
+    code = ['' if line == '#' else line for line in code]
+    if not code[-1].startswith('ui.run('):
+        code.append('')
+        code.append('ui.run()')
+    full_code = isort.code('\n'.join(code), no_sections=True, lines_after_imports=1)
+    return full_code

website/documentation/demo.py

@@ -1,41 +1,16 @@
-import inspect
-import re
 from typing import Callable, Optional, Union
 
-import isort
-
 from nicegui import helpers, json, ui
 
+from .code_extraction import get_full_code
 from .intersection_observer import IntersectionObserver as intersection_observer
 from .windows import browser_window, python_window
 
-UNCOMMENT_PATTERN = re.compile(r'^(\s*)# ?')
-
-
-def _uncomment(text: str) -> str:
-    return UNCOMMENT_PATTERN.sub(r'\1', text)  # NOTE: non-executed lines should be shown in the code examples
-
 
 def demo(f: Callable, *, lazy: bool = True, tab: Optional[Union[str, Callable]] = None) -> Callable:
     """Render a callable as a demo with Python code and browser window."""
     with ui.column().classes('w-full items-stretch gap-8 no-wrap min-[1500px]:flex-row'):
-        code = inspect.getsource(f).split('# END OF DEMO', 1)[0].strip().splitlines()
-        code = [line for line in code if not line.endswith('# HIDE')]
-        while not code[0].strip().startswith(('def', 'async def')):
-            del code[0]
-        del code[0]
-        if code[0].strip().startswith('"""'):
-            while code[0].strip() != '"""':
-                del code[0]
-            del code[0]
-        indentation = len(code[0]) - len(code[0].lstrip())
-        code = [line[indentation:] for line in code]
-        code = ['from nicegui import ui'] + [_uncomment(line) for line in code]
-        code = ['' if line == '#' else line for line in code]
-        if not code[-1].startswith('ui.run('):
-            code.append('')
-            code.append('ui.run()')
-        full_code = isort.code('\n'.join(code), no_sections=True, lines_after_imports=1)
+        full_code = get_full_code(f)
         with python_window(classes='w-full max-w-[44rem]'):
             ui.markdown(f'````python\n{full_code}\n````')
             ui.icon('content_copy', size='xs') \

website/documentation/search.py

@@ -6,10 +6,13 @@ from fastapi.responses import JSONResponse
 from nicegui import app
 
 from ..examples import examples
+from .code_extraction import get_full_code
 from .content import registry
 
 PATH = Path(__file__).parent.parent / 'static' / 'search_index.json'
 search_index: List[Dict[str, str]] = []
+sitewide_index: List[Dict[str, str]] = []
+examples_index: List[Dict[str, str]] = []
 
 
 @app.get('/static/search_index.json')
@@ -17,20 +20,39 @@ def _get_search_index() -> JSONResponse:
     return JSONResponse(search_index)
 
 
+@app.get('/static/sitewide_index.json')
+def _get_sitewide_index() -> JSONResponse:
+    return JSONResponse(sitewide_index)
+
+
+@app.get('/static/examples_index.json')
+def _get_examples_index() -> JSONResponse:
+    return JSONResponse(examples_index)
+
+
 def build_search_index() -> None:
     """Build search index."""
-    search_index.clear()
-    search_index.extend([
+    search_index[:] = _collect_documentation_parts(include_code=False) + _collect_examples()
+    sitewide_index[:] = _collect_documentation_parts(include_code=True)
+    examples_index[:] = _collect_examples()
+
+
+def _collect_documentation_parts(*, include_code: bool = False) -> List[Dict[str, str]]:
+    return [
         {
             'title': f'{documentation.heading.replace("*", "")}: {part.title}',
             'content': part.description or part.search_text or '',
             'format': part.description_format,
+            **({'demo': get_full_code(part.demo.function) if part.demo is not None else ''} if include_code else {}),
             'url': f'/documentation/{documentation.name}#{part.link_target}',
         }
         for documentation in registry.values()
         for part in documentation.parts
-    ])
-    search_index.extend([
+    ]
+
+
+def _collect_examples() -> List[Dict[str, str]]:
+    return [
         {
             'title': f'Example: {example.title}',
             'content': example.description,
@@ -38,4 +60,4 @@ def build_search_index() -> None:
             'url': example.url,
         }
         for example in examples
-    ])
+    ]