Benchmark with app harness (#2774)

.github/workflows/benchmarks.yml

@@ -21,11 +21,10 @@ env:
   PYTHONIOENCODING: 'utf8'
   TELEMETRY_ENABLED: false
   NODE_OPTIONS: '--max_old_space_size=4096'
+  DATABASE_URL: ${{ secrets.DATABASE_URL }}
 
 jobs:
   reflex-web:
-    env:
-      DATABASE_URL: ${{ secrets.DATABASE_URL }}
     strategy:
       fail-fast: false
       matrix:
@@ -70,8 +69,58 @@ jobs:
       - name: Run Benchmarks
         # Only run if the database creds are available in this context.
         if: ${{ env.DATABASE_URL }}
-        working-directory: ./integration/benchmarks
-        run: poetry run python benchmarks.py "$GITHUB_SHA" .lighthouseci
+        run: poetry run python scripts/lighthouse_score_upload.py "$GITHUB_SHA" ./integration/benchmarks/.lighthouseci
         env:
           GITHUB_SHA: ${{ github.sha }}
           PR_TITLE: ${{ github.event.pull_request.title }}
+
+  simple-apps-benchmarks:
+    env:
+      OUTPUT_FILE: benchmarks.json
+    timeout-minutes: 50
+    strategy:
+      # Prioritize getting more information out of the workflow (even if something fails)
+      fail-fast: false
+      matrix:
+        # Show OS combos first in GUI
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        python-version: ['3.8.18', '3.9.18', '3.10.13', '3.11.5', '3.12.0']
+        exclude:
+          - os: windows-latest
+            python-version: '3.10.13'
+          - os: windows-latest
+            python-version: '3.9.18'
+          - os: windows-latest
+            python-version: '3.8.18'
+        include:
+          - os: windows-latest
+            python-version: '3.10.11'
+          - os: windows-latest
+            python-version: '3.9.13'
+          - os: windows-latest
+            python-version: '3.8.10'
+
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup_build_env
+        with:
+          python-version: ${{ matrix.python-version }}
+          run-poetry-install: true
+          create-venv-at-path: .venv
+      - name: Install additional dependencies for DB access
+        run: poetry run pip install psycopg2-binary
+      - name: Run benchmark tests
+        env:
+          APP_HARNESS_HEADLESS: 1
+          PYTHONUNBUFFERED: 1
+        run: |
+          poetry run pytest -v benchmarks/ --benchmark-json=${{ env.OUTPUT_FILE }} -s
+      - name: Upload benchmark results
+        # Only run if the database creds are available in this context.
+        if: ${{ env.DATABASE_URL }}
+        run: poetry run python scripts/simple_app_benchmark_upload.py --os "${{ matrix.os }}"
+          --python-version "${{ matrix.python-version }}" --commit-sha "${{ github.sha }}" 
+          --benchmark-json "${{ env.OUTPUT_FILE }}" --pr-title "${{ github.event.pull_request.title }}"
+          --db-url "${{ env.DATABASE_URL }}" --branch-name "${{ github.head_ref || github.ref_name }}"
+          --event-type "${{ github.event_name }}" --actor "${{ github.actor }}"

.pre-commit-config.yaml

@@ -10,6 +10,7 @@ repos:
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]
+        exclude: '^integration/benchmarks/'
 
   - repo: https://github.com/RobertCraigie/pyright-python
     rev: v1.1.313

benchmarks/__init__.py

@@ -0,0 +1,3 @@
+"""Reflex benchmarks."""
+
+WINDOWS_SKIP_REASON = "Takes too much time as a result of npm"

benchmarks/conftest.py

@@ -0,0 +1,20 @@
+"""Shared conftest for all benchmark tests."""
+
+import pytest
+
+from reflex.testing import AppHarness, AppHarnessProd
+
+
+@pytest.fixture(
+    scope="session", params=[AppHarness, AppHarnessProd], ids=["dev", "prod"]
+)
+def app_harness_env(request):
+    """Parametrize the AppHarness class to use for the test, either dev or prod.
+
+    Args:
+        request: The pytest fixture request object.
+
+    Returns:
+        The AppHarness class to use for the test.
+    """
+    return request.param

benchmarks/test_benchmark_compile_components.py

@@ -0,0 +1,370 @@
+"""Benchmark tests for apps with varying component numbers."""
+
+from __future__ import annotations
+
+import functools
+import time
+from typing import Generator
+
+import pytest
+
+from benchmarks import WINDOWS_SKIP_REASON
+from reflex import constants
+from reflex.compiler import utils
+from reflex.testing import AppHarness, chdir
+from reflex.utils import build
+
+
+def render_component(num: int):
+    """Generate a number of components based on num.
+
+    Args:
+        num: number of components to produce.
+
+    Returns:
+        The rendered number of components.
+    """
+    import reflex as rx
+
+    return [
+        rx.fragment(
+            rx.box(
+                rx.accordion.root(
+                    rx.accordion.item(
+                        header="Full Ingredients",  # type: ignore
+                        content="Yes. It's built with accessibility in mind.",  # type: ignore
+                        font_size="3em",
+                    ),
+                    rx.accordion.item(
+                        header="Applications",  # type: ignore
+                        content="Yes. It's unstyled by default, giving you freedom over the look and feel.",  # type: ignore
+                    ),
+                    collapsible=True,
+                    variant="ghost",
+                    width="25rem",
+                ),
+                padding_top="20px",
+            ),
+            rx.box(
+                rx.drawer.root(
+                    rx.drawer.trigger(
+                        rx.button("Open Drawer with snap points"), as_child=True
+                    ),
+                    rx.drawer.overlay(),
+                    rx.drawer.portal(
+                        rx.drawer.content(
+                            rx.flex(
+                                rx.drawer.title("Drawer Content"),
+                                rx.drawer.description("Drawer description"),
+                                rx.drawer.close(
+                                    rx.button("Close Button"),
+                                    as_child=True,
+                                ),
+                                direction="column",
+                                margin="5em",
+                                align_items="center",
+                            ),
+                            top="auto",
+                            height="100%",
+                            flex_direction="column",
+                            background_color="var(--green-3)",
+                        ),
+                    ),
+                    snap_points=["148px", "355px", 1],
+                ),
+            ),
+            rx.box(
+                rx.callout(
+                    "You will need admin privileges to install and access this application.",
+                    icon="info",
+                    size="3",
+                ),
+            ),
+            rx.box(
+                rx.table.root(
+                    rx.table.header(
+                        rx.table.row(
+                            rx.table.column_header_cell("Full name"),
+                            rx.table.column_header_cell("Email"),
+                            rx.table.column_header_cell("Group"),
+                        ),
+                    ),
+                    rx.table.body(
+                        rx.table.row(
+                            rx.table.row_header_cell("Danilo Sousa"),
+                            rx.table.cell("danilo@example.com"),
+                            rx.table.cell("Developer"),
+                        ),
+                        rx.table.row(
+                            rx.table.row_header_cell("Zahra Ambessa"),
+                            rx.table.cell("zahra@example.com"),
+                            rx.table.cell("Admin"),
+                        ),
+                        rx.table.row(
+                            rx.table.row_header_cell("Jasper Eriksson"),
+                            rx.table.cell("jasper@example.com"),
+                            rx.table.cell("Developer"),
+                        ),
+                    ),
+                )
+            ),
+        )
+    ] * num
+
+
+def AppWithTenComponentsOnePage():
+    """A reflex app with roughly 10 components on one page."""
+    import reflex as rx
+
+    def index() -> rx.Component:
+        return rx.center(rx.vstack(*render_component(1)))
+
+    app = rx.App(state=rx.State)
+    app.add_page(index)
+
+
+def AppWithHundredComponentOnePage():
+    """A reflex app with roughly 100 components on one page."""
+    import reflex as rx
+
+    def index() -> rx.Component:
+        return rx.center(rx.vstack(*render_component(100)))
+
+    app = rx.App(state=rx.State)
+    app.add_page(index)
+
+
+def AppWithThousandComponentsOnePage():
+    """A reflex app with roughly 1000 components on one page."""
+    import reflex as rx
+
+    def index() -> rx.Component:
+        return rx.center(rx.vstack(*render_component(1000)))
+
+    app = rx.App(state=rx.State)
+    app.add_page(index)
+
+
+@pytest.fixture(scope="session")
+def app_with_10_components(
+    tmp_path_factory,
+) -> Generator[AppHarness, None, None]:
+    """Start Blank Template app at tmp_path via AppHarness.
+
+    Args:
+        tmp_path_factory: pytest tmp_path_factory fixture
+
+    Yields:
+        running AppHarness instance
+    """
+    root = tmp_path_factory.mktemp("app10components")
+
+    yield AppHarness.create(
+        root=root,
+        app_source=functools.partial(
+            AppWithTenComponentsOnePage, render_component=render_component  # type: ignore
+        ),
+    )  # type: ignore
+
+
+@pytest.fixture(scope="session")
+def app_with_100_components(
+    tmp_path_factory,
+) -> Generator[AppHarness, None, None]:
+    """Start Blank Template app at tmp_path via AppHarness.
+
+    Args:
+        tmp_path_factory: pytest tmp_path_factory fixture
+
+    Yields:
+        running AppHarness instance
+    """
+    root = tmp_path_factory.mktemp("app100components")
+
+    yield AppHarness.create(
+        root=root,
+        app_source=functools.partial(
+            AppWithHundredComponentOnePage, render_component=render_component  # type: ignore
+        ),
+    )  # type: ignore
+
+
+@pytest.fixture(scope="session")
+def app_with_1000_components(
+    tmp_path_factory,
+) -> Generator[AppHarness, None, None]:
+    """Create an app with 1000 components at tmp_path via AppHarness.
+
+    Args:
+        tmp_path_factory: pytest tmp_path_factory fixture
+
+    Yields:
+        an AppHarness instance
+    """
+    root = tmp_path_factory.mktemp("app1000components")
+
+    yield AppHarness.create(
+        root=root,
+        app_source=functools.partial(
+            AppWithThousandComponentsOnePage, render_component=render_component  # type: ignore
+        ),
+    )  # type: ignore
+
+
+@pytest.mark.skipif(constants.IS_WINDOWS, reason=WINDOWS_SKIP_REASON)
+@pytest.mark.benchmark(
+    group="Compile time of varying component numbers",
+    timer=time.perf_counter,
+    disable_gc=True,
+    warmup=False,
+)
+def test_app_10_compile_time_cold(benchmark, app_with_10_components):
+    """Test the compile time on a cold start for an app with roughly 10 components.
+
+    Args:
+        benchmark: The benchmark fixture.
+        app_with_10_components: The app harness.
+    """
+
+    def setup():
+        with chdir(app_with_10_components.app_path):
+            utils.empty_dir(constants.Dirs.WEB_PAGES, keep_files=["_app.js"])
+            app_with_10_components._initialize_app()
+            build.setup_frontend(app_with_10_components.app_path)
+
+    def benchmark_fn():
+        with chdir(app_with_10_components.app_path):
+            app_with_10_components.app_instance.compile_()
+
+    benchmark.pedantic(benchmark_fn, setup=setup, rounds=10)
+
+
+@pytest.mark.benchmark(
+    group="Compile time of varying component numbers",
+    min_rounds=5,
+    timer=time.perf_counter,
+    disable_gc=True,
+    warmup=False,
+)
+def test_app_10_compile_time_warm(benchmark, app_with_10_components):
+    """Test the compile time on a warm start for an app with roughly 10 components.
+
+    Args:
+        benchmark: The benchmark fixture.
+        app_with_10_components: The app harness.
+    """
+    with chdir(app_with_10_components.app_path):
+        app_with_10_components._initialize_app()
+        build.setup_frontend(app_with_10_components.app_path)
+
+    def benchmark_fn():
+        with chdir(app_with_10_components.app_path):
+            app_with_10_components.app_instance.compile_()
+
+    benchmark(benchmark_fn)
+
+
+@pytest.mark.skipif(constants.IS_WINDOWS, reason=WINDOWS_SKIP_REASON)
+@pytest.mark.benchmark(
+    group="Compile time of varying component numbers",
+    timer=time.perf_counter,
+    disable_gc=True,
+    warmup=False,
+)
+def test_app_100_compile_time_cold(benchmark, app_with_100_components):
+    """Test the compile time on a cold start for an app with roughly 100 components.
+
+    Args:
+        benchmark: The benchmark fixture.
+        app_with_100_components: The app harness.
+    """
+
+    def setup():
+        with chdir(app_with_100_components.app_path):
+            utils.empty_dir(constants.Dirs.WEB_PAGES, keep_files=["_app.js"])
+            app_with_100_components._initialize_app()
+            build.setup_frontend(app_with_100_components.app_path)
+
+    def benchmark_fn():
+        with chdir(app_with_100_components.app_path):
+            app_with_100_components.app_instance.compile_()
+
+    benchmark.pedantic(benchmark_fn, setup=setup, rounds=5)
+
+
+@pytest.mark.benchmark(
+    group="Compile time of varying component numbers",
+    min_rounds=5,
+    timer=time.perf_counter,
+    disable_gc=True,
+    warmup=False,
+)
+def test_app_100_compile_time_warm(benchmark, app_with_100_components):
+    """Test the compile time on a warm start for an app with roughly 100 components.
+
+    Args:
+        benchmark: The benchmark fixture.
+        app_with_100_components: The app harness.
+    """
+    with chdir(app_with_100_components.app_path):
+        app_with_100_components._initialize_app()
+        build.setup_frontend(app_with_100_components.app_path)
+
+    def benchmark_fn():
+        with chdir(app_with_100_components.app_path):
+            app_with_100_components.app_instance.compile_()
+
+    benchmark(benchmark_fn)
+
+
+@pytest.mark.skipif(constants.IS_WINDOWS, reason=WINDOWS_SKIP_REASON)
+@pytest.mark.benchmark(
+    group="Compile time of varying component numbers",
+    timer=time.perf_counter,
+    disable_gc=True,
+    warmup=False,
+)
+def test_app_1000_compile_time_cold(benchmark, app_with_1000_components):
+    """Test the compile time on a cold start for an app with roughly 1000 components.
+
+    Args:
+        benchmark: The benchmark fixture.
+        app_with_1000_components: The app harness.
+    """
+
+    def setup():
+        with chdir(app_with_1000_components.app_path):
+            utils.empty_dir(constants.Dirs.WEB_PAGES, keep_files=["_app.js"])
+            app_with_1000_components._initialize_app()
+            build.setup_frontend(app_with_1000_components.app_path)
+
+    def benchmark_fn():
+        with chdir(app_with_1000_components.app_path):
+            app_with_1000_components.app_instance.compile_()
+
+    benchmark.pedantic(benchmark_fn, setup=setup, rounds=5)
+
+
+@pytest.mark.benchmark(
+    group="Compile time of varying component numbers",
+    min_rounds=5,
+    timer=time.perf_counter,
+    disable_gc=True,
+    warmup=False,
+)
+def test_app_1000_compile_time_warm(benchmark, app_with_1000_components):
+    """Test the compile time on a warm start for an app with roughly 1000 components.
+
+    Args:
+        benchmark: The benchmark fixture.
+        app_with_1000_components: The app harness.
+    """
+    with chdir(app_with_1000_components.app_path):
+        app_with_1000_components._initialize_app()
+        build.setup_frontend(app_with_1000_components.app_path)
+
+    def benchmark_fn():
+        with chdir(app_with_1000_components.app_path):
+            app_with_1000_components.app_instance.compile_()
+
+    benchmark(benchmark_fn)

benchmarks/test_benchmark_compile_pages.py

@@ -0,0 +1,557 @@
+"""Benchmark tests for apps with varying page numbers."""
+
+from __future__ import annotations
+
+import functools
+import time
+from typing import Generator
+
+import pytest
+
+from benchmarks import WINDOWS_SKIP_REASON
+from reflex import constants
+from reflex.compiler import utils
+from reflex.testing import AppHarness, chdir
+from reflex.utils import build
+
+
+def render_multiple_pages(app, num: int):
+    """Add multiple pages based on num.
+
+    Args:
+        app: The App object.
+        num: number of pages to render.
+
+    """
+    from typing import Tuple
+
+    from rxconfig import config  # type: ignore
+
+    import reflex as rx
+
+    docs_url = "https://reflex.dev/docs/getting-started/introduction/"
+    filename = f"{config.app_name}/{config.app_name}.py"
+    college = [
+        "Stanford University",
+        "Arizona",
+        "Arizona state",
+        "Baylor",
+        "Boston College",
+        "Boston University",
+    ]
+
+    class State(rx.State):
+        """The app state."""
+
+        position: str
+        college: str
+        age: Tuple[int, int] = (18, 50)
+        salary: Tuple[int, int] = (0, 25000000)
+
+    comp1 = rx.center(
+        rx.theme_panel(),
+        rx.vstack(
+            rx.heading("Welcome to Reflex!", size="9"),
+            rx.text("Get started by editing ", rx.code(filename)),
+            rx.button(
+                "Check out our docs!",
+                on_click=lambda: rx.redirect(docs_url),
+                size="4",
+            ),
+            align="center",
+            spacing="7",
+            font_size="2em",
+        ),
+        height="100vh",
+    )
+
+    comp2 = rx.vstack(
+        rx.hstack(
+            rx.vstack(
+                rx.select(
+                    ["C", "PF", "SF", "PG", "SG"],
+                    placeholder="Select a position. (All)",
+                    on_change=State.set_position,  # type: ignore
+                    size="3",
+                ),
+                rx.select(
+                    college,
+                    placeholder="Select a college. (All)",
+                    on_change=State.set_college,  # type: ignore
+                    size="3",
+                ),
+            ),
+            rx.vstack(
+                rx.vstack(
+                    rx.hstack(
+                        rx.badge("Min Age: ", State.age[0]),
+                        rx.divider(orientation="vertical"),
+                        rx.badge("Max Age: ", State.age[1]),
+                    ),
+                    rx.slider(
+                        default_value=[18, 50],
+                        min=18,
+                        max=50,
+                        on_value_commit=State.set_age,  # type: ignore
+                    ),
+                    align_items="left",
+                    width="100%",
+                ),
+                rx.vstack(
+                    rx.hstack(
+                        rx.badge("Min Sal: ", State.salary[0] // 1000000, "M"),
+                        rx.divider(orientation="vertical"),
+                        rx.badge("Max Sal: ", State.salary[1] // 1000000, "M"),
+                    ),
+                    rx.slider(
+                        default_value=[0, 25000000],
+                        min=0,
+                        max=25000000,
+                        on_value_commit=State.set_salary,  # type: ignore
+                    ),
+                    align_items="left",
+                    width="100%",
+                ),
+            ),
+            spacing="4",
+        ),
+        width="100%",
+    )
+
+    for i in range(1, num + 1):
+        if i % 2 == 1:
+            app.add_page(comp1, route=f"page{i}")
+        else:
+            app.add_page(comp2, route=f"page{i}")
+
+
+def AppWithOnePage():
+    """A reflex app with one page."""
+    from rxconfig import config  # type: ignore
+
+    import reflex as rx
+
+    docs_url = "https://reflex.dev/docs/getting-started/introduction/"
+    filename = f"{config.app_name}/{config.app_name}.py"
+
+    class State(rx.State):
+        """The app state."""
+
+        pass
+
+    def index() -> rx.Component:
+        return rx.center(
+            rx.chakra.input(
+                id="token", value=State.router.session.client_token, is_read_only=True
+            ),
+            rx.vstack(
+                rx.heading("Welcome to Reflex!", size="9"),
+                rx.text("Get started by editing ", rx.code(filename)),
+                rx.button(
+                    "Check out our docs!",
+                    on_click=lambda: rx.redirect(docs_url),
+                    size="4",
+                ),
+                align="center",
+                spacing="7",
+                font_size="2em",
+            ),
+            height="100vh",
+        )
+
+    app = rx.App(state=rx.State)
+    app.add_page(index)
+
+
+def AppWithTenPages():
+    """A reflex app with 10 pages."""
+    import reflex as rx
+
+    app = rx.App(state=rx.State)
+    render_multiple_pages(app, 10)
+
+
+def AppWithHundredPages():
+    """A reflex app with 100 pages."""
+    import reflex as rx
+
+    app = rx.App(state=rx.State)
+    render_multiple_pages(app, 100)
+
+
+def AppWithThousandPages():
+    """A reflex app with Thousand pages."""
+    import reflex as rx
+
+    app = rx.App(state=rx.State)
+    render_multiple_pages(app, 1000)
+
+
+def AppWithTenThousandPages():
+    """A reflex app with ten thousand pages."""
+    import reflex as rx
+
+    app = rx.App(state=rx.State)
+    render_multiple_pages(app, 10000)
+
+
+@pytest.fixture(scope="session")
+def app_with_one_page(
+    tmp_path_factory,
+) -> Generator[AppHarness, None, None]:
+    """Create an app with 10000 pages at tmp_path via AppHarness.
+
+    Args:
+        tmp_path_factory: pytest tmp_path_factory fixture
+
+    Yields:
+        an AppHarness instance
+    """
+    root = tmp_path_factory.mktemp(f"app1")
+
+    yield AppHarness.create(root=root, app_source=AppWithOnePage)  # type: ignore
+
+
+@pytest.fixture(scope="session")
+def app_with_ten_pages(
+    tmp_path_factory,
+) -> Generator[AppHarness, None, None]:
+    """Create an app with 10 pages at tmp_path via AppHarness.
+
+    Args:
+        tmp_path_factory: pytest tmp_path_factory fixture
+
+    Yields:
+        an AppHarness instance
+    """
+    root = tmp_path_factory.mktemp(f"app10")
+    yield AppHarness.create(root=root, app_source=functools.partial(AppWithTenPages, render_comp=render_multiple_pages))  # type: ignore
+
+
+@pytest.fixture(scope="session")
+def app_with_hundred_pages(
+    tmp_path_factory,
+) -> Generator[AppHarness, None, None]:
+    """Create an app with 100 pages at tmp_path via AppHarness.
+
+    Args:
+        tmp_path_factory: pytest tmp_path_factory fixture
+
+    Yields:
+        an AppHarness instance
+    """
+    root = tmp_path_factory.mktemp(f"app100")
+
+    yield AppHarness.create(
+        root=root,
+        app_source=functools.partial(
+            AppWithHundredPages, render_comp=render_multiple_pages  # type: ignore
+        ),
+    )  # type: ignore
+
+
+@pytest.fixture(scope="session")
+def app_with_thousand_pages(
+    tmp_path_factory,
+) -> Generator[AppHarness, None, None]:
+    """Create an app with 1000 pages at tmp_path via AppHarness.
+
+    Args:
+        tmp_path_factory: pytest tmp_path_factory fixture
+
+    Yields:
+        an AppHarness instance
+    """
+    root = tmp_path_factory.mktemp(f"app1000")
+
+    yield AppHarness.create(
+        root=root,
+        app_source=functools.partial(  # type: ignore
+            AppWithThousandPages, render_comp=render_multiple_pages  # type: ignore
+        ),
+    )  # type: ignore
+
+
+@pytest.fixture(scope="session")
+def app_with_ten_thousand_pages(
+    tmp_path_factory,
+) -> Generator[AppHarness, None, None]:
+    """Create an app with 10000 pages at tmp_path via AppHarness.
+
+    Args:
+        tmp_path_factory: pytest tmp_path_factory fixture
+
+    Yields:
+        running AppHarness instance
+    """
+    root = tmp_path_factory.mktemp(f"app10000")
+
+    yield AppHarness.create(
+        root=root,
+        app_source=functools.partial(
+            AppWithTenThousandPages, render_comp=render_multiple_pages  # type: ignore
+        ),
+    )  # type: ignore
+
+
+@pytest.mark.skipif(constants.IS_WINDOWS, reason=WINDOWS_SKIP_REASON)
+@pytest.mark.benchmark(
+    group="Compile time of varying page numbers",
+    timer=time.perf_counter,
+    disable_gc=True,
+    warmup=False,
+)
+def test_app_1_compile_time_cold(benchmark, app_with_one_page):
+    """Test the compile time on a cold start for an app with 1 page.
+
+    Args:
+        benchmark: The benchmark fixture.
+        app_with_one_page: The app harness.
+    """
+
+    def setup():
+        with chdir(app_with_one_page.app_path):
+            utils.empty_dir(constants.Dirs.WEB_PAGES, keep_files=["_app.js"])
+            app_with_one_page._initialize_app()
+            build.setup_frontend(app_with_one_page.app_path)
+
+    def benchmark_fn():
+        with chdir(app_with_one_page.app_path):
+            app_with_one_page.app_instance.compile_()
+
+    benchmark.pedantic(benchmark_fn, setup=setup, rounds=5)
+
+
+@pytest.mark.benchmark(
+    group="Compile time of varying page numbers",
+    min_rounds=5,
+    timer=time.perf_counter,
+    disable_gc=True,
+    warmup=False,
+)
+def test_app_1_compile_time_warm(benchmark, app_with_one_page):
+    """Test the compile time on a warm start for an app with 1 page.
+
+    Args:
+        benchmark: The benchmark fixture.
+        app_with_one_page: The app harness.
+    """
+    with chdir(app_with_one_page.app_path):
+        app_with_one_page._initialize_app()
+        build.setup_frontend(app_with_one_page.app_path)
+
+    def benchmark_fn():
+        with chdir(app_with_one_page.app_path):
+            app_with_one_page.app_instance.compile_()
+
+    benchmark(benchmark_fn)
+
+
+@pytest.mark.skipif(constants.IS_WINDOWS, reason=WINDOWS_SKIP_REASON)
+@pytest.mark.benchmark(
+    group="Compile time of varying page numbers",
+    timer=time.perf_counter,
+    disable_gc=True,
+    warmup=False,
+)
+def test_app_10_compile_time_cold(benchmark, app_with_ten_pages):
+    """Test the compile time on a cold start for an app with 10 page.
+
+    Args:
+        benchmark: The benchmark fixture.
+        app_with_ten_pages: The app harness.
+    """
+
+    def setup():
+        with chdir(app_with_ten_pages.app_path):
+            utils.empty_dir(constants.Dirs.WEB_PAGES, keep_files=["_app.js"])
+            app_with_ten_pages._initialize_app()
+            build.setup_frontend(app_with_ten_pages.app_path)
+
+    def benchmark_fn():
+        with chdir(app_with_ten_pages.app_path):
+            app_with_ten_pages.app_instance.compile_()
+
+    benchmark.pedantic(benchmark_fn, setup=setup, rounds=5)
+
+
+@pytest.mark.benchmark(
+    group="Compile time of varying page numbers",
+    min_rounds=5,
+    timer=time.perf_counter,
+    disable_gc=True,
+    warmup=False,
+)
+def test_app_10_compile_time_warm(benchmark, app_with_ten_pages):
+    """Test the compile time on a warm start for an app with 10 page.
+
+    Args:
+        benchmark: The benchmark fixture.
+        app_with_ten_pages: The app harness.
+    """
+    with chdir(app_with_ten_pages.app_path):
+        app_with_ten_pages._initialize_app()
+        build.setup_frontend(app_with_ten_pages.app_path)
+
+    def benchmark_fn():
+        with chdir(app_with_ten_pages.app_path):
+            app_with_ten_pages.app_instance.compile_()
+
+    benchmark(benchmark_fn)
+
+
+@pytest.mark.skipif(constants.IS_WINDOWS, reason=WINDOWS_SKIP_REASON)
+@pytest.mark.benchmark(
+    group="Compile time of varying page numbers",
+    timer=time.perf_counter,
+    disable_gc=True,
+    warmup=False,
+)
+def test_app_100_compile_time_cold(benchmark, app_with_hundred_pages):
+    """Test the compile time on a cold start for an app with 100 page.
+
+    Args:
+        benchmark: The benchmark fixture.
+        app_with_hundred_pages: The app harness.
+    """
+
+    def setup():
+        with chdir(app_with_hundred_pages.app_path):
+            utils.empty_dir(constants.Dirs.WEB_PAGES, keep_files=["_app.js"])
+            app_with_hundred_pages._initialize_app()
+            build.setup_frontend(app_with_hundred_pages.app_path)
+
+    def benchmark_fn():
+        with chdir(app_with_hundred_pages.app_path):
+            app_with_hundred_pages.app_instance.compile_()
+
+    benchmark.pedantic(benchmark_fn, setup=setup, rounds=5)
+
+
+@pytest.mark.benchmark(
+    group="Compile time of varying page numbers",
+    min_rounds=5,
+    timer=time.perf_counter,
+    disable_gc=True,
+    warmup=False,
+)
+def test_app_100_compile_time_warm(benchmark, app_with_hundred_pages):
+    """Test the compile time on a warm start for an app with 100 page.
+
+    Args:
+        benchmark: The benchmark fixture.
+        app_with_hundred_pages: The app harness.
+    """
+    with chdir(app_with_hundred_pages.app_path):
+        app_with_hundred_pages._initialize_app()
+        build.setup_frontend(app_with_hundred_pages.app_path)
+
+    def benchmark_fn():
+        with chdir(app_with_hundred_pages.app_path):
+            app_with_hundred_pages.app_instance.compile_()
+
+    benchmark(benchmark_fn)
+
+
+@pytest.mark.skipif(constants.IS_WINDOWS, reason=WINDOWS_SKIP_REASON)
+@pytest.mark.benchmark(
+    group="Compile time of varying page numbers",
+    timer=time.perf_counter,
+    disable_gc=True,
+    warmup=False,
+)
+def test_app_1000_compile_time_cold(benchmark, app_with_thousand_pages):
+    """Test the compile time on a cold start for an app with 1000 page.
+
+    Args:
+        benchmark: The benchmark fixture.
+        app_with_thousand_pages: The app harness.
+    """
+
+    def setup():
+        with chdir(app_with_thousand_pages.app_path):
+            utils.empty_dir(constants.Dirs.WEB_PAGES, keep_files=["_app.js"])
+            app_with_thousand_pages._initialize_app()
+            build.setup_frontend(app_with_thousand_pages.app_path)
+
+    def benchmark_fn():
+        with chdir(app_with_thousand_pages.app_path):
+            app_with_thousand_pages.app_instance.compile_()
+
+    benchmark.pedantic(benchmark_fn, setup=setup, rounds=5)
+
+
+@pytest.mark.benchmark(
+    group="Compile time of varying page numbers",
+    min_rounds=5,
+    timer=time.perf_counter,
+    disable_gc=True,
+    warmup=False,
+)
+def test_app_1000_compile_time_warm(benchmark, app_with_thousand_pages):
+    """Test the compile time on a warm start for an app with 1000 page.
+
+    Args:
+        benchmark: The benchmark fixture.
+        app_with_thousand_pages: The app harness.
+    """
+    with chdir(app_with_thousand_pages.app_path):
+        app_with_thousand_pages._initialize_app()
+        build.setup_frontend(app_with_thousand_pages.app_path)
+
+    def benchmark_fn():
+        with chdir(app_with_thousand_pages.app_path):
+            app_with_thousand_pages.app_instance.compile_()
+
+    benchmark(benchmark_fn)
+
+
+@pytest.mark.skip
+@pytest.mark.benchmark(
+    group="Compile time of varying page numbers",
+    timer=time.perf_counter,
+    disable_gc=True,
+    warmup=False,
+)
+def test_app_10000_compile_time_cold(benchmark, app_with_ten_thousand_pages):
+    """Test the compile time on a cold start for an app with 10000 page.
+
+    Args:
+        benchmark: The benchmark fixture.
+        app_with_ten_thousand_pages: The app harness.
+    """
+
+    def setup():
+        with chdir(app_with_ten_thousand_pages.app_path):
+            utils.empty_dir(constants.Dirs.WEB_PAGES, keep_files=["_app.js"])
+            app_with_ten_thousand_pages._initialize_app()
+            build.setup_frontend(app_with_ten_thousand_pages.app_path)
+
+    def benchmark_fn():
+        with chdir(app_with_ten_thousand_pages.app_path):
+            app_with_ten_thousand_pages.app_instance.compile_()
+
+    benchmark.pedantic(benchmark_fn, setup=setup, rounds=5)
+
+
+@pytest.mark.skip
+@pytest.mark.benchmark(
+    group="Compile time of varying page numbers",
+    min_rounds=5,
+    timer=time.perf_counter,
+    disable_gc=True,
+    warmup=False,
+)
+def test_app_10000_compile_time_warm(benchmark, app_with_ten_thousand_pages):
+    """Test the compile time on a warm start for an app with 10000 page.
+
+    Args:
+        benchmark: The benchmark fixture.
+        app_with_ten_thousand_pages: The app harness.
+    """
+
+    def benchmark_fn():
+        with chdir(app_with_ten_thousand_pages.app_path):
+            app_with_ten_thousand_pages.app_instance.compile_()
+
+    benchmark(benchmark_fn)

integration/benchmarks/helpers.py

@@ -1,49 +0,0 @@
-"""Helper functions for the benchmarking integration."""
-
-import json
-from datetime import datetime
-
-import psycopg2
-
-
-def insert_benchmarking_data(
-    db_connection_url: str,
-    lighthouse_data: dict,
-    performance_data: list[dict],
-    commit_sha: str,
-    pr_title: str,
-):
-    """Insert the benchmarking data into the database.
-
-    Args:
-        db_connection_url: The URL to connect to the database.
-        lighthouse_data: The Lighthouse data to insert.
-        performance_data: The performance data to insert.
-        commit_sha: The commit SHA to insert.
-        pr_title: The PR title to insert.
-    """
-    # Serialize the JSON data
-    lighthouse_json = json.dumps(lighthouse_data)
-    performance_json = json.dumps(performance_data)
-
-    # Get the current timestamp
-    current_timestamp = datetime.now()
-
-    # Connect to the database and insert the data
-    with psycopg2.connect(db_connection_url) as conn, conn.cursor() as cursor:
-        insert_query = """
-            INSERT INTO benchmarks (lighthouse, performance, commit_sha, pr_title, time)
-            VALUES (%s, %s, %s, %s, %s);
-            """
-        cursor.execute(
-            insert_query,
-            (
-                lighthouse_json,
-                performance_json,
-                commit_sha,
-                pr_title,
-                current_timestamp,
-            ),
-        )
-        # Commit the transaction
-        conn.commit()

integration/benchmarks/test_compile_benchmark.py

@@ -1,121 +0,0 @@
-"""Benchmark the time it takes to compile a reflex app."""
-
-import importlib
-
-import reflex
-
-rx = reflex
-
-
-class State(rx.State):
-    """A simple state class with a count variable."""
-
-    count: int = 0
-
-    def increment(self):
-        """Increment the count."""
-        self.count += 1
-
-    def decrement(self):
-        """Decrement the count."""
-        self.count -= 1
-
-
-class SliderVariation(State):
-    """A simple state class with a count variable."""
-
-    value: int = 50
-
-    def set_end(self, value: int):
-        """Increment the count.
-
-        Args:
-            value: The value of the slider.
-        """
-        self.value = value
-
-
-def sample_small_page() -> rx.Component:
-    """A simple page with a button that increments the count.
-
-    Returns:
-        A reflex component.
-    """
-    return rx.vstack(
-        *[rx.button(State.count, font_size="2em") for i in range(100)],
-        gap="1em",
-    )
-
-
-def sample_large_page() -> rx.Component:
-    """A large page with a slider that increments the count.
-
-    Returns:
-        A reflex component.
-    """
-    return rx.vstack(
-        *[
-            rx.vstack(
-                rx.heading(SliderVariation.value),
-                rx.slider(on_change_end=SliderVariation.set_end),
-                width="100%",
-            )
-            for i in range(100)
-        ],
-        gap="1em",
-    )
-
-
-def add_small_pages(app: rx.App):
-    """Add 10 small pages to the app.
-
-    Args:
-        app: The reflex app to add the pages to.
-    """
-    for i in range(10):
-        app.add_page(sample_small_page, route=f"/{i}")
-
-
-def add_large_pages(app: rx.App):
-    """Add 10 large pages to the app.
-
-    Args:
-        app: The reflex app to add the pages to.
-    """
-    for i in range(10):
-        app.add_page(sample_large_page, route=f"/{i}")
-
-
-def test_mean_import_time(benchmark):
-    """Test that the mean import time is less than 1 second.
-
-    Args:
-        benchmark: The benchmark fixture.
-    """
-
-    def import_reflex():
-        importlib.reload(reflex)
-
-    # Benchmark the import
-    benchmark(import_reflex)
-
-
-def test_mean_add_small_page_time(benchmark):
-    """Test that the mean add page time is less than 1 second.
-
-    Args:
-        benchmark: The benchmark fixture.
-    """
-    app = rx.App(state=State)
-    benchmark(add_small_pages, app)
-
-
-def test_mean_add_large_page_time(benchmark):
-    """Test that the mean add page time is less than 1 second.
-
-    Args:
-        benchmark: The benchmark fixture.
-    """
-    app = rx.App(state=State)
-    results = benchmark(add_large_pages, app)
-    print(results)

integration/conftest.py

@@ -1,4 +1,5 @@
 """Shared conftest for all integration tests."""
+
 import os
 import re
 from pathlib import Path
@@ -20,7 +21,7 @@ def xvfb():
     Yields:
         the pyvirtualdisplay object that the browser will be open on
     """
-    if os.environ.get("GITHUB_ACTIONS"):
+    if os.environ.get("GITHUB_ACTIONS") and not os.environ.get("APP_HARNESS_HEADLESS"):
         from pyvirtualdisplay.smartdisplay import (  # pyright: ignore [reportMissingImports]
             SmartDisplay,
         )

reflex/testing.py

@@ -211,7 +211,9 @@ class AppHarness:
             # get the source from a function or module object
             source_code = "\n".join(
                 [
-                    "\n".join(f"{k} = {v!r}" for k, v in app_globals.items()),
+                    "\n".join(
+                        self.get_app_global_source(k, v) for k, v in app_globals.items()
+                    ),
                     self._get_source_from_app_source(self.app_source),
                 ]
             )
@@ -331,6 +333,24 @@ class AppHarness:
         self._wait_frontend()
         return self
 
+    @staticmethod
+    def get_app_global_source(key, value):
+        """Get the source code of a global object.
+        If value is a function or class we render the actual
+        source of value otherwise we assign value to key.
+
+        Args:
+            key: variable name to assign value to.
+            value: value of the global variable.
+
+        Returns:
+            The rendered app global code.
+
+        """
+        if not inspect.isclass(value) and not inspect.isfunction(value):
+            return f"{key} = {value!r}"
+        return inspect.getsource(value)
+
     def __enter__(self) -> "AppHarness":
         """Contextmanager protocol for `start()`.
 

integration/benchmarks/benchmarks.py → scripts/lighthouse_score_upload.py

@@ -1,11 +1,52 @@
 """Runs the benchmarks and inserts the results into the database."""
 
+from __future__ import annotations
+
 import json
 import os
 import sys
+from datetime import datetime
+
+import psycopg2
+
+
+def insert_benchmarking_data(
+    db_connection_url: str,
+    lighthouse_data: dict,
+    commit_sha: str,
+    pr_title: str,
+):
+    """Insert the benchmarking data into the database.
 
-import pytest
-from helpers import insert_benchmarking_data
+    Args:
+        db_connection_url: The URL to connect to the database.
+        lighthouse_data: The Lighthouse data to insert.
+        commit_sha: The commit SHA to insert.
+        pr_title: The PR title to insert.
+    """
+    # Serialize the JSON data
+    lighthouse_json = json.dumps(lighthouse_data)
+
+    # Get the current timestamp
+    current_timestamp = datetime.now()
+
+    # Connect to the database and insert the data
+    with psycopg2.connect(db_connection_url) as conn, conn.cursor() as cursor:
+        insert_query = """
+            INSERT INTO benchmarks (lighthouse, commit_sha, pr_title, time)
+            VALUES (%s, %s, %s, %s);
+            """
+        cursor.execute(
+            insert_query,
+            (
+                lighthouse_json,
+                commit_sha,
+                pr_title,
+                current_timestamp,
+            ),
+        )
+        # Commit the transaction
+        conn.commit()
 
 
 def get_lighthouse_scores(directory_path: str) -> dict:
@@ -44,70 +85,6 @@ def get_lighthouse_scores(directory_path: str) -> dict:
     return scores
 
 
-def run_pytest_and_get_results(test_path=None) -> dict:
-    """Runs pytest and returns the results.
-
-    Args:
-        test_path: The path to the tests to run.
-
-    Returns:
-        dict: The results of the tests.
-    """
-    # Set the default path to the current directory if no path is provided
-    if not test_path:
-        test_path = os.getcwd()
-    # Ensure you have installed the pytest-json plugin before running this
-    pytest_args = ["-v", "--benchmark-json=benchmark_report.json", test_path]
-
-    # Run pytest with the specified arguments
-    pytest.main(pytest_args)
-
-    # Print ls of the current directory
-    print(os.listdir())
-
-    with open("benchmark_report.json", "r") as file:
-        pytest_results = json.load(file)
-
-    return pytest_results
-
-
-def extract_stats_from_json(json_data) -> list[dict]:
-    """Extracts the stats from the JSON data and returns them as a list of dictionaries.
-
-    Args:
-        json_data: The JSON data to extract the stats from.
-
-    Returns:
-        list[dict]: The stats for each test.
-    """
-    # Load the JSON data if it is a string, otherwise assume it's already a dictionary
-    data = json.loads(json_data) if isinstance(json_data, str) else json_data
-
-    # Initialize an empty list to store the stats for each test
-    test_stats = []
-
-    # Iterate over each test in the 'benchmarks' list
-    for test in data.get("benchmarks", []):
-        stats = test.get("stats", {})
-        test_name = test.get("name", "Unknown Test")
-        min_value = stats.get("min", None)
-        max_value = stats.get("max", None)
-        mean_value = stats.get("mean", None)
-        stdev_value = stats.get("stddev", None)
-
-        test_stats.append(
-            {
-                "test_name": test_name,
-                "min": min_value,
-                "max": max_value,
-                "mean": mean_value,
-                "stdev": stdev_value,
-            }
-        )
-
-    return test_stats
-
-
 def main():
     """Runs the benchmarks and inserts the results into the database."""
     # Get the commit SHA and JSON directory from the command line arguments
@@ -121,17 +98,11 @@ def main():
     if db_url is None or pr_title is None:
         sys.exit("Missing environment variables")
 
-    # Run pytest and get the results
-    results = run_pytest_and_get_results()
-    cleaned_results = extract_stats_from_json(results)
-
     # Get the Lighthouse scores
     lighthouse_scores = get_lighthouse_scores(json_dir)
 
     # Insert the data into the database
-    insert_benchmarking_data(
-        db_url, lighthouse_scores, cleaned_results, commit_sha, pr_title
-    )
+    insert_benchmarking_data(db_url, lighthouse_scores, commit_sha, pr_title)
 
 
 if __name__ == "__main__":

scripts/simple_app_benchmark_upload.py

@@ -0,0 +1,166 @@
+"""Runs the benchmarks and inserts the results into the database."""
+
+from __future__ import annotations
+
+import argparse
+import json
+from datetime import datetime
+
+import psycopg2
+
+
+def extract_stats_from_json(json_file: str) -> list[dict]:
+    """Extracts the stats from the JSON data and returns them as a list of dictionaries.
+
+    Args:
+        json_file: The JSON file to extract the stats data from.
+
+    Returns:
+        list[dict]: The stats for each test.
+    """
+    with open(json_file, "r") as file:
+        json_data = json.load(file)
+
+    # Load the JSON data if it is a string, otherwise assume it's already a dictionary
+    data = json.loads(json_data) if isinstance(json_data, str) else json_data
+
+    # Initialize an empty list to store the stats for each test
+    test_stats = []
+
+    # Iterate over each test in the 'benchmarks' list
+    for test in data.get("benchmarks", []):
+        stats = test.get("stats", {})
+        test_name = test.get("name", "Unknown Test")
+        min_value = stats.get("min", None)
+        max_value = stats.get("max", None)
+        mean_value = stats.get("mean", None)
+        stdev_value = stats.get("stddev", None)
+
+        test_stats.append(
+            {
+                "test_name": test_name,
+                "min": min_value,
+                "max": max_value,
+                "mean": mean_value,
+                "stdev": stdev_value,
+            }
+        )
+    return test_stats
+
+
+def insert_benchmarking_data(
+    db_connection_url: str,
+    os_type_version: str,
+    python_version: str,
+    performance_data: list[dict],
+    commit_sha: str,
+    pr_title: str,
+    branch_name: str,
+    event_type: str,
+    actor: str,
+):
+    """Insert the benchmarking data into the database.
+
+    Args:
+        db_connection_url: The URL to connect to the database.
+        os_type_version: The OS type and version to insert.
+        python_version: The Python version to insert.
+        performance_data: The performance data of reflex web to insert.
+        commit_sha: The commit SHA to insert.
+        pr_title: The PR title to insert.
+        branch_name: The name of the branch.
+        event_type: Type of github event(push, pull request, etc)
+        actor: Username of the user that triggered the run.
+    """
+    # Serialize the JSON data
+    simple_app_performance_json = json.dumps(performance_data)
+
+    # Get the current timestamp
+    current_timestamp = datetime.now()
+
+    # Connect to the database and insert the data
+    with psycopg2.connect(db_connection_url) as conn, conn.cursor() as cursor:
+        insert_query = """
+            INSERT INTO simple_app_benchmarks (os, python_version, commit_sha, time, pr_title, branch_name, event_type, actor, performance)
+            VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s);
+            """
+        cursor.execute(
+            insert_query,
+            (
+                os_type_version,
+                python_version,
+                commit_sha,
+                current_timestamp,
+                pr_title,
+                branch_name,
+                event_type,
+                actor,
+                simple_app_performance_json,
+            ),
+        )
+        # Commit the transaction
+        conn.commit()
+
+
+def main():
+    """Runs the benchmarks and inserts the results."""
+    # Get the commit SHA and JSON directory from the command line arguments
+    parser = argparse.ArgumentParser(description="Run benchmarks and process results.")
+    parser.add_argument(
+        "--os", help="The OS type and version to insert into the database."
+    )
+    parser.add_argument(
+        "--python-version", help="The Python version to insert into the database."
+    )
+    parser.add_argument(
+        "--commit-sha", help="The commit SHA to insert into the database."
+    )
+    parser.add_argument(
+        "--benchmark-json",
+        help="The JSON file containing the benchmark results.",
+    )
+    parser.add_argument(
+        "--db-url",
+        help="The URL to connect to the database.",
+        required=True,
+    )
+    parser.add_argument(
+        "--pr-title",
+        help="The PR title to insert into the database.",
+        required=True,
+    )
+    parser.add_argument(
+        "--branch-name",
+        help="The current branch",
+        required=True,
+    )
+    parser.add_argument(
+        "--event-type",
+        help="The github event type",
+        required=True,
+    )
+    parser.add_argument(
+        "--actor",
+        help="Username of the user that triggered the run.",
+        required=True,
+    )
+    args = parser.parse_args()
+
+    # Get the results of pytest benchmarks
+    cleaned_benchmark_results = extract_stats_from_json(args.benchmark_json)
+    # Insert the data into the database
+    insert_benchmarking_data(
+        db_connection_url=args.db_url,
+        os_type_version=args.os,
+        python_version=args.python_version,
+        performance_data=cleaned_benchmark_results,
+        commit_sha=args.commit_sha,
+        pr_title=args.pr_title,
+        branch_name=args.branch_name,
+        event_type=args.event_type,
+        actor=args.actor,
+    )
+
+
+if __name__ == "__main__":
+    main()