remove some benchmarks from CI (#4812)

.github/workflows/benchmarks.yml

@@ -70,56 +70,6 @@ jobs:
         env:
           GITHUB_SHA: ${{ github.sha }}
 
-  simple-apps-benchmarks: # This app tests the compile times of various compoonents and pages
-    if: github.event.pull_request.merged == true
-    env:
-      OUTPUT_FILE: benchmarks.json
-    timeout-minutes: 50
-    strategy:
-      # Prioritize getting more information out of the workflow (even if something fails)
-      fail-fast: false
-      matrix:
-        # Show OS combos first in GUI
-        os: [ubuntu-latest, windows-latest, macos-latest]
-        python-version: ["3.10.16", "3.11.11", "3.12.8"]
-        exclude:
-          - os: windows-latest
-            python-version: "3.10.16"
-          - os: windows-latest
-            python-version: "3.11.11"
-          # keep only one python version for MacOS
-          - os: macos-latest
-            python-version: "3.10.16"
-          - os: macos-latest
-            python-version: "3.11.11"
-        include:
-          - os: windows-latest
-            python-version: "3.10.11"
-          - os: windows-latest
-            python-version: "3.11.9"
-
-    runs-on: ${{ matrix.os }}
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/setup_build_env
-        with:
-          python-version: ${{ matrix.python-version }}
-          run-poetry-install: true
-          create-venv-at-path: .venv
-      - name: Run benchmark tests
-        env:
-          APP_HARNESS_HEADLESS: 1
-          PYTHONUNBUFFERED: 1
-        run: |
-          poetry run pytest -v benchmarks/ --benchmark-json=${{ env.OUTPUT_FILE }} -s
-      - name: Upload benchmark results
-        # Only run if the database creds are available in this context.
-        run:
-          poetry run python benchmarks/benchmark_compile_times.py --os "${{ matrix.os }}"
-          --python-version "${{ matrix.python-version }}" --commit-sha "${{ github.sha }}"
-          --benchmark-json "${{ env.OUTPUT_FILE }}" --branch-name "${{ github.head_ref || github.ref_name }}"
-          --event-type "${{ github.event_name }}" --pr-id "${{ github.event.pull_request.id }}"
-
   reflex-dist-size: # This job is used to calculate the size of the Reflex distribution (wheel file)
     if: github.event.pull_request.merged == true
     timeout-minutes: 30

.github/workflows/integration_tests.yml

@@ -94,26 +94,6 @@ jobs:
           # Check that npm is home
           npm -v
           poetry run bash scripts/integration.sh ./reflex-examples/counter dev
-      - name: Measure and upload .web size
-        run:
-          poetry run python benchmarks/benchmark_web_size.py --os "${{ matrix.os }}"
-          --python-version "${{ matrix.python-version }}" --commit-sha "${{ github.sha }}"
-          --pr-id "${{ github.event.pull_request.id }}"
-          --branch-name "${{ github.head_ref || github.ref_name }}"
-          --path ./reflex-examples/counter/.web
-          --app-name "counter"
-      - name: Install hyperfine
-        run: cargo install hyperfine
-      - name: Benchmark imports
-        working-directory: ./reflex-examples/counter
-        run: hyperfine --warmup 3 "export POETRY_VIRTUALENVS_PATH=../../.venv; poetry run python counter/counter.py" --show-output --export-json "${{ env.OUTPUT_FILE }}" --shell bash
-      - name: Upload Benchmarks
-        run:
-          poetry run python benchmarks/benchmark_imports.py --os "${{ matrix.os }}"
-          --python-version "${{ matrix.python-version }}" --commit-sha "${{ github.sha }}"
-          --benchmark-json "./reflex-examples/counter/${{ env.OUTPUT_FILE }}"
-          --branch-name "${{ github.head_ref || github.ref_name }}" --pr-id "${{ github.event.pull_request.id }}"
-          --app-name "counter"
       - name: Install requirements for nba proxy example
         working-directory: ./reflex-examples/nba-proxy
         run: |
@@ -174,12 +154,6 @@ jobs:
           # Check that npm is home
           npm -v
           poetry run bash scripts/integration.sh ./reflex-web prod
-      - name: Measure and upload .web size
-        run:
-          poetry run python benchmarks/benchmark_web_size.py --os "${{ matrix.os }}"
-          --python-version "${{ matrix.python-version }}" --commit-sha "${{ github.sha }}"
-          --pr-id "${{ github.event.pull_request.id }}" --branch-name "${{ github.head_ref || github.ref_name }}"
-          --app-name "reflex-web" --path ./reflex-web/.web
 
   rx-shout-from-template:
     strategy:
@@ -243,9 +217,3 @@ jobs:
           # Check that npm is home
           npm -v
           poetry run bash scripts/integration.sh ./reflex-web prod
-      - name: Measure and upload .web size
-        run:
-          poetry run python benchmarks/benchmark_web_size.py --os "${{ matrix.os }}"
-          --python-version "${{ matrix.python-version }}" --commit-sha "${{ github.sha }}"
-          --pr-id "${{ github.event.pull_request.id }}" --branch-name "${{ github.head_ref || github.ref_name }}"
-          --app-name "reflex-web" --path ./reflex-web/.web

benchmarks/test_benchmark_compile_components.py

@@ -1,376 +0,0 @@
-"""Benchmark tests for apps with varying component numbers."""
-
-from __future__ import annotations
-
-import functools
-import time
-from typing import Generator
-
-import pytest
-
-from benchmarks import WINDOWS_SKIP_REASON
-from reflex import constants
-from reflex.compiler import utils
-from reflex.testing import AppHarness, chdir
-from reflex.utils import build
-from reflex.utils.prerequisites import get_web_dir
-
-web_pages = get_web_dir() / constants.Dirs.PAGES
-
-
-def render_component(num: int):
-    """Generate a number of components based on num.
-
-    Args:
-        num: number of components to produce.
-
-    Returns:
-        The rendered number of components.
-    """
-    import reflex as rx
-
-    return [
-        rx.fragment(
-            rx.box(
-                rx.accordion.root(
-                    rx.accordion.item(
-                        header="Full Ingredients",
-                        content="Yes. It's built with accessibility in mind.",
-                        font_size="3em",
-                    ),
-                    rx.accordion.item(
-                        header="Applications",
-                        content="Yes. It's unstyled by default, giving you freedom over the look and feel.",
-                    ),
-                    collapsible=True,
-                    variant="ghost",
-                    width="25rem",
-                ),
-                padding_top="20px",
-            ),
-            rx.box(
-                rx.drawer.root(
-                    rx.drawer.trigger(
-                        rx.button("Open Drawer with snap points"), as_child=True
-                    ),
-                    rx.drawer.overlay(),
-                    rx.drawer.portal(
-                        rx.drawer.content(
-                            rx.flex(
-                                rx.drawer.title("Drawer Content"),
-                                rx.drawer.description("Drawer description"),
-                                rx.drawer.close(
-                                    rx.button("Close Button"),
-                                    as_child=True,
-                                ),
-                                direction="column",
-                                margin="5em",
-                                align_items="center",
-                            ),
-                            top="auto",
-                            height="100%",
-                            flex_direction="column",
-                            background_color="var(--green-3)",
-                        ),
-                    ),
-                    snap_points=["148px", "355px", 1],
-                ),
-            ),
-            rx.box(
-                rx.callout(
-                    "You will need admin privileges to install and access this application.",
-                    icon="info",
-                    size="3",
-                ),
-            ),
-            rx.box(
-                rx.table.root(
-                    rx.table.header(
-                        rx.table.row(
-                            rx.table.column_header_cell("Full name"),
-                            rx.table.column_header_cell("Email"),
-                            rx.table.column_header_cell("Group"),
-                        ),
-                    ),
-                    rx.table.body(
-                        rx.table.row(
-                            rx.table.row_header_cell("Danilo Sousa"),
-                            rx.table.cell("danilo@example.com"),
-                            rx.table.cell("Developer"),
-                        ),
-                        rx.table.row(
-                            rx.table.row_header_cell("Zahra Ambessa"),
-                            rx.table.cell("zahra@example.com"),
-                            rx.table.cell("Admin"),
-                        ),
-                        rx.table.row(
-                            rx.table.row_header_cell("Jasper Eriksson"),
-                            rx.table.cell("jasper@example.com"),
-                            rx.table.cell("Developer"),
-                        ),
-                    ),
-                )
-            ),
-        )
-    ] * num
-
-
-def AppWithTenComponentsOnePage():
-    """A reflex app with roughly 10 components on one page."""
-    import reflex as rx
-
-    def index() -> rx.Component:
-        return rx.center(rx.vstack(*render_component(1)))
-
-    app = rx.App(_state=rx.State)
-    app.add_page(index)
-
-
-def AppWithHundredComponentOnePage():
-    """A reflex app with roughly 100 components on one page."""
-    import reflex as rx
-
-    def index() -> rx.Component:
-        return rx.center(rx.vstack(*render_component(100)))
-
-    app = rx.App(_state=rx.State)
-    app.add_page(index)
-
-
-def AppWithThousandComponentsOnePage():
-    """A reflex app with roughly 1000 components on one page."""
-    import reflex as rx
-
-    def index() -> rx.Component:
-        return rx.center(rx.vstack(*render_component(1000)))
-
-    app = rx.App(_state=rx.State)
-    app.add_page(index)
-
-
-@pytest.fixture(scope="session")
-def app_with_10_components(
-    tmp_path_factory,
-) -> Generator[AppHarness, None, None]:
-    """Start Blank Template app at tmp_path via AppHarness.
-
-    Args:
-        tmp_path_factory: pytest tmp_path_factory fixture
-
-    Yields:
-        running AppHarness instance
-    """
-    root = tmp_path_factory.mktemp("app10components")
-
-    yield AppHarness.create(
-        root=root,
-        app_source=functools.partial(
-            AppWithTenComponentsOnePage,
-            render_component=render_component,  # pyright: ignore [reportCallIssue]
-        ),
-    )
-
-
-@pytest.fixture(scope="session")
-def app_with_100_components(
-    tmp_path_factory,
-) -> Generator[AppHarness, None, None]:
-    """Start Blank Template app at tmp_path via AppHarness.
-
-    Args:
-        tmp_path_factory: pytest tmp_path_factory fixture
-
-    Yields:
-        running AppHarness instance
-    """
-    root = tmp_path_factory.mktemp("app100components")
-
-    yield AppHarness.create(
-        root=root,
-        app_source=functools.partial(
-            AppWithHundredComponentOnePage,
-            render_component=render_component,  # pyright: ignore [reportCallIssue]
-        ),
-    )
-
-
-@pytest.fixture(scope="session")
-def app_with_1000_components(
-    tmp_path_factory,
-) -> Generator[AppHarness, None, None]:
-    """Create an app with 1000 components at tmp_path via AppHarness.
-
-    Args:
-        tmp_path_factory: pytest tmp_path_factory fixture
-
-    Yields:
-        an AppHarness instance
-    """
-    root = tmp_path_factory.mktemp("app1000components")
-
-    yield AppHarness.create(
-        root=root,
-        app_source=functools.partial(
-            AppWithThousandComponentsOnePage,
-            render_component=render_component,  # pyright: ignore [reportCallIssue]
-        ),
-    )
-
-
-@pytest.mark.skipif(constants.IS_WINDOWS, reason=WINDOWS_SKIP_REASON)
-@pytest.mark.benchmark(
-    group="Compile time of varying component numbers",
-    timer=time.perf_counter,
-    disable_gc=True,
-    warmup=False,
-)
-def test_app_10_compile_time_cold(benchmark, app_with_10_components):
-    """Test the compile time on a cold start for an app with roughly 10 components.
-
-    Args:
-        benchmark: The benchmark fixture.
-        app_with_10_components: The app harness.
-    """
-
-    def setup():
-        with chdir(app_with_10_components.app_path):
-            utils.empty_dir(web_pages, ["_app.js"])
-            app_with_10_components._initialize_app()
-            build.setup_frontend(app_with_10_components.app_path)
-
-    def benchmark_fn():
-        with chdir(app_with_10_components.app_path):
-            app_with_10_components.app_instance._compile()
-
-    benchmark.pedantic(benchmark_fn, setup=setup, rounds=10)
-
-
-@pytest.mark.benchmark(
-    group="Compile time of varying component numbers",
-    min_rounds=5,
-    timer=time.perf_counter,
-    disable_gc=True,
-    warmup=False,
-)
-def test_app_10_compile_time_warm(benchmark, app_with_10_components):
-    """Test the compile time on a warm start for an app with roughly 10 components.
-
-    Args:
-        benchmark: The benchmark fixture.
-        app_with_10_components: The app harness.
-    """
-    with chdir(app_with_10_components.app_path):
-        app_with_10_components._initialize_app()
-        build.setup_frontend(app_with_10_components.app_path)
-
-    def benchmark_fn():
-        with chdir(app_with_10_components.app_path):
-            app_with_10_components.app_instance._compile()
-
-    benchmark(benchmark_fn)
-
-
-@pytest.mark.skipif(constants.IS_WINDOWS, reason=WINDOWS_SKIP_REASON)
-@pytest.mark.benchmark(
-    group="Compile time of varying component numbers",
-    timer=time.perf_counter,
-    disable_gc=True,
-    warmup=False,
-)
-def test_app_100_compile_time_cold(benchmark, app_with_100_components):
-    """Test the compile time on a cold start for an app with roughly 100 components.
-
-    Args:
-        benchmark: The benchmark fixture.
-        app_with_100_components: The app harness.
-    """
-
-    def setup():
-        with chdir(app_with_100_components.app_path):
-            utils.empty_dir(web_pages, ["_app.js"])
-            app_with_100_components._initialize_app()
-            build.setup_frontend(app_with_100_components.app_path)
-
-    def benchmark_fn():
-        with chdir(app_with_100_components.app_path):
-            app_with_100_components.app_instance._compile()
-
-    benchmark.pedantic(benchmark_fn, setup=setup, rounds=5)
-
-
-@pytest.mark.benchmark(
-    group="Compile time of varying component numbers",
-    min_rounds=5,
-    timer=time.perf_counter,
-    disable_gc=True,
-    warmup=False,
-)
-def test_app_100_compile_time_warm(benchmark, app_with_100_components):
-    """Test the compile time on a warm start for an app with roughly 100 components.
-
-    Args:
-        benchmark: The benchmark fixture.
-        app_with_100_components: The app harness.
-    """
-    with chdir(app_with_100_components.app_path):
-        app_with_100_components._initialize_app()
-        build.setup_frontend(app_with_100_components.app_path)
-
-    def benchmark_fn():
-        with chdir(app_with_100_components.app_path):
-            app_with_100_components.app_instance._compile()
-
-    benchmark(benchmark_fn)
-
-
-@pytest.mark.skipif(constants.IS_WINDOWS, reason=WINDOWS_SKIP_REASON)
-@pytest.mark.benchmark(
-    group="Compile time of varying component numbers",
-    timer=time.perf_counter,
-    disable_gc=True,
-    warmup=False,
-)
-def test_app_1000_compile_time_cold(benchmark, app_with_1000_components):
-    """Test the compile time on a cold start for an app with roughly 1000 components.
-
-    Args:
-        benchmark: The benchmark fixture.
-        app_with_1000_components: The app harness.
-    """
-
-    def setup():
-        with chdir(app_with_1000_components.app_path):
-            utils.empty_dir(web_pages, keep_files=["_app.js"])
-            app_with_1000_components._initialize_app()
-            build.setup_frontend(app_with_1000_components.app_path)
-
-    def benchmark_fn():
-        with chdir(app_with_1000_components.app_path):
-            app_with_1000_components.app_instance._compile()
-
-    benchmark.pedantic(benchmark_fn, setup=setup, rounds=5)
-
-
-@pytest.mark.benchmark(
-    group="Compile time of varying component numbers",
-    min_rounds=5,
-    timer=time.perf_counter,
-    disable_gc=True,
-    warmup=False,
-)
-def test_app_1000_compile_time_warm(benchmark, app_with_1000_components):
-    """Test the compile time on a warm start for an app with roughly 1000 components.
-
-    Args:
-        benchmark: The benchmark fixture.
-        app_with_1000_components: The app harness.
-    """
-    with chdir(app_with_1000_components.app_path):
-        app_with_1000_components._initialize_app()
-        build.setup_frontend(app_with_1000_components.app_path)
-
-    def benchmark_fn():
-        with chdir(app_with_1000_components.app_path):
-            app_with_1000_components.app_instance._compile()
-
-    benchmark(benchmark_fn)

benchmarks/test_benchmark_compile_pages.py

@@ -1,595 +0,0 @@
-"""Benchmark tests for apps with varying page numbers."""
-
-from __future__ import annotations
-
-import functools
-import time
-from typing import Generator
-
-import pytest
-
-from benchmarks import WINDOWS_SKIP_REASON
-from reflex import constants
-from reflex.compiler import utils
-from reflex.testing import AppHarness, chdir
-from reflex.utils import build
-from reflex.utils.prerequisites import get_web_dir
-
-web_pages = get_web_dir() / constants.Dirs.PAGES
-
-
-def render_multiple_pages(app, num: int):
-    """Add multiple pages based on num.
-
-    Args:
-        app: The App object.
-        num: number of pages to render.
-
-    """
-    from typing import Tuple
-
-    from rxconfig import config  # pyright: ignore [reportMissingImports]
-
-    import reflex as rx
-
-    docs_url = "https://reflex.dev/docs/getting-started/introduction/"
-    filename = f"{config.app_name}/{config.app_name}.py"
-    college = [
-        "Stanford University",
-        "Arizona",
-        "Arizona state",
-        "Baylor",
-        "Boston College",
-        "Boston University",
-    ]
-
-    class State(rx.State):
-        """The app state."""
-
-        position: rx.Field[str]
-        college: rx.Field[str]
-        age: rx.Field[Tuple[int, int]] = rx.field((18, 50))
-        salary: rx.Field[Tuple[int, int]] = rx.field((0, 25000000))
-
-        @rx.event
-        def set_position(self, value: str):
-            self.position = value
-
-        @rx.event
-        def set_college(self, value: str):
-            self.college = value
-
-        @rx.event
-        def set_age(self, value: list[int]):
-            self.age = (value[0], value[1])
-
-        @rx.event
-        def set_salary(self, value: list[int]):
-            self.salary = (value[0], value[1])
-
-    comp1 = rx.center(
-        rx.theme_panel(),
-        rx.vstack(
-            rx.heading("Welcome to Reflex!", size="9"),
-            rx.text("Get started by editing ", rx.code(filename)),
-            rx.button(
-                "Check out our docs!",
-                on_click=lambda: rx.redirect(docs_url),
-                size="4",
-            ),
-            align="center",
-            spacing="7",
-            font_size="2em",
-        ),
-        height="100vh",
-    )
-
-    comp2 = rx.vstack(
-        rx.hstack(
-            rx.vstack(
-                rx.select(
-                    ["C", "PF", "SF", "PG", "SG"],
-                    placeholder="Select a position. (All)",
-                    on_change=State.set_position,
-                    size="3",
-                ),
-                rx.select(
-                    college,
-                    placeholder="Select a college. (All)",
-                    on_change=State.set_college,
-                    size="3",
-                ),
-            ),
-            rx.vstack(
-                rx.vstack(
-                    rx.hstack(
-                        rx.badge("Min Age: ", State.age[0]),
-                        rx.divider(orientation="vertical"),
-                        rx.badge("Max Age: ", State.age[1]),
-                    ),
-                    rx.slider(
-                        default_value=[18, 50],
-                        min=18,
-                        max=50,
-                        on_value_commit=State.set_age,
-                    ),
-                    align_items="left",
-                    width="100%",
-                ),
-                rx.vstack(
-                    rx.hstack(
-                        rx.badge("Min Sal: ", State.salary[0] // 1000000, "M"),
-                        rx.divider(orientation="vertical"),
-                        rx.badge("Max Sal: ", State.salary[1] // 1000000, "M"),
-                    ),
-                    rx.slider(
-                        default_value=[0, 25000000],
-                        min=0,
-                        max=25000000,
-                        on_value_commit=State.set_salary,
-                    ),
-                    align_items="left",
-                    width="100%",
-                ),
-            ),
-            spacing="4",
-        ),
-        width="100%",
-    )
-
-    for i in range(1, num + 1):
-        if i % 2 == 1:
-            app.add_page(comp1, route=f"page{i}")
-        else:
-            app.add_page(comp2, route=f"page{i}")
-
-
-def AppWithOnePage():
-    """A reflex app with one page."""
-    from rxconfig import config  # pyright: ignore [reportMissingImports]
-
-    import reflex as rx
-
-    docs_url = "https://reflex.dev/docs/getting-started/introduction/"
-    filename = f"{config.app_name}/{config.app_name}.py"
-
-    class State(rx.State):
-        """The app state."""
-
-        pass
-
-    def index() -> rx.Component:
-        return rx.center(
-            rx.input(
-                id="token", value=State.router.session.client_token, is_read_only=True
-            ),
-            rx.vstack(
-                rx.heading("Welcome to Reflex!", size="9"),
-                rx.text("Get started by editing ", rx.code(filename)),
-                rx.button(
-                    "Check out our docs!",
-                    on_click=lambda: rx.redirect(docs_url),
-                    size="4",
-                ),
-                align="center",
-                spacing="7",
-                font_size="2em",
-            ),
-            height="100vh",
-        )
-
-    app = rx.App(_state=rx.State)
-    app.add_page(index)
-
-
-def AppWithTenPages():
-    """A reflex app with 10 pages."""
-    import reflex as rx
-
-    app = rx.App(_state=rx.State)
-    render_multiple_pages(app, 10)
-
-
-def AppWithHundredPages():
-    """A reflex app with 100 pages."""
-    import reflex as rx
-
-    app = rx.App(_state=rx.State)
-    render_multiple_pages(app, 100)
-
-
-def AppWithThousandPages():
-    """A reflex app with Thousand pages."""
-    import reflex as rx
-
-    app = rx.App(_state=rx.State)
-    render_multiple_pages(app, 1000)
-
-
-def AppWithTenThousandPages():
-    """A reflex app with ten thousand pages."""
-    import reflex as rx
-
-    app = rx.App(_state=rx.State)
-    render_multiple_pages(app, 10000)
-
-
-@pytest.fixture(scope="session")
-def app_with_one_page(
-    tmp_path_factory,
-) -> Generator[AppHarness, None, None]:
-    """Create an app with 10000 pages at tmp_path via AppHarness.
-
-    Args:
-        tmp_path_factory: pytest tmp_path_factory fixture
-
-    Yields:
-        an AppHarness instance
-    """
-    root = tmp_path_factory.mktemp("app1")
-
-    yield AppHarness.create(root=root, app_source=AppWithOnePage)
-
-
-@pytest.fixture(scope="session")
-def app_with_ten_pages(
-    tmp_path_factory,
-) -> Generator[AppHarness, None, None]:
-    """Create an app with 10 pages at tmp_path via AppHarness.
-
-    Args:
-        tmp_path_factory: pytest tmp_path_factory fixture
-
-    Yields:
-        an AppHarness instance
-    """
-    root = tmp_path_factory.mktemp("app10")
-    yield AppHarness.create(
-        root=root,
-        app_source=functools.partial(
-            AppWithTenPages,
-            render_comp=render_multiple_pages,  # pyright: ignore [reportCallIssue]
-        ),
-    )
-
-
-@pytest.fixture(scope="session")
-def app_with_hundred_pages(
-    tmp_path_factory,
-) -> Generator[AppHarness, None, None]:
-    """Create an app with 100 pages at tmp_path via AppHarness.
-
-    Args:
-        tmp_path_factory: pytest tmp_path_factory fixture
-
-    Yields:
-        an AppHarness instance
-    """
-    root = tmp_path_factory.mktemp("app100")
-
-    yield AppHarness.create(
-        root=root,
-        app_source=functools.partial(
-            AppWithHundredPages,
-            render_comp=render_multiple_pages,  # pyright: ignore [reportCallIssue]
-        ),
-    )
-
-
-@pytest.fixture(scope="session")
-def app_with_thousand_pages(
-    tmp_path_factory,
-) -> Generator[AppHarness, None, None]:
-    """Create an app with 1000 pages at tmp_path via AppHarness.
-
-    Args:
-        tmp_path_factory: pytest tmp_path_factory fixture
-
-    Yields:
-        an AppHarness instance
-    """
-    root = tmp_path_factory.mktemp("app1000")
-
-    yield AppHarness.create(
-        root=root,
-        app_source=functools.partial(
-            AppWithThousandPages,
-            render_comp=render_multiple_pages,  # pyright: ignore [reportCallIssue]
-        ),
-    )
-
-
-@pytest.fixture(scope="session")
-def app_with_ten_thousand_pages(
-    tmp_path_factory,
-) -> Generator[AppHarness, None, None]:
-    """Create an app with 10000 pages at tmp_path via AppHarness.
-
-    Args:
-        tmp_path_factory: pytest tmp_path_factory fixture
-
-    Yields:
-        running AppHarness instance
-    """
-    root = tmp_path_factory.mktemp("app10000")
-
-    yield AppHarness.create(
-        root=root,
-        app_source=functools.partial(
-            AppWithTenThousandPages,
-            render_comp=render_multiple_pages,  # pyright: ignore [reportCallIssue]
-        ),
-    )
-
-
-@pytest.mark.skipif(constants.IS_WINDOWS, reason=WINDOWS_SKIP_REASON)
-@pytest.mark.benchmark(
-    group="Compile time of varying page numbers",
-    timer=time.perf_counter,
-    disable_gc=True,
-    warmup=False,
-)
-def test_app_1_compile_time_cold(benchmark, app_with_one_page):
-    """Test the compile time on a cold start for an app with 1 page.
-
-    Args:
-        benchmark: The benchmark fixture.
-        app_with_one_page: The app harness.
-    """
-
-    def setup():
-        with chdir(app_with_one_page.app_path):
-            utils.empty_dir(web_pages, keep_files=["_app.js"])
-            app_with_one_page._initialize_app()
-            build.setup_frontend(app_with_one_page.app_path)
-
-    def benchmark_fn():
-        with chdir(app_with_one_page.app_path):
-            app_with_one_page.app_instance._compile()
-
-    benchmark.pedantic(benchmark_fn, setup=setup, rounds=5)
-    app_with_one_page._reload_state_module()
-
-
-@pytest.mark.benchmark(
-    group="Compile time of varying page numbers",
-    min_rounds=5,
-    timer=time.perf_counter,
-    disable_gc=True,
-    warmup=False,
-)
-def test_app_1_compile_time_warm(benchmark, app_with_one_page):
-    """Test the compile time on a warm start for an app with 1 page.
-
-    Args:
-        benchmark: The benchmark fixture.
-        app_with_one_page: The app harness.
-    """
-    with chdir(app_with_one_page.app_path):
-        app_with_one_page._initialize_app()
-        build.setup_frontend(app_with_one_page.app_path)
-
-    def benchmark_fn():
-        with chdir(app_with_one_page.app_path):
-            app_with_one_page.app_instance._compile()
-
-    benchmark(benchmark_fn)
-    app_with_one_page._reload_state_module()
-
-
-@pytest.mark.skipif(constants.IS_WINDOWS, reason=WINDOWS_SKIP_REASON)
-@pytest.mark.benchmark(
-    group="Compile time of varying page numbers",
-    timer=time.perf_counter,
-    disable_gc=True,
-    warmup=False,
-)
-def test_app_10_compile_time_cold(benchmark, app_with_ten_pages):
-    """Test the compile time on a cold start for an app with 10 page.
-
-    Args:
-        benchmark: The benchmark fixture.
-        app_with_ten_pages: The app harness.
-    """
-
-    def setup():
-        with chdir(app_with_ten_pages.app_path):
-            utils.empty_dir(web_pages, keep_files=["_app.js"])
-            app_with_ten_pages._initialize_app()
-            build.setup_frontend(app_with_ten_pages.app_path)
-
-    def benchmark_fn():
-        with chdir(app_with_ten_pages.app_path):
-            app_with_ten_pages.app_instance._compile()
-
-    benchmark.pedantic(benchmark_fn, setup=setup, rounds=5)
-    app_with_ten_pages._reload_state_module()
-
-
-@pytest.mark.benchmark(
-    group="Compile time of varying page numbers",
-    min_rounds=5,
-    timer=time.perf_counter,
-    disable_gc=True,
-    warmup=False,
-)
-def test_app_10_compile_time_warm(benchmark, app_with_ten_pages):
-    """Test the compile time on a warm start for an app with 10 page.
-
-    Args:
-        benchmark: The benchmark fixture.
-        app_with_ten_pages: The app harness.
-    """
-    with chdir(app_with_ten_pages.app_path):
-        app_with_ten_pages._initialize_app()
-        build.setup_frontend(app_with_ten_pages.app_path)
-
-    def benchmark_fn():
-        with chdir(app_with_ten_pages.app_path):
-            app_with_ten_pages.app_instance._compile()
-
-    benchmark(benchmark_fn)
-    app_with_ten_pages._reload_state_module()
-
-
-@pytest.mark.skipif(constants.IS_WINDOWS, reason=WINDOWS_SKIP_REASON)
-@pytest.mark.benchmark(
-    group="Compile time of varying page numbers",
-    timer=time.perf_counter,
-    disable_gc=True,
-    warmup=False,
-)
-def test_app_100_compile_time_cold(benchmark, app_with_hundred_pages):
-    """Test the compile time on a cold start for an app with 100 page.
-
-    Args:
-        benchmark: The benchmark fixture.
-        app_with_hundred_pages: The app harness.
-    """
-
-    def setup():
-        with chdir(app_with_hundred_pages.app_path):
-            utils.empty_dir(web_pages, keep_files=["_app.js"])
-            app_with_hundred_pages._initialize_app()
-            build.setup_frontend(app_with_hundred_pages.app_path)
-
-    def benchmark_fn():
-        with chdir(app_with_hundred_pages.app_path):
-            app_with_hundred_pages.app_instance._compile()
-
-    benchmark.pedantic(benchmark_fn, setup=setup, rounds=5)
-    app_with_hundred_pages._reload_state_module()
-
-
-@pytest.mark.benchmark(
-    group="Compile time of varying page numbers",
-    min_rounds=5,
-    timer=time.perf_counter,
-    disable_gc=True,
-    warmup=False,
-)
-def test_app_100_compile_time_warm(benchmark, app_with_hundred_pages):
-    """Test the compile time on a warm start for an app with 100 page.
-
-    Args:
-        benchmark: The benchmark fixture.
-        app_with_hundred_pages: The app harness.
-    """
-    with chdir(app_with_hundred_pages.app_path):
-        app_with_hundred_pages._initialize_app()
-        build.setup_frontend(app_with_hundred_pages.app_path)
-
-    def benchmark_fn():
-        with chdir(app_with_hundred_pages.app_path):
-            app_with_hundred_pages.app_instance._compile()
-
-    benchmark(benchmark_fn)
-    app_with_hundred_pages._reload_state_module()
-
-
-@pytest.mark.skipif(constants.IS_WINDOWS, reason=WINDOWS_SKIP_REASON)
-@pytest.mark.benchmark(
-    group="Compile time of varying page numbers",
-    timer=time.perf_counter,
-    disable_gc=True,
-    warmup=False,
-)
-def test_app_1000_compile_time_cold(benchmark, app_with_thousand_pages):
-    """Test the compile time on a cold start for an app with 1000 page.
-
-    Args:
-        benchmark: The benchmark fixture.
-        app_with_thousand_pages: The app harness.
-    """
-
-    def setup():
-        with chdir(app_with_thousand_pages.app_path):
-            utils.empty_dir(web_pages, keep_files=["_app.js"])
-            app_with_thousand_pages._initialize_app()
-            build.setup_frontend(app_with_thousand_pages.app_path)
-
-    def benchmark_fn():
-        with chdir(app_with_thousand_pages.app_path):
-            app_with_thousand_pages.app_instance._compile()
-
-    benchmark.pedantic(benchmark_fn, setup=setup, rounds=5)
-    app_with_thousand_pages._reload_state_module()
-
-
-@pytest.mark.benchmark(
-    group="Compile time of varying page numbers",
-    min_rounds=5,
-    timer=time.perf_counter,
-    disable_gc=True,
-    warmup=False,
-)
-def test_app_1000_compile_time_warm(benchmark, app_with_thousand_pages):
-    """Test the compile time on a warm start for an app with 1000 page.
-
-    Args:
-        benchmark: The benchmark fixture.
-        app_with_thousand_pages: The app harness.
-    """
-    with chdir(app_with_thousand_pages.app_path):
-        app_with_thousand_pages._initialize_app()
-        build.setup_frontend(app_with_thousand_pages.app_path)
-
-    def benchmark_fn():
-        with chdir(app_with_thousand_pages.app_path):
-            app_with_thousand_pages.app_instance._compile()
-
-    benchmark(benchmark_fn)
-    app_with_thousand_pages._reload_state_module()
-
-
-@pytest.mark.skip
-@pytest.mark.benchmark(
-    group="Compile time of varying page numbers",
-    timer=time.perf_counter,
-    disable_gc=True,
-    warmup=False,
-)
-def test_app_10000_compile_time_cold(benchmark, app_with_ten_thousand_pages):
-    """Test the compile time on a cold start for an app with 10000 page.
-
-    Args:
-        benchmark: The benchmark fixture.
-        app_with_ten_thousand_pages: The app harness.
-    """
-
-    def setup():
-        with chdir(app_with_ten_thousand_pages.app_path):
-            utils.empty_dir(web_pages, keep_files=["_app.js"])
-            app_with_ten_thousand_pages._initialize_app()
-            build.setup_frontend(app_with_ten_thousand_pages.app_path)
-
-    def benchmark_fn():
-        with chdir(app_with_ten_thousand_pages.app_path):
-            app_with_ten_thousand_pages.app_instance._compile()
-
-    benchmark.pedantic(benchmark_fn, setup=setup, rounds=5)
-    app_with_ten_thousand_pages._reload_state_module()
-
-
-@pytest.mark.skip
-@pytest.mark.benchmark(
-    group="Compile time of varying page numbers",
-    min_rounds=5,
-    timer=time.perf_counter,
-    disable_gc=True,
-    warmup=False,
-)
-def test_app_10000_compile_time_warm(benchmark, app_with_ten_thousand_pages):
-    """Test the compile time on a warm start for an app with 10000 page.
-
-    Args:
-        benchmark: The benchmark fixture.
-        app_with_ten_thousand_pages: The app harness.
-    """
-
-    def benchmark_fn():
-        with chdir(app_with_ten_thousand_pages.app_path):
-            app_with_ten_thousand_pages.app_instance._compile()
-
-    benchmark(benchmark_fn)
-    app_with_ten_thousand_pages._reload_state_module()