[REF-1682][REF-1683][REF-1684][REF-2283]Benchmark reflex package size and .web folder (#2880)

* remove codspeed.yml

* test upload job

* minor edits to get upload job working

* perhaps this works

* upload needs relex-install-size

* retrigger pipeline

* test only on ubuntu

* change to save to db directly

* oops

* size benchmarks

* .web for counter

* its timeout-minutes

* se integration.sh to run and kill process

* install psycopg2

* move .web runs to integration_tests.yml to save runners

* fix measurement-type for reflex-web

* add database url to env

* psycopg2

* test run ids

* commit sha gets the job done

* refactor

* add more matrices

* move reflex package size to integration_test.yml

* fix venv path

* test fix

* test fix

* use hyphen

* testing reflex build size

* ls for temp debug

* fix typo in command

* possible fix

* possible fix for windows

* remove dead code

* remove dead code

* remove unwanted comments

* refactor

* rebase on main

* pr_title

* remove pr_title from args

* debug

* should work now

* precommit fix

* print out package size for

* add shell

* test

* trying again

* dont use cached poetry to have accurate measurement of deps

* remove reflex deps calculation step from integration job

* fix script path

* precommit fix

* no real difference on different python versions so use 3.11.5

* remove ls keyword

.github/workflows/benchmarks.yml

@@ -22,6 +22,8 @@ env:
   TELEMETRY_ENABLED: false
   NODE_OPTIONS: '--max_old_space_size=4096'
   DATABASE_URL: ${{ secrets.DATABASE_URL }}
+  PR_TITLE: ${{ github.event.pull_request.title }}
+
 
 jobs:
   reflex-web:
@@ -63,16 +65,15 @@ jobs:
         run: |
           # Check that npm is home
           npm -v
-          poetry run bash scripts/benchmarks.sh ./reflex-web prod
+          poetry run bash scripts/benchmarks/benchmarks.sh ./reflex-web prod
         env:
           LHCI_GITHUB_APP_TOKEN: $
       - name: Run Benchmarks
         # Only run if the database creds are available in this context.
         if: ${{ env.DATABASE_URL }}
-        run: poetry run python scripts/lighthouse_score_upload.py "$GITHUB_SHA" ./integration/benchmarks/.lighthouseci
+        run: poetry run python scripts/benchmarks/lighthouse_score_upload.py "$GITHUB_SHA" ./integration/benchmarks/.lighthouseci
         env:
           GITHUB_SHA: ${{ github.sha }}
-          PR_TITLE: ${{ github.event.pull_request.title }}
 
   simple-apps-benchmarks:
     env:
@@ -119,11 +120,75 @@ jobs:
       - name: Upload benchmark results
         # Only run if the database creds are available in this context.
         if: ${{ env.DATABASE_URL }}
-        env:
-          PR_TITLE: ${{ github.event.pull_request.title }}
         run:
-          poetry run python scripts/simple_app_benchmark_upload.py --os "${{ matrix.os }}"
+          poetry run python scripts/benchmarks/simple_app_benchmark_upload.py --os "${{ matrix.os }}"
           --python-version "${{ matrix.python-version }}" --commit-sha "${{ github.sha }}"
           --benchmark-json "${{ env.OUTPUT_FILE }}"
           --db-url "${{ env.DATABASE_URL }}" --branch-name "${{ github.head_ref || github.ref_name }}"
           --event-type "${{ github.event_name }}" --actor "${{ github.actor }}" --pr-id "${{ github.event.pull_request.id }}"
+
+  reflex-build-size:
+    timeout-minutes: 30
+    strategy:
+      # Prioritize getting more information out of the workflow (even if something fails)
+      fail-fast: false
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup_build_env
+        with:
+          python-version: 3.11.5
+          run-poetry-install: true
+          create-venv-at-path: .venv
+      - name: Install additional dependencies for DB access
+        run: poetry run pip install psycopg2-binary
+      - name: Build reflex
+        run: | 
+          poetry build
+      - name: Upload benchmark results
+        # Only run if the database creds are available in this context.
+        if: ${{ env.DATABASE_URL }}
+        run: poetry run python scripts/benchmarks/benchmark_reflex_size.py --os ubuntu-latest
+          --python-version 3.11.5 --commit-sha "${{ github.sha }}" --pr-id "${{ github.event.pull_request.id }}" 
+          --db-url "${{ env.DATABASE_URL }}" --branch-name "${{ github.head_ref || github.ref_name }}"
+          --measurement-type "reflex-build" --path ./dist
+
+  reflex-plus-dependency-size:
+    timeout-minutes: 30
+    strategy:
+      # Prioritize getting more information out of the workflow (even if something fails)
+      fail-fast: false
+      matrix:
+        # Show OS combos first in GUI
+        os: [ ubuntu-latest, windows-latest, macos-latest ]
+        python-version: [ '3.11.5']
+
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version : 1.3.1
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+          virtualenvs-path: .venv
+
+      - name: Run poetry install
+        shell: bash
+        run: |
+          python -m venv .venv
+          source .venv/*/activate
+          poetry install --without dev --no-interaction --no-root  
+
+      - name: Install additional dependencies for DB access
+        run: poetry run pip install psycopg2-binary
+
+      - if: ${{ env.DATABASE_URL }}
+        name: calculate and upload size
+        run: poetry run python scripts/benchmarks/benchmark_reflex_size.py --os "${{ matrix.os }}"
+          --python-version "${{ matrix.python-version }}" --commit-sha "${{ github.sha }}"
+          --pr-id "${{ github.event.pull_request.id }}" --db-url "${{ env.DATABASE_URL }}"
+          --branch-name "${{ github.head_ref || github.ref_name }}"
+          --measurement-type "reflex-package" --path ./.venv

.github/workflows/integration_tests.yml

@@ -26,6 +26,9 @@ env:
   PYTHONIOENCODING: 'utf8'
   TELEMETRY_ENABLED: false
   NODE_OPTIONS: '--max_old_space_size=4096'
+  DATABASE_URL: ${{ secrets.DATABASE_URL }}
+  PR_TITLE: ${{ github.event.pull_request.title }}
+
 
 jobs:
   example-counter:
@@ -60,17 +63,17 @@ jobs:
           python-version: ${{ matrix.python-version }}
           run-poetry-install: true
           create-venv-at-path: .venv
-
       - name: Clone Reflex Examples Repo
         uses: actions/checkout@v4
         with:
           repository: reflex-dev/reflex-examples
           path: reflex-examples
-
       - name: Install requirements for counter example
         working-directory: ./reflex-examples/counter
         run: |
           poetry run pip install -r requirements.txt
+      - name: Install additional dependencies for DB access
+        run: poetry run pip install psycopg2-binary
       - name: Check export --backend-only before init for counter example
         working-directory: ./reflex-examples/counter
         run: |
@@ -91,6 +94,13 @@ jobs:
           # Check that npm is home
           npm -v
           poetry run bash scripts/integration.sh ./reflex-examples/counter dev
+      - name: Measure and upload .web size
+        if: ${{ env.DATABASE_URL }}
+        run: poetry run python scripts/benchmarks/benchmark_reflex_size.py --os "${{ matrix.os }}"
+          --python-version "${{ matrix.python-version }}" --commit-sha "${{ github.sha }}"
+          --pr-id "${{ github.event.pull_request.id }}" --db-url "${{ env.DATABASE_URL }}" 
+          --branch-name "${{ github.head_ref || github.ref_name }}"
+          --measurement-type "counter-app-dot-web" --path ./reflex-examples/counter/.web
 
   reflex-web:
     strategy:
@@ -121,6 +131,8 @@ jobs:
       - name: Install Requirements for reflex-web
         working-directory: ./reflex-web
         run: poetry run pip install -r requirements.txt
+      - name: Install additional dependencies for DB access
+        run: poetry run pip install psycopg2-binary
       - name: Init Website for reflex-web
         working-directory: ./reflex-web
         run: poetry run reflex init
@@ -129,3 +141,10 @@ jobs:
           # Check that npm is home
           npm -v
           poetry run bash scripts/integration.sh ./reflex-web prod
+      - name: Measure and upload .web size
+        if: ${{ env.DATABASE_URL }}
+        run: poetry run python scripts/benchmarks/benchmark_reflex_size.py --os "${{ matrix.os }}"
+          --python-version "${{ matrix.python-version }}" --commit-sha "${{ github.sha }}"
+          --pr-id "${{ github.event.pull_request.id }}" 
+          --db-url "${{ env.DATABASE_URL }}" --branch-name "${{ github.head_ref || github.ref_name }}"
+          --measurement-type "reflex-web-dot-web" --path ./reflex-web/.web

scripts/benchmarks/benchmark_reflex_size.py

@@ -0,0 +1,206 @@
+"""Checks the size of a specific directory and uploads result."""
+import argparse
+import os
+import subprocess
+from datetime import datetime
+
+import psycopg2
+
+
+def get_directory_size(directory):
+    """Get the size of a directory in bytes.
+
+    Args:
+        directory: The directory to check.
+
+    Returns:
+        The size of the dir in bytes.
+    """
+    total_size = 0
+    for dirpath, _, filenames in os.walk(directory):
+        for f in filenames:
+            fp = os.path.join(dirpath, f)
+            total_size += os.path.getsize(fp)
+    return total_size
+
+
+def get_python_version(venv_path, os_name):
+    """Get the python version of python in a virtual env.
+
+    Args:
+        venv_path: Path to virtual environment.
+        os_name: Name of os.
+
+    Returns:
+        The python version.
+    """
+    python_executable = (
+        os.path.join(venv_path, "bin", "python")
+        if "windows" not in os_name
+        else os.path.join(venv_path, "Scripts", "python.exe")
+    )
+    try:
+        output = subprocess.check_output(
+            [python_executable, "--version"], stderr=subprocess.STDOUT
+        )
+        python_version = output.decode("utf-8").strip().split()[1]
+        return ".".join(python_version.split(".")[:-1])
+    except subprocess.CalledProcessError:
+        return None
+
+
+def get_package_size(venv_path, os_name):
+    """Get the size of a specified package.
+
+    Args:
+        venv_path: The path to the venv.
+        os_name: Name of os.
+
+    Returns:
+        The total size of the package in bytes.
+
+    Raises:
+        ValueError: when venv does not exist or python version is None.
+    """
+    python_version = get_python_version(venv_path, os_name)
+    if python_version is None:
+        raise ValueError("Error: Failed to determine Python version.")
+
+    is_windows = "windows" in os_name
+
+    full_path = (
+        ["lib", f"python{python_version}", "site-packages"]
+        if not is_windows
+        else ["Lib", "site-packages"]
+    )
+
+    package_dir = os.path.join(venv_path, *full_path)
+    if not os.path.exists(package_dir):
+        raise ValueError(
+            "Error: Virtual environment does not exist or is not activated."
+        )
+
+    total_size = get_directory_size(package_dir)
+    return total_size
+
+
+def insert_benchmarking_data(
+    db_connection_url: str,
+    os_type_version: str,
+    python_version: str,
+    measurement_type: str,
+    commit_sha: str,
+    pr_title: str,
+    branch_name: str,
+    pr_id: str,
+    path: str,
+):
+    """Insert the benchmarking data into the database.
+
+    Args:
+        db_connection_url: The URL to connect to the database.
+        os_type_version: The OS type and version to insert.
+        python_version: The Python version to insert.
+        measurement_type: The type of metric to measure.
+        commit_sha: The commit SHA to insert.
+        pr_title: The PR title to insert.
+        branch_name: The name of the branch.
+        pr_id: The id of the PR.
+        path: The path to the dir or file to check size.
+    """
+    if measurement_type == "reflex-package":
+        size = get_package_size(path, os_type_version)
+    else:
+        size = get_directory_size(path)
+
+    # Get the current timestamp
+    current_timestamp = datetime.now()
+
+    # Connect to the database and insert the data
+    with psycopg2.connect(db_connection_url) as conn, conn.cursor() as cursor:
+        insert_query = """
+            INSERT INTO size_benchmarks (os, python_version, commit_sha, created_at, pr_title, branch_name, pr_id, measurement_type, size)
+            VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s);
+            """
+        cursor.execute(
+            insert_query,
+            (
+                os_type_version,
+                python_version,
+                commit_sha,
+                current_timestamp,
+                pr_title,
+                branch_name,
+                pr_id,
+                measurement_type,
+                round(
+                    size / (1024 * 1024), 3
+                ),  # save size in mb and round to 3 places.
+            ),
+        )
+        # Commit the transaction
+        conn.commit()
+
+
+def main():
+    """Runs the benchmarks and inserts the results."""
+    parser = argparse.ArgumentParser(description="Run benchmarks and process results.")
+    parser.add_argument(
+        "--os", help="The OS type and version to insert into the database."
+    )
+    parser.add_argument(
+        "--python-version", help="The Python version to insert into the database."
+    )
+    parser.add_argument(
+        "--commit-sha", help="The commit SHA to insert into the database."
+    )
+    parser.add_argument(
+        "--db-url",
+        help="The URL to connect to the database.",
+        required=True,
+    )
+    parser.add_argument(
+        "--pr-title",
+        help="The PR title to insert into the database.",
+    )
+    parser.add_argument(
+        "--branch-name",
+        help="The current branch",
+        required=True,
+    )
+    parser.add_argument(
+        "--pr-id",
+        help="The pr id",
+        required=True,
+    )
+    parser.add_argument(
+        "--measurement-type",
+        help="The type of metric to be checked.",
+        required=True,
+    )
+    parser.add_argument(
+        "--path",
+        help="the current path to check size.",
+        required=True,
+    )
+    args = parser.parse_args()
+
+    # Get the PR title from env or the args. For the PR merge or push event, there is no PR title, leaving it empty.
+    pr_title = args.pr_title or os.getenv("PR_TITLE", "")
+
+    # Insert the data into the database
+    insert_benchmarking_data(
+        db_connection_url=args.db_url,
+        os_type_version=args.os,
+        python_version=args.python_version,
+        measurement_type=args.measurement_type,
+        commit_sha=args.commit_sha,
+        pr_title=pr_title,
+        branch_name=args.branch_name,
+        pr_id=args.pr_id,
+        path=args.path,
+    )
+
+
+if __name__ == "__main__":
+    main()