Remove curl and parallelize node/bun install (#1458)

reflex/constants.py

@@ -45,54 +45,57 @@ MODULE_NAME = "reflex"
 # The current version of Reflex.
 VERSION = metadata.version(MODULE_NAME)
 
-# Project dependencies.
+# Files and directories used to init a new project.
 # The directory to store reflex dependencies.
-REFLEX_DIR = os.path.expandvars("$HOME/.reflex")
+REFLEX_DIR = os.path.expandvars(os.path.join("$HOME", f".{MODULE_NAME}"))
+# The root directory of the reflex library.
+ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+# The name of the assets directory.
+APP_ASSETS_DIR = "assets"
+# The template directory used during reflex init.
+TEMPLATE_DIR = os.path.join(ROOT_DIR, MODULE_NAME, ".templates")
+# The web subdirectory of the template directory.
+WEB_TEMPLATE_DIR = os.path.join(TEMPLATE_DIR, "web")
+# The assets subdirectory of the template directory.
+ASSETS_TEMPLATE_DIR = os.path.join(TEMPLATE_DIR, APP_ASSETS_DIR)
+# The jinja template directory.
+JINJA_TEMPLATE_DIR = os.path.join(TEMPLATE_DIR, "jinja")
 
 # Bun config.
 # The Bun version.
 BUN_VERSION = "0.7.0"
 # The directory to store the bun.
-BUN_ROOT_PATH = f"{REFLEX_DIR}/.bun"
+BUN_ROOT_PATH = os.path.join(REFLEX_DIR, ".bun")
 # The bun path.
-BUN_PATH = f"{BUN_ROOT_PATH}/bin/bun"
-# Command to install bun.
-INSTALL_BUN = f"curl -fsSL https://bun.sh/install | env BUN_INSTALL={BUN_ROOT_PATH} bash -s -- bun-v{BUN_VERSION}"
+BUN_PATH = os.path.join(BUN_ROOT_PATH, "bin", "bun")
+# URL to bun install script.
+BUN_INSTALL_URL = "https://bun.sh/install"
 
 # NVM / Node config.
+# The NVM version.
+NVM_VERSION = "0.39.1"
 # The Node version.
 NODE_VERSION = "18.17.0"
 # The minimum required node version.
-MIN_NODE_VERSION = "16.8.0"
+NODE_VERSION_MIN = "16.8.0"
 # The directory to store nvm.
-NVM_ROOT_PATH = f"{REFLEX_DIR}/.nvm"
+NVM_DIR = os.path.join(REFLEX_DIR, ".nvm")
 # The nvm path.
-NVM_PATH = f"{NVM_ROOT_PATH}/nvm.sh"
+NVM_PATH = os.path.join(NVM_DIR, "nvm.sh")
 # The node bin path.
-NODE_BIN_PATH = f"{NVM_ROOT_PATH}/versions/node/v{NODE_VERSION}/bin"
+NODE_BIN_PATH = os.path.join(NVM_DIR, "versions", "node", f"v{NODE_VERSION}", "bin")
 # The default path where node is installed.
-NODE_PATH = "node" if platform.system() == "Windows" else f"{NODE_BIN_PATH}/node"
+NODE_PATH = (
+    "node" if platform.system() == "Windows" else os.path.join(NODE_BIN_PATH, "node")
+)
 # The default path where npm is installed.
-NPM_PATH = "npm" if platform.system() == "Windows" else f"{NODE_BIN_PATH}/npm"
-# Command to install nvm.
-INSTALL_NVM = f"curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.1/install.sh | env NVM_DIR={NVM_ROOT_PATH} bash"
-# Command to install node.
-INSTALL_NODE = f'bash -c "export NVM_DIR={NVM_ROOT_PATH} && . {NVM_ROOT_PATH}/nvm.sh && nvm install {NODE_VERSION}"'
-
-
-# Files and directories used to init a new project.
-# The root directory of the reflex library.
-ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-# The name of the assets directory.
-APP_ASSETS_DIR = "assets"
-# The template directory used during reflex init.
-TEMPLATE_DIR = os.path.join(ROOT_DIR, MODULE_NAME, ".templates")
-# The web subdirectory of the template directory.
-WEB_TEMPLATE_DIR = os.path.join(TEMPLATE_DIR, "web")
-# The assets subdirectory of the template directory.
-ASSETS_TEMPLATE_DIR = os.path.join(TEMPLATE_DIR, APP_ASSETS_DIR)
-# The jinja template directory.
-JINJA_TEMPLATE_DIR = os.path.join(TEMPLATE_DIR, "jinja")
+NPM_PATH = (
+    "npm" if platform.system() == "Windows" else os.path.join(NODE_BIN_PATH, "npm")
+)
+# The URL to the nvm install script.
+NVM_INSTALL_URL = (
+    f"https://raw.githubusercontent.com/nvm-sh/nvm/v{NVM_VERSION}/install.sh"
+)
 
 # The frontend directories in a project.
 # The web folder where the NextJS app is compiled to.

reflex/utils/build.py

@@ -147,7 +147,7 @@ def export_app(
                 # Check for special strings and update the progress bar.
                 for special_string in checkpoints:
                     if special_string in line:
-                        if special_string == "Export successful":
+                        if special_string == checkpoints[-1]:
                             progress.update(task, completed=len(checkpoints))
                             break  # Exit the loop if the completion message is found
                         else:

reflex/utils/prerequisites.py

@@ -9,12 +9,15 @@ import platform
 import re
 import subprocess
 import sys
+import tempfile
+import threading
 from datetime import datetime
 from fileinput import FileInput
 from pathlib import Path
 from types import ModuleType
 from typing import Optional
 
+import httpx
 import typer
 from alembic.util.exc import CommandError
 from packaging import version
@@ -44,7 +47,7 @@ def check_node_version():
         current_version = version.parse(result.stdout.decode())
         # Compare the version numbers
         return (
-            current_version >= version.parse(constants.MIN_NODE_VERSION)
+            current_version >= version.parse(constants.NODE_VERSION_MIN)
             if IS_WINDOWS
             else current_version == version.parse(constants.NODE_VERSION)
         )
@@ -273,39 +276,68 @@ def initialize_node():
         install_node()
 
 
+def download_and_run(url: str, *args, **env):
+    """Download and run a script.
+
+
+    Args:
+        url: The url of the script.
+        args: The arguments to pass to the script.
+        env: The environment variables to use.
+
+
+    Raises:
+        Exit: if installation failed
+    """
+    # Download the script
+    response = httpx.get(url)
+    if response.status_code != httpx.codes.OK:
+        response.raise_for_status()
+
+    # Save the script to a temporary file.
+    script = tempfile.NamedTemporaryFile()
+    with open(script.name, "w") as f:
+        f.write(response.text)
+
+    # Run the script.
+    env = {
+        **os.environ,
+        **env,
+    }
+    result = subprocess.run(["bash", f.name, *args], env=env)
+    if result.returncode != 0:
+        raise typer.Exit(code=result.returncode)
+
+
 def install_node():
     """Install nvm and nodejs for use by Reflex.
        Independent of any existing system installations.
 
     Raises:
-        FileNotFoundError: if unzip or curl packages are not found.
         Exit: if installation failed
     """
+    # NVM is not supported on Windows.
     if IS_WINDOWS:
         console.print(
             f"[red]Node.js version {constants.NODE_VERSION} or higher is required to run Reflex."
         )
         raise typer.Exit()
 
-    # Only install if bun is not already installed.
-    console.log("Installing nvm...")
-
-    # Check if curl is installed
-    # TODO no need to shell out to curl
-    curl_path = path_ops.which("curl")
-    if curl_path is None:
-        raise FileNotFoundError("Reflex requires curl to be installed.")
-
     # Create the nvm directory and install.
-    path_ops.mkdir(constants.NVM_ROOT_PATH)
-    result = subprocess.run(constants.INSTALL_NVM, shell=True)
-
-    if result.returncode != 0:
-        raise typer.Exit(code=result.returncode)
-
-    console.log("Installing node...")
-    result = subprocess.run(constants.INSTALL_NODE, shell=True)
-
+    path_ops.mkdir(constants.NVM_DIR)
+    env = {**os.environ, "NVM_DIR": constants.NVM_DIR}
+    download_and_run(constants.NVM_INSTALL_URL, **env)
+
+    # Install node.
+    # We use bash -c as we need to source nvm.sh to use nvm.
+    result = subprocess.run(
+        [
+            "bash",
+            "-c",
+            f". {constants.NVM_DIR}/nvm.sh && nvm install {constants.NODE_VERSION}",
+        ],
+        env=env,
+    )
     if result.returncode != 0:
         raise typer.Exit(code=result.returncode)
 
@@ -314,32 +346,27 @@ def install_bun():
     """Install bun onto the user's system.
 
     Raises:
-        FileNotFoundError: if unzip or curl packages are not found.
-        Exit: if installation failed
+        FileNotFoundError: If required packages are not found.
     """
     # Bun is not supported on Windows.
     if IS_WINDOWS:
-        console.log("Skipping bun installation on Windows.")
         return
 
-    # Only install if bun is not already installed.
-    if not os.path.exists(constants.BUN_PATH):
-        console.log("Installing bun...")
-
-        # Check if curl is installed
-        curl_path = path_ops.which("curl")
-        if curl_path is None:
-            raise FileNotFoundError("Reflex requires curl to be installed.")
-
-        # Check if unzip is installed
-        unzip_path = path_ops.which("unzip")
-        if unzip_path is None:
-            raise FileNotFoundError("Reflex requires unzip to be installed.")
+    # Skip if bun is already installed.
+    if os.path.exists(constants.BUN_PATH):
+        return
 
-        result = subprocess.run(constants.INSTALL_BUN, shell=True)
+    # Check if unzip is installed
+    unzip_path = path_ops.which("unzip")
+    if unzip_path is None:
+        raise FileNotFoundError("Reflex requires unzip to be installed.")
 
-        if result.returncode != 0:
-            raise typer.Exit(code=result.returncode)
+    # Run the bun install script.
+    download_and_run(
+        constants.BUN_INSTALL_URL,
+        f"bun-v{constants.BUN_VERSION}",
+        BUN_INSTALL=constants.BUN_ROOT_PATH,
+    )
 
 
 def install_frontend_packages():
@@ -417,12 +444,20 @@ def initialize_frontend_dependencies():
     path_ops.mkdir(constants.REFLEX_DIR)
 
     # Install the frontend dependencies.
-    initialize_bun()
-    initialize_node()
+    threads = [
+        threading.Thread(target=initialize_bun),
+        threading.Thread(target=initialize_node),
+    ]
+    for thread in threads:
+        thread.start()
 
     # Set up the web directory.
     initialize_web_directory()
 
+    # Wait for the threads to finish.
+    for thread in threads:
+        thread.join()
+
 
 def check_admin_settings():
     """Check if admin settings are set and valid for logging in cli app."""

reflex/utils/processes.py

@@ -134,13 +134,15 @@ def new_process(args, **kwargs):
     Returns:
         Execute a child program in a new process.
     """
-    env = os.environ.copy()
-    env["PATH"] = os.pathsep.join([constants.NODE_BIN_PATH, env["PATH"]])
+    env = {
+        **os.environ,
+        "PATH": os.pathsep.join([constants.NODE_BIN_PATH, os.environ["PATH"]]),
+    }
     kwargs = {
         "env": env,
         "stderr": subprocess.STDOUT,
-        "stdout": subprocess.PIPE,  # Redirect stdout to a pipe
-        "universal_newlines": True,  # Set universal_newlines to True for text mode
+        "stdout": subprocess.PIPE,
+        "universal_newlines": True,
         "encoding": "UTF-8",
         **kwargs,
     }

tests/test_utils.py

@@ -9,6 +9,7 @@ import typer
 from packaging import version
 
 from reflex import Env, constants
+from reflex.base import Base
 from reflex.utils import build, format, imports, prerequisites, types
 from reflex.vars import Var
 
@@ -527,13 +528,20 @@ def test_node_install_windows(mocker):
 def test_node_install_unix(tmp_path, mocker):
     nvm_root_path = tmp_path / ".reflex" / ".nvm"
 
-    mocker.patch("reflex.utils.prerequisites.constants.NVM_ROOT_PATH", nvm_root_path)
+    mocker.patch("reflex.utils.prerequisites.constants.NVM_DIR", nvm_root_path)
     subprocess_run = mocker.patch(
         "reflex.utils.prerequisites.subprocess.run",
         return_value=subprocess.CompletedProcess(args="", returncode=0),
     )
     mocker.patch("reflex.utils.prerequisites.IS_WINDOWS", False)
 
+    class Resp(Base):
+        status_code = 200
+        text = "test"
+
+    mocker.patch("httpx.get", return_value=Resp())
+    mocker.patch("reflex.utils.prerequisites.download_and_run")
+
     prerequisites.install_node()
 
     assert nvm_root_path.exists()
@@ -541,14 +549,15 @@ def test_node_install_unix(tmp_path, mocker):
     subprocess_run.call_count = 2
 
 
-def test_node_install_without_curl(mocker):
-    """Test that an error is thrown when installing node with curl not installed.
+def test_bun_install_without_unzip(mocker):
+    """Test that an error is thrown when installing bun with unzip not installed.
 
     Args:
         mocker: Pytest mocker object.
     """
-    mocker.patch("reflex.utils.prerequisites.path_ops.which", return_value=None)
+    mocker.patch("reflex.utils.path_ops.which", return_value=None)
+    mocker.patch("os.path.exists", return_value=False)
     mocker.patch("reflex.utils.prerequisites.IS_WINDOWS", False)
 
     with pytest.raises(FileNotFoundError):
-        prerequisites.install_node()
+        prerequisites.install_bun()