Cache dependencies on github actions (#1296) (#1337)

.github/actions/install/action.yml

@@ -22,28 +22,28 @@ outputs:
 runs:
   using: "composite"
   steps:
+    - name: Replace Pipfile if pipfile-version = max
+      if: inputs.pipfile-version == 'max'
+      run: |
+        # Override the current Pipfile with the Pipfile containing the max versions
+        rm Pipfile
+        mv tools/packages/pipfiles/Pipfile${{ inputs.python-version }}.max Pipfile
+      shell: bash
+
     - uses: actions/setup-python@v5
       with:
         python-version: ${{ inputs.python-version }}
+        cache: 'pipenv'
+        cache-dependency-path: 'Pipfile'
 
     - name: Install pipenv
       run: pip install pipenv --upgrade
       shell: bash
 
-    - name: Install minimum supported dependencies
-      if: inputs.pipfile-version == 'min'
+    - name: Install pipenv dependencies
       run: pipenv install --dev --python=${{ inputs.python-version }}
       shell: bash
 
-    - name: Install maximum version supported dependencies
-      if: inputs.pipfile-version == 'max'
-      run: |
-        # Override the current Pipfile with the Pipfile containing the max versions
-        rm Pipfile
-        mv tools/packages/pipfiles/Pipfile${{ inputs.python-version }}.max Pipfile
-        pipenv install --dev --python=${{ inputs.python-version }}
-      shell: bash
-
     - name: Setup LibMagic (MacOS)
       if: inputs.os == 'macos-13' && inputs.install-gui
       run: brew install libmagic
@@ -53,9 +53,30 @@ runs:
       if: inputs.install-gui
       with:
         node-version: 20
+        cache: 'npm'
+        cache-dependency-path: '**/package-lock.json'
 
-    - name: Frontend Bundle Build
+    - name: Hash frontend source code
       if: inputs.install-gui
+      id: hash-frontend
+      run: |
+        pipenv run python tools/frontend/hash_source.py
+        echo "HASH=$(cat hash.txt)" >> $GITHUB_OUTPUT
+        rm hash.txt
+      shell: bash
+
+    - name: Restore cached frontend build
+      if: inputs.install-gui
+      id: cache-fe-build
+      uses: actions/cache@v4
+      with:
+        path: |
+          taipy/gui/webapp
+          taipy/gui_core/lib
+        key: frontend-build-${{ runner.os }}-${{ steps.hash-frontend.outputs.HASH }}
+
+    - name: Frontend Bundle Build
+      if: inputs.install-gui && steps.cache-fe-build.outputs.cache-hit != 'true'
       run: pipenv run python tools/frontend/bundle_build.py
       shell: bash
 

.github/workflows/build-and-release-single-package.yml

@@ -53,7 +53,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           ssh-key: ${{secrets.DEPLOY_KEY}}
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: 3.9
       - uses: actions/setup-node@v4

.github/workflows/build-and-release.yml

@@ -50,7 +50,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           ssh-key: ${{secrets.DEPLOY_KEY}}
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: 3.9
       - uses: actions/setup-node@v4

.github/workflows/frontend.yml

@@ -24,17 +24,41 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
       - name: npm build and test with node ${{ matrix.node-version }} on ${{ matrix.os }}
         uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node-version }}
+          cache: 'npm'
+          cache-dependency-path: '**/package-lock.json'
+
+      - name: Hash taipy-gui source code
+        id: hash-gui-fe
+        working-directory: ./
+        run: |
+          python tools/frontend/hash_source.py --taipy-gui-only
+          echo "HASH=$(cat hash.txt)" >> $GITHUB_OUTPUT
+          rm hash.txt
+        shell: bash
+      - name: Restore cached frontend build
+        id: cache-gui-fe-build
+        uses: actions/cache@v4
+        with:
+          path: taipy/gui/webapp
+          key: taipy-gui-frontend-build-${{ runner.os }}-${{ steps.hash-gui-fe.outputs.HASH }}
 
       - name: Install dom dependencies
+        if: steps.cache-gui-fe-build.outputs.cache-hit != 'true'
         working-directory: ./frontend/taipy-gui/dom
         run: npm ci
       - name: Install dependencies
+        if: steps.cache-gui-fe-build.outputs.cache-hit != 'true'
         run: npm ci --omit=optional
-      - run: npm run build --if-present
+      - if: steps.cache-gui-fe-build.outputs.cache-hit != 'true'
+        run: npm run build --if-present
+
       - run: npm test
 
       - name: Code coverage

.github/workflows/partial-tests.yml

@@ -51,6 +51,8 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           python-version: ${{matrix.python-version}}
+          cache: 'pipenv'
+          cache-dependency-path: 'Pipfile'
       - name: Install Setuptools and wheel
         run: pip install --upgrade setuptools wheel
 
@@ -64,11 +66,34 @@ jobs:
         if: matrix.os == 'macos-13'
         run: brew install libmagic
 
-      - uses: actions/setup-node@v4
+      - name: Setup Node
+        if: steps.changes.outputs.gui == 'true' || steps.changes.outputs.gui-core == 'true'
+        uses: actions/setup-node@v4
         with:
           node-version: 20
-      - name: Frontend Bundle Build
+          cache: 'npm'
+          cache-dependency-path: '**/package-lock.json'
+
+      - name: Hash frontend source code
         if: steps.changes.outputs.gui == 'true' || steps.changes.outputs.gui-core == 'true'
+        id: hash-frontend
+        run: |
+          pipenv run python tools/frontend/hash_source.py
+          echo "HASH=$(cat hash.txt)" >> $GITHUB_OUTPUT
+          rm hash.txt
+        shell: bash
+      - name: Restore cached frontend build
+        if: steps.changes.outputs.gui == 'true' || steps.changes.outputs.gui-core == 'true'
+        id: cache-fe-build
+        uses: actions/cache@v4
+        with:
+          path: |
+            taipy/gui/webapp
+            taipy/gui_core/lib
+          key: frontend-build-${{ runner.os }}-${{ steps.hash-frontend.outputs.HASH }}
+
+      - name: Frontend Bundle Build
+        if: (steps.changes.outputs.gui == 'true' || steps.changes.outputs.gui-core == 'true') && steps.cache-fe-build.outputs.cache-hit != 'true'
         run: pipenv run python tools/frontend/bundle_build.py
 
       - name: Install Playwright
@@ -128,9 +153,13 @@ jobs:
             core:
               - 'taipy/core/**'
 
-      - uses: actions/setup-python@v5
+      - name: Setup Python with cache
+        if: steps.changes.outputs.core == 'true'
+        uses: actions/setup-python@v5
         with:
           python-version: ${{matrix.python-version}}
+          cache: 'pipenv'
+          cache-dependency-path: 'Pipfile'
 
       - name: Install pipenv
         if: steps.changes.outputs.core == 'true'
@@ -167,9 +196,13 @@ jobs:
             core:
               - 'taipy/core/**'
 
-      - uses: actions/setup-python@v5
+      - name: Setup Python with cache
+        if: steps.changes.outputs.core == 'true'
+        uses: actions/setup-python@v5
         with:
           python-version: ${{matrix.python-version}}
+          cache: 'pipenv'
+          cache-dependency-path: 'Pipfile'
 
       - name: Install pipenv
         if: steps.changes.outputs.core == 'true'

.github/workflows/publish.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: 3.8
 
@@ -109,7 +109,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-versions }}
 

frontend/taipy-gui/base/src/app.ts

@@ -43,6 +43,7 @@ export class TaipyApp {
         this.routes = undefined;
         this.path = path;
         this.socket = socket;
+        // Init socket io connection
         initSocket(socket, this);
     }
 

tools/frontend/hash_source.py

@@ -0,0 +1,75 @@
+# Copyright 2021-2024 Avaiga Private Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+# an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+
+import hashlib
+import os
+import sys
+
+ignore_folder = ["node_modules", "dist", "coverage", "build", "scripts", "test-config"]
+ignore_file_name = [
+    "DS_Store",
+    "env.local",
+    "env.development.local",
+    "env.test.local",
+    "env.production.local",
+    ".eslintrc.js",
+    ".gitignore",
+    "jest.config.js",
+    "typedoc-mkdocs.json",
+    ".env",
+    "dev.env",
+]
+ignore_file_extension = [".md"]
+
+
+def hash_file(file_path):
+    hasher = hashlib.sha256()
+    with open(file_path, "rb") as f:
+        buf = f.read()
+        hasher.update(buf)
+    return hasher.hexdigest()
+
+
+def hash_files_in_frontend_folder(frontend_folder):
+    combined_hasher = hashlib.sha256()
+    file_hashes = {}
+    lookup_fe_folder = [f"{frontend_folder}{os.sep}taipy", f"{frontend_folder}{os.sep}taipy-gui"]
+    if len(sys.argv) > 1 and sys.argv[1] == "--taipy-gui-only":
+        lookup_fe_folder.pop(0)
+    for root_folder in lookup_fe_folder:
+        # Sort before looping to ensure consistent cache key
+        for root, _, files in sorted(os.walk(root_folder)):
+            if any(ignore in root for ignore in ignore_folder):
+                continue
+            # Sort before looping to ensure consistent cache key
+            for file in sorted(files):
+                if any(ignore in file for ignore in ignore_file_name):
+                    continue
+                if any(file.endswith(ignore) for ignore in ignore_file_extension):
+                    continue
+                file_path = os.path.join(root, file)
+                file_hash = hash_file(file_path)
+                file_hashes[file_path] = file_hash
+                combined_hasher.update(file_hash.encode())
+
+    combined_hash = combined_hasher.hexdigest()
+    return file_hashes, combined_hash
+
+
+if __name__ == "__main__":
+    frontend_folder = "frontend"
+    file_hashes, combined_hash = hash_files_in_frontend_folder(frontend_folder)
+    # for path, file_hash in sorted(file_hashes.items()):
+    #     print(f"{path}: {file_hash}")
+    print(f"Taipy Frontend hash {combined_hash}")  # noqa: T201
+    # write combined hash to file
+    with open("hash.txt", "w") as f:
+        f.write(combined_hash)