Merge pull request #164 from adferrand/include_local_wheels

Create a local_wheels parameter to include wheel files matching one or more glob paths

README.rst

@@ -36,8 +36,8 @@ Quickstart
             beautifulsoup4==4.6.0
             html5lib==0.999999999
 
-       # To bundle packages which don't publish wheels, see the docs on the
-       # config file.
+       # To bundle packages which don't publish wheels, or to include directly wheel files
+       # from a directory, see the docs on the config file.
 
        # Other files and folders that should be installed
        files = LICENSE

doc/cfgfile.rst

@@ -223,6 +223,20 @@ the line with the key:
 
    .. versionadded:: 2.0
 
+.. describe:: local_wheels (optional)
+
+   One or more glob paths that match one or more wheel files located on the
+   local filesystem. All matching wheel files will be included in the installer.
+   For instance ``wheels\*.whl`` will include all wheel files from the relative
+   folder ``wheels``. 
+   
+   Any included wheel corresponding to the same distribution
+   of a wheel specified in ``pypi_wheels`` will raise an error.
+
+   Relative glob paths are from the directory containing the config file.
+
+   .. versionadded:: 2.2
+
 .. describe:: packages (optional)
 
    A list of importable package and module names to include in the installer.

nsist/__init__.py

@@ -78,6 +78,8 @@ class InstallerBuilder(object):
     :param list pypi_wheel_reqs: Package specifications to fetch from PyPI as wheels
     :param extra_wheel_sources: Directory paths to find wheels in.
     :type extra_wheel_sources: list of Path objects
+    :param local_wheels: Glob paths matching wheel files to include
+    :type local_wheels: list of str
     :param list extra_files: List of 2-tuples (file, destination) of files to include
     :param list exclude: Paths of files to exclude that would otherwise be included
     :param str py_version: Full version of Python to bundle
@@ -96,7 +98,7 @@ class InstallerBuilder(object):
                 py_format='bundled', inc_msvcrt=True, build_dir=DEFAULT_BUILD_DIR,
                 installer_name=None, nsi_template=None,
                 exclude=None, pypi_wheel_reqs=None, extra_wheel_sources=None,
-                commands=None, license_file=None):
+                local_wheels=None, commands=None, license_file=None):
         self.appname = appname
         self.version = version
         self.publisher = publisher
@@ -107,6 +109,7 @@ class InstallerBuilder(object):
         self.extra_files = extra_files or []
         self.pypi_wheel_reqs = pypi_wheel_reqs or []
         self.extra_wheel_sources = extra_wheel_sources or []
+        self.local_wheels = local_wheels or []
         self.commands = commands or {}
         self.license_file = license_file
 
@@ -347,8 +350,8 @@ if __name__ == '__main__':
         else:
             os.mkdir(build_pkg_dir)
 
-        # 2. Wheels from PyPI
-        fetch_pypi_wheels(self.pypi_wheel_reqs, build_pkg_dir,
+        # 2. Wheels specified in pypi_wheel_reqs or in paths of local_wheels
+        fetch_pypi_wheels(self.pypi_wheel_reqs, self.local_wheels, build_pkg_dir,
                           py_version=self.py_version, bitness=self.py_bitness,
                           extra_sources=self.extra_wheel_sources,
                           exclude=self.exclude)

nsist/configreader.py

@@ -76,6 +76,7 @@ CONFIG_VALIDATORS = {
         ('extra_wheel_sources', False),
         ('files', False),
         ('exclude', False),
+        ('local_wheels', False)
     ]),
     'Python': SectionValidator([
         ('version', False),
@@ -235,4 +236,5 @@ def get_installer_builder_args(config):
     args['installer_name'] = config.get('Build', 'installer_name', fallback=None)
     args['nsi_template'] = config.get('Build', 'nsi_template', fallback=None)
     args['exclude'] = config.get('Include', 'exclude', fallback='').strip().splitlines()
+    args['local_wheels'] = config.get('Include', 'local_wheels', fallback='').strip().splitlines()
     return args

nsist/pypi.py

@@ -2,14 +2,16 @@
 import fnmatch
 import hashlib
 import logging
-from pathlib import Path
+import glob
+import os
 import re
 import shutil
-from tempfile import mkdtemp
+import yarg
 import zipfile
 
-import yarg
+from pathlib import Path
 from requests_download import download, HashTracker
+from tempfile import mkdtemp
 
 from .util import get_cache_dir, normalize_path
 
@@ -82,8 +84,8 @@ class WheelLocator(object):
         Returns a Path or None.
         """
         whl_filename_prefix = '{name}-{version}-'.format(
-            name=re.sub("[^\w\d.]+", "_", self.name),
-            version=re.sub("[^\w\d.]+", "_", self.version),
+            name=re.sub(r'[^\w\d.]+', '_', self.name),
+            version=re.sub(r'[^\w\d.]+', '_', self.version),
         )
         for source in self.extra_sources:
             candidates = [CachedRelease(p.name)
@@ -92,7 +94,6 @@ class WheelLocator(object):
             rel = self.pick_best_wheel(candidates)
             if rel:
                 path = source / rel.filename
-                logger.info('Using wheel from extra directory: %s', path)
                 return path
 
     def check_cache(self):
@@ -109,7 +110,6 @@ class WheelLocator(object):
         if rel is None:
             return None
 
-        logger.info('Using cached wheel: %s', rel.filename)
         return release_dir / rel.filename
 
     def get_from_pypi(self):
@@ -158,10 +158,12 @@ class WheelLocator(object):
         """Find and return a compatible wheel (main interface)"""
         p = self.check_extra_sources()
         if p is not None:
+            logger.info('Using wheel from extra directory: %s', p)
             return p
 
         p = self.check_cache()
         if p is not None:
+            logger.info('Using cached wheel: %s', p)
             return p
 
         return self.get_from_pypi()
@@ -250,12 +252,58 @@ def extract_wheel(whl_file, target_dir, exclude=None):
     shutil.rmtree(str(td))
 
 
-def fetch_pypi_wheels(requirements, target_dir, py_version, bitness,
-                      extra_sources=None, exclude=None):
-    for req in requirements:
+def fetch_pypi_wheels(wheels_requirements, wheels_paths, target_dir, py_version,
+                      bitness, extra_sources=None, exclude=None):
+    """
+    Gather wheels included explicitly by wheels_pypi parameter 
+    or matching glob paths given in local_wheels parameter.
+    """
+    distributions = []
+    # We try to get the wheels from wheels_pypi requirements parameter
+    for req in wheels_requirements:
         wl = WheelLocator(req, py_version, bitness, extra_sources)
-        whl_file = wl.fetch()
+        whl_file = wl.fetch() 
         extract_wheel(whl_file, target_dir, exclude=exclude)
+        distributions.append(wl.name)
+    # Then from the local_wheels paths parameter
+    for glob_path in wheels_paths:
+        paths = glob.glob(glob_path)
+        if not paths:
+            raise ValueError('Error, glob path {0} does not match any wheel file'.format(glob_path))
+        for path in paths:
+            logger.info('Collecting wheel file: %s (from: %s)', os.path.basename(path), glob_path)
+            validate_wheel(path, distributions, py_version, bitness)
+            extract_wheel(path, target_dir, exclude=exclude)
+
+
+def extract_distribution_and_version(wheel_name):
+    """Extract distribution and version from a wheel file name"""
+    search = re.search(r'^([^-]+)-([^-]+)-.*\.whl$', wheel_name)
+    if not search:
+        raise ValueError('Invalid wheel file name: {0}'.format(wheel_name))
+
+    return (search.group(1), search.group(2))
+
+
+def validate_wheel(whl_path, distributions, py_version, bitness):
+    """
+    Verify that the given wheel can safely be included in the current installer.
+    If so, the given wheel info will be included in the given wheel info array.
+    If not, an exception will be raised.
+    """
+    wheel_name = os.path.basename(whl_path)
+    (distribution, version) = extract_distribution_and_version(wheel_name)
+
+    # Check that a distribution of same name has not been included before
+    if distribution in distributions:
+        raise ValueError('Error, wheel distribution {0} already included'.format(distribution))
+
+    # Check that the wheel is compatible with the installer environment
+    locator = WheelLocator('{0}=={1}'.format(distribution, version), py_version, bitness, [Path(os.path.dirname(whl_path))])
+    if not locator.check_extra_sources():
+        raise ValueError('Error, wheel {0} is not compatible with Python {1} for Windows'.format(wheel_name, py_version))
+
+    distributions.append(distribution)
 
 
 def is_excluded(path, exclude):

nsist/tests/test_local_wheels.py

@@ -0,0 +1,68 @@
+import unittest
+import os
+import platform
+import subprocess
+import glob
+
+from testpath.tempdir import TemporaryDirectory
+from testpath import assert_isfile, assert_isdir
+from nsist.pypi import fetch_pypi_wheels
+
+class TestLocalWheels(unittest.TestCase):
+    def test_matching_one_pattern(self):
+        with TemporaryDirectory() as td1:
+            subprocess.call(['pip', 'wheel', 'requests==2.19.1', '-w', td1])
+
+            with TemporaryDirectory() as td2:
+                fetch_pypi_wheels([], [os.path.join(td1, '*.whl')], td2, platform.python_version(), 64)
+
+                assert_isdir(os.path.join(td2, 'requests'))
+                assert_isfile(os.path.join(td2, 'requests-2.19.1.dist-info', 'METADATA'))
+
+                assert_isdir(os.path.join(td2, 'urllib3'))
+                self.assertTrue(glob.glob(os.path.join(td2, 'urllib3*.dist-info')))
+
+    def test_duplicate_wheel_files_raise(self):
+        with TemporaryDirectory() as td1:
+            subprocess.call(['pip', 'wheel', 'requests==2.19.1', '-w', td1])
+
+            with TemporaryDirectory() as td2:
+                with self.assertRaisesRegex(ValueError, 'wheel distribution requests already included'):
+                    fetch_pypi_wheels(['requests==2.19.1'], [os.path.join(td1, '*.whl')], td2, platform.python_version(), 64)
+
+    def test_invalid_wheel_file_raise(self):
+        with TemporaryDirectory() as td1:
+            open(os.path.join(td1, 'notawheel.txt'), 'w+')
+
+            with TemporaryDirectory() as td2:
+                with self.assertRaisesRegex(ValueError, 'Invalid wheel file name: notawheel.txt'):
+                    fetch_pypi_wheels([], [os.path.join(td1, '*')], td2, platform.python_version(), 64)
+
+    def test_incompatible_plateform_wheel_file_raise(self):
+        with TemporaryDirectory() as td1:
+            open(os.path.join(td1, 'incompatiblewheel-1.0.0-py2.py3-none-linux_x86_64.whl'), 'w+')
+
+            with TemporaryDirectory() as td2:
+                with self.assertRaisesRegex(ValueError, '{0} is not compatible with Python {1} for Windows'
+                .format('incompatiblewheel-1.0.0-py2.py3-none-linux_x86_64.whl', platform.python_version())):
+                    fetch_pypi_wheels([], [os.path.join(td1, '*.whl')], td2, platform.python_version(), 64)
+
+    def test_incompatible_python_wheel_file_raise(self):
+        with TemporaryDirectory() as td1:
+            open(os.path.join(td1, 'incompatiblewheel-1.0.0-py26-none-any.whl'), 'w+')
+
+            with TemporaryDirectory() as td2:
+                with self.assertRaisesRegex(ValueError, '{0} is not compatible with Python {1} for Windows'
+                .format('incompatiblewheel-1.0.0-py26-none-any.whl', platform.python_version())):
+                    fetch_pypi_wheels([], [os.path.join(td1, '*.whl')], td2, platform.python_version(), 64)
+
+    def test_useless_wheel_glob_path_raise(self):
+        with TemporaryDirectory() as td1:
+            with TemporaryDirectory() as td2:
+                with self.assertRaisesRegex(ValueError, 'does not match any wheel file'):
+                    fetch_pypi_wheels([], [os.path.join(td1, '*.whl')], td2, platform.python_version(), 64)
+
+
+# To exclude these, run:  nosetests -a '!network'
+TestLocalWheels.test_matching_one_pattern.network = 1
+TestLocalWheels.test_duplicate_wheel_files_raise.network = 1