|
@@ -1,10 +1,21 @@
|
|
|
-'''
|
|
|
+# Copyright 2021-2024 Avaiga Private Limited
|
|
|
+#
|
|
|
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
|
|
+# the License. You may obtain a copy of the License at
|
|
|
+#
|
|
|
+# http://www.apache.org/licenses/LICENSE-2.0
|
|
|
+#
|
|
|
+# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
|
|
|
+# an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
|
|
+# specific language governing permissions and limitations under the License.
|
|
|
+
|
|
|
+"""
|
|
|
This script is a helper on the dependencies management of the project.
|
|
|
It can be used:
|
|
|
- To check that the same version of a package is set across files.
|
|
|
- To generate a Pipfile and requirements files with the latest version installables.
|
|
|
- To display a summary of the dependencies to update.
|
|
|
-'''
|
|
|
+"""
|
|
|
import glob
|
|
|
import itertools
|
|
|
import sys
|
|
@@ -22,6 +33,7 @@ class Release:
|
|
|
"""
|
|
|
Information about a release of a package.
|
|
|
"""
|
|
|
+
|
|
|
version: str
|
|
|
upload_date: datetime.date
|
|
|
|
|
@@ -31,6 +43,7 @@ class Package:
|
|
|
"""
|
|
|
Information about a package.
|
|
|
"""
|
|
|
+
|
|
|
# Package name
|
|
|
name: str
|
|
|
# Min version of the package set as requirements.
|
|
@@ -69,10 +82,7 @@ class Package:
|
|
|
"""
|
|
|
import requests # pylint: disable=import-outside-toplevel
|
|
|
|
|
|
- releases = requests.get(
|
|
|
- f"https://pypi.org/pypi/{self.name}/json",
|
|
|
- timeout=5
|
|
|
- ).json().get('releases', {})
|
|
|
+ releases = requests.get(f"https://pypi.org/pypi/{self.name}/json", timeout=5).json().get("releases", {})
|
|
|
|
|
|
for version, info in releases.items():
|
|
|
# Ignore old releases without upload time.
|
|
@@ -81,7 +91,7 @@ class Package:
|
|
|
# Ignore pre and post releases.
|
|
|
if any(str.isalpha(c) for c in version):
|
|
|
continue
|
|
|
- date = datetime.strptime(info[0]['upload_time'], "%Y-%m-%dT%H:%M:%S").date()
|
|
|
+ date = datetime.strptime(info[0]["upload_time"], "%Y-%m-%dT%H:%M:%S").date()
|
|
|
release = Release(version, date)
|
|
|
self.releases.append(release)
|
|
|
if self.min_version == version:
|
|
@@ -106,11 +116,11 @@ class Package:
|
|
|
|
|
|
if with_version:
|
|
|
if self.installation_markers:
|
|
|
- return f'{name}>={self.min_version},<={self.max_version};{self.installation_markers}'
|
|
|
- return f'{name}>={self.min_version},<={self.max_version}'
|
|
|
+ return f"{name}>={self.min_version},<={self.max_version};{self.installation_markers}"
|
|
|
+ return f"{name}>={self.min_version},<={self.max_version}"
|
|
|
|
|
|
if self.installation_markers:
|
|
|
- return f'{name};{self.installation_markers}'
|
|
|
+ return f"{name};{self.installation_markers}"
|
|
|
return name
|
|
|
|
|
|
def as_pipfile_line(self) -> str:
|
|
@@ -124,10 +134,10 @@ class Package:
|
|
|
line += f', markers="{self.installation_markers}"'
|
|
|
|
|
|
if self.extras_dependencies:
|
|
|
- dep = ','.join(f'"{p}"' for p in self.extras_dependencies)
|
|
|
- line += f', extras=[{dep}]'
|
|
|
+ dep = ",".join(f'"{p}"' for p in self.extras_dependencies)
|
|
|
+ line += f", extras=[{dep}]"
|
|
|
|
|
|
- line += '}'
|
|
|
+ line += "}"
|
|
|
return line
|
|
|
|
|
|
@classmethod
|
|
@@ -135,9 +145,9 @@ class Package:
|
|
|
"""
|
|
|
Check if a package definition is correctly formatted.
|
|
|
"""
|
|
|
- if '>=' not in package or '<' not in package:
|
|
|
+ if ">=" not in package or "<" not in package:
|
|
|
# Only Taipy dependencies can be without version.
|
|
|
- if 'taipy' not in package:
|
|
|
+ if "taipy" not in package:
|
|
|
raise Exception(f"Invalid package: {package}")
|
|
|
|
|
|
@classmethod
|
|
@@ -149,18 +159,18 @@ class Package:
|
|
|
try:
|
|
|
# Lower the name to avoid case issues.
|
|
|
name = extract_name(package).lower()
|
|
|
- is_taipy = 'taipy' in name
|
|
|
+ is_taipy = "taipy" in name
|
|
|
return cls(
|
|
|
name,
|
|
|
- extract_min_version(package) if not is_taipy else '',
|
|
|
- extract_max_version(package) if not is_taipy else '',
|
|
|
- extract_installation_markers(package) if not is_taipy else '',
|
|
|
+ extract_min_version(package) if not is_taipy else "",
|
|
|
+ extract_max_version(package) if not is_taipy else "",
|
|
|
+ extract_installation_markers(package) if not is_taipy else "",
|
|
|
is_taipy,
|
|
|
extract_extras_dependencies(package),
|
|
|
- [filename]
|
|
|
+ [filename],
|
|
|
)
|
|
|
except Exception as e:
|
|
|
- print(f"Error while parsing package {package}: {e}") # noqa: T201
|
|
|
+ print(f"Error while parsing package {package}: {e}") # noqa: T201
|
|
|
raise
|
|
|
|
|
|
|
|
@@ -169,9 +179,9 @@ def extract_installation_markers(package: str) -> str:
|
|
|
Extract the installation markers of a package from a requirements line.
|
|
|
ex: "pandas>=1.0.0,<2.0.0;python_version<'3.9'" -> "python_version<'3.9'"
|
|
|
"""
|
|
|
- if ';' not in package:
|
|
|
- return ''
|
|
|
- return package.split(';')[1]
|
|
|
+ if ";" not in package:
|
|
|
+ return ""
|
|
|
+ return package.split(";")[1]
|
|
|
|
|
|
|
|
|
def extract_min_version(package: str) -> str:
|
|
@@ -180,14 +190,14 @@ def extract_min_version(package: str) -> str:
|
|
|
ex: "pandas>=1.0.0,<2.0.0;python_version<'3.9'" -> "1.0.0"
|
|
|
"""
|
|
|
# The max version is the defined version if it is a fixed version.
|
|
|
- if '==' in package:
|
|
|
- version = package.split('==')[1]
|
|
|
- if ';' in version:
|
|
|
+ if "==" in package:
|
|
|
+ version = package.split("==")[1]
|
|
|
+ if ";" in version:
|
|
|
# Remove installation markers.
|
|
|
- version = version.split(';')[0]
|
|
|
+ version = version.split(";")[0]
|
|
|
return version
|
|
|
|
|
|
- return package.split('>=')[1].split(',')[0]
|
|
|
+ return package.split(">=")[1].split(",")[0]
|
|
|
|
|
|
|
|
|
def extract_max_version(package: str) -> str:
|
|
@@ -200,23 +210,23 @@ def extract_max_version(package: str) -> str:
|
|
|
- pandas>=1.0.0,<2.0.0;python_version<'3.9' -> 2.0.0
|
|
|
"""
|
|
|
# The max version is the defined version if it is a fixed version.
|
|
|
- if '==' in package:
|
|
|
- version = package.split('==')[1]
|
|
|
- if ';' in version:
|
|
|
+ if "==" in package:
|
|
|
+ version = package.split("==")[1]
|
|
|
+ if ";" in version:
|
|
|
# Remove installation markers.
|
|
|
- version = version.split(';')[0]
|
|
|
+ version = version.split(";")[0]
|
|
|
return version
|
|
|
|
|
|
version = None
|
|
|
|
|
|
- if ',<=' in package:
|
|
|
- version = package.split(',<=')[1]
|
|
|
+ if ",<=" in package:
|
|
|
+ version = package.split(",<=")[1]
|
|
|
else:
|
|
|
- version = package.split(',<')[1]
|
|
|
+ version = package.split(",<")[1]
|
|
|
|
|
|
- if ';' in version:
|
|
|
+ if ";" in version:
|
|
|
# Remove installation markers.
|
|
|
- version = version.split(';')[0]
|
|
|
+ version = version.split(";")[0]
|
|
|
|
|
|
return version
|
|
|
|
|
@@ -230,15 +240,15 @@ def extract_name(package: str) -> str:
|
|
|
- pandas==1.0.0;python_version<'3.9' -> pandas
|
|
|
- pandas>=1.0.0,<2.0.0;python_version<'3.9' -> pandas
|
|
|
"""
|
|
|
- if '==' in package:
|
|
|
- return package.split('==')[0]
|
|
|
+ if "==" in package:
|
|
|
+ return package.split("==")[0]
|
|
|
|
|
|
- name = package.split('>=')[0]
|
|
|
+ name = package.split(">=")[0]
|
|
|
|
|
|
# Remove optional dependencies.
|
|
|
# Ex: "pandas[sql]" -> "pandas"
|
|
|
- if '[' in name:
|
|
|
- name = name.split('[')[0]
|
|
|
+ if "[" in name:
|
|
|
+ name = name.split("[")[0]
|
|
|
return name
|
|
|
|
|
|
|
|
@@ -248,10 +258,10 @@ def extract_extras_dependencies(package: str) -> List[str]:
|
|
|
Ex:
|
|
|
- pymongo[srv]>=4.2.0,<=4.6.1 -> ["srv"]
|
|
|
"""
|
|
|
- if '[' not in package:
|
|
|
+ if "[" not in package:
|
|
|
return []
|
|
|
|
|
|
- return package.split('[')[1].split(']')[0].split(',')
|
|
|
+ return package.split("[")[1].split("]")[0].split(",")
|
|
|
|
|
|
|
|
|
def load_dependencies(requirements_filenames: List[str], enforce_format: bool) -> Dict[str, Package]:
|
|
@@ -279,8 +289,10 @@ def load_dependencies(requirements_filenames: List[str], enforce_format: bool) -
|
|
|
# In that case, do not load the releases again but ensure versions are the same.
|
|
|
if package.name in dependencies:
|
|
|
existing_package = dependencies[package.name]
|
|
|
- if not existing_package.min_version == package.min_version or \
|
|
|
- not existing_package.max_version == package.max_version:
|
|
|
+ if (
|
|
|
+ not existing_package.min_version == package.min_version
|
|
|
+ or not existing_package.max_version == package.max_version
|
|
|
+ ):
|
|
|
raise Exception(
|
|
|
f"Inconsistent version of '{package.name}' between '{filename}' and {','.join(package.files)}."
|
|
|
)
|
|
@@ -308,27 +320,29 @@ def display_dependencies_versions(dependencies: Dict[str, Package]):
|
|
|
# Load the latest releases of the package.
|
|
|
package.load_releases()
|
|
|
|
|
|
- to_print.append((
|
|
|
- package_name,
|
|
|
- f'{package.min_version} ({package.min_release.upload_date if package.min_release else "N.A."})',
|
|
|
- f'{package.max_version} ({package.max_release.upload_date if package.max_release else "N.C."})',
|
|
|
- f'{package.releases[0].version} ({package.releases[0].upload_date})',
|
|
|
- len(list(itertools.takewhile(lambda x: x.version != package.max_version, package.releases))), # noqa: B023
|
|
|
- ))
|
|
|
+ to_print.append(
|
|
|
+ (
|
|
|
+ package_name,
|
|
|
+ f'{package.min_version} ({package.min_release.upload_date if package.min_release else "N.A."})',
|
|
|
+ f'{package.max_version} ({package.max_release.upload_date if package.max_release else "N.C."})',
|
|
|
+ f"{package.releases[0].version} ({package.releases[0].upload_date})",
|
|
|
+ len(list(itertools.takewhile(lambda x: x.version != package.max_version, package.releases))), # noqa: B023
|
|
|
+ )
|
|
|
+ )
|
|
|
|
|
|
to_print.sort(key=lambda x: x[0])
|
|
|
- h = ['name', 'version-min', 'version-max', 'current-version', 'nb-releases-behind']
|
|
|
- print(tabulate.tabulate(to_print, headers=h, tablefmt='pretty')) # noqa: T201
|
|
|
+ h = ["name", "version-min", "version-max", "current-version", "nb-releases-behind"]
|
|
|
+ print(tabulate.tabulate(to_print, headers=h, tablefmt="pretty")) # noqa: T201
|
|
|
|
|
|
|
|
|
def update_dependencies(
|
|
|
- # Dependencies installed in the environment.
|
|
|
- dependencies_installed: Dict[str, Package],
|
|
|
- # Dependencies set in requirements files.
|
|
|
- dependencies_set: Dict[str, Package],
|
|
|
- # Requirements files to update.
|
|
|
- requirements_filenames: List[str]
|
|
|
- ):
|
|
|
+ # Dependencies installed in the environment.
|
|
|
+ dependencies_installed: Dict[str, Package],
|
|
|
+ # Dependencies set in requirements files.
|
|
|
+ dependencies_set: Dict[str, Package],
|
|
|
+ # Requirements files to update.
|
|
|
+ requirements_filenames: List[str],
|
|
|
+):
|
|
|
"""
|
|
|
Display and updates dependencies.
|
|
|
"""
|
|
@@ -342,32 +356,26 @@ def update_dependencies(
|
|
|
di = dependencies_installed.get(name)
|
|
|
# Some package as 'gitignore-parser' becomes 'gitignore_parser' during the installation.
|
|
|
if not di:
|
|
|
- di = dependencies_installed.get(name.replace('-', '_'))
|
|
|
+ di = dependencies_installed.get(name.replace("-", "_"))
|
|
|
|
|
|
if di:
|
|
|
if di.max_version != ds.max_version:
|
|
|
- to_print.append((
|
|
|
- name,
|
|
|
- di.max_version,
|
|
|
- ','.join(f.split('/')[0] for f in ds.files)
|
|
|
- ))
|
|
|
+ to_print.append((name, di.max_version, ",".join(f.split("/")[0] for f in ds.files)))
|
|
|
# Save the new dependency version.
|
|
|
ds.max_version = di.max_version
|
|
|
|
|
|
# Print the dependencies to update.
|
|
|
to_print.sort(key=lambda x: x[0])
|
|
|
- print(tabulate.tabulate(to_print, headers=['name', 'version', 'files'], tablefmt='pretty')) # noqa: T201
|
|
|
+ print(tabulate.tabulate(to_print, headers=["name", "version", "files"], tablefmt="pretty")) # noqa: T201
|
|
|
|
|
|
# Update requirements files.
|
|
|
for fd in requirements_filenames:
|
|
|
- requirements = '\n'.join(
|
|
|
- d.as_requirements_line()
|
|
|
- for d in sorted(dependencies_set.values(), key=lambda d: d.name)
|
|
|
- if fd in d.files
|
|
|
+ requirements = "\n".join(
|
|
|
+ d.as_requirements_line() for d in sorted(dependencies_set.values(), key=lambda d: d.name) if fd in d.files
|
|
|
)
|
|
|
# Add a new line at the end of the file.
|
|
|
- requirements += '\n'
|
|
|
- Path(fd).write_text(requirements, 'UTF-8')
|
|
|
+ requirements += "\n"
|
|
|
+ Path(fd).write_text(requirements, "UTF-8")
|
|
|
|
|
|
|
|
|
def generate_raw_requirements_txt(dependencies: Dict[str, Package]):
|
|
@@ -376,7 +384,7 @@ def generate_raw_requirements_txt(dependencies: Dict[str, Package]):
|
|
|
"""
|
|
|
for package in dependencies.values():
|
|
|
if not package.is_taipy:
|
|
|
- print(package.as_requirements_line(with_version=False)) # noqa: T201
|
|
|
+ print(package.as_requirements_line(with_version=False)) # noqa: T201
|
|
|
|
|
|
|
|
|
def update_pipfile(pipfile: str, dependencies_version: Dict[str, Package]):
|
|
@@ -389,13 +397,13 @@ def update_pipfile(pipfile: str, dependencies_version: Dict[str, Package]):
|
|
|
dependencies_str = ""
|
|
|
pipfile_obj = toml.load(pipfile)
|
|
|
|
|
|
- packages = pipfile_obj.pop('packages')
|
|
|
+ packages = pipfile_obj.pop("packages")
|
|
|
for name, dep in packages.items():
|
|
|
# Find the package in use.
|
|
|
rp = dependencies_version.get(name)
|
|
|
# Some package as 'gitignore-parser' becomes 'gitignore_parser' during the installation.
|
|
|
if not rp:
|
|
|
- rp = dependencies_version.get(name.replace('-', '_'))
|
|
|
+ rp = dependencies_version.get(name.replace("-", "_"))
|
|
|
if rp:
|
|
|
# Change for the real name of the package.
|
|
|
rp.name = name
|
|
@@ -404,48 +412,48 @@ def update_pipfile(pipfile: str, dependencies_version: Dict[str, Package]):
|
|
|
# Package not found. Can be due to python version.
|
|
|
# Ex: backports.zoneinfo
|
|
|
if isinstance(dep, dict):
|
|
|
- new_dep = ''
|
|
|
+ new_dep = ""
|
|
|
# Format as a Pipfile line.
|
|
|
new_dep = f'version="{dep["version"]}"'
|
|
|
- if dep.get('markers'):
|
|
|
+ if dep.get("markers"):
|
|
|
new_dep += f', markers="{dep["markers"]}"'
|
|
|
- if dep.get('extras'):
|
|
|
+ if dep.get("extras"):
|
|
|
new_dep += f', extras={dep["extras"]}'
|
|
|
dep = f"{{{new_dep}}}"
|
|
|
dependencies_str += f'"{name}" = {dep}\n'
|
|
|
else:
|
|
|
if isinstance(dep, dict):
|
|
|
# Requirements does not have installation markers and extras.
|
|
|
- rp.installation_markers = dep.get('markers', '')
|
|
|
- rp.extras_dependencies = [dep.get('extras')[0]] if dep.get('extras') else []
|
|
|
- dependencies_str += f'{rp.as_pipfile_line()}\n'
|
|
|
+ rp.installation_markers = dep.get("markers", "")
|
|
|
+ rp.extras_dependencies = [dep.get("extras")[0]] if dep.get("extras") else []
|
|
|
+ dependencies_str += f"{rp.as_pipfile_line()}\n"
|
|
|
|
|
|
toml_str = toml.dumps(pipfile_obj)
|
|
|
- Path(pipfile).write_text(f'{toml_str}\n\n[packages]\n{dependencies_str}', 'UTF-8')
|
|
|
+ Path(pipfile).write_text(f"{toml_str}\n\n[packages]\n{dependencies_str}", "UTF-8")
|
|
|
|
|
|
|
|
|
-if __name__ == '__main__':
|
|
|
- if sys.argv[1] == 'ensure-same-version':
|
|
|
+if __name__ == "__main__":
|
|
|
+ if sys.argv[1] == "ensure-same-version":
|
|
|
# Load dependencies from requirements files.
|
|
|
# Verify that the same version is set for the same package across files.
|
|
|
- _requirements_filenames = glob.glob('taipy*/*requirements.txt')
|
|
|
+ _requirements_filenames = glob.glob("taipy*/*requirements.txt")
|
|
|
_dependencies = load_dependencies(_requirements_filenames, True)
|
|
|
display_dependencies_versions(_dependencies)
|
|
|
- if sys.argv[1] == 'dependencies-summary':
|
|
|
+ if sys.argv[1] == "dependencies-summary":
|
|
|
# Load and compare dependencies from requirements files.
|
|
|
# The first file is the reference to the other.
|
|
|
# Display the differences including new version available on Pypi.
|
|
|
- _requirements_filenames = glob.glob('taipy*/*requirements.txt')
|
|
|
+ _requirements_filenames = glob.glob("taipy*/*requirements.txt")
|
|
|
_dependencies_installed = load_dependencies([sys.argv[2]], False)
|
|
|
_dependencies_set = load_dependencies(_requirements_filenames, False)
|
|
|
update_dependencies(_dependencies_installed, _dependencies_set, _requirements_filenames)
|
|
|
- if sys.argv[1] == 'generate-raw-requirements':
|
|
|
+ if sys.argv[1] == "generate-raw-requirements":
|
|
|
# Load dependencies from requirements files.
|
|
|
# Print the dependencies as requirements lines without born.
|
|
|
- _requirements_filenames = glob.glob('taipy*/*requirements.txt')
|
|
|
+ _requirements_filenames = glob.glob("taipy*/*requirements.txt")
|
|
|
_dependencies = load_dependencies(_requirements_filenames, False)
|
|
|
generate_raw_requirements_txt(_dependencies)
|
|
|
- if sys.argv[1] == 'generate-pipfile':
|
|
|
+ if sys.argv[1] == "generate-pipfile":
|
|
|
# Generate a new Pipfile from requirements files using dependencies versions
|
|
|
# set in the requirement file.
|
|
|
_pipfile_path = sys.argv[2]
|