diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml new file mode 100644 index 00000000..b062493f --- /dev/null +++ b/.github/workflows/run-unit-tests.yml @@ -0,0 +1,35 @@ +name: Run unit tests + +on: + workflow_dispatch: + pull_request: + push: + branches: + - master + +jobs: + test: + runs-on: ubuntu-24.04 + + strategy: + max-parallel: 4 + matrix: + python-version: [ "3.10", "3.11", "3.12", "3.13", "3.14" ] + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: make dev + + - name: Validate code format + run: make check + + - name: Run tests + run: make test diff --git a/AUTHORS.rst b/AUTHORS.rst index f60077ba..a3907a35 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -5,4 +5,4 @@ The following organizations or individuals have contributed to this repo: - Philippe Ombredanne @ pombredanne - Steven Esser @ majurg - Tushar Goel @ TG1999 - +- Thomas Druez @ tdruez diff --git a/Makefile b/Makefile index 2a9b0324..233be652 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,7 @@ isort: black: @echo "-> Apply black code formatter" - ${VENV}/bin/black -l 100 --exclude=src/fetchcode/vcs/pip src tests setup.py + ${VENV}/bin/black -l 100 --extend-exclude=src/fetchcode/vcs/pip/* . doc8: @echo "-> Run doc8 validation" @@ -33,11 +33,11 @@ valid: isort black check: @echo "-> Run pycodestyle (PEP8) validation" - @${ACTIVATE} pycodestyle --max-line-length=100 --exclude=.eggs,venv,lib,thirdparty,docs,migrations,settings.py,.cache,etc,src/fetchcode/vcs/pip,tests/data/ . + @${ACTIVATE} pycodestyle --max-line-length=100 --exclude=.eggs,venv,lib,thirdparty,docs,migrations,settings.py,.cache,etc,src/fetchcode/vcs/pip,tests/ . @echo "-> Run isort imports ordering validation" - @${ACTIVATE} isort --sl --check-only -l 100 --skip=src/fetchcode/vcs/pip setup.py src tests + @${ACTIVATE} isort --sl --check-only -l 100 --skip=src/fetchcode/vcs/pip setup.py tests @echo "-> Run black validation" - @${ACTIVATE} black --check --check -l 100 --exclude=src/fetchcode/vcs/pip src tests setup.py + @${ACTIVATE} black --check -l 100 --extend-exclude=src/fetchcode/vcs/pip/* . clean: @echo "-> Clean the Python env" diff --git a/configure b/configure index 22d92885..b6055e86 100755 --- a/configure +++ b/configure @@ -28,9 +28,9 @@ CLI_ARGS=$1 ################################ # Requirement arguments passed to pip and used by default or with --dev. -REQUIREMENTS="--editable . --constraint requirements.txt" -DEV_REQUIREMENTS="--editable .[testing] --constraint requirements.txt --constraint requirements-dev.txt" -DOCS_REQUIREMENTS="--editable .[docs] --constraint requirements.txt" +REQUIREMENTS="--editable ." +DEV_REQUIREMENTS="--editable .[testing]" +DOCS_REQUIREMENTS="--editable .[docs]" # where we create a virtualenv VIRTUALENV_DIR=venv diff --git a/etc/ci/azure-container-deb.yml b/etc/ci/azure-container-deb.yml deleted file mode 100644 index 85b611d3..00000000 --- a/etc/ci/azure-container-deb.yml +++ /dev/null @@ -1,50 +0,0 @@ -parameters: - job_name: '' - container: '' - python_path: '' - python_version: '' - package_manager: apt-get - install_python: '' - install_packages: | - set -e -x - sudo apt-get -y update - sudo apt-get -y install \ - build-essential \ - xz-utils zlib1g bzip2 libbz2-1.0 tar \ - sqlite3 libxml2-dev libxslt1-dev \ - software-properties-common openssl - test_suite: '' - test_suite_label: '' - - -jobs: - - job: ${{ parameters.job_name }} - - pool: - vmImage: 'ubuntu-16.04' - - container: - image: ${{ parameters.container }} - options: '--name ${{ parameters.job_name }} -e LANG=C.UTF-8 -e LC_ALL=C.UTF-8 -v /usr/bin/docker:/tmp/docker:ro' - - steps: - - checkout: self - fetchDepth: 10 - - - script: /tmp/docker exec -t -e LANG=C.UTF-8 -e LC_ALL=C.UTF-8 -u 0 ${{ parameters.job_name }} $(Build.SourcesDirectory)/etc/ci/install_sudo.sh ${{ parameters.package_manager }} - displayName: Install sudo - - - script: ${{ parameters.install_packages }} - displayName: Install required packages - - - script: ${{ parameters.install_python }} - displayName: 'Install Python ${{ parameters.python_version }}' - - - script: ${{ parameters.python_path }} --version - displayName: 'Show Python version' - - - script: PYTHON_EXE=${{ parameters.python_path }} ./configure --dev - displayName: 'Run Configure' - - - script: ${{ parameters.test_suite }} - displayName: 'Run ${{ parameters.test_suite_label }} tests with py${{ parameters.python_version }} on ${{ parameters.job_name }}' diff --git a/etc/ci/azure-container-rpm.yml b/etc/ci/azure-container-rpm.yml deleted file mode 100644 index 1e6657d0..00000000 --- a/etc/ci/azure-container-rpm.yml +++ /dev/null @@ -1,51 +0,0 @@ -parameters: - job_name: '' - image_name: 'ubuntu-16.04' - container: '' - python_path: '' - python_version: '' - package_manager: yum - install_python: '' - install_packages: | - set -e -x - sudo yum groupinstall -y "Development Tools" - sudo yum install -y \ - openssl openssl-devel \ - sqlite-devel zlib-devel xz-devel bzip2-devel \ - bzip2 tar unzip zip \ - libxml2-devel libxslt-devel - test_suite: '' - test_suite_label: '' - - -jobs: - - job: ${{ parameters.job_name }} - - pool: - vmImage: ${{ parameters.image_name }} - - container: - image: ${{ parameters.container }} - options: '--name ${{ parameters.job_name }} -e LANG=C.UTF-8 -e LC_ALL=C.UTF-8 -v /usr/bin/docker:/tmp/docker:ro' - - steps: - - checkout: self - fetchDepth: 10 - - - script: /tmp/docker exec -t -e LANG=C.UTF-8 -e LC_ALL=C.UTF-8 -u 0 ${{ parameters.job_name }} $(Build.SourcesDirectory)/etc/ci/install_sudo.sh ${{ parameters.package_manager }} - displayName: Install sudo - - - script: ${{ parameters.install_packages }} - displayName: Install required packages - - - script: ${{ parameters.install_python }} - displayName: 'Install Python ${{ parameters.python_version }}' - - - script: ${{ parameters.python_path }} --version - displayName: 'Show Python version' - - - script: PYTHON_EXE=${{ parameters.python_path }} ./configure --dev - displayName: 'Run Configure' - - - script: ${{ parameters.test_suite }} - displayName: 'Run ${{ parameters.test_suite_label }} tests with py${{ parameters.python_version }} on ${{ parameters.job_name }}' diff --git a/etc/ci/azure-posix.yml b/etc/ci/azure-posix.yml deleted file mode 100644 index 9fdc7f15..00000000 --- a/etc/ci/azure-posix.yml +++ /dev/null @@ -1,39 +0,0 @@ -parameters: - job_name: '' - image_name: '' - python_versions: [] - test_suites: {} - python_architecture: x64 - -jobs: - - job: ${{ parameters.job_name }} - - pool: - vmImage: ${{ parameters.image_name }} - - strategy: - matrix: - ${{ each tsuite in parameters.test_suites }}: - ${{ tsuite.key }}: - test_suite_label: ${{ tsuite.key }} - test_suite: ${{ tsuite.value }} - - steps: - - checkout: self - fetchDepth: 10 - - - ${{ each pyver in parameters.python_versions }}: - - task: UsePythonVersion@0 - inputs: - versionSpec: '${{ pyver }}' - architecture: '${{ parameters.python_architecture }}' - displayName: '${{ pyver }} - Install Python' - - - script: | - python${{ pyver }} --version - echo "python${{ pyver }}" > PYTHON_EXECUTABLE - ./configure --clean && ./configure --dev - displayName: '${{ pyver }} - Configure' - - - script: $(test_suite) - displayName: '${{ pyver }} - $(test_suite_label) on ${{ parameters.job_name }}' diff --git a/etc/ci/azure-win.yml b/etc/ci/azure-win.yml deleted file mode 100644 index 26b41116..00000000 --- a/etc/ci/azure-win.yml +++ /dev/null @@ -1,39 +0,0 @@ -parameters: - job_name: '' - image_name: '' - python_versions: [] - test_suites: {} - python_architecture: x64 - -jobs: - - job: ${{ parameters.job_name }} - - pool: - vmImage: ${{ parameters.image_name }} - - strategy: - matrix: - ${{ each tsuite in parameters.test_suites }}: - ${{ tsuite.key }}: - test_suite_label: ${{ tsuite.key }} - test_suite: ${{ tsuite.value }} - - steps: - - checkout: self - fetchDepth: 10 - - - ${{ each pyver in parameters.python_versions }}: - - task: UsePythonVersion@0 - inputs: - versionSpec: '${{ pyver }}' - architecture: '${{ parameters.python_architecture }}' - displayName: '${{ pyver }} - Install Python' - - - script: | - python --version - echo | set /p=python> PYTHON_EXECUTABLE - configure --clean && configure --dev - displayName: '${{ pyver }} - Configure' - - - script: $(test_suite) - displayName: '${{ pyver }} - $(test_suite_label) on ${{ parameters.job_name }}' diff --git a/etc/ci/install_sudo.sh b/etc/ci/install_sudo.sh deleted file mode 100644 index 77f4210d..00000000 --- a/etc/ci/install_sudo.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -set -e - - -if [[ "$1" == "apt-get" ]]; then - apt-get update -y - apt-get -o DPkg::Options::="--force-confold" install -y sudo - -elif [[ "$1" == "yum" ]]; then - yum install -y sudo - -elif [[ "$1" == "dnf" ]]; then - dnf install -y sudo - -fi diff --git a/etc/ci/macports-ci b/etc/ci/macports-ci deleted file mode 100644 index ac474e4e..00000000 --- a/etc/ci/macports-ci +++ /dev/null @@ -1,304 +0,0 @@ -#! /bin/bash - -# Copyright (c) 2019 Giovanni Bussi - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -export COLUMNS=80 - -if [ "$GITHUB_ACTIONS" = true ] ; then - echo "COLUMNS=$COLUMNS" >> "$GITHUB_ENV" -fi - -# file to be source at the end of subshell: -export MACPORTS_CI_SOURCEME="$(mktemp)" - -( -# start subshell -# this allows to use the script in two ways: -# 1. as ./macports-ci -# 2. as source ./macports-ci -# as of now, choice 2 only changes the env var COLUMNS. - -MACPORTS_VERSION=2.6.4 -MACPORTS_PREFIX=/opt/local -MACPORTS_SYNC=tarball - -action=$1 -shift - -case "$action" in -(install) - -echo "macports-ci: install" - -KEEP_BREW=yes - -for opt -do - case "$opt" in - (--source) SOURCE=yes ;; - (--binary) SOURCE=no ;; - (--keep-brew) KEEP_BREW=yes ;; - (--remove-brew) KEEP_BREW=no ;; - (--version=*) MACPORTS_VERSION="${opt#--version=}" ;; - (--prefix=*) MACPORTS_PREFIX="${opt#--prefix=}" ;; - (--sync=*) MACPORTS_SYNC="${opt#--sync=}" ;; - (*) echo "macports-ci: unknown option $opt" - exit 1 ;; - esac -done - -if test "$KEEP_BREW" = no ; then - echo "macports-ci: removing homebrew" - pushd "$(mktemp -d)" - curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/uninstall > uninstall - chmod +x uninstall - ./uninstall --force - popd -else - echo "macports-ci: keeping HomeBrew" -fi - -echo "macports-ci: prefix=$MACPORTS_PREFIX" - -if test "$MACPORTS_PREFIX" != /opt/local ; then - echo "macports-ci: Installing on non standard prefix $MACPORTS_PREFIX can be only made from sources" - SOURCE=yes -fi - -if test "$SOURCE" = yes ; then - echo "macports-ci: Installing from source" -else - echo "macports-ci: Installing from binary" -fi - -echo "macports-ci: Sync mode=$MACPORTS_SYNC" - -pushd "$(mktemp -d)" - -OSX_VERSION="$(sw_vers -productVersion | grep -o '^[0-9][0-9]*\.[0-9][0-9]*')" - -if test "$OSX_VERSION" == 10.10 ; then - OSX_NAME=Yosemite -elif test "$OSX_VERSION" == 10.11 ; then - OSX_NAME=ElCapitan -elif test "$OSX_VERSION" == 10.12 ; then - OSX_NAME=Sierra -elif test "$OSX_VERSION" == 10.13 ; then - OSX_NAME=HighSierra -elif test "$OSX_VERSION" == 10.14 ; then - OSX_NAME=Mojave -elif test "$OSX_VERSION" == 10.15 ; then - OSX_NAME=Catalina -else - echo "macports-ci: Unknown OSX version $OSX_VERSION" - exit 1 -fi - -echo "macports-ci: OSX version $OSX_VERSION $OSX_NAME" - -MACPORTS_PKG=MacPorts-${MACPORTS_VERSION}-${OSX_VERSION}-${OSX_NAME}.pkg - -# this is a workaround needed because binary installer MacPorts-2.6.3-10.12-Sierra.pkg is broken -if [ "$SOURCE" != yes ] && [ "$MACPORTS_PKG" = "MacPorts-2.6.3-10.12-Sierra.pkg" ] ; then - echo "macports-ci: WARNING $MACPORTS_PKG installer is broken" - echo "macports-ci: reverting to 2.6.2 installer followed by selfupdate" - MACPORTS_VERSION=2.6.2 - MACPORTS_PKG=MacPorts-${MACPORTS_VERSION}-${OSX_VERSION}-${OSX_NAME}.pkg -fi - -URL="https://distfiles.macports.org/MacPorts" -URL="https://github.com/macports/macports-base/releases/download/v$MACPORTS_VERSION/" - -echo "macports-ci: Base URL is $URL" - -if test "$SOURCE" = yes ; then -# download source: - curl -LO $URL/MacPorts-${MACPORTS_VERSION}.tar.bz2 - tar xjf MacPorts-${MACPORTS_VERSION}.tar.bz2 - cd MacPorts-${MACPORTS_VERSION} -# install - ./configure --prefix="$MACPORTS_PREFIX" --with-applications-dir="$MACPORTS_PREFIX/Applications" >/dev/null && - sudo make install >/dev/null -else - -# download installer: - curl -LO $URL/$MACPORTS_PKG -# install: - sudo installer -verbose -pkg $MACPORTS_PKG -target / -fi - -# update: -export PATH="$MACPORTS_PREFIX/bin:$PATH" - -echo "PATH=\"$MACPORTS_PREFIX/bin:\$PATH\"" > "$MACPORTS_CI_SOURCEME" - -if [ "$GITHUB_ACTIONS" = true ] ; then - echo "$MACPORTS_PREFIX/bin" >> "$GITHUB_PATH" -fi - - -SOURCES="${MACPORTS_PREFIX}"/etc/macports/sources.conf - -case "$MACPORTS_SYNC" in -(rsync) - echo "macports-ci: Using rsync" - ;; -(github) - echo "macports-ci: Using github" - pushd "$MACPORTS_PREFIX"/var/macports/sources - sudo mkdir -p github.com/macports/macports-ports/ - sudo chown -R $USER:admin github.com - git clone https://github.com/macports/macports-ports.git github.com/macports/macports-ports/ - awk '{if($NF=="[default]") print "file:///opt/local/var/macports/sources/github.com/macports/macports-ports/"; else print}' "$SOURCES" > $HOME/$$.tmp - sudo mv -f $HOME/$$.tmp "$SOURCES" - popd - ;; -(tarball) - echo "macports-ci: Using tarball" - awk '{if($NF=="[default]") print "https://distfiles.macports.org/ports.tar.gz [default]"; else print}' "$SOURCES" > $$.tmp - sudo mv -f $$.tmp "$SOURCES" - ;; -(*) - echo "macports-ci: Unknown sync mode $MACPORTS_SYNC" - ;; -esac - -i=1 -# run through a while to retry upon failure -while true -do - echo "macports-ci: Trying to selfupdate (iteration $i)" -# here I test for the presence of a known portfile -# this check confirms that ports were installed -# notice that port -N selfupdate && break is not sufficient as a test -# (sometime it returns a success even though ports have not been installed) -# for some misterious reasons, running without "-d" does not work in some case - sudo port -d -N selfupdate 2>&1 | grep -v DEBUG | awk '{if($1!="x")print}' - port info xdrfile > /dev/null && break || true - sleep 5 - i=$((i+1)) - if ((i>20)) ; then - echo "macports-ci: Failed after $i iterations" - exit 1 - fi -done - -echo "macports-ci: Selfupdate successful after $i iterations" - -dir="$PWD" -popd -sudo rm -fr $dir - -;; - -(localports) - -echo "macports-ci: localports" - -for opt -do - case "$opt" in - (*) ports="$opt" ;; - esac -done - -if ! test -d "$ports" ; then - echo "macports-ci: Please provide a port directory" - exit 1 -fi - -w=$(which port) - -MACPORTS_PREFIX="${w%/bin/port}" - -cd "$ports" - -ports="$(pwd)" - -echo "macports-ci: Portdir fullpath: $ports" -SOURCES="${MACPORTS_PREFIX}"/etc/macports/sources.conf - -awk -v repo="file://$ports" '{if($NF=="[default]") print repo; print}' "$SOURCES" > $$.tmp -sudo mv -f $$.tmp "$SOURCES" - -portindex - -;; - -(ccache) -w=$(which port) -MACPORTS_PREFIX="${w%/bin/port}" - -echo "macports-ci: ccache" - -ccache_do=install - -for opt -do - case "$opt" in - (--save) ccache_do=save ;; - (--install) ccache_do=install ;; - (*) echo "macports-ci: ccache: unknown option $opt" - exit 1 ;; - esac -done - - -case "$ccache_do" in -(install) -# first install ccache -sudo port -N install ccache -# then tell macports to use it -CONF="${MACPORTS_PREFIX}"/etc/macports/macports.conf -awk '{if(match($0,"configureccache")) print "configureccache yes" ; else print }' "$CONF" > $$.tmp -sudo mv -f $$.tmp "$CONF" - -# notice that cache size is set to 512Mb, same as it is set by Travis-CI on linux -# might be changed in the future -test -f "$HOME"/.macports-ci-ccache/ccache.conf && - sudo rm -fr "$MACPORTS_PREFIX"/var/macports/build/.ccache && - sudo mkdir -p "$MACPORTS_PREFIX"/var/macports/build/.ccache && - sudo cp -a "$HOME"/.macports-ci-ccache/* "$MACPORTS_PREFIX"/var/macports/build/.ccache/ && - sudo echo "max_size = 512M" > "$MACPORTS_PREFIX"/var/macports/build/.ccache/ccache.conf && - sudo chown -R macports:admin "$MACPORTS_PREFIX"/var/macports/build/.ccache - -;; -(save) - -sudo rm -fr "$HOME"/.macports-ci-ccache -sudo mkdir -p "$HOME"/.macports-ci-ccache -sudo cp -a "$MACPORTS_PREFIX"/var/macports/build/.ccache/* "$HOME"/.macports-ci-ccache/ - -esac - -CCACHE_DIR="$MACPORTS_PREFIX"/var/macports/build/.ccache/ ccache -s - -;; - -(*) -echo "macports-ci: unknown action $action" - -esac - -) - -# allows setting env var if necessary: -source "$MACPORTS_CI_SOURCEME" diff --git a/etc/ci/macports-ci.ABOUT b/etc/ci/macports-ci.ABOUT deleted file mode 100644 index 60a11f8e..00000000 --- a/etc/ci/macports-ci.ABOUT +++ /dev/null @@ -1,16 +0,0 @@ -about_resource: macports-ci -name: macports-ci -version: c9676e67351a3a519e37437e196cd0ee9c2180b8 -download_url: https://raw.githubusercontent.com/GiovanniBussi/macports-ci/c9676e67351a3a519e37437e196cd0ee9c2180b8/macports-ci -description: Simplify MacPorts setup on Travis-CI -homepage_url: https://github.com/GiovanniBussi/macports-ci -license_expression: mit -copyright: Copyright (c) Giovanni Bussi -attribute: yes -checksum_md5: 5d31d479132502f80acdaed78bed9e23 -checksum_sha1: 74b15643bd1a528d91b4a7c2169c6fc656f549c2 -package_url: pkg:github/giovannibussi/macports-ci@c9676e67351a3a519e37437e196cd0ee9c2180b8#macports-ci -licenses: - - key: mit - name: MIT License - file: mit.LICENSE diff --git a/etc/ci/mit.LICENSE b/etc/ci/mit.LICENSE deleted file mode 100644 index e662c786..00000000 --- a/etc/ci/mit.LICENSE +++ /dev/null @@ -1,5 +0,0 @@ -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/etc/scripts/check_thirdparty.py b/etc/scripts/check_thirdparty.py index 2daded94..a2493617 100644 --- a/etc/scripts/check_thirdparty.py +++ b/etc/scripts/check_thirdparty.py @@ -17,8 +17,7 @@ @click.option( "-d", "--dest", - type=click.Path(exists=True, readable=True, - path_type=str, file_okay=False), + type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), required=True, help="Path to the thirdparty directory to check.", ) diff --git a/etc/scripts/fetch_thirdparty.py b/etc/scripts/fetch_thirdparty.py index 3f9ff527..46cd606d 100644 --- a/etc/scripts/fetch_thirdparty.py +++ b/etc/scripts/fetch_thirdparty.py @@ -55,8 +55,7 @@ "-d", "--dest", "dest_dir", - type=click.Path(exists=True, readable=True, - path_type=str, file_okay=False), + type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), metavar="DIR", default=utils_thirdparty.THIRDPARTY_DIR, show_default=True, @@ -121,7 +120,7 @@ show_default=False, multiple=True, help="Package name(s) that come only in sdist format (no wheels). " - "The command will not fail and exit if no wheel exists for these names", + "The command will not fail and exit if no wheel exists for these names", ) @click.option( "--wheel-only", @@ -132,7 +131,7 @@ show_default=False, multiple=True, help="Package name(s) that come only in wheel format (no sdist). " - "The command will not fail and exit if no sdist exists for these names", + "The command will not fail and exit if no sdist exists for these names", ) @click.option( "--no-dist", @@ -143,7 +142,7 @@ show_default=False, multiple=True, help="Package name(s) that do not come either in wheel or sdist format. " - "The command will not fail and exit if no distribution exists for these names", + "The command will not fail and exit if no distribution exists for these names", ) @click.help_option("-h", "--help") def fetch_thirdparty( @@ -225,8 +224,7 @@ def fetch_thirdparty( environments = None if wheels: evts = itertools.product(python_versions, operating_systems) - environments = [utils_thirdparty.Environment.from_pyver_and_os( - pyv, os) for pyv, os in evts] + environments = [utils_thirdparty.Environment.from_pyver_and_os(pyv, os) for pyv, os in evts] # Collect PyPI repos repos = [] @@ -262,14 +260,11 @@ def fetch_thirdparty( repos=repos, ) if not fetched: - wheels_or_sdist_not_found[f"{name}=={version}"].append( - environment) + wheels_or_sdist_not_found[f"{name}=={version}"].append(environment) if TRACE: print(f" NOT FOUND") - if (sdists or - (f"{name}=={version}" in wheels_or_sdist_not_found and name in sdist_only) - ): + if sdists or (f"{name}=={version}" in wheels_or_sdist_not_found and name in sdist_only): if TRACE: print(f" ==> Fetching sdist: {name}=={version}") @@ -292,8 +287,7 @@ def fetch_thirdparty( sdist_missing = sdists and "sdist" in dists and not name in wheel_only if sdist_missing: mia.append(f"SDist missing: {nv} {dists}") - wheels_missing = wheels and any( - d for d in dists if d != "sdist") and not name in sdist_only + wheels_missing = wheels and any(d for d in dists if d != "sdist") and not name in sdist_only if wheels_missing: mia.append(f"Wheels missing: {nv} {dists}") @@ -303,8 +297,7 @@ def fetch_thirdparty( raise Exception(mia) print(f"==> FETCHING OR CREATING ABOUT AND LICENSE FILES") - utils_thirdparty.fetch_abouts_and_licenses( - dest_dir=dest_dir, use_cached_index=use_cached_index) + utils_thirdparty.fetch_abouts_and_licenses(dest_dir=dest_dir, use_cached_index=use_cached_index) utils_thirdparty.clean_about_files(dest_dir=dest_dir) # check for problems diff --git a/etc/scripts/utils_dejacode.py b/etc/scripts/utils_dejacode.py index 652252d4..b77c9735 100644 --- a/etc/scripts/utils_dejacode.py +++ b/etc/scripts/utils_dejacode.py @@ -33,8 +33,7 @@ def can_do_api_calls(): if not DEJACODE_API_KEY and DEJACODE_API_URL: - print( - "DejaCode DEJACODE_API_KEY and DEJACODE_API_URL not configured. Doing nothing") + print("DejaCode DEJACODE_API_KEY and DEJACODE_API_URL not configured. Doing nothing") return False else: return True @@ -69,8 +68,7 @@ def get_package_data(distribution): return results[0] elif len_results > 1: - print( - f"More than 1 entry exists, review at: {DEJACODE_API_URL_PACKAGES}") + print(f"More than 1 entry exists, review at: {DEJACODE_API_URL_PACKAGES}") else: print("Could not find package:", distribution.download_url) @@ -151,8 +149,7 @@ def find_latest_dejacode_package(distribution): # there was no exact match, find the latest version # TODO: consider the closest version rather than the latest # or the version that has the best data - with_versions = [(packaging_version.parse(p["version"]), p) - for p in packages] + with_versions = [(packaging_version.parse(p["version"]), p) for p in packages] with_versions = sorted(with_versions) latest_version, latest_package_version = sorted(with_versions)[-1] print( diff --git a/etc/scripts/utils_requirements.py b/etc/scripts/utils_requirements.py index 1c502390..e1927493 100644 --- a/etc/scripts/utils_requirements.py +++ b/etc/scripts/utils_requirements.py @@ -102,8 +102,7 @@ def lock_dev_requirements( all_req_nvs = get_required_name_versions(all_req_lines) dev_only_req_nvs = {n: v for n, v in all_req_nvs if n not in main_names} - new_reqs = "\n".join( - f"{n}=={v}" for n, v in sorted(dev_only_req_nvs.items())) + new_reqs = "\n".join(f"{n}=={v}" for n, v in sorted(dev_only_req_nvs.items())) with open(dev_requirements_file, "w") as fo: fo.write(new_reqs) @@ -114,12 +113,10 @@ def get_installed_reqs(site_packages_dir): as a text. """ if not os.path.exists(site_packages_dir): - raise Exception( - f"site_packages directory: {site_packages_dir!r} does not exists") + raise Exception(f"site_packages directory: {site_packages_dir!r} does not exists") # Also include these packages in the output with --all: wheel, distribute, # setuptools, pip - args = ["pip", "freeze", "--exclude-editable", - "--all", "--path", site_packages_dir] + args = ["pip", "freeze", "--exclude-editable", "--all", "--path", site_packages_dir] return subprocess.check_output(args, encoding="utf-8") diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index 47ac5b67..02203e57 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -249,11 +249,9 @@ def download_wheel(name, version, environment, dest_dir=THIRDPARTY_DIR, repos=tu package = repo.get_package_version(name=name, version=version) if not package: if TRACE_DEEP: - print( - f" download_wheel: No package in {repo.index_url} for {name}=={version}") + print(f" download_wheel: No package in {repo.index_url} for {name}=={version}") continue - supported_wheels = list( - package.get_supported_wheels(environment=environment)) + supported_wheels = list(package.get_supported_wheels(environment=environment)) if not supported_wheels: if TRACE_DEEP: print( @@ -297,8 +295,7 @@ def download_sdist(name, version, dest_dir=THIRDPARTY_DIR, repos=tuple()): if not package: if TRACE_DEEP: - print( - f" download_sdist: No package in {repo.index_url} for {name}=={version}") + print(f" download_sdist: No package in {repo.index_url} for {name}=={version}") continue sdist = package.sdist if not sdist: @@ -307,8 +304,7 @@ def download_sdist(name, version, dest_dir=THIRDPARTY_DIR, repos=tuple()): continue if TRACE_DEEP: - print( - f" download_sdist: Getting sdist from index (or cache): {sdist.download_url}") + print(f" download_sdist: Getting sdist from index (or cache): {sdist.download_url}") fetched_sdist_filename = package.sdist.download(dest_dir=dest_dir) if fetched_sdist_filename: @@ -541,8 +537,7 @@ def get_best_download_url(self, repos=tuple()): repos = DEFAULT_PYPI_REPOS for repo in repos: - package = repo.get_package_version( - name=self.name, version=self.version) + package = repo.get_package_version(name=self.name, version=self.version) if not package: if TRACE: print( @@ -781,8 +776,7 @@ def load_remote_about_data(self): if notice_text: about_data["notice_text"] = notice_text except RemoteNotFetchedException: - print( - f"Failed to fetch NOTICE file: {self.notice_download_url}") + print(f"Failed to fetch NOTICE file: {self.notice_download_url}") return self.load_about_data(about_data) def get_checksums(self, dest_dir=THIRDPARTY_DIR): @@ -831,11 +825,9 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR, use_cached_index=False): Fetch license files if missing in `dest_dir`. Return True if license files were fetched. """ - urls = LinksRepository.from_url( - use_cached_index=use_cached_index).links + urls = LinksRepository.from_url(use_cached_index=use_cached_index).links errors = [] - extra_lic_names = [l.get("file") - for l in self.extra_data.get("licenses", {})] + extra_lic_names = [l.get("file") for l in self.extra_data.get("licenses", {})] extra_lic_names += [self.extra_data.get("license_file")] extra_lic_names = [ln for ln in extra_lic_names if ln] lic_names = [f"{key}.LICENSE" for key in self.get_license_keys()] @@ -846,8 +838,7 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR, use_cached_index=False): try: # try remotely first - lic_url = get_license_link_for_filename( - filename=filename, urls=urls) + lic_url = get_license_link_for_filename(filename=filename, urls=urls) fetch_and_save( path_or_url=lic_url, @@ -924,8 +915,7 @@ def load_pkginfo_data(self, dest_dir=THIRDPARTY_DIR): c for c in classifiers if c.startswith("License") ] license_expression = get_license_expression(declared_license) - other_classifiers = [ - c for c in classifiers if not c.startswith("License")] + other_classifiers = [c for c in classifiers if not c.startswith("License")] holder = raw_data["Author"] holder_contact = raw_data["Author-email"] @@ -967,8 +957,7 @@ def update(self, data, overwrite=False, keep_extra=True): package_url = data.get("package_url") if package_url: purl_from_data = packageurl.PackageURL.from_string(package_url) - purl_from_self = packageurl.PackageURL.from_string( - self.package_url) + purl_from_self = packageurl.PackageURL.from_string(self.package_url) if purl_from_data != purl_from_self: print( f"Invalid dist update attempt, no same same purl with dist: " @@ -1018,8 +1007,7 @@ def get_license_link_for_filename(filename, urls): if not path_or_url: raise Exception(f"Missing link to file: {filename}") if not len(path_or_url) == 1: - raise Exception( - f"Multiple links to file: {filename}: \n" + "\n".join(path_or_url)) + raise Exception(f"Multiple links to file: {filename}: \n" + "\n".join(path_or_url)) return path_or_url[0] @@ -1145,7 +1133,6 @@ def to_filename(self): @attr.attributes class Wheel(Distribution): - """ Represents a wheel file. @@ -1413,8 +1400,7 @@ def packages_from_dir(cls, directory): """ base = os.path.abspath(directory) - paths = [os.path.join(base, f) - for f in os.listdir(base) if f.endswith(EXTENSIONS)] + paths = [os.path.join(base, f) for f in os.listdir(base) if f.endswith(EXTENSIONS)] if TRACE_ULTRA_DEEP: print("packages_from_dir: paths:", paths) @@ -1475,8 +1461,7 @@ def dists_from_paths_or_urls(cls, paths_or_urls): dists = [] if TRACE_ULTRA_DEEP: print(" ###paths_or_urls:", paths_or_urls) - installable = [f for f in paths_or_urls if f.endswith( - EXTENSIONS_INSTALLABLE)] + installable = [f for f in paths_or_urls if f.endswith(EXTENSIONS_INSTALLABLE)] for path_or_url in installable: try: dist = Distribution.from_path_or_url(path_or_url) @@ -1494,8 +1479,7 @@ def dists_from_paths_or_urls(cls, paths_or_urls): ) except InvalidDistributionFilename: if TRACE_DEEP: - print( - f" Skipping invalid distribution from: {path_or_url}") + print(f" Skipping invalid distribution from: {path_or_url}") continue return dists @@ -1544,8 +1528,7 @@ class Environment: implementation = attr.ib( type=str, default="cp", - metadata=dict( - help="Python implementation supported by this environment."), + metadata=dict(help="Python implementation supported by this environment."), repr=False, ) @@ -1559,8 +1542,7 @@ class Environment: platforms = attr.ib( type=list, default=attr.Factory(list), - metadata=dict( - help="List of platform tags supported by this environment."), + metadata=dict(help="List of platform tags supported by this environment."), repr=False, ) @@ -1644,8 +1626,7 @@ class PypiSimpleRepository: fetched_package_normalized_names = attr.ib( type=set, default=attr.Factory(set), - metadata=dict( - help="A set of already fetched package normalized names."), + metadata=dict(help="A set of already fetched package normalized names."), ) use_cached_index = attr.ib( @@ -1676,12 +1657,10 @@ def _get_package_versions_map(self, name): self.packages[normalized_name] = versions except RemoteNotFetchedException as e: if TRACE: - print( - f"failed to fetch package name: {name} from: {self.index_url}:\n{e}") + print(f"failed to fetch package name: {name} from: {self.index_url}:\n{e}") if not versions and TRACE: - print( - f"WARNING: package {name} not found in repo: {self.index_url}") + print(f"WARNING: package {name} not found in repo: {self.index_url}") return versions @@ -1866,8 +1845,7 @@ def get(self, path_or_url, as_text=True, force=False): if force or not os.path.exists(cached): if TRACE_DEEP: print(f" FILE CACHE MISS: {path_or_url}") - content = get_file_content( - path_or_url=path_or_url, as_text=as_text) + content = get_file_content(path_or_url=path_or_url, as_text=as_text) wmode = "w" if as_text else "wb" with open(cached, wmode) as fo: fo.write(content) @@ -1889,8 +1867,7 @@ def get_file_content(path_or_url, as_text=True): if path_or_url.startswith("https://"): if TRACE_DEEP: print(f"Fetching: {path_or_url}") - _headers, content = get_remote_file_content( - url=path_or_url, as_text=as_text) + _headers, content = get_remote_file_content(url=path_or_url, as_text=as_text) return content elif path_or_url.startswith("file://") or ( @@ -1956,8 +1933,7 @@ def get_remote_file_content( ) else: - raise RemoteNotFetchedException( - f"Failed HTTP request from {url} with {status}") + raise RemoteNotFetchedException(f"Failed HTTP request from {url} with {status}") if headers_only: return response.headers, None @@ -2048,8 +2024,7 @@ def get_other_dists(_package, _dist): # if has key data we may look to improve later, but we can move on if local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir=dest_dir) - local_dist.fetch_license_files( - dest_dir=dest_dir, use_cached_index=use_cached_index) + local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index) continue # lets try to get from another dist of the same local package @@ -2061,8 +2036,7 @@ def get_other_dists(_package, _dist): # if has key data we may look to improve later, but we can move on if local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir=dest_dir) - local_dist.fetch_license_files( - dest_dir=dest_dir, use_cached_index=use_cached_index) + local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index) continue # try to get another version of the same package that is not our version @@ -2073,8 +2047,7 @@ def get_other_dists(_package, _dist): ] other_local_version = other_local_packages and other_local_packages[-1] if other_local_version: - latest_local_dists = list( - other_local_version.get_distributions()) + latest_local_dists = list(other_local_version.get_distributions()) for latest_local_dist in latest_local_dists: latest_local_dist.load_about_data(dest_dir=dest_dir) if not latest_local_dist.has_key_metadata(): @@ -2100,8 +2073,7 @@ def get_other_dists(_package, _dist): # if has key data we may look to improve later, but we can move on if local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir=dest_dir) - local_dist.fetch_license_files( - dest_dir=dest_dir, use_cached_index=use_cached_index) + local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index) continue # try to get a latest version of the same package that is not our version @@ -2142,8 +2114,7 @@ def get_other_dists(_package, _dist): # if local_dist.has_key_metadata() or not local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir) - lic_errs = local_dist.fetch_license_files( - dest_dir, use_cached_index=use_cached_index) + lic_errs = local_dist.fetch_license_files(dest_dir, use_cached_index=use_cached_index) if not local_dist.has_key_metadata(): print(f"Unable to add essential ABOUT data for: {local_dist}") @@ -2291,8 +2262,7 @@ def find_problems( for dist in package.get_distributions(): dist.load_about_data(dest_dir=dest_dir) - abpth = os.path.abspath(os.path.join( - dest_dir, dist.about_filename)) + abpth = os.path.abspath(os.path.join(dest_dir, dist.about_filename)) if not dist.has_key_metadata(): print(f" Missing key ABOUT data in file://{abpth}") if "classifiers" in dist.extra_data: diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index 5a2b6959..00000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,24 +0,0 @@ -aboutcode-toolkit==7.0.2 -bleach==4.1.0 -build==0.7.0 -commonmark==0.9.1 -docutils==0.18.1 -et-xmlfile==1.1.0 -execnet==1.9.0 -iniconfig==1.1.1 -jeepney==0.7.1 -keyring==23.4.1 -openpyxl==3.0.9 -pep517==0.12.0 -pkginfo==1.8.2 -py==1.11.0 -pytest==7.0.1 -pytest-forked==1.4.0 -pytest-xdist==2.5.0 -readme-renderer==34.0 -requests-toolbelt==0.9.1 -rfc3986==1.5.0 -rich==12.3.0 -secretstorage==3.3.2 -tomli==1.2.3 -twine==3.8.0 diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 092117eb..00000000 --- a/requirements.txt +++ /dev/null @@ -1,81 +0,0 @@ -attrs==21.4.0 -banal==1.0.6 -beautifulsoup4==4.11.1 -binaryornot==0.4.4 -boolean.py==3.8 -certifi==2024.7.4 -cffi==1.15.0 -chardet==4.0.0 -charset-normalizer==2.0.12 -click==8.0.4 -colorama==0.4.4 -commoncode==30.2.0 -construct==2.10.68 -container-inspector==31.0.0 -cryptography==44.0.1 -python-dateutil==2.8.2 -debian-inspector==30.0.0 -dockerfile-parse==1.2.0 -dparse2==0.6.1 -extractcode==31.0.0 -extractcode-7z==16.5.210531 -extractcode-libarchive==3.5.1.210531 -fasteners==0.17.3 -fingerprints==1.0.3 -ftfy==6.0.3 -future==0.18.3 -gemfileparser==0.8.0 -html5lib==1.1 -idna==3.7 -importlib-metadata==4.8.3 -inflection==0.5.1 -intbitset==3.1.0 -isodate==0.6.1 -jaraco.functools==3.4.0 -javaproperties==0.8.1 -Jinja2==3.1.6 -jsonstreams==0.6.0 -license-expression==21.6.14 -lxml==4.9.1 -MarkupSafe==2.0.1 -more-itertools==8.13.0 -normality==2.3.3 -packagedcode-msitools==0.101.210706 -packageurl-python==0.17.4 -packaging==24.0 -parameter-expansion-patched==0.3.1 -patch==1.16 -pdfminer-six==20220506 -pefile==2021.9.3 -pip-requirements-parser==31.2.0 -pkginfo2==30.0.0 -pluggy==1.0.0 -plugincode==30.0.0 -ply==3.11 -publicsuffix2==2.20191221 -pyahocorasick==2.0.0b1 -pycparser==2.21 -pygmars==0.7.0 -Pygments==2.15.0 -pymaven-patch==0.3.0 -pyparsing==3.0.8 -pytz==2022.1 -PyYAML==6.0 -rdflib==5.0.0 -regipy==2.3.1 -requests==2.32.4 -rpm-inspector-rpm==4.16.1.3.210404 -saneyaml==0.5.2 -six==1.16.0 -soupsieve==2.3.1 -spdx-tools==0.7.0a3 -text-unidecode==1.3 -toml==0.10.2 -typecode==30.0.0 -typecode-libmagic==5.39.210531 -urllib3==1.26.19 -urlpy==0.5 -wcwidth==0.2.5 -webencodings==0.5.1 -xmltodict==0.12.0 -zipp==3.19.1 diff --git a/setup.cfg b/setup.cfg index fc6a3855..59d1a8d9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -67,7 +67,7 @@ where = src [options.extras_require] testing = - pytest >= 6, != 7.0.0 + pytest >= 8.0.0 pytest-xdist >= 2 aboutcode-toolkit >= 7.0.2 pycodestyle >= 2.8.0 diff --git a/src/fetchcode/composer.py b/src/fetchcode/composer.py index 3188d000..8de333f0 100644 --- a/src/fetchcode/composer.py +++ b/src/fetchcode/composer.py @@ -46,11 +46,9 @@ def get_download_url(cls, purl): return for package in data["packages"][name]: - if ( - package.get("version") == purl.version - or package.get("version") == f"v{purl.version}" - or package.get("version_normalized") == purl.version - or package.get("version_normalized") == f"v{purl.version}" + version = purl.version + if any( + package.get(field) in (version, f"v{version}") + for field in ("version", "version_normalized") ): - download_url = package["dist"].get("url") - return download_url + return package["dist"].get("url") diff --git a/src/fetchcode/package.py b/src/fetchcode/package.py index e4140d02..1b0f18d4 100644 --- a/src/fetchcode/package.py +++ b/src/fetchcode/package.py @@ -388,7 +388,8 @@ def get_cocoapods_data_from_purl(purl): gh_repo_owner = None gh_repo_name = name - podspec_api_url = f"https://raw.githubusercontent.com/CocoaPods/Specs/master/Specs/{hashed_path}/{name}/{tag}/{name}.podspec.json" + base_url = "https://raw.githubusercontent.com/CocoaPods/Specs/master/Specs" + podspec_api_url = f"{base_url}/{hashed_path}/{name}/{tag}/{name}.podspec.json" podspec_api_response = get_response(podspec_api_url) podspec_homepage = podspec_api_response.get("homepage") diff --git a/src/fetchcode/package_util.py b/src/fetchcode/package_util.py index aedfd427..5284b85f 100644 --- a/src/fetchcode/package_util.py +++ b/src/fetchcode/package_util.py @@ -303,7 +303,7 @@ def get_cocoapod_tags(spec, name): data_list.pop(0) return data_list return None - except: + except: # noqa: E722 return None @@ -338,7 +338,8 @@ def construct_cocoapods_package( bug_tracking_url = f"{github_url}/{gh_repo_owner}/{gh_repo_name}/issues" code_view_url = f"{github_url}/{gh_repo_owner}/{gh_repo_name}" - podspec_api_url = f"https://raw.githubusercontent.com/CocoaPods/Specs/master/Specs/{hashed_path}/{name}/{tag}/{name}.podspec.json" + base_url = "https://raw.githubusercontent.com/CocoaPods/Specs/master/Specs" + podspec_api_url = f"{base_url}/{hashed_path}/{name}/{tag}/{name}.podspec.json" podspec_api_response = utils.get_response(podspec_api_url) homepage_url = podspec_api_response.get("homepage") diff --git a/src/fetchcode/utils.py b/src/fetchcode/utils.py index 43345d9c..13f14217 100644 --- a/src/fetchcode/utils.py +++ b/src/fetchcode/utils.py @@ -189,7 +189,7 @@ def make_head_request(url, headers=None): try: resp = requests.head(url, headers=headers) return resp - except: + except requests.RequestException: raise Exception(f"Failed to fetch: {url}")