diff --git a/README.rst b/README.rst index 59c963c8b..0599a238f 100644 --- a/README.rst +++ b/README.rst @@ -5,6 +5,17 @@ .. image:: https://governance.openstack.org/tc/badges/requirements.svg :target: https://governance.openstack.org/tc/reference/tags/index.html +Security Warning +================ + +OpenStack makes no security guarantees about third-party +dependencies listed here, and does not keep track of any +vulnerabilities they contain. Versions of these dependencies are +frozen at each coordinated release in order to stabilize upstream +testing, and can contain known vulnerabilities. Consumers are +*STRONGLY* encouraged to rely on curated distributions of OpenStack +or manage security patching of dependencies themselves. + Resources and Documentation =========================== diff --git a/global-requirements.txt b/global-requirements.txt index ba16a77b5..5c7080c6b 100644 --- a/global-requirements.txt +++ b/global-requirements.txt @@ -1,3 +1,11 @@ +### WARNING: OpenStack makes no security guarantees about third-party +### dependencies listed here, and does not keep track of any +### vulnerabilities they contain. Versions of these dependencies are +### frozen at each coordinated release in order to stabilize upstream +### testing, and can contain known vulnerabilities. Consumers are +### *STRONGLY* encouraged to rely on curated distributions of OpenStack +### or manage security patching of dependencies themselves. + ## section:general aiomysql # MIT License @@ -72,7 +80,6 @@ influxdb!=5.3.0 # MIT influxdb-client # MIT infoblox-client # Apache-2.0 iso8601 # MIT -jaeger-client # Apache-2.0 Jinja2 # BSD License (3 clause) jira # BSD License (2 clause) jmespath # MIT @@ -81,7 +88,6 @@ jsonschema # MIT kazoo # Apache-2.0 kombu!=4.0.2 # BSD kubernetes # Apache-2.0 -ldap3 # LGPLv3 libsass # MIT libvirt-python!=4.1.0,!=4.2.0 # LGPLv2+ lxml!=3.7.0 # BSD @@ -137,7 +143,6 @@ python-ldap # PSF python-memcached # PSF pytz # MIT pyudev # LGPLv2.1+ -pywinrm # MIT PyYAML # MIT pyzabbix # LGPL qrcode # BSD @@ -277,6 +282,7 @@ types-simplejson # Apache-2.0 typing # PSF typing-extensions # PSF tzdata # MIT +valkey # MIT virtualbmc # Apache-2.0 virtualenv!=16.3.0 # MIT WebTest # MIT @@ -329,6 +335,7 @@ ironic-lib!=4.6.0 # Apache-2.0 keystoneauth1 # Apache-2.0 keystonemiddleware # Apache-2.0 kuryr-lib # Apache-2.0 +lark # MIT metalsmith # Apache-2.0 microversion-parse # Apache-2.0 mistral-lib # Apache-2.0 @@ -384,7 +391,7 @@ pbr!=2.1.0 # Apache-2.0 sherlock # MIT stevedore!=3.0.0 # Apache-2.0 tap-as-a-service # Apache-2.0 -taskflow # Apache-2.0 +taskflow!=6.0.0 # Apache-2.0 tempest # Apache-2.0 tooz # Apache-2.0 tosca-parser # Apache-2.0 diff --git a/openstack_requirements/cmds/build_lower_constraints.py b/openstack_requirements/cmds/build_lower_constraints.py deleted file mode 100644 index 324660404..000000000 --- a/openstack_requirements/cmds/build_lower_constraints.py +++ /dev/null @@ -1,68 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Merge multiple lower-constraints.txt files to find the highest values. - -""" - -import argparse -import collections - -from openstack_requirements.utils import read_requirements_file - - -import packaging.specifiers -import packaging.version - - -def get_requirements_version(req): - """Find the version for a requirement. - - Use the version attached to >=, ==, or ===, depending on the type - of input requirement. - - """ - for specifier in packaging.specifiers.SpecifierSet(req.specifiers): - if '>=' in specifier.operator or '==' in specifier.operator: - return packaging.version.parse(specifier.version) - raise ValueError('could not find version for {}'.format(req)) - - -def merge_constraints_sets(constraints_sets): - "Generator of Requirements with the maximum version for each constraint." - all_constraints = collections.defaultdict(list) - for constraints_set in constraints_sets: - for constraint_name, constraint in constraints_set.items(): - if constraint_name: - all_constraints[constraint_name].extend(constraint) - for constraint_name, constraints in sorted(all_constraints.items()): - val = max((c[0] for c in constraints), key=get_requirements_version) - yield val.to_line() - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument( - 'lower_constraints', - nargs='+', - help='lower-constraints.txt files', - ) - args = parser.parse_args() - - constraints_sets = [ - read_requirements_file(filename) - for filename in args.lower_constraints - ] - - merged = list(merge_constraints_sets(constraints_sets)) - print(''.join(merged)) diff --git a/openstack_requirements/cmds/check_py2.py b/openstack_requirements/cmds/check_py2.py deleted file mode 100755 index 09148547b..000000000 --- a/openstack_requirements/cmds/check_py2.py +++ /dev/null @@ -1,77 +0,0 @@ -#! /usr/bin/env python - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import argparse - -import pkg_resources -import requests - - -_url_template = 'https://pypi.org/project/{dist}/{version}/json' - - -def _get_metadata(dist, version): - try: - url = _url_template.format(dist=dist, version=version) - response = requests.get(url) - return response.json() - except ValueError: - return {} - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument( - '--verbose', '-v', - default=False, - action='store_true', - help='turn on noisy output', - ) - parser.add_argument( - '--requirements', - default='upper-constraints.txt', - help='the list of constrained requirements to check', - ) - args = parser.parse_args() - - for line in open(args.requirements, 'r'): - try: - req = pkg_resources.Requirement.parse(line) - except ValueError: - # Assume this is a comment and skip it. - continue - # req.specifier is a set so we can't get an item out of it - # directly. Turn it into a list and take the first (and only) - # value. That gives us an _IndividualSpecifier which has a - # version attribute that is not smart enough to filter out the - # selector value for things like python version, so drop - # anything after the first semicolon. - version = list(req.specifier)[0].version.split(';')[0] - data = _get_metadata(req.project_name, version) - classifiers = data.get('info', {}).get('classifiers', []) - for classifier in classifiers: - if classifier.startswith('Programming Language :: Python :: 2'): - if args.verbose: - print('{}==={} {!r}'.format( - req.project_name, version, classifier)) - break - else: - print('\nNo "Python :: 2" classifier found for {}==={}'.format( - req.project_name, version)) - for classifier in classifiers: - print(' {}'.format(classifier)) - - -if __name__ == '__main__': - main() diff --git a/openstack_requirements/cmds/edit_constraint.py b/openstack_requirements/cmds/edit_constraint.py index 68f619bb6..1db62f807 100644 --- a/openstack_requirements/cmds/edit_constraint.py +++ b/openstack_requirements/cmds/edit_constraint.py @@ -68,7 +68,7 @@ def main(argv=None, stdout=None): content = open(args[0], 'rt').read() reqs = requirement.parse(content, permit_urls=True) out_reqs = edit(reqs, args[1], args[2]) - out = requirement.to_content(out_reqs, prefix=False) + out = requirement.to_content(out_reqs) with open(args[0] + '.tmp', 'wt') as f: f.write(out) if os.path.exists(args[0]): diff --git a/openstack_requirements/cmds/generate.py b/openstack_requirements/cmds/generate.py index e80d8888a..43090832b 100644 --- a/openstack_requirements/cmds/generate.py +++ b/openstack_requirements/cmds/generate.py @@ -26,6 +26,17 @@ from openstack_requirements import requirement +SECURITY_WARNING = [ + "# WARNING: OpenStack makes no security guarantees about third-party", + "# dependencies listed here, and does not keep track of any", + "# vulnerabilities they contain. Versions of these dependencies are", + "# frozen at each coordinated release in order to stabilize upstream", + "# testing, and can contain known vulnerabilities. Consumers are", + "# *STRONGLY* encouraged to rely on curated distributions of OpenStack", + "# or manage security patching of dependencies themselves.", + ] + + def _parse_freeze(text): """Parse a freeze into structured data. @@ -257,5 +268,5 @@ def main(argv=None, stdout=None): denylist = _parse_denylist(options.denylist) frozen = [ *sorted(_combine_freezes(freezes, denylist), key=_make_sort_key)] - stdout.writelines(frozen) + stdout.writelines(SECURITY_WARNING + frozen) stdout.flush() diff --git a/openstack_requirements/project.py b/openstack_requirements/project.py index f3ac9f510..8fdaec34b 100644 --- a/openstack_requirements/project.py +++ b/openstack_requirements/project.py @@ -15,43 +15,11 @@ """The project abstraction.""" -import collections import configparser import errno import io import os -from parsley import makeGrammar - -from openstack_requirements import requirement - -# PURE logic from here until the IO marker below. - - -_Comment = collections.namedtuple('Comment', ['line']) -_Extra = collections.namedtuple('Extra', ['name', 'content']) - - -_extras_grammar = """ -ini = (line*:p extras?:e line*:l final:s) -> (''.join(p), e, ''.join(l+[s])) -line = ~extras <(~'\\n' anything)* '\\n'> -final = <(~'\\n' anything)* > -extras = '[' 'e' 'x' 't' 'r' 'a' 's' ']' '\\n'+ body*:b -> b -body = comment | extra -comment = <'#' (~'\\n' anything)* '\\n'>:c '\\n'* -> comment(c) -extra = name:n ' '* '=' line:l cont*:c '\\n'* -> extra(n, ''.join([l] + c)) -name = <(anything:x ?(x not in '\\n \\t='))+> -cont = ' '+ <(~'\\n' anything)* '\\n'> -""" -_extras_compiled = makeGrammar( - _extras_grammar, {"comment": _Comment, "extra": _Extra}) - - -Error = collections.namedtuple('Error', ['message']) -File = collections.namedtuple('File', ['filename', 'content']) -StdOut = collections.namedtuple('StdOut', ['message']) -Verbose = collections.namedtuple('Verbose', ['message']) - def extras(project): """Return a dict of extra-name:content for the extras in setup.cfg.""" @@ -64,41 +32,6 @@ def extras(project): return dict(c.items('extras')) -def merge_setup_cfg(old_content, new_extras): - # This is ugly. All the existing libraries handle setup.cfg's poorly. - prefix, extras, suffix = _extras_compiled(old_content).ini() - out_extras = [] - if extras is not None: - for extra in extras: - if type(extra) is _Comment: - out_extras.append(extra) - elif type(extra) is _Extra: - if extra.name not in new_extras: - out_extras.append(extra) - continue - e = _Extra( - extra.name, - requirement.to_content( - new_extras[extra.name], ':', ' ', False)) - out_extras.append(e) - else: - raise TypeError('unknown type %r' % extra) - if out_extras: - extras_str = ['[extras]\n'] - for extra in out_extras: - if type(extra) is _Comment: - extras_str.append(extra.line) - else: - extras_str.append(extra.name + ' =') - extras_str.append(extra.content) - if suffix: - extras_str.append('\n') - extras_str = ''.join(extras_str) - else: - extras_str = '' - return prefix + extras_str + suffix - - # IO from here to the end of the file. def _safe_read(project, filename, output=None): @@ -143,47 +76,3 @@ def read(root): result['lower-constraints.txt'] = None _safe_read(result, 'lower-constraints.txt') return result - - -def write(project, actions, stdout, verbose, noop=False): - """Write actions into project. - - :param project: A project metadata dict. - :param actions: A list of action tuples - File or Verbose - that describe - what actions are to be taken. - Error objects write a message to stdout and trigger an exception at - the end of _write_project. - File objects describe a file to have content placed in it. - StdOut objects describe a message to write to stdout. - Verbose objects will write a message to stdout when verbose is True. - :param stdout: Where to write content for stdout. - :param verbose: If True Verbose actions will be written to stdout. - :param noop: If True nothing will be written to disk. - :return None: - :raises IOError: If the IO operations fail, IOError is raised. If this - happens some actions may have been applied and others not. - """ - error = False - for action in actions: - if type(action) is Error: - error = True - stdout.write(action.message + '\n') - elif type(action) is File: - if noop: - continue - fullname = os.path.join(project['root'], action.filename) - tmpname = fullname + '.tmp' - with open(tmpname, 'wt') as f: - f.write(action.content) - if os.path.exists(fullname): - os.remove(fullname) - os.rename(tmpname, fullname) - elif type(action) is StdOut: - stdout.write(action.message) - elif type(action) is Verbose: - if verbose: - stdout.write(u"%s\n" % (action.message,)) - else: - raise Exception("Invalid action %r" % (action,)) - if error: - raise Exception("Error occurred processing %s" % (project['root'])) diff --git a/openstack_requirements/requirement.py b/openstack_requirements/requirement.py index 287d063f5..dd6aaac36 100644 --- a/openstack_requirements/requirement.py +++ b/openstack_requirements/requirement.py @@ -21,17 +21,6 @@ import re -# A header for the requirements file(s). -# TODO(lifeless): Remove this once constraints are in use. -_REQS_HEADER = [ - '# The order of packages is significant, because pip processes ' - 'them in the order\n', - '# of appearance. Changing the order has an impact on the overall ' - 'integration\n', - '# process, which may cause wedges in the gate later.\n', -] - - def key_specifier(a): weight = {'>=': 0, '>': 0, '===': 1, '==': 1, '~=': 1, '!=': 1, @@ -160,10 +149,8 @@ def parse_line(req_line, permit_urls=False): return Requirement(name, location, specifier, markers, comment, extras) -def to_content(reqs, marker_sep=';', line_prefix='', prefix=True): +def to_content(reqs, marker_sep=';', line_prefix=''): lines = [] - if prefix: - lines += _REQS_HEADER for req in reqs.reqs: lines.append(req.to_line(marker_sep, line_prefix)) return u''.join(lines) diff --git a/openstack_requirements/tests/test_build_lower_constraints.py b/openstack_requirements/tests/test_build_lower_constraints.py deleted file mode 100644 index 5f820f488..000000000 --- a/openstack_requirements/tests/test_build_lower_constraints.py +++ /dev/null @@ -1,69 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import testtools - -from openstack_requirements.cmds import build_lower_constraints -from openstack_requirements import requirement - - -class BuildLowerConstraintsTest(testtools.TestCase): - - def test_one_input_file(self): - inputs = [ - requirement.parse('package==1.2.3'), - ] - expected = [ - 'package==1.2.3\n', - ] - self.assertEqual( - expected, - list(build_lower_constraints.merge_constraints_sets(inputs)) - ) - - def test_two_input_file_same(self): - inputs = [ - requirement.parse('package==1.2.3'), - requirement.parse('package==1.2.3'), - ] - expected = [ - 'package==1.2.3\n', - ] - self.assertEqual( - expected, - list(build_lower_constraints.merge_constraints_sets(inputs)) - ) - - def test_two_input_file_differ(self): - inputs = [ - requirement.parse('package==1.2.3'), - requirement.parse('package==4.5.6'), - ] - expected = [ - 'package==4.5.6\n', - ] - self.assertEqual( - expected, - list(build_lower_constraints.merge_constraints_sets(inputs)) - ) - - def test_one_input_file_with_comments(self): - inputs = [ - requirement.parse('package==1.2.3\n # package2==0.9.8'), - ] - expected = [ - 'package==1.2.3\n', - ] - self.assertEqual( - expected, - list(build_lower_constraints.merge_constraints_sets(inputs)) - ) diff --git a/openstack_requirements/tests/test_project.py b/openstack_requirements/tests/test_project.py index 204f16066..fb046d0e1 100644 --- a/openstack_requirements/tests/test_project.py +++ b/openstack_requirements/tests/test_project.py @@ -10,17 +10,14 @@ # License for the specific language governing permissions and limitations # under the License. -import io import textwrap import fixtures -import parsley import testscenarios import testtools from testtools import matchers from openstack_requirements import project -from openstack_requirements import requirement from openstack_requirements.tests import common @@ -73,211 +70,3 @@ def test_none(self): def test_no_setup_cfg(self): proj = {} self.assertEqual({}, project.extras(proj)) - - -class TestExtrasParsing(testtools.TestCase): - - def test_none(self): - old_content = textwrap.dedent(u""" - [metadata] - # something something - name = fred - - [entry_points] - console_scripts = - foo = bar:quux - """) - ini = project._extras_compiled(old_content).ini() - self.assertEqual(ini, (old_content, None, '')) - - def test_no_eol(self): - old_content = textwrap.dedent(u""" - [metadata] - # something something - name = fred - - [entry_points] - console_scripts = - foo = bar:quux""") - expected1 = textwrap.dedent(u""" - [metadata] - # something something - name = fred - - [entry_points] - console_scripts = - """) - suffix = ' foo = bar:quux' - ini = project._extras_compiled(old_content).ini() - self.assertEqual(ini, (expected1, None, suffix)) - - def test_two_extras_raises(self): - old_content = textwrap.dedent(u""" - [metadata] - # something something - name = fred - - [extras] - a = b - [extras] - b = c - - [entry_points] - console_scripts = - foo = bar:quux - """) - with testtools.ExpectedException(parsley.ParseError): - project._extras_compiled(old_content).ini() - - def test_extras(self): - # We get an AST for extras we can use to preserve comments. - old_content = textwrap.dedent(u""" - [metadata] - # something something - name = fred - - [extras] - # comment1 - a = - b - c - # comment2 - # comment3 - d = - e - # comment4 - - [entry_points] - console_scripts = - foo = bar:quux - """) - prefix = textwrap.dedent(u""" - [metadata] - # something something - name = fred - - """) - suffix = textwrap.dedent(u"""\ - [entry_points] - console_scripts = - foo = bar:quux - """) - extras = [ - project._Comment('# comment1\n'), - project._Extra('a', '\nb\nc\n'), - project._Comment('# comment2\n'), - project._Comment('# comment3\n'), - project._Extra('d', '\ne\n'), - project._Comment('# comment4\n')] - ini = project._extras_compiled(old_content).ini() - self.assertEqual(ini, (prefix, extras, suffix)) - - -class TestMergeSetupCfg(testtools.TestCase): - - def test_merge_none(self): - old_content = textwrap.dedent(u""" - [metadata] - # something something - name = fred - - [entry_points] - console_scripts = - foo = bar:quux - """) - merged = project.merge_setup_cfg(old_content, {}) - self.assertEqual(old_content, merged) - - def test_merge_extras(self): - old_content = textwrap.dedent(u""" - [metadata] - name = fred - - [extras] - # Comment - a = - b - # comment - c = - d - - [entry_points] - console_scripts = - foo = bar:quux - """) - blank = requirement.Requirement('', '', '', '', '') - r1 = requirement.Requirement( - 'b', '', '>=1', "python_version=='2.7'", '') - r2 = requirement.Requirement('d', '', '', '', '# BSD') - reqs = { - 'a': requirement.Requirements([blank, r1]), - 'c': requirement.Requirements([blank, r2])} - merged = project.merge_setup_cfg(old_content, reqs) - expected = textwrap.dedent(u""" - [metadata] - name = fred - - [extras] - # Comment - a = - b>=1:python_version=='2.7' - # comment - c = - d # BSD - - [entry_points] - console_scripts = - foo = bar:quux - """) - self.assertEqual(expected, merged) - - -class TestWriteProject(testtools.TestCase): - - def test_smoke(self): - stdout = io.StringIO() - root = self.useFixture(fixtures.TempDir()).path - proj = {'root': root} - actions = [ - project.File('foo', '123\n'), - project.File('bar', '456\n'), - project.Verbose(u'fred')] - project.write(proj, actions, stdout, True) - foo = open(root + '/foo', 'rt').read() - self.expectThat(foo, matchers.Equals('123\n')) - bar = open(root + '/bar', 'rt').read() - self.expectThat(bar, matchers.Equals('456\n')) - self.expectThat(stdout.getvalue(), matchers.Equals('fred\n')) - - def test_non_verbose(self): - stdout = io.StringIO() - root = self.useFixture(fixtures.TempDir()).path - proj = {'root': root} - actions = [project.Verbose(u'fred')] - project.write(proj, actions, stdout, False) - self.expectThat(stdout.getvalue(), matchers.Equals('')) - - def test_bad_action(self): - root = self.useFixture(fixtures.TempDir()).path - stdout = io.StringIO() - proj = {'root': root} - actions = [('foo', 'bar')] - with testtools.ExpectedException(Exception): - project.write(proj, actions, stdout, True) - - def test_stdout(self): - stdout = io.StringIO() - root = self.useFixture(fixtures.TempDir()).path - proj = {'root': root} - actions = [project.StdOut(u'fred\n')] - project.write(proj, actions, stdout, True) - self.expectThat(stdout.getvalue(), matchers.Equals('fred\n')) - - def test_errors(self): - stdout = io.StringIO() - root = self.useFixture(fixtures.TempDir()).path - proj = {'root': root} - actions = [project.Error(u'fred')] - with testtools.ExpectedException(Exception): - project.write(proj, actions, stdout, True) - self.expectThat(stdout.getvalue(), matchers.Equals('fred\n')) diff --git a/openstack_requirements/tests/test_requirement.py b/openstack_requirements/tests/test_requirement.py index 97c5130c2..f0a4ed5da 100644 --- a/openstack_requirements/tests/test_requirement.py +++ b/openstack_requirements/tests/test_requirement.py @@ -116,8 +116,7 @@ def test_smoke(self): 'foo', '', '<=1', "python_version=='2.7'", '# BSD')]), marker_sep='!') self.assertEqual( - ''.join(requirement._REQS_HEADER - + ["foo<=1!python_version=='2.7' # BSD\n"]), + "foo<=1!python_version=='2.7' # BSD\n", reqs) def test_location(self): @@ -125,8 +124,7 @@ def test_location(self): [requirement.Requirement( 'foo', 'file://foo', '', "python_version=='2.7'", '# BSD')])) self.assertEqual( - ''.join(requirement._REQS_HEADER - + ["file://foo#egg=foo;python_version=='2.7' # BSD\n"]), + "file://foo#egg=foo;python_version=='2.7' # BSD\n", reqs) diff --git a/requirements.txt b/requirements.txt index ddd20cc3e..8984054ca 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,4 @@ fixtures>=3.0.0 # Apache-2.0/BSD -Parsley>=1.2 # MIT packaging!=20.5,!=20.6,!=20.7,>=16.5 # Apache-2.0 requests>=2.14.2 # Apache-2.0 PyYAML>=3.12 # MIT diff --git a/setup.cfg b/setup.cfg index 975d1e9e9..af56b935d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -33,6 +33,4 @@ console_scripts = validate-constraints = openstack_requirements.cmds.validate:main validate-projects = openstack_requirements.cmds.validate_projects:main normalize-requirements = openstack_requirements.cmds.normalize_requirements:main - check-python2-support = openstack_requirements.cmds.check_py2:main check-constraints = openstack_requirements.cmds.check_exists:main - build-lower-constraints = openstack_requirements.cmds.build_lower_constraints:main diff --git a/test-requirements.txt b/test-requirements.txt index bc9ec5a31..69ff89422 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -4,4 +4,3 @@ stestr>=1.0.0 # Apache-2.0 testscenarios>=0.4 # Apache-2.0/BSD testtools>=2.2.0 # MIT virtualenv>=14.0.6 # MIT -bashate>=0.5.1 # Apache-2.0 diff --git a/tools/fix-lower-constraints.py b/tools/fix-lower-constraints.py deleted file mode 100755 index 7301b6863..000000000 --- a/tools/fix-lower-constraints.py +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env python -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -""" -Instructions: - - 1. virtualenv venv - 2. source venv/bin/activate - 3. pip install /path/to/local/copy/of/requirements/repository - 4. cd /path/to/project/to/fix - 5. .../requirements/tools/fix-lower-constraints.py > new-lc.txt - 6. mv new-lc.txt lower-constraints.txt - 7. Update the patch and resubmit it to gerrit. -""" - -import io - -from openstack_requirements import requirement - - -def read_file(name): - with io.open(name, 'r', encoding='utf-8') as f: - return requirement.parse(f.read()) - - -requirements = read_file('requirements.txt') -requirements.update(read_file('test-requirements.txt')) -constraints = read_file('lower-constraints.txt') - -output = [] - -for const in constraints.values(): - const = const[0][0] - actual = const.specifiers.lstrip('=') - name = const.package.lower() - if name not in requirements: - # Ignore secondary dependencies - output.append(const.to_line()) - continue - for req, _ in requirements[name]: - min = [ - s - for s in req.specifiers.split(',') - if '>' in s - ] - if not min: - # If there is no lower bound, assume the constraint is - # right. - output.append(const.to_line()) - continue - required = min[0].lstrip('>=') - if required != actual: - output.append('{}=={}\n'.format( - const.package, required)) - else: - output.append(const.to_line()) - -for line in sorted(output, key=lambda x: x.lower()): - if not line.strip(): - continue - print(line.rstrip()) diff --git a/tox.ini b/tox.ini index 404bf9e84..d0a0fc3c9 100644 --- a/tox.ini +++ b/tox.ini @@ -1,14 +1,13 @@ [tox] minversion = 4.11.0 envlist = validate,py3,pep8,pip-install -ignore_basepython_conflict=true [testenv] -basepython = python3 -usedevelop = True -deps = -c{toxinidir}/upper-constraints.txt - -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt +usedevelop = true +deps = + -c{toxinidir}/upper-constraints.txt + -r{toxinidir}/requirements.txt + -r{toxinidir}/test-requirements.txt commands = stestr run {posargs} @@ -40,59 +39,42 @@ allowlist_externals = generate-constraints description = Regenerates upper-constraints.txt # Generate needs an unconstrained install to get new dependencies -deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt +deps = + -r{toxinidir}/requirements.txt + -r{toxinidir}/test-requirements.txt commands = generate-constraints {posargs: -d denylist.txt -r global-requirements.txt -p python3.9 -p python3.10 -p python3.11 -p python3.12 -p python3.13 > upper-constraints.txt} [testenv:validate] allowlist_externals = validate-constraints commands = - validate-constraints {toxinidir}/global-requirements.txt {toxinidir}/upper-constraints.txt {toxinidir}/denylist.txt + validate-constraints {toxinidir}/global-requirements.txt {toxinidir}/upper-constraints.txt {toxinidir}/denylist.txt [testenv:validate-projects] allowlist_externals = validate-projects commands = validate-projects {toxinidir}/projects.txt -# TODO remove once zuul reconfigured to run linters on gate [testenv:pep8] -deps = {[testenv:linters]deps} -allowlist_externals = - bash -commands = - flake8 - bash -c "find {toxinidir}/tools \ - -type f \ - -name \*.sh \ - -print0 | xargs -0 bashate -v -iE006,E010" - bash -c 'sed -e "s,===,==," upper-constraints.txt > {envtmpdir}/safety-check.txt' - -safety check --json -r {envtmpdir}/safety-check.txt - -[testenv:linters] description = Perform linting +skip_install = true deps = - hacking>=1.0.0 - bashate>=0.5.1 - safety + hacking~=7.0 # Apache-2.0 + bashate~=2.1 # Apache-2.0 allowlist_externals = bash commands = - flake8 - bash -c "find {toxinidir}/tools \ - -type f \ - -name \*.sh \ - -print0 | xargs -0 bashate -v -iE006,E010" - bash -c 'sed -e "s,===,==," upper-constraints.txt > {envtmpdir}/safety-check.txt' - -safety check --json -r {envtmpdir}/safety-check.txt + flake8 + bash -c "find {toxinidir}/tools \ + -type f \ + -name \*.sh \ + -print0 | xargs -0 bashate -v -iE006,E010" [testenv:bindep] # Do not install any requirements. We want this to be fast and work even if # system dependencies are missing, since it's used to tell you what system -# dependencies are missing! This also means that bindep must be installed -# separately, outside of the requirements files, and develop mode disabled -# explicitly to avoid unnecessarily installing the checked-out repo too (this -# further relies on "tox.skipsdist = True" above). +# dependencies are missing! +skip_install = true deps = bindep commands = bindep test usedevelop = False @@ -100,12 +82,13 @@ usedevelop = False [testenv:docs] allowlist_externals = sphinx-build -deps = -c{env:TOX_CONSTRAINTS_FILE:{toxinidir}/upper-constraints.txt} - -r{toxinidir}/doc/requirements.txt +deps = + -c{env:TOX_CONSTRAINTS_FILE:{toxinidir}/upper-constraints.txt} + -r{toxinidir}/doc/requirements.txt commands = sphinx-build -W -b html doc/source doc/build/html [testenv:pip-install] -recreate = True +recreate = true deps = . commands = python {toxinidir}/tools/check-install.py @@ -117,6 +100,7 @@ commands = {toxinidir}/playbooks/files/project-requirements-change.py --local {posargs} [testenv:babel] +skip_install = true # Use the local upper-constraints.txt file allowlist_externals = {toxinidir}/tools/babel-test.sh diff --git a/upper-constraints.txt b/upper-constraints.txt index e44e71a82..03c348fb3 100644 --- a/upper-constraints.txt +++ b/upper-constraints.txt @@ -247,6 +247,7 @@ cliff===4.9.1;python_version=='3.9' cliff===4.11.0;python_version>='3.10' os-brick===6.13.0 scp===0.15.0 +lark===1.3.0 python-zaqarclient===3.0.1;python_version=='3.9' python-zaqarclient===4.1.0;python_version>='3.10' ldappool===3.0.0 @@ -328,7 +329,6 @@ XStatic-moment===2.8.4.3 autopage===0.5.2 gitdb===4.0.12 python-monascaclient===2.8.0 -ldap3===2.9.1 opentelemetry-api===1.36.0 automaton===3.2.0 types-urllib3===1.26.25.14 @@ -353,7 +353,7 @@ elasticsearch===2.4.1 django-nose===1.4.7 asgiref===3.9.1 XStatic-JQuery.TableSorter===2.14.5.2 -pifpaf===3.3.0 +pifpaf===3.4.0 blockdiag===3.0.0 testtools===2.7.2 infi.dtypes.iqn===0.4.0 @@ -368,7 +368,6 @@ rich===14.1.0 os-traits===3.5.0 typepy===1.3.4 SecretStorage===3.3.3 -opentracing===2.4.0 XStatic-Rickshaw===1.5.1.0 iso8601===2.1.0 tooz===7.0.0 @@ -439,7 +438,6 @@ tomli===2.2.1;python_version=='3.9' oslo.upgradecheck===2.6.0 sherlock===0.4.1 stevedore===5.5.0 -pywinrm===0.5.0 botocore===1.40.1 xmltodict===0.14.2 pyasn1===0.6.0 @@ -449,6 +447,7 @@ pexpect===4.9.0 cmd2===2.7.0 python-json-logger===3.3.0 redis===6.2.0 +valkey===6.1.1 jmespath===1.0.1 click===8.1.8;python_version=='3.9' click===8.2.2;python_version>='3.10' @@ -511,7 +510,6 @@ pytest-cov===4.1.0 reactivex===4.0.4 Paste===3.10.1 pytest-django===4.11.1 -jaeger-client===4.8.0 XStatic-Json2yaml===0.1.1.0 boto===2.49.0 hyperlink===21.0.0 @@ -530,7 +528,8 @@ sphinxcontrib-qthelp===2.0.0 keystoneauth1===5.12.0 statsd===4.0.1 proto-plus===1.26.1 -python-keystoneclient===5.7.0 +python-keystoneclient===5.6.0;python_version=='3.9' +python-keystoneclient===5.7.0;python_version>='3.10' diskimage-builder===3.39.0 heat-translator===3.3.0 python-magnumclient===4.9.0