From f42d43648e882f72cbd3454ab3b1973bed1371dc Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Wed, 26 Mar 2025 14:18:31 +0100 Subject: [PATCH 01/11] A ResourceOpts is always to added to callback execution by Resource.parse Fixes errors like: ERROR pyff.resource Failed handling resource: MDServiceListParser.parse.._update_entities() takes 1 positional argument but 2 were given --- src/pyff/samlmd.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pyff/samlmd.py b/src/pyff/samlmd.py index 9be47176..d1f772dc 100644 --- a/src/pyff/samlmd.py +++ b/src/pyff/samlmd.py @@ -184,9 +184,9 @@ def parse(self, resource: Resource, content: str) -> SAMLParserInfo: resource.expire_time = expire_time info.expiration_time = str(expire_time) - def _extra_md(_t, info, **kwargs): + def _extra_md(_t, resource_opts, **kwargs): entityID = kwargs.get('entityID') - if info['alias'] != entityID: + if resource_opts['alias'] != entityID: return _t sp_entities = kwargs.get('sp_entities') location = kwargs.get('location') @@ -316,7 +316,7 @@ def parse(self, resource: Resource, content: str) -> EidasMDParserInfo: r = resource.add_child(location, child_opts) # this is specific post-processing for MDSL files - def _update_entities(_t, **kwargs): + def _update_entities(_t, resource_opts, **kwargs): _country_code = kwargs.get('country_code') _hide_from_discovery = kwargs.get('hide_from_discovery') for e in iter_entities(_t): From 0e460b090153c5f0b2c33614c18df7b14bccb4b7 Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Thu, 10 Apr 2025 15:57:31 +0200 Subject: [PATCH 02/11] Store opts unique Without the deep copy all opts were added to the origin. E.g causing Lambas to be run on each "entity" instead of just the selected child. --- src/pyff/resource.py | 4 ++++ src/pyff/samlmd.py | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/pyff/resource.py b/src/pyff/resource.py index fe5ce6a3..0c8fbd14 100644 --- a/src/pyff/resource.py +++ b/src/pyff/resource.py @@ -339,6 +339,10 @@ def _replace(self, r: Resource) -> None: raise ValueError("Resource {} not present - use add_child".format(r.url)) def add_child(self, url: str, opts: ResourceOpts) -> Resource: + """ + Spent 3 man days. Make sure to make a deep copy of opts. + + """ r = Resource(url, opts) if r in self.children: log.debug(f'\n\n{self}:\nURL {url}\nReplacing child {r}') diff --git a/src/pyff/samlmd.py b/src/pyff/samlmd.py index d1f772dc..ddddf42f 100644 --- a/src/pyff/samlmd.py +++ b/src/pyff/samlmd.py @@ -209,7 +209,7 @@ def _extra_md(_t, resource_opts, **kwargs): if md_source is not None: location = md_source.attrib.get('src') if location is not None: - child_opts = resource.opts.copy(update={'alias': entityID}) + child_opts = resource.opts.model_copy(update={'alias': entityID}, deep=True) r = resource.add_child(location, child_opts) kwargs = { 'entityID': entityID, @@ -311,7 +311,7 @@ def parse(self, resource: Resource, content: str) -> EidasMDParserInfo: info.scheme_territory, location, fp, args.get('country_code') ) ) - child_opts = resource.opts.copy(update={'alias': None}) + child_opts = resource.opts.model_copy(update={'alias': None}, deep=True) child_opts.verify = fp r = resource.add_child(location, child_opts) From 9313a3713bc626c779adff4c8ff5a461b805eabd Mon Sep 17 00:00:00 2001 From: Johan Wassberg Date: Thu, 10 Apr 2025 16:14:13 +0200 Subject: [PATCH 03/11] Intitial test of the MDSL stuff --- src/pyff/test/data/eidas/countries/FR.xml | 55 ++++++++++ src/pyff/test/data/eidas/countries/GR.xml | 63 +++++++++++ src/pyff/test/data/eidas/eidas.xml | 18 ++++ src/pyff/test/test_mdsl.py | 124 ++++++++++++++++++++++ 4 files changed, 260 insertions(+) create mode 100644 src/pyff/test/data/eidas/countries/FR.xml create mode 100644 src/pyff/test/data/eidas/countries/GR.xml create mode 100644 src/pyff/test/data/eidas/eidas.xml create mode 100644 src/pyff/test/test_mdsl.py diff --git a/src/pyff/test/data/eidas/countries/FR.xml b/src/pyff/test/data/eidas/countries/FR.xml new file mode 100644 index 00000000..4db01c9e --- /dev/null +++ b/src/pyff/test/data/eidas/countries/FR.xml @@ -0,0 +1,55 @@ + + + + + + + MIIECTCCAnGgAwIBAgIBATANBgkqhkiG9w0BAQsFADAzMQswCQYDVQQGEwJGUjEkMCIGA1UECgwb +RlItRElOVU0tZUlEQVMtbm9kZS1wcmVwcm9kMB4XDTIyMDkyMjE0NDgwNloXDTMyMDkxOTE0NDgw +NlowTDELMAkGA1UEBhMCRlIxJDAiBgNVBAoMG0ZSLURJTlVNLWVJREFTLW5vZGUtcHJlcHJvZDEX +MBUGA1UEAwwOZUlEQVMtbWV0YWRhdGEwggGiMA0GCSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQDO +B8ffvjFTk3rBzcyBiwFFMDlYPxIlj+i1BLdLvvzSJdUHKZhegOhzNuRVCeJfoyvml9vtlwK5tfzD +iY4znZsFO/qIB1wROxrPVRq8AEw0LiPCfP1Ie1rvK2Ddaw0wUEI6wFn4ViAStP5wI3/yaOqN1cFf +JceXUbvgVfjRS5ETRVZK7UER5vMeMIPn4ESkf86d+GB0rvZoipNOyymXfgs9RU/dvjd40lrRs5rZ +Nc/l4dOrFUpxHXq/AfLsWKjmmOGx959qMmYtsK9KcQdEAQs2L/adKM+yLJL0JyHxyNnWuaUJDBwW +xV5PK/hWnkLkebkgpeB5loF7f7Ra2MnB1uzZlaAa69tSILjycdw0cOcY5+hnH6QFq74JDL4WPDR8 +FzCdcN+TY7/HvG6cTR4lPdW/iY3eZQycQqkEccQiFITAeewwVwZbQddbWhHxdtJE8P73QoZl3iSp +/TzP33Kr3im+IEX1o2PV6Ur36/cDNmc9WQWsRKTeydoIY+AOmV/odt0CAwEAAaMPMA0wCwYDVR0P +BAQDAgeAMA0GCSqGSIb3DQEBCwUAA4IBgQB2FpL/Xen8z318X2To+3nEdRthc/9OwPF9ZNzftcce +QP4Q+lxuwKFacklt5VoXURy/JMsgPtSHhar5kiFZTm2SMXVSEqVDj4UzAobVnUutiZPsFoyPWr/c +iEsWu0VSthsI31AUOvSTisy0w81rPKjNkRuCU5V+AS1Z5rBqMlkwOEiUxUMci+pZQ4VhH+mqg1oH +1rMuJ1gysYf/zMjSWGlraSbcHApIMBAjs24uSTH/O6io+9k9jLhZGZv9LpssoeSPVz3wJY6h1LHT +D5OHZYDtelO4X5ZCVRFx5kCaUpKtb4mvo72oWN9KlGvrqp6SkUF1ogovTl16sEf2FZHoN4r0+5ZN +/vS/plTsCGbe46QPGuj+FO5yN6KclM+eonmBp0JSyoABfYN1T28a4dRwE8CPWCOgJmzxuwqSOH+P +XlUbPftbx5nGsE3mRBAU1Jga2S4S+zJPHirc2lfZGh6ggdYG9kkqp7X7kiTyPyPuWU644+YaCOwv +x7804cuLUcQ8VSQ= + + + + + + + + MIIECTCCAnGgAwIBAgIBATANBgkqhkiG9w0BAQsFADAzMQswCQYDVQQGEwJGUjEkMCIGA1UECgwb +RlItRElOVU0tZUlEQVMtbm9kZS1wcmVwcm9kMB4XDTIyMDkyMjE0NDgwNloXDTMyMDkxOTE0NDgw +NlowTDELMAkGA1UEBhMCRlIxJDAiBgNVBAoMG0ZSLURJTlVNLWVJREFTLW5vZGUtcHJlcHJvZDEX +MBUGA1UEAwwOZUlEQVMtbWV0YWRhdGEwggGiMA0GCSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQDO +B8ffvjFTk3rBzcyBiwFFMDlYPxIlj+i1BLdLvvzSJdUHKZhegOhzNuRVCeJfoyvml9vtlwK5tfzD +iY4znZsFO/qIB1wROxrPVRq8AEw0LiPCfP1Ie1rvK2Ddaw0wUEI6wFn4ViAStP5wI3/yaOqN1cFf +JceXUbvgVfjRS5ETRVZK7UER5vMeMIPn4ESkf86d+GB0rvZoipNOyymXfgs9RU/dvjd40lrRs5rZ +Nc/l4dOrFUpxHXq/AfLsWKjmmOGx959qMmYtsK9KcQdEAQs2L/adKM+yLJL0JyHxyNnWuaUJDBwW +xV5PK/hWnkLkebkgpeB5loF7f7Ra2MnB1uzZlaAa69tSILjycdw0cOcY5+hnH6QFq74JDL4WPDR8 +FzCdcN+TY7/HvG6cTR4lPdW/iY3eZQycQqkEccQiFITAeewwVwZbQddbWhHxdtJE8P73QoZl3iSp +/TzP33Kr3im+IEX1o2PV6Ur36/cDNmc9WQWsRKTeydoIY+AOmV/odt0CAwEAAaMPMA0wCwYDVR0P +BAQDAgeAMA0GCSqGSIb3DQEBCwUAA4IBgQB2FpL/Xen8z318X2To+3nEdRthc/9OwPF9ZNzftcce +QP4Q+lxuwKFacklt5VoXURy/JMsgPtSHhar5kiFZTm2SMXVSEqVDj4UzAobVnUutiZPsFoyPWr/c +iEsWu0VSthsI31AUOvSTisy0w81rPKjNkRuCU5V+AS1Z5rBqMlkwOEiUxUMci+pZQ4VhH+mqg1oH +1rMuJ1gysYf/zMjSWGlraSbcHApIMBAjs24uSTH/O6io+9k9jLhZGZv9LpssoeSPVz3wJY6h1LHT +D5OHZYDtelO4X5ZCVRFx5kCaUpKtb4mvo72oWN9KlGvrqp6SkUF1ogovTl16sEf2FZHoN4r0+5ZN +/vS/plTsCGbe46QPGuj+FO5yN6KclM+eonmBp0JSyoABfYN1T28a4dRwE8CPWCOgJmzxuwqSOH+P +XlUbPftbx5nGsE3mRBAU1Jga2S4S+zJPHirc2lfZGh6ggdYG9kkqp7X7kiTyPyPuWU644+YaCOwv +x7804cuLUcQ8VSQ= + + + + diff --git a/src/pyff/test/data/eidas/countries/GR.xml b/src/pyff/test/data/eidas/countries/GR.xml new file mode 100644 index 00000000..a8e1626e --- /dev/null +++ b/src/pyff/test/data/eidas/countries/GR.xml @@ -0,0 +1,63 @@ + + + + + + + MIIE5zCCA0+gAwIBAgIULMraEI5oF6p3T3LH0x/21gVFOvcwDQYJKoZIhvcNAQELBQAwgYkxCzAJ +BgNVBAYTAkdSMScwJQYDVQQKDB5NaW5pc3RyeSBvZiBEaWdpdGFsIEdvdmVybmFuY2UxPTA7BgNV +BAsMNGVJREFTIE5vZGUgTWV0YWRhdGEgU2lnbmluZyB0ZXN0aW5nIGVudmlyb25tZW50IDIwMjMx +EjAQBgNVBAMMCUNvbm5lY3RvcjAeFw0yMzA5MjAxODIxMDlaFw0zMzA5MTcxODIxMDlaMIGJMQsw +CQYDVQQGEwJHUjEnMCUGA1UECgweTWluaXN0cnkgb2YgRGlnaXRhbCBHb3Zlcm5hbmNlMT0wOwYD +VQQLDDRlSURBUyBOb2RlIE1ldGFkYXRhIFNpZ25pbmcgdGVzdGluZyBlbnZpcm9ubWVudCAyMDIz +MRIwEAYDVQQDDAlDb25uZWN0b3IwggGiMA0GCSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQC0PU5P +pibubglyAQJDW0Q+iWiKXnfoBV6K5yFvzzE91UT/lE2UvX3haqqj8PwsfsJsPG7M805ky2UPY5nO +U7VcnaQ3MWzz35kjNI1fvlXkR06YEkhBkr64WJ/7JTxf0wLO83ZRqBcvMqlDCqf8bPHXdjar89sS +e866Wd2rthi7Tu6Ah6buF96lXpS2d9vwnf1S9mhOmtykQ53Vs5zgqMVOaHfKwCThefoYOCyzuqNP +S1G0dQaRbXkIDpbniT96aap8Ksf0/Yx5E+o7BnvbnEsCJPHTYudTKqN7ljD8+Q4M2UNpMT05qIR7 +Zd2KfpufEV8o5YJzMCIftZH6sndBYbpDYkqM+Cd6qy84HBy2/UWgZDt2iQ1eML/Szk59wSm21szd +Q4mshSv4rTwHuTtHg+ZeiCzwJGgvnQen3WR+TGm0YyHijfI9DqIvzbM8yXF8Abp00l3sA3Grm7wo +E0YW+EbkiiJGUFrs88+P+cpcXeNkLltXSvhVosgF0JLcgoLl62UCAwEAAaNFMEMwEgYDVR0TAQH/ +BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCB4AwHQYDVR0OBBYEFDu6OnhYjoq3nx9jmT8vwQ5tjFWD +MA0GCSqGSIb3DQEBCwUAA4IBgQARw/klO118DF5BNunXqsiExeQSQqb+n/qcBXDYLypQJomO/JAs +VEg4FEn05Z26QxyacwWxVqWhInVLFVhXgBJ5vnRTwK7RUkJCDQ+9u00oHd3JeOvygFjc7DoC5SWJ +q9hnN+pX1qK9yI+R9hs7cKPKpQ+MaNPWl0yVQCE53GZrWk+Skgl2T5/s0rY+LTYUx5d21kxq1tKE +cpzy9di+33w05uv7ozh7xcbL91wj7zbdDSibxpXmFdlY6C8BH4DOI0kRoYjWKpbuQnPrEbhQwZjI +V9S5OcEaPyiYkVB9n4Z0z3AMTD5G4X52gKmdSh/iTuZuYP4Vj1hbgNYpMHI7B5fnXisa26e2DzT4 +OFEots8gDlsf26WBEHQcJnrNzqCPTd4Zyl5Jpzg+Vo/dwknIESuZYn2l+Iu1GJCMIV+RrI/LoPD4 +FRGrw9YbcFOqAgmSdqxRj6fSb2W5WanIvc7OAT0hKQjPu1jYHDGIeXipKf1rBLjpRF/xtzU/xb4m +JDmr+yA= + + + + + + + + MIIE7TCCA1WgAwIBAgIUTo2pg82nNFjKFzNyHnCbQu1rEVYwDQYJKoZIhvcNAQELBQAwgYwxCzAJ +BgNVBAYTAkdSMScwJQYDVQQKDB5NaW5pc3RyeSBvZiBEaWdpdGFsIEdvdmVybmFuY2UxPTA7BgNV +BAsMNGVJREFTIE5vZGUgTWV0YWRhdGEgU2lnbmluZyB0ZXN0aW5nIGVudmlyb25tZW50IDIwMjMx +FTATBgNVBAMMDFByb3h5U2VydmljZTAeFw0yMzA5MjAxODM5MDZaFw0zMzA5MTcxODM5MDZaMIGM +MQswCQYDVQQGEwJHUjEnMCUGA1UECgweTWluaXN0cnkgb2YgRGlnaXRhbCBHb3Zlcm5hbmNlMT0w +OwYDVQQLDDRlSURBUyBOb2RlIE1ldGFkYXRhIFNpZ25pbmcgdGVzdGluZyBlbnZpcm9ubWVudCAy +MDIzMRUwEwYDVQQDDAxQcm94eVNlcnZpY2UwggGiMA0GCSqGSIb3DQEBAQUAA4IBjwAwggGKAoIB +gQCqNGRL80G3XrF5wgtJIOCglAg8361zga2Hup8G+w6eQdP5bUyX9JZskTr9IUNeajUw7sRGN9GC +iflAILmzHqySnspOZZer4bUaOEKhRE0RZnhWoCyeZgdp1j9bwe2uLgRJtLQWpeGq3kEzCoSqul70 +iLegKd85f8i0S5ZgzdpjSBJcetGwQxV8bw1+3IT4/OXrL467Z2tBvPE1AClf0ETkw9y5vfI1O+Vr +OHJg8ywIKUgLdfgpCpjHGzljOVlA1ZbCplPvKOOkjRKx7BWAFwFqUbSLYjVDrhIHkCv+EsGeHLK4 +aLmxAVVpwj5qhlnxJ7vweKUEorw7GUGHhAmiM9bem5Wc3jakt06Hd8vA6/kn7Yr17feZtaBfWHAP +HoH0nvGLHk6+WU3W3/i89KLnYH+JsGfY8vSOQesaavmZy6WTEmXk6AkDrUocdC0IrMhq8duOiN3u +KrzN9t0vaRb6KMr35x7l9Sq6hxyiIjfZW+qB8434HmxKyZODCcXEncpNK8MCAwEAAaNFMEMwEgYD +VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCB4AwHQYDVR0OBBYEFM6pVXDExVEGIVCJbVKV +/S9m7zFtMA0GCSqGSIb3DQEBCwUAA4IBgQCmADqhrXNb4iOsSOkHBghndhsT/lowKPtCicLrwfmm +twQtwA/H1JILe11r97LxIuDmFi0wdSIwx+E0ioA3BF4kcddXrYDimQD2LDQZIWFYHaMhAII89Kfv +VXnn9Ox048cJHsKrteMchCOKw1xxZYsOKr1dFPiQQOX+mP/S1aBnj+7Sr4GuTSkWXC4OiO+BaZQW +4mWe83DsngdUjHmCDrvnBT3xzUbJx5hEky2lnU1ZFZtusH6v6VElQ1KPgDBov1dTktB+r44v47DO +WZ7GUkwvoOVS0lpl7nCpD9QBBch0JmAyPI9RvVRD5vaPghD+Jv5JEiQtMJwcE2l1GYLcwo5Q1BOu +FsLJQEEmD7PUm0ZisV7GTQbRqh2YcLSHMrXOMvbvBlNdEuTDrdMvKFK/FMGh+4DHh/J3+hklGw+e +X+U++5GGUbOS6nZtDwmW/KBYMSnanWyWJ1cnhos7A2JkL429B6uhZsPIHAQn26LBz8yGx3vxze8D +wBm+w1VVhk19CHc= + + + + diff --git a/src/pyff/test/data/eidas/eidas.xml b/src/pyff/test/data/eidas/eidas.xml new file mode 100644 index 00000000..f9d28c92 --- /dev/null +++ b/src/pyff/test/data/eidas/eidas.xml @@ -0,0 +1,18 @@ + + + + Swedish E-Identification Board + urn:se:elegnamnden:eidas:mdlist:local + SE + + + + + + https://qa.md.eidas.swedenconnect.se/mdservicelist-aggregate.xml + + diff --git a/src/pyff/test/test_mdsl.py b/src/pyff/test/test_mdsl.py new file mode 100644 index 00000000..3ec44d75 --- /dev/null +++ b/src/pyff/test/test_mdsl.py @@ -0,0 +1,124 @@ +import json +import os +import shutil +import sys +import tempfile + +import pytest +import six +import yaml +from mako.lookup import TemplateLookup +from mock import patch + +from pyff import builtins +from pyff.exceptions import MetadataException +from pyff.parse import ParserException +from pyff.pipes import PipeException, Plumbing, plumbing +from pyff.repo import MDRepository +from pyff.resource import ResourceException +from pyff.test import ExitException, SignerTestCase +from pyff.utils import hash_id, parse_xml, resource_filename, root, dumptree +from pyff.constants import NS + + +__author__ = 'leifj' + +# The 'builtins' import appears unused to static analysers, ensure it isn't removed +assert builtins is not None + + +class PipeLineTest(SignerTestCase): + @pytest.fixture(autouse=True) + def _capsys(self, capsys): + self._capsys = capsys + + @property + def captured_stdout(self) -> str: + """ Return anything written to STDOUT during this test """ + out, _err = self._capsys.readouterr() # type: ignore + return out + + @property + def captured_stderr(self) -> str: + """ Return anything written to STDERR during this test """ + _out, err = self._capsys.readouterr() # type: ignore + return err + + @pytest.fixture(autouse=True) + def _caplog(self, caplog): + """ Return anything written to the logging system during this test """ + self._caplog = caplog + + @property + def captured_log_text(self) -> str: + return self._caplog.text # type: ignore + + def run_pipeline(self, pl_name, ctx=None, md=None): + if ctx is None: + ctx = dict() + + if md is None: + md = MDRepository() + + templates = TemplateLookup(directories=[os.path.join(self.datadir, 'simple-pipeline')]) + pipeline = tempfile.NamedTemporaryFile('w').name + template = templates.get_template(pl_name) + with open(pipeline, "w") as fd: + fd.write(template.render(ctx=ctx)) + res = plumbing(pipeline).process(md, state={'batch': True, 'stats': {}}) + os.unlink(pipeline) + return res, md, ctx + + def exec_pipeline(self, pstr): + md = MDRepository() + p = yaml.safe_load(six.StringIO(pstr)) + print("\n{}".format(yaml.dump(p))) + pl = Plumbing(p, pid="test") + res = pl.process(md, state={'batch': True, 'stats': {}}) + return res, md + + @classmethod + def setUpClass(cls): + SignerTestCase.setUpClass() + + def setUp(self): + SignerTestCase.setUpClass() + self.templates = TemplateLookup(directories=[os.path.join(self.datadir, 'simple-pipeline')]) + + +class ParseTest(PipeLineTest): + def test_eidas_country(self): + tmpfile = tempfile.NamedTemporaryFile('w').name + try: + self.exec_pipeline(f""" +- when eidas: + - xslt: + stylesheet: eidas-cleanup.xsl + - break + +- load: + - file://{self.datadir}/eidas/eidas.xml cleanup eidas +- select +- publish: {tmpfile} +""" + ) + xml = parse_xml(tmpfile) + assert xml is not None + entityID = "https://pre.eidas.gov.gr/EidasNode/ServiceMetadata" + with_hide_from_discovery = xml.find("{%s}EntityDescriptor[@entityID='%s']" % (NS['md'], entityID)) + assert with_hide_from_discovery is not None + search = "{%s}Extensions/{%s}EntityAttributes/{%s}Attribute[@Name='%s']" % (NS['md'], NS['mdattr'], NS['saml'],'http://macedir.org/entity-category') + ecs = with_hide_from_discovery.find(search) + assert ecs is not None + entityID2 = "https://eidas.pp.dev-franceconnect.fr/EidasNode/ServiceMetadata" + without_hide_from_discovery = xml.find("{%s}EntityDescriptor[@entityID='%s']" % (NS['md'], entityID2)) + ecs2 = without_hide_from_discovery.find(search) + assert ecs2 is None + except IOError: + pass + finally: + try: + #os.unlink(tmpfile) + pass + except (IOError, OSError): + pass From a637a2b0a252fbe80c31627e08f53bab77a753d0 Mon Sep 17 00:00:00 2001 From: Mikael Frykholm Date: Wed, 2 Apr 2025 14:55:41 +0200 Subject: [PATCH 04/11] Update python-package.yml --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index e84c09a2..20f9e0b4 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -8,7 +8,7 @@ on: branches: [ "master" ] pull_request: branches: [ "master" ] - + workflow_dispatch: jobs: build: From 2a78c77ff2ad61bfc302d30758efcb242fdf6d62 Mon Sep 17 00:00:00 2001 From: Dick Visser Date: Fri, 11 Oct 2024 16:27:54 +0200 Subject: [PATCH 05/11] fix 276 --- src/pyff/parse.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pyff/parse.py b/src/pyff/parse.py index cb0ad1bf..9162d743 100644 --- a/src/pyff/parse.py +++ b/src/pyff/parse.py @@ -4,6 +4,7 @@ from typing import Any, Dict, List, Optional from pydantic import BaseModel, Field +from urllib.parse import quote as urlescape from xmlsec.crypto import CertDict from pyff.constants import NS @@ -84,7 +85,7 @@ def parse(self, resource: Resource, content: str) -> ParserInfo: n = 0 for fn in find_matching_files(content, self.extensions): child_opts = resource.opts.copy(update={'alias': None}) - resource.add_child("file://" + fn, child_opts) + resource.add_child("file://" + urlescape(fn), child_opts) n += 1 if n == 0: From b7fc40a8afbbd78ff39836be5d95a412570c50a2 Mon Sep 17 00:00:00 2001 From: Mikael Frykholm Date: Wed, 2 Apr 2025 15:39:08 +0200 Subject: [PATCH 06/11] Never use live data in tests! will fix later.. --- src/pyff/test/test_md_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pyff/test/test_md_api.py b/src/pyff/test/test_md_api.py index 471bd87b..68cd3714 100644 --- a/src/pyff/test/test_md_api.py +++ b/src/pyff/test/test_md_api.py @@ -141,7 +141,7 @@ def test_load_and_query(self): assert r.status_code == 200 data = r.json() info = data[0] - assert 'https://box-idp.nordu.net/simplesaml/module.php/saml/sp/discoresp.php' in info['discovery_responses'] + assert 'https://box-idp.nordu.net/simplesaml/module.php/saml/sp/discoResponse' in info['discovery_responses'] class PyFFAPITestResources(PipeLineTest): """ From 0cc52a39145fe697eda055551682fe429e99778d Mon Sep 17 00:00:00 2001 From: Mikael Frykholm Date: Mon, 7 Apr 2025 14:30:16 +0200 Subject: [PATCH 07/11] Switch to ruff for formatting instead of black. This helps Language Servers keep the formatting sane. --- Makefile | 3 +-- pyproject.toml | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 pyproject.toml diff --git a/Makefile b/Makefile index d1b20d9f..3ab85c56 100644 --- a/Makefile +++ b/Makefile @@ -15,8 +15,7 @@ test_coverage: coverage combine reformat: - isort --line-width 120 --atomic --project eduid_scimapi --recursive $(SOURCE) - black --line-length 120 --target-version py37 --skip-string-normalization $(SOURCE) + ruff --format $(SOURCE) typecheck: mypy --ignore-missing-imports $(SOURCE) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..a9d009aa --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,32 @@ +[project] +name = "pyFF" +version = "2.1.5" +readme = "README.rst" +description = "Federation Feeder" +requires-python = ">=3.7" +license = {file = "LICENSE"} + +authors = [ + {name = "Leif Johansson", email = "leifj@sunet.se"}, + {name = "Fredrik Thulin", email = "redrik@thulin.net"}, + {name = "Enrique Pérez Arnaud"}, + {name = "Mikael Frykholm", email = "mifr@sunet.se"}, +] +maintainers = [ + {name = "Mikael Frykholm", email = "mifr@sunet.se"} +] + +[tool.ruff] +# Allow lines to be as long as 120. +line-length = 120 +target-version = "py37" +[tool.ruff.format] +quote-style = "preserve" + +[tool.build_sphinx] +source-dir = "docs/" +build-dir = "docs/build" +all_files = "1" + +[tool.upload_sphinx] +upload-dir = "docs/build/html" From 5f4f763f4846e5e8c436f317d467c651b8ded865 Mon Sep 17 00:00:00 2001 From: Mikael Frykholm Date: Mon, 7 Apr 2025 14:41:57 +0200 Subject: [PATCH 08/11] Fix Pydantic deprecation copy > model_copy --- src/pyff/builtins.py | 13 ++++++------- src/pyff/parse.py | 8 +++++--- src/pyff/samlmd.py | 23 ++++++++++++++++------- 3 files changed, 27 insertions(+), 17 deletions(-) diff --git a/src/pyff/builtins.py b/src/pyff/builtins.py index 8bdd3a2d..b00cbd63 100644 --- a/src/pyff/builtins.py +++ b/src/pyff/builtins.py @@ -256,7 +256,7 @@ def fork(req: Plumbing.Request, *opts): **parsecopy** Due to a hard to find bug, fork which uses deepcopy can lose some namespaces. The parsecopy argument is a workaround. - It uses a brute force serialisation and deserialisation to get around the bug. + It uses a brute force serialisation and deserialisation to get around the bug. .. code-block:: yaml @@ -676,7 +676,7 @@ def load(req: Plumbing.Request, *opts): url = r.pop(0) # Copy parent node opts as a starting point - child_opts = req.md.rm.opts.copy(update={"via": [], "cleanup": [], "verify": None, "alias": url}) + child_opts = req.md.rm.opts.model_copy(update={"via": [], "cleanup": [], "verify": None, "alias": url}) while len(r) > 0: elt = r.pop(0) @@ -702,7 +702,7 @@ def load(req: Plumbing.Request, *opts): child_opts.verify = elt # override anything in child_opts with what is in opts - child_opts = child_opts.copy(update=_opts) + child_opts = child_opts.model_copy(update=_opts) req.md.rm.add_child(url, child_opts) @@ -814,7 +814,7 @@ def select(req: Plumbing.Request, *opts): else: _opts['as'] = opts[i] if i + 1 < len(opts): - more_opts = opts[i + 1:] + more_opts = opts[i + 1 :] _opts.update(dict(list(zip(more_opts[::2], more_opts[1::2])))) break @@ -835,7 +835,6 @@ def select(req: Plumbing.Request, *opts): entities = resolve_entities(args, lookup_fn=req.md.store.select, dedup=dedup) if req.state.get('match', None): # TODO - allow this to be passed in via normal arguments - match = req.state['match'] if isinstance(match, six.string_types): @@ -1304,7 +1303,7 @@ def xslt(req: Plumbing.Request, *opts): if stylesheet is None: raise PipeException("xslt requires stylesheet") - params = dict((k, "\'%s\'" % v) for (k, v) in list(req.args.items())) + params = dict((k, "'%s'" % v) for (k, v) in list(req.args.items())) del params['stylesheet'] try: return root(xslt_transform(req.t, stylesheet, params)) @@ -1312,6 +1311,7 @@ def xslt(req: Plumbing.Request, *opts): log.debug(traceback.format_exc()) raise ex + @pipe def indent(req: Plumbing.Request, *opts): """ @@ -1710,7 +1710,6 @@ def finalize(req: Plumbing.Request, *opts): if name is None or 0 == len(name): name = req.state.get('url', None) if name and 'baseURL' in req.args: - try: name_url = urlparse(name) base_url = urlparse(req.args.get('baseURL')) diff --git a/src/pyff/parse.py b/src/pyff/parse.py index 9162d743..7737f43b 100644 --- a/src/pyff/parse.py +++ b/src/pyff/parse.py @@ -9,7 +9,7 @@ from pyff.constants import NS from pyff.logs import get_log -from pyff.resource import Resource,ResourceInfo +from pyff.resource import Resource, ResourceInfo from pyff.utils import find_matching_files, parse_xml, root, unicode_stream, utc_now __author__ = 'leifj' @@ -30,8 +30,10 @@ def _format_key(k: str) -> str: res = {_format_key(k): v for k, v in self.dict().items()} return res + ResourceInfo.model_rebuild() + class ParserException(Exception): def __init__(self, msg, wrapped=None, data=None): self._wraped = wrapped @@ -84,7 +86,7 @@ def parse(self, resource: Resource, content: str) -> ParserInfo: info = ParserInfo(description='Directory', expiration_time='never expires') n = 0 for fn in find_matching_files(content, self.extensions): - child_opts = resource.opts.copy(update={'alias': None}) + child_opts = resource.opts.model_copy(update={'alias': None}) resource.add_child("file://" + urlescape(fn), child_opts) n += 1 @@ -122,7 +124,7 @@ def parse(self, resource: Resource, content: str) -> ParserInfo: if len(fingerprints) > 0: fp = fingerprints[0] log.debug("XRD: {} verified by {}".format(link_href, fp)) - child_opts = resource.opts.copy(update={'alias': None}) + child_opts = resource.opts.model_copy(update={'alias': None}) resource.add_child(link_href, child_opts) resource.last_seen = utc_now().replace(microsecond=0) resource.expire_time = None diff --git a/src/pyff/samlmd.py b/src/pyff/samlmd.py index ddddf42f..20363cd4 100644 --- a/src/pyff/samlmd.py +++ b/src/pyff/samlmd.py @@ -86,7 +86,10 @@ def find_merge_strategy(strategy_name): def parse_saml_metadata( - source: BytesIO, opts: ResourceOpts, base_url=None, validation_errors: Optional[Dict[str, Any]] = None, + source: BytesIO, + opts: ResourceOpts, + base_url=None, + validation_errors: Optional[Dict[str, Any]] = None, ): """Parse a piece of XML and return an EntitiesDescriptor element after validation. @@ -192,7 +195,10 @@ def _extra_md(_t, resource_opts, **kwargs): location = kwargs.get('location') sp_entity = sp_entities.find("{%s}EntityDescriptor[@entityID='%s']" % (NS['md'], entityID)) if sp_entity is not None: - md_source = sp_entity.find("{%s}SPSSODescriptor/{%s}Extensions/{%s}TrustInfo/{%s}MetadataSource[@src='%s']" % (NS['md'], NS['md'], NS['ti'], NS['ti'], location)) + md_source = sp_entity.find( + "{%s}SPSSODescriptor/{%s}Extensions/{%s}TrustInfo/{%s}MetadataSource[@src='%s']" + % (NS['md'], NS['md'], NS['ti'], NS['ti'], location) + ) for e in iter_entities(_t): md_source.append(e) return etree.Element("{%s}EntitiesDescriptor" % NS['md']) @@ -205,7 +211,10 @@ def _extra_md(_t, resource_opts, **kwargs): entityID = e.get('entityID') info.entities.append(entityID) - md_source = e.find("{%s}SPSSODescriptor/{%s}Extensions/{%s}TrustInfo/{%s}MetadataSource" % (NS['md'], NS['md'], NS['ti'], NS['ti'])) + md_source = e.find( + "{%s}SPSSODescriptor/{%s}Extensions/{%s}TrustInfo/{%s}MetadataSource" + % (NS['md'], NS['md'], NS['ti'], NS['ti']) + ) if md_source is not None: location = md_source.attrib.get('src') if location is not None: @@ -725,7 +734,6 @@ def entity_domains(entity): def entity_extended_display_i18n(entity, default_lang=None): - name_dict = lang_dict(entity.iter("{%s}OrganizationName" % NS['md']), lambda e: e.text, default_lang=default_lang) name_dict.update( lang_dict(entity.iter("{%s}OrganizationDisplayName" % NS['md']), lambda e: e.text, default_lang=default_lang) @@ -981,7 +989,9 @@ def discojson_sp(e, global_trust_info=None, global_md_sources=None): sp['entityID'] = e.get('entityID', None) - md_sources = e.findall("{%s}SPSSODescriptor/{%s}Extensions/{%s}TrustInfo/{%s}MetadataSource" % (NS['md'], NS['md'], NS['ti'], NS['ti'])) + md_sources = e.findall( + "{%s}SPSSODescriptor/{%s}Extensions/{%s}TrustInfo/{%s}MetadataSource" % (NS['md'], NS['md'], NS['ti'], NS['ti']) + ) sp['extra_md'] = {} for md_source in md_sources: @@ -1041,7 +1051,6 @@ def discojson_sp(e, global_trust_info=None, global_md_sources=None): def discojson_sp_attr(e): - attribute = "https://refeds.org/entity-selection-profile" b64_trustinfos = entity_attribute(e, attribute) if b64_trustinfos is None: @@ -1395,7 +1404,7 @@ def get_key(e): except AttributeError: pass except IndexError: - log.warning("Sort pipe: unable to sort entity by '%s'. " "Entity '%s' has no such value" % (sxp, eid)) + log.warning("Sort pipe: unable to sort entity by '%s'. Entity '%s' has no such value" % (sxp, eid)) except TypeError: pass From 2254a585236d7aa3bf3384118d4704fcd8182d6a Mon Sep 17 00:00:00 2001 From: Mikael Frykholm Date: Mon, 7 Apr 2025 15:57:10 +0200 Subject: [PATCH 09/11] Started removing pkg_resources. Formatting fixes. --- src/pyff/__init__.py | 9 ++------- src/pyff/api.py | 12 ++++++------ src/pyff/test/__init__.py | 9 +++++---- 3 files changed, 13 insertions(+), 17 deletions(-) diff --git a/src/pyff/__init__.py b/src/pyff/__init__.py index 5e45b2b6..7d54b185 100644 --- a/src/pyff/__init__.py +++ b/src/pyff/__init__.py @@ -2,11 +2,6 @@ pyFF is a SAML metadata aggregator. """ -import pkg_resources +import importlib -__author__ = 'Leif Johansson' -__copyright__ = "Copyright 2009-2018 SUNET and the IdentityPython Project" -__license__ = "BSD" -__maintainer__ = "leifj@sunet.se" -__status__ = "Production" -__version__ = pkg_resources.require("pyFF")[0].version +__version__ = importlib.metadata.version('pyFF') diff --git a/src/pyff/api.py b/src/pyff/api.py index 1050efbf..8b443409 100644 --- a/src/pyff/api.py +++ b/src/pyff/api.py @@ -4,7 +4,6 @@ from json import dumps from typing import Any, Dict, Generator, Iterable, List, Mapping, Optional, Tuple -import pkg_resources import pyramid.httpexceptions as exc import pytz import requests @@ -26,12 +25,13 @@ from pyff.resource import Resource from pyff.samlmd import entity_display_name from pyff.utils import b2u, dumptree, hash_id, json_serializer, utc_now +from pyff import __version__ log = get_log(__name__) class NoCache(object): - """ Dummy implementation for when caching isn't enabled """ + """Dummy implementation for when caching isn't enabled""" def __init__(self) -> None: pass @@ -70,7 +70,7 @@ def status_handler(request: Request) -> Response: if 'Validation Errors' in r.info and r.info['Validation Errors']: d[r.url] = r.info['Validation Errors'] _status = dict( - version=pkg_resources.require("pyFF")[0].version, + version=__version__, invalids=d, icon_store=dict(size=request.registry.md.icon_store.size()), jobs=[dict(id=j.id, next_run_time=j.next_run_time) for j in request.registry.scheduler.get_jobs()], @@ -163,7 +163,7 @@ def process_handler(request: Request) -> Response: _ctypes = {'xml': 'application/samlmetadata+xml;application/xml;text/xml', 'json': 'application/json'} def _d(x: Optional[str], do_split: bool = True) -> Tuple[Optional[str], Optional[str]]: - """ Split a path into a base component and an extension. """ + """Split a path into a base component and an extension.""" if x is not None: x = x.strip() @@ -214,7 +214,7 @@ def _d(x: Optional[str], do_split: bool = True) -> Tuple[Optional[str], Optional pfx = request.registry.aliases.get(alias, None) if pfx is None: log.debug("alias {} not found - passing to storage lookup".format(alias)) - path=alias #treat as path + path = alias # treat as path # content_negotiation_policy is one of three values: # 1. extension - current default, inspect the path and if it ends in @@ -478,7 +478,7 @@ def cors_headers(request: Request, response: Response) -> None: { 'Access-Control-Allow-Origin': '*', 'Access-Control-Allow-Methods': 'POST,GET,DELETE,PUT,OPTIONS', - 'Access-Control-Allow-Headers': ('Origin, Content-Type, Accept, ' 'Authorization'), + 'Access-Control-Allow-Headers': ('Origin, Content-Type, Accept, Authorization'), 'Access-Control-Allow-Credentials': 'true', 'Access-Control-Max-Age': '1728000', } diff --git a/src/pyff/test/__init__.py b/src/pyff/test/__init__.py index c39abfda..51cd14ef 100644 --- a/src/pyff/test/__init__.py +++ b/src/pyff/test/__init__.py @@ -6,9 +6,8 @@ import tempfile from unittest import TestCase -import pkg_resources +import importlib.resources import six - from pyff import __version__ as pyffversion # range of ports where available ports can be found @@ -118,7 +117,6 @@ def _p(args, outf=None, ignore_exit=False): class SignerTestCase(TestCase): - datadir = None private_keyspec = None public_keyspec = None @@ -128,7 +126,10 @@ def sys_exit(self, code): @classmethod def setUpClass(cls): - cls.datadir = pkg_resources.resource_filename(__name__, 'data') + with importlib.resources.path( + __name__, 'data' + ) as context: # We just want the path for now to be compatible downstream + cls.datadir = context.as_posix() cls.private_keyspec = tempfile.NamedTemporaryFile('w').name cls.public_keyspec = tempfile.NamedTemporaryFile('w').name From d45e7d8f6a112dce2ec8aa62b5a1abfd1bb09ce0 Mon Sep 17 00:00:00 2001 From: Mikael Frykholm Date: Tue, 8 Apr 2025 15:34:48 +0200 Subject: [PATCH 10/11] Make it work with python3.9. --- pyproject.toml | 4 ++-- src/pyff/test/__init__.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a9d009aa..317d1cb1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ name = "pyFF" version = "2.1.5" readme = "README.rst" description = "Federation Feeder" -requires-python = ">=3.7" +requires-python = ">=3.9" license = {file = "LICENSE"} authors = [ @@ -19,7 +19,7 @@ maintainers = [ [tool.ruff] # Allow lines to be as long as 120. line-length = 120 -target-version = "py37" +target-version = "py39" [tool.ruff.format] quote-style = "preserve" diff --git a/src/pyff/test/__init__.py b/src/pyff/test/__init__.py index 51cd14ef..be32c588 100644 --- a/src/pyff/test/__init__.py +++ b/src/pyff/test/__init__.py @@ -126,10 +126,10 @@ def sys_exit(self, code): @classmethod def setUpClass(cls): - with importlib.resources.path( - __name__, 'data' - ) as context: # We just want the path for now to be compatible downstream - cls.datadir = context.as_posix() + cls.datadir = importlib.resources.files( + __name__, + ).joinpath('data') + cls.private_keyspec = tempfile.NamedTemporaryFile('w').name cls.public_keyspec = tempfile.NamedTemporaryFile('w').name From 64c0b01565ed9d6be4ffa3f5214e6ce03052c78c Mon Sep 17 00:00:00 2001 From: Mikael Frykholm Date: Tue, 8 Apr 2025 16:11:10 +0200 Subject: [PATCH 11/11] ubuntu-20.04 is deprecated by github runners. --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 20f9e0b4..aa18e2d9 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -15,7 +15,7 @@ jobs: strategy: fail-fast: false matrix: - os: ["ubuntu-24.04", "ubuntu-22.04", "ubuntu-20.04"] + os: ["ubuntu-24.04", "ubuntu-22.04"] python: ["3.9", "3.10", "3.11", "3.12"] runs-on: ${{ matrix.os }} steps: