From 2ac3dae3b7a6e2a76154898ab05271c4a0b3d358 Mon Sep 17 00:00:00 2001 From: Alex Welsh Date: Thu, 20 Nov 2025 11:51:59 +0000 Subject: [PATCH 01/13] Bump Ansible to 13 --- .github/workflows/pull_request.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index ac4afa4..704326a 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -27,7 +27,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install ansible==5.* jmespath pulp-glue==0.21.* + pip install ansible==13.* jmespath pulp-glue==0.21.* ansible-galaxy collection install git+file://$(pwd) - name: Run Pulp in one From b8643b336d52bc6343b8bddf6e54d703fb0539f5 Mon Sep 17 00:00:00 2001 From: Alex Welsh Date: Thu, 20 Nov 2025 11:56:03 +0000 Subject: [PATCH 02/13] Update Readme --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2e3d63b..756f022 100644 --- a/README.md +++ b/README.md @@ -7,12 +7,15 @@ Note: Pulp server installation is out of this collection's scope - for this purp ## Tested with Ansible -Tested with the current Ansible 2.9-2.10 releases. +Tested with the current Ansible 13 releases. ## Included content pulp_contentguard role pulp_repository role +pulp_distribution role +pulp_django_user role +pulp_group role ## Using this collection From ef62109a9d925c6ef73acaeeeadc9f290ef9c93b Mon Sep 17 00:00:00 2001 From: Alex Welsh Date: Thu, 20 Nov 2025 11:57:40 +0000 Subject: [PATCH 03/13] Bump tested pulp-in-one version --- .github/workflows/pull_request.yml | 2 +- tests/pulp-in-one.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 704326a..a9bd194 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -16,8 +16,8 @@ jobs: fail-fast: false matrix: pulp: - - "3.21" - "3.45" + - "3.81" steps: # Checks-out the repository under $GITHUB_WORKSPACE, so it's accessible to the job - uses: actions/checkout@v3 diff --git a/tests/pulp-in-one.sh b/tests/pulp-in-one.sh index be985c0..abc2db8 100755 --- a/tests/pulp-in-one.sh +++ b/tests/pulp-in-one.sh @@ -8,7 +8,7 @@ set -o pipefail mkdir -p settings -PULP_TAG=${PULP_TAG:-"3.45"} +PULP_TAG=${PULP_TAG:-"3.81"} cat << EOF > settings/settings.py CONTENT_ORIGIN='http://$(hostname):8080' From 766703e7595ad42f4aa91cb0699cfd8fd75abe09 Mon Sep 17 00:00:00 2001 From: Alex Welsh Date: Fri, 21 Nov 2025 14:13:40 +0000 Subject: [PATCH 04/13] dnm: test new lint version --- .github/workflows/lint-collection.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-collection.yml b/.github/workflows/lint-collection.yml index 1cda5e6..1d32579 100644 --- a/.github/workflows/lint-collection.yml +++ b/.github/workflows/lint-collection.yml @@ -4,4 +4,4 @@ name: Ansible collection linters pull_request: jobs: lint: - uses: stackhpc/.github/.github/workflows/lint-collection.yml@main + uses: stackhpc/.github/.github/workflows/lint-collection.yml@bump-ansible From c2a34137229cd93cb9e118c07186cf07167c9275 Mon Sep 17 00:00:00 2001 From: Alex Welsh Date: Fri, 21 Nov 2025 16:15:00 +0000 Subject: [PATCH 05/13] Rewrite pulp_container_content updated sqeezer --- galaxy.yml | 5 +- plugins/modules/pulp_container_content.py | 223 ------------------ roles/pulp_container_content/tasks/main.yml | 34 +-- .../tasks/process_content.yml | 132 +++++++++++ tests/test_container_content.yml | 12 +- 5 files changed, 143 insertions(+), 263 deletions(-) delete mode 100644 plugins/modules/pulp_container_content.py create mode 100644 roles/pulp_container_content/tasks/process_content.yml diff --git a/galaxy.yml b/galaxy.yml index f69ae2a..0cc7dd6 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -2,14 +2,15 @@ namespace: stackhpc name: pulp description: > Roles and plugins Pulp repository server configuration -version: "0.5.5" +version: "0.6.0" readme: "README.md" authors: - "Piotr Parczewski" - "MichaƂ Nasiadka" - "Mark Goddard" + - "Alex Welsh" dependencies: - "pulp.squeezer": "*" + "pulp.squeezer": ">=0.20.0" license: - "Apache-2.0" tags: diff --git a/plugins/modules/pulp_container_content.py b/plugins/modules/pulp_container_content.py deleted file mode 100644 index 9bf1887..0000000 --- a/plugins/modules/pulp_container_content.py +++ /dev/null @@ -1,223 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- - -# GNU General Public License v3.0+ (see LICENSE or https://www.gnu.org/licenses/gpl-3.0.txt) - -from __future__ import absolute_import, division, print_function - -__metaclass__ = type - - -DOCUMENTATION = r""" ---- -module: pulp_container_content -short_description: Manage container content of a pulp api server instance -description: - - "This performs CRUD operations on container content in a pulp api server instance." -options: - allow_missing: - description: - - Whether to allow missing tags when state is present. - type: bool - default: false - is_push: - description: - - Whether repository is a container-push repository. - type: bool - default: false - src_repo: - description: - - Name of the repository to copy content from when state is present. - type: str - src_is_push: - description: - - Whether src_repo is a container-push repository. - type: bool - default: false - repository: - description: - - Name of the repository to add or remove content - type: str - required: true - state: - description: - - State the entity should be in - type: str - default: present - choices: - - present - - absent - - read - tags: - description: - - List of tags to add or remove - type: list - elements: str - required: true - wait: - description: - - Whether to wait for completion of the operation - type: bool - default: true -extends_documentation_fragment: - - pulp.squeezer.pulp - - pulp.squeezer.pulp.entity_state -author: - - Mark Goddard (@markgoddard) -""" - -EXAMPLES = r""" -- name: Copy tag1 and tag2 from repo1 to repo2 - pulp_container_content: - pulp_url: https://pulp.example.org - username: admin - password: password - repository: repo2 - src_repo: repo1 - tags: - - tag1 - - tag2 - -- name: Remove tag3 from repo3 - pulp_container_content: - pulp_url: https://pulp.example.org - username: admin - password: password - repository: repo3 - tags: - - tag3 - state: absent -""" - -RETURN = r""" - repository_version: - description: Created container repository version - type: dict - returned: when content is added or removed -""" - - -from ansible_collections.pulp.squeezer.plugins.module_utils.pulp import ( - PAGE_LIMIT, - PulpContainerRepository, - PulpEntityAnsibleModule, - PulpTask, - SqueezerException, -) - - -class PulpContainerRepositoryContent(PulpContainerRepository): - _add_id = "repositories_container_container_add" - _remove_id = "repositories_container_container_remove" - _container_tags_list_id = "content_container_tags_list" - - _name_singular = "repository_version" - - def get_src_repo(self): - # Query source repository. - natural_key = {"name": self.module.params["src_repo"]} - repo = PulpContainerRepository(self.module, natural_key) - if self.module.params["state"] == "present" and self.module.params["src_is_push"]: - repo._list_id = "repositories_container_container_push_list" - # find populates repo.entity. - repo.find(failsafe=False) - return repo - - def get_content_units(self, repo): - # Query container tags with matching names in repo. - # Pagination code adapted from PulpEntity.list(). - tags = [] - offset = 0 - search_result = {"next": True} - while search_result["next"]: - parameters = { - "limit": PAGE_LIMIT, - "offset": offset, - "name__in": ",".join(self.module.params["tags"]), - "repository_version": repo.entity["latest_version_href"] - } - search_result = self.module.pulp_api.call( - self._container_tags_list_id, parameters=parameters - ) - tags.extend(search_result["results"]) - offset += PAGE_LIMIT - - tag_names = [tag["name"] for tag in tags] - if (self.module.params["state"] in ["present", "read"] and - not self.module.params["allow_missing"] and - len(tag_names) != len(self.module.params["tags"])): - missing = ", ".join(set(self.module.params["tags"]) - set(tag_names)) - raise SqueezerException(f"Some tags not found in source repository: {missing}") - return [result["pulp_href"] for result in tags] - - def add_or_remove(self, add_or_remove_id, content_units): - body = {"content_units": content_units} - if not self.module.check_mode: - parameters = {"container_container_repository_href": self.entity["pulp_href"]} - response = self.module.pulp_api.call( - add_or_remove_id, body=body, uploads=self.uploads, parameters=parameters - ) - if response and "task" in response: - if self.module.params["wait"]: - task = PulpTask(self.module, {"pulp_href": response["task"]}).wait_for() - # Adding or removing content results in creation of a new repository version - if task["created_resources"]: - self.entity = {"pulp_href": task["created_resources"][0]} - self.module.set_changed() - else: - self.entity = None - else: - self._name_singular = "task" - self.entity = {"pulp_href": response["task"]} - else: - self.entity = response - else: - # Assume changed in check mode - self.module.set_changed() - - def add(self): - src_repo = self.get_src_repo() - self.add_or_remove(self._add_id, self.get_content_units(src_repo)) - - def remove(self): - self.add_or_remove(self._remove_id, self.get_content_units(self)) - - def read(self): - self.get_content_units(self) - - def process(self): - if self.module.params["state"] == "read" and self.module.params["is_push"]: - self._list_id = "repositories_container_container_push_list" - # Populate self.entity. - self.find(failsafe=False) - if self.module.params["state"] == "present": - self.add() - elif self.module.params["state"] == "absent": - self.remove() - elif self.module.params["state"] == "read": - self.read() - else: - raise SqueezerException("Unexpected state") - self.module.set_result(self._name_singular, self.presentation(self.entity)) - - -def main(): - with PulpEntityAnsibleModule( - argument_spec=dict( - allow_missing={"type": "bool", "default": False}, - is_push={"type": "bool", "default": False}, - repository={"required": True}, - src_repo={}, - src_is_push={"type": "bool", "default": False}, - state={"default": "present", "choices": ["present", "absent", "read"]}, - tags={"type": "list", "elements": "str", "required": True}, - wait={"type": "bool", "default": True}, - ), - required_if=[("state", "present", ["src_repo"])], - ) as module: - natural_key = {"name": module.params["repository"]} - PulpContainerRepositoryContent(module, natural_key).process() - - -if __name__ == "__main__": - main() diff --git a/roles/pulp_container_content/tasks/main.yml b/roles/pulp_container_content/tasks/main.yml index 6f91868..88a89a0 100644 --- a/roles/pulp_container_content/tasks/main.yml +++ b/roles/pulp_container_content/tasks/main.yml @@ -1,36 +1,6 @@ --- - name: Add or remove content units - stackhpc.pulp.pulp_container_content: - pulp_url: "{{ pulp_url }}" - username: "{{ pulp_username }}" - password: "{{ pulp_password }}" - validate_certs: "{{ pulp_validate_certs | bool }}" - allow_missing: "{{ item.allow_missing | default(omit) }}" - is_push: "{{ item.is_push | default(omit) }}" - src_repo: "{{ item.src_repo | default(omit) }}" - src_is_push: "{{ item.src_is_push | default(omit) }}" - repository: "{{ item.repository }}" - tags: "{{ item.tags }}" - state: "{{ item.state | default(omit) }}" - wait: "{{ pulp_container_content_wait | bool }}" + include_tasks: process_content.yml loop: "{{ pulp_container_content }}" - register: pulp_container_content_result - -- name: Wait for tasks to complete - pulp.squeezer.task: - pulp_url: "{{ pulp_url }}" - username: "{{ pulp_username }}" - password: "{{ pulp_password }}" - validate_certs: "{{ pulp_validate_certs | bool }}" - pulp_href: "{{ content_result.task.pulp_href }}" - state: "completed" - loop: "{{ pulp_container_content }}" - when: - - not pulp_container_content_wait | bool - - "'task' in content_result" - changed_when: pulp_container_content_wait_result.task.created_resources | default([]) | length > 0 - register: pulp_container_content_wait_result loop_control: - index_var: result_index - vars: - content_result: "{{ pulp_container_content_result.results[result_index] }}" + loop_var: content_item diff --git a/roles/pulp_container_content/tasks/process_content.yml b/roles/pulp_container_content/tasks/process_content.yml new file mode 100644 index 0000000..3531d65 --- /dev/null +++ b/roles/pulp_container_content/tasks/process_content.yml @@ -0,0 +1,132 @@ +--- +- name: Get destination repository href + pulp.squeezer.api_call: + pulp_url: "{{ pulp_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + validate_certs: "{{ pulp_validate_certs | bool }}" + operation_id: repositories_container_container_list + parameters: + name: "{{ content_item.repository }}" + register: dest_repo_result + +- name: Fail if destination repository not found + fail: + msg: "Destination repository '{{ content_item.repository }}' not found." + when: dest_repo_result.response.count == 0 + +- name: Set destination repo href and ID + set_fact: + dest_repo_href: "{{ dest_repo_result.response.results[0].pulp_href }}" + # Extract the UUID from the HREF (assuming standard format .../uuid/) + dest_repo_id: "{{ dest_repo_result.response.results[0].pulp_href.strip('/').split('/')[-1] }}" + +- name: Get source repository info + pulp.squeezer.api_call: + pulp_url: "{{ pulp_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + validate_certs: "{{ pulp_validate_certs | bool }}" + operation_id: "{{ 'repositories_container_container_push_list' if content_item.src_is_push | default(false) else 'repositories_container_container_list' }}" + parameters: + name: "{{ content_item.src_repo }}" + register: src_repo_result + when: + - content_item.state | default('present') == 'present' + - content_item.src_repo is defined + +- name: Fail if source repository not found + fail: + msg: "Source repository '{{ content_item.src_repo }}' not found." + when: + - content_item.state | default('present') == 'present' + - content_item.src_repo is defined + - src_repo_result.response.count == 0 + +- name: Determine reference repository version + set_fact: + ref_repo_version: >- + {% if content_item.state | default('present') == 'present' %} + {{ src_repo_result.response.results[0].latest_version_href }} + {% else %} + {{ dest_repo_result.response.results[0].latest_version_href }} + {% endif %} + when: + - content_item.state | default('present') != 'present' or content_item.src_repo is defined + +- name: Resolve tags + block: + - name: List tags from reference repository + pulp.squeezer.api_call: + pulp_url: "{{ pulp_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + validate_certs: "{{ pulp_validate_certs | bool }}" + operation_id: content_container_tags_list + parameters: + name__in: "{{ content_item.tags }}" + repository_version: "{{ ref_repo_version }}" + register: tags_result + + - name: Check for missing tags + vars: + found_tags: "{{ tags_result.response.results | map(attribute='name') | list }}" + missing_tags: "{{ content_item.tags | difference(found_tags) }}" + fail: + msg: "Some tags not found in source repository: {{ missing_tags | join(', ') }}" + when: + - content_item.state | default('present') in ['present', 'read'] + - not content_item.allow_missing | default(false) + - missing_tags | length > 0 + + - name: Set content units to process + set_fact: + content_units: "{{ tags_result.response.results | map(attribute='pulp_href') | list }}" + when: ref_repo_version is defined + +- name: Add content to repository + pulp.squeezer.api_call: + pulp_url: "{{ pulp_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + validate_certs: "{{ pulp_validate_certs | bool }}" + operation_id: repositories_container_container_add + parameters: + container_container_repository_href: "{{ dest_repo_href }}" + body: + content_units: "{{ content_units }}" + register: add_result + when: + - content_item.state | default('present') == 'present' + - content_units | length > 0 + changed_when: true + +- name: Remove content from repository + pulp.squeezer.api_call: + pulp_url: "{{ pulp_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + validate_certs: "{{ pulp_validate_certs | bool }}" + operation_id: repositories_container_container_remove + parameters: + container_container_repository_href: "{{ dest_repo_href }}" + body: + content_units: "{{ content_units }}" + register: remove_result + when: + - content_item.state | default('present') == 'absent' + - content_units | length > 0 + changed_when: true + +- name: Wait for task completion + pulp.squeezer.task: + pulp_url: "{{ pulp_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + validate_certs: "{{ pulp_validate_certs | bool }}" + pulp_href: "{{ item.response.pulp_href }}" + state: completed + loop: "{{ [add_result, remove_result] }}" + when: + - item.changed | default(false) + - pulp_container_content_wait | bool diff --git a/tests/test_container_content.yml b/tests/test_container_content.yml index 47ed5fe..cc36ca5 100644 --- a/tests/test_container_content.yml +++ b/tests/test_container_content.yml @@ -182,14 +182,14 @@ state: present rescue: - set_fact: - failed_task: "{{ ansible_failed_task }}" + failed_msg: "{{ ansible_failed_result.msg }}" always: - name: Assert that adding a missing tag failed assert: that: - - failed_task.name == "Add or remove content units" + - failed_msg is search("Some tags not found in source repository") - set_fact: - failed_task: + failed_msg: # Repeat the above test with state=read - block: @@ -203,14 +203,14 @@ state: read rescue: - set_fact: - failed_task: "{{ ansible_failed_task }}" + failed_msg: "{{ ansible_failed_result.msg }}" always: - name: Assert that querying a missing tag failed assert: that: - - failed_task.name == "Add or remove content units" + - failed_msg is search("Some tags not found in source repository") - set_fact: - failed_task: + failed_msg: - include_role: name: pulp_repository From 05b0d8964fe7fe757a809b274812cbf61d4ebdaa Mon Sep 17 00:00:00 2001 From: Alex Welsh Date: Wed, 26 Nov 2025 17:11:08 +0000 Subject: [PATCH 06/13] Fix various tests --- .github/workflows/pull_request.yml | 2 +- tests/test_container_distribution.yml | 4 ++-- tests/test_container_repository.yml | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index a9bd194..6e679cf 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -27,7 +27,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install ansible==13.* jmespath pulp-glue==0.21.* + pip install ansible==13.* jmespath pulp-glue==0.33.* pulp-glue-deb==0.3.* ansible-galaxy collection install git+file://$(pwd) - name: Run Pulp in one diff --git a/tests/test_container_distribution.yml b/tests/test_container_distribution.yml index fc8fef0..69101d0 100644 --- a/tests/test_container_distribution.yml +++ b/tests/test_container_distribution.yml @@ -65,14 +65,14 @@ - dist_result.distribution.name == "test_container_distribution" - dist_result.distribution.base_path == "test_container_distribution" - dist_result.distribution.repository == repo_result.repository.pulp_href - - dist_result.distribution.repository_version is none + - dist_result.distribution.repository_version == "" - name: Verify distribution creation assert: that: - dist_version_1_result.distribution.name == "test_container_distribution_version_1" - dist_version_1_result.distribution.base_path == "test_container_distribution_version_1" - - dist_version_1_result.distribution.repository is none + - dist_version_1_result.distribution.repository == "" - dist_version_1_result.distribution.repository_version == repo_result.repository.latest_version_href - include_role: diff --git a/tests/test_container_repository.yml b/tests/test_container_repository.yml index bb0af06..f6d7aba 100644 --- a/tests/test_container_repository.yml +++ b/tests/test_container_repository.yml @@ -94,22 +94,22 @@ rescue: - set_fact: - failed_task: "{{ ansible_failed_task }}" + failed_result: "{{ ansible_failed_result }}" always: - name: Assert that syncing from a URL that returns 404 fails assert: that: - - failed_task.name == "Sync container remotes into repositories" + - failed_result.msg is search("One or more items failed") - name: Assert that syncing from a URL that returns 404 is retried the correct number of times assert: that: - - pulp_repository_container_repos_sync.results[0].attempts == pulp_repository_container_repos_sync_retries + - failed_result.results[0].attempts == pulp_repository_container_repos_sync_retries - include_role: name: pulp_repository vars: - pulp_repository_deb_repos: + pulp_repository_container_repos: - name: test_container_repo_bad_url state: absent From 2bb20ae53991c76d25c5603caa3a9106c941e2ad Mon Sep 17 00:00:00 2001 From: Alex Welsh Date: Thu, 27 Nov 2025 16:34:46 +0000 Subject: [PATCH 07/13] Revert to Ansible 11 --- .github/workflows/pull_request.yml | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 6e679cf..ce22402 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -27,7 +27,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install ansible==13.* jmespath pulp-glue==0.33.* pulp-glue-deb==0.3.* + pip install ansible==11.* jmespath pulp-glue==0.33.* pulp-glue-deb==0.3.* ansible-galaxy collection install git+file://$(pwd) - name: Run Pulp in one diff --git a/README.md b/README.md index 756f022..4dd0fb1 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ Note: Pulp server installation is out of this collection's scope - for this purp ## Tested with Ansible -Tested with the current Ansible 13 releases. +Tested with the current Ansible 11 release. ## Included content From a005536c465e3862ddf37030d7ea76fcdbe37b17 Mon Sep 17 00:00:00 2001 From: Alex Welsh Date: Thu, 27 Nov 2025 16:39:13 +0000 Subject: [PATCH 08/13] Pin to squeezer 0.2.3 --- galaxy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy.yml b/galaxy.yml index 0cc7dd6..1d1c5ff 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -10,7 +10,7 @@ authors: - "Mark Goddard" - "Alex Welsh" dependencies: - "pulp.squeezer": ">=0.20.0" + "pulp.squeezer": "0.2.3" license: - "Apache-2.0" tags: From be3578747bdeebda10367d07a198d160177fb489 Mon Sep 17 00:00:00 2001 From: Alex Welsh Date: Thu, 27 Nov 2025 16:43:49 +0000 Subject: [PATCH 09/13] Remove query var name --- tests/test_content_guard_rbac.yml | 50 +++++++++++++++---------------- tests/test_group.yml | 4 +-- tests/test_user.yml | 4 +-- 3 files changed, 29 insertions(+), 29 deletions(-) diff --git a/tests/test_content_guard_rbac.yml b/tests/test_content_guard_rbac.yml index 681b1af..82f5704 100644 --- a/tests/test_content_guard_rbac.yml +++ b/tests/test_content_guard_rbac.yml @@ -23,8 +23,8 @@ - name: Exit if version < 3.17 meta: end_play vars: - query: "[?component=='core'].version" - when: status_result.json.versions | json_query(query) | first is version('3.17', '<') + query_string: "[?component=='core'].version" + when: status_result.json.versions | json_query(query_string) | first is version('3.17', '<') - name: Query groups uri: @@ -84,31 +84,31 @@ - name: Evaluate results for test-rbac_cg-1 vars: - query: "[?name=='test-rbac_cg-1']" + query_string: "[?name=='test-rbac_cg-1']" assert: that: - - rbac_cg_list.json.results | json_query(query) | length == 1 - - (rbac_cg_list.json.results | json_query(query) | first).name == 'test-rbac_cg-1' - - (rbac_cg_list.json.results | json_query(query) | first).groups | length == 2 - - (rbac_cg_list.json.results | json_query(query) | first).groups[0].name in ['test_group1', 'test_group2'] - - (rbac_cg_list.json.results | json_query(query) | first).groups[1].name in ['test_group1', 'test_group2'] + - rbac_cg_list.json.results | json_query(query_string) | length == 1 + - (rbac_cg_list.json.results | json_query(query_string) | first).name == 'test-rbac_cg-1' + - (rbac_cg_list.json.results | json_query(query_string) | first).groups | length == 2 + - (rbac_cg_list.json.results | json_query(query_string) | first).groups[0].name in ['test_group1', 'test_group2'] + - (rbac_cg_list.json.results | json_query(query_string) | first).groups[1].name in ['test_group1', 'test_group2'] - > - (rbac_cg_list.json.results | json_query(query) | first).groups[0].name != - (rbac_cg_list.json.results | json_query(query) | first).groups[1].name + (rbac_cg_list.json.results | json_query(query_string) | first).groups[0].name != + (rbac_cg_list.json.results | json_query(query_string) | first).groups[1].name - name: Evaluate results for test-rbac_cg-2 vars: - query: "[?name=='test-rbac_cg-2']" + query_string: "[?name=='test-rbac_cg-2']" assert: that: - - rbac_cg_list.json.results | json_query(query) | length == 1 - - (rbac_cg_list.json.results | json_query(query) | first).name == 'test-rbac_cg-2' - - (rbac_cg_list.json.results | json_query(query) | first).groups | length == 2 - - (rbac_cg_list.json.results | json_query(query) | first).groups[0].name in ['test_group1', 'test_group2'] - - (rbac_cg_list.json.results | json_query(query) | first).groups[1].name in ['test_group1', 'test_group2'] + - rbac_cg_list.json.results | json_query(query_string) | length == 1 + - (rbac_cg_list.json.results | json_query(query_string) | first).name == 'test-rbac_cg-2' + - (rbac_cg_list.json.results | json_query(query_string) | first).groups | length == 2 + - (rbac_cg_list.json.results | json_query(query_string) | first).groups[0].name in ['test_group1', 'test_group2'] + - (rbac_cg_list.json.results | json_query(query_string) | first).groups[1].name in ['test_group1', 'test_group2'] - > - (rbac_cg_list.json.results | json_query(query) | first).groups[0].name != - (rbac_cg_list.json.results | json_query(query) | first).groups[1].name + (rbac_cg_list.json.results | json_query(query_string) | first).groups[0].name != + (rbac_cg_list.json.results | json_query(query_string) | first).groups[1].name # update content guards - include_role: @@ -136,21 +136,21 @@ - name: Evaluate results for test-rbac_cg-1 vars: - query: "[?name=='test-rbac_cg-1']" + query_string: "[?name=='test-rbac_cg-1']" assert: that: - - rbac_cg_list.json.results | json_query(query) | length == 1 - - (rbac_cg_list.json.results | json_query(query) | first).name == 'test-rbac_cg-1' - - (rbac_cg_list.json.results | json_query(query) | first).groups | length == 1 - - (rbac_cg_list.json.results | json_query(query) | first).groups[0].name == 'test_group1' + - rbac_cg_list.json.results | json_query(query_string) | length == 1 + - (rbac_cg_list.json.results | json_query(query_string) | first).name == 'test-rbac_cg-1' + - (rbac_cg_list.json.results | json_query(query_string) | first).groups | length == 1 + - (rbac_cg_list.json.results | json_query(query_string) | first).groups[0].name == 'test_group1' - name: Evaluate results for test-rbac_cg-2 vars: - query: "[?name=='test-rbac_cg-2']" + query_string: "[?name=='test-rbac_cg-2']" assert: that: - - rbac_cg_list.json.results | json_query(query) | length == 0 + - rbac_cg_list.json.results | json_query(query_string) | length == 0 - name: Cleanup include_role: diff --git a/tests/test_group.yml b/tests/test_group.yml index 3b8a379..5ccf77b 100644 --- a/tests/test_group.yml +++ b/tests/test_group.yml @@ -21,8 +21,8 @@ - name: Exit if version < 3.17 meta: end_play vars: - query: "[?component=='core'].version" - when: status_result.json.versions | json_query(query) | first is version('3.17', '<') + query_string: "[?component=='core'].version" + when: status_result.json.versions | json_query(query_string) | first is version('3.17', '<') - name: Query groups uri: diff --git a/tests/test_user.yml b/tests/test_user.yml index 54d28f2..73586f8 100644 --- a/tests/test_user.yml +++ b/tests/test_user.yml @@ -22,8 +22,8 @@ - name: Exit if version < 3.17 meta: end_play vars: - query: "[?component=='core'].version" - when: status_result.json.versions | json_query(query) | first is version('3.17', '<') + query_string: "[?component=='core'].version" + when: status_result.json.versions | json_query(query_string) | first is version('3.17', '<') - name: Query groups uri: From 83ddd5db899690c011fb013b9d495419ae735d62 Mon Sep 17 00:00:00 2001 From: Alex Welsh Date: Thu, 27 Nov 2025 16:51:52 +0000 Subject: [PATCH 10/13] make linter happy --- .ansible-lint | 4 ++++ roles/pulp_container_content/tasks/process_content.yml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.ansible-lint b/.ansible-lint index e697c81..0fbcd38 100644 --- a/.ansible-lint +++ b/.ansible-lint @@ -6,3 +6,7 @@ skip_list: - galaxy[version-incorrect] - meta-runtime[unsupported-version] - fqcn[action-core] + +# Ensure dependent collections are not linted +exclude_paths: + - .ansible/ diff --git a/roles/pulp_container_content/tasks/process_content.yml b/roles/pulp_container_content/tasks/process_content.yml index 3531d65..c604c65 100644 --- a/roles/pulp_container_content/tasks/process_content.yml +++ b/roles/pulp_container_content/tasks/process_content.yml @@ -55,6 +55,7 @@ - content_item.state | default('present') != 'present' or content_item.src_repo is defined - name: Resolve tags + when: ref_repo_version is defined block: - name: List tags from reference repository pulp.squeezer.api_call: @@ -82,7 +83,6 @@ - name: Set content units to process set_fact: content_units: "{{ tags_result.response.results | map(attribute='pulp_href') | list }}" - when: ref_repo_version is defined - name: Add content to repository pulp.squeezer.api_call: From f4aec06c10cda2d103159ff6752ef7049dee7660 Mon Sep 17 00:00:00 2001 From: Alex Welsh Date: Fri, 28 Nov 2025 12:23:44 +0000 Subject: [PATCH 11/13] Parallelise pulp_container_content --- roles/pulp_container_content/tasks/main.yml | 15 +++++++++++++++ .../tasks/process_content.yml | 14 +++++--------- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/roles/pulp_container_content/tasks/main.yml b/roles/pulp_container_content/tasks/main.yml index 88a89a0..e6ca370 100644 --- a/roles/pulp_container_content/tasks/main.yml +++ b/roles/pulp_container_content/tasks/main.yml @@ -1,6 +1,21 @@ --- +- name: Initialize active tasks list + set_fact: + pulp_container_active_tasks: [] + - name: Add or remove content units include_tasks: process_content.yml loop: "{{ pulp_container_content }}" loop_control: loop_var: content_item + +- name: Wait for tasks to complete + pulp.squeezer.task: + pulp_url: "{{ pulp_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + validate_certs: "{{ pulp_validate_certs | bool }}" + pulp_href: "{{ item }}" + state: completed + loop: "{{ pulp_container_active_tasks }}" + when: pulp_container_content_wait | bool diff --git a/roles/pulp_container_content/tasks/process_content.yml b/roles/pulp_container_content/tasks/process_content.yml index c604c65..81b5bdc 100644 --- a/roles/pulp_container_content/tasks/process_content.yml +++ b/roles/pulp_container_content/tasks/process_content.yml @@ -118,15 +118,11 @@ - content_units | length > 0 changed_when: true -- name: Wait for task completion - pulp.squeezer.task: - pulp_url: "{{ pulp_url }}" - username: "{{ pulp_username }}" - password: "{{ pulp_password }}" - validate_certs: "{{ pulp_validate_certs | bool }}" - pulp_href: "{{ item.response.pulp_href }}" - state: completed +- name: Register active tasks + set_fact: + pulp_container_active_tasks: "{{ pulp_container_active_tasks + [item.response.pulp_href] }}" loop: "{{ [add_result, remove_result] }}" when: + - item is defined - item.changed | default(false) - - pulp_container_content_wait | bool + - item.response.pulp_href is defined From b8c1eaaab73e48e6089e5dd35ef2b9bbdf142d6c Mon Sep 17 00:00:00 2001 From: Alex Welsh Date: Fri, 28 Nov 2025 16:20:39 +0000 Subject: [PATCH 12/13] Rename Ansible lint workflow --- .github/workflows/lint-collection.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-collection.yml b/.github/workflows/lint-collection.yml index 1d32579..8ea2410 100644 --- a/.github/workflows/lint-collection.yml +++ b/.github/workflows/lint-collection.yml @@ -4,4 +4,4 @@ name: Ansible collection linters pull_request: jobs: lint: - uses: stackhpc/.github/.github/workflows/lint-collection.yml@bump-ansible + uses: stackhpc/.github/.github/workflows/lint-collection.yml@ansible-2.18-2.20 From 83db0c48efa0d385d8f48a5863962c38759309fd Mon Sep 17 00:00:00 2001 From: Alex Welsh Date: Fri, 28 Nov 2025 17:21:45 +0000 Subject: [PATCH 13/13] testo --- roles/pulp_repository/tasks/container.yml | 190 +++++++++++++++++----- 1 file changed, 146 insertions(+), 44 deletions(-) diff --git a/roles/pulp_repository/tasks/container.yml b/roles/pulp_repository/tasks/container.yml index 821b39a..f956ead 100644 --- a/roles/pulp_repository/tasks/container.yml +++ b/roles/pulp_repository/tasks/container.yml @@ -1,19 +1,83 @@ --- +- name: List existing container repositories + pulp.squeezer.api_call: + pulp_url: "{{ pulp_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + validate_certs: "{{ pulp_validate_certs | bool }}" + operation_id: repositories_container_container_list + parameters: + limit: 10000 + fields: + - name + - pulp_href + register: existing_container_repos + +- name: List existing container remotes + pulp.squeezer.api_call: + pulp_url: "{{ pulp_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + validate_certs: "{{ pulp_validate_certs | bool }}" + operation_id: remotes_container_container_list + parameters: + limit: 10000 + fields: + - name + - pulp_href + register: existing_container_remotes + +- name: Build repository and remote lookup maps + set_fact: + # Map 'name' to 'pulp_href' for fast lookup + _existing_repo_map: "{{ dict(existing_container_repos.response.results | map(attribute='name') | zip(existing_container_repos.response.results | map(attribute='pulp_href'))) }}" + _existing_remote_map: "{{ dict(existing_container_remotes.response.results | map(attribute='name') | zip(existing_container_remotes.response.results | map(attribute='pulp_href'))) }}" + +- name: Calculate items to process + set_fact: + _repos_to_process: >- + {{ + pulp_repository_container_repos + | selectattr('state', 'defined') | selectattr('state', 'equalto', 'absent') + | selectattr('name', 'in', _existing_repo_map.keys()) | list + + + pulp_repository_container_repos + | selectattr('state', 'defined') | rejectattr('state', 'equalto', 'absent') + | rejectattr('name', 'in', _existing_repo_map.keys()) | list + + + pulp_repository_container_repos + | rejectattr('state', 'defined') + | rejectattr('name', 'in', _existing_repo_map.keys()) | list + }} + _remotes_to_process: >- + {{ + pulp_repository_container_repos + | selectattr('state', 'defined') | selectattr('state', 'equalto', 'absent') + | selectattr('name', 'in', _existing_remote_map.keys() | map('regex_replace', '-remote$', '') ) | list + + + pulp_repository_container_repos + | selectattr('url', 'defined') + | selectattr('state', 'defined') | rejectattr('state', 'equalto', 'absent') + | rejectattr('name', 'in', _existing_remote_map.keys() | map('regex_replace', '-remote$', '') ) | list + + + pulp_repository_container_repos + | selectattr('url', 'defined') + | rejectattr('state', 'defined') + | rejectattr('name', 'in', _existing_remote_map.keys() | map('regex_replace', '-remote$', '') ) | list + }} + - name: Setup container repositories pulp.squeezer.container_repository: pulp_url: "{{ pulp_url }}" username: "{{ pulp_username }}" password: "{{ pulp_password }}" validate_certs: "{{ pulp_validate_certs | bool }}" - name: "{{ pulp_repository_container_repos[repository_index].name }}" - state: "{{ pulp_repository_container_repos[repository_index].state }}" - loop: "{{ pulp_repository_container_repos | map(attribute='name') }}" + name: "{{ item.name }}" + state: "{{ item.state | default('present') }}" + loop: "{{ _repos_to_process }}" loop_control: - index_var: repository_index - register: pulp_repository_container_repositories - until: "pulp_repository_container_repositories is not failed" - retries: "{{ pulp_repository_container_repositories_retries }}" - delay: 1 + label: "{{ item.name }}" + register: pulp_repo_results - name: Setup container remotes pulp.squeezer.container_remote: @@ -21,49 +85,87 @@ username: "{{ pulp_username }}" password: "{{ pulp_password }}" validate_certs: "{{ pulp_validate_certs | bool }}" - name: "{{ pulp_repository_container_repos[repository_index].name }}-remote" - ca_cert: "{{ pulp_repository_container_repos[repository_index].ca_cert | default(omit) }}" - client_cert: "{{ pulp_repository_container_repos[repository_index].client_cert | default(omit) }}" - client_key: "{{ pulp_repository_container_repos[repository_index].client_key | default(omit) }}" - download_concurrency: "{{ pulp_repository_container_repos[repository_index].download_concurrency | default(omit) }}" - exclude_tags: "{{ pulp_repository_container_repos[repository_index].exclude_tags | default(omit) }}" - include_tags: "{{ pulp_repository_container_repos[repository_index].include_tags | default(omit) }}" - policy: "{{ pulp_repository_container_repos[repository_index].policy | default(omit) }}" - proxy_url: "{{ pulp_repository_container_repos[repository_index].proxy_url | default(omit) }}" - proxy_username: "{{ pulp_repository_container_repos[repository_index].proxy_username | default(omit) }}" - proxy_password: "{{ pulp_repository_container_repos[repository_index].proxy_password | default(omit) }}" - remote_username: "{{ pulp_repository_container_repos[repository_index].remote_username | default(omit) }}" - remote_password: "{{ pulp_repository_container_repos[repository_index].remote_password | default(omit) }}" - tls_validation: "{{ pulp_repository_container_repos[repository_index].tls_validation | default(omit) }}" - upstream_name: "{{ pulp_repository_container_repos[repository_index].upstream_name | default(pulp_repository_container_repos[repository_index].name) }}" - url: "{{ pulp_repository_container_repos[repository_index].url | default(omit) }}" - state: "{{ pulp_repository_container_repos[repository_index].state }}" - when: > - pulp_repository_container_repos[repository_index].state == "absent" or - pulp_repository_container_repos[repository_index].url is defined - loop: "{{ pulp_repository_container_repos | map(attribute='name') }}" + name: "{{ item.name }}-remote" + ca_cert: "{{ item.ca_cert | default(omit) }}" + client_cert: "{{ item.client_cert | default(omit) }}" + client_key: "{{ item.client_key | default(omit) }}" + download_concurrency: "{{ item.download_concurrency | default(omit) }}" + exclude_tags: "{{ item.exclude_tags | default(omit) }}" + include_tags: "{{ item.include_tags | default(omit) }}" + policy: "{{ item.policy | default(omit) }}" + proxy_url: "{{ item.proxy_url | default(omit) }}" + proxy_username: "{{ item.proxy_username | default(omit) }}" + proxy_password: "{{ item.proxy_password | default(omit) }}" + remote_username: "{{ item.remote_username | default(omit) }}" + remote_password: "{{ item.remote_password | default(omit) }}" + tls_validation: "{{ item.tls_validation | default(omit) }}" + upstream_name: "{{ item.upstream_name | default(item.name) }}" + url: "{{ item.url | default(omit) }}" + state: "{{ item.state | default('present') }}" + loop: "{{ _remotes_to_process }}" loop_control: - index_var: repository_index - register: pulp_repository_container_remotes - until: "pulp_repository_container_remotes is not failed" - retries: "{{ pulp_repository_container_remotes_retries }}" - delay: 1 + label: "{{ item.name }}" + register: pulp_remote_results + +- name: Update lookup maps with new items + set_fact: + _all_repo_map: >- + {{ + _existing_repo_map | combine( + dict(pulp_repo_results.results | selectattr('repository', 'defined') | map(attribute='item.name') | zip(pulp_repo_results.results | selectattr('repository', 'defined') | map(attribute='repository.pulp_href'))) + ) + }} + _all_remote_map: >- + {{ + _existing_remote_map | combine( + dict(pulp_remote_results.results | selectattr('remote', 'defined') | map(attribute='item.name') | map('regex_replace', '$', '-remote') | zip(pulp_remote_results.results | selectattr('remote', 'defined') | map(attribute='remote.pulp_href'))) + ) + }} -- name: Sync container remotes into repositories - pulp.squeezer.container_sync: +- name: Initialize container sync tasks list + set_fact: + pulp_container_sync_tasks: [] + +- name: Trigger container repository syncs + pulp.squeezer.api_call: pulp_url: "{{ pulp_url }}" username: "{{ pulp_username }}" password: "{{ pulp_password }}" validate_certs: "{{ pulp_validate_certs | bool }}" - repository: "{{ pulp_repository_container_repos[repository_index].name }}" - remote: "{{ pulp_repository_container_repos[repository_index].name }}-remote" + operation_id: repositories_container_container_sync + parameters: + container_container_repository_href: "{{ _all_repo_map[item.name] }}" + body: + remote: "{{ _all_remote_map[item.name + '-remote'] }}" when: - - pulp_repository_container_repos[repository_index].url is defined - - pulp_repository_container_repos[repository_index].state == "present" - loop: "{{ pulp_repository_container_repos | map(attribute='name') }}" + - item.url is defined + - item.state | default('present') == 'present' + - item.name in _all_repo_map + - (item.name + '-remote') in _all_remote_map + loop: "{{ pulp_repository_container_repos }}" loop_control: - index_var: repository_index - register: pulp_repository_container_repos_sync - until: "pulp_repository_container_repos_sync is not failed" + label: "{{ item.name }}" + register: pulp_container_sync_trigger + changed_when: true + +- name: Collect container sync task HREFs + set_fact: + pulp_container_sync_tasks: "{{ pulp_container_sync_tasks + [item.response.pulp_href] }}" + loop: "{{ pulp_container_sync_trigger.results }}" + when: + - item.skipped is not defined or not item.skipped + - item.response.pulp_href is defined + +- name: Wait for container sync tasks to complete + pulp.squeezer.task: + pulp_url: "{{ pulp_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + validate_certs: "{{ pulp_validate_certs | bool }}" + pulp_href: "{{ item }}" + state: completed + loop: "{{ pulp_container_sync_tasks }}" + register: pulp_container_sync_results + until: "pulp_container_sync_results is not failed" retries: "{{ pulp_repository_container_repos_sync_retries }}" delay: 1