diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..6a2a3c77 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,8 @@ + include LICENSE + include README.md + include pyproject.toml + include setup.py + recursive-include charon *.py *.json + recursive-include tests *.py *.txt *.tgz *.zip *.json *.sha1 + exclude .github .gitignore + diff --git a/README.md b/README.md index c0cb8377..7b9e9cd3 100644 --- a/README.md +++ b/README.md @@ -53,12 +53,14 @@ to configure AWS access credentials. ### charon-upload: upload a repo to S3 ```bash -usage: charon upload $tarball --product/-p ${prod} --version/-v ${ver} [--root_path] [--ignore_patterns] [--debug] [--contain_signature] [--key] +usage: charon upload $tarball [$tarball*] --product/-p ${prod} --version/-v ${ver} [--root_path] [--ignore_patterns] [--debug] [--contain_signature] [--key] ``` This command will upload the repo in tarball to S3. It will auto-detect if the tarball is for maven or npm +**New in 1.3.5**: For Maven archives, this command now supports uploading multiple zip files at once. When multiple Maven zips are provided, they will be merged intelligently, including proper handling of archetype catalog files and duplicate artifact detection. + * For maven type, it will: * Scan the tarball for all paths and collect them all. @@ -99,11 +101,13 @@ This command will delete some paths from repo in S3. ### charon-index: refresh the index.html for the specified path ```bash -usage: charon index $PATH [-t, --target] [-D, --debug] [-q, --quiet] +usage: charon index $PATH [-t, --target] [-D, --debug] [-q, --quiet] [--recursive] ``` This command will refresh the index.html for the specified path. +**New in 1.3.5**: Added `--recursive` flag to support recursive indexing under the specified path. + * Note that if the path is a NPM metadata path which contains package.json, this refreshment will not work because this type of folder will display the package.json instead of the index.html in http request. ### charon-cf-check: check the invalidation status of the specified invalidation id for AWS CloudFront diff --git a/charon.spec b/charon.spec index fb9f2073..d1ea5170 100644 --- a/charon.spec +++ b/charon.spec @@ -64,22 +64,28 @@ export LANG=en_US.UTF-8 LANGUAGE=en_US.en LC_ALL=en_US.UTF-8 %changelog +* Wed Oct 29 2025 Gang Li +- 1.3.5 release +- Support recursive indexing for index function +- Accept multiple maven zips for uploading + * Fri Jun 27 2025 Gang Li - 1.4.0 release - Add RADAS signature support * Mon Jun 23 2025 Gang Li - 1.3.4 release -- Fix the sorting problem of index page items +- Add --version flag to support version check +- Bug fix: MMENG-4362 re-sort the indexing page items +- Add pyproject.toml * Mon Dec 16 2024 Gang Li - 1.3.3 release -- Fix npm del error when deleting a package which has overlapped name with others -- Some code refinement +- Bug fix: MMENG-4284 npm del error when deleting a package which has overlapped name with others -* Thu Jul 11 2024 Gang Li +* Wed Jul 10 2024 Gang Li - 1.3.2 release -- Some updates in the Containerfile. +- Container file update * Tue May 7 2024 Gang Li - 1.3.1 release diff --git a/charon/cache.py b/charon/cache.py index 5b8d1227..3d216b60 100644 --- a/charon/cache.py +++ b/charon/cache.py @@ -86,8 +86,8 @@ def invalidate_paths( The default value is 3000 which is the maximum number in official doc: https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/Invalidation.html#InvalidationLimits """ - INPRO_W_SECS = 5 - NEXT_W_SECS = 1 + INPRO_W_SECS = 10 + NEXT_W_SECS = 2 real_paths = [paths] # Split paths into batches by batch_size if batch_size: diff --git a/charon/cmd/__init__.py b/charon/cmd/__init__.py index 985d7f79..f3d027b9 100644 --- a/charon/cmd/__init__.py +++ b/charon/cmd/__init__.py @@ -20,6 +20,7 @@ from charon.cmd.cmd_checksum import init_checksum, checksum from charon.cmd.cmd_cache import init_cf, cf from charon.cmd.cmd_sign import sign +from charon.cmd.cmd_merge import merge @group() @@ -47,3 +48,6 @@ def cli(ctx): # radas sign cmd cli.add_command(sign) + +# maven zips merge cmd +cli.add_command(merge) diff --git a/charon/cmd/cmd_index.py b/charon/cmd/cmd_index.py index e5dd11a5..7d4c07a6 100644 --- a/charon/cmd/cmd_index.py +++ b/charon/cmd/cmd_index.py @@ -42,6 +42,13 @@ """, required=True ) +@option( + "--recursive", + "-r", + help="If do indexing recursively under $path", + is_flag=True, + default=False +) @option( "--config", "-c", @@ -69,6 +76,7 @@ def index( path: str, target: str, + recursive: bool = False, config: str = None, debug: bool = False, quiet: bool = False, @@ -120,7 +128,15 @@ def index( if not aws_bucket: logger.error("No bucket specified for target %s!", target) else: - re_index(b, path, package_type, aws_profile, dryrun) + args = { + "target": b, + "path": path, + "package_type": package_type, + "aws_profile": aws_profile, + "recursive": recursive, + "dry_run": dryrun + } + re_index(**args) # type: ignore except Exception: print(traceback.format_exc()) diff --git a/charon/cmd/cmd_merge.py b/charon/cmd/cmd_merge.py new file mode 100644 index 00000000..3371ef74 --- /dev/null +++ b/charon/cmd/cmd_merge.py @@ -0,0 +1,156 @@ +""" +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from typing import List + +from charon.utils.archive import detect_npm_archives, NpmArchiveType +from charon.cmd.internal import _get_local_repos, _decide_mode +from charon.pkgs.maven import _extract_tarballs +from click import command, option, argument +from zipfile import ZipFile, ZIP_DEFLATED +from tempfile import mkdtemp + +import logging +import os +import sys + +logger = logging.getLogger(__name__) + + +@argument( + "repos", + type=str, + nargs=-1 # This allows multiple arguments for zip urls +) +@option( + "--product", + "-p", + help=""" + The product key, will combine with version to decide + the metadata of the files in tarball. + """, + nargs=1, + required=True, + multiple=False, +) +@option( + "--version", + "-v", + help=""" + The product version, will combine with key to decide + the metadata of the files in tarball. + """, + required=True, + multiple=False, +) +@option( + "--root_path", + "-r", + default="maven-repository", + help=""" + The root path in the tarball before the real maven paths, + will be trailing off before uploading. + """, +) +@option( + "--work_dir", + "-w", + help=""" + The temporary working directory into which archives should + be extracted, when needed. + """, +) +@option( + "--merge_result", + "-m", + help=""" + The path of the final merged zip file will be compressed and saved. + Default is the ZIP file which is created in a temporary directory based on work_dir. + e.g. /tmp/work/jboss-eap-8.1.0_merged_a1b2c3/jboss-eap-8.1.0_merged.zip + """, +) +@option( + "--debug", + "-D", + help="Debug mode, will print all debug logs for problem tracking.", + is_flag=True, + default=False +) +@option( + "--quiet", + "-q", + help="Quiet mode, will shrink most of the logs except warning and errors.", + is_flag=True, + default=False +) +@command() +def merge( + repos: List[str], + product: str, + version: str, + root_path="maven-repository", + work_dir=None, + merge_result=None, + debug=False, + quiet=False +): + """Merge multiple Maven ZIP archives and compress the result into a single ZIP file. + The merged file is stored locally as specified by merge_result. + + Note: This function does not support merging single archive, NPM archives, + or archives of inconsistent types. + """ + _decide_mode(product, version, is_quiet=quiet, is_debug=debug) + if len(repos) == 1: + logger.info("Skip merge step, single archive detected, no merge needed") + sys.exit(0) + + product_key = f"{product}-{version}" + archive_paths = _get_local_repos(repos) + archive_types = detect_npm_archives(archive_paths) + + maven_count = archive_types.count(NpmArchiveType.NOT_NPM) + npm_count = len(archive_types) - maven_count + if maven_count == len(archive_types): + tmp_root = _extract_tarballs(archive_paths, root_path, product_key, dir__=work_dir) + _create_merged_zip(tmp_root, merge_result, product_key, work_dir) + elif npm_count == len(archive_types): + logger.error("Skip merge step for the npm archives") + sys.exit(1) + else: + logger.error("Skip merge step since the types are not consistent") + sys.exit(1) + + +def _create_merged_zip( + root_path: str, + merge_result: str, + product_key: str, + work_dir: str +): + zip_path = merge_result + if not merge_result: + merge_path = mkdtemp(prefix=f"{product_key}_merged_", dir=work_dir) + zip_path = os.path.join(merge_path, f"{product_key}_merged.zip") + + # pylint: disable=unused-variable + with ZipFile(zip_path, 'w', ZIP_DEFLATED) as zipf: + for root, dirs, files in os.walk(root_path): + for file in files: + file_path = os.path.join(root, file) + # Calculate relative path to preserve directory structure + arcname = os.path.relpath(file_path, root_path) + zipf.write(file_path, arcname) + logger.info("Done for the merged zip generation: %s", zip_path) diff --git a/charon/cmd/cmd_upload.py b/charon/cmd/cmd_upload.py index 3a0e6990..17bf2aa5 100644 --- a/charon/cmd/cmd_upload.py +++ b/charon/cmd/cmd_upload.py @@ -16,12 +16,12 @@ from typing import List from charon.config import get_config -from charon.utils.archive import detect_npm_archive, NpmArchiveType +from charon.utils.archive import detect_npm_archives, NpmArchiveType from charon.pkgs.maven import handle_maven_uploading from charon.pkgs.npm import handle_npm_uploading from charon.cmd.internal import ( _decide_mode, _validate_prod_key, - _get_local_repo, _get_targets, + _get_local_repos, _get_targets, _get_ignore_patterns, _safe_delete ) from click import command, option, argument @@ -35,8 +35,9 @@ @argument( - "repo", + "repos", type=str, + nargs=-1 # This allows multiple arguments for zip urls ) @option( "--product", @@ -146,7 +147,7 @@ ) @command() def upload( - repo: str, + repos: List[str], product: str, version: str, targets: List[str], @@ -161,9 +162,10 @@ def upload( dryrun=False, sign_result_file=None, ): - """Upload all files from a released product REPO to Ronda - Service. The REPO points to a product released tarball which - is hosted in a remote url or a local path. + """Upload all files from released product REPOs to Ronda + Service. The REPOs point to a product released tarballs which + are hosted in remote urls or local paths. + Notes: It does not support multiple repos for NPM archives """ tmp_dir = work_dir try: @@ -182,8 +184,8 @@ def upload( logger.error("No AWS profile specified!") sys.exit(1) - archive_path = _get_local_repo(repo) - npm_archive_type = detect_npm_archive(archive_path) + archive_paths = _get_local_repos(repos) + archive_types = detect_npm_archives(archive_paths) product_key = f"{product}-{version}" manifest_bucket_name = conf.get_manifest_bucket() targets_ = _get_targets(targets, conf) @@ -194,23 +196,10 @@ def upload( " are set correctly.", targets_ ) sys.exit(1) - if npm_archive_type != NpmArchiveType.NOT_NPM: - logger.info("This is a npm archive") - tmp_dir, succeeded = handle_npm_uploading( - archive_path, - product_key, - targets=targets_, - aws_profile=aws_profile, - dir_=work_dir, - gen_sign=contain_signature, - cf_enable=conf.is_aws_cf_enable(), - key=sign_key, - dry_run=dryrun, - manifest_bucket_name=manifest_bucket_name - ) - if not succeeded: - sys.exit(1) - else: + + maven_count = archive_types.count(NpmArchiveType.NOT_NPM) + npm_count = len(archive_types) - maven_count + if maven_count == len(archive_types): ignore_patterns_list = None if ignore_patterns: ignore_patterns_list = ignore_patterns @@ -218,7 +207,7 @@ def upload( ignore_patterns_list = _get_ignore_patterns(conf) logger.info("This is a maven archive") tmp_dir, succeeded = handle_maven_uploading( - archive_path, + archive_paths, product_key, ignore_patterns_list, root=root_path, @@ -235,6 +224,28 @@ def upload( ) if not succeeded: sys.exit(1) + elif npm_count == len(archive_types) and len(archive_types) == 1: + logger.info("This is a npm archive") + tmp_dir, succeeded = handle_npm_uploading( + archive_paths[0], + product_key, + targets=targets_, + aws_profile=aws_profile, + dir_=work_dir, + gen_sign=contain_signature, + cf_enable=conf.is_aws_cf_enable(), + key=sign_key, + dry_run=dryrun, + manifest_bucket_name=manifest_bucket_name + ) + if not succeeded: + sys.exit(1) + elif npm_count == len(archive_types) and len(archive_types) > 1: + logger.error("Doesn't support multiple upload for npm") + sys.exit(1) + else: + logger.error("Upload types are not consistent") + sys.exit(1) except Exception: print(traceback.format_exc()) sys.exit(2) # distinguish between exception and bad config or bad state diff --git a/charon/cmd/internal.py b/charon/cmd/internal.py index e7e7d14a..89d4ea1b 100644 --- a/charon/cmd/internal.py +++ b/charon/cmd/internal.py @@ -75,6 +75,14 @@ def _get_local_repo(url: str) -> str: return archive_path +def _get_local_repos(urls: list) -> list: + archive_paths = [] + for url in urls: + archive_path = _get_local_repo(url) + archive_paths.append(archive_path) + return archive_paths + + def _validate_prod_key(product: str, version: str) -> bool: if not product or product.strip() == "": logger.error("Error: product can not be empty!") diff --git a/charon/pkgs/checksum_http.py b/charon/pkgs/checksum_http.py index e57dab34..e30a373e 100644 --- a/charon/pkgs/checksum_http.py +++ b/charon/pkgs/checksum_http.py @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. """ -from charon.utils.files import digest, HashType +from charon.utils.files import digest, HashType, overwrite_file from charon.storage import S3Client from typing import Tuple, List, Dict, Optional from html.parser import HTMLParser @@ -169,9 +169,10 @@ def _check_and_remove_file(file_name: str): def _write_one_col_file(items: List[str], file_name: str): if items and len(items) > 0: _check_and_remove_file(file_name) - with open(file_name, "w") as f: - for i in items: - f.write(i + "\n") + content = "" + for i in items: + content = content + i + "\n" + overwrite_file(file_name, content) logger.info("The report file %s is generated.", file_name) _write_one_col_file(content[0], os.path.join(work_dir, "mismatched_files.csv")) @@ -180,10 +181,9 @@ def _write_one_col_file(items: List[str], file_name: str): if content[2] and len(content[2]) > 0: error_file = os.path.join(work_dir, "error_files.csv") _check_and_remove_file(error_file) - with open(error_file, "w") as f: - f.write("path,error\n") - for d in content[2]: - f.write("{path},{error}\n".format(path=d["path"], error=d["error"])) + f_content_lines: List[str] = [] + f_content = "path,error\n" + "\n".join(f_content_lines) + overwrite_file(error_file, f_content) logger.info("The report file %s is generated.", error_file) diff --git a/charon/pkgs/indexing.py b/charon/pkgs/indexing.py index 4710cdab..6794a478 100644 --- a/charon/pkgs/indexing.py +++ b/charon/pkgs/indexing.py @@ -19,7 +19,7 @@ # from charon.pkgs.pkg_utils import invalidate_cf_paths from charon.constants import (INDEX_HTML_TEMPLATE, NPM_INDEX_HTML_TEMPLATE, PACKAGE_TYPE_MAVEN, PACKAGE_TYPE_NPM, PROD_INFO_SUFFIX) -from charon.utils.files import digest_content +from charon.utils.files import digest_content, overwrite_file from jinja2 import Template import os import logging @@ -155,8 +155,7 @@ def __to_html(package_type: str, contents: List[str], folder: str, top_level: st if folder == "/": html_path = os.path.join(top_level, "index.html") os.makedirs(os.path.dirname(html_path), exist_ok=True) - with open(html_path, 'w', encoding='utf-8') as html: - html.write(html_content) + overwrite_file(html_path, html_content) return html_path @@ -267,7 +266,7 @@ def re_index( path: str, package_type: str, aws_profile: str = None, - # cf_enable: bool = False, + recursive: bool = False, dry_run: bool = False ): """Refresh the index.html for the specified folder in the bucket. @@ -307,6 +306,7 @@ def re_index( logger.debug("The re-indexed page content: %s", index_content) if not dry_run: index_path = os.path.join(path, "index.html") + logger.info("Start re-indexing %s in bucket %s", index_path, bucket_name) if path == "/": index_path = "index.html" s3_client.simple_delete_file(index_path, (bucket_name, real_prefix)) @@ -314,10 +314,23 @@ def re_index( index_path, index_content, (bucket_name, real_prefix), "text/html", digest_content(index_content) ) - # We will not invalidate index.html per cost consideration - # if cf_enable: - # cf_client = CFClient(aws_profile=aws_profile) - # invalidate_cf_paths(cf_client, bucket, [index_path]) + logger.info("%s re-indexing finished", index_path) + if recursive: + for c in contents: + if c.endswith("/"): + sub_path = c.removeprefix(real_prefix).strip() + if sub_path.startswith("/"): + sub_path = sub_path.removeprefix("/") + logger.debug("subpath: %s", sub_path) + args = { + "target": target, + "path": sub_path, + "package_type": package_type, + "aws_profile": aws_profile, + "recursive": recursive, + "dry_run": dry_run + } + re_index(**args) # type: ignore else: logger.warning( "The path %s does not contain any contents in bucket %s. " diff --git a/charon/pkgs/maven.py b/charon/pkgs/maven.py index 5ccee694..6b047f27 100644 --- a/charon/pkgs/maven.py +++ b/charon/pkgs/maven.py @@ -33,11 +33,12 @@ META_FILE_FAILED, MAVEN_METADATA_TEMPLATE, ARCHETYPE_CATALOG_TEMPLATE, ARCHETYPE_CATALOG_FILENAME, PACKAGE_TYPE_MAVEN) -from typing import Dict, List, Tuple +from typing import Dict, List, Tuple, Union from jinja2 import Template from datetime import datetime from zipfile import ZipFile, BadZipFile from tempfile import mkdtemp +from shutil import rmtree, copy2 from defusedxml import ElementTree import os @@ -217,7 +218,8 @@ def parse_gavs(pom_paths: List[str], root="/") -> Dict[str, Dict[str, List[str]] return gavs -def gen_meta_file(group_id, artifact_id: str, versions: list, root="/", digest=True) -> List[str]: +def gen_meta_file(group_id, artifact_id: str, + versions: list, root="/", do_digest=True) -> List[str]: content = MavenMetadata( group_id, artifact_id, versions ).generate_meta_file_content() @@ -229,7 +231,7 @@ def gen_meta_file(group_id, artifact_id: str, versions: list, root="/", digest=T meta_files.append(final_meta_path) except FileNotFoundError as e: raise e - if digest: + if do_digest: meta_files.extend(__gen_all_digest_files(final_meta_path)) return meta_files @@ -262,7 +264,7 @@ def __gen_digest_file(hash_file_path, meta_file_path: str, hashtype: HashType) - def handle_maven_uploading( - repo: str, + repos: List[str], prod_key: str, ignore_patterns=None, root="maven-repository", @@ -296,8 +298,9 @@ def handle_maven_uploading( """ if targets is None: targets = [] - # 1. extract tarball - tmp_root = _extract_tarball(repo, prod_key, dir__=dir_) + + # 1. extract tarballs + tmp_root = _extract_tarballs(repos, root, prod_key, dir__=dir_) # 2. scan for paths and filter out the ignored paths, # and also collect poms for later metadata generation @@ -702,6 +705,195 @@ def _extract_tarball(repo: str, prefix="", dir__=None) -> str: sys.exit(1) +def _extract_tarballs(repos: List[str], root: str, prefix="", dir__=None) -> str: + """ Extract multiple zip archives to a temporary directory. + * repos are the list of repo paths to extract + * root is a prefix in the tarball to identify which path is + the beginning of the maven GAV path + * prefix is the prefix for temporary directory name + * dir__ is the directory where temporary directories will be created. + + Returns the path to the merged temporary directory containing all extracted files + """ + # Create final merge directory + final_tmp_root = mkdtemp(prefix=f"charon-{prefix}-final-", dir=dir__) + + total_copied = 0 + total_duplicated = 0 + total_merged = 0 + total_processed = 0 + + # Collect all extracted directories first + extracted_dirs = [] + + for repo in repos: + if os.path.exists(repo): + try: + logger.info("Extracting tarball %s", repo) + repo_zip = ZipFile(repo) + tmp_root = mkdtemp(prefix=f"charon-{prefix}-", dir=dir__) + extract_zip_all(repo_zip, tmp_root) + extracted_dirs.append(tmp_root) + + except BadZipFile as e: + logger.error("Tarball extraction error for repo %s: %s", repo, e) + sys.exit(1) + else: + logger.error("Error: archive %s does not exist", repo) + sys.exit(1) + + # Merge all extracted directories + if extracted_dirs: + # Create merged directory name + merged_dir_name = "merged_repositories" + merged_dest_dir = os.path.join(final_tmp_root, merged_dir_name) + + # Merge content from all extracted directories + for extracted_dir in extracted_dirs: + copied, duplicated, merged, processed = _merge_directories_with_rename( + extracted_dir, merged_dest_dir, root + ) + total_copied += copied + total_duplicated += duplicated + total_merged += merged + total_processed += processed + + # Clean up temporary extraction directory + rmtree(extracted_dir) + + logger.info( + "All zips merged! Total copied: %s, Total duplicated: %s, " + "Total merged: %s, Total processed: %s", + total_copied, + total_duplicated, + total_merged, + total_processed, + ) + return final_tmp_root + + +def _merge_directories_with_rename(src_dir: str, dest_dir: str, root: str): + """ Recursively copy files from src_dir to dest_dir, overwriting existing files. + * src_dir is the source directory to copy from + * dest_dir is the destination directory to copy to. + + Returns Tuple of (copied_count, duplicated_count, merged_count, processed_count) + """ + copied_count = 0 + duplicated_count = 0 + merged_count = 0 + processed_count = 0 + + # Find the actual content directory + content_root = src_dir + for item in os.listdir(src_dir): + item_path = os.path.join(src_dir, item) + # Check the root maven-repository subdirectory existence + maven_repo_path = os.path.join(item_path, root) + if os.path.isdir(item_path) and os.path.exists(maven_repo_path): + content_root = item_path + break + + # pylint: disable=unused-variable + for root_dir, dirs, files in os.walk(content_root): + # Calculate relative path from content root + rel_path = os.path.relpath(root_dir, content_root) + dest_root = os.path.join(dest_dir, rel_path) if rel_path != '.' else dest_dir + + # Create destination directory if it doesn't exist + os.makedirs(dest_root, exist_ok=True) + + # Copy all files, skip existing ones + for file in files: + src_file = os.path.join(root_dir, file) + dest_file = os.path.join(dest_root, file) + + if file == ARCHETYPE_CATALOG_FILENAME: + _handle_archetype_catalog_merge(src_file, dest_file) + merged_count += 1 + logger.debug("Merged archetype catalog: %s -> %s", src_file, dest_file) + elif os.path.exists(dest_file): + duplicated_count += 1 + logger.debug("Duplicated: %s, skipped", dest_file) + else: + copy2(src_file, dest_file) + copied_count += 1 + logger.debug("Copied: %s -> %s", src_file, dest_file) + + processed_count += 1 + + logger.info( + "One zip merged! Files copied: %s, Files duplicated: %s, " + "Files merged: %s, Total files processed: %s", + copied_count, + duplicated_count, + merged_count, + processed_count, + ) + return copied_count, duplicated_count, merged_count, processed_count + + +def _handle_archetype_catalog_merge(src_catalog: str, dest_catalog: str): + """ + Handle merging of archetype-catalog.xml files during directory merge. + + Args: + src_catalog: Source archetype-catalog.xml file path + dest_catalog: Destination archetype-catalog.xml file path + """ + try: + with open(src_catalog, "rb") as sf: + src_archetypes = _parse_archetypes(sf.read()) + except ElementTree.ParseError as e: + logger.warning("Failed to read source archetype catalog %s: %s", src_catalog, e) + return + + if len(src_archetypes) < 1: + logger.warning( + "No archetypes found in source archetype-catalog.xml: %s, " + "even though the file exists! Skipping.", + src_catalog + ) + return + + # Copy directly if dest_catalog doesn't exist + if not os.path.exists(dest_catalog): + copy2(src_catalog, dest_catalog) + return + + try: + with open(dest_catalog, "rb") as df: + dest_archetypes = _parse_archetypes(df.read()) + except ElementTree.ParseError as e: + logger.warning("Failed to read dest archetype catalog %s: %s", dest_catalog, e) + return + + if len(dest_archetypes) < 1: + logger.warning( + "No archetypes found in dest archetype-catalog.xml: %s, " + "even though the file exists! Copy directly from the src_catalog, %s.", + dest_catalog, src_catalog + ) + copy2(src_catalog, dest_catalog) + return + + else: + original_dest_size = len(dest_archetypes) + for sa in src_archetypes: + if sa not in dest_archetypes: + dest_archetypes.append(sa) + else: + logger.debug("DUPLICATE ARCHETYPE: %s", sa) + + if len(dest_archetypes) != original_dest_size: + content = MavenArchetypeCatalog(dest_archetypes).generate_meta_file_content() + try: + overwrite_file(dest_catalog, content) + except Exception as e: + logger.error("Failed to merge archetype catalog: %s", dest_catalog) + raise e + + def _scan_paths(files_root: str, ignore_patterns: List[str], root: str) -> Tuple[str, List[str], List[str], List[str]]: # 2. scan for paths and filter out the ignored paths, @@ -870,17 +1062,16 @@ def _generate_rollback_archetype_catalog( else: # Re-render the result of our archetype un-merge to the # local file, in preparation for upload. - with open(local, 'wb') as f: - content = MavenArchetypeCatalog(remote_archetypes)\ - .generate_meta_file_content() - try: - overwrite_file(local, content) - except FileNotFoundError as e: - logger.error( - "Error: Can not create file %s because of some missing folders", - local, - ) - raise e + content = MavenArchetypeCatalog(remote_archetypes)\ + .generate_meta_file_content() + try: + overwrite_file(local, content) + except FileNotFoundError as e: + logger.error( + "Error: Can not create file %s because of some missing folders", + local, + ) + raise e __gen_all_digest_files(local) return 1 @@ -986,17 +1177,16 @@ def _generate_upload_archetype_catalog( # Re-render the result of our archetype merge / # un-merge to the local file, in preparation for # upload. - with open(local, 'wb') as f: - content = MavenArchetypeCatalog(remote_archetypes)\ - .generate_meta_file_content() - try: - overwrite_file(local, content) - except FileNotFoundError as e: - logger.error( - "Error: Can not create file %s because of some missing folders", - local, - ) - raise e + content = MavenArchetypeCatalog(remote_archetypes)\ + .generate_meta_file_content() + try: + overwrite_file(local, content) + except FileNotFoundError as e: + logger.error( + "Error: Can not create file %s because of some missing folders", + local, + ) + raise e __gen_all_digest_files(local) return True @@ -1143,8 +1333,8 @@ def __wildcard_metadata_paths(paths: List[str]) -> List[str]: new_paths.append(path[:-len(".xml")] + ".*") elif path.endswith(".md5")\ or path.endswith(".sha1")\ - or path.endswith(".sha128")\ - or path.endswith(".sha256"): + or path.endswith(".sha256")\ + or path.endswith(".sha512"): continue else: new_paths.append(path) @@ -1153,7 +1343,7 @@ def __wildcard_metadata_paths(paths: List[str]) -> List[str]: class VersionCompareKey: 'Used as key function for version sorting' - def __init__(self, obj): + def __init__(self, obj: str): self.obj = obj def __lt__(self, other): @@ -1184,36 +1374,61 @@ def __compare(self, other) -> int: big = max(len(xitems), len(yitems)) for i in range(big): try: - xitem = xitems[i] + xitem: Union[str, int] = xitems[i] except IndexError: return -1 try: - yitem = yitems[i] + yitem: Union[str, int] = yitems[i] except IndexError: return 1 - if xitem.isnumeric() and yitem.isnumeric(): + if (isinstance(xitem, str) and isinstance(yitem, str) and + xitem.isnumeric() and yitem.isnumeric()): xitem = int(xitem) yitem = int(yitem) - elif xitem.isnumeric() and not yitem.isnumeric(): - return 1 - elif not xitem.isnumeric() and yitem.isnumeric(): - return -1 - if xitem > yitem: + elif (isinstance(xitem, str) and xitem.isnumeric() and + (not isinstance(yitem, str) or not yitem.isnumeric())): return 1 - elif xitem < yitem: + elif (isinstance(yitem, str) and yitem.isnumeric() and + (not isinstance(xitem, str) or not xitem.isnumeric())): return -1 + # At this point, both are the same type (both int or both str) + if isinstance(xitem, int) and isinstance(yitem, int): + if xitem > yitem: + return 1 + elif xitem < yitem: + return -1 + elif isinstance(xitem, str) and isinstance(yitem, str): + if xitem > yitem: + return 1 + elif xitem < yitem: + return -1 else: continue return 0 -class ArchetypeCompareKey(VersionCompareKey): - 'Used as key function for GAV sorting' - def __init__(self, gav): - super().__init__(gav.version) +class ArchetypeCompareKey: + def __init__(self, gav: ArchetypeRef): self.gav = gav - # pylint: disable=unused-private-member + def __lt__(self, other): + return self.__compare(other) < 0 + + def __gt__(self, other): + return self.__compare(other) > 0 + + def __le__(self, other): + return self.__compare(other) <= 0 + + def __ge__(self, other): + return self.__compare(other) >= 0 + + def __eq__(self, other): + return self.__compare(other) == 0 + + def __hash__(self): + return self.gav.__hash__() + def __compare(self, other) -> int: x = self.gav.group_id + ":" + self.gav.artifact_id y = other.gav.group_id + ":" + other.gav.artifact_id diff --git a/charon/utils/archive.py b/charon/utils/archive.py index 4a1f256c..058fa17e 100644 --- a/charon/utils/archive.py +++ b/charon/utils/archive.py @@ -182,6 +182,19 @@ def detect_npm_archive(repo): return NpmArchiveType.NOT_NPM +def detect_npm_archives(repos): + """Detects, if the archives need to have npm workflow. + :parameter repos list of repository directories + :return list of NpmArchiveType values + """ + results = [] + for repo in repos: + result = detect_npm_archive(repo) + results.append(result) + + return results + + def download_archive(url: str, base_dir=None) -> str: dir_ = base_dir if not dir_ or not os.path.isdir(dir_): diff --git a/charon/utils/files.py b/charon/utils/files.py index d811200b..dca71444 100644 --- a/charon/utils/files.py +++ b/charon/utils/files.py @@ -17,7 +17,9 @@ import os import hashlib import errno -from typing import List, Tuple +import tempfile +import shutil +from typing import List, Tuple, Optional from charon.constants import MANIFEST_SUFFIX @@ -32,24 +34,37 @@ class HashType(Enum): def get_hash_type(type_str: str) -> HashType: """Get hash type from string""" - if type_str.lower() == "md5": + type_str_low = type_str.lower() + if type_str_low == "md5": return HashType.MD5 - elif type_str.lower() == "sha1": + elif type_str_low == "sha1": return HashType.SHA1 - elif type_str.lower() == "sha256": + elif type_str_low == "sha256": return HashType.SHA256 - elif type_str.lower() == "sha512": + elif type_str_low == "sha512": return HashType.SHA512 else: raise ValueError("Unsupported hash type: {}".format(type_str)) -def overwrite_file(file_path: str, content: str): - if not os.path.isfile(file_path): - with open(file_path, mode="a", encoding="utf-8"): - pass - with open(file_path, mode="w", encoding="utf-8") as f: - f.write(content) +def overwrite_file(file_path: str, content: str) -> None: + parent_dir: Optional[str] = os.path.dirname(file_path) + if parent_dir: + if not os.path.exists(parent_dir): + os.makedirs(parent_dir, exist_ok=True) + else: + parent_dir = None # None explicitly means current directory for tempfile + + # Write to temporary file first, then atomically rename + fd, temp_path = tempfile.mkstemp(dir=parent_dir, text=True) + try: + with os.fdopen(fd, 'w', encoding='utf-8') as f: + f.write(content) + shutil.move(temp_path, file_path) + except Exception: + if os.path.exists(temp_path): + os.unlink(temp_path) + raise def read_sha1(file: str) -> str: @@ -97,7 +112,6 @@ def digest_content(content: str, hash_type=HashType.SHA1) -> str: def _hash_object(hash_type: HashType): - hash_obj = None if hash_type == HashType.SHA1: hash_obj = hashlib.sha1() elif hash_type == HashType.SHA256: @@ -107,7 +121,7 @@ def _hash_object(hash_type: HashType): elif hash_type == HashType.SHA512: hash_obj = hashlib.sha512() else: - raise Exception("Error: Unknown hash type for digesting.") + raise ValueError("Error: Unknown hash type for digesting.") return hash_obj @@ -116,15 +130,8 @@ def write_manifest(paths: List[str], root: str, product_key: str) -> Tuple[str, manifest_path = os.path.join(root, manifest_name) artifacts = [] for path in paths: - if path.startswith(root): - path = path[len(root):] - if path.startswith("/"): - path = path[1:] - artifacts.append(path) - - if not os.path.isfile(manifest_path): - with open(manifest_path, mode="a", encoding="utf-8"): - pass - with open(manifest_path, mode="w", encoding="utf-8") as f: - f.write('\n'.join(artifacts)) + rel_path = os.path.relpath(path, root) + artifacts.append(rel_path) + + overwrite_file(manifest_path, '\n'.join(artifacts)) return manifest_name, manifest_path diff --git a/pyproject.toml b/pyproject.toml index b667868d..0693668d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,12 +10,11 @@ authors = [ ] readme = "README.md" keywords = ["charon", "mrrc", "maven", "npm", "build", "java"] -license-files = ["LICENSE"] +license = {text="Apache-2.0"} requires-python = ">=3.9" classifiers = [ "Development Status :: 1 - Planning", "Intended Audience :: Developers", - "License :: OSI Approved :: Apache Software License", "Topic :: Software Development :: Build Tools", "Topic :: Utilities", "Programming Language :: Python :: 3 :: Only", @@ -52,7 +51,7 @@ dev = [ test = [ "flexmock>=0.10.6", "responses>=0.9.0", - "pytest<=7.1.3", + "pytest<=9.0.1", "pytest-cov", "pytest-html", "requests-mock", @@ -63,8 +62,8 @@ test = [ [project.scripts] charon = "charon.cmd:cli" -[tool.setuptools] -packages = ["charon"] +[tool.setuptools.packages.find] +include = ["charon*"] [tool.setuptools_scm] fallback_version = "1.3.4+dev.fallback" diff --git a/setup.py b/setup.py index 3935d97b..2de9c2e6 100755 --- a/setup.py +++ b/setup.py @@ -32,33 +32,28 @@ classifiers=[ "Development Status :: 1 - Planning", "Intended Audience :: Developers", - "License :: OSI Approved :: Apache Software License", "Programming Language :: Python :: 3", "Topic :: Software Development :: Build Tools", "Topic :: Utilities", ], keywords="charon mrrc maven npm build java", author="RedHat EXD SPMM", - license="APLv2", packages=find_packages(exclude=["ez_setup", "examples", "tests"]), package_data={'charon': ['schemas/*.json']}, - test_suite="tests", entry_points={ "console_scripts": ["charon = charon.cmd:cli"], }, - install_requires=[ - "Jinja2>=3.1.3", - "boto3>=1.18.35", - "botocore>=1.21.35", - "click>=8.1.3", - "requests>=2.25.0", - "PyYAML>=5.4.1", - "defusedxml>=0.7.1", - "subresource-integrity>=0.2", - "jsonschema>=4.9.1", - "urllib3>=1.25.10", - "semantic-version>=2.10.0", - "oras<=0.2.31", - "python-qpid-proton>=0.39.0" - ], + # install_requires=[ + # "Jinja2>=3.1.3", + # "boto3>=1.18.35", + # "botocore>=1.21.35", + # "click>=8.1.3", + # "requests>=2.25.0", + # "PyYAML>=5.4.1", + # "defusedxml>=0.7.1", + # "subresource-integrity>=0.2", + # "jsonschema>=4.9.1", + # "urllib3>=1.25.10", + # "semantic-version>=2.10.0" + # ], ) diff --git a/tests/requirements.txt b/tests/requirements.txt index 408de626..b0e35b7f 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,6 +1,6 @@ flexmock>=0.10.6 responses>=0.9.0 -pytest<=7.1.3 +pytest<=9.0.1 pytest-cov pytest-html requests-mock diff --git a/tests/test_archive.py b/tests/test_archive.py index 0e2ac09a..22cf48fd 100644 --- a/tests/test_archive.py +++ b/tests/test_archive.py @@ -1,5 +1,5 @@ from tests.base import BaseTest -from charon.utils.archive import NpmArchiveType, detect_npm_archive +from charon.utils.archive import NpmArchiveType, detect_npm_archive, detect_npm_archives import os from tests.constants import INPUTS @@ -12,5 +12,36 @@ def test_detect_package(self): npm_tarball = os.path.join(INPUTS, "code-frame-7.14.5.tgz") self.assertEqual(NpmArchiveType.TAR_FILE, detect_npm_archive(npm_tarball)) + def test_detect_packages(self): + mvn_tarballs = [ + os.path.join(INPUTS, "commons-client-4.5.6.zip"), + os.path.join(INPUTS, "commons-client-4.5.9.zip") + ] + archive_types = detect_npm_archives(mvn_tarballs) + self.assertEqual(2, archive_types.count(NpmArchiveType.NOT_NPM)) + + npm_tarball = [ + os.path.join(INPUTS, "code-frame-7.14.5.tgz") + ] + archive_types = detect_npm_archives(npm_tarball) + self.assertEqual(1, archive_types.count(NpmArchiveType.TAR_FILE)) + + npm_tarballs = [ + os.path.join(INPUTS, "code-frame-7.14.5.tgz"), + os.path.join(INPUTS, "code-frame-7.15.8.tgz") + ] + archive_types = detect_npm_archives(npm_tarballs) + self.assertEqual(2, archive_types.count(NpmArchiveType.TAR_FILE)) + + tarballs = [ + os.path.join(INPUTS, "commons-client-4.5.6.zip"), + os.path.join(INPUTS, "commons-client-4.5.9.zip"), + os.path.join(INPUTS, "code-frame-7.14.5.tgz"), + os.path.join(INPUTS, "code-frame-7.15.8.tgz") + ] + archive_types = detect_npm_archives(tarballs) + self.assertEqual(2, archive_types.count(NpmArchiveType.NOT_NPM)) + self.assertEqual(2, archive_types.count(NpmArchiveType.TAR_FILE)) + def test_download_archive(self): pass diff --git a/tests/test_cf_maven_ops.py b/tests/test_cf_maven_ops.py index b8cb03c1..ca5ac361 100644 --- a/tests/test_cf_maven_ops.py +++ b/tests/test_cf_maven_ops.py @@ -31,7 +31,7 @@ def test_cf_after_upload(self): test_zip = os.path.join(INPUTS, "commons-client-4.5.6.zip") product = "commons-client-4.5.6" handle_maven_uploading( - test_zip, product, + [test_zip], product, targets=[('', TEST_BUCKET, 'ga', '', 'maven.repository.redhat.com')], dir_=self.tempdir, do_index=True, @@ -52,7 +52,7 @@ def test_cf_after_del(self): test_zip = os.path.join(INPUTS, "commons-client-4.5.6.zip") product_456 = "commons-client-4.5.6" handle_maven_uploading( - test_zip, product_456, + [test_zip], product_456, targets=[('', TEST_BUCKET, 'ga', '', 'maven.repository.redhat.com')], dir_=self.tempdir, do_index=True diff --git a/tests/test_cf_reindex.py b/tests/test_cf_reindex.py index 944a86f2..941793fd 100644 --- a/tests/test_cf_reindex.py +++ b/tests/test_cf_reindex.py @@ -40,7 +40,7 @@ def test_cf_maven_after_reindex(self): test_zip = os.path.join(INPUTS, "commons-client-4.5.6.zip") product_456 = "commons-client-4.5.6" handle_maven_uploading( - test_zip, product_456, + [test_zip], product_456, targets=[('', TEST_BUCKET, 'ga', '', 'maven.repository.redhat.com')], dir_=self.tempdir ) diff --git a/tests/test_extract_tarballs.py b/tests/test_extract_tarballs.py new file mode 100644 index 00000000..53a96f63 --- /dev/null +++ b/tests/test_extract_tarballs.py @@ -0,0 +1,28 @@ +from tests.base import BaseTest +from charon.pkgs.maven import _extract_tarballs +import os + +from tests.constants import INPUTS + + +class ArchiveTest(BaseTest): + def test_extract_tarballs(self): + mvn_tarballs = [ + os.path.join(INPUTS, "commons-client-4.5.6.zip"), + os.path.join(INPUTS, "commons-client-4.5.9.zip"), + ] + final_merged_path = _extract_tarballs(mvn_tarballs, "maven-repository") + expected_dir = os.path.join( + final_merged_path, "merged_repositories", "maven-repository" + ) + self.assertTrue(os.path.exists(expected_dir)) + + expected_files = [ + "org/apache/httpcomponents/httpclient/4.5.9/httpclient-4.5.9.jar", + "org/apache/httpcomponents/httpclient/4.5.9/httpclient-4.5.9.pom", + "org/apache/httpcomponents/httpclient/4.5.6/httpclient-4.5.6.jar", + "org/apache/httpcomponents/httpclient/4.5.6/httpclient-4.5.6.pom", + ] + for expected_file in expected_files: + file_path = os.path.join(expected_dir, expected_file) + self.assertTrue(os.path.exists(file_path)) diff --git a/tests/test_manifest_del.py b/tests/test_manifest_del.py index 7a81be3c..c47c7602 100644 --- a/tests/test_manifest_del.py +++ b/tests/test_manifest_del.py @@ -77,7 +77,7 @@ def __prepare_maven_content(self): test_zip = os.path.join(INPUTS, "commons-client-4.5.6.zip") product = "commons-client-4.5.6" handle_maven_uploading( - test_zip, product, + [test_zip], product, targets=[(TEST_TARGET, TEST_BUCKET, '', '')], dir_=self.tempdir, do_index=False, diff --git a/tests/test_manifest_upload.py b/tests/test_manifest_upload.py index c7e801b2..520f0679 100644 --- a/tests/test_manifest_upload.py +++ b/tests/test_manifest_upload.py @@ -36,7 +36,7 @@ def test_maven_manifest_upload(self): test_zip = os.path.join(INPUTS, "commons-client-4.5.6.zip") product = "commons-client-4.5.6" handle_maven_uploading( - test_zip, product, + [test_zip], product, targets=[(TEST_TARGET, TEST_BUCKET, '', '')], dir_=self.tempdir, do_index=False, diff --git a/tests/test_maven_del.py b/tests/test_maven_del.py index 5b565adc..86425724 100644 --- a/tests/test_maven_del.py +++ b/tests/test_maven_del.py @@ -190,7 +190,7 @@ def __prepare_content(self, prefix=None): test_zip = os.path.join(INPUTS, "commons-client-4.5.6.zip") product_456 = "commons-client-4.5.6" handle_maven_uploading( - test_zip, product_456, + [test_zip], product_456, targets=[('', TEST_BUCKET, prefix, '')], dir_=self.tempdir, do_index=False @@ -199,7 +199,7 @@ def __prepare_content(self, prefix=None): test_zip = os.path.join(INPUTS, "commons-client-4.5.9.zip") product_459 = "commons-client-4.5.9" handle_maven_uploading( - test_zip, product_459, + [test_zip], product_459, targets=[('', TEST_BUCKET, prefix, '')], dir_=self.tempdir, do_index=False diff --git a/tests/test_maven_del_multi_tgts.py b/tests/test_maven_del_multi_tgts.py index 26fa11cc..2a7d042f 100644 --- a/tests/test_maven_del_multi_tgts.py +++ b/tests/test_maven_del_multi_tgts.py @@ -259,7 +259,7 @@ def __prepare_content(self, prefix=None): product_456 = "commons-client-4.5.6" targets_ = [('', TEST_BUCKET, prefix, ''), ('', TEST_BUCKET_2, prefix, '')] handle_maven_uploading( - test_zip, product_456, + [test_zip], product_456, targets=targets_, dir_=self.tempdir, do_index=False @@ -268,7 +268,7 @@ def __prepare_content(self, prefix=None): test_zip = os.path.join(INPUTS, "commons-client-4.5.9.zip") product_459 = "commons-client-4.5.9" handle_maven_uploading( - test_zip, product_459, + [test_zip], product_459, targets=targets_, dir_=self.tempdir, do_index=False diff --git a/tests/test_maven_index.py b/tests/test_maven_index.py index a5cd1ed2..33533337 100644 --- a/tests/test_maven_index.py +++ b/tests/test_maven_index.py @@ -37,7 +37,7 @@ def test_uploading_index(self): test_zip = os.path.join(INPUTS, "commons-client-4.5.6.zip") product = "commons-client-4.5.6" handle_maven_uploading( - test_zip, product, + [test_zip], product, targets=[('', TEST_BUCKET, '', '')], dir_=self.tempdir ) @@ -79,7 +79,7 @@ def test_overlap_upload_index(self): test_zip = os.path.join(INPUTS, "commons-client-4.5.6.zip") product_456 = "commons-client-4.5.6" handle_maven_uploading( - test_zip, product_456, + [test_zip], product_456, targets=[('', TEST_BUCKET, '', '')], dir_=self.tempdir ) @@ -87,7 +87,7 @@ def test_overlap_upload_index(self): test_zip = os.path.join(INPUTS, "commons-client-4.5.9.zip") product_459 = "commons-client-4.5.9" handle_maven_uploading( - test_zip, product_459, + [test_zip], product_459, targets=[('', TEST_BUCKET, '', '')], dir_=self.tempdir ) @@ -130,7 +130,7 @@ def test_re_index(self): test_zip = os.path.join(INPUTS, "commons-client-4.5.6.zip") product = "commons-client-4.5.6" handle_maven_uploading( - test_zip, product, + [test_zip], product, targets=[('', TEST_BUCKET, '', '')], dir_=self.tempdir ) @@ -221,7 +221,7 @@ def __test_upload_index_with_prefix(self, prefix: str): test_zip = os.path.join(INPUTS, "commons-client-4.5.6.zip") product = "commons-client-4.5.6" handle_maven_uploading( - test_zip, product, + [test_zip], product, targets=[('', TEST_BUCKET, prefix, '')], dir_=self.tempdir ) @@ -403,7 +403,7 @@ def __prepare_content(self, prefix=None): test_zip = os.path.join(INPUTS, "commons-client-4.5.6.zip") product_456 = "commons-client-4.5.6" handle_maven_uploading( - test_zip, product_456, + [test_zip], product_456, targets=[('', TEST_BUCKET, prefix, '')], dir_=self.tempdir ) @@ -411,7 +411,7 @@ def __prepare_content(self, prefix=None): test_zip = os.path.join(INPUTS, "commons-client-4.5.9.zip") product_459 = "commons-client-4.5.9" handle_maven_uploading( - test_zip, product_459, + [test_zip], product_459, targets=[('', TEST_BUCKET, prefix, '')], dir_=self.tempdir ) diff --git a/tests/test_maven_index_multi_tgts.py b/tests/test_maven_index_multi_tgts.py index cc9d0718..44f921bf 100644 --- a/tests/test_maven_index_multi_tgts.py +++ b/tests/test_maven_index_multi_tgts.py @@ -46,7 +46,7 @@ def test_uploading_index(self): test_zip = os.path.join(INPUTS, "commons-client-4.5.6.zip") product = "commons-client-4.5.6" handle_maven_uploading( - test_zip, product, + [test_zip], product, targets=targets_, dir_=self.tempdir ) @@ -106,7 +106,7 @@ def test_overlap_upload_index(self): test_zip = os.path.join(INPUTS, "commons-client-4.5.6.zip") product_456 = "commons-client-4.5.6" handle_maven_uploading( - test_zip, product_456, + [test_zip], product_456, targets=targets_, dir_=self.tempdir ) @@ -114,7 +114,7 @@ def test_overlap_upload_index(self): test_zip = os.path.join(INPUTS, "commons-client-4.5.9.zip") product_459 = "commons-client-4.5.9" handle_maven_uploading( - test_zip, product_459, + [test_zip], product_459, targets=targets_, dir_=self.tempdir ) @@ -194,7 +194,7 @@ def __test_upload_index_with_prefix(self, prefix: str): test_zip = os.path.join(INPUTS, "commons-client-4.5.6.zip") product = "commons-client-4.5.6" handle_maven_uploading( - test_zip, product, + [test_zip], product, targets=targets_, dir_=self.tempdir ) @@ -417,7 +417,7 @@ def __prepare_content(self, prefix=None): test_zip = os.path.join(INPUTS, "commons-client-4.5.6.zip") product_456 = "commons-client-4.5.6" handle_maven_uploading( - test_zip, product_456, + [test_zip], product_456, targets=targets_, dir_=self.tempdir ) @@ -425,7 +425,7 @@ def __prepare_content(self, prefix=None): test_zip = os.path.join(INPUTS, "commons-client-4.5.9.zip") product_459 = "commons-client-4.5.9" handle_maven_uploading( - test_zip, product_459, + [test_zip], product_459, targets=targets_, dir_=self.tempdir ) diff --git a/tests/test_maven_sign.py b/tests/test_maven_sign.py index f60ee54d..834326bf 100644 --- a/tests/test_maven_sign.py +++ b/tests/test_maven_sign.py @@ -32,7 +32,7 @@ def test_uploading_sign(self): test_zip = os.path.join(INPUTS, "commons-client-4.5.6.zip") product = "commons-client-4.5.6" handle_maven_uploading( - test_zip, product, + [test_zip], product, targets=[('', TEST_BUCKET, '', '')], dir_=self.tempdir, gen_sign=True, @@ -63,7 +63,7 @@ def test_overlap_upload_index(self): test_zip = os.path.join(INPUTS, "commons-client-4.5.6.zip") product_456 = "commons-client-4.5.6" handle_maven_uploading( - test_zip, product_456, + [test_zip], product_456, targets=[('', TEST_BUCKET, '', '')], dir_=self.tempdir, gen_sign=True, @@ -73,7 +73,7 @@ def test_overlap_upload_index(self): test_zip = os.path.join(INPUTS, "commons-client-4.5.9.zip") product_459 = "commons-client-4.5.9" handle_maven_uploading( - test_zip, product_459, + [test_zip], product_459, targets=[('', TEST_BUCKET, '', '')], dir_=self.tempdir, gen_sign=True, diff --git a/tests/test_maven_upload.py b/tests/test_maven_upload.py index 629a9e3f..ab36c76f 100644 --- a/tests/test_maven_upload.py +++ b/tests/test_maven_upload.py @@ -47,7 +47,7 @@ def test_overlap_upload(self): test_zip = os.path.join(INPUTS, "commons-client-4.5.6.zip") product_456 = "commons-client-4.5.6" handle_maven_uploading( - test_zip, product_456, + [test_zip], product_456, targets=[('', TEST_BUCKET, '', '')], dir_=self.tempdir, do_index=False ) @@ -55,7 +55,7 @@ def test_overlap_upload(self): test_zip = os.path.join(INPUTS, "commons-client-4.5.9.zip") product_459 = "commons-client-4.5.9" handle_maven_uploading( - test_zip, product_459, + [test_zip], product_459, targets=[('', TEST_BUCKET, '', '')], dir_=self.tempdir, do_index=False ) @@ -110,11 +110,74 @@ def test_overlap_upload(self): self.assertIn("httpclient", cat_content) self.assertIn("org.apache.httpcomponents", cat_content) + def test_multi_zips_upload(self): + mvn_tarballs = [ + os.path.join(INPUTS, "commons-client-4.5.6.zip"), + os.path.join(INPUTS, "commons-client-4.5.9.zip") + ] + product_45 = "commons-client-4.5" + + handle_maven_uploading( + mvn_tarballs, product_45, + targets=[('', TEST_BUCKET, '', '')], + dir_=self.tempdir, do_index=False + ) + + objs = list(self.test_bucket.objects.all()) + actual_files = [obj.key for obj in objs] + # need to double mvn num because of .prodinfo files + self.assertEqual( + COMMONS_CLIENT_MVN_NUM * 2 + COMMONS_CLIENT_META_NUM, + len(actual_files) + ) + + filesets = [ + COMMONS_CLIENT_METAS, COMMONS_CLIENT_456_FILES, + COMMONS_CLIENT_459_FILES, + ARCHETYPE_CATALOG_FILES + ] + for fileset in filesets: + for f in fileset: + self.assertIn(f, actual_files) + + product_mix = [product_45] + for f in COMMONS_LOGGING_FILES: + self.assertIn(f, actual_files) + self.check_product(f, product_mix) + for f in COMMONS_LOGGING_METAS: + self.assertIn(f, actual_files) + + meta_obj_client = self.test_bucket.Object(COMMONS_CLIENT_METAS[0]) + meta_content_client = str(meta_obj_client.get()["Body"].read(), "utf-8") + self.assertIn( + "org.apache.httpcomponents", meta_content_client + ) + self.assertIn("httpclient", meta_content_client) + self.assertIn("4.5.9", meta_content_client) + self.assertIn("4.5.9", meta_content_client) + self.assertIn("4.5.6", meta_content_client) + self.assertIn("4.5.9", meta_content_client) + + meta_obj_logging = self.test_bucket.Object(COMMONS_LOGGING_METAS[0]) + meta_content_logging = str(meta_obj_logging.get()["Body"].read(), "utf-8") + self.assertIn("commons-logging", meta_content_logging) + self.assertIn("commons-logging", meta_content_logging) + self.assertIn("1.2", meta_content_logging) + self.assertIn("1.2", meta_content_logging) + self.assertIn("1.2", meta_content_logging) + + catalog = self.test_bucket.Object(ARCHETYPE_CATALOG) + cat_content = str(catalog.get()["Body"].read(), "utf-8") + self.assertIn("4.5.6", cat_content) + self.assertIn("4.5.9", cat_content) + self.assertIn("httpclient", cat_content) + self.assertIn("org.apache.httpcomponents", cat_content) + def test_ignore_upload(self): test_zip = os.path.join(INPUTS, "commons-client-4.5.6.zip") product_456 = "commons-client-4.5.6" handle_maven_uploading( - test_zip, product_456, [".*.sha1"], + [test_zip], product_456, [".*.sha1"], targets=[('', TEST_BUCKET, '', '')], dir_=self.tempdir, do_index=False ) @@ -143,7 +206,7 @@ def __test_prefix_upload(self, prefix: str): test_zip = os.path.join(INPUTS, "commons-client-4.5.6.zip") product = "commons-client-4.5.6" handle_maven_uploading( - test_zip, product, + [test_zip], product, targets=[('', TEST_BUCKET, prefix, '')], dir_=self.tempdir, do_index=False diff --git a/tests/test_maven_upload_multi_tgts.py b/tests/test_maven_upload_multi_tgts.py index 35aa49d4..f6eb289e 100644 --- a/tests/test_maven_upload_multi_tgts.py +++ b/tests/test_maven_upload_multi_tgts.py @@ -68,7 +68,7 @@ def test_overlap_upload(self): ('', TEST_BUCKET, '', ''), ('', TEST_BUCKET_2, '', '') ] handle_maven_uploading( - test_zip, product_456, + [test_zip], product_456, targets=targets_, dir_=self.tempdir, do_index=False ) @@ -76,7 +76,7 @@ def test_overlap_upload(self): test_zip = os.path.join(INPUTS, "commons-client-4.5.9.zip") product_459 = "commons-client-4.5.9" handle_maven_uploading( - test_zip, product_459, + [test_zip], product_459, targets=targets_, dir_=self.tempdir, do_index=False ) @@ -186,7 +186,7 @@ def test_ignore_upload(self): ('', TEST_BUCKET, '', ''), ('', TEST_BUCKET_2, '', '') ] handle_maven_uploading( - test_zip, product_456, [".*.sha1"], + [test_zip], product_456, [".*.sha1"], targets=targets_, dir_=self.tempdir, do_index=False ) @@ -221,7 +221,7 @@ def __test_prefix_upload(self, targets: List[Tuple[str, str, str, str]]): test_zip = os.path.join(INPUTS, "commons-client-4.5.6.zip") product = "commons-client-4.5.6" handle_maven_uploading( - test_zip, product, + [test_zip], product, targets=targets, dir_=self.tempdir, do_index=False diff --git a/tests/test_pkgs_dryrun.py b/tests/test_pkgs_dryrun.py index 46061734..c49ad14d 100644 --- a/tests/test_pkgs_dryrun.py +++ b/tests/test_pkgs_dryrun.py @@ -30,7 +30,7 @@ def test_maven_upload_dry_run(self): test_zip = os.path.join(INPUTS, "commons-client-4.5.6.zip") product = "commons-client-4.5.6" handle_maven_uploading( - test_zip, product, + [test_zip], product, targets=[('', TEST_BUCKET, '', '')], dir_=self.tempdir, dry_run=True @@ -90,7 +90,7 @@ def __prepare_maven_content(self): test_zip = os.path.join(INPUTS, "commons-client-4.5.6.zip") product_456 = "commons-client-4.5.6" handle_maven_uploading( - test_zip, product_456, + [test_zip], product_456, targets=[('', TEST_BUCKET, '', '')], dir_=self.tempdir ) @@ -98,7 +98,7 @@ def __prepare_maven_content(self): test_zip = os.path.join(INPUTS, "commons-client-4.5.9.zip") product_459 = "commons-client-4.5.9" handle_maven_uploading( - test_zip, product_459, + [test_zip], product_459, targets=[('', TEST_BUCKET, '', '')], dir_=self.tempdir ) diff --git a/tests/test_radas_sign_generation.py b/tests/test_radas_sign_generation.py index ccc448a2..8e64d76c 100644 --- a/tests/test_radas_sign_generation.py +++ b/tests/test_radas_sign_generation.py @@ -20,7 +20,6 @@ import os import json import shutil -import builtins from unittest import mock from charon.utils.files import overwrite_file from charon.pkgs.radas_sign import generate_radas_sign @@ -68,20 +67,14 @@ def test_sign_files_generation_with_failure(self): expected_asc1 = os.path.join(self.__repo_dir, "foo/bar/1.0/foo-bar-1.0.jar.asc") expected_asc2 = os.path.join(self.__repo_dir, "foo/bar/2.0/foo-bar-2.0.jar.asc") - # simulate expected_asc1 can not open to write properly - real_open = builtins.open - with mock.patch("builtins.open") as mock_open: - def side_effect(path, *args, **kwargs): - # this is for pylint check - mode = "r" - if len(args) > 0: - mode = args[0] - elif "mode" in kwargs: - mode = kwargs["mode"] - if path == expected_asc1 and "w" in mode: + # simulate expected_asc1 can not be written properly + real_overwrite = overwrite_file + with mock.patch("charon.pkgs.radas_sign.files.overwrite_file") as mock_overwrite: + def side_effect(path, content): + if path == expected_asc1: raise IOError("mock write error") - return real_open(path, *args, **kwargs) - mock_open.side_effect = side_effect + return real_overwrite(path, content) + mock_overwrite.side_effect = side_effect failed, generated = generate_radas_sign(self.__repo_dir, self.__sign_result_file) self.assertEqual(len(failed), 1)