Produce Source Distribution and Documentation as tar.gz (Azure#31419)

* remove all references to *.zip files throughout CI
* source distribution and documentation artifact are now all tar.gz files
* update all references to artifact .zip -> .tar.gz

Co-authored-by: McCoy Patiño <39780829+mccoyp@users.noreply.github.com>

Author: scbedd

eng/pipelines/templates/jobs/smoke.tests.yml

@@ -159,10 +159,10 @@ jobs:
           timeoutInMinutes: 5
 
         - pwsh: |
-            $packages = Get-ChildItem "$(Pipeline.Workspace)/${{ parameters.ArtifactName }}/${{ parameters.Artifact.name }}/*.zip"
+            $packages = Get-ChildItem "$(Pipeline.Workspace)/${{ parameters.ArtifactName }}/${{ parameters.Artifact.name }}/*.tar.gz"
             Write-Host "Artifacts found:"
             $artifacts = $packages | ForEach-Object {
-              if ($_.Name -match "([a-zA-Z\-]+)\-(.*).zip") {
+              if ($_.Name -match "([a-zA-Z\-]+)\-(.*).tar.gz") {
                 Write-Host "$($matches[1]): $($matches[2])"
                 return @{ "name" = $matches[1]; "version" = $matches[2] }
               }

eng/pipelines/templates/stages/archetype-python-release.yml

@@ -106,7 +106,7 @@ stages:
                           New-Item -ItemType Directory -Force -Path $esrpDirectory
 
                           Get-ChildItem -Path "$(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}}" `
-                            | Where-Object { ($_.Name -like "*.zip" -or $_.Name -like "*.whl") } `
+                            | Where-Object { ($_.Name -like "*.tar.gz" -or $_.Name -like "*.whl") } `
                             | Copy-Item -Destination $esrpDirectory
 
                           Get-ChildItem $esrpDirectory
@@ -129,7 +129,7 @@ stages:
                           set -e
                           twine upload --repository ${{parameters.DevFeedName}} --config-file $(PYPIRC_PATH) $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}}/*.whl
                           echo "Uploaded whl to devops feed"
-                          twine upload --repository ${{parameters.DevFeedName}} --config-file $(PYPIRC_PATH) $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}}/*.zip
+                          twine upload --repository ${{parameters.DevFeedName}} --config-file $(PYPIRC_PATH) $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}}/*.tar.gz
                           echo "Uploaded sdist to devops feed"
                         displayName: 'Publish package to feed: ${{parameters.DevFeedName}}'
 
@@ -275,7 +275,7 @@ stages:
 
               twine upload --repository $(DevFeedName) --config-file $(PYPIRC_PATH) $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}}/*-*a*.whl
               echo "Uploaded whl to devops feed $(DevFeedName)"
-              twine upload --repository $(DevFeedName) --config-file $(PYPIRC_PATH) $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}}/*-*a*.zip
+              twine upload --repository $(DevFeedName) --config-file $(PYPIRC_PATH) $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}}/*-*a*.tar.gz
               echo "Uploaded sdist to devops feed $(DevFeedName)"
             displayName: 'Publish ${{artifact.name}} alpha package'
 

eng/pipelines/templates/steps/build-package-artifacts.yml

@@ -67,7 +67,7 @@ steps:
 
   - script: |
       twine check $(Build.ArtifactStagingDirectory)/**/*.whl
-      twine check $(Build.ArtifactStagingDirectory)/**/*.zip
+      twine check $(Build.ArtifactStagingDirectory)/**/*.tar.gz
     displayName: 'Verify Readme'
 
   - ${{if eq(variables['System.TeamProject'], 'internal') }}:

eng/scripts/Language-Settings.ps1

@@ -1,7 +1,7 @@
 $Language = "python"
 $LanguageDisplayName = "Python"
 $PackageRepository = "PyPI"
-$packagePattern = "*.zip"
+$packagePattern = "*.tar.gz"
 $MetadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/main/_data/releases/latest/python-packages.csv"
 $BlobStorageUrl = "https://azuresdkdocs.blob.core.windows.net/%24web?restype=container&comp=list&prefix=python%2F&delimiter=%2F"
 $GithubUri = "https://github.com/Azure/azure-sdk-for-python"
@@ -86,7 +86,7 @@ function IsPythonPackageVersionPublished($pkgId, $pkgVersion)
   }
 }
 
-# Parse out package publishing information given a python sdist of ZIP format.
+# Parse out package publishing information given a python sdist of tar.gz format.
 function Get-python-PackageInfoFromPackageFile ($pkg, $workingDirectory)
 {
   $pkg.Basename -match $SDIST_PACKAGE_REGEX | Out-Null
@@ -101,7 +101,8 @@ function Get-python-PackageInfoFromPackageFile ($pkg, $workingDirectory)
   $readmeContent = ""
 
   New-Item -ItemType Directory -Force -Path $workFolder
-  Expand-Archive -Path $pkg -DestinationPath $workFolder
+  Write-Host "tar -zxvf $pkg -C $workFolder"
+  tar -zxvf $pkg -C $workFolder
 
   $changeLogLoc = @(Get-ChildItem -Path $workFolder -Recurse -Include "CHANGELOG.md")[0]
   if ($changeLogLoc) {
@@ -130,16 +131,26 @@ function Get-python-PackageInfoFromPackageFile ($pkg, $workingDirectory)
 # Stage and Upload Docs to blob Storage
 function Publish-python-GithubIODocs ($DocLocation, $PublicArtifactLocation)
 {
-  $PublishedDocs = Get-ChildItem "$DocLocation" | Where-Object -FilterScript {$_.Name.EndsWith(".zip")}
+  $PublishedDocs = Get-ChildItem "$DocLocation" | Where-Object -FilterScript {$_.Name.EndsWith(".tar.gz")}
 
   foreach ($Item in $PublishedDocs)
   {
-    $PkgName = $Item.BaseName
+    $PkgName = $Item.BaseName.Replace(".tar", "")
     $ZippedDocumentationPath = Join-Path -Path $DocLocation -ChildPath $Item.Name
     $UnzippedDocumentationPath = Join-Path -Path $DocLocation -ChildPath $PkgName
     $VersionFileLocation = Join-Path -Path $UnzippedDocumentationPath -ChildPath "version.txt"
 
-    Expand-Archive -Force -Path $ZippedDocumentationPath -DestinationPath $UnzippedDocumentationPath
+    if (!(Test-Path $UnzippedDocumentationPath)) {
+      New-Item -Path $UnzippedDocumentationPath -ItemType Directory
+    }
+
+    Write-Host "tar -zxvf $ZippedDocumentationPath -C $UnzippedDocumentationPath"
+    tar -zxvf $ZippedDocumentationPath -C $UnzippedDocumentationPath
+
+    if ($LASTEXITCODE -ne 0) {
+      Write-Error "tar failed with exit code $LASTEXITCODE."
+      exit $LASTEXITCODE
+    }
 
     $Version = $(Get-Content $VersionFileLocation).Trim()
 

eng/tox/create_package_and_install.py

@@ -76,7 +76,7 @@ def build_and_discover_package(setuppy_path, dist_dir, target_setup, package_typ
         create_package(setuppy_path, dist_dir, enable_wheel=False)
 
     prebuilt_packages = [
-        f for f in os.listdir(args.distribution_directory) if f.endswith(".whl" if package_type == "wheel" else ".zip")
+        f for f in os.listdir(args.distribution_directory) if f.endswith(".whl" if package_type == "wheel" else ".tar.gz")
     ]
 
     if not in_ci():

eng/tox/run_sphinx_build.py

@@ -32,12 +32,12 @@
 def in_ci():
     return os.getenv('TF_BUILD', False)
 
-def move_output_and_zip(target_dir, package_dir, package_name):
+def move_output_and_compress(target_dir, package_dir, package_name):
     if not os.path.exists(ci_doc_dir):
         os.mkdir(ci_doc_dir)
 
     individual_zip_location = os.path.join(ci_doc_dir, package_name, package_name)
-    shutil.make_archive(individual_zip_location, 'zip', target_dir)
+    shutil.make_archive(individual_zip_location, 'gztar', target_dir)
 
 def sphinx_build(target_dir, output_dir):
     command_array = [
@@ -114,6 +114,6 @@ def sphinx_build(target_dir, output_dir):
         sphinx_build(target_dir, output_dir)
 
         if in_ci() or args.in_ci:
-            move_output_and_zip(output_dir, package_dir, pkg_details.name)
+            move_output_and_compress(output_dir, package_dir, pkg_details.name)
     else:
         logging.info("Skipping sphinx build for {}".format(pkg_details.name))

eng/tox/tox.ini

@@ -379,7 +379,7 @@ deps =
     {[packaging]pkgs}
 commands =
     python -m pip install {repository_root}/tools/azure-sdk-tools --no-deps
-    python {tox_root}/setup.py --q sdist --format zip -d {envtmpdir}
+    python {tox_root}/setup.py --q sdist -d {envtmpdir}
     python {repository_root}/eng/tox/verify_sdist.py -d {envtmpdir} -t {tox_root}
 
 

eng/tox/tox_helper_tasks.py

@@ -10,19 +10,13 @@
 import shutil
 import sys
 import logging
-import ast
 import os
-import textwrap
-import io
 import glob
 import zipfile
-import fnmatch
+import tarfile
 import subprocess
 import re
 
-from packaging.specifiers import SpecifierSet
-from pkg_resources import Requirement, parse_version
-
 logging.getLogger().setLevel(logging.INFO)
 
 
@@ -50,19 +44,27 @@ def get_pip_list_output():
     return collected_output
 
 
-def unzip_sdist_to_directory(containing_folder):
-    # grab the first one
-    path_to_zip_file = glob.glob(os.path.join(containing_folder, "*.zip"))[0]
-    return unzip_file_to_directory(path_to_zip_file, containing_folder)
+def unzip_sdist_to_directory(containing_folder: str) -> str:
+    zips = glob.glob(os.path.join(containing_folder, "*.zip"))
+
+    if zips:
+        return unzip_file_to_directory(zips[0], containing_folder)
+    else:
+        tars = glob.glob(os.path.join(containing_folder, "*.tar.gz"))
+        return unzip_file_to_directory(tars[0], containing_folder)
 
 
-def unzip_file_to_directory(path_to_zip_file, extract_location):
-    # unzip file in given path
-    # dump into given path
-    with zipfile.ZipFile(path_to_zip_file, "r") as zip_ref:
-        zip_ref.extractall(extract_location)
-        extracted_dir = os.path.basename(os.path.splitext(path_to_zip_file)[0])
-        return os.path.join(extract_location, extracted_dir)
+def unzip_file_to_directory(path_to_zip_file: str, extract_location: str) -> str:
+    if path_to_zip_file.endswith(".zip"):
+        with zipfile.ZipFile(path_to_zip_file, "r") as zip_ref:
+            zip_ref.extractall(extract_location)
+            extracted_dir = os.path.basename(os.path.splitext(path_to_zip_file)[0])
+            return os.path.join(extract_location, extracted_dir)
+    else:
+        with tarfile.open(path_to_zip_file) as tar_ref:
+            tar_ref.extractall(extract_location)
+            extracted_dir = os.path.basename(path_to_zip_file).replace(".tar.gz", "")
+            return os.path.join(extract_location, extracted_dir)
 
 
 def move_and_rename(source_location):

eng/tox/verify_sdist.py

@@ -42,7 +42,7 @@ def get_root_directories_in_sdist(dist_dir: str, version: str) -> List[str]:
     """
     # find sdist zip file
     # extract sdist and find list of directories in sdist
-    path_to_zip = glob.glob(os.path.join(dist_dir, "*{}*.zip".format(version)))[0]
+    path_to_zip = glob.glob(os.path.join(dist_dir, "*{}*.tar.gz".format(version)))[0]
     extract_location = os.path.join(dist_dir, "unzipped")
     # Cleanup any files in unzipped
     cleanup(extract_location)

eng/tox/verify_whl.py

@@ -12,8 +12,6 @@
 import glob
 import shutil
 from tox_helper_tasks import (
-    unzip_sdist_to_directory,
-    move_and_rename,
     unzip_file_to_directory,
 )
 
@@ -35,8 +33,8 @@ def extract_whl(dist_dir, version):
     # Find whl for the package
     path_to_whl = glob.glob(os.path.join(dist_dir, "*{}*.whl".format(version)))[0]
 
-    # Cleanup any existing stale files if any and rename whl file to tar.gz
-    zip_file = path_to_whl.replace(".whl", ".tar.gz")
+    # Cleanup any existing stale files if any and rename whl file to zip for extraction later
+    zip_file = path_to_whl.replace(".whl", ".zip")
     cleanup(zip_file)
     os.rename(path_to_whl, zip_file)
 
@@ -48,7 +46,7 @@ def extract_whl(dist_dir, version):
 
 
 def verify_whl_root_directory(dist_dir, expected_top_level_module, version):
-    # This method ensures root directory in whl is the directoy indicated by our top level namespace
+    # This method ensures root directory in whl is the directory indicated by our top level namespace
     extract_location = extract_whl(dist_dir, version)
     root_folders = os.listdir(extract_location)