Add environment.yml to unify conda versioning (Azure#18141)

* add an environment.yml. consume in both build_conda_artifacts as well as in the meta.yml package definitions.

* enhance conda_build_artifacts.py to extend the `summary` attribute via setting an environment variable

* update to get_tagged_code to use mkdir -p instead of testing any paths.

Author: scbedd

doc/dev/conda-builds.md

@@ -16,6 +16,21 @@ A Conda Artifact defines:
 
 ## How to Build an Azure SDK Conda Package Locally
 
+### Set up your conda environment
+
+You will notice that all the azure-sdk conda distributions have the **same** version number and requirement set. This is due to the fact that the azure-sdk team pushes our conda packages out in waves. To support this, all versions are set via a common environment variable `AZURESDK_CONDA_VERSION`.
+
+We keep this environment variable set properly across all our builds by using a common `conda_env.yml` when creating our build environment. This environment definition ensures that: 
+
+1. Our channel `https://azuresdkconda.blob.core.windows.net/channel1/` is added to the set to download packages
+2. The environment variable `AZURESDK_CONDA_VERSION` will be set exactly once.
+  
+
+Reference the `conda_env.yml` in your local build by pass `-f <path to conda_env.yml>` when you create your conda environment.
+
+```
+conda env create --yes --quiet --name ${{ artifact.name }} -f $(Build.SourcesDirectory)/eng/conda_env.yml
+```
 
 ### Create Your Build Directory
 Given how Conda packages are comprised of multiple source distributions _combined_, the buildable source does not exist directly within the azure-sdk-for-python repo. Currently, there is _some_ manual work that needs to be done.
@@ -84,6 +99,8 @@ python `build_conda_artifacts.py`
     -r "azure/storage"
     -n "azure-storage"
     -s "storage"
+    -e "<resolvable path to repo root>/eng/conda_env.yml"
+    -c "<resolvable path to sdk/storage/ci.yml>"
 ```
 
 ### Generate the Conda Package

eng/conda_env.yml

@@ -0,0 +1,2 @@
+variables:
+  AZURESDK_CONDA_VERSION: '2021.05.01'

eng/pipelines/templates/steps/build-conda-artifacts.yml

@@ -53,21 +53,28 @@ steps:
       displayName: 'Build Source Distribution for ${{ artifact.name }}'
       inputs:
         scriptPath: 'scripts/devops_tasks/build_conda_artifacts.py'
-        arguments: '-d "$(conda.output)" -b "$(conda.build)" -m "$(Build.SourcesDirectory)/sdk/${{ parameters.ServiceDirectory }}/${{ artifact.meta_source }}" -r "${{ artifact.common_root }}" -n "${{ artifact.name }}" -s "${{ parameters.ServiceDirectory }}" -o "${{ upper(parameters.ServiceDirectory) }}_SOURCE_DISTRIBUTION"'
+        arguments: >-
+          -d "$(conda.output)"
+          -b "$(conda.build)"
+          -m "$(Build.SourcesDirectory)/sdk/${{ parameters.ServiceDirectory }}/${{ artifact.meta_source }}"
+          -r "${{ artifact.common_root }}"
+          -n "${{ artifact.name }}"
+          -s "${{ parameters.ServiceDirectory }}"
+          -e "$(Build.SourcesDirectory)/eng/conda_env.yml"
+          -c "$(Build.SourcesDirectory)/sdk/${{ parameters.ServiceDirectory }}/ci.yml"
 
     - bash: |
         echo "##vso[task.prependpath]$CONDA/bin"
       displayName: 'Prepend PATH with Conda and INIT'
 
     - bash: |
-        conda create --yes --quiet --name ${{ artifact.name }}
-        source activate ${{ artifact.name }}
+        conda env create --name ${{ artifact.name }} --file $(Build.SourcesDirectory)/eng/conda_env.yml
         conda install --yes --quiet --name ${{ artifact.name }} conda-build
       displayName: 'Prepare Conda Environment for building ${{ artifact.name }}'
 
     - bash: |
         source activate ${{ artifact.name }}
-        conda-build . --output-folder "$(Agent.BuildDirectory)/conda/output" -c https://azuresdkconda.blob.core.windows.net/channel1/
+        conda-build . --output-folder "$(Agent.BuildDirectory)/conda/output" -c $(AzureSDKCondaChannel)
       displayName: 'Activate Conda Environment and Build ${{ artifact.name }}'
       workingDirectory: $(Build.SourcesDirectory)/sdk/${{ parameters.ServiceDirectory }}
   

eng/pipelines/templates/steps/get-tagged-code.yml

@@ -15,9 +15,7 @@ parameters:
 steps:
   - pwsh: |
       $targetPath = "$(Agent.TempDirectory)/${{ parameters.Package }}"
-      if (!(Test-Path $targetPath)) {
-        mkdir $targetPath
-      }
+      mkdir -p $targetPath
 
       Write-Host "##vso[task.setvariable variable=Package.Clone]$targetPath"
     displayName: 'Prep for Sparse Checkout'

eng/pipelines/templates/variables/globals.yml

@@ -1,3 +1,4 @@
 variables:
   PythonVersion: '3.6'
   skipComponentGovernanceDetection: true
+  AzureSDKCondaChannel: https://azuresdkconda.blob.core.windows.net/channel1/

scripts/devops_tasks/build_conda_artifacts.py

@@ -25,12 +25,15 @@
 import os
 import shutil
 import re
+import yaml
 
 from common_tasks import process_glob_string, run_check_call, str_to_bool, parse_setup
 from subprocess import check_call
 from distutils.dir_util import copy_tree
 
-VERSION_REGEX = re.compile(r"\{\%\s*set\s*version\s*=\s*\"(.*)\"\s*\%\}")
+VERSION_REGEX = re.compile(r"\s*AZURESDK_CONDA_VERSION\s*:\s*[\'](.*)[\']\s*")
+
+SUMMARY_TEMPLATE = "- Generated from {}."
 
 NAMESPACE_EXTENSION_TEMPLATE = """__path__ = __import__('pkgutil').extend_path(__path__, __name__)  # type: str
 """
@@ -143,8 +146,8 @@ def create_sdist_skeleton(build_directory, artifact_name, common_root):
                 shutil.copytree(src, dest)
 
 
-def get_version_from_meta(meta_yaml_location):
-    with open(os.path.abspath((meta_yaml_location)), "r") as f:
+def get_version_from_config(environment_config):
+    with open(os.path.abspath((environment_config)), "r") as f:
         lines = f.readlines()
     for line in lines:
         result = VERSION_REGEX.match(line)
@@ -165,15 +168,17 @@ def get_manifest_includes(common_root):
     return breadcrumbs
 
 
-def create_setup_files(build_directory, common_root, artifact_name, service, meta_yaml):
+def create_setup_files(
+    build_directory, common_root, artifact_name, service, meta_yaml, environment_config
+):
     sdist_directory = os.path.join(build_directory, artifact_name)
     setup_location = os.path.join(sdist_directory, "setup.py")
     manifest_location = os.path.join(sdist_directory, "MANIFEST.in")
     cfg_location = os.path.join(sdist_directory, "setup.cfg")
 
     setup_template = CONDA_PKG_SETUP_TEMPLATE.format(
         conda_package_name=artifact_name,
-        version=get_version_from_meta(meta_yaml),
+        version=get_version_from_config(environment_config),
         service=service,
         package_excludes="'azure', 'tests', '{}'".format(common_root.replace("/", ".")),
     )
@@ -182,7 +187,9 @@ def create_setup_files(build_directory, common_root, artifact_name, service, met
         f.write(setup_template)
 
     manifest_template = MANIFEST_TEMPLATE.format(
-        namespace_includes="\n".join(["include " + ns for ns in get_manifest_includes(common_root)])
+        namespace_includes="\n".join(
+            ["include " + ns for ns in get_manifest_includes(common_root)]
+        )
     )
 
     with open(manifest_location, "w") as f:
@@ -193,7 +200,13 @@ def create_setup_files(build_directory, common_root, artifact_name, service, met
 
 
 def create_combined_sdist(
-    output_directory, build_directory, artifact_name, common_root, service, meta_yaml
+    output_directory,
+    build_directory,
+    artifact_name,
+    common_root,
+    service,
+    meta_yaml,
+    environment_config,
 ):
     singular_dependency = (
         len(get_pkgs_from_build_directory(build_directory, artifact_name)) == 0
@@ -202,7 +215,12 @@ def create_combined_sdist(
     if not singular_dependency:
         create_sdist_skeleton(build_directory, artifact_name, common_root)
         create_setup_files(
-            build_directory, common_root, artifact_name, service, meta_yaml
+            build_directory,
+            common_root,
+            artifact_name,
+            service,
+            meta_yaml,
+            environment_config,
         )
 
     sdist_location = os.path.join(build_directory, artifact_name)
@@ -222,6 +240,28 @@ def create_combined_sdist(
     return output_location
 
 
+def get_summary(ci_yml, artifact_name):
+    pkg_list = []
+    with open(ci_yml, "r") as f:
+        data = f.read()
+
+    config = yaml.safe_load(data)
+
+    conda_artifact = [
+        conda_artifact
+        for conda_artifact in config["extends"]["parameters"]["CondaArtifacts"]
+        if conda_artifact["name"] == artifact_name
+    ]
+
+    if conda_artifact:
+        dependencies = conda_artifact[0]["checkout"]
+
+    for dep in dependencies:
+        pkg_list.append("{}=={}".format(dep["package"], dep["version"]))
+
+    return SUMMARY_TEMPLATE.format(", ".join(pkg_list))
+
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
         description="Build a Conda Package, given a properly formatted build directory, and input configuration. This script assumes that the build directory has been set up w/ the necessary sdists in each location."
@@ -276,11 +316,19 @@ def create_combined_sdist(
     )
 
     parser.add_argument(
-        "-o",
-        "--output_var",
-        dest="output_var",
-        help="The name of the environment variable that will be set in azure devops. The contents will be the final location of the output artifact. Local users will need to grab this value and set their env manually.",
-        required=False,
+        "-e",
+        "--environment_config",
+        dest="environment_config",
+        help="The location of the yml config file used to create the conda environments. This file has necessary common configuration information within.",
+        required=True,
+    )
+
+    parser.add_argument(
+        "-c",
+        "--ci_yml",
+        dest="ci_yml",
+        help="The location of the ci.yml that is used to define our conda artifacts. Used when to easily grab summary information.",
+        required=True,
     )
 
     args = parser.parse_args()
@@ -291,11 +339,21 @@ def create_combined_sdist(
         args.common_root,
         args.service,
         args.meta_yml,
+        args.environment_config,
     )
 
-    if args.output_var:
+    summary = get_summary(args.ci_yml, args.artifact_name)
+
+    if output_source_location:
+        print(
+            "##vso[task.setvariable variable={}]{}".format(
+                args.service.upper() + "_SOURCE_DISTRIBUTION", output_source_location
+            )
+        )
+
+    if summary:
         print(
             "##vso[task.setvariable variable={}]{}".format(
-                args.output_var, output_source_location
+                args.service.upper() + "_SUMMARY", summary
             )
         )

sdk/core/meta.yaml

@@ -1,9 +1,8 @@
 {% set name = "azure-core" %}
-{% set version = "2021.05.01" %}
 
 package:
   name: "{{ name|lower }}"
-  version: "{{ version }}"
+  version: {{ environ.get('AZURESDK_CONDA_VERSION', '0.0.0') }}
 
 source:
   url: {{ environ.get('CORE_SOURCE_DISTRIBUTION', '') }}
@@ -39,10 +38,10 @@ about:
   license: MIT
   license_family: MIT
   license_file: 
-  summary: "Microsoft Azure Core Library for Python"
+  summary: Microsoft Azure Core Library for Python{{ environ.get('CORE_SUMMARY', '') }}
   doc_url: 
   dev_url: 
 
 extra:
   recipe-maintainers:
-    - lmazuel,xiangyan99
+    - lmazuel,xiangyan99,scbedd

sdk/storage/meta.yaml

@@ -1,9 +1,8 @@
 {% set name = "azure-storage" %}
-{% set version = "2021.05.01" %}
 
 package:
   name: "{{ name|lower }}"
-  version: "{{ version }}"
+  version: {{ environ.get('AZURESDK_CONDA_VERSION', '0.0.0') }}
 
 source:
   url: {{ environ.get('STORAGE_SOURCE_DISTRIBUTION', '') }}
@@ -17,14 +16,14 @@ build:
 
 requirements:
   host:
-    - azure-core >=2021.05.01
+    - azure-core >={{ environ.get('AZURESDK_CONDA_VERSION', '0.0.0') }}
     - cryptography >=2.1.4
     - msrest >=2021.05.01
     - pip
     - python
     - aiohttp
   run:
-    - azure-core >=2021.05.01
+    - azure-core >={{ environ.get('AZURESDK_CONDA_VERSION', '0.0.0') }}
     - cryptography >=2.1.4
     - msrest >=2021.05.01
     - python
@@ -42,11 +41,11 @@ about:
   home: "https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/storage"
   license: MIT
   license_family: MIT
-  license_file: 
-  summary: "Microsoft Azure Storage Client Library for Python"
+  license_file:
+  summary: Microsoft Azure Storage Client Library for Python{{ environ.get('STORAGE_SUMMARY', '') }}
   doc_url: 
   dev_url: 
 
 extra:
   recipe-maintainers:
-    - your-github-id-here
+    - lmazuel,xiangyan99,scbedd