Resolve From Source Compilation Failure Caused by Spark Modules (Azure#26628)

Resolve From Source Compilation Failure Caused by Spark Modules

Author: alzimmermsft

eng/.docsettings.yml

@@ -44,7 +44,6 @@ known_presence_issues:
   - ['sdk/cognitiveservices/ms-azure-cs-videosearch', '#2847']
   - ['sdk/cognitiveservices/ms-azure-cs-visualsearch', '#2847']
   - ['sdk/cognitiveservices/ms-azure-cs-websearch', '#2847']
-  - ['sdk/cosmos/azure-cosmos-spark_3_2-12', '#2847']
   - ['sdk/eventhubs/microsoft-azure-eventhubs-extensions', '#2847']
   - ['sdk/keyvault/microsoft-azure-keyvault', '#2847']
   - ['sdk/keyvault/microsoft-azure-keyvault-core', '#2847']
@@ -111,6 +110,7 @@ known_content_issues:
   - ['sdk/cosmos/azure-cosmos-examples/README.md', '#3113']
   - ['sdk/cosmos/azure-cosmos/README.md', '#3113']
   - ['sdk/cosmos/azure-cosmos-encryption/README.md', '#3113']
+  - ['sdk/cosmos/azure-cosmos-spark_3_2-12/README.md', '#3113']
   - ['sdk/cosmos/azure-cosmos-spark_3-1_2-12/README.md', '#3113']
   - ['sdk/cosmos/azure-cosmos-spark_3-2_2-12/README.md', '#3113']
   - ['sdk/cosmos/azure-cosmos-spark_3_2-12/dev/README.md', '#3113']

eng/pipelines/scripts/generate_overview_from_readme.py

@@ -34,6 +34,8 @@ def generate_overview(readme_file, version, overview_file_path):
         print('{} does not exist'.format(readme_file))
 
     if overview_file_path:
+        if not os.path.exists(overview_file_path):
+            os.makedirs(overview_file_path)
         html_overview_file = overview_file_path + 'readme_overview.html'
     else:
         html_overview_file = str(readme_file).lower().replace('readme.md', 'readme_overview.html')

eng/pipelines/templates/jobs/ci.tests.yml

@@ -19,7 +19,7 @@ parameters:
     default: []
   - name: BuildOptions
     type: string
-    default: '$(DefaultOptions) -T 2C -DskipTests -Dgpg.skip -DtrimStackTrace=false -Dmaven.javadoc.skip=true -Dcodesnippet.skip=true -Dspotbugs.skip=true -Dcheckstyle.skip=true -Drevapi.skip=true -Djacoco.skip=true'
+    default: '$(DefaultOptions) -T 2C --quiet -DskipTests -Dgpg.skip -DtrimStackTrace=false -Dmaven.javadoc.skip=true -Dcodesnippet.skip=true -Dspotbugs.skip=true -Dcheckstyle.skip=true -Drevapi.skip=true -Djacoco.skip=true'
   - name: TestOptions
     type: string
     default: '$(DefaultTestOptions) -Dparallel-test-playback -DAZURE_TEST_DEBUG=$(IsDebug)'

eng/pipelines/templates/jobs/ci.yml

@@ -151,7 +151,7 @@ jobs:
         inputs:
           mavenPomFile: pom.xml
           goals: 'deploy'
-          options: '$(DefaultOptions) -T 1C -DskipTests -Dgenerate-overview -Dspotbugs.skip=true -Dcheckstyle.skip=true -Drevapi.skip=true -pl $(ProjectList) -am -DaltDeploymentRepository=id::default::file://$(System.DefaultWorkingDirectory)/build' # We include template-module so we ensure it always builds in CI
+          options: '$(DefaultOptions) -T 1C --quiet -DskipTests -Dgenerate-overview -Dspotbugs.skip=true -Dcheckstyle.skip=true -Drevapi.skip=true -pl $(ProjectList) -am -DaltDeploymentRepository=id::default::file://$(System.DefaultWorkingDirectory)/build' # We include template-module so we ensure it always builds in CI
           mavenOptions: '$(MemoryOptions) $(LoggingOptions)'
           javaHomeOption: 'JDKVersion'
           jdkVersionOption: $(JavaBuildVersion)
@@ -300,7 +300,7 @@ jobs:
         inputs:
           mavenPomFile: pom.xml
           goals: 'install'
-          options: '$(DefaultOptions) -T 2C -DskipTests -Dgpg.skip -Dmaven.javadoc.skip=true -Dcodesnippet.skip=true -Dcheckstyle.skip=true -Dspotbugs.skip=true -Drevapi.skip=true -pl $(ProjectList) -am'
+          options: '$(DefaultOptions) -T 2C --quiet -DskipTests -Dgpg.skip -Dmaven.javadoc.skip=true -Dcodesnippet.skip=true -Dcheckstyle.skip=true -Dspotbugs.skip=true -Drevapi.skip=true -pl $(ProjectList) -am'
           mavenOptions: '$(MemoryOptions) $(LoggingOptions)'
           javaHomeOption: 'JDKVersion'
           jdkVersionOption: $(JavaBuildVersion)

eng/pipelines/templates/stages/cosmos-emulator-matrix.json

@@ -29,7 +29,7 @@
           "AdditionalArgs": "-DargLine=\"-DACCOUNT_HOST=https://localhost:8081/\""
       },
       "Spark Integration Tests targeting Cosmos Emulator'": {
-          "ProfileFlag": "-PsparkE2E",
+          "ProfileFlag": "-Dspark-e2e",
           "PROTOCOLS": "[\"Tcp\"]",
           "DESIRED_CONSISTENCIES": "[\"Session\"]",
           "AdditionalArgs": "-DargLine=\"-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop\""

eng/scripts/generate_from_source_pom.py

@@ -46,10 +46,10 @@ def add_dependent(self, dependent: str):
 
 # azure-client-sdk-parent, azure-perf-test-parent, spring-boot-starter-parent, and azure-spring-boot-test-parent are
 # valid parent POMs for Track 2 libraries.
-valid_parents = ['com.azure:azure-client-sdk-parent', 'com.azure:azure-perf-test-parent', 'org.springframework.boot:spring-boot-starter-parent', 'com.azure.spring:azure-spring-boot-test-parent']
+valid_parents = ['com.azure:azure-client-sdk-parent', 'com.azure:azure-perf-test-parent', 'org.springframework.boot:spring-boot-starter-parent', 'com.azure.spring:azure-spring-boot-test-parent', 'com.azure.cosmos.spark:azure-cosmos-spark_3_2-12']
 
 # List of parent POMs that should be retained as projects to create a full from source POM.
-parent_pom_identifiers = ['com.azure:azure-sdk-parent', 'com.azure:azure-client-sdk-parent', 'com.azure:azure-perf-test-parent', 'com.azure.spring:azure-spring-boot-test-parent']
+parent_pom_identifiers = ['com.azure:azure-sdk-parent', 'com.azure:azure-client-sdk-parent', 'com.azure:azure-perf-test-parent', 'com.azure.spring:azure-spring-boot-test-parent', 'com.azure.cosmos.spark:azure-cosmos-spark_3_2-12']
 
 # From this file get to the root path of the repo.
 root_path = os.path.normpath(os.path.abspath(__file__) + '/../../../')
@@ -109,7 +109,7 @@ def create_from_source_pom(project_list: str, set_pipeline_variable: str, set_sk
     add_source_projects(source_projects, dependent_modules, projects)
     add_source_projects(source_projects, dependency_modules, projects)
 
-    modules = list(set(sorted([p.module_path for p in source_projects])))
+    modules = sorted(list(set([p.module_path for p in source_projects])))
     with open(file=client_from_source_pom_path, mode='w') as fromSourcePom:
         fromSourcePom.write(pom_file_start)
 

eng/versioning/external_dependencies.txt

@@ -332,6 +332,8 @@ cosmos_org.scalatest:scalatest-maven-plugin;2.0.2
 cosmos_net.alchim31.maven:scala-maven-plugin;4.5.4
 cosmos_org.scalastyle:scalastyle-maven-plugin;1.0.0
 cosmos_org.apache.maven.plugins:maven-shade-plugin;1.2.7
+cosmos_org.codehaus.mojo:flatten-maven-plugin;1.2.7
+cosmos_com.coderplus.maven.plugins:copy-rename-maven-plugin;1.0.1
 
 # sdk\core\azure-core-serializer-avro-jackson\pom.xml
 # This dependency is needed since Jackson Avro uses an older dependency on Apache Avro which is another library.

eng/versioning/version_client.txt

@@ -81,6 +81,7 @@ com.azure:azure-core-tracing-opentelemetry;1.0.0-beta.19;1.0.0-beta.20
 com.azure:azure-cosmos;4.25.0;4.26.0-beta.1
 com.azure:azure-cosmos-benchmark;4.0.1-beta.1;4.0.1-beta.1
 com.azure:azure-cosmos-dotnet-benchmark;4.0.1-beta.1;4.0.1-beta.1
+com.azure.cosmos.spark:azure-cosmos-spark_3_2-12;1.0.0-beta.1;1.0.0-beta.1
 com.azure.cosmos.spark:azure-cosmos-spark_3-1_2-12;4.6.0-beta.1;4.6.0-beta.1
 com.azure.cosmos.spark:azure-cosmos-spark_3-2_2-12;4.6.0-beta.1;4.6.0-beta.1
 com.azure:azure-cosmos-encryption;1.0.0-beta.9;1.0.0-beta.10

sdk/communication/pom.xml

@@ -31,7 +31,7 @@
         <dependency>
           <groupId>com.azure</groupId>
           <artifactId>azure-communication-chat</artifactId>
-          <version>1.2.0-beta.1</version> <!-- {x-version-update;com.azure:azure-communication-chat;current} -->
+          <version>1.1.3</version> <!-- {x-version-update;com.azure:azure-communication-chat;current} -->
         </dependency>
         <dependency>
           <groupId>com.azure</groupId>