Update Javadoc Linking and Groups (Azure#20058)

alzimmermsft · web-flow · commit f7a3b48f15e7 · 2021-03-24T17:12:14.000Z
Added missing Javadoc links, parallelized build and deploy job, and updated test jobs
diff --git a/eng/pipelines/scripts/inject_codesnippets.py b/eng/pipelines/scripts/inject_codesnippets.py
@@ -60,7 +60,7 @@ def finalize_snippet(self, key):
 
 
 def re_space_snippet(snippet_list):
-    # find identation (or whitespace characters) on the left side
+    # find indentation (or whitespace characters) on the left side
     white_space = [
         re.match(WHITESPACE_EXTRACTION, line).groupdict()["leadingspace"]
         for line in snippet_list
@@ -74,25 +74,28 @@ def re_space_snippet(snippet_list):
     return [line.replace(white_space_for_replacement, "", 1) for line in snippet_list]
 
 
-def get_snippets_from_file(file):
+def get_snippets_from_file(file, verbose):
     finished_snippets = {}
     running_dict = SnippetDict()
 
     with open(file, "r", encoding="utf-8") as source:
-        print(file)
+        if verbose:
+            print(file)
         for line in source.readlines():
 
             begin = re.match(SNIPPET_BEGIN, line)
             end = re.match(SNIPPET_END, line)
 
             if begin:
                 id_beginning = begin.groupdict()["id"]
-                print("beginning {}".format(id_beginning))
+                if verbose:
+                    print("beginning {}".format(id_beginning))
                 running_dict.begin_snippet(id_beginning)
             elif end:
                 id_ending = end.groupdict()["id"]
                 ending = running_dict.finalize_snippet(id_ending)
-                print("ending {}".format(id_ending))
+                if verbose:
+                    print("ending {}".format(id_ending))
                 finished_snippets[id_ending] = ending
             elif running_dict:
                 running_dict.process_line(line)
@@ -118,6 +121,14 @@ def check_exclusion(file_name, exclusion_array):
         help="The path to the directory containing our package or service. Essentially used as a scoping mechanism for the replacement of the snippets.",
         required=True,
     )
+    parser.add_argument(
+        "--verbose",
+        "-v",
+        dest="verbose",
+        help="Flag indicating if verbose, debug level, output should be printed.",
+        required=False,
+        default=False
+    )
     args = parser.parse_args()
 
     # walk the codebase, find all java files
@@ -132,7 +143,7 @@ def check_exclusion(file_name, exclusion_array):
     snippets = {}
 
     for file in snippet_files:
-        snippet_dict = get_snippets_from_file(file)
+        snippet_dict = get_snippets_from_file(file, args.verbose)
         snippets.update(snippet_dict)
 
     for file in all_files:
@@ -174,4 +185,3 @@ def check_exclusion(file_name, exclusion_array):
             with open(file, "w", encoding="utf-8") as out_file:
                 for line in amended_file:
                     out_file.write(line)
- 
diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml
@@ -17,12 +17,12 @@ parameters:
   - name: PreTestSteps
     type: object
     default: []
+  - name: BuildOptions
+    type: string
+    default: '$(DefaultOptions) -T 1C -DskipTests -Dgpg.skip -DtrimStackTrace=false -Dmaven.javadoc.skip=true -Dspotbugs.skip=true -Dcheckstyle.skip=true -Drevapi.skip=true -Djacoco.skip=true'
   - name: TestOptions
     type: string
     default: '$(DefaultOptions) -DtrimStackTrace=false -Dmaven.javadoc.skip=true -Dspotbugs.skip=true -Dcheckstyle.skip=true -Drevapi.skip=true -Dparallel-test-playback'
-  - name: TestGoals
-    type: string
-    default: verify
   - name: Matrix
     type: object
   - name: CloudConfig
@@ -106,17 +106,31 @@ jobs:
           arguments: '--project-list $(ProjectList)'
         condition: and(succeeded(), eq(variables['ShouldRunSourceTests'],'true'))
 
+      - task: Maven@3
+        displayName: 'Build for non-From Source run'
+        inputs:
+          mavenPomFile: pom.xml
+          options: ${{ parameters.BuildOptions }} -pl $(PLSkipOptions)$(ProjectList) -am
+          mavenOptions: '$(MemoryOptions) $(LoggingOptions)'
+          javaHomeOption: 'JDKVersion'
+          jdkVersionOption: $(JavaTestVersion)
+          jdkArchitectureOption: 'x64'
+          publishJUnitResults: false
+          goals: 'install'
+        # we want to run this when we aren't running from source and aren't generating Jacoco test coverage
+        condition: and(succeeded(), ne(variables['TestFromSource'], 'true'), not(and(eq(variables['Agent.OS'], 'Windows_NT'), eq(variables['JavaTestVersion'], '1.11'), eq('${{ parameters.SDKType }}', 'client'))))
+
       - task: Maven@3
         displayName: 'Run tests'
         inputs:
           mavenPomFile: pom.xml
-          options: ${{ parameters.TestOptions }} -pl $(PLSkipOptions)$(ProjectList) -am $(AdditionalOptions)
+          options: ${{ parameters.TestOptions }} -pl $(PLSkipOptions)$(ProjectList) $(AdditionalOptions)
           mavenOptions: '$(MemoryOptions) $(LoggingOptions)'
           javaHomeOption: 'JDKVersion'
           jdkVersionOption: $(JavaTestVersion)
           jdkArchitectureOption: 'x64'
           publishJUnitResults: false
-          goals: ${{ parameters.TestGoals }}
+          goals: $(TestGoals)
         # we want to run this when TestFromSource isn't true
         condition: and(succeeded(), ne(variables['TestFromSource'],'true'))
 
@@ -146,7 +160,7 @@ jobs:
           # dependency. Note: The -T 1C says spin up a build thread per core. Since we're building
           # all of the track 2, client, libraries (which is now over 100) this effectively cuts the
           # the build time in half.
-          options: '$(DefaultOptions) -T 1C -DskipTests -Dgpg.skip -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dspotbugs.skip=true -Drevapi.skip=true'
+          options: ${{ parameters.BuildOptions }}
           mavenOptions: '$(MemoryOptions) $(LoggingOptions)'
           javaHomeOption: 'JDKVersion'
           jdkVersionOption: $(JavaBuildVersion)
@@ -170,7 +184,7 @@ jobs:
           # as well as libraries which depend on them and not the other stages in the Maven
           # lifecycle that test or verify targets would execute. The libraries were already
           # compiled and installed in the previous task, this tasks only runs tests.
-          goals: 'surefire:test'
+          goals: $(TestGoals)
         condition: and(succeeded(), and(eq(variables['TestFromSource'],'true'), eq(variables['ShouldRunSourceTests'],'true')))
 
       - template: ../steps/upload-repository-on-failure.yml
diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml
@@ -17,12 +17,6 @@ parameters:
   - name: PreTestSteps
     type: object
     default: []
-  - name: TestOptions
-    type: string
-    default: '$(DefaultOptions) -DtrimStackTrace=false -Dmaven.javadoc.skip=true -Dspotbugs.skip=true -Dcheckstyle.skip=true -Drevapi.skip=true -Dparallel-test-playback'
-  - name: TestGoals
-    type: string
-    default: verify
   - name: MatrixConfigs
     type: object
   - name: AdditionalMatrixConfigs
@@ -105,11 +99,11 @@ jobs:
           AdditionalModules: ${{parameters.AdditionalModules}}
 
       - task: Maven@3
-        displayName: 'Build and Package , JDK Version: $(JavaBuildVersion)'
+        displayName: 'Build and Package, JDK Version: $(JavaBuildVersion)'
         inputs:
           mavenPomFile: pom.xml
           goals: 'deploy'
-          options: '$(DefaultOptions) -DskipTests -Dinject-codesnippets -Dgenerate-overview -Dspotbugs.skip=true -Dcheckstyle.skip=true -Drevapi.skip=true -pl $(ProjectList) -am -DaltDeploymentRepository=id::default::file://$(System.DefaultWorkingDirectory)/build' # We include template-module so we ensure it always builds in CI
+          options: '$(DefaultOptions) -T 1C -DskipTests -Dinject-codesnippets -Dgenerate-overview -Dspotbugs.skip=true -Dcheckstyle.skip=true -Drevapi.skip=true -pl $(ProjectList) -am -DaltDeploymentRepository=id::default::file://$(System.DefaultWorkingDirectory)/build' # We include template-module so we ensure it always builds in CI
           mavenOptions: '$(MemoryOptions) $(LoggingOptions)'
           javaHomeOption: 'JDKVersion'
           jdkVersionOption: $(JavaBuildVersion)
@@ -242,5 +236,3 @@ jobs:
         Artifacts: ${{ parameters.Artifacts }}
         AdditionalModules: ${{ parameters.AdditionalModules }}
         PreTestSteps: ${{ parameters.PreTestSteps }}
-        TestOptions: ${{ parameters.TestOptions }}
-        TestGoals: ${{ parameters.TestGoals }}
diff --git a/eng/pipelines/templates/stages/platform-matrix.json b/eng/pipelines/templates/stages/platform-matrix.json
@@ -10,8 +10,15 @@
     },
     "JavaTestVersion": [ "1.8", "1.11" ],
     "AZURE_TEST_HTTP_CLIENTS": [ "okhttp", "netty" ],
+    "TestGoals": "surefire:test",
     "TestOptions": ""
   },
+  "exclude": [
+    {
+      "Pool": "azsdk-pool-mms-win-2019-general",
+      "JavaTestVersion": "1.11"
+    }
+  ],
   "include": [
     {
       "Agent": {
@@ -20,6 +27,16 @@
       "JavaTestVersion": "1.11",
       "AZURE_TEST_HTTP_CLIENTS": "netty",
       "TestFromSource": true,
+      "TestGoals": "surefire:test",
+      "TestOptions": ""
+    },
+    {
+      "Agent": {
+        "windows-2019": { "OSVmImage": "MMS2019", "Pool": "azsdk-pool-mms-win-2019-general" }
+      },
+      "JavaTestVersion": "1.11",
+      "AZURE_TEST_HTTP_CLIENTS": "netty",
+      "TestGoals": "verify",
       "TestOptions": ""
     }
   ]
diff --git a/sdk/core/azure-core-http-netty/pom.xml b/sdk/core/azure-core-http-netty/pom.xml
@@ -199,6 +199,19 @@
         </configuration>
       </plugin>
 
+      <!-- This plugin generates Javadocs -->
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-javadoc-plugin</artifactId>
+        <version>3.1.1</version> <!-- {x-version-update;org.apache.maven.plugins:maven-javadoc-plugin;external_dependency} -->
+        <configuration>
+          <links>
+            <link>https://netty.io/4.1/api/</link>
+            <link>https://projectreactor.io/docs/netty/release/api/</link>
+          </links>
+        </configuration>
+      </plugin>
+
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-compiler-plugin</artifactId>
diff --git a/sdk/core/azure-core-serializer-avro-apache/pom.xml b/sdk/core/azure-core-serializer-avro-apache/pom.xml
@@ -115,6 +115,18 @@
         </configuration>
       </plugin>
 
+      <!-- This plugin generates Javadocs -->
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-javadoc-plugin</artifactId>
+        <version>3.1.1</version> <!-- {x-version-update;org.apache.maven.plugins:maven-javadoc-plugin;external_dependency} -->
+        <configuration>
+          <links>
+            <link>https://avro.apache.org/docs/current/api/java/</link>
+          </links>
+        </configuration>
+      </plugin>
+
       <plugin>
         <groupId>org.apache.avro</groupId>
         <artifactId>avro-maven-plugin</artifactId>
diff --git a/sdk/core/azure-core-serializer-json-jackson/pom.xml b/sdk/core/azure-core-serializer-json-jackson/pom.xml
@@ -105,6 +105,20 @@
           </compilerArgs>
         </configuration>
       </plugin>
+
+      <!-- This plugin generates Javadocs -->
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-javadoc-plugin</artifactId>
+        <version>3.1.1</version> <!-- {x-version-update;org.apache.maven.plugins:maven-javadoc-plugin;external_dependency} -->
+        <configuration>
+          <links>
+            <link>https://www.javadoc.io/doc/com.fasterxml.jackson.core/jackson-annotations/latest/</link>
+            <link>https://www.javadoc.io/doc/com.fasterxml.jackson.core/jackson-core/latest/</link>
+            <link>https://www.javadoc.io/doc/com.fasterxml.jackson.core/jackson-databind/latest/</link>
+          </links>
+        </configuration>
+      </plugin>
     </plugins>
   </build>
 
diff --git a/sdk/core/azure-core/pom.xml b/sdk/core/azure-core/pom.xml
@@ -202,6 +202,20 @@
         </configuration>
       </plugin>
 
+      <!-- This plugin generates Javadocs -->
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-javadoc-plugin</artifactId>
+        <version>3.1.1</version> <!-- {x-version-update;org.apache.maven.plugins:maven-javadoc-plugin;external_dependency} -->
+        <configuration>
+          <links>
+            <link>https://www.javadoc.io/doc/com.fasterxml.jackson.core/jackson-annotations/latest/</link>
+            <link>https://www.javadoc.io/doc/com.fasterxml.jackson.core/jackson-core/latest/</link>
+            <link>https://www.javadoc.io/doc/com.fasterxml.jackson.core/jackson-databind/latest/</link>
+          </links>
+        </configuration>
+      </plugin>
+
       <!-- This plugin scans reports spotbugs in the code -->
       <plugin>
         <groupId>com.github.spotbugs</groupId>
diff --git a/sdk/parents/azure-client-sdk-parent/pom.xml b/sdk/parents/azure-client-sdk-parent/pom.xml
diff --git a/sdk/parents/azure-data-sdk-parent/pom.xml b/sdk/parents/azure-data-sdk-parent/pom.xml
