[dev-tool] Enable sample folders to use nested directory structures. (Azure#15110)

* [dev-tool] Added support for nested sample files.

* Tweak regexp for removing empty lines at beginning of JSDoc comments

* Fix table generation to allow nested filenames for link tags.

Author: witemple-msft

common/tools/dev-tool/src/commands/samples/publish.ts

@@ -37,6 +37,8 @@ import instantiateSampleReadme from "../../templates/sampleReadme.md";
 import { convert } from "./tsToJs";
 
 import devToolPackageJson from "../../../package.json";
+import { findMatchingFiles } from "../../util/findMatchingFiles";
+import { EOL } from "os";
 
 const log = createPrinter("publish");
 
@@ -194,23 +196,26 @@ async function processSources(
         const tags = ts.getJSDocTags(node);
 
         // Look for the @summary jsdoc tag block as well
-        if (summary === undefined && tags.some(({ tagName: { text } }) => text === "summary")) {
+        if (summary === undefined) {
           for (const tag of tags) {
+            log.debug(`File ${relativeSourcePath} has tag ${tag.tagName.text}`);
             if (tag.tagName.text === "summary") {
               log.debug("Found summary tag on node:", node.getText(sourceFile));
               // Replace is required due to multi-line splitting messing with table formatting
               summary = tag.comment?.replace(/\s*\r?\n\s*/g, " ");
-            } else if (
-              tag.tagName.text.startsWith(`${AZSDK_META_TAG_PREFIX}`) &&
-              tag.comment !== undefined
-            ) {
+            } else if (tag.tagName.text.startsWith(`${AZSDK_META_TAG_PREFIX}`)) {
               // We ran into an `azsdk` directive in the metadata
               const metaTag = tag.tagName.text.replace(
                 new RegExp(`^${AZSDK_META_TAG_PREFIX}`),
                 ""
               ) as keyof AzSdkMetaTags;
+              log.debug(`File ${relativeSourcePath} has azsdk tag ${tag.tagName.text}`);
               if (VALID_AZSDK_META_TAGS.includes(metaTag)) {
-                azSdkTags[metaTag as keyof AzSdkMetaTags] = JSON.parse(tag.comment);
+                const comment = tag.comment?.trim();
+                // If there was _no_ comment, then we can assume it is a boolean tag
+                // and so being specified at all is an indication that we should use
+                // `true`
+                azSdkTags[metaTag as keyof AzSdkMetaTags] = comment ? JSON.parse(comment) : true;
               } else {
                 log.warn(
                   `Invalid azsdk tag ${metaTag}. Valid tags include ${VALID_AZSDK_META_TAGS}`
@@ -228,17 +233,26 @@ async function processSources(
       return sourceFile;
     };
 
+    const jsModuleText = convert(sourceText, {
+      fileName: source,
+      transformers: {
+        before: [sourceProcessor]
+      }
+    });
+
+    if (summary === undefined && azSdkTags.util !== true) {
+      log.debug(azSdkTags.util, summary);
+      fail(
+        `${relativeSourcePath} does not include an @summary tag and is not marked as a util (using @azsdk-util true).`
+      );
+    }
+
     return {
       filePath: source,
       relativeSourcePath,
       text: sourceText,
-      jsModuleText: convert(sourceText, {
-        fileName: source,
-        transformers: {
-          before: [sourceProcessor]
-        }
-      }),
-      summary: summary ?? fail(`${relativeSourcePath} does not include an @summary tag.`),
+      jsModuleText,
+      summary,
       importedModules: importedModules.filter(isDependency),
       usedEnvironmentVariables,
       azSdkTags
@@ -248,6 +262,16 @@ async function processSources(
   return Promise.all(jobs);
 }
 
+async function collect<T>(i: AsyncIterableIterator<T>): Promise<T[]> {
+  const out = [];
+
+  for await (const v of i) {
+    out.push(v);
+  }
+
+  return out;
+}
+
 /**
  * Extracts the sample generation metainformation from the sample sources and
  * configuration in package.json.
@@ -261,9 +285,10 @@ async function makeSampleGenerationInfo(
   onError: () => void
 ): Promise<SampleGenerationInfo> {
   const sampleSourcesPath = path.join(projectInfo.path, DEV_SAMPLES_BASE);
-  const sampleSources = (await fs.readdir(sampleSourcesPath))
-    .filter((name) => name.endsWith(".ts"))
-    .map((name) => path.join(sampleSourcesPath, name));
+
+  const sampleSources = await collect(
+    findMatchingFiles(sampleSourcesPath, (name) => name.endsWith(".ts"))
+  );
 
   const sampleConfiguration = getSampleConfiguration(projectInfo.packageJson);
 
@@ -398,7 +423,8 @@ function createReadme(outputKind: OutputKind, info: SampleGenerationInfo): strin
         },
     publicationDirectory: PUBLIC_SAMPLES_BASE + "/" + info.topLevelDirectory,
     useTypeScript: outputKind === OutputKind.TypeScript,
-    ...info
+    ...info,
+    moduleInfos: info.moduleInfos.filter((mod) => mod.summary !== undefined)
   });
 }
 
@@ -440,7 +466,18 @@ async function makeSamplesFactory(
    */
   function postProcess(moduleText: string | Buffer): string {
     const content = Buffer.isBuffer(moduleText) ? moduleText.toString("utf8") : moduleText;
-    return content.replace(new RegExp(`^\\s*\\*\\s*@${AZSDK_META_TAG_PREFIX}.*\n`, "gm"), "");
+    return (
+      content
+        .replace(new RegExp(`^\\s*\\*\\s*@${AZSDK_META_TAG_PREFIX}.*\n`, "gm"), "")
+        // We also need to clean up extra blank lines that might be left behind by
+        // removing azsdk tags. These regular expressions are extremely frustrating
+        // because they deal almost exclusively in the literal "/" and "*" characters.
+        .replace(/(\s+\*)+\//s, EOL + " */")
+        // Clean up blank lines at the beginning
+        .replace(/\/\*\*(\s+\*)*/s, `/**${EOL} *`)
+        // Finally remove empty doc comments.
+        .replace(/\s*\/\*\*(\s+\*)*\/\s*/s, EOL + EOL)
+    );
   }
 
   // We use a tempdir at the outer layer to avoid creating dirty trees
@@ -455,8 +492,8 @@ async function makeSamplesFactory(
         // We copy the samples sources in to the `src` folder on the typescript side
         dir(
           "src",
-          info.moduleInfos.map(({ filePath }) =>
-            file(path.basename(filePath), () => postProcess(fs.readFileSync(filePath)))
+          info.moduleInfos.map(({ relativeSourcePath, filePath }) =>
+            file(relativeSourcePath, () => postProcess(fs.readFileSync(filePath)))
           )
         )
       ]),
@@ -465,8 +502,8 @@ async function makeSamplesFactory(
         file("package.json", () => jsonify(createPackageJson(info, OutputKind.JavaScript))),
         copy("sample.env", path.join(projectInfo.path, "sample.env")),
         // Extract the JS Module Text from the module info structures
-        ...info.moduleInfos.map(({ filePath, jsModuleText }) =>
-          file(path.basename(filePath).replace(/\.ts$/, ".js"), () => postProcess(jsModuleText))
+        ...info.moduleInfos.map(({ relativeSourcePath, jsModuleText }) =>
+          file(relativeSourcePath.replace(/\.ts$/, ".js"), () => postProcess(jsModuleText))
         )
       ]),
       // Copy extraFiles by reducing all configured destinations for each input file

common/tools/dev-tool/src/config/rollup.base.config.ts

@@ -39,25 +39,22 @@ export function openTelemetryCommonJs(): Record<string, string[]> {
     ];
   }
 
-  const releasedOpenTelemetryVersions = [
-    "0.10.2",
-    "1.0.0-rc.0"
-  ];
+  const releasedOpenTelemetryVersions = ["0.10.2", "1.0.0-rc.0"];
 
   for (const version of releasedOpenTelemetryVersions) {
     namedExports[
       // working around a limitation in the rollup common.js plugin - it's not able to resolve these modules so the named exports listed above will not get applied. We have to drill down to the actual path.
       `../../../common/temp/node_modules/.pnpm/@opentelemetry/api@${version}/node_modules/@opentelemetry/api/build/src/index.js`
     ] = [
-        "SpanKind",
-        "TraceFlags",
-        "getSpan",
-        "setSpan",
-        "StatusCode",
-        "CanonicalCode",
-        "getSpanContext",
-        "setSpanContext"
-      ];
+      "SpanKind",
+      "TraceFlags",
+      "getSpan",
+      "setSpan",
+      "StatusCode",
+      "CanonicalCode",
+      "getSpanContext",
+      "setSpanContext"
+    ];
   }
 
   return namedExports;

common/tools/dev-tool/src/templates/sampleReadme.md.ts

@@ -37,10 +37,10 @@ function fence(language: string, ...contents: string[]): string {
  * Ex. recognizePii.ts -> recognizepii
  */
 function sampleLinkTag(filePath: string): string {
-  return path
-    .basename(filePath)
+  return filePath
+    .split(path.sep)
+    .join("_")
     .replace(/\.[a-z]*$/, "")
-    .replace(/\//, "_")
     .toLowerCase();
 }
 
@@ -55,12 +55,12 @@ function fileLinks(info: SampleReadmeConfiguration) {
   ].join("/");
 
   return filterModules(info)
-    .map(({ filePath, relativeSourcePath }) => {
+    .map(({ relativeSourcePath }) => {
       const sourcePath = info.useTypeScript
         ? relativeSourcePath
         : relativeSourcePath.replace(/\.ts$/, ".js");
       return `[${sampleLinkTag(
-        filePath
+        relativeSourcePath
       )}]: https://github.com/Azure/azure-sdk-for-js/blob/master/${packageSamplesPathFragment}/${sourcePath}`;
     })
     .join("\n");
@@ -115,11 +115,11 @@ function filterModules(info: SampleReadmeConfiguration): SampleReadmeConfigurati
  * Renders the sample file table.
  */
 function table(info: SampleReadmeConfiguration) {
-  const contents = filterModules(info).map(({ filePath, summary, relativeSourcePath }) => {
+  const contents = filterModules(info).map(({ summary, relativeSourcePath }) => {
     const fileName = info.useTypeScript
       ? relativeSourcePath
       : relativeSourcePath.replace(/\.ts$/, ".js");
-    return `| [${fileName}][${sampleLinkTag(filePath)}] | ${summary} |`;
+    return `| [${fileName}][${sampleLinkTag(relativeSourcePath)}] | ${summary} |`;
   });
 
   return [

common/tools/dev-tool/src/util/fileTree.ts

@@ -104,5 +104,9 @@ export function file(name: string, contents: FileContents): FileTreeFactory {
       : Buffer.from(immediateContents, "utf8");
   };
 
-  return async (basePath) => fs.writeFile(path.join(basePath, name), getContentsAsBuffer());
+  return async (basePath) => {
+    const dirName = path.resolve(basePath, path.dirname(name));
+    await fs.ensureDir(dirName);
+    return fs.writeFile(path.join(basePath, name), getContentsAsBuffer());
+  };
 }

common/tools/dev-tool/src/util/sampleGenerationInfo.ts

@@ -121,7 +121,7 @@ export interface ModuleInfo {
   /**
    * The description provided by the first doc comment.
    */
-  summary: string;
+  summary?: string;
   /**
    * A list of module specifiers that are imported by this
    * source file.
@@ -186,6 +186,10 @@ export interface AzSdkMetaTags {
    * Causes the sample file to be ignored entirely (will skip publication).
    */
   ignore?: boolean;
+  /**
+   * Causes the file to be omitted from the generated sample index (README).
+   */
+  util?: boolean;
   /**
    * Causes the sample file to skip JavaScript output.
    */
@@ -206,6 +210,7 @@ export const AZSDK_META_TAG_PREFIX = "azsdk-";
 export const VALID_AZSDK_META_TAGS: Array<keyof AzSdkMetaTags> = [
   "weight",
   "ignore",
+  "util",
   "skip-javascript"
 ];