add transformer pattern & rest transformer for API (#58388)

Co-authored-by: Kevin Heis <heiskr@users.noreply.github.com>

Author: Ebonsignori

src/article-api/README.md

@@ -13,12 +13,24 @@ Article API endpoints allow consumers to query GitHub Docs for listings of curre
 
 The `/api/article/meta` endpoint powers hovercards, which provide a preview for internal links on <docs.github.com>.
 
+The `/api/article/body` endpoint can serve markdown for both regular articles and autogenerated content (such as REST API documentation) using specialized transformers.
+
 ## How it works
 
 The `/api/article` endpoints return information about a page by `pathname`. 
 
 `api/article/meta` is highly cached, in JSON format.
 
+### Autogenerated Content Transformers
+
+For autogenerated pages (REST, landing pages, audit logs, webhooks, GraphQL, etc), the Article API uses specialized transformers to convert the rendered content into markdown format. These transformers are located in `src/article-api/transformers/` and use an extensible architecture:
+
+To add a new transformer for other autogenerated content types:
+1. Create a new transformer file implementing the `PageTransformer` interface
+2. Register it in `transformers/index.ts`
+3. Create a template in `templates/` to configure how the transformer will organize the autogenerated content 
+4. The transformer will automatically be used by `/api/article/body`
+
 ## How to get help
 
 For internal folks ask in the Docs Engineering slack channel. 
@@ -34,12 +46,13 @@ Get article metadata and content in a single object. Equivalent to calling `/art
 
 **Parameters**:
 - **pathname** (string) - Article path (e.g. '/en/get-started/article-name')
+- **[apiVersion]** (string) - API version for REST pages (optional, defaults to latest)
 
 **Returns**: (object) - JSON object with article metadata and content (`meta` and `body` keys)
 
 **Throws**:
 - (Error): 403 - If the article body cannot be retrieved. Reason is given in the error message.
-- (Error): 400 - If pathname parameter is invalid.
+- (Error): 400 - If pathname or apiVersion parameters are invalid.
 - (Error): 404 - If the path is valid, but the page couldn't be resolved.
 
 **Example**:
@@ -63,12 +76,13 @@ Get the contents of an article's body.
 
 **Parameters**:
 - **pathname** (string) - Article path (e.g. '/en/get-started/article-name')
+- **[apiVersion]** (string) - API version (optional, defaults to latest)
 
 **Returns**: (string) - Article body content in markdown format.
 
 **Throws**:
 - (Error): 403 - If the article body cannot be retrieved. Reason is given in the error message.
-- (Error): 400 - If pathname parameter is invalid.
+- (Error): 400 - If pathname or apiVersion parameters are invalid.
 - (Error): 404 - If the path is valid, but the page couldn't be resolved.
 
 **Example**:

src/article-api/liquid-renderers/index.ts

@@ -0,0 +1,16 @@
+/**
+ * API Transformer Liquid Tags
+ *
+ * This module contains custom Liquid tags used by article-api transformers
+ * to render API documentation in a consistent format.
+ */
+
+import { restTags } from './rest-tags'
+
+// Export all API transformer tags for registration
+export const apiTransformerTags = {
+  ...restTags,
+}
+
+// Re-export individual tag modules for direct access if needed
+export { restTags } from './rest-tags'

src/article-api/liquid-renderers/rest-tags.ts

@@ -0,0 +1,230 @@
+import type { TagToken, Context as LiquidContext } from 'liquidjs'
+import { fastTextOnly } from '@/content-render/unified/text-only'
+import { renderContent } from '@/content-render/index'
+import type { Context } from '@/types'
+import type { Parameter, BodyParameter, ChildParameter, StatusCode } from '@/rest/components/types'
+import { createLogger } from '@/observability/logger'
+
+const logger = createLogger('article-api/liquid-renderers/rest-tags')
+
+/**
+ * Custom Liquid tag for rendering REST API parameters
+ * Usage: {% rest_parameter param %}
+ */
+export class RestParameter {
+  private paramName: string
+
+  constructor(
+    token: TagToken,
+    remainTokens: TagToken[],
+    liquid: { options: any; parser: any },
+    private liquidContext?: LiquidContext,
+  ) {
+    // The tag receives the parameter object from the template context
+    this.paramName = token.args.trim()
+  }
+
+  async render(ctx: LiquidContext, emitter: any): Promise<void> {
+    const param = ctx.get([this.paramName]) as Parameter
+    const context = ctx.get(['context']) as Context
+
+    if (!param) {
+      emitter.write('')
+      return
+    }
+
+    const lines: string[] = []
+    const required = param.required ? ' (required)' : ''
+    const type = param.schema?.type || 'string'
+
+    lines.push(`- **\`${param.name}\`** (${type})${required}`)
+
+    if (param.description) {
+      const description = await htmlToMarkdown(param.description, context)
+      lines.push(`  ${description}`)
+    }
+
+    if (param.schema?.default !== undefined) {
+      lines.push(`  Default: \`${param.schema.default}\``)
+    }
+
+    if (param.schema?.enum && param.schema.enum.length > 0) {
+      lines.push(`  Can be one of: ${param.schema.enum.map((v) => `\`${v}\``).join(', ')}`)
+    }
+
+    emitter.write(lines.join('\n'))
+  }
+}
+
+/**
+ * Custom Liquid tag for rendering REST API body parameters
+ * Usage: {% rest_body_parameter param indent %}
+ */
+export class RestBodyParameter {
+  constructor(
+    token: TagToken,
+    remainTokens: TagToken[],
+    liquid: { options: any; parser: any },
+    private liquidContext?: LiquidContext,
+  ) {
+    // Parse arguments - param name and optional indent level
+    const args = token.args.trim().split(/\s+/)
+    this.param = args[0]
+    this.indent = args[1] ? parseInt(args[1]) : 0
+  }
+
+  private param: string
+  private indent: number
+
+  async render(ctx: LiquidContext, emitter: any): Promise<void> {
+    const param = ctx.get([this.param]) as BodyParameter
+    const context = ctx.get(['context']) as Context
+    const indent = this.indent
+
+    if (!param) {
+      emitter.write('')
+      return
+    }
+
+    const lines: string[] = []
+    const prefix = '  '.repeat(indent)
+    const required = param.isRequired ? ' (required)' : ''
+    const type = param.type || 'string'
+
+    lines.push(`${prefix}- **\`${param.name}\`** (${type})${required}`)
+
+    if (param.description) {
+      const description = await htmlToMarkdown(param.description, context)
+      lines.push(`${prefix}  ${description}`)
+    }
+
+    if (param.default !== undefined) {
+      lines.push(`${prefix}  Default: \`${param.default}\``)
+    }
+
+    if (param.enum && param.enum.length > 0) {
+      lines.push(`${prefix}  Can be one of: ${param.enum.map((v) => `\`${v}\``).join(', ')}`)
+    }
+
+    // Handle nested parameters
+    if (param.childParamsGroups && param.childParamsGroups.length > 0) {
+      for (const childGroup of param.childParamsGroups) {
+        lines.push(await renderChildParameter(childGroup, context, indent + 1))
+      }
+    }
+
+    emitter.write(lines.join('\n'))
+  }
+}
+
+/**
+ * Custom Liquid tag for rendering REST API status codes
+ * Usage: {% rest_status_code statusCode %}
+ */
+export class RestStatusCode {
+  private statusCodeName: string
+
+  constructor(
+    token: TagToken,
+    remainTokens: TagToken[],
+    liquid: { options: any; parser: any },
+    private liquidContext?: LiquidContext,
+  ) {
+    this.statusCodeName = token.args.trim()
+  }
+
+  async render(ctx: LiquidContext, emitter: any): Promise<void> {
+    const statusCode = ctx.get([this.statusCodeName]) as StatusCode
+    const context = ctx.get(['context']) as Context
+
+    if (!statusCode) {
+      emitter.write('')
+      return
+    }
+
+    const lines: string[] = []
+
+    if (statusCode.description) {
+      const description = await htmlToMarkdown(statusCode.description, context)
+      lines.push(`- **${statusCode.httpStatusCode}**`)
+      if (description.trim()) {
+        lines.push(`  ${description.trim()}`)
+      }
+    } else if (statusCode.httpStatusMessage) {
+      lines.push(`- **${statusCode.httpStatusCode}** - ${statusCode.httpStatusMessage}`)
+    } else {
+      lines.push(`- **${statusCode.httpStatusCode}**`)
+    }
+
+    emitter.write(lines.join('\n'))
+  }
+}
+
+/**
+ * Helper function to render child parameters recursively
+ */
+async function renderChildParameter(
+  param: ChildParameter,
+  context: Context,
+  indent: number,
+): Promise<string> {
+  const lines: string[] = []
+  const prefix = '  '.repeat(indent)
+  const required = param.isRequired ? ' (required)' : ''
+  const type = param.type || 'string'
+
+  lines.push(`${prefix}- **\`${param.name}\`** (${type})${required}`)
+
+  if (param.description) {
+    const description = await htmlToMarkdown(param.description, context)
+    lines.push(`${prefix}  ${description}`)
+  }
+
+  if (param.default !== undefined) {
+    lines.push(`${prefix}  Default: \`${param.default}\``)
+  }
+
+  if (param.enum && param.enum.length > 0) {
+    lines.push(`${prefix}  Can be one of: ${param.enum.map((v: string) => `\`${v}\``).join(', ')}`)
+  }
+
+  // Recursively handle nested parameters
+  if (param.childParamsGroups && param.childParamsGroups.length > 0) {
+    for (const child of param.childParamsGroups) {
+      lines.push(await renderChildParameter(child, context, indent + 1))
+    }
+  }
+
+  return lines.join('\n')
+}
+
+/**
+ * Helper function to convert HTML to markdown
+ */
+async function htmlToMarkdown(html: string, context: Context): Promise<string> {
+  if (!html) return ''
+
+  try {
+    const rendered = await renderContent(html, context, { textOnly: false })
+    return fastTextOnly(rendered)
+  } catch (error) {
+    logger.error('Failed to render HTML content to markdown in REST tag', {
+      error,
+      html: html.substring(0, 100), // First 100 chars for context
+      contextInfo: context && context.page ? { page: context.page.relativePath } : undefined,
+    })
+    // In non-production, re-throw to aid debugging
+    if (process.env.NODE_ENV !== 'production') {
+      throw error
+    }
+    // Fallback to simple text extraction
+    return fastTextOnly(html)
+  }
+}
+
+// Export tag names for registration
+export const restTags = {
+  rest_parameter: RestParameter,
+  rest_body_parameter: RestBodyParameter,
+  rest_status_code: RestStatusCode,
+}

src/article-api/middleware/article-body.ts

@@ -3,20 +3,15 @@ import type { Response } from 'express'
 import { Context } from '@/types'
 import { ExtendedRequestWithPageInfo } from '@/article-api/types'
 import contextualize from '@/frame/middleware/context/context'
+import { transformerRegistry } from '@/article-api/transformers'
+import { allVersions } from '@/versions/lib/all-versions'
+import type { Page } from '@/types'
 
-export async function getArticleBody(req: ExtendedRequestWithPageInfo) {
-  // req.pageinfo is set from pageValidationMiddleware and pathValidationMiddleware
-  // and is in the ExtendedRequestWithPageInfo
-  const { page, pathname, archived } = req.pageinfo
-
-  if (archived?.isArchived)
-    throw new Error(`Page ${pathname} is archived and can't be rendered in markdown.`)
-  // for anything that's not an article (like index pages), don't try to render and
-  // tell the user what's going on
-  if (page.documentType !== 'article') {
-    throw new Error(`Page ${pathname} isn't yet available in markdown.`)
-  }
-  // these parts allow us to render the page
+/**
+ * Creates a mocked rendering request and contextualizes it.
+ * This is used to prepare a request for rendering pages in markdown format.
+ */
+async function createContextualizedRenderingRequest(pathname: string, page: Page) {
   const mockedContext: Context = {}
   const renderingReq = {
     path: pathname,
@@ -29,9 +24,51 @@ export async function getArticleBody(req: ExtendedRequestWithPageInfo) {
     },
   }
 
-  // contextualize and render the page
+  // contextualize the request to get proper version info
   await contextualize(renderingReq as ExtendedRequestWithPageInfo, {} as Response, () => {})
   renderingReq.context.page = page
+
+  return renderingReq
+}
+
+export async function getArticleBody(req: ExtendedRequestWithPageInfo) {
+  // req.pageinfo is set from pageValidationMiddleware and pathValidationMiddleware
+  // and is in the ExtendedRequestWithPageInfo
+  const { page, pathname, archived } = req.pageinfo
+
+  if (archived?.isArchived)
+    throw new Error(`Page ${pathname} is archived and can't be rendered in markdown.`)
+
+  // Extract apiVersion from query params if provided
+  const apiVersion = req.query.apiVersion as string | undefined
+
+  // Check if there's a transformer for this page type (e.g., REST, webhooks, etc.)
+  const transformer = transformerRegistry.findTransformer(page)
+
+  if (transformer) {
+    // Use the transformer for autogenerated pages
+    const renderingReq = await createContextualizedRenderingRequest(pathname, page)
+
+    // Determine the API version to use (provided or latest)
+    // Validation is handled by apiVersionValidationMiddleware
+    const currentVersion = renderingReq.context.currentVersion
+    let effectiveApiVersion = apiVersion
+
+    // Use latest version if not provided
+    if (!effectiveApiVersion && currentVersion && allVersions[currentVersion]) {
+      effectiveApiVersion = allVersions[currentVersion].latestApiVersion || undefined
+    }
+
+    return await transformer.transform(page, pathname, renderingReq.context, effectiveApiVersion)
+  }
+
+  // For regular articles (non-autogenerated)
+  if (page.documentType !== 'article') {
+    throw new Error(`Page ${pathname} isn't yet available in markdown.`)
+  }
+
+  // these parts allow us to render the page
+  const renderingReq = await createContextualizedRenderingRequest(pathname, page)
   renderingReq.context.markdownRequested = true
   return await page.render(renderingReq.context)
 }