diff --git a/Package.swift b/Package.swift index f5162a432b..86249f0bf4 100644 --- a/Package.swift +++ b/Package.swift @@ -44,6 +44,7 @@ let package = Package( name: "SwiftDocC", dependencies: [ .target(name: "DocCCommon"), + .target(name: "DocCHTML"), .product(name: "Markdown", package: "swift-markdown"), .product(name: "SymbolKit", package: "swift-docc-symbolkit"), .product(name: "CLMDB", package: "swift-lmdb"), diff --git a/Sources/SwiftDocC/CMakeLists.txt b/Sources/SwiftDocC/CMakeLists.txt index c5e6ef0e26..ce74f882ea 100644 --- a/Sources/SwiftDocC/CMakeLists.txt +++ b/Sources/SwiftDocC/CMakeLists.txt @@ -175,6 +175,8 @@ add_library(SwiftDocC Model/Rendering/Diffing/Differences.swift Model/Rendering/Diffing/RenderNode+Diffable.swift Model/Rendering/DocumentationContentRenderer.swift + Model/Rendering/HTML/HTMLContentConsumer.swift + Model/Rendering/HTML/HTMLRenderer.swift Model/Rendering/LinkTitleResolver.swift "Model/Rendering/Navigation Tree/RenderHierarchy.swift" "Model/Rendering/Navigation Tree/RenderHierarchyChapter.swift" @@ -465,6 +467,8 @@ add_library(SwiftDocC Utility/Version.swift) target_link_libraries(SwiftDocC PRIVATE DocCCommon) +target_link_libraries(SwiftDocC PRIVATE + DocCHTML) target_link_libraries(SwiftDocC PUBLIC SwiftMarkdown::Markdown DocC::SymbolKit diff --git a/Sources/SwiftDocC/Infrastructure/ConvertActionConverter.swift b/Sources/SwiftDocC/Infrastructure/ConvertActionConverter.swift index c18d64004b..d82fcce906 100644 --- a/Sources/SwiftDocC/Infrastructure/ConvertActionConverter.swift +++ b/Sources/SwiftDocC/Infrastructure/ConvertActionConverter.swift @@ -26,6 +26,7 @@ package enum ConvertActionConverter { /// - Parameters: /// - context: The context that the bundle is a part of. /// - outputConsumer: The consumer that the conversion passes outputs of the conversion to. + /// - htmlContentConsumer: The consumer for HTML content that the conversion produces, or `nil` if the conversion shouldn't produce any HTML content. /// - sourceRepository: The source repository where the documentation's sources are hosted. /// - emitDigest: Whether the conversion should pass additional metadata output––such as linkable entities information, indexing information, or asset references by asset type––to the consumer. /// - documentationCoverageOptions: The level of experimental documentation coverage information that the conversion should pass to the consumer. @@ -33,6 +34,7 @@ package enum ConvertActionConverter { package static func convert( context: DocumentationContext, outputConsumer: some ConvertOutputConsumer & ExternalNodeConsumer, + htmlContentConsumer: (any HTMLContentConsumer)?, sourceRepository: SourceRepository?, emitDigest: Bool, documentationCoverageOptions: DocumentationCoverageOptions @@ -103,7 +105,7 @@ package enum ConvertActionConverter { let renderSignpostHandle = signposter.beginInterval("Render", id: signposter.makeSignpostID(), "Render \(context.knownPages.count) pages") - var conversionProblems: [Problem] = context.knownPages.concurrentPerform { identifier, results in + var conversionProblems: [Problem] = context.knownPages.concurrentPerform { [htmlContentConsumer] identifier, results in // If cancelled skip all concurrent conversion work in this block. guard !Task.isCancelled else { return } @@ -111,7 +113,19 @@ package enum ConvertActionConverter { autoreleasepool { do { let entity = try context.entity(with: identifier) - + + if let htmlContentConsumer { + var renderer = HTMLRenderer(reference: identifier, context: context, goal: .conciseness) + + if let symbol = entity.semantic as? Symbol { + let renderedPageInfo = renderer.renderSymbol(symbol) + try htmlContentConsumer.consume(pageInfo: renderedPageInfo, forPage: identifier) + } else if let article = entity.semantic as? Article { + let renderedPageInfo = renderer.renderArticle(article) + try htmlContentConsumer.consume(pageInfo: renderedPageInfo, forPage: identifier) + } + } + guard let renderNode = converter.renderNode(for: entity) else { // No render node was produced for this entity, so just skip it. return @@ -247,3 +261,16 @@ package enum ConvertActionConverter { return conversionProblems } } + +private extension HTMLContentConsumer { + func consume(pageInfo: HTMLRenderer.RenderedPageInfo, forPage reference: ResolvedTopicReference) throws { + try consume( + mainContent: pageInfo.content, + metadata: ( + title: pageInfo.metadata.title, + description: pageInfo.metadata.plainDescription + ), + forPage: reference + ) + } +} diff --git a/Sources/SwiftDocC/Infrastructure/ConvertOutputConsumer.swift b/Sources/SwiftDocC/Infrastructure/ConvertOutputConsumer.swift index f5e1ebd432..288e84334e 100644 --- a/Sources/SwiftDocC/Infrastructure/ConvertOutputConsumer.swift +++ b/Sources/SwiftDocC/Infrastructure/ConvertOutputConsumer.swift @@ -8,8 +8,6 @@ See https://swift.org/CONTRIBUTORS.txt for Swift project authors */ -import Foundation - /// A consumer for output produced by a documentation conversion. /// /// Types that conform to this protocol manage what to do with documentation conversion products, for example persist them to disk diff --git a/Sources/SwiftDocC/Model/Rendering/HTML/HTMLContentConsumer.swift b/Sources/SwiftDocC/Model/Rendering/HTML/HTMLContentConsumer.swift new file mode 100644 index 0000000000..877e6932dd --- /dev/null +++ b/Sources/SwiftDocC/Model/Rendering/HTML/HTMLContentConsumer.swift @@ -0,0 +1,40 @@ +/* + This source file is part of the Swift.org open source project + + Copyright (c) 2025 Apple Inc. and the Swift project authors + Licensed under Apache License v2.0 with Runtime Library Exception + + See https://swift.org/LICENSE.txt for license information + See https://swift.org/CONTRIBUTORS.txt for Swift project authors +*/ + +#if canImport(FoundationXML) +// TODO: Consider other HTML rendering options as a future improvement (rdar://165755530) +package import FoundationXML +#else +package import Foundation +#endif + +/// A consumer for HTML content produced during documentation conversion. +package protocol HTMLContentConsumer { + // One reason that this is its own protocol, rather than an extension of ConvertOutputConsumer, is so that we can avoid exposing `XMLNode` in any public API. + // That way, we are completely free to replace the entire internal HTML rendering implementation with something else in the future, without breaking API. + + /// Consumes the HTML content and metadata for a given page. + /// + /// The content and metadata doesn't make up a full valid HTML page. + /// It's the consumers responsibility to insert the information into a template or skeletal structure to produce a valid HTML file for each page. + /// + /// - Parameters: + /// - mainContent: The contents for this page as an XHTML node. + /// - metadata: Metadata information (title and description) about this page. + /// - reference: The resolved topic reference that identifies this page. + func consume( + mainContent: XMLNode, + metadata: ( + title: String, + description: String? + ), + forPage reference: ResolvedTopicReference + ) throws +} diff --git a/Sources/SwiftDocC/Model/Rendering/HTML/HTMLRenderer.swift b/Sources/SwiftDocC/Model/Rendering/HTML/HTMLRenderer.swift new file mode 100644 index 0000000000..ce5472b3ca --- /dev/null +++ b/Sources/SwiftDocC/Model/Rendering/HTML/HTMLRenderer.swift @@ -0,0 +1,291 @@ +/* + This source file is part of the Swift.org open source project + + Copyright (c) 2025 Apple Inc. and the Swift project authors + Licensed under Apache License v2.0 with Runtime Library Exception + + See https://swift.org/LICENSE.txt for license information + See https://swift.org/CONTRIBUTORS.txt for Swift project authors +*/ + +#if canImport(FoundationXML) +// TODO: Consider other HTML rendering options as a future improvement (rdar://165755530) +import FoundationXML +import FoundationEssentials +#else +import Foundation +#endif +import DocCHTML +import Markdown +import SymbolKit + +/// A link provider that provider structured information about resolved links and assets to the underlying HTML renderer. +private struct ContextLinkProvider: LinkProvider { + let reference: ResolvedTopicReference + let context: DocumentationContext + + func element(for url: URL) -> LinkedElement? { + guard url.scheme == "doc", + let rawBundleID = url.host, + // TODO: Support returning information about external pages (rdar://165912415) + let node = context.documentationCache[ResolvedTopicReference(bundleID: .init(rawValue: rawBundleID), path: url.path, fragment: url.fragment, sourceLanguage: .swift /* The reference's language doesn't matter */)] + else { + return nil + } + + let names: LinkedElement.Names + if let symbol = node.semantic as? Symbol, + case .symbol(let primaryTitle) = node.name + { + // Check if this symbol has any language-specific titles + let titles = symbol.titleVariants.allValues + if titles.contains(where: { _, title in title != primaryTitle }) { + // This symbol has multiple unique names + let titles = [SourceLanguage: String]( + titles.map { trait, title in + (trait.sourceLanguage ?? .swift, title) + }, + uniquingKeysWith: { _, new in new } + ) + + names = .languageSpecificSymbol(titles) + } else { + // There are multiple names, but the're all the same + names = .single(.symbol(primaryTitle)) + } + } else { + let name: LinkedElement.Name = switch node.name { + case .conceptual(let title): .conceptual(title) + case .symbol(name: let title): .symbol(title) + } + names = .single(name) + } + + // A helper function that transforms SymbolKit fragments into renderable identifier/decorator fragments + func convert(_ fragments: [SymbolGraph.Symbol.DeclarationFragments.Fragment]) -> [LinkedElement.SymbolNameFragment] { + func convert(kind: SymbolGraph.Symbol.DeclarationFragments.Fragment.Kind) -> LinkedElement.SymbolNameFragment.Kind { + switch kind { + case .identifier, .externalParameter: .identifier + default: .decorator + } + } + guard var current = fragments.first.map({ LinkedElement.SymbolNameFragment(text: $0.spelling, kind: convert(kind: $0.kind)) }) else { + return [] + } + + // Join together multiple fragments of the same identifier/decorator kind to produce a smaller output. + var result: [LinkedElement.SymbolNameFragment] = [] + for fragment in fragments.dropFirst() { + let kind = convert(kind: fragment.kind) + if kind == current.kind { + current.text += fragment.spelling + } else { + result.append(current) + current = .init(text: fragment.spelling, kind: kind) + } + } + result.append(current) + return result + } + + let subheadings: DocCHTML.LinkedElement.Subheadings + if let symbol = node.semantic as? Symbol { + // Check if this symbol has any language-specific _sub headings_ + let primarySubheading = symbol.subHeading + let allSubheadings = symbol.subHeadingVariants.allValues + + if allSubheadings.contains(where: { _, title in title != primarySubheading }) { + // This symbol has multiple unique subheadings + subheadings = .languageSpecificSymbol(.init( + allSubheadings.map { trait, subheading in ( + key: trait.sourceLanguage ?? .swift, + value: convert(subheading) + )}, + uniquingKeysWith: { _, new in new } + )) + } else { + // There are multiple subheadings, but the're all the same + subheadings = .single(.symbol(convert(primarySubheading ?? []))) + } + } else { + subheadings = .single(.conceptual(node.name.plainText)) + } + + return .init( + path: Self.filePath(for: node.reference), + names: names, + subheadings: subheadings, + abstract: (node.semantic as? (any Abstracted))?.abstract + ) + } + + func pathForSymbolID(_ usr: String) -> URL? { + context.localOrExternalReference(symbolID: usr).map { + Self.filePath(for: $0) + } + } + + func assetNamed(_ assetName: String) -> LinkedAsset? { + guard let asset = context.resolveAsset(named: assetName, in: reference) else { + // The context + return nil + } + + var files = [LinkedAsset.ColorStyle: [Int: URL]]() + for (traits, url) in asset.variants { + let scale = (traits.displayScale ?? .standard).scaleFactor + + files[traits.userInterfaceStyle == .dark ? .dark : .light, default: [:]][scale] = url + } + + return .init(files: files) + } + + func fallbackLinkText(linkString: String) -> String { + // For unresolved links, especially to symbols, prefer to display only the the last link component without its disambiguation + PathHierarchy.PathParser.parse(path: linkString).components.last.map { String($0.name) } ?? linkString + } + + static func filePath(for reference: ResolvedTopicReference) -> URL { + reference.url.withoutHostAndPortAndScheme().appendingPathComponent("index.html") + } +} + +/// A type that renders documentation pages into semantic HTML elements. +struct HTMLRenderer { + let reference: ResolvedTopicReference + let context: DocumentationContext + let goal: RenderGoal + + private let renderer: MarkdownRenderer + + init(reference: ResolvedTopicReference, context: DocumentationContext, goal: RenderGoal) { + self.reference = reference + self.context = context + self.goal = goal + self.renderer = MarkdownRenderer( + path: ContextLinkProvider.filePath(for: reference), + goal: goal, + linkProvider: ContextLinkProvider(reference: reference, context: context) + ) + } + + /// Information about a rendered page + struct RenderedPageInfo { + /// The HTML content of the page as an XMLNode hierarchy. + /// + /// The string representation of those node hierarchy is intended to be inserted _somewhere_ inside the `` HTML element. + /// It _doesn't_ include a page header, footer, navigator, etc. and may be an insufficient representation of the "entire" page + var content: XMLNode + /// The title and description/abstract of the page. + var metadata: Metadata + /// Meta information about the page that belongs in the HTML `` element. + struct Metadata { + /// The plain text title of this page, suitable as content for the HTML `` element. + var title: String + /// The plain text description/abstract of this page, suitable a data for a `<meta>` element for sharing purposes. + var plainDescription: String? + } + } + + mutating func renderArticle(_ article: Article) -> RenderedPageInfo { + let node = context.documentationCache[reference]! + + let main = XMLElement(name: "main") + let articleElement = XMLElement(name: "article") + main.addChild(articleElement) + + let hero = XMLElement(name: "section") + articleElement.addChild(hero) + + // Title + hero.addChild( + .element(named: "h1", children: [.text(node.name.plainText)]) + ) + + // Abstract + if let abstract = article.abstract { + let paragraph = renderer.visit(abstract) as! XMLElement + if goal == .richness { + paragraph.addAttribute(XMLNode.attribute(withName: "id", stringValue: "abstract") as! XMLNode) + } + hero.addChild(paragraph) + } + + return RenderedPageInfo( + content: goal == .richness ? main : articleElement, + metadata: .init( + title: article.title?.plainText ?? node.name.plainText, + plainDescription: article.abstract?.plainText + ) + ) + } + + mutating func renderSymbol(_ symbol: Symbol) -> RenderedPageInfo { + let node = context.documentationCache[reference]! + + let main = XMLElement(name: "main") + let articleElement = XMLElement(name: "article") + main.addChild(articleElement) + + let hero = XMLElement(name: "section") + articleElement.addChild(hero) + + // Title + let titleVariants = symbol.titleVariants.allValues.sorted(by: { $0.trait < $1.trait }) + for (trait, languageSpecificTitle) in titleVariants { + guard let language = trait.sourceLanguage else { continue } + + let attributes: [String: String]? + if goal == .richness, titleVariants.count < 1 { + attributes = ["class": "\(language.id)-only"] + } else { + attributes = nil + } + + hero.addChild( + .element(named: "h1", children: renderer.wordBreak(symbolName: languageSpecificTitle), attributes: attributes) + ) + } + + // Abstract + if let abstract = symbol.abstract { + let paragraph = renderer.visit(abstract) as! XMLElement + if goal == .richness { + paragraph.addAttribute(XMLNode.attribute(withName: "id", stringValue: "abstract") as! XMLNode) + } + hero.addChild(paragraph) + } + + return RenderedPageInfo( + content: goal == .richness ? main : articleElement, + metadata: .init( + title: symbol.title, + plainDescription: symbol.abstract?.plainText + ) + ) + } + + // TODO: As a future enhancement, add another layer on top of this that creates complete HTML pages (both `<head>` and `<body>`) (rdar://165912669) +} + +// Note; this isn't a Comparable conformance so that it can remain private to this file. +private extension DocumentationDataVariantsTrait { + static func < (lhs: DocumentationDataVariantsTrait, rhs: DocumentationDataVariantsTrait) -> Bool { + if let lhs = lhs.sourceLanguage { + if let rhs = rhs.sourceLanguage { + return lhs < rhs + } + return true // nil is after anything + } + return false // nil is after anything + } +} + +private extension XMLElement { + func addChildren(_ nodes: [XMLNode]) { + for node in nodes { + addChild(node) + } + } +} diff --git a/Sources/SwiftDocCTestUtilities/SymbolGraphCreation.swift b/Sources/SwiftDocCTestUtilities/SymbolGraphCreation.swift index 6a15e4596b..8bd48ead94 100644 --- a/Sources/SwiftDocCTestUtilities/SymbolGraphCreation.swift +++ b/Sources/SwiftDocCTestUtilities/SymbolGraphCreation.swift @@ -90,6 +90,7 @@ extension XCTestCase { location: (position: SymbolGraph.LineList.SourceRange.Position, url: URL)? = (defaultSymbolPosition, defaultSymbolURL), signature: SymbolGraph.Symbol.FunctionSignature? = nil, availability: [SymbolGraph.Symbol.Availability.AvailabilityItem]? = nil, + declaration: [SymbolGraph.Symbol.DeclarationFragments.Fragment]? = nil, otherMixins: [any Mixin] = [] ) -> SymbolGraph.Symbol { precondition(!pathComponents.isEmpty, "Need at least one path component to name the symbol") @@ -104,10 +105,24 @@ extension XCTestCase { if let availability { mixins.append(SymbolGraph.Symbol.Availability(availability: availability)) } + if let declaration { + mixins.append(SymbolGraph.Symbol.DeclarationFragments(declarationFragments: declaration)) + } + + let names = if let declaration { + SymbolGraph.Symbol.Names( + title: pathComponents.last!, // Verified above to exist + navigator: declaration, + subHeading: declaration, + prose: nil + ) + } else { + makeSymbolNames(name: pathComponents.last!) // Verified above to exist + } return SymbolGraph.Symbol( identifier: SymbolGraph.Symbol.Identifier(precise: id, interfaceLanguage: language.id), - names: makeSymbolNames(name: pathComponents.last!), + names: names, pathComponents: pathComponents, docComment: docComment.map { makeLineList( diff --git a/Sources/SwiftDocCUtilities/Action/Actions/Convert/ConvertAction.swift b/Sources/SwiftDocCUtilities/Action/Actions/Convert/ConvertAction.swift index b6c98ecedd..62d2baf744 100644 --- a/Sources/SwiftDocCUtilities/Action/Actions/Convert/ConvertAction.swift +++ b/Sources/SwiftDocCUtilities/Action/Actions/Convert/ConvertAction.swift @@ -320,6 +320,7 @@ public struct ConvertAction: AsyncAction { try ConvertActionConverter.convert( context: context, outputConsumer: outputConsumer, + htmlContentConsumer: nil, sourceRepository: sourceRepository, emitDigest: emitDigest, documentationCoverageOptions: documentationCoverageOptions diff --git a/Sources/SwiftDocCUtilities/Action/Actions/Convert/FileWritingHTMLContentConsumer.swift b/Sources/SwiftDocCUtilities/Action/Actions/Convert/FileWritingHTMLContentConsumer.swift new file mode 100644 index 0000000000..2fcaf2eae7 --- /dev/null +++ b/Sources/SwiftDocCUtilities/Action/Actions/Convert/FileWritingHTMLContentConsumer.swift @@ -0,0 +1,119 @@ +/* + This source file is part of the Swift.org open source project + + Copyright (c) 2025 Apple Inc. and the Swift project authors + Licensed under Apache License v2.0 with Runtime Library Exception + + See https://swift.org/LICENSE.txt for license information + See https://swift.org/CONTRIBUTORS.txt for Swift project authors +*/ + +#if canImport(FoundationXML) +// TODO: Consider other HTML rendering options as a future improvement (rdar://165755530) +import FoundationXML +import FoundationEssentials +#else +import Foundation +#endif + +import SwiftDocC +import DocCHTML + +struct FileWritingHTMLContentConsumer: HTMLContentConsumer { + var targetFolder: URL + var fileManager: any FileManagerProtocol + var prettyPrintOutput: Bool + + private struct HTMLTemplate { + var original: String + var contentReplacementRange: Range<String.Index> + var titleReplacementRange: Range<String.Index> + var descriptionReplacementRange: Range<String.Index> + + init(data: Data) throws { + let content = String(decoding: data, as: UTF8.self) + + // ???: Should we parse the content with XMLParser instead? If so, what do we do if it's not valid XHTML? + let noScriptStart = content.utf8.firstRange(of: "<noscript>".utf8)!.upperBound + let noScriptEnd = content.utf8.firstRange(of: "</noscript>".utf8)!.lowerBound + + let titleStart = content.utf8.firstRange(of: "<title>".utf8)!.upperBound + let titleEnd = content.utf8.firstRange(of: "".utf8)!.lowerBound + + let beforeHeadEnd = content.utf8.firstRange(of: "".utf8)!.lowerBound + + original = content + // TODO: If the template doesn't already contain a