Merge pull request #27 from JuliaComputing/sp/lowfi

feat: add lowfi mode for flexsearch

Author: pfitzseb

assets/default/flexsearch_integration.js

@@ -8,16 +8,9 @@
             index: [
                 {
                     field: 'content',
-                    tokenize: 'forward',
-                    minlength: 3,
-                    resolution: 9
                 }
             ]
         },
-        encoder: 'simple',
-        fastupdate: false,
-        optimize: true,
-        context: true,
     });
 
     let importDone = null

flexsearch/gensearch-lowfi.js

@@ -0,0 +1,42 @@
+const { Document } = require('flexsearch')
+const fs = require('fs')
+const process = require('process')
+const path = require('path')
+
+var flexsearchIdx = new Document({
+    document: {
+        id: 'id',
+        store: ['title', 'pagetitle', 'ref'],
+        index: [
+            {
+                field: 'content',
+                tokenize: 'forward',
+                minlength: 3,
+                resolution: 5
+            }
+        ]
+    },
+    encoder: 'advanced',
+    fastupdate: false,
+    optimize: true,
+    context: false,
+});
+
+const idx = require(process.cwd() + '/index.json')
+
+idx.forEach(doc => {
+    flexsearchIdx.add(doc)
+})
+
+fs.mkdirSync(path.join(process.cwd(), 'search-data'))
+flexsearchIdx.export((key, data) => {
+    if (data) {
+        const p = path.join(process.cwd(), 'search-data', key + '.json')
+        try {
+            fs.writeFileSync(p, data)
+            console.log('  ' + key)
+        } catch (err) {
+            console.error(err)
+        }
+    }
+})

flexsearch/gensearch.js

@@ -12,14 +12,14 @@ var flexsearchIdx = new Document({
                 field: 'content',
                 tokenize: 'forward',
                 minlength: 3,
-                resolution: 9
+                resolution: 5
             }
         ]
     },
-    encoder: 'simple',
+    encoder: 'advanced',
     fastupdate: false,
     optimize: true,
-    context: true,
+    context: false,
 });
 
 const idx = require(process.cwd() + '/index.json')

src/MultiDocumenter.jl

@@ -4,16 +4,18 @@ import Gumbo, AbstractTrees
 using HypertextLiteral
 
 """
-    SearchConfig(index_versions = ["stable"], engine = MultiDocumenter.FlexSearch)
+    SearchConfig(index_versions = ["stable"], engine = MultiDocumenter.FlexSearch, lowfi = false)
 
 `index_versions` is a vector of relative paths used for generating the search index. Only
 the first matching path is considered.
 `engine` may be `MultiDocumenter.FlexSearch`, `MultiDocumenter.Stork`, or a module that conforms
 to the expected API (which is currently undocumented).
+`lowfi = true` will try to minimize search index size. Only relevant for flexsearch.
 """
 Base.@kwdef mutable struct SearchConfig
     index_versions = ["stable", "dev"]
     engine = FlexSearch
+    lowfi = false
 end
 
 struct MultiDocRef
@@ -280,7 +282,7 @@ function inject_styles_and_global_navigation(
     end
     pushfirst!(custom_stylesheets, joinpath("assets", "default", "multidoc.css"))
 
-    for (root, _, files) in walkdir(dir)
+    @sync for (root, _, files) in walkdir(dir)
         for file in files
             path = joinpath(root, file)
             if file == "documenter.js"
@@ -294,60 +296,61 @@ function inject_styles_and_global_navigation(
 
             islink(path) && continue
             isfile(path) || continue
-
-            stylesheets = make_global_stylesheet(custom_stylesheets, relpath(dir, root))
-            scripts = make_global_scripts(custom_scripts, relpath(dir, root))
-
-
             page = read(path, String)
             if startswith(
                 page,
                 "<!--This file is automatically generated by Documenter.jl-->",
             )
                 continue
             end
-            doc = Gumbo.parsehtml(page)
-            injected = 0
-
-            for el in AbstractTrees.PreOrderDFS(doc.root)
-                injected >= 2 && break
 
-                if el isa Gumbo.HTMLElement
-                    if Gumbo.tag(el) == :head
-                        for stylesheet in stylesheets
-                            stylesheet.parent = el
-                            push!(el.children, stylesheet)
+            Threads.@spawn begin
+                stylesheets = make_global_stylesheet(custom_stylesheets, relpath(dir, root))
+                scripts = make_global_scripts(custom_scripts, relpath(dir, root))
+
+                doc = Gumbo.parsehtml(page)
+                injected = 0
+
+                for el in AbstractTrees.PreOrderDFS(doc.root)
+                    injected >= 2 && break
+
+                    if el isa Gumbo.HTMLElement
+                        if Gumbo.tag(el) == :head
+                            for stylesheet in stylesheets
+                                stylesheet.parent = el
+                                push!(el.children, stylesheet)
+                            end
+                            for script in scripts
+                                script.parent = el
+                                pushfirst!(el.children, script)
+                            end
+                            injected += 1
+                        elseif Gumbo.tag(el) == :body && !isempty(el.children)
+                            documenter_div = first(el.children)
+                            if documenter_div isa Gumbo.HTMLElement &&
+                            Gumbo.getattr(documenter_div, "id", "") == "documenter"
+                                @debug "Could not detect Documenter page layout in $path. This may be due to an old version of Documenter."
+                            end
+                            # inject global navigation as first element in body
+
+                            global_nav = make_global_nav(
+                                dir,
+                                docs,
+                                root,
+                                brand_image,
+                                search_engine,
+                                prettyurls,
+                            )
+                            global_nav.parent = el
+                            pushfirst!(el.children, global_nav)
+                            injected += 1
                         end
-                        for script in scripts
-                            script.parent = el
-                            pushfirst!(el.children, script)
-                        end
-                        injected += 1
-                    elseif Gumbo.tag(el) == :body && !isempty(el.children)
-                        documenter_div = first(el.children)
-                        if documenter_div isa Gumbo.HTMLElement &&
-                           Gumbo.getattr(documenter_div, "id", "") == "documenter"
-                            @debug "Could not detect Documenter page layout in $path. This may be due to an old version of Documenter."
-                        end
-                        # inject global navigation as first element in body
-
-                        global_nav = make_global_nav(
-                            dir,
-                            docs,
-                            root,
-                            brand_image,
-                            search_engine,
-                            prettyurls,
-                        )
-                        global_nav.parent = el
-                        pushfirst!(el.children, global_nav)
-                        injected += 1
                     end
                 end
-            end
 
-            open(path, "w") do io
-                print(io, doc)
+                open(path, "w") do io
+                    print(io, doc)
+                end
             end
         end
     end

src/search/flexsearch.jl

@@ -143,10 +143,11 @@ function build_search_index(root, docs, config)
     ID[] = 0
     idx = generate_index(root, docs, config)
     to_json_index(idx, joinpath(root, "index.json"))
-    println("Writing flexsearch index:")
+    file = config.lowfi ? "gensearch-lowfi.js" : "gensearch.js"
+    println("Writing $(config.lowfi ? "lowfi" : "") flexsearch index:")
     cd(root) do
         run(
-            `$(NodeJS.nodejs_cmd()) $(joinpath(@__DIR__, "..", "..", "flexsearch", "gensearch.js"))`,
+            `$(NodeJS.nodejs_cmd()) $(joinpath(@__DIR__, "..", "..", "flexsearch", file))`,
         )
     end
     return nothing