Improve search relevance and ensure less janky 'as you type' results (#2304)

* Improve search relevance and ensure less janky 'as you type' results

  - Improve search relevance with exact title matching
   and prefix search capabilities
  - Add title prefix boosting for smoother
  autocomplete behavior:
    - Single character queries (e.g., "A") get high
  boost (100x) to show only titles starting with that
  letter
    - Two character queries get medium boost (4x)
    - Up to 10 characters uses a term query with no
  additional boost
    - This creates a less janky autocomplete
  experience where results don't shift dramatically as
   you type
  - Add bidirectional synonym expansion for query-time
   boosting
    - Ensures both .NET and dotnet score high when the
   page title is just .NET
    - Synonyms are expanded in both directions so
  searching either term finds the relevant page
  - Apply keyword normalizer to all keyword fields for
   consistent case-insensitive matching
  - Reorganize and expand config/search.yml with new
  synonyms (edr, eks, kpi, logsdb, rag, tsvb) and
  diminish terms (curator, hadoop, glossary)
  - Remove unused url_segment_count field from
  document mapping

  Test plan

  - Verify all search relevance tests pass with
  updated expected results
  - Test prefix searches with short queries (1-2
  characters) return sensible results with titles
  starting with those characters
  - Verify typing progressively longer queries
  produces stable results without excessive shifting
  - Verify synonym expansion works bidirectionally
  (searching "dotnet" finds ".NET" pages and vice
  versa)
  - Confirm exact title matches are properly boosted
  in search results

* Code QoL improvements

* fix test

Author: Mpdreamz

config/search.yml

@@ -1,40 +1,14 @@
-synonyms:
-  - [ ".net", "c#", "csharp", "dotnet", "net" ]
-  - [ "esql", "es|ql => esql" ]
-  - [ "data-stream", "data stream", "datastream => data-streams"]
-  - [ "data-streams", "data streams", "datastreams"]
-  - [ "motlp", "managed otlp" ]
-  - [ "s3", "aws s3", "amazon s3" ]
-  - [ "es", "elasticsearch" ]
-  - [ "elastic learned sparse encoder", "elser" ]
-  - [ "ccs", "cross cluster search", "cross-cluster search", "spaghetti" ]
-  - [ "cps", "cross project search", "cross-project search" ]
-  - [ "apm", "application performance monitoring" ]
-  - [ "ecctl", "elastic cloud control" ]
-  - [ "opentelemetry", "otel" ]
-  - [ "elastic distributions of opentelemetry", "edot" ]
-  - [ "eck", "elastic cloud on kubernetes" ]
-  - [ "ece", "elastic cloud enterprise" ]
-  - [ "elv2", "elastic license v2" ]
-  - [ "kql", "kibana query language" ]
-  - [ "ccr", "cross-cluster replication", "cross cluster replication" ]
-  - [ "esaas", "elastic stack as a service" ]
-  - [ "knn", "k-nearest neighbors" ]
-  - [ "ech", "elastic cloud hosted", "ess" ]
-  - [ "elasticsearch service", "elastic cloud" ]
-  - [ "aws", "amazon" ]
-  - [ "gcp", "google cloud platform" ]
-  - [ "ilm", "index lifecycle management" ]
-  - [ "javascript", "js", "node", "nodejs", "node.js" ]
-  - [ "edot", "elastic distribution of opentelemetry" ]
-  - [ "k8s", "kubernetes" ]
-  - [ "ecs", "elastic common schema" ]
-  - [ "machine-learning", "machine learning", "ml => machine learning" ]
-  - [ "eis", "elastic inference service" ]
-  - [ "traffic filter", "network security" ]
-  - [ "sso", "single sign-on" ]
-  - [ "querydsl", "query dsl", "query dsl"]
+# Dampen the impact of search results that contain these terms.
+# Use sparingly, in general our query relevance should be good enough.
+diminish_terms:
+  - plugin
+  - client
+  - integration
+  - curator
+  - hadoop
+  - glossary
 
+# Query rules, to promote certain results over others.
 # Use sparingly, in general our query relevance should be good enough.
 rules:
   # datastreams require this special handling because `datastream` is really unique vs the other spellings. 
@@ -65,8 +39,47 @@ rules:
       ids:
         - /docs/reference/logstash
 
-diminish_terms:
-  - plugin
-  - client
-  - integration
-  - glossary
+
+synonyms:
+  - [ ".net", "c#", "csharp", "dotnet", "net" ]
+  - [ "apm", "application performance monitoring" ]
+  - [ "aws", "amazon" ]
+  - [ "ccr", "cross-cluster replication", "cross cluster replication" ]
+  - [ "ccs", "cross cluster search", "cross-cluster search", "spaghetti" ]
+  - [ "cps", "cross project search", "cross-project search" ]
+  - [ "data-stream", "data stream", "datastream => data-streams"]
+  - [ "data-streams", "data streams", "datastreams"]
+  - [ "ecctl", "elastic cloud control" ]
+  - [ "ece", "elastic cloud enterprise" ]
+  - [ "ech", "elastic cloud hosted", "ess" ]
+  - [ "eck", "elastic cloud on kubernetes" ]
+  - [ "ecs", "elastic common schema" ]
+  - [ "edot", "elastic distribution of opentelemetry" ]
+  - [ "edr", "endpoint detection response" ]
+  - [ "eis", "elastic inference service" ]
+  - [ "eks", "elastic kubernetes service" ]
+  - [ "elastic distributions of opentelemetry", "edot" ]
+  - [ "elastic learned sparse encoder", "elser" ]
+  - [ "elasticsearch service", "elastic cloud" ]
+  - [ "elv2", "elastic license v2" ]
+  - [ "es", "elasticsearch" ]
+  - [ "esaas", "elastic stack as a service" ]
+  - [ "esql", "es|ql => esql" ]
+  - [ "gcp", "google cloud platform" ]
+  - [ "ilm", "index lifecycle management" ]
+  - [ "javascript", "js", "node", "nodejs", "node.js" ]
+  - [ "k8s", "kubernetes" ]
+  - [ "knn", "k-nearest neighbors" ]
+  - [ "kpi", "key performance indicator" ]
+  - [ "kql", "kibana query language" ]
+  - [ "logsdb", "logs datastream", "logs data stream" ]
+  - [ "machine-learning", "machine learning", "ml => machine learning" ]
+  - [ "motlp", "managed otlp" ]
+  - [ "opentelemetry", "otel" ]
+  - [ "querydsl", "query dsl", "query dsl"]
+  - [ "rag", "retrieval augmented generation" ]
+  - [ "s3", "aws s3", "amazon s3" ]
+  - [ "sso", "single sign-on" ]
+  - [ "traffic filter", "network security" ]
+  - [ "tsvb", "time series visual builder" ]
+

src/Elastic.ApiExplorer/Elasticsearch/OpenApiDocumentExporter.cs

@@ -165,6 +165,7 @@ private IEnumerable<DocumentationDocument> ConvertToDocuments(OpenApiDocument op
 					Type = "api",
 					Url = url,
 					Title = title,
+					SearchTitle = title,
 					Description = description,
 					Body = body,
 					StrippedBody = body,

src/Elastic.Documentation.Configuration/Search/SearchConfiguration.cs

@@ -3,12 +3,46 @@
 // See the LICENSE file in the project root for more information
 
 using System.Collections.Immutable;
+using System.Diagnostics.CodeAnalysis;
 
 namespace Elastic.Documentation.Configuration.Search;
 
 public record SearchConfiguration
 {
-	public required IReadOnlyCollection<string> Synonyms { get; init; }
+	private readonly IReadOnlyDictionary<string, string[]> _synonyms;
+
+	public required IReadOnlyDictionary<string, string[]> Synonyms
+	{
+		get => _synonyms;
+		[MemberNotNull(nameof(_synonyms))]
+		init
+		{
+			_synonyms = value;
+			SynonymBiDirectional = value
+				.Select(kv => kv.Value.Concat([kv.Key]).ToArray())
+				.SelectMany(a =>
+				{
+					var targets = new List<string[]>();
+					foreach (var s in a)
+					{
+						if (s.Contains(' ') || s.Contains("=>"))
+							continue;
+
+						List<string> newTarget = [s];
+						newTarget.AddRange(a.Except([s]));
+						targets.Add(newTarget.ToArray());
+					}
+
+					return targets;
+				})
+				.Where(a => a.Length > 1)
+				.DistinctBy(a => a[0])
+				.ToDictionary(a => a[0], a => a.Skip(1).ToArray(), StringComparer.OrdinalIgnoreCase);
+		}
+	}
+
+	public IReadOnlyDictionary<string, string[]> SynonymBiDirectional { get; private set; } = new Dictionary<string, string[]>();
+
 	public required IReadOnlyCollection<QueryRule> Rules { get; init; }
 	public required IReadOnlyCollection<string> DiminishTerms { get; init; }
 }
@@ -78,15 +112,18 @@ public static class SearchConfigurationExtensions
 	public static SearchConfiguration CreateSearchConfiguration(this ConfigurationFileProvider provider)
 	{
 		var searchFile = provider.SearchFile;
+		var synonyms = new Dictionary<string, string[]>();
 
 		if (!searchFile.Exists)
-			return new SearchConfiguration { Synonyms = [], Rules = [], DiminishTerms = [] };
+			return new SearchConfiguration { Synonyms = synonyms, Rules = [], DiminishTerms = [] };
 
 		var searchDto = ConfigurationFileProvider.Deserializer.Deserialize<SearchConfigDto>(searchFile.OpenText());
-		var flattenedSynonyms = searchDto.Synonyms.Select(sl => string.Join(',', sl)).ToImmutableArray();
+		synonyms = searchDto.Synonyms
+			.Where(s => s.Count > 1)
+			.ToDictionary(k => k[0], sl => sl.Skip(1).ToArray(), StringComparer.OrdinalIgnoreCase);
 		var rules = searchDto.Rules.Select(ParseRule).ToImmutableArray();
 		var diminishTerms = searchDto.DiminishTerms.ToImmutableArray();
-		return new SearchConfiguration { Synonyms = flattenedSynonyms, Rules = rules, DiminishTerms = diminishTerms };
+		return new SearchConfiguration { Synonyms = synonyms, Rules = rules, DiminishTerms = diminishTerms };
 	}
 
 	private static QueryRule ParseRule(QueryRuleDto dto) =>

src/Elastic.Documentation/Search/DocumentationDocument.cs

@@ -10,20 +10,32 @@ namespace Elastic.Documentation.Search;
 public record ParentDocument
 {
 	[JsonPropertyName("title")]
-	public string? Title { get; set; }
+	public required string Title { get; set; }
 
 	[JsonPropertyName("url")]
-	public string? Url { get; set; }
+	public required string Url { get; set; }
 }
 
 public record DocumentationDocument
 {
+	[JsonPropertyName("title")]
+	public required string Title { get; set; }
+
+	/// <summary>
+	/// Search title is a combination of the title and the url components.
+	/// This is used for querying to not reward documents with short titles contributing to heavily to scoring
+	/// </summary>
+	[JsonPropertyName("search_title")]
+	public required string SearchTitle { get; set; }
+
 	[JsonPropertyName("type")]
-	public string Type { get; set; } = "doc";
+	public required string Type { get; set; } = "doc";
 
-	// TODO make this required once all doc_sets have published again
 	[JsonPropertyName("url")]
-	public string Url { get; set; } = string.Empty;
+	public required string Url { get; set; } = string.Empty;
+
+	[JsonPropertyName("hash")]
+	public string Hash { get; set; } = string.Empty;
 
 	[JsonPropertyName("navigation_depth")]
 	public int NavigationDepth { get; set; } = 50; //default to a high number so that omission gets penalized.
@@ -43,20 +55,6 @@ public record DocumentationDocument
 	[JsonPropertyName("last_updated")]
 	public DateTimeOffset LastUpdated { get; set; }
 
-	// TODO make this required once all doc_sets have published again
-	[JsonPropertyName("hash")]
-	public string Hash { get; set; } = string.Empty;
-
-	/// <summary>
-	/// Search title is a combination of the title and the url components.
-	/// This is used for querying to not reward documents with short titles contributing to heavily to scoring
-	/// </summary>
-	[JsonPropertyName("search_title")]
-	public string? SearchTitle { get; set; }
-
-	[JsonPropertyName("title")]
-	public string? Title { get; set; }
-
 	[JsonPropertyName("description")]
 	public string? Description { get; set; }
 
@@ -72,7 +70,7 @@ public record DocumentationDocument
 	[JsonPropertyName("body")]
 	public string? Body { get; set; }
 
-	// Stripped body is the body with Markdown removed, suitable for search indexing
+	/// Stripped body is the body with Markdown removed, suitable for search indexing
 	[JsonPropertyName("stripped_body")]
 	public string? StrippedBody { get; set; }