From 08611ab78a1bb5ddb1bf20b9ecc78cb9f60950b5 Mon Sep 17 00:00:00 2001 From: Andrey Stolyarov Date: Tue, 18 Nov 2025 21:47:51 +0300 Subject: [PATCH] Add fuzzy filters --- .../ru/d10xa/jsonlogviewer/ViewElement.scala | 2 + .../decline/yaml/ConfigYamlLoaderImpl.scala | 6 + .../ru/d10xa/jsonlogviewer/FuzzyFilter.scala | 108 +++++++ .../d10xa/jsonlogviewer/FuzzyTokenizer.scala | 48 ++++ .../d10xa/jsonlogviewer/LogViewerStream.scala | 4 + .../jsonlogviewer/config/ResolvedConfig.scala | 8 + .../jsonlogviewer/decline/yaml/Feed.scala | 2 + .../d10xa/jsonlogviewer/FuzzyFilterTest.scala | 270 ++++++++++++++++++ .../jsonlogviewer/FuzzyTokenizerTest.scala | 118 ++++++++ .../jsonlogviewer/JsonDetectorTest.scala | 2 +- .../jsonlogviewer/JsonPrefixPostfixTest.scala | 4 +- .../LogViewerStreamIntegrationTest.scala | 4 + .../LogViewerStreamLiveReloadTest.scala | 4 + .../ParseResultKeysGetByKeyTest.scala | 2 + .../jsonlogviewer/ParseResultKeysTest.scala | 4 + .../YamlCommandExecutionTest.scala | 4 + .../config/ConfigResolverTest.scala | 4 + .../csv/CsvLogLineParserTest.scala | 2 + .../formatout/ColorLineFormatterTest.scala | 2 + .../query/LogLineQueryPredicateImplTest.scala | 2 + 20 files changed, 597 insertions(+), 3 deletions(-) create mode 100644 json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/FuzzyFilter.scala create mode 100644 json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/FuzzyTokenizer.scala create mode 100644 json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/FuzzyFilterTest.scala create mode 100644 json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/FuzzyTokenizerTest.scala diff --git a/frontend-laminar/src/main/scala/ru/d10xa/jsonlogviewer/ViewElement.scala b/frontend-laminar/src/main/scala/ru/d10xa/jsonlogviewer/ViewElement.scala index 80327fd..16303a0 100644 --- a/frontend-laminar/src/main/scala/ru/d10xa/jsonlogviewer/ViewElement.scala +++ b/frontend-laminar/src/main/scala/ru/d10xa/jsonlogviewer/ViewElement.scala @@ -38,6 +38,8 @@ object ViewElement { formatIn = config.formatIn, rawInclude = None, rawExclude = None, + fuzzyInclude = None, + fuzzyExclude = None, excludeFields = None, fieldNames = None, showEmptyFields = None diff --git a/json-log-viewer/jvm/src/main/scala/ru/d10xa/jsonlogviewer/decline/yaml/ConfigYamlLoaderImpl.scala b/json-log-viewer/jvm/src/main/scala/ru/d10xa/jsonlogviewer/decline/yaml/ConfigYamlLoaderImpl.scala index 0cc2637..f13be5b 100644 --- a/json-log-viewer/jvm/src/main/scala/ru/d10xa/jsonlogviewer/decline/yaml/ConfigYamlLoaderImpl.scala +++ b/json-log-viewer/jvm/src/main/scala/ru/d10xa/jsonlogviewer/decline/yaml/ConfigYamlLoaderImpl.scala @@ -201,6 +201,10 @@ class ConfigYamlLoaderImpl extends ConfigYamlLoader { parseOptionalListString(feedFields, "rawInclude") val rawExcludeValidated = parseOptionalListString(feedFields, "rawExclude") + val fuzzyIncludeValidated = + parseOptionalListString(feedFields, "fuzzyInclude") + val fuzzyExcludeValidated = + parseOptionalListString(feedFields, "fuzzyExclude") val excludeFieldsValidated = parseOptionalListString( feedFields, @@ -218,6 +222,8 @@ class ConfigYamlLoaderImpl extends ConfigYamlLoader { fieldNamesValidated, rawIncludeValidated, rawExcludeValidated, + fuzzyIncludeValidated, + fuzzyExcludeValidated, excludeFieldsValidated, showEmptyFieldsValidated ) diff --git a/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/FuzzyFilter.scala b/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/FuzzyFilter.scala new file mode 100644 index 0000000..696b133 --- /dev/null +++ b/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/FuzzyFilter.scala @@ -0,0 +1,108 @@ +package ru.d10xa.jsonlogviewer + +import ru.d10xa.jsonlogviewer.config.ResolvedConfig + +/** Fuzzy filter that searches for patterns across all fields in a parsed log + * entry using token-based matching. + * + * Unlike rawFilter (regex on raw strings) or SQL filters (exact field + * matching), fuzzy filter: + * - Works after JSON parsing + * - Searches across all fields (level, message, stackTrace, etc.) + * - Uses tokenization to ignore punctuation + * - Is case-insensitive + * - Supports partial token matching + * + * Example: {{{ fuzzyInclude: ["error timeout"] // Will match: {"level": + * "ERROR", "message": "Connection timeout"} {"message": "Error: request + * timeout occurred"} {"error_code": "500", "details": "timeout"} }}} + * + * @param config + * Resolved configuration containing fuzzyInclude and fuzzyExclude patterns + */ +class FuzzyFilter(config: ResolvedConfig) { + + /** Collects all values from the parsed log entry into a single searchable + * string. + * + * Includes standard fields (level, message, etc.) and all custom attributes + * from otherAttributes. + * + * @param parseResult + * Parsed log entry + * @return + * Space-separated concatenation of all field values + */ + private def collectAllValues(parseResult: ParseResult): String = + parseResult.parsed match { + case None => parseResult.raw // Fallback to raw string if parsing failed + case Some(parsed) => + val standardFields = List( + parsed.timestamp, + parsed.level, + parsed.message, + parsed.stackTrace, + parsed.loggerName, + parsed.threadName + ).flatten + + val otherValues = parsed.otherAttributes.values + + (standardFields ++ otherValues).mkString(" ") + } + + /** Token-based fuzzy matching: checks if all tokens from the pattern exist in + * the text. + * + * Uses partial matching: pattern token "timeout" will match text tokens + * "timeout", "timeouts", "timeout_ms", etc. + * + * @param text + * Text to search in (typically all log field values concatenated) + * @param pattern + * Search pattern (e.g., "error timeout") + * @return + * true if all pattern tokens are found in text tokens + */ + private def tokenBasedMatch(text: String, pattern: String): Boolean = { + val textTokens = FuzzyTokenizer.tokenize(text) + val patternTokens = FuzzyTokenizer.tokenize(pattern) + + // All pattern tokens must be present in text tokens (with partial matching) + patternTokens.forall { patternToken => + textTokens.exists(textToken => textToken.contains(patternToken)) + } + } + + /** Tests whether the parsed log entry matches fuzzyInclude and fuzzyExclude + * patterns. + * + * Logic: + * - fuzzyInclude: At least one pattern must match (OR logic) + * - fuzzyExclude: No pattern should match (AND NOT logic) + * - If fuzzyInclude is empty or None, all entries pass + * + * @param parseResult + * Parsed log entry to test + * @return + * true if entry should be included in output + */ + def test(parseResult: ParseResult): Boolean = { + val allValues = collectAllValues(parseResult) + + val includeMatches = config.fuzzyInclude match { + case None => true + case Some(patterns) if patterns.isEmpty => true + case Some(patterns) => + patterns.exists(pattern => tokenBasedMatch(allValues, pattern)) + } + + val excludeMatches = config.fuzzyExclude match { + case None => true + case Some(patterns) => + patterns.forall(pattern => !tokenBasedMatch(allValues, pattern)) + } + + includeMatches && excludeMatches + } +} diff --git a/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/FuzzyTokenizer.scala b/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/FuzzyTokenizer.scala new file mode 100644 index 0000000..4b946f5 --- /dev/null +++ b/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/FuzzyTokenizer.scala @@ -0,0 +1,48 @@ +package ru.d10xa.jsonlogviewer + +/** Tokenizer for fuzzy search that handles punctuation, quotes, and special + * characters in log messages. + * + * Rules: + * - Splits text into words while preserving meaningful characters + * - Keeps dots and underscores inside words (e.g., john.doe, user_id) + * - Removes standalone punctuation + * - Converts to lowercase for case-insensitive matching + * - Filters tokens shorter than 2 characters + * + * Examples: {{{ tokenize("User 'john.doe' timeout") → Set("user", "john.doe", + * "timeout") tokenize("ERROR: database.query() failed") → Set("error", + * "database.query", "failed") tokenize("card_number=1234") → + * Set("card_number", "1234") }}} + */ +object FuzzyTokenizer { + + /** Tokenizes text into a set of searchable words. + * + * @param text + * Text to tokenize + * @return + * Set of normalized tokens (lowercase, minimum 2 characters) + */ + def tokenize(text: String): Set[String] = { + // Pattern matches alphanumeric characters, dots, and underscores + // This preserves: user_id, john.doe, 192.168.1.1, etc. + val tokenPattern = """[\w._]+""".r + + tokenPattern + .findAllIn(text.toLowerCase) + .toSet + .filter(_.length >= 2) + .filterNot(isOnlyPunctuation) + } + + /** Checks if a token consists only of non-alphanumeric characters. + * + * @param token + * Token to check + * @return + * true if token contains only punctuation + */ + private def isOnlyPunctuation(token: String): Boolean = + token.forall(c => !c.isLetterOrDigit) +} diff --git a/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/LogViewerStream.scala b/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/LogViewerStream.scala index 7a85103..267bd17 100644 --- a/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/LogViewerStream.scala +++ b/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/LogViewerStream.scala @@ -123,6 +123,7 @@ object LogViewerStream { val timestampFilter = TimestampFilter() val parseResultKeys = ParseResultKeys(resolvedConfig) val logLineFilter = LogLineFilter(resolvedConfig, parseResultKeys) + val fuzzyFilter = new FuzzyFilter(resolvedConfig) val outputLineFormatter = resolvedConfig.formatOut match { case Some(Config.FormatOut.Raw) => RawFormatter() @@ -142,6 +143,7 @@ object LogViewerStream { .map(parser.parse) .filter(logLineFilter.grep) .filter(logLineFilter.logLineQueryPredicate) + .filter(fuzzyFilter.test) .through( timestampFilter.filterTimestampAfter(resolvedConfig.timestampAfter) ) @@ -165,6 +167,7 @@ object LogViewerStream { val timestampFilter = TimestampFilter() val parseResultKeys = ParseResultKeys(resolvedConfig) val logLineFilter = LogLineFilter(resolvedConfig, parseResultKeys) + val fuzzyFilter = new FuzzyFilter(resolvedConfig) val outputLineFormatter = resolvedConfig.formatOut match { case Some(Config.FormatOut.Raw) => RawFormatter() @@ -183,6 +186,7 @@ object LogViewerStream { .map(csvHeaderParser.parse) .filter(logLineFilter.grep) .filter(logLineFilter.logLineQueryPredicate) + .filter(fuzzyFilter.test) .through( timestampFilter.filterTimestampAfter(resolvedConfig.timestampAfter) ) diff --git a/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/config/ResolvedConfig.scala b/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/config/ResolvedConfig.scala index a63eff0..17f4bf2 100644 --- a/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/config/ResolvedConfig.scala +++ b/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/config/ResolvedConfig.scala @@ -30,6 +30,8 @@ final case class ResolvedConfig( // Feed-specific settings rawInclude: Option[List[String]], rawExclude: Option[List[String]], + fuzzyInclude: Option[List[String]], + fuzzyExclude: Option[List[String]], excludeFields: Option[List[String]], // Timestamp settings @@ -89,6 +91,8 @@ object ConfigResolver { fieldNames = feedFieldNames, rawInclude = feed.rawInclude, rawExclude = feed.rawExclude, + fuzzyInclude = feed.fuzzyInclude, + fuzzyExclude = feed.fuzzyExclude, excludeFields = feed.excludeFields, timestampAfter = config.timestamp.after, timestampBefore = config.timestamp.before, @@ -109,6 +113,8 @@ object ConfigResolver { fieldNames = globalFieldNames, rawInclude = None, rawExclude = None, + fuzzyInclude = None, + fuzzyExclude = None, excludeFields = None, timestampAfter = config.timestamp.after, timestampBefore = config.timestamp.before, @@ -130,6 +136,8 @@ object ConfigResolver { fieldNames = config.fieldNames, rawInclude = None, rawExclude = None, + fuzzyInclude = None, + fuzzyExclude = None, excludeFields = None, timestampAfter = config.timestamp.after, timestampBefore = config.timestamp.before, diff --git a/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/decline/yaml/Feed.scala b/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/decline/yaml/Feed.scala index ce42164..96c28ce 100644 --- a/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/decline/yaml/Feed.scala +++ b/json-log-viewer/shared/src/main/scala/ru/d10xa/jsonlogviewer/decline/yaml/Feed.scala @@ -12,6 +12,8 @@ case class Feed( fieldNames: Option[FieldNames], rawInclude: Option[List[String]], rawExclude: Option[List[String]], + fuzzyInclude: Option[List[String]], + fuzzyExclude: Option[List[String]], excludeFields: Option[List[String]], showEmptyFields: Option[Boolean] ) diff --git a/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/FuzzyFilterTest.scala b/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/FuzzyFilterTest.scala new file mode 100644 index 0000000..cfc3735 --- /dev/null +++ b/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/FuzzyFilterTest.scala @@ -0,0 +1,270 @@ +package ru.d10xa.jsonlogviewer + +import munit.FunSuite +import ru.d10xa.jsonlogviewer.config.ResolvedConfig +import ru.d10xa.jsonlogviewer.decline.FieldNamesConfig + +class FuzzyFilterTest extends FunSuite { + + private def createResolvedConfig( + fuzzyInclude: Option[List[String]] = None, + fuzzyExclude: Option[List[String]] = None + ): ResolvedConfig = ResolvedConfig( + feedName = None, + commands = List.empty, + inlineInput = None, + filter = None, + formatIn = None, + formatOut = None, + fieldNames = FieldNamesConfig( + timestampFieldName = "@timestamp", + levelFieldName = "level", + messageFieldName = "message", + stackTraceFieldName = "stack_trace", + loggerNameFieldName = "logger_name", + threadNameFieldName = "thread_name" + ), + rawInclude = None, + rawExclude = None, + fuzzyInclude = fuzzyInclude, + fuzzyExclude = fuzzyExclude, + excludeFields = None, + timestampAfter = None, + timestampBefore = None, + grep = List.empty, + showEmptyFields = false + ) + + private def createParseResult( + level: Option[String] = None, + message: Option[String] = None, + otherAttributes: Map[String, String] = Map.empty + ): ParseResult = { + val parsed = ParsedLine( + timestamp = Some("2024-01-01T10:00:00Z"), + level = level, + message = message, + stackTrace = None, + loggerName = None, + threadName = None, + otherAttributes = otherAttributes + ) + ParseResult( + raw = """{"level":"ERROR","message":"Connection timeout"}""", + parsed = Some(parsed), + middle = "", + prefix = None, + postfix = None + ) + } + + test("should match pattern across different fields") { + val config = createResolvedConfig( + fuzzyInclude = Some(List("error timeout")) + ) + val filter = new FuzzyFilter(config) + + val parseResult = createParseResult( + level = Some("ERROR"), + message = Some("Connection timeout") + ) + + assert(filter.test(parseResult)) + } + + test("should be case insensitive") { + val config = createResolvedConfig( + fuzzyInclude = Some(List("error timeout")) + ) + val filter = new FuzzyFilter(config) + + val parseResult = createParseResult( + level = Some("ERROR"), + message = Some("TIMEOUT occurred") + ) + + assert(filter.test(parseResult)) + } + + test("should work with word order independence") { + val config = createResolvedConfig( + fuzzyInclude = Some(List("timeout error")) + ) + val filter = new FuzzyFilter(config) + + val parseResult = createParseResult( + level = Some("ERROR"), + message = Some("Connection timeout") + ) + + assert(filter.test(parseResult)) + } + + test("should search in otherAttributes") { + val config = createResolvedConfig( + fuzzyInclude = Some(List("error timeout")) + ) + val filter = new FuzzyFilter(config) + + val parseResult = createParseResult( + level = Some("INFO"), + message = Some("Processing request"), + otherAttributes = Map("status" -> "error", "duration" -> "timeout") + ) + + assert(filter.test(parseResult)) + } + + test("fuzzyExclude should exclude matches") { + val config = createResolvedConfig( + fuzzyInclude = Some(List("error timeout")), + fuzzyExclude = Some(List("retry succeeded")) + ) + val filter = new FuzzyFilter(config) + + val parseResult = createParseResult( + level = Some("ERROR"), + message = Some("Connection timeout but retry succeeded") + ) + + assert(!filter.test(parseResult)) + } + + test("should pass when fuzzyInclude is empty") { + val config = createResolvedConfig( + fuzzyInclude = Some(List.empty) + ) + val filter = new FuzzyFilter(config) + + val parseResult = createParseResult( + level = Some("ERROR"), + message = Some("Any message") + ) + + assert(filter.test(parseResult)) + } + + test("should pass when fuzzyInclude is None") { + val config = createResolvedConfig( + fuzzyInclude = None + ) + val filter = new FuzzyFilter(config) + + val parseResult = createParseResult( + level = Some("ERROR"), + message = Some("Any message") + ) + + assert(filter.test(parseResult)) + } + + test("should support multiple include patterns (OR logic)") { + val config = createResolvedConfig( + fuzzyInclude = Some(List("database timeout", "connection refused")) + ) + val filter = new FuzzyFilter(config) + + val parseResult1 = createParseResult( + level = Some("ERROR"), + message = Some("Database query timeout") + ) + assert(filter.test(parseResult1)) + + val parseResult2 = createParseResult( + level = Some("ERROR"), + message = Some("Connection refused by server") + ) + assert(filter.test(parseResult2)) + + val parseResult3 = createParseResult( + level = Some("ERROR"), + message = Some("Unknown error") + ) + assert(!filter.test(parseResult3)) + } + + test("should support multiple exclude patterns (AND NOT logic)") { + val config = createResolvedConfig( + fuzzyExclude = Some(List("test debug", "health check")) + ) + val filter = new FuzzyFilter(config) + + val parseResult1 = createParseResult( + level = Some("INFO"), + message = Some("Test environment debug mode") + ) + assert(!filter.test(parseResult1)) + + val parseResult2 = createParseResult( + level = Some("INFO"), + message = Some("Health check succeeded") + ) + assert(!filter.test(parseResult2)) + + val parseResult3 = createParseResult( + level = Some("ERROR"), + message = Some("Production error") + ) + assert(filter.test(parseResult3)) + } + + test("should handle punctuation in log messages") { + val config = createResolvedConfig( + fuzzyInclude = Some(List("database query failed")) + ) + val filter = new FuzzyFilter(config) + + val parseResult = createParseResult( + level = Some("ERROR"), + message = Some("ERROR: database.query() failed - connection lost") + ) + + assert(filter.test(parseResult)) + } + + test("should support partial token matching") { + val config = createResolvedConfig( + fuzzyInclude = Some(List("timeout error")) + ) + val filter = new FuzzyFilter(config) + + val parseResult = createParseResult( + level = Some("ERROR"), + message = Some("Connection timeouts and errors occurred") + ) + + assert(filter.test(parseResult)) + } + + test("should handle missing parsed data by falling back to raw") { + val config = createResolvedConfig( + fuzzyInclude = Some(List("error timeout")) + ) + val filter = new FuzzyFilter(config) + + val parseResult = ParseResult( + raw = """ERROR: Connection timeout""", + parsed = None, + middle = "", + prefix = None, + postfix = None + ) + + assert(filter.test(parseResult)) + } + + test("should not match when pattern words are missing") { + val config = createResolvedConfig( + fuzzyInclude = Some(List("database timeout user")) + ) + val filter = new FuzzyFilter(config) + + val parseResult = createParseResult( + level = Some("ERROR"), + message = Some("Database timeout occurred") + // Missing "user" + ) + + assert(!filter.test(parseResult)) + } +} diff --git a/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/FuzzyTokenizerTest.scala b/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/FuzzyTokenizerTest.scala new file mode 100644 index 0000000..1eda74e --- /dev/null +++ b/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/FuzzyTokenizerTest.scala @@ -0,0 +1,118 @@ +package ru.d10xa.jsonlogviewer + +import munit.FunSuite + +class FuzzyTokenizerTest extends FunSuite { + + test("tokenize should handle basic words") { + val result = FuzzyTokenizer.tokenize("error timeout user") + assertEquals(result, Set("error", "timeout", "user")) + } + + test("tokenize should handle punctuation") { + val result = FuzzyTokenizer.tokenize("User 'john.doe' timeout") + assert(result.contains("user")) + assert(result.contains("john.doe")) + assert(result.contains("timeout")) + } + + test("tokenize should preserve dots in words") { + val result = FuzzyTokenizer.tokenize("database.query() failed") + assert(result.contains("database.query")) + assert(result.contains("failed")) + } + + test("tokenize should preserve underscores") { + val result = FuzzyTokenizer.tokenize("card_number=1234 user_id=5678") + assert(result.contains("card_number")) + assert(result.contains("user_id")) + assert(result.contains("1234")) + assert(result.contains("5678")) + } + + test("tokenize should handle quoted strings") { + val result = + FuzzyTokenizer.tokenize("reason=\"insufficient funds\" error") + assert(result.contains("reason")) + assert(result.contains("insufficient")) + assert(result.contains("funds")) + assert(result.contains("error")) + } + + test("tokenize should filter short tokens") { + val result = FuzzyTokenizer.tokenize("a b c error") + assert(!result.contains("a")) + assert(!result.contains("b")) + assert(!result.contains("c")) + assert(result.contains("error")) + } + + test("tokenize should be case insensitive") { + val result1 = FuzzyTokenizer.tokenize("ERROR Timeout USER") + val result2 = FuzzyTokenizer.tokenize("error timeout user") + assertEquals(result1, result2) + assertEquals(result1, Set("error", "timeout", "user")) + } + + test("tokenize should handle complex punctuation") { + val result = FuzzyTokenizer.tokenize( + "ERROR: database.query() failed - timeout=30s (retry: 3)" + ) + assert(result.contains("error")) + assert(result.contains("database.query")) + assert(result.contains("failed")) + assert(result.contains("timeout")) + assert(result.contains("30s")) + assert(result.contains("retry")) + } + + test("tokenize should handle brackets and parens") { + val result = FuzzyTokenizer.tokenize("User[admin] login(failed)") + assert(result.contains("user")) + assert(result.contains("admin")) + assert(result.contains("login")) + assert(result.contains("failed")) + } + + test("tokenize should handle URLs and paths") { + val result = + FuzzyTokenizer.tokenize("Request /api/v1/users?id=123 returned 500") + assert(result.contains("request")) + assert(result.contains("api")) + assert(result.contains("v1")) + assert(result.contains("users")) + assert(result.contains("id")) + assert(result.contains("123")) + assert(result.contains("returned")) + assert(result.contains("500")) + } + + test("tokenize should handle email addresses") { + val result = FuzzyTokenizer.tokenize("User john.doe@example.com logged in") + assert(result.contains("user")) + assert(result.contains("john.doe")) + assert(result.contains("example.com")) + assert(result.contains("logged")) + assert(result.contains("in")) + } + + test("tokenize should handle empty string") { + val result = FuzzyTokenizer.tokenize("") + assertEquals(result, Set.empty[String]) + } + + test("tokenize should handle only punctuation") { + val result = FuzzyTokenizer.tokenize("!@#$%^&*()") + assertEquals(result, Set.empty[String]) + } + + test("tokenize should handle mixed case with special chars") { + val result = FuzzyTokenizer.tokenize( + "PaymentError: card_declined (code=INSUFFICIENT_FUNDS)" + ) + assert(result.contains("paymenterror")) + assert(result.contains("card_declined")) + assert(result.contains("code")) + assert(result.contains("insufficient_funds")) + } +} diff --git a/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/JsonDetectorTest.scala b/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/JsonDetectorTest.scala index d729440..df3d46c 100644 --- a/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/JsonDetectorTest.scala +++ b/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/JsonDetectorTest.scala @@ -5,7 +5,7 @@ class JsonDetectorTest extends munit.FunSuite { test("valid json") { val stringWithJsonInside = """abc {"x":"y"}a""" - val Some((start, end)) = jsonDetector.detectJson(stringWithJsonInside) + val Some((start, end)) = jsonDetector.detectJson(stringWithJsonInside): @unchecked assertEquals(stringWithJsonInside.charAt(start), '{') assertEquals(stringWithJsonInside.charAt(end), '}') } diff --git a/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/JsonPrefixPostfixTest.scala b/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/JsonPrefixPostfixTest.scala index 3be0146..aa1366f 100644 --- a/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/JsonPrefixPostfixTest.scala +++ b/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/JsonPrefixPostfixTest.scala @@ -4,14 +4,14 @@ class JsonPrefixPostfixTest extends munit.FunSuite { val jsonPrefixPostfix = new JsonPrefixPostfix(new JsonDetector) test("extract prefix and postfix") { val (json, Some(prefix), Some(postfix)) = - jsonPrefixPostfix.detectJson("""a{"x":"y"}b""") + jsonPrefixPostfix.detectJson("""a{"x":"y"}b"""): @unchecked assertEquals(json, """{"x":"y"}""") assertEquals(prefix, "a") assertEquals(postfix, "b") } test("extract prefix") { val (json, Some(prefix), postfixOpt) = - jsonPrefixPostfix.detectJson("""a{"x":"y"}""") + jsonPrefixPostfix.detectJson("""a{"x":"y"}"""): @unchecked assertEquals(json, """{"x":"y"}""") assertEquals(prefix, "a") assertEquals(postfixOpt, None) diff --git a/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/LogViewerStreamIntegrationTest.scala b/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/LogViewerStreamIntegrationTest.scala index 49afff7..4527c1e 100644 --- a/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/LogViewerStreamIntegrationTest.scala +++ b/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/LogViewerStreamIntegrationTest.scala @@ -62,6 +62,8 @@ class LogViewerStreamIntegrationTest extends CatsEffectSuite { fieldNames = None, rawInclude = None, rawExclude = None, + fuzzyInclude = None, + fuzzyExclude = None, excludeFields = None, showEmptyFields = None ) @@ -167,6 +169,8 @@ class LogViewerStreamIntegrationTest extends CatsEffectSuite { fieldNames = None, rawInclude = None, rawExclude = None, + fuzzyInclude = None, + fuzzyExclude = None, excludeFields = None, showEmptyFields = None ) diff --git a/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/LogViewerStreamLiveReloadTest.scala b/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/LogViewerStreamLiveReloadTest.scala index 9e298ec..57d12ed 100644 --- a/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/LogViewerStreamLiveReloadTest.scala +++ b/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/LogViewerStreamLiveReloadTest.scala @@ -60,6 +60,8 @@ class LogViewerStreamLiveReloadTest extends CatsEffectSuite { fieldNames = None, rawInclude = None, rawExclude = None, + fuzzyInclude = None, + fuzzyExclude = None, excludeFields = None, showEmptyFields = None ) @@ -135,6 +137,8 @@ class LogViewerStreamLiveReloadTest extends CatsEffectSuite { fieldNames = None, rawInclude = None, rawExclude = None, + fuzzyInclude = None, + fuzzyExclude = None, excludeFields = None, showEmptyFields = None ) diff --git a/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/ParseResultKeysGetByKeyTest.scala b/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/ParseResultKeysGetByKeyTest.scala index bf924d1..904a061 100644 --- a/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/ParseResultKeysGetByKeyTest.scala +++ b/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/ParseResultKeysGetByKeyTest.scala @@ -23,6 +23,8 @@ class ParseResultKeysGetByKeyTest extends FunSuite { ), rawInclude = None, rawExclude = None, + fuzzyInclude = None, + fuzzyExclude = None, excludeFields = None, timestampAfter = None, timestampBefore = None, diff --git a/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/ParseResultKeysTest.scala b/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/ParseResultKeysTest.scala index f4f8744..aac9a36 100644 --- a/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/ParseResultKeysTest.scala +++ b/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/ParseResultKeysTest.scala @@ -26,6 +26,8 @@ class ParseResultKeysTest extends FunSuite { ), rawInclude = None, rawExclude = None, + fuzzyInclude = None, + fuzzyExclude = None, excludeFields = None, timestampAfter = None, timestampBefore = None, @@ -50,6 +52,8 @@ class ParseResultKeysTest extends FunSuite { ), rawInclude = None, rawExclude = None, + fuzzyInclude = None, + fuzzyExclude = None, excludeFields = None, timestampAfter = None, timestampBefore = None, diff --git a/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/YamlCommandExecutionTest.scala b/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/YamlCommandExecutionTest.scala index 9e8d049..3e67785 100644 --- a/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/YamlCommandExecutionTest.scala +++ b/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/YamlCommandExecutionTest.scala @@ -62,6 +62,8 @@ class YamlCommandExecutionTest extends CatsEffectSuite { fieldNames = None, rawInclude = None, rawExclude = None, + fuzzyInclude = None, + fuzzyExclude = None, excludeFields = None, showEmptyFields = None ) @@ -121,6 +123,8 @@ class YamlCommandExecutionTest extends CatsEffectSuite { fieldNames = None, rawInclude = None, rawExclude = None, + fuzzyInclude = None, + fuzzyExclude = None, excludeFields = None, showEmptyFields = None ) diff --git a/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/config/ConfigResolverTest.scala b/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/config/ConfigResolverTest.scala index e466a93..be56879 100644 --- a/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/config/ConfigResolverTest.scala +++ b/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/config/ConfigResolverTest.scala @@ -105,6 +105,8 @@ class ConfigResolverTest extends FunSuite { ), rawInclude = None, rawExclude = None, + fuzzyInclude = None, + fuzzyExclude = None, excludeFields = None, showEmptyFields = None ), @@ -117,6 +119,8 @@ class ConfigResolverTest extends FunSuite { fieldNames = None, rawInclude = None, rawExclude = None, + fuzzyInclude = None, + fuzzyExclude = None, excludeFields = None, showEmptyFields = None ) diff --git a/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/csv/CsvLogLineParserTest.scala b/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/csv/CsvLogLineParserTest.scala index 42e3ea9..57b22b8 100644 --- a/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/csv/CsvLogLineParserTest.scala +++ b/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/csv/CsvLogLineParserTest.scala @@ -23,6 +23,8 @@ class CsvLogLineParserTest extends FunSuite { formatOut = None, rawInclude = None, rawExclude = None, + fuzzyInclude = None, + fuzzyExclude = None, excludeFields = None, timestampAfter = None, timestampBefore = None, diff --git a/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/formatout/ColorLineFormatterTest.scala b/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/formatout/ColorLineFormatterTest.scala index b9f4563..8891b55 100644 --- a/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/formatout/ColorLineFormatterTest.scala +++ b/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/formatout/ColorLineFormatterTest.scala @@ -25,6 +25,8 @@ class ColorLineFormatterTest extends FunSuite { ), rawInclude = None, rawExclude = None, + fuzzyInclude = None, + fuzzyExclude = None, excludeFields = None, timestampAfter = None, timestampBefore = None, diff --git a/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/query/LogLineQueryPredicateImplTest.scala b/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/query/LogLineQueryPredicateImplTest.scala index 638eaa2..8948223 100644 --- a/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/query/LogLineQueryPredicateImplTest.scala +++ b/json-log-viewer/shared/src/test/scala/ru/d10xa/jsonlogviewer/query/LogLineQueryPredicateImplTest.scala @@ -100,6 +100,8 @@ class LogLineQueryPredicateImplTest extends munit.FunSuite { formatOut = None, rawInclude = None, rawExclude = None, + fuzzyInclude = None, + fuzzyExclude = None, excludeFields = None, timestampAfter = None, timestampBefore = None,