diff --git a/deep-filtering.go b/deep-filtering.go index 74803d0..ab99b8f 100644 --- a/deep-filtering.go +++ b/deep-filtering.go @@ -4,9 +4,13 @@ import ( "errors" "fmt" "reflect" + "regexp" + "strconv" + "strings" "sync" "github.com/survivorbat/go-tsyncmap" + gormqonvert "github.com/survivorbat/gorm-query-convert" "gorm.io/gorm/schema" "gorm.io/gorm" @@ -81,11 +85,27 @@ func AddDeepFilters(db *gorm.DB, objectType any, filters ...map[string]any) (*go relationalTypesInfo := getDatabaseFieldsOfType(db.NamingStrategy, schemaInfo) simpleFilter := map[string]any{} + totalFilterString := "" + functionRegex := regexp.MustCompile(`.*\((.*?)\)+`) + qonvertMap := map[string]string{} + + if _, ok := db.Plugins[gormqonvert.New(gormqonvert.CharacterConfig{}).Name()]; ok { + qonvertPlugin := db.Plugins[gormqonvert.New(gormqonvert.CharacterConfig{}).Name()] + qonvertPluginConfig := reflect.ValueOf(qonvertPlugin).Elem().FieldByName("config") + qonvertMap[qonvertPluginConfig.FieldByName("GreaterThanPrefix").String()] = ">" + qonvertMap[qonvertPluginConfig.FieldByName("GreaterOrEqualToPrefix").String()] = ">=" + qonvertMap[qonvertPluginConfig.FieldByName("LessThanPrefix").String()] = "<" + qonvertMap[qonvertPluginConfig.FieldByName("LessOrEqualToPrefix").String()] = "<=" + qonvertMap[qonvertPluginConfig.FieldByName("NotEqualToPrefix").String()] = "!=" + qonvertMap[qonvertPluginConfig.FieldByName("LikePrefix").String()] = "%s LIKE '%%%s%%'" + qonvertMap[qonvertPluginConfig.FieldByName("NotLikePrefix").String()] = "%s NOT LIKE '%%%s%%'" + } // Go through the filters for _, filterObject := range filters { // Go through all the keys of the filters for fieldName, givenFilter := range filterObject { + filterString := "" switch givenFilter.(type) { // WithFilters for relational objects case map[string]any: @@ -105,6 +125,116 @@ func AddDeepFilters(db *gorm.DB, objectType any, filters ...map[string]any) (*go // Simple filters (string, int, bool etc.) default: + // If the simple filter contains a function, build the query different + if functionRegex.MatchString(fieldName) { + checkFieldName := functionRegex.ReplaceAllString(fieldName, "$1") + if _, ok := schemaInfo.FieldsByDBName[checkFieldName]; !ok { + return nil, fmt.Errorf("failed to add filters for '%s.%s': %w", schemaInfo.Table, checkFieldName, ErrFieldDoesNotExist) + } + + if _, ok := givenFilter.([]string); ok { + for _, filter := range givenFilter.([]string) { + containedQonvert := false + qonvertOperator := "" + qonvertFilter := "" + for qonvertField, qonvertValue := range qonvertMap { + // Find longest possible prefix match in filter + // (e.g. to make sure <= is fully matched and not overwritten by <) + if strings.HasPrefix(filter, qonvertField) && len(qonvertField) > len(qonvertFilter) { + qonvertOperator = qonvertValue + qonvertFilter = qonvertField + containedQonvert = true + } + } + + if !containedQonvert { + if filterString == "" { + filterString = fmt.Sprintf("%s = '%s'", fieldName, filter) + } else { + filterString = fmt.Sprintf("%s OR %s", filterString, fmt.Sprintf("%s = '%s'", fieldName, filter)) + } + } else { + filter = strings.Replace(filter, qonvertFilter, "", 1) + if strings.Contains(qonvertOperator, "%") { + if filterString == "" { + filterString = fmt.Sprintf(qonvertOperator, fieldName, filter) + } else { + filterString = fmt.Sprintf("%s OR %s", filterString, fmt.Sprintf(qonvertOperator, fieldName, filter)) + } + } else { + if filterString == "" { + filterString = prepareFilterValue(fieldName, qonvertOperator, filter) + } else { + filterString = fmt.Sprintf("%s OR %s", filterString, prepareFilterValue(fieldName, qonvertOperator, filter)) + } + } + } + } + + filterString = fmt.Sprintf("(%s)", filterString) + if totalFilterString != "" { + totalFilterString += " AND " + } + totalFilterString += filterString + break + } + + if _, ok := givenFilter.([]int); ok { + for _, filter := range givenFilter.([]int) { + if filterString == "" { + filterString = fmt.Sprintf("%s = %d", fieldName, filter) + } else { + filterString = fmt.Sprintf("%s OR %s", filterString, fmt.Sprintf("%s = %d", fieldName, filter)) + } + } + + filterString = fmt.Sprintf("(%s)", filterString) + if totalFilterString != "" { + totalFilterString += " AND " + } + totalFilterString += filterString + break + } + + containedQonvert := false + qonvertOperator := "" + qonvertFilter := "" + for qonvertField, qonvertValue := range qonvertMap { + // Find longest possible prefix match in filter + // (e.g. to make sure <= is fully matched and not overwritten by <) + if filterStrCast, castOk := givenFilter.(string); castOk && strings.HasPrefix(filterStrCast, qonvertField) && len(qonvertField) > len(qonvertFilter) { + qonvertOperator = qonvertValue + qonvertFilter = qonvertField + containedQonvert = true + } + + } + + if containedQonvert { + givenFilter = strings.Replace(givenFilter.(string), qonvertFilter, "", 1) + if strings.Contains(qonvertOperator, "%") { + if filterString == "" { + filterString = fmt.Sprintf(qonvertOperator, fieldName, givenFilter) + } else { + filterString = fmt.Sprintf("%s OR %s", filterString, fmt.Sprintf(qonvertOperator, fieldName, givenFilter)) + } + } else { + filterString = prepareFilterValue(fieldName, qonvertOperator, givenFilter.(string)) + } + } + + if filterString == "" { + filterString = prepareFilterValueCast(fieldName, "=", givenFilter) + } + + filterString = fmt.Sprintf("(%s)", filterString) + if totalFilterString != "" { + totalFilterString += " AND " + } + totalFilterString += filterString + break + } + if _, ok := schemaInfo.FieldsByDBName[fieldName]; !ok { return nil, fmt.Errorf("failed to add filters for '%s.%s': %w", schemaInfo.Table, fieldName, ErrFieldDoesNotExist) } @@ -114,6 +244,9 @@ func AddDeepFilters(db *gorm.DB, objectType any, filters ...map[string]any) (*go } // Add simple filters + if totalFilterString != "" { + db = db.Where(totalFilterString) + } db = db.Where(simpleFilter) return db, nil @@ -148,6 +281,31 @@ type iKind[T any] interface { Elem() T } +// prepareFilterValue checks if the given filter can be converted to an int or bool and gives back the correct SQL value for it +func prepareFilterValue(fieldName string, operator string, filterValue string) string { + if value, err := strconv.Atoi(filterValue); err == nil { + return fmt.Sprintf("%s %s %d", fieldName, operator, value) + } + + if value, err := strconv.ParseBool(filterValue); err == nil { + return fmt.Sprintf("%s %s %t", fieldName, operator, value) + } + + return fmt.Sprintf("%s %s '%s'", fieldName, operator, filterValue) +} + +// prepareFilterValue checks if the given filter can be converted to an int or bool and gives back the correct SQL value for it +func prepareFilterValueCast(fieldName string, operator string, filterValue any) string { + if filterIntCast, castOk := filterValue.(int); castOk { + return fmt.Sprintf("%s %s %d", fieldName, operator, filterIntCast) + } + if filterBoolCast, castOk := filterValue.(bool); castOk { + return fmt.Sprintf("%s %s %t", fieldName, operator, filterBoolCast) + } + + return fmt.Sprintf("%s %s '%s'", fieldName, operator, filterValue.(string)) +} + // ensureConcrete ensures that the given value is a value and not a pointer, if it is, convert it to its element type func ensureConcrete[T iKind[T]](value T) T { if value.Kind() == reflect.Ptr { diff --git a/deep-filtering_test.go b/deep-filtering_test.go index 7e57494..ee2f110 100644 --- a/deep-filtering_test.go +++ b/deep-filtering_test.go @@ -7,8 +7,11 @@ import ( "testing" "github.com/ing-bank/gormtestutil" + "gorm.io/gorm" "gorm.io/gorm/schema" + gormqonvert "github.com/survivorbat/gorm-query-convert" + "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -688,6 +691,332 @@ func TestAddDeepFilters_ReturnsErrorOnUnknownFieldInformation(t *testing.T) { } } +func TestAddDeepFilters_AddsSimplelFiltersWithFunctions(t *testing.T) { + // t.Parallel() + t.Cleanup(cleanupCache) + type SimpleStruct6 struct { + Name string + Occupation string + } + + tests := map[string]struct { + records []*SimpleStruct6 + expected []*SimpleStruct6 + filters []map[string]any + expectedQuery string + }{ + "simple filter": { + records: []*SimpleStruct6{ + { + Occupation: "Dev", + Name: "John", + }, + { + Occupation: "Ops", + Name: "Jennifer", + }, + }, + expected: []*SimpleStruct6{ + { + Occupation: "Ops", + Name: "Jennifer", + }, + }, + filters: []map[string]any{ + { + "LOWER(UPPER(occupation))": "ops", + }, + }, + expectedQuery: "SELECT * FROM `simple_struct6` WHERE (LOWER(UPPER(occupation)) = 'ops')", + }, + "simple like filter": { + records: []*SimpleStruct6{ + { + Occupation: "Dev", + Name: "John", + }, + { + Occupation: "Ops", + Name: "Jennifer", + }, + }, + expected: []*SimpleStruct6{ + { + Occupation: "Ops", + Name: "Jennifer", + }, + }, + filters: []map[string]any{ + { + "LOWER(occupation)": "~op", + }, + }, + expectedQuery: "SELECT * FROM `simple_struct6` WHERE (LOWER(occupation) LIKE '%op%')", + }, + "simple like filter with or": { + records: []*SimpleStruct6{ + { + Occupation: "Dev", + Name: "John", + }, + { + Occupation: "Ops", + Name: "Jennifer", + }, + }, + expected: []*SimpleStruct6{ + { + Occupation: "Dev", + Name: "John", + }, + { + Occupation: "Ops", + Name: "Jennifer", + }, + }, + filters: []map[string]any{ + { + "LOWER(occupation)": []string{"~op", "~de"}, + }, + }, + expectedQuery: "SELECT * FROM `simple_struct6` WHERE (LOWER(occupation) LIKE '%op%' OR LOWER(occupation) LIKE '%de%')", + }, + "and filter with qonvert": { + records: []*SimpleStruct6{ + { + Occupation: "Dev", + Name: "John", + }, + { + Occupation: "Ops", + Name: "Jennifer", + }, + }, + expected: []*SimpleStruct6{ + { + Occupation: "Ops", + Name: "Jennifer", + }, + }, + filters: []map[string]any{ + { + "occupation": "Ops", + }, + { + "LENGTH(name)": ">4", + }, + }, + expectedQuery: "SELECT * FROM `simple_struct6` WHERE (LENGTH(name) > 4) AND `simple_struct6`.`occupation` = \"Ops\"", + }, + "simple or filter strings": { + records: []*SimpleStruct6{ + { + Occupation: "Dev", + Name: "John", + }, + { + Occupation: "Ops", + Name: "Jennifer", + }, + }, + expected: []*SimpleStruct6{ + { + Occupation: "Dev", + Name: "John", + }, + { + Occupation: "Ops", + Name: "Jennifer", + }, + }, + filters: []map[string]any{ + { + "UPPER(occupation)": []string{"OPS", "DEV"}, + }, + }, + expectedQuery: "SELECT * FROM `simple_struct6` WHERE (UPPER(occupation) = 'OPS' OR UPPER(occupation) = 'DEV')", + }, + "simple or filter int": { + records: []*SimpleStruct6{ + { + Occupation: "Dev", + Name: "John", + }, + { + Occupation: "Ops", + Name: "Jennifer", + }, + }, + expected: []*SimpleStruct6{ + { + Occupation: "Dev", + Name: "John", + }, + { + Occupation: "Ops", + Name: "Jennifer", + }, + }, + filters: []map[string]any{ + { + "LENGTH(name)": []int{4, 8}, + }, + }, + expectedQuery: "SELECT * FROM `simple_struct6` WHERE (LENGTH(name) = 4 OR LENGTH(name) = 8)", + }, + "or and filter with convert": { + records: []*SimpleStruct6{ + { + Occupation: "Dev", + Name: "John", + }, + { + Occupation: "Ops", + Name: "Jennifer", + }, + }, + expected: []*SimpleStruct6{ + { + Occupation: "Dev", + Name: "John", + }, + }, + filters: []map[string]any{ + { + "UPPER(occupation)": []string{"OPS", "DEV"}, + }, + { + "LENGTH(name)": []string{"<=4", ">8"}, + }, + }, + expectedQuery: "SELECT * FROM `simple_struct6` WHERE (UPPER(occupation) = 'OPS' OR UPPER(occupation) = 'DEV') AND (LENGTH(name) <= 4 OR LENGTH(name) > 8)", + }, + "simple filter with like prefix": { + records: []*SimpleStruct6{ + { + Occupation: "Dev", + Name: "John", + }, + { + Occupation: "Ops", + Name: "Jennifer", + }, + }, + expected: []*SimpleStruct6{ + { + Occupation: "Dev", + Name: "John", + }, + { + Occupation: "Ops", + Name: "Jennifer", + }, + }, + filters: []map[string]any{ + { + "UPPER(name)": []string{"~J"}, + }, + }, + expectedQuery: "SELECT * FROM `simple_struct6` WHERE (UPPER(name) LIKE '%J%')", + }, + "simple or filter with like prefix": { + records: []*SimpleStruct6{ + { + Occupation: "Dev", + Name: "John", + }, + { + Occupation: "Ops", + Name: "Jennifer", + }, + }, + expected: []*SimpleStruct6{ + { + Occupation: "Dev", + Name: "John", + }, + { + Occupation: "Ops", + Name: "Jennifer", + }, + }, + filters: []map[string]any{ + { + "UPPER(occupation)": []string{"~EV", "~OP"}, + }, + }, + expectedQuery: "SELECT * FROM `simple_struct6` WHERE (UPPER(occupation) LIKE '%EV%' OR UPPER(occupation) LIKE '%OP%')", + }, + "and or filter with like prefix": { + records: []*SimpleStruct6{ + { + Occupation: "Dev", + Name: "John", + }, + { + Occupation: "Ops", + Name: "Jennifer", + }, + }, + expected: []*SimpleStruct6{ + { + Occupation: "Ops", + Name: "Jennifer", + }, + }, + filters: []map[string]any{ + { + "UPPER(occupation)": []string{"!=DEV", "!~TEST"}, + }, + { + "LENGTH(name)": []int{3, 8}, + }, + }, + expectedQuery: "SELECT * FROM `simple_struct6` WHERE (UPPER(occupation) != 'DEV' OR UPPER(occupation) NOT LIKE '%TEST%') AND (LENGTH(name) = 3 OR LENGTH(name) = 8)", + }, + } + + for name, testData := range tests { + testData := testData + t.Run(name, func(t *testing.T) { + // t.Parallel() + // Arrange + database := gormtestutil.NewMemoryDatabase(t, gormtestutil.WithName(t.Name())) + database.Use(gormqonvert.New(gormqonvert.CharacterConfig{ + GreaterThanPrefix: ">", + GreaterOrEqualToPrefix: ">=", + LessThanPrefix: "<", + LessOrEqualToPrefix: "<=", + NotEqualToPrefix: "!=", + LikePrefix: "~", + NotLikePrefix: "!~", + })) + _ = database.AutoMigrate(&SimpleStruct6{}) + + database.CreateInBatches(testData.records, len(testData.records)) + + // Act + sqlQuery := database.ToSQL(func(tx *gorm.DB) *gorm.DB { + deepFilterQuery, err := AddDeepFilters(tx, SimpleStruct6{}, testData.filters...) + assert.Nil(t, err) + return deepFilterQuery.Find([]*SimpleStruct6{}) + }) + query, err := AddDeepFilters(database, SimpleStruct6{}, testData.filters...) + + // Assert + assert.Nil(t, err) + + if assert.NotNil(t, query) { + var result []*SimpleStruct6 + query.Preload(clause.Associations).Find(&result) + + assert.EqualValues(t, testData.expected, result) + assert.Equal(t, testData.expectedQuery, sqlQuery) + } + }) + } +} + func TestAddDeepFilters_AddsSimpleFilters(t *testing.T) { t.Parallel() t.Cleanup(cleanupCache) @@ -697,106 +1026,358 @@ func TestAddDeepFilters_AddsSimpleFilters(t *testing.T) { } tests := map[string]struct { - records []*SimpleStruct6 - expected []*SimpleStruct6 - filterMap map[string]any + records []*SimpleStruct6 + expected []*SimpleStruct6 + filterMap map[string]any + }{ + "first": { + records: []*SimpleStruct6{ + { + Occupation: "Dev", + Name: "John", + }, + { + Occupation: "Ops", + Name: "Jennifer", + }, + }, + expected: []*SimpleStruct6{ + { + Occupation: "Ops", + Name: "Jennifer", + }, + }, + filterMap: map[string]any{ + "occupation": "Ops", + }, + }, + "second": { + records: []*SimpleStruct6{ + { + Occupation: "Dev", + Name: "John", + }, + { + Occupation: "Ops", + Name: "Jennifer", + }, + { + Occupation: "Ops", + Name: "Roy", + }, + }, + expected: []*SimpleStruct6{ + { + Occupation: "Ops", + Name: "Jennifer", + }, + { + Occupation: "Ops", + Name: "Roy", + }, + }, + filterMap: map[string]any{ + "occupation": "Ops", + }, + }, + "third": { + records: []*SimpleStruct6{ + { + Occupation: "Dev", + Name: "John", + }, + { + Occupation: "Ops", + Name: "Jennifer", + }, + }, + expected: []*SimpleStruct6{ + { + Occupation: "Ops", + Name: "Jennifer", + }, + }, + filterMap: map[string]any{ + "occupation": "Ops", + "name": "Jennifer", + }, + }, + "fourth": { + records: []*SimpleStruct6{ + { + Occupation: "Dev", + Name: "John", + }, + { + Occupation: "Ops", + Name: "Jennifer", + }, + }, + expected: []*SimpleStruct6{ + { + Occupation: "Dev", + Name: "John", + }, + { + Occupation: "Ops", + Name: "Jennifer", + }, + }, + filterMap: map[string]any{ + "occupation": []string{"Ops", "Dev"}, + }, + }, + } + + for name, testData := range tests { + testData := testData + t.Run(name, func(t *testing.T) { + t.Parallel() + // Arrange + database := gormtestutil.NewMemoryDatabase(t, gormtestutil.WithName(t.Name())) + database.Use(gormqonvert.New(gormqonvert.CharacterConfig{ + GreaterThanPrefix: ">", + GreaterOrEqualToPrefix: ">=", + LessThanPrefix: "<", + LessOrEqualToPrefix: "<=", + NotEqualToPrefix: "!=", + LikePrefix: "~", + NotLikePrefix: "!~", + })) + _ = database.AutoMigrate(&SimpleStruct6{}) + + database.CreateInBatches(testData.records, len(testData.records)) + + // Act + query, err := AddDeepFilters(database, SimpleStruct6{}, testData.filterMap) + + // Assert + assert.Nil(t, err) + + if assert.NotNil(t, query) { + var result []*SimpleStruct6 + query.Preload(clause.Associations).Find(&result) + + assert.EqualValues(t, result, testData.expected) + } + }) + } +} + +func TestAddDeepFilters_AddsDeepFiltersWithOneToManyWithFunctions(t *testing.T) { + t.Parallel() + t.Cleanup(cleanupCache) + tests := map[string]struct { + records []*ComplexStruct1 + expected []ComplexStruct1 + filters []map[string]any + expectedQuery string }{ - "first": { - records: []*SimpleStruct6{ + "simple filter with function": { + records: []*ComplexStruct1{ { - Occupation: "Dev", - Name: "John", + ID: uuid.MustParse("59aa5a8f-c5de-44fa-9355-080650481687"), + Value: 1, + NestedRef: uuid.MustParse("71766db4-eb17-4457-a85c-8b89af5a319d"), // A + Nested: &NestedStruct4{ + ID: uuid.MustParse("71766db4-eb17-4457-a85c-8b89af5a319d"), // A + Name: "Johan", + Occupation: "Dev", + }, }, { - Occupation: "Ops", - Name: "Jennifer", + ID: uuid.MustParse("23292d51-4768-4c41-8475-6d4c9f0c6f69"), + Value: 11, + NestedRef: uuid.MustParse("4604bb79-ee05-4a09-b874-c3af8964d8c4"), // BObject + Nested: &NestedStruct4{ + + ID: uuid.MustParse("4604bb79-ee05-4a09-b874-c3af8964d8c4"), // BObject + Name: "Katherina", + Occupation: "Dev", + }, }, }, - expected: []*SimpleStruct6{ + expected: []ComplexStruct1{ { - Occupation: "Ops", - Name: "Jennifer", + ID: uuid.MustParse("23292d51-4768-4c41-8475-6d4c9f0c6f69"), + Value: 11, + NestedRef: uuid.MustParse("4604bb79-ee05-4a09-b874-c3af8964d8c4"), // BObject + Nested: &NestedStruct4{ + ID: uuid.MustParse("4604bb79-ee05-4a09-b874-c3af8964d8c4"), // BObject + Name: "Katherina", + Occupation: "Dev", + }, }, }, - filterMap: map[string]any{ - "occupation": "Ops", - }, - }, - "second": { - records: []*SimpleStruct6{ + filters: []map[string]any{ { - Occupation: "Dev", - Name: "John", + "nested": map[string]any{ + "LOWER(UPPER(name))": "katherina", + }, }, + }, + expectedQuery: "SELECT * FROM `complex_struct1` WHERE nested_ref IN (SELECT `id` FROM `nested_struct4` WHERE (LOWER(UPPER(name)) = 'katherina'))", + }, + "more complex query with simple filter function": { + records: []*ComplexStruct1{ { - Occupation: "Ops", - Name: "Jennifer", + ID: uuid.MustParse("59aa5a8f-c5de-44fa-9355-080650481687"), + Value: 1, + NestedRef: uuid.MustParse("71766db4-eb17-4457-a85c-8b89af5a319d"), // A + Nested: &NestedStruct4{ + ID: uuid.MustParse("71766db4-eb17-4457-a85c-8b89af5a319d"), // A + Name: "Johan", + Occupation: "Dev", + }, }, { - Occupation: "Ops", - Name: "Roy", + ID: uuid.MustParse("23292d51-4768-4c41-8475-6d4c9f0c6f69"), + Value: 11, + NestedRef: uuid.MustParse("4604bb79-ee05-4a09-b874-c3af8964d8c4"), // BObject + Nested: &NestedStruct4{ + + ID: uuid.MustParse("4604bb79-ee05-4a09-b874-c3af8964d8c4"), // BObject + Name: "Katherina", + Occupation: "Dev", + }, }, }, - expected: []*SimpleStruct6{ + expected: []ComplexStruct1{ { - Occupation: "Ops", - Name: "Jennifer", + ID: uuid.MustParse("23292d51-4768-4c41-8475-6d4c9f0c6f69"), + Value: 11, + NestedRef: uuid.MustParse("4604bb79-ee05-4a09-b874-c3af8964d8c4"), // BObject + Nested: &NestedStruct4{ + ID: uuid.MustParse("4604bb79-ee05-4a09-b874-c3af8964d8c4"), // BObject + Name: "Katherina", + Occupation: "Dev", + }, }, + }, + filters: []map[string]any{ { - Occupation: "Ops", - Name: "Roy", + "nested": map[string]any{ + "UPPER(name)": "KATHERINA", + }, + "value": 11, }, }, - filterMap: map[string]any{ - "occupation": "Ops", - }, + expectedQuery: "SELECT * FROM `complex_struct1` WHERE nested_ref IN (SELECT `id` FROM `nested_struct4` WHERE (UPPER(name) = 'KATHERINA')) AND `complex_struct1`.`value` = 11", }, - "third": { - records: []*SimpleStruct6{ + "or query with function": { + records: []*ComplexStruct1{ { - Occupation: "Dev", - Name: "John", + ID: uuid.MustParse("c98dc9f2-bfa5-4ab5-9cbb-76800e09e512"), + Value: 4, + NestedRef: uuid.MustParse("71766db4-eb17-4457-a85c-8b89af5a319d"), // A + Nested: &NestedStruct4{ + ID: uuid.MustParse("71766db4-eb17-4457-a85c-8b89af5a319d"), // A + Name: "Vanessa", + Occupation: "Ops", + }, }, { - Occupation: "Ops", - Name: "Jennifer", + ID: uuid.MustParse("2ad6a4fe-e0a4-4791-8f10-df6317cdb8b5"), + Value: 193, + NestedRef: uuid.MustParse("4604bb79-ee05-4a09-b874-c3af8964d8c4"), // BObject + Nested: &NestedStruct4{ + ID: uuid.MustParse("4604bb79-ee05-4a09-b874-c3af8964d8c4"), // BObject + Name: "Vanessa", + Occupation: "Dev", + }, + }, + { + ID: uuid.MustParse("5cc022ae-43a1-44d8-8ab5-31350e68d0b1"), + Value: 1593, + NestedRef: uuid.MustParse("4604bb79-ee05-4a09-b874-c3af8964d8c5"), // C + Nested: &NestedStruct4{ + ID: uuid.MustParse("4604bb79-ee05-4a09-b874-c3af8964d8c5"), // C + Name: "Derek", + Occupation: "Dev", + }, }, }, - expected: []*SimpleStruct6{ + expected: []ComplexStruct1{ { - Occupation: "Ops", - Name: "Jennifer", + ID: uuid.MustParse("c98dc9f2-bfa5-4ab5-9cbb-76800e09e512"), + Value: 4, + NestedRef: uuid.MustParse("71766db4-eb17-4457-a85c-8b89af5a319d"), // A + Nested: &NestedStruct4{ + ID: uuid.MustParse("71766db4-eb17-4457-a85c-8b89af5a319d"), // A + Name: "Vanessa", + Occupation: "Ops", + }, + }, + { + ID: uuid.MustParse("2ad6a4fe-e0a4-4791-8f10-df6317cdb8b5"), + Value: 193, + NestedRef: uuid.MustParse("4604bb79-ee05-4a09-b874-c3af8964d8c4"), // BObject + Nested: &NestedStruct4{ + ID: uuid.MustParse("4604bb79-ee05-4a09-b874-c3af8964d8c4"), // BObject + Name: "Vanessa", + Occupation: "Dev", + }, }, }, - filterMap: map[string]any{ - "occupation": "Ops", - "name": "Jennifer", + filters: []map[string]any{ + { + "nested": map[string]any{ + "LOWER(name)": "vanessa", + }, + }, + { + "nested": map[string]any{ + "UPPER(occupation)": []string{"DEV", "OPS"}, + }, + }, }, + expectedQuery: "SELECT * FROM `complex_struct1` WHERE nested_ref IN (SELECT `id` FROM `nested_struct4` WHERE (LOWER(name) = 'vanessa')) AND nested_ref IN (SELECT `id` FROM `nested_struct4` WHERE (UPPER(occupation) = 'DEV' OR UPPER(occupation) = 'OPS'))", }, - "fourth": { - records: []*SimpleStruct6{ + "query with qonvert and function": { + records: []*ComplexStruct1{ { - Occupation: "Dev", - Name: "John", + ID: uuid.MustParse("59aa5a8f-c5de-44fa-9355-080650481687"), + Value: 1, + NestedRef: uuid.MustParse("71766db4-eb17-4457-a85c-8b89af5a319d"), // A + Nested: &NestedStruct4{ + ID: uuid.MustParse("71766db4-eb17-4457-a85c-8b89af5a319d"), // A + Name: "Coat", + Occupation: "Product Owner", + }, }, { - Occupation: "Ops", - Name: "Jennifer", + ID: uuid.MustParse("23292d51-4768-4c41-8475-6d4c9f0c6f69"), + Value: 2, + NestedRef: uuid.MustParse("4604bb79-ee05-4a09-b874-c3af8964d8c4"), // BObject + Nested: &NestedStruct4{ + ID: uuid.MustParse("4604bb79-ee05-4a09-b874-c3af8964d8c4"), // BObject + Name: "Joke", + Occupation: "Ops", + }, }, }, - expected: []*SimpleStruct6{ + expected: []ComplexStruct1{ { - Occupation: "Dev", - Name: "John", + ID: uuid.MustParse("23292d51-4768-4c41-8475-6d4c9f0c6f69"), + Value: 2, + NestedRef: uuid.MustParse("4604bb79-ee05-4a09-b874-c3af8964d8c4"), // BObject + Nested: &NestedStruct4{ + ID: uuid.MustParse("4604bb79-ee05-4a09-b874-c3af8964d8c4"), // BObject + Name: "Joke", + Occupation: "Ops", + }, }, + }, + filters: []map[string]any{ { - Occupation: "Ops", - Name: "Jennifer", + "nested": map[string]any{ + "UPPER(name)": []string{"~OK", "~AT"}, + "LENGTH(occupation)": ">=3", + }, + "value": 2, }, }, - filterMap: map[string]any{ - "occupation": []string{"Ops", "Dev"}, - }, + expectedQuery: "SELECT * FROM `complex_struct1` WHERE nested_ref IN (SELECT `id` FROM `nested_struct4` WHERE (UPPER(name) LIKE '%OK%' OR UPPER(name) LIKE '%AT%') AND (LENGTH(occupation) >= 3)) AND `complex_struct1`.`value` = 2", }, } @@ -806,21 +1387,39 @@ func TestAddDeepFilters_AddsSimpleFilters(t *testing.T) { t.Parallel() // Arrange database := gormtestutil.NewMemoryDatabase(t, gormtestutil.WithName(t.Name())) - _ = database.AutoMigrate(&SimpleStruct6{}) + database.Use(gormqonvert.New(gormqonvert.CharacterConfig{ + GreaterThanPrefix: ">", + GreaterOrEqualToPrefix: ">=", + LessThanPrefix: "<", + LessOrEqualToPrefix: "<=", + NotEqualToPrefix: "!=", + LikePrefix: "~", + NotLikePrefix: "!~", + })) + _ = database.AutoMigrate(&ComplexStruct1{}, &NestedStruct4{}) + // Crate records database.CreateInBatches(testData.records, len(testData.records)) // Act - query, err := AddDeepFilters(database, SimpleStruct6{}, testData.filterMap) + sqlQuery := database.ToSQL(func(tx *gorm.DB) *gorm.DB { + deepFilterQuery, _ := AddDeepFilters(tx, ComplexStruct1{}, testData.filters...) + return deepFilterQuery.Find(&ComplexStruct1{}) + }) + query, err := AddDeepFilters(database, ComplexStruct1{}, testData.filters...) // Assert assert.Nil(t, err) if assert.NotNil(t, query) { - var result []*SimpleStruct6 - query.Preload(clause.Associations).Find(&result) + var result []ComplexStruct1 + res := query.Preload(clause.Associations).Find(&result) - assert.EqualValues(t, result, testData.expected) + // Handle error + assert.Nil(t, res.Error) + + assert.Equal(t, testData.expected, result) + assert.Equal(t, testData.expectedQuery, sqlQuery) } }) } @@ -1782,6 +2381,52 @@ func TestAddDeepFilters_ReturnsErrorOnNonExistingFields(t *testing.T) { }, expectedErrorMsg: "failed to add filters for 'tag_values.key': field does not exist", }, + "one to many filter with function": { + records: []*ComplexStruct3{ + { + ID: uuid.MustParse("59aa5a8f-c5de-44fa-9355-080650481687"), // A + Name: "Python", + Tags: []*Tag{ + { + ID: uuid.MustParse("1c83a7c9-e95d-4dba-b858-5eb4e34ebcf2"), + ComplexStructRef: uuid.MustParse("59aa5a8f-c5de-44fa-9355-080650481687"), + Key: "type", + Value: "interpreted", + TagValue: &TagValue{ + ID: uuid.MustParse("38769e29-e945-451f-a551-3e5811a5d363"), + Value: "test-python-value", + }, + }, + }, + }, + { + ID: uuid.MustParse("23292d51-4768-4c41-8475-6d4c9f0c6f69"), // BObject + Name: "Go", + Tags: []*Tag{ + { + ID: uuid.MustParse("17983ba8-2d26-4e36-bb6b-6c5a04b6606e"), + ComplexStructRef: uuid.MustParse("23292d51-4768-4c41-8475-6d4c9f0c6f69"), + Key: "type", + Value: "compiled", + TagValue: &TagValue{ + ID: uuid.MustParse("e75a2f7e-0e1c-4f9c-a8ce-af90f1b64baa"), + Value: "test-go-value", + }, + }, + }, + }, + }, + filterMap: []map[string]any{ + { + "tags": map[string]any{ + "tag_value": map[string]any{ + "LOWER(key)": "test-python-value", + }, + }, + }, + }, + expectedErrorMsg: "failed to add filters for 'tag_values.key': field does not exist", + }, "many to one filter": { records: []*ComplexStruct3{ { diff --git a/go.mod b/go.mod index b28cf4e..7ce16b6 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/ing-bank/gormtestutil v0.0.0 github.com/stretchr/testify v1.8.1 github.com/survivorbat/go-tsyncmap v0.0.0 + github.com/survivorbat/gorm-query-convert v0.1.0 gorm.io/driver/sqlite v1.5.2 gorm.io/gorm v1.30.0 ) @@ -15,7 +16,6 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect - github.com/kr/text v0.2.0 // indirect github.com/mattn/go-sqlite3 v1.14.17 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect golang.org/x/text v0.25.0 // indirect diff --git a/go.sum b/go.sum index e5768f0..839f7a0 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,3 @@ -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -29,6 +28,8 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/survivorbat/go-tsyncmap v0.0.0 h1:XTc1+uXyuw//1Hhpg4IxW6tEe3Tvd2d5vM/6IPqmkeg= github.com/survivorbat/go-tsyncmap v0.0.0/go.mod h1:zKe2CuXEo+c1d9DVT5L7AG2jPTdWi7QQN/Gk+26Vecg= +github.com/survivorbat/gorm-query-convert v0.1.0 h1:ct05m9K79EbYj45sfLpiYRay+7ZGlg+aGZpuGzFeqtU= +github.com/survivorbat/gorm-query-convert v0.1.0/go.mod h1:JbZVdQDRMhGsdzRpkmvYHxp8goY0bKKUrY3dxnq1d9w= golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=