From 1674abae7321365cacfc836b3e70a5788eb06291 Mon Sep 17 00:00:00 2001 From: ston1th Date: Wed, 30 Aug 2017 22:01:19 +0200 Subject: [PATCH] added sort support --- README.md | 146 ++++++++++++++++++++++++++++++++----------------- cmd/main.go | 27 +++++++-- filter.go | 93 ++++++++++++++++++++++++++++--- filter_test.go | 146 ++++++++++++++++++++++++++++++++++++++++++++++++- helper.go | 31 +++++++++++ 5 files changed, 377 insertions(+), 66 deletions(-) diff --git a/README.md b/README.md index 584d55c..ed7926c 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,61 @@ -# filter - a simple struct field filter for go +# filter - a simple struct field filter and sorter for go -## Usage +## Query Syntax + +### Matching + +Explicit match: `:` + +Wildcard match (use ** to match the '*' character): + +``` +:* +:* +:** +``` + +Nested structs: `.:` + +Multiple matches (seperated by a single whitespace): + +``` +: :* +``` + +Bool does not support the wildcard operator, only the literal strings `true` and `false`. + +### Sorting + +The sort statement consists of the literal string `sort`, the field name (nesting like above is supported) and the literal strings `asc` or `desc`. + +``` +sort::asc +sort::desc +sort:.:desc +``` + +### Struct fields + +Supported struct field values are: + +* `int`, `int8`, `int16`, `int32`, `int64` +* `float32`, `float64` +* `string` +* `bool` + +For nested structs: + +* `struct` + +### Filter logic + +Supported logical matching: + +* `AND` - all statements have to match +* `OR` - only one statement has to match +* `NOT` - none of the statements have to match + +## Examples ``` // Example data structure @@ -35,27 +90,6 @@ var data = Filter{ } ``` -### Struct fields - -Supported struct field values are: - -* `int`, `int8`, `int16`, `int32`, `int64` -* `float32`, `float64` -* `string` -* `bool` - -For nested structs: - -* `struct` - -### Filter logic - -Supported logical matching: - -* `AND` - all of the queries have to match -* `OR` - only one query has to match -* `NOT` - none of the queries have to match - ### Explicit match ``` @@ -91,35 +125,45 @@ filter.NewQuery("Struct.String:*2 Bool:true", filter.AND).Filter(&data) data = [{2str 42 true {s2 {2}}}] ``` +### Sorting + +The `GenericFilter` does not support sorting. + +#### Sort + +``` +filter.NewQuery("sort:Bool:desc", filter.AND).Filter(&data) + +data = [{str1 42 false {s1 {1}}} {2str 42 true {s2 {2}}} {str3 42 true {s3 {3}}}] +``` + +#### Sort and Filter + +``` +filter.NewQuery("sort:Bool:desc String:str*", filter.AND).Filter(&data) + +data = [{str1 42 false {s1 {1}}} {str3 42 true {s3 {3}}}] +``` + ## Benchmarks ``` -go test -cpu 1,2,4 --bench . -BenchmarkNewQuerySimple 3000000 401 ns/op 176 B/op 7 allocs/op -BenchmarkNewQuerySimple-2 5000000 390 ns/op 176 B/op 7 allocs/op -BenchmarkNewQuerySimple-4 2000000 792 ns/op 176 B/op 7 allocs/op -BenchmarkNewQueryComplex 1000000 1416 ns/op 656 B/op 17 allocs/op -BenchmarkNewQueryComplex-2 1000000 1384 ns/op 656 B/op 17 allocs/op -BenchmarkNewQueryComplex-4 1000000 2488 ns/op 656 B/op 17 allocs/op -BenchmarkNewQueryComplexOr 1000000 1192 ns/op 640 B/op 17 allocs/op -BenchmarkNewQueryComplexOr-2 1000000 1158 ns/op 640 B/op 17 allocs/op -BenchmarkNewQueryComplexOr-4 1000000 1963 ns/op 640 B/op 17 allocs/op -BenchmarkFilterSimple 2000000 907 ns/op 432 B/op 12 allocs/op -BenchmarkFilterSimple-2 2000000 899 ns/op 432 B/op 12 allocs/op -BenchmarkFilterSimple-4 1000000 1468 ns/op 432 B/op 12 allocs/op -BenchmarkFilterComplex 500000 3129 ns/op 672 B/op 48 allocs/op -BenchmarkFilterComplex-2 500000 3208 ns/op 672 B/op 48 allocs/op -BenchmarkFilterComplex-4 500000 6644 ns/op 672 B/op 48 allocs/op -BenchmarkFilterComplexOr 500000 2698 ns/op 816 B/op 39 allocs/op -BenchmarkFilterComplexOr-2 500000 2832 ns/op 816 B/op 39 allocs/op -BenchmarkFilterComplexOr-4 500000 5509 ns/op 816 B/op 39 allocs/op -BenchmarkGenericFilterSimple 500000 2386 ns/op 1200 B/op 24 allocs/op -BenchmarkGenericFilterSimple-2 1000000 2360 ns/op 1200 B/op 24 allocs/op -BenchmarkGenericFilterSimple-4 1000000 3656 ns/op 1200 B/op 24 allocs/op -BenchmarkGenericFilterComplex 200000 7178 ns/op 1680 B/op 96 allocs/op -BenchmarkGenericFilterComplex-2 200000 7102 ns/op 1680 B/op 96 allocs/op -BenchmarkGenericFilterComplex-4 200000 12602 ns/op 1680 B/op 96 allocs/op -BenchmarkGenericFilterComplexOr 200000 5454 ns/op 2456 B/op 61 allocs/op -BenchmarkGenericFilterComplexOr-2 300000 5444 ns/op 2456 B/op 61 allocs/op -BenchmarkGenericFilterComplexOr-4 300000 9970 ns/op 2456 B/op 61 allocs/op +go test -bench . +BenchmarkNewQuerySimple-4 2000000 974 ns/op 272 B/op 8 allocs/op +BenchmarkNewQueryComplex-4 500000 2779 ns/op 848 B/op 20 allocs/op +BenchmarkNewQueryComplexOr-4 500000 2677 ns/op 832 B/op 20 allocs/op +BenchmarkNewQuerySort-4 3000000 576 ns/op 160 B/op 4 allocs/op +BenchmarkNewQuerySortSimple-4 1000000 1192 ns/op 304 B/op 9 allocs/op +BenchmarkNewQuerySortComplex-4 500000 2598 ns/op 784 B/op 19 allocs/op +BenchmarkNewQuerySortComplexOr-4 500000 2326 ns/op 768 B/op 19 allocs/op +BenchmarkFilterSimple-4 1000000 2123 ns/op 432 B/op 12 allocs/op +BenchmarkFilterComplex-4 200000 6840 ns/op 672 B/op 42 allocs/op +BenchmarkFilterComplexOr-4 300000 5929 ns/op 744 B/op 30 allocs/op +BenchmarkSort-4 200000 7859 ns/op 1864 B/op 51 allocs/op +BenchmarkSortFilterSimple-4 200000 6294 ns/op 1432 B/op 39 allocs/op +BenchmarkSortFilterComplex-4 200000 11990 ns/op 1672 B/op 69 allocs/op +BenchmarkSortFilterComplexOr-4 200000 12138 ns/op 2176 B/op 69 allocs/op +BenchmarkGenericFilterSimple-4 300000 5939 ns/op 1200 B/op 24 allocs/op +BenchmarkGenericFilterComplex-4 100000 16638 ns/op 1680 B/op 84 allocs/op +BenchmarkGenericFilterComplexOr-4 200000 11846 ns/op 2360 B/op 49 allocs/op ``` diff --git a/cmd/main.go b/cmd/main.go index 572b144..be87247 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -29,14 +29,17 @@ type Filter []Example func (v Filter) Len() int { return len(v) } func (v Filter) Index(i int) interface{} { return v[i] } -func (v *Filter) Remove(i, j int) { *v = append((*v)[:i], (*v)[j:]...) } + +//func (v Filter) Remove(i, j int) { v = append(v[:i], v[j:]...) } + +func (v *Filter) Remove(i, j int) { *v = append((*v)[:i], (*v)[j:]...) } // Example data var data = Filter{ - {"str1", 42, true, Struct1{"s1", Struct2{1}}}, + {"str3", 42, true, Struct1{"s3", Struct2{3}}}, {"2str", 42, true, Struct1{"s2", Struct2{2}}}, - {"str3", 42, false, Struct1{"s3", Struct2{3}}}, - {":*", 42, false, Struct1{"s3", Struct2{3}}}, + {"str1", 42, false, Struct1{"s1", Struct2{1}}}, + {":*", 42, false, Struct1{"s4", Struct2{4}}}, } func main() { @@ -48,8 +51,22 @@ func main() { } } f, _ = filter.NewQuery("Struct.Struct.Int:1 Struct.Struct.Int:2 String::", filter.OR) - f.Filter(&data) + err := f.Filter(&data) + if err != nil { + fmt.Println(err) + } for _, v := range data { fmt.Println("Interface:", v) } + s, err := filter.NewQuery("sort:Bool:desc", filter.AND) + if err != nil { + fmt.Println(err) + } + err = s.Filter(&data) + if err != nil { + fmt.Println(err) + } + for _, v := range data { + fmt.Println("Sort:", v) + } } diff --git a/filter.go b/filter.go index e39b552..cddc35b 100644 --- a/filter.go +++ b/filter.go @@ -6,6 +6,7 @@ package filter import ( "errors" "reflect" + "sort" "strings" ) @@ -27,25 +28,39 @@ type match struct { bf func(bool) bool } +type sorter struct { + enabled bool + only bool + desc bool + fields []string +} + +const ( + desc = "desc" + asc = "asc" +) + // Logic is the filter logic type Logic int const ( - // AND all of the queries have to match + // AND all statements have to match AND Logic = iota - // OR only one query has to match + // OR only one statement has to match OR - // NOT none of the queries have to match + // NOT none of the statements have to match NOT ) // Query is the filter query type Query struct { + q string m []match l Logic + s sorter } -// NewQuery parses the filter query and match logic +// NewQuery parses the filter/sort query and match logic // Examples: // Explicit match: : // Wildcard match (use ** to match the '*' character): @@ -55,14 +70,21 @@ type Query struct { // Nested structs: // .: // Multiple matches: +// Multiple statemenmts are seperated by a single whitespace character: ' ' // : :* // Supported data types are: string, int, float, bool // Bool does not support the wildcard operator +// Sorting: +// The sort statement consists of the literal string 'sort', the field name (nesting like above is supported) and the literal strings 'asc' or 'desc'. +// Examples: +// sort::asc +// sort::desc +// sort:.:desc func NewQuery(s string, l Logic) (q *Query, err error) { if l > 2 || l < 0 { l = AND } - q = &Query{l: l} + q = &Query{q: s, l: l} if s == "" { err = errors.New("query is empty") return @@ -73,6 +95,28 @@ func NewQuery(s string, l Logic) (q *Query, err error) { if v == "" { continue } + if !q.s.enabled { + s := strings.SplitN(v, ":", 3) + if len(s) == 3 { + if s[0] == "sort" { + switch s[2] { + case asc, desc: + q.s.fields = strings.Split(s[1], ".") + if len(q.s.fields) > 0 { + if s[2] == desc { + q.s.desc = true + } + q.s.enabled = true + } + continue + default: + err = errors.New("invalid sort: " + v) + return + } + + } + } + } m := strings.SplitN(v, ":", 2) if len(m) != 2 { err = errors.New("invalid match: " + v) @@ -91,9 +135,19 @@ func NewQuery(s string, l Logic) (q *Query, err error) { boolFunc(m[1]), }) } + if len(q.m) == 0 { + q.s.only = true + } + if len(q.m) == 1 && q.l != NOT { + q.l = OR + } return } +func (q *Query) String() string { + return q.q +} + func (q *Query) check(oks []bool) (b bool) { b = true switch q.l { @@ -164,6 +218,7 @@ func (q *Query) match(v reflect.Value) (ok bool) { // GenericFilter takes any slice of structs value and returns the same slice type containing only the filter matches. // Returns nil if interface{} is not a slice. +// GenericFilter does not support sorting. func (q *Query) GenericFilter(i interface{}) interface{} { v := reflect.ValueOf(i) if v.Kind() != reflect.Slice { @@ -179,9 +234,29 @@ func (q *Query) GenericFilter(i interface{}) interface{} { return ret.Interface() } +func (q Query) sort(f Filter) { + if f.Len() == 0 { + return + } + l := makeLess(f, q.s.fields) + if q.s.desc { + l = reverse(l) + } + sort.SliceStable(reflect.Indirect(reflect.ValueOf(f)).Interface(), l) +} + // Filter takes a pointer to the Filter interface as an argument. -// The original slice is modified in the filtering process. -func (q Query) Filter(f Filter) { +// The original slice is modified in the filtering/sorting process. +func (q Query) Filter(f Filter) error { + v := reflect.ValueOf(f) + if v.Kind() != reflect.Ptr { + return errors.New("filter: non-pointer " + v.Type().String()) + } + + if q.s.enabled && q.s.only { + q.sort(f) + return nil + } count := 0 i := 0 for ; i < f.Len(); i++ { @@ -198,4 +273,8 @@ func (q Query) Filter(f Filter) { if count > 0 { f.Remove(i-count, i) } + if q.s.enabled { + q.sort(f) + } + return nil } diff --git a/filter_test.go b/filter_test.go index 0290bed..96dc2e1 100644 --- a/filter_test.go +++ b/filter_test.go @@ -64,6 +64,11 @@ func TestNewQuery(t *testing.T) { if err == nil { t.Error("invalid match") } + + _, err = NewQuery("String:1 sort:Bool:ASC", AND) + if err == nil { + t.Error("invalid sort") + } } const ( @@ -118,6 +123,61 @@ func TestFilter(t *testing.T) { } } +const ( + sort1 = "sort:Int:desc" + sort2 = "sort:Int:desc Bool:true" +) + +func TestSortFilter(t *testing.T) { + f1 := makefilter(benchData) + q1, err := NewQuery(sort1, AND) + if err != nil { + t.Error(err) + } + q1.Filter(&f1) + if len(f1) != 12 { + t.Error("f1 len is:", len(f1)) + } + if f1[0] != benchData[11] { + t.Error("f1[0] != benchData[11]") + } + if f1[11] != benchData[0] { + t.Error("f1[11] != benchData[0]") + } + + f2 := makefilter(benchData) + q2, err := NewQuery(sort2, OR) + if err != nil { + t.Error(err) + } + q2.Filter(&f2) + if len(f2) != 6 { + t.Error("f2 len is:", len(f2)) + } + if f2[0] != benchData[9] { + t.Error("f2[0] != benchData[9]") + } + if f2[5] != benchData[0] { + t.Error("f2[5] != benchData[0]") + } + + f3 := makefilter(benchData) + q3, err := NewQuery(sort2, NOT) + if err != nil { + t.Error(err) + } + q3.Filter(&f3) + if len(f3) != 6 { + t.Error("f3 len is:", len(f3)) + } + if f3[0] != benchData[11] { + t.Error("f3[0] != benchData[11]") + } + if f3[5] != benchData[2] { + t.Error("f3[5] != benchData[2]") + } +} + func TestGenericFilter(t *testing.T) { q1, err := NewQuery(filter1, AND) if err != nil { @@ -175,9 +235,13 @@ func TestGenericFilter(t *testing.T) { } const ( - simple = "Bool:true" - comp = "String:*st* Bool:false Struct.Struct.Int:1" - compOr = "Int:47 Bool:false Struct.String:s1*" + simple = "Bool:true" + comp = "String:*st* Bool:false Struct.Struct.Int:1" + compOr = "Int:47 Bool:false Struct.String:s1*" + sortOnly = "sort:Int:desc" + simpleSort = "sort:Int:desc Bool:true" + compSort = "sort:Int:desc String:*st* Bool:false Struct.Struct.Int:1" + compOrSort = "sort:Int:desc Int:47 Bool:false Struct.String:s1*" ) func BenchmarkNewQuerySimple(b *testing.B) { @@ -201,6 +265,34 @@ func BenchmarkNewQueryComplexOr(b *testing.B) { } } +func BenchmarkNewQuerySort(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + NewQuery(sortOnly, OR) + } +} + +func BenchmarkNewQuerySortSimple(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + NewQuery(simpleSort, OR) + } +} + +func BenchmarkNewQuerySortComplex(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + NewQuery(compSort, AND) + } +} + +func BenchmarkNewQuerySortComplexOr(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + NewQuery(compOrSort, OR) + } +} + func BenchmarkFilterSimple(b *testing.B) { b.ReportAllocs() q, err := NewQuery(simple, OR) @@ -237,6 +329,54 @@ func BenchmarkFilterComplexOr(b *testing.B) { } } +func BenchmarkSort(b *testing.B) { + b.ReportAllocs() + q, err := NewQuery(sortOnly, OR) + if err != nil { + b.Error(err) + } + fd := makefilter(benchData) + for i := 0; i < b.N; i++ { + q.Filter(&fd) + } +} + +func BenchmarkSortFilterSimple(b *testing.B) { + b.ReportAllocs() + q, err := NewQuery(simpleSort, OR) + if err != nil { + b.Error(err) + } + fd := makefilter(benchData) + for i := 0; i < b.N; i++ { + q.Filter(&fd) + } +} + +func BenchmarkSortFilterComplex(b *testing.B) { + b.ReportAllocs() + q, err := NewQuery(compSort, AND) + if err != nil { + b.Error(err) + } + fd := makefilter(benchData) + for i := 0; i < b.N; i++ { + q.Filter(&fd) + } +} + +func BenchmarkSortFilterComplexOr(b *testing.B) { + b.ReportAllocs() + q, err := NewQuery(compOrSort, OR) + if err != nil { + b.Error(err) + } + fd := makefilter(benchData) + for i := 0; i < b.N; i++ { + q.Filter(&fd) + } +} + func BenchmarkGenericFilterSimple(b *testing.B) { b.ReportAllocs() q, err := NewQuery(simple, OR) diff --git a/helper.go b/helper.go index 7acdcf3..7b8786f 100644 --- a/helper.go +++ b/helper.go @@ -12,6 +12,37 @@ const ( anys = string(any) ) +type less func(i, j int) bool + +func reverse(f less) less { + return func(i, j int) bool { + return f(j, i) + } +} + +func index(f Filter, i int, fields []string) (val reflect.Value) { + val = reflect.ValueOf(f.Index(i)).FieldByName(fields[0]) + for j := 1; j < len(fields); j++ { + val = val.FieldByName(fields[j]) + } + return +} + +func makeLess(f Filter, fields []string) less { + switch index(f, 0, fields).Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, + reflect.Int32, reflect.Int64: + return func(i, j int) bool { return index(f, i, fields).Int() < index(f, j, fields).Int() } + case reflect.Float32, reflect.Float64: + return func(i, j int) bool { return index(f, i, fields).Float() < index(f, j, fields).Float() } + case reflect.String: + return func(i, j int) bool { return index(f, i, fields).String() < index(f, j, fields).String() } + case reflect.Bool: + return func(i, _ int) bool { return index(f, i, fields).Bool() } + } + return nil +} + func getValue(v reflect.Value) (s string, b bool, err error) { switch v.Kind() { case reflect.Int, reflect.Int8, reflect.Int16,