ston1th/filter
1
0
Fork 0
Code Issues 1 Releases Activity

added sort support

Browse source
This commit is contained in:
ston1th 2017-08-30 22:01:19 +02:00
commit 1674abae73
5 changed files with 377 additions and 66 deletions
Download patch file Download diff file Expand all files Collapse all files

146
README.md
View file

@ -1,6 +1,61 @@
# filter - a simple struct field filter for go
# filter - a simple struct field filter and sorter for go
## Usage
## Query Syntax
### Matching
Explicit match: `<struct field>:<struct value>`
Wildcard match (use ** to match the '*' character):
```
<struct field>:*<right part of struct value>
<struct field>:<left part of struct value>*
<struct field>:*<middl of struct value>*
```
Nested structs: `<struct field>.<struct field>:<struct value>`
Multiple matches (seperated by a single whitespace):
```
<struct field>:<struct value> <struct field>:<struct value>*
```
Bool does not support the wildcard operator, only the literal strings `true` and `false`.
### Sorting
The sort statement consists of the literal string `sort`, the field name (nesting like above is supported) and the literal strings `asc` or `desc`.
```
sort:<struct field>:asc
sort:<struct field>:desc
sort:<struct field>.<struct field>:desc
```
### Struct fields
Supported struct field values are:
* `int`, `int8`, `int16`, `int32`, `int64`
* `float32`, `float64`
* `string`
* `bool`
For nested structs:
* `struct`
### Filter logic
Supported logical matching:
* `AND` - all statements have to match
* `OR` - only one statement has to match
* `NOT` - none of the statements have to match
## Examples
```
// Example data structure
@ -35,27 +90,6 @@ var data = Filter{
}
```
### Struct fields
Supported struct field values are:
* `int`, `int8`, `int16`, `int32`, `int64`
* `float32`, `float64`
* `string`
* `bool`
For nested structs:
* `struct`
### Filter logic
Supported logical matching:
* `AND` - all of the queries have to match
* `OR` - only one query has to match
* `NOT` - none of the queries have to match
### Explicit match
```
@ -91,35 +125,45 @@ filter.NewQuery("Struct.String:*2 Bool:true", filter.AND).Filter(&data)
data = [{2str 42 true {s2 {2}}}]
```
### Sorting
The `GenericFilter` does not support sorting.
#### Sort
```
filter.NewQuery("sort:Bool:desc", filter.AND).Filter(&data)
data = [{str1 42 false {s1 {1}}} {2str 42 true {s2 {2}}} {str3 42 true {s3 {3}}}]
```
#### Sort and Filter
```
filter.NewQuery("sort:Bool:desc String:str*", filter.AND).Filter(&data)
data = [{str1 42 false {s1 {1}}} {str3 42 true {s3 {3}}}]
```
## Benchmarks
```
go test -cpu 1,2,4 --bench .
BenchmarkNewQuerySimple 3000000 401 ns/op 176 B/op 7 allocs/op
BenchmarkNewQuerySimple-2 5000000 390 ns/op 176 B/op 7 allocs/op
BenchmarkNewQuerySimple-4 2000000 792 ns/op 176 B/op 7 allocs/op
BenchmarkNewQueryComplex 1000000 1416 ns/op 656 B/op 17 allocs/op
BenchmarkNewQueryComplex-2 1000000 1384 ns/op 656 B/op 17 allocs/op
BenchmarkNewQueryComplex-4 1000000 2488 ns/op 656 B/op 17 allocs/op
BenchmarkNewQueryComplexOr 1000000 1192 ns/op 640 B/op 17 allocs/op
BenchmarkNewQueryComplexOr-2 1000000 1158 ns/op 640 B/op 17 allocs/op
BenchmarkNewQueryComplexOr-4 1000000 1963 ns/op 640 B/op 17 allocs/op
BenchmarkFilterSimple 2000000 907 ns/op 432 B/op 12 allocs/op
BenchmarkFilterSimple-2 2000000 899 ns/op 432 B/op 12 allocs/op
BenchmarkFilterSimple-4 1000000 1468 ns/op 432 B/op 12 allocs/op
BenchmarkFilterComplex 500000 3129 ns/op 672 B/op 48 allocs/op
BenchmarkFilterComplex-2 500000 3208 ns/op 672 B/op 48 allocs/op
BenchmarkFilterComplex-4 500000 6644 ns/op 672 B/op 48 allocs/op
BenchmarkFilterComplexOr 500000 2698 ns/op 816 B/op 39 allocs/op
BenchmarkFilterComplexOr-2 500000 2832 ns/op 816 B/op 39 allocs/op
BenchmarkFilterComplexOr-4 500000 5509 ns/op 816 B/op 39 allocs/op
BenchmarkGenericFilterSimple 500000 2386 ns/op 1200 B/op 24 allocs/op
BenchmarkGenericFilterSimple-2 1000000 2360 ns/op 1200 B/op 24 allocs/op
BenchmarkGenericFilterSimple-4 1000000 3656 ns/op 1200 B/op 24 allocs/op
BenchmarkGenericFilterComplex 200000 7178 ns/op 1680 B/op 96 allocs/op
BenchmarkGenericFilterComplex-2 200000 7102 ns/op 1680 B/op 96 allocs/op
BenchmarkGenericFilterComplex-4 200000 12602 ns/op 1680 B/op 96 allocs/op
BenchmarkGenericFilterComplexOr 200000 5454 ns/op 2456 B/op 61 allocs/op
BenchmarkGenericFilterComplexOr-2 300000 5444 ns/op 2456 B/op 61 allocs/op
BenchmarkGenericFilterComplexOr-4 300000 9970 ns/op 2456 B/op 61 allocs/op
go test -bench .
BenchmarkNewQuerySimple-4 2000000 974 ns/op 272 B/op 8 allocs/op
BenchmarkNewQueryComplex-4 500000 2779 ns/op 848 B/op 20 allocs/op
BenchmarkNewQueryComplexOr-4 500000 2677 ns/op 832 B/op 20 allocs/op
BenchmarkNewQuerySort-4 3000000 576 ns/op 160 B/op 4 allocs/op
BenchmarkNewQuerySortSimple-4 1000000 1192 ns/op 304 B/op 9 allocs/op
BenchmarkNewQuerySortComplex-4 500000 2598 ns/op 784 B/op 19 allocs/op
BenchmarkNewQuerySortComplexOr-4 500000 2326 ns/op 768 B/op 19 allocs/op
BenchmarkFilterSimple-4 1000000 2123 ns/op 432 B/op 12 allocs/op
BenchmarkFilterComplex-4 200000 6840 ns/op 672 B/op 42 allocs/op
BenchmarkFilterComplexOr-4 300000 5929 ns/op 744 B/op 30 allocs/op
BenchmarkSort-4 200000 7859 ns/op 1864 B/op 51 allocs/op
BenchmarkSortFilterSimple-4 200000 6294 ns/op 1432 B/op 39 allocs/op
BenchmarkSortFilterComplex-4 200000 11990 ns/op 1672 B/op 69 allocs/op
BenchmarkSortFilterComplexOr-4 200000 12138 ns/op 2176 B/op 69 allocs/op
BenchmarkGenericFilterSimple-4 300000 5939 ns/op 1200 B/op 24 allocs/op
BenchmarkGenericFilterComplex-4 100000 16638 ns/op 1680 B/op 84 allocs/op
BenchmarkGenericFilterComplexOr-4 200000 11846 ns/op 2360 B/op 49 allocs/op
```

27
cmd/main.go
View file

@ -29,14 +29,17 @@ type Filter []Example
func (v Filter) Len() int { return len(v) }
func (v Filter) Index(i int) interface{} { return v[i] }
func (v *Filter) Remove(i, j int) { *v = append((*v)[:i], (*v)[j:]...) }
//func (v Filter) Remove(i, j int) { v = append(v[:i], v[j:]...) }
func (v *Filter) Remove(i, j int) { *v = append((*v)[:i], (*v)[j:]...) }
// Example data
var data = Filter{
{"str1", 42, true, Struct1{"s1", Struct2{1}}},
{"str3", 42, true, Struct1{"s3", Struct2{3}}},
{"2str", 42, true, Struct1{"s2", Struct2{2}}},
{"str3", 42, false, Struct1{"s3", Struct2{3}}},
{":*", 42, false, Struct1{"s3", Struct2{3}}},
{"str1", 42, false, Struct1{"s1", Struct2{1}}},
{":*", 42, false, Struct1{"s4", Struct2{4}}},
}
func main() {
@ -48,8 +51,22 @@ func main() {
}
}
f, _ = filter.NewQuery("Struct.Struct.Int:1 Struct.Struct.Int:2 String::", filter.OR)
f.Filter(&data)
err := f.Filter(&data)
if err != nil {
fmt.Println(err)
}
for _, v := range data {
fmt.Println("Interface:", v)
}
s, err := filter.NewQuery("sort:Bool:desc", filter.AND)
if err != nil {
fmt.Println(err)
}
err = s.Filter(&data)
if err != nil {
fmt.Println(err)
}
for _, v := range data {
fmt.Println("Sort:", v)
}
}

93
filter.go
View file

@ -6,6 +6,7 @@ package filter
import (
"errors"
"reflect"
"sort"
"strings"
)
@ -27,25 +28,39 @@ type match struct {
bf func(bool) bool
}
type sorter struct {
enabled bool
only bool
desc bool
fields []string
}
const (
desc = "desc"
asc = "asc"
)
// Logic is the filter logic
type Logic int
const (
// AND all of the queries have to match
// AND all statements have to match
AND Logic = iota
// OR only one query has to match
// OR only one statement has to match
OR
// NOT none of the queries have to match
// NOT none of the statements have to match
NOT
)
// Query is the filter query
type Query struct {
q string
m []match
l Logic
s sorter
}
// NewQuery parses the filter query and match logic
// NewQuery parses the filter/sort query and match logic
// Examples:
// Explicit match: <struct field>:<struct value>
// Wildcard match (use ** to match the '*' character):
@ -55,14 +70,21 @@ type Query struct {
// Nested structs:
// <struct field>.<struct field>:<struct value>
// Multiple matches:
// Multiple statemenmts are seperated by a single whitespace character: ' '
// <struct field>:<struct value> <struct field>:<struct value>*
// Supported data types are: string, int, float, bool
// Bool does not support the wildcard operator
// Sorting:
// The sort statement consists of the literal string 'sort', the field name (nesting like above is supported) and the literal strings 'asc' or 'desc'.
// Examples:
// sort:<struct field>:asc
// sort:<struct field>:desc
// sort:<struct field>.<struct field>:desc
func NewQuery(s string, l Logic) (q *Query, err error) {
if l > 2 || l < 0 {
l = AND
}
q = &Query{l: l}
q = &Query{q: s, l: l}
if s == "" {
err = errors.New("query is empty")
return
@ -73,6 +95,28 @@ func NewQuery(s string, l Logic) (q *Query, err error) {
if v == "" {
continue
}
if !q.s.enabled {
s := strings.SplitN(v, ":", 3)
if len(s) == 3 {
if s[0] == "sort" {
switch s[2] {
case asc, desc:
q.s.fields = strings.Split(s[1], ".")
if len(q.s.fields) > 0 {
if s[2] == desc {
q.s.desc = true
}
q.s.enabled = true
}
continue
default:
err = errors.New("invalid sort: " + v)
return
}
}
}
}
m := strings.SplitN(v, ":", 2)
if len(m) != 2 {
err = errors.New("invalid match: " + v)
@ -91,9 +135,19 @@ func NewQuery(s string, l Logic) (q *Query, err error) {
boolFunc(m[1]),
})
}
if len(q.m) == 0 {
q.s.only = true
}
if len(q.m) == 1 && q.l != NOT {
q.l = OR
}
return
}
func (q *Query) String() string {
return q.q
}
func (q *Query) check(oks []bool) (b bool) {
b = true
switch q.l {
@ -164,6 +218,7 @@ func (q *Query) match(v reflect.Value) (ok bool) {
// GenericFilter takes any slice of structs value and returns the same slice type containing only the filter matches.
// Returns nil if interface{} is not a slice.
// GenericFilter does not support sorting.
func (q *Query) GenericFilter(i interface{}) interface{} {
v := reflect.ValueOf(i)
if v.Kind() != reflect.Slice {
@ -179,9 +234,29 @@ func (q *Query) GenericFilter(i interface{}) interface{} {
return ret.Interface()
}
func (q Query) sort(f Filter) {
if f.Len() == 0 {
return
}
l := makeLess(f, q.s.fields)
if q.s.desc {
l = reverse(l)
}
sort.SliceStable(reflect.Indirect(reflect.ValueOf(f)).Interface(), l)
}
// Filter takes a pointer to the Filter interface as an argument.
// The original slice is modified in the filtering process.
func (q Query) Filter(f Filter) {
// The original slice is modified in the filtering/sorting process.
func (q Query) Filter(f Filter) error {
v := reflect.ValueOf(f)
if v.Kind() != reflect.Ptr {
return errors.New("filter: non-pointer " + v.Type().String())
}
if q.s.enabled && q.s.only {
q.sort(f)
return nil
}
count := 0
i := 0
for ; i < f.Len(); i++ {
@ -198,4 +273,8 @@ func (q Query) Filter(f Filter) {
if count > 0 {
f.Remove(i-count, i)
}
if q.s.enabled {
q.sort(f)
}
return nil
}

146
filter_test.go
View file

@ -64,6 +64,11 @@ func TestNewQuery(t *testing.T) {
if err == nil {
t.Error("invalid match")
}
_, err = NewQuery("String:1 sort:Bool:ASC", AND)
if err == nil {
t.Error("invalid sort")
}
}
const (
@ -118,6 +123,61 @@ func TestFilter(t *testing.T) {
}
}
const (
sort1 = "sort:Int:desc"
sort2 = "sort:Int:desc Bool:true"
)
func TestSortFilter(t *testing.T) {
f1 := makefilter(benchData)
q1, err := NewQuery(sort1, AND)
if err != nil {
t.Error(err)
}
q1.Filter(&f1)
if len(f1) != 12 {
t.Error("f1 len is:", len(f1))
}
if f1[0] != benchData[11] {
t.Error("f1[0] != benchData[11]")
}
if f1[11] != benchData[0] {
t.Error("f1[11] != benchData[0]")
}
f2 := makefilter(benchData)
q2, err := NewQuery(sort2, OR)
if err != nil {
t.Error(err)
}
q2.Filter(&f2)
if len(f2) != 6 {
t.Error("f2 len is:", len(f2))
}
if f2[0] != benchData[9] {
t.Error("f2[0] != benchData[9]")
}
if f2[5] != benchData[0] {
t.Error("f2[5] != benchData[0]")
}
f3 := makefilter(benchData)
q3, err := NewQuery(sort2, NOT)
if err != nil {
t.Error(err)
}
q3.Filter(&f3)
if len(f3) != 6 {
t.Error("f3 len is:", len(f3))
}
if f3[0] != benchData[11] {
t.Error("f3[0] != benchData[11]")
}
if f3[5] != benchData[2] {
t.Error("f3[5] != benchData[2]")
}
}
func TestGenericFilter(t *testing.T) {
q1, err := NewQuery(filter1, AND)
if err != nil {
@ -175,9 +235,13 @@ func TestGenericFilter(t *testing.T) {
}
const (
simple = "Bool:true"
comp = "String:*st* Bool:false Struct.Struct.Int:1"
compOr = "Int:47 Bool:false Struct.String:s1*"
simple = "Bool:true"
comp = "String:*st* Bool:false Struct.Struct.Int:1"
compOr = "Int:47 Bool:false Struct.String:s1*"
sortOnly = "sort:Int:desc"
simpleSort = "sort:Int:desc Bool:true"
compSort = "sort:Int:desc String:*st* Bool:false Struct.Struct.Int:1"
compOrSort = "sort:Int:desc Int:47 Bool:false Struct.String:s1*"
)
func BenchmarkNewQuerySimple(b *testing.B) {
@ -201,6 +265,34 @@ func BenchmarkNewQueryComplexOr(b *testing.B) {
}
}
func BenchmarkNewQuerySort(b *testing.B) {
b.ReportAllocs()
for i := 0; i < b.N; i++ {
NewQuery(sortOnly, OR)
}
}
func BenchmarkNewQuerySortSimple(b *testing.B) {
b.ReportAllocs()
for i := 0; i < b.N; i++ {
NewQuery(simpleSort, OR)
}
}
func BenchmarkNewQuerySortComplex(b *testing.B) {
b.ReportAllocs()
for i := 0; i < b.N; i++ {
NewQuery(compSort, AND)
}
}
func BenchmarkNewQuerySortComplexOr(b *testing.B) {
b.ReportAllocs()
for i := 0; i < b.N; i++ {
NewQuery(compOrSort, OR)
}
}
func BenchmarkFilterSimple(b *testing.B) {
b.ReportAllocs()
q, err := NewQuery(simple, OR)
@ -237,6 +329,54 @@ func BenchmarkFilterComplexOr(b *testing.B) {
}
}
func BenchmarkSort(b *testing.B) {
b.ReportAllocs()
q, err := NewQuery(sortOnly, OR)
if err != nil {
b.Error(err)
}
fd := makefilter(benchData)
for i := 0; i < b.N; i++ {
q.Filter(&fd)
}
}
func BenchmarkSortFilterSimple(b *testing.B) {
b.ReportAllocs()
q, err := NewQuery(simpleSort, OR)
if err != nil {
b.Error(err)
}
fd := makefilter(benchData)
for i := 0; i < b.N; i++ {
q.Filter(&fd)
}
}
func BenchmarkSortFilterComplex(b *testing.B) {
b.ReportAllocs()
q, err := NewQuery(compSort, AND)
if err != nil {
b.Error(err)
}
fd := makefilter(benchData)
for i := 0; i < b.N; i++ {
q.Filter(&fd)
}
}
func BenchmarkSortFilterComplexOr(b *testing.B) {
b.ReportAllocs()
q, err := NewQuery(compOrSort, OR)
if err != nil {
b.Error(err)
}
fd := makefilter(benchData)
for i := 0; i < b.N; i++ {
q.Filter(&fd)
}
}
func BenchmarkGenericFilterSimple(b *testing.B) {
b.ReportAllocs()
q, err := NewQuery(simple, OR)

31
helper.go
View file

@ -12,6 +12,37 @@ const (
anys = string(any)
)
type less func(i, j int) bool
func reverse(f less) less {
return func(i, j int) bool {
return f(j, i)
}
}
func index(f Filter, i int, fields []string) (val reflect.Value) {
val = reflect.ValueOf(f.Index(i)).FieldByName(fields[0])
for j := 1; j < len(fields); j++ {
val = val.FieldByName(fields[j])
}
return
}
func makeLess(f Filter, fields []string) less {
switch index(f, 0, fields).Kind() {
case reflect.Int, reflect.Int8, reflect.Int16,
reflect.Int32, reflect.Int64:
return func(i, j int) bool { return index(f, i, fields).Int() < index(f, j, fields).Int() }
case reflect.Float32, reflect.Float64:
return func(i, j int) bool { return index(f, i, fields).Float() < index(f, j, fields).Float() }
case reflect.String:
return func(i, j int) bool { return index(f, i, fields).String() < index(f, j, fields).String() }
case reflect.Bool:
return func(i, _ int) bool { return index(f, i, fields).Bool() }
}
return nil
}
func getValue(v reflect.Value) (s string, b bool, err error) {
switch v.Kind() {
case reflect.Int, reflect.Int8, reflect.Int16,