diff --git a/pkg/server/templates/userTotp.html b/templates/userTotp.html
similarity index 100%
rename from pkg/server/templates/userTotp.html
rename to templates/userTotp.html
diff --git a/vendor/git.giftfish.de/ston1th/authdav/LICENSE b/vendor/git.giftfish.de/ston1th/authdav/LICENSE
deleted file mode 100644
index 45785b1..0000000
--- a/vendor/git.giftfish.de/ston1th/authdav/LICENSE
+++ /dev/null
@@ -1,24 +0,0 @@
-Copyright (C) 2019 Marius Schellenberger
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
- * The names of the authors and/or contributors may not be used to
- endorse or promote products derived from this software without
- specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL ston1th BE LIABLE FOR ANY
-DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/vendor/git.giftfish.de/ston1th/authdav/README.md b/vendor/git.giftfish.de/ston1th/authdav/README.md
deleted file mode 100644
index aca18a0..0000000
--- a/vendor/git.giftfish.de/ston1th/authdav/README.md
+++ /dev/null
@@ -1,59 +0,0 @@
-# authdav - a x/net/webdav wrapper
-
-Wrappers:
-* HTTP Basic Auth wrapper for `webdav.Handler`
-* write only once FileSystem wrapper for `webdav.FileSystem`
-
-## Usage
-
-### HTTP Basic Auth
-
-```
-package main
-
-import (
- "git.giftfish.de/ston1th/authdav"
- "golang.org/x/net/webdav"
- "log"
- "net/http"
-)
-
-type exampleAuthenticator struct {}
-
-func (exampleAuthenticator) BasicAuth(user, password string) bool {
- if user == "user" && password == "password" {
- return true
- }
- return false
-}
-
-func main() {
- auth := exampleAuthenticator{}
- handler := authdav.NewWebdavBasicAuth("", webdav.Dir("."), nil, nil, auth, "webdav with basic auth")
- log.Fatal(http.ListenAndServe(":8080", handler))
-}
-```
-
-### Write Only
-
-```
-package main
-
-import (
- "git.giftfish.de/ston1th/authdav"
- "golang.org/x/net/webdav"
- "log"
- "net/http"
-)
-
-func main() {
- fs := authdav.NewWriteOnlyOnceFileSystem(webdav.Dir("."))
- handler := &webdav.Handler{
- Prefix: "",
- FileSystem: fs,
- LockSystem: webdav.NewMemLS(),
- Logger: nil,
- }
- log.Fatal(http.ListenAndServe(":8080", handler))
-}
-```
diff --git a/vendor/git.giftfish.de/ston1th/authdav/authdav.go b/vendor/git.giftfish.de/ston1th/authdav/authdav.go
deleted file mode 100644
index 268bc4a..0000000
--- a/vendor/git.giftfish.de/ston1th/authdav/authdav.go
+++ /dev/null
@@ -1,57 +0,0 @@
-// Copyright (C) 2019 Marius Schellenberger
-
-package authdav
-
-import (
- "golang.org/x/net/webdav"
- "net/http"
-)
-
-// DefaultRealm is the HTTP Basic Auth default realm
-const DefaultRealm = "WebDav"
-
-// Authenticator is the interface for Webdav HTTP Basic Auth
-type Authenticator interface {
- BasicAuth(username string, password string) bool
-}
-
-// WebdavBasicAuth is a WebDav wrapper for HTTP Basic Auth and implements the http.Handler interface
-type WebdavBasicAuth struct {
- dav *webdav.Handler
- auth Authenticator
- realm string
-}
-
-// NewWebdavBasicAuth returns a new WebdavBasicAuth wrapper
-// prefix can be empty
-// ls and log can be nil
-// an empty authenticator disables HTTP Basic Auth
-// realm can be empty and defaults to authdav.DefaultRealm
-func NewWebdavBasicAuth(prefix string, fs webdav.FileSystem, ls webdav.LockSystem, log func(*http.Request, error), auth Authenticator, realm string) *WebdavBasicAuth {
- if ls == nil {
- ls = webdav.NewMemLS()
- }
- if realm == "" {
- realm = DefaultRealm
- }
- dav := &webdav.Handler{
- Prefix: prefix,
- FileSystem: fs,
- LockSystem: ls,
- Logger: log,
- }
- return &WebdavBasicAuth{dav, auth, `Basic realm="` + realm + `"`}
-}
-
-func (h *WebdavBasicAuth) ServeHTTP(w http.ResponseWriter, r *http.Request) {
- if h.auth != nil {
- u, p, ok := r.BasicAuth()
- if !ok || !h.auth.BasicAuth(u, p) {
- w.Header().Set("WWW-Authenticate", h.realm)
- w.WriteHeader(http.StatusUnauthorized)
- w.Write([]byte("Unauthorised\n"))
- return
- }
- }
- h.dav.ServeHTTP(w, r)
-}
diff --git a/vendor/git.giftfish.de/ston1th/authdav/file.go b/vendor/git.giftfish.de/ston1th/authdav/file.go
deleted file mode 100644
index 75e2364..0000000
--- a/vendor/git.giftfish.de/ston1th/authdav/file.go
+++ /dev/null
@@ -1,46 +0,0 @@
-// Copyright (C) 2019 Marius Schellenberger
-
-package authdav
-
-import (
- "golang.org/x/net/webdav"
- "io"
- "os"
-)
-
-// File is a wrapper for webdav.File providing write only functionality
-type File struct {
- f webdav.File
-}
-
-func (f *File) Close() error {
- return f.f.Close()
-}
-
-func (f *File) Read(p []byte) (int, error) {
- return 0, io.EOF
-}
-
-func (f *File) Seek(offset int64, whence int) (int64, error) {
- return 0, nil
-}
-
-func (f *File) Readdir(count int) (fi []os.FileInfo, err error) {
- ffi, err := f.f.Readdir(count)
- for _, f := range ffi {
- fi = append(fi, &FileInfo{f})
- }
- return
-}
-
-func (f *File) Stat() (os.FileInfo, error) {
- fi, err := f.f.Stat()
- if err != nil {
- return nil, err
- }
- return &FileInfo{fi}, nil
-}
-
-func (f *File) Write(p []byte) (int, error) {
- return f.f.Write(p)
-}
diff --git a/vendor/git.giftfish.de/ston1th/authdav/fileinfo.go b/vendor/git.giftfish.de/ston1th/authdav/fileinfo.go
deleted file mode 100644
index 3b4589f..0000000
--- a/vendor/git.giftfish.de/ston1th/authdav/fileinfo.go
+++ /dev/null
@@ -1,44 +0,0 @@
-// Copyright (C) 2019 Marius Schellenberger
-
-package authdav
-
-import (
- "os"
- "time"
-)
-
-// FileInfo is a wrapper for os.FileInfo which always raturns a zero length for regular files
-type FileInfo struct {
- fi os.FileInfo
-}
-
-func (fi *FileInfo) Name() string {
- return fi.fi.Name()
-}
-
-func (fi *FileInfo) RealSize() int64 {
- return fi.fi.Size()
-}
-
-func (fi *FileInfo) Size() int64 {
- if fi.Mode().IsRegular() {
- return 0
- }
- return fi.fi.Size()
-}
-
-func (fi *FileInfo) Mode() os.FileMode {
- return fi.fi.Mode()
-}
-
-func (fi *FileInfo) ModTime() time.Time {
- return fi.fi.ModTime()
-}
-
-func (fi *FileInfo) IsDir() bool {
- return fi.fi.IsDir()
-}
-
-func (fi *FileInfo) Sys() interface{} {
- return fi.fi.Sys()
-}
diff --git a/vendor/git.giftfish.de/ston1th/authdav/filesystem.go b/vendor/git.giftfish.de/ston1th/authdav/filesystem.go
deleted file mode 100644
index 447b7ef..0000000
--- a/vendor/git.giftfish.de/ston1th/authdav/filesystem.go
+++ /dev/null
@@ -1,79 +0,0 @@
-// Copyright (C) 2019 Marius Schellenberger
-
-package authdav
-
-import (
- "context"
- "errors"
- "golang.org/x/net/webdav"
- "os"
-)
-
-// ErrWonly indicates forbidden operation
-var ErrWonly = errors.New("authdav: only file write once operations permitted")
-
-const writeFlag = os.O_RDWR | os.O_CREATE | os.O_TRUNC
-
-// WriteOnlyOnceFileSystem is a file write only wrapper for a webdav.FileSystem
-type WriteOnlyOnceFileSystem struct {
- Filters []Filter
- fs webdav.FileSystem
-}
-
-// NewWriteOnlyOnceFileSystem returns a new WriteOnlyOnceFileSystem wrapping a webdav.FileSystem
-func NewWriteOnlyOnceFileSystem(fs webdav.FileSystem) *WriteOnlyOnceFileSystem {
- return &WriteOnlyOnceFileSystem{fs: fs}
-}
-
-// Mkdir is not permitted
-func (fs *WriteOnlyOnceFileSystem) Mkdir(ctx context.Context, name string, perm os.FileMode) error {
- return ErrWonly
-}
-
-// OpenFile allows write only once operations
-func (fs *WriteOnlyOnceFileSystem) OpenFile(ctx context.Context, name string, flag int, perm os.FileMode) (webdav.File, error) {
- fi, err := fs.Stat(ctx, name)
- if err == nil && fi.Mode().IsRegular() {
- f, ok := fi.(*FileInfo)
- if ok && flag == writeFlag && f.RealSize() > 0 {
- return nil, ErrWonly
- }
- }
- for _, filter := range fs.Filters {
- err = filter.Filter(name)
- if err != nil {
- return nil, err
- }
- }
- f, err := fs.fs.OpenFile(ctx, name, flag, perm)
- if err != nil {
- return nil, err
- }
- return &File{f}, nil
-}
-
-// RemoveAll only permits deleting empty files
-func (fs *WriteOnlyOnceFileSystem) RemoveAll(ctx context.Context, name string) error {
- fi, err := fs.fs.Stat(ctx, name)
- if err != nil {
- return err
- }
- if fi.Mode().IsRegular() && fi.Size() == 0 {
- return fs.fs.RemoveAll(ctx, name)
- }
- return ErrWonly
-}
-
-// Rename is not permitted
-func (fs *WriteOnlyOnceFileSystem) Rename(ctx context.Context, oldName, newName string) error {
- return ErrWonly
-}
-
-// Stat returns os.FileInfo wrapped in FileInfo
-func (fs *WriteOnlyOnceFileSystem) Stat(ctx context.Context, name string) (os.FileInfo, error) {
- fi, err := fs.fs.Stat(ctx, name)
- if err != nil {
- return nil, err
- }
- return &FileInfo{fi}, nil
-}
diff --git a/vendor/git.giftfish.de/ston1th/authdav/filter.go b/vendor/git.giftfish.de/ston1th/authdav/filter.go
deleted file mode 100644
index 8b039a6..0000000
--- a/vendor/git.giftfish.de/ston1th/authdav/filter.go
+++ /dev/null
@@ -1,50 +0,0 @@
-// Copyright (C) 2019 Marius Schellenberger
-
-package authdav
-
-import (
- "errors"
- "path/filepath"
- "strings"
-)
-
-// Matcher is the matcher function type
-type Matcher func(path string) bool
-
-// Filter defines an interface to filter paths or files
-type Filter interface {
- Filter(path string) error
-}
-
-// ErrMacOSFilter error returned by the MacOSFilter
-var ErrMacOSFilter = errors.New("authdav: MacOS filter: access denied")
-
-// MacOSFilter implements a filter for MacOS specific files
-type MacOSFilter struct {
- match []Matcher
-}
-
-// NewMacOSFilter returns a new MacOSFilter
-func NewMacOSFilter() *MacOSFilter {
- return &MacOSFilter{
- match: []Matcher{
- func(p string) bool {
- return p == ".DS_Store"
- },
- func(p string) bool {
- return strings.HasPrefix(p, "._")
- },
- },
- }
-}
-
-// Filter implements the Filter interface
-func (f *MacOSFilter) Filter(path string) error {
- _, file := filepath.Split(path)
- for _, m := range f.match {
- if m(file) {
- return ErrMacOSFilter
- }
- }
- return nil
-}
diff --git a/vendor/git.giftfish.de/ston1th/godrop/v2/LICENSE b/vendor/git.giftfish.de/ston1th/godrop/v2/LICENSE
index aa39852..2bb060c 100644
--- a/vendor/git.giftfish.de/ston1th/godrop/v2/LICENSE
+++ b/vendor/git.giftfish.de/ston1th/godrop/v2/LICENSE
@@ -1,4 +1,4 @@
-Copyright (C) 2022 Marius Schellenberger
+Copyright (C) 2017 Marius Schellenberger
All rights reserved.
Redistribution and use in source and binary forms, with or without
diff --git a/vendor/git.giftfish.de/ston1th/godrop/v2/go.mod b/vendor/git.giftfish.de/ston1th/godrop/v2/go.mod
new file mode 100644
index 0000000..a41c9af
--- /dev/null
+++ b/vendor/git.giftfish.de/ston1th/godrop/v2/go.mod
@@ -0,0 +1,3 @@
+module git.giftfish.de/ston1th/godrop/v2
+
+require golang.org/x/sys v0.0.0-20181026144532-2772b66316d2 // indirect
diff --git a/vendor/git.giftfish.de/ston1th/godrop/v2/go.sum b/vendor/git.giftfish.de/ston1th/godrop/v2/go.sum
new file mode 100644
index 0000000..36d0abd
--- /dev/null
+++ b/vendor/git.giftfish.de/ston1th/godrop/v2/go.sum
@@ -0,0 +1,2 @@
+golang.org/x/sys v0.0.0-20181026144532-2772b66316d2 h1:W7CqTdBJ1CmxLKe7LptKDnBYV6PHrVLiGnoyBjaG/JQ=
+golang.org/x/sys v0.0.0-20181026144532-2772b66316d2/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
diff --git a/vendor/git.giftfish.de/ston1th/godrop/v2/godrop.go b/vendor/git.giftfish.de/ston1th/godrop/v2/godrop.go
index 37c7e75..55f39ec 100644
--- a/vendor/git.giftfish.de/ston1th/godrop/v2/godrop.go
+++ b/vendor/git.giftfish.de/ston1th/godrop/v2/godrop.go
@@ -1,6 +1,5 @@
-// Copyright (C) 2022 Marius Schellenberger
+// Copyright (C) 2018 Marius Schellenberger
-//go:build go1.11
// +build go1.11
// Package godrop provides a simple library to drop privileges on Linux and OpenBSD.
diff --git a/vendor/git.giftfish.de/ston1th/godrop/v2/lookup.go b/vendor/git.giftfish.de/ston1th/godrop/v2/lookup.go
index 725dc82..2d0e1a9 100644
--- a/vendor/git.giftfish.de/ston1th/godrop/v2/lookup.go
+++ b/vendor/git.giftfish.de/ston1th/godrop/v2/lookup.go
@@ -1,6 +1,5 @@
-// Copyright (C) 2022 Marius Schellenberger
+// Copyright (C) 2018 Marius Schellenberger
-//go:build go1.11
// +build go1.11
package godrop
diff --git a/vendor/git.giftfish.de/ston1th/godrop/v2/pledge.go b/vendor/git.giftfish.de/ston1th/godrop/v2/pledge.go
index 8c97aba..d0e361e 100644
--- a/vendor/git.giftfish.de/ston1th/godrop/v2/pledge.go
+++ b/vendor/git.giftfish.de/ston1th/godrop/v2/pledge.go
@@ -1,6 +1,5 @@
-// Copyright (C) 2022 Marius Schellenberger
+// Copyright (C) 2018 Marius Schellenberger
-//go:build !openbsd
// +build !openbsd
package godrop
diff --git a/vendor/git.giftfish.de/ston1th/godrop/v2/pledge_openbsd.go b/vendor/git.giftfish.de/ston1th/godrop/v2/pledge_openbsd.go
index 0af656a..7369b03 100644
--- a/vendor/git.giftfish.de/ston1th/godrop/v2/pledge_openbsd.go
+++ b/vendor/git.giftfish.de/ston1th/godrop/v2/pledge_openbsd.go
@@ -1,6 +1,5 @@
-// Copyright (C) 2022 Marius Schellenberger
+// Copyright (C) 2018 Marius Schellenberger
-//go:build openbsd
// +build openbsd
package godrop
diff --git a/vendor/git.giftfish.de/ston1th/godrop/v2/unveil.go b/vendor/git.giftfish.de/ston1th/godrop/v2/unveil.go
index 412851b..06c8c90 100644
--- a/vendor/git.giftfish.de/ston1th/godrop/v2/unveil.go
+++ b/vendor/git.giftfish.de/ston1th/godrop/v2/unveil.go
@@ -1,6 +1,5 @@
-// Copyright (C) 2022 Marius Schellenberger
+// Copyright (C) 2018 Marius Schellenberger
-//go:build !openbsd
// +build !openbsd
package godrop
diff --git a/vendor/git.giftfish.de/ston1th/godrop/v2/unveil_openbsd.go b/vendor/git.giftfish.de/ston1th/godrop/v2/unveil_openbsd.go
index f1e4bbc..f32d630 100644
--- a/vendor/git.giftfish.de/ston1th/godrop/v2/unveil_openbsd.go
+++ b/vendor/git.giftfish.de/ston1th/godrop/v2/unveil_openbsd.go
@@ -1,6 +1,5 @@
-// Copyright (C) 2022 Marius Schellenberger
+// Copyright (C) 2018 Marius Schellenberger
-//go:build openbsd
// +build openbsd
package godrop
diff --git a/vendor/git.giftfish.de/ston1th/jwt/v3/LICENSE b/vendor/git.giftfish.de/ston1th/jwt/v3/LICENSE
index 74cee2b..115e96e 100644
--- a/vendor/git.giftfish.de/ston1th/jwt/v3/LICENSE
+++ b/vendor/git.giftfish.de/ston1th/jwt/v3/LICENSE
@@ -1,4 +1,4 @@
-Copyright (C) 2025 Marius Schellenberger
+Copyright (C) 2018 Marius Schellenberger
All rights reserved.
Redistribution and use in source and binary forms, with or without
diff --git a/vendor/git.giftfish.de/ston1th/jwt/v3/blacklist.go b/vendor/git.giftfish.de/ston1th/jwt/v3/blacklist.go
index 180c5fd..b60a928 100644
--- a/vendor/git.giftfish.de/ston1th/jwt/v3/blacklist.go
+++ b/vendor/git.giftfish.de/ston1th/jwt/v3/blacklist.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2025 Marius Schellenberger
+// Copyright (C) 2018 Marius Schellenberger
package jwt
diff --git a/vendor/git.giftfish.de/ston1th/jwt/v3/claims.go b/vendor/git.giftfish.de/ston1th/jwt/v3/claims.go
index a5f4364..10cbcfe 100644
--- a/vendor/git.giftfish.de/ston1th/jwt/v3/claims.go
+++ b/vendor/git.giftfish.de/ston1th/jwt/v3/claims.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2025 Marius Schellenberger
+// Copyright (C) 2018 Marius Schellenberger
package jwt
@@ -30,31 +30,40 @@ func (c Claims) GetBool(key string) (b bool) {
// GetInt returns an int from the claims map
func (c Claims) GetInt(key string) (i int) {
- i = int(c.GetInt64(key))
+ if f, ok := c.getFloat64(key); ok {
+ return int(f)
+ }
+ if v, ok := c.Get(key); ok && v != nil {
+ i, _ = v.(int)
+ }
return
}
// GetInt64 returns an int64 from the claims map
func (c Claims) GetInt64(key string) (i int64) {
+ if f, ok := c.getFloat64(key); ok {
+ return int64(f)
+ }
if v, ok := c.Get(key); ok && v != nil {
- switch val := v.(type) {
- case int64:
- i = val
- case float64:
- i = int64(val)
- }
+ i, _ = v.(int64)
}
return
}
// getFloat64 returns a float64 and ok from the claims map
-func (c Claims) GetFloat64(key string) (f float64) {
+func (c Claims) getFloat64(key string) (f float64, fok bool) {
if v, ok := c.Get(key); ok && v != nil {
- f, _ = v.(float64)
+ f, fok = v.(float64)
}
return
}
+// GetFloat64 returns a float64 from the claims map
+func (c Claims) GetFloat64(key string) (f float64) {
+ f, _ = c.getFloat64(key)
+ return
+}
+
// Set sets the value of key in the claims map, if not nil
func (c Claims) Set(key string, v interface{}) {
if c == nil {
diff --git a/vendor/git.giftfish.de/ston1th/jwt/v3/encoding.go b/vendor/git.giftfish.de/ston1th/jwt/v3/encoding.go
index 3693f5f..73b81cf 100644
--- a/vendor/git.giftfish.de/ston1th/jwt/v3/encoding.go
+++ b/vendor/git.giftfish.de/ston1th/jwt/v3/encoding.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2025 Marius Schellenberger
+// Copyright (C) 2018 Marius Schellenberger
package jwt
diff --git a/vendor/git.giftfish.de/ston1th/jwt/v3/go.mod b/vendor/git.giftfish.de/ston1th/jwt/v3/go.mod
new file mode 100644
index 0000000..6752b42
--- /dev/null
+++ b/vendor/git.giftfish.de/ston1th/jwt/v3/go.mod
@@ -0,0 +1 @@
+module git.giftfish.de/ston1th/jwt/v3
diff --git a/vendor/git.giftfish.de/ston1th/jwt/v3/hash.go b/vendor/git.giftfish.de/ston1th/jwt/v3/hash.go
index e47c0cc..2c224ab 100644
--- a/vendor/git.giftfish.de/ston1th/jwt/v3/hash.go
+++ b/vendor/git.giftfish.de/ston1th/jwt/v3/hash.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2025 Marius Schellenberger
+// Copyright (C) 2018 Marius Schellenberger
package jwt
@@ -14,25 +14,19 @@ const (
HS512Name = "HS512"
)
-// NewHash returns the Hash type equal to the input string
-func NewHash(alg string) Hash {
+// ParseHash returns the Hash type equal to the input string
+func ParseHash(alg string) Hash {
switch alg {
case HS256Name:
- return hs256
+ return NewHS256()
case HS384Name:
- return hs384
+ return NewHS384()
case HS512Name:
- return hs512
+ return NewHS512()
}
return nil
}
-var (
- hs256 = HS256{}
- hs384 = HS384{}
- hs512 = HS512{}
-)
-
// Hash is the hashsum interface for signing the jwt
type Hash interface {
Hash() hash.Hash
@@ -42,6 +36,11 @@ type Hash interface {
// HS256 implements the Hash interface with SHA256
type HS256 struct{}
+// NewHS256 returns a new HS256 instance
+func NewHS256() Hash {
+ return HS256{}
+}
+
// Alg returns the algorithm name "HS256"
func (HS256) Alg() string {
return HS256Name
@@ -55,6 +54,11 @@ func (HS256) Hash() hash.Hash {
// HS384 implements the Hash interface with SHA384
type HS384 struct{}
+// NewHS384 returns a new HS384 instance
+func NewHS384() Hash {
+ return HS384{}
+}
+
// Alg returns the algorithm name "HS384"
func (HS384) Alg() string {
return HS384Name
@@ -68,6 +72,11 @@ func (HS384) Hash() hash.Hash {
// HS512 implements the Hash interface with SHA512
type HS512 struct{}
+// NewHS512 returns a new HS512 instance
+func NewHS512() Hash {
+ return HS512{}
+}
+
// Alg returns the algorithm name "HS512"
func (HS512) Alg() string {
return HS512Name
diff --git a/vendor/git.giftfish.de/ston1th/jwt/v3/jwt.go b/vendor/git.giftfish.de/ston1th/jwt/v3/jwt.go
index f9ce6af..44f82ec 100644
--- a/vendor/git.giftfish.de/ston1th/jwt/v3/jwt.go
+++ b/vendor/git.giftfish.de/ston1th/jwt/v3/jwt.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2025 Marius Schellenberger
+// Copyright (C) 2018 Marius Schellenberger
// Package jwt provides a easy to use JSON Web Token and blacklisting library
package jwt
@@ -31,8 +31,6 @@ const (
ExpClaim = "exp"
// NbfClaim is the nbf claim name
NbfClaim = "nbf"
- // NonceClaim
- NonceClaim = "nonce"
// TokenSeparator is the tokens separator char
TokenSeparator = "."
)
@@ -59,39 +57,14 @@ var (
ErrInvalidKeySize = errors.New(jwtErr + "invalid secret key size")
)
-type JWTOption func(*JWT)
-
-func WithExpiry(expiry time.Duration) JWTOption {
- return func(jwt *JWT) {
- jwt.expiry = expiry
- }
-}
-
-func WithBlacklist(blacklist Blacklist) JWTOption {
- return func(jwt *JWT) {
- jwt.blacklist = blacklist
- }
-}
-func WithSecret(secret io.Reader) JWTOption {
- return func(jwt *JWT) {
- jwt.secretReader = secret
- }
-}
-func WithNonce() JWTOption {
- return func(jwt *JWT) {
- jwt.nonce = true
- }
-}
-
// JWT represents the JSON Web Token signing and blacklisting infrastructure
type JWT struct {
- secretReader io.Reader
- key []byte
- expiry time.Duration
- blacklist Blacklist
- nonce bool
- stopOnce sync.Once
- done chan struct{}
+ key []byte
+ expiry time.Duration
+
+ blacklist Blacklist
+ stopOnce sync.Once
+ done chan struct{}
}
// New returns a new JWT object with the given expiry timeout.
@@ -100,18 +73,14 @@ type JWT struct {
// If secret is nil the DefaultSecretReader is used.
// If blacklisting is enabled, the JWT object leaks a goroutine to garbage-collect expired blacklisted tokens.
// Call the Stop() method to exit the goroutine.
-func New(options ...JWTOption) (*JWT, error) {
- jwt := &JWT{}
- for _, option := range options {
- option(jwt)
+func New(expiry time.Duration, blacklist Blacklist, secret io.Reader) (*JWT, error) {
+ if expiry <= 0 {
+ expiry = DefaultExpiry
}
- if jwt.expiry <= 0 {
- jwt.expiry = DefaultExpiry
+ if secret == nil {
+ secret = DefaultSecretReader
}
- if jwt.secretReader == nil {
- jwt.secretReader = DefaultSecretReader
- }
- secret := io.LimitReader(jwt.secretReader, KeySize)
+ secret = io.LimitReader(secret, KeySize)
key := make([]byte, KeySize)
i, err := secret.Read(key)
if err != nil {
@@ -120,8 +89,12 @@ func New(options ...JWTOption) (*JWT, error) {
if i < KeySize {
return nil, ErrInvalidKeySize
}
- jwt.key = key
- if jwt.blacklist != nil {
+ jwt := &JWT{
+ key: key,
+ expiry: expiry,
+ blacklist: blacklist,
+ }
+ if blacklist != nil {
jwt.done = make(chan struct{})
go jwt.clean()
}
@@ -155,7 +128,7 @@ func (jwt *JWT) Invalidate(t *Token) error {
if t.Header.GetString(TypClaim) != Typ {
return ErrNoJWT
}
- h := NewHash(t.Header.GetString(AlgClaim))
+ h := ParseHash(t.Header.GetString(AlgClaim))
if h == nil {
return ErrUnsupportedAlg
}
@@ -217,7 +190,7 @@ func (jwt *JWT) Sign(t *Token) (err error) {
if t.Header.GetString(TypClaim) != Typ {
return ErrNoJWT
}
- h := NewHash(t.Header.GetString(AlgClaim))
+ h := ParseHash(t.Header.GetString(AlgClaim))
if h == nil {
return ErrUnsupportedAlg
}
@@ -228,9 +201,6 @@ func (jwt *JWT) Sign(t *Token) (err error) {
if _, ok := t.Claims.Get(NbfClaim); !ok {
t.Claims.Set(NbfClaim, NewNbf(now))
}
- if jwt.nonce {
- t.Claims.Set(NonceClaim, enc.EncodeToString(key16()))
- }
head, err := json.Marshal(t.Header)
if err != nil {
return
@@ -260,7 +230,7 @@ func (jwt *JWT) Verify(t *Token) error {
if t.Header.GetString(TypClaim) != Typ {
return ErrNoJWT
}
- h := NewHash(t.Header.GetString(AlgClaim))
+ h := ParseHash(t.Header.GetString(AlgClaim))
if h == nil {
return ErrUnsupportedAlg
}
diff --git a/vendor/git.giftfish.de/ston1th/jwt/v3/rand.go b/vendor/git.giftfish.de/ston1th/jwt/v3/rand.go
deleted file mode 100644
index bd1b191..0000000
--- a/vendor/git.giftfish.de/ston1th/jwt/v3/rand.go
+++ /dev/null
@@ -1,11 +0,0 @@
-// Copyright (C) 2025 Marius Schellenberger
-
-package jwt
-
-import "crypto/rand"
-
-func key16() []byte {
- b := make([]byte, 16)
- rand.Read(b)
- return b
-}
diff --git a/vendor/git.giftfish.de/ston1th/jwt/v3/time.go b/vendor/git.giftfish.de/ston1th/jwt/v3/time.go
index 77e151f..71b10ca 100644
--- a/vendor/git.giftfish.de/ston1th/jwt/v3/time.go
+++ b/vendor/git.giftfish.de/ston1th/jwt/v3/time.go
@@ -1,5 +1,3 @@
-// Copyright (C) 2025 Marius Schellenberger
-
package jwt
import "time"
diff --git a/vendor/git.giftfish.de/ston1th/jwt/v3/token.go b/vendor/git.giftfish.de/ston1th/jwt/v3/token.go
index 772415c..a1f1dd1 100644
--- a/vendor/git.giftfish.de/ston1th/jwt/v3/token.go
+++ b/vendor/git.giftfish.de/ston1th/jwt/v3/token.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2025 Marius Schellenberger
+// Copyright (C) 2018 Marius Schellenberger
package jwt
@@ -45,7 +45,7 @@ func NewToken(claims Claims, hash Hash) *Token {
claims = make(Claims)
}
if hash == nil {
- hash = NewHash(HS256Name)
+ hash = NewHS256()
}
return &Token{
Header: Claims{
diff --git a/vendor/github.com/RoaringBitmap/roaring/.drone.yml b/vendor/github.com/RoaringBitmap/roaring/.drone.yml
deleted file mode 100644
index 7936bfe..0000000
--- a/vendor/github.com/RoaringBitmap/roaring/.drone.yml
+++ /dev/null
@@ -1,19 +0,0 @@
-kind: pipeline
-name: default
-
-workspace:
- base: /go
- path: src/github.com/RoaringBitmap/roaring
-
-steps:
-- name: test
- image: golang
- commands:
- - go get -t
- - go test
- - go build -tags appengine
- - go test -tags appengine
- - GOARCH=386 go build
- - GOARCH=386 go test
- - GOARCH=arm go build
- - GOARCH=arm64 go build
diff --git a/vendor/github.com/RoaringBitmap/roaring/.gitignore b/vendor/github.com/RoaringBitmap/roaring/.gitignore
index 851f323..b7943ab 100644
--- a/vendor/github.com/RoaringBitmap/roaring/.gitignore
+++ b/vendor/github.com/RoaringBitmap/roaring/.gitignore
@@ -3,3 +3,4 @@ roaring-fuzz.zip
workdir
coverage.out
testdata/all3.classic
+testdata/all3.msgp.snappy
diff --git a/vendor/github.com/RoaringBitmap/roaring/.travis.yml b/vendor/github.com/RoaringBitmap/roaring/.travis.yml
new file mode 100644
index 0000000..32ceaa6
--- /dev/null
+++ b/vendor/github.com/RoaringBitmap/roaring/.travis.yml
@@ -0,0 +1,32 @@
+language: go
+sudo: false
+install:
+- go get -t github.com/RoaringBitmap/roaring
+- go get -t golang.org/x/tools/cmd/cover
+- go get -t github.com/mattn/goveralls
+- go get -t github.com/mschoch/smat
+notifications:
+ email: false
+go:
+- "1.7.x"
+- "1.8.x"
+- "1.9.x"
+- "1.10.x"
+- "1.11.x"
+- "1.12.x"
+- tip
+
+# whitelist
+branches:
+ only:
+ - master
+script:
+- goveralls -v -service travis-ci -ignore arraycontainer_gen.go,bitmapcontainer_gen.go,rle16_gen.go,rle_gen.go,roaringarray_gen.go,rle.go || go test
+- go test -race -run TestConcurrent*
+- GOARCH=arm64 go build
+- GOARCH=386 go build
+- GOARCH=386 go test
+- GOARCH=arm go build
+matrix:
+ allow_failures:
+ - go: tip
diff --git a/vendor/github.com/RoaringBitmap/roaring/CONTRIBUTORS b/vendor/github.com/RoaringBitmap/roaring/CONTRIBUTORS
index 1a8da9c..4a8bda2 100644
--- a/vendor/github.com/RoaringBitmap/roaring/CONTRIBUTORS
+++ b/vendor/github.com/RoaringBitmap/roaring/CONTRIBUTORS
@@ -12,7 +12,4 @@ Vali Malinoiu (@0x4139),
Forud Ghafouri (@fzerorubigd),
Joe Nall (@joenall),
(@fredim),
-Edd Robinson (@e-dard),
-Alexander Petrov (@alldroll),
-Guy Molinari (@guymolinari),
-Ling Jin (@JinLingChristopher)
+Edd Robinson (@e-dard)
diff --git a/vendor/github.com/RoaringBitmap/roaring/Makefile b/vendor/github.com/RoaringBitmap/roaring/Makefile
new file mode 100644
index 0000000..c489376
--- /dev/null
+++ b/vendor/github.com/RoaringBitmap/roaring/Makefile
@@ -0,0 +1,111 @@
+.PHONY: help all test format fmtcheck vet lint qa deps clean nuke ser fetch-real-roaring-datasets
+
+
+
+
+
+
+
+
+# Display general help about this command
+help:
+ @echo ""
+ @echo "The following commands are available:"
+ @echo ""
+ @echo " make qa : Run all the tests"
+ @echo " make test : Run the unit tests"
+ @echo ""
+ @echo " make format : Format the source code"
+ @echo " make fmtcheck : Check if the source code has been formatted"
+ @echo " make vet : Check for suspicious constructs"
+ @echo " make lint : Check for style errors"
+ @echo ""
+ @echo " make deps : Get the dependencies"
+ @echo " make clean : Remove any build artifact"
+ @echo " make nuke : Deletes any intermediate file"
+ @echo ""
+ @echo " make fuzz-smat : Fuzzy testing with smat"
+ @echo " make fuzz-stream : Fuzzy testing with stream deserialization"
+ @echo " make fuzz-buffer : Fuzzy testing with buffer deserialization"
+ @echo ""
+
+# Alias for help target
+all: help
+test:
+ go test
+ go test -race -run TestConcurrent*
+# Format the source code
+format:
+ @find ./ -type f -name "*.go" -exec gofmt -w {} \;
+
+# Check if the source code has been formatted
+fmtcheck:
+ @mkdir -p target
+ @find ./ -type f -name "*.go" -exec gofmt -d {} \; | tee target/format.diff
+ @test ! -s target/format.diff || { echo "ERROR: the source code has not been formatted - please use 'make format' or 'gofmt'"; exit 1; }
+
+# Check for syntax errors
+vet:
+ GOPATH=$(GOPATH) go vet ./...
+
+# Check for style errors
+lint:
+ GOPATH=$(GOPATH) PATH=$(GOPATH)/bin:$(PATH) golint ./...
+
+
+
+
+
+# Alias to run all quality-assurance checks
+qa: fmtcheck test vet lint
+
+# --- INSTALL ---
+
+# Get the dependencies
+deps:
+ GOPATH=$(GOPATH) go get github.com/smartystreets/goconvey/convey
+ GOPATH=$(GOPATH) go get github.com/willf/bitset
+ GOPATH=$(GOPATH) go get github.com/golang/lint/golint
+ GOPATH=$(GOPATH) go get github.com/mschoch/smat
+ GOPATH=$(GOPATH) go get github.com/dvyukov/go-fuzz/go-fuzz
+ GOPATH=$(GOPATH) go get github.com/dvyukov/go-fuzz/go-fuzz-build
+ GOPATH=$(GOPATH) go get github.com/glycerine/go-unsnap-stream
+ GOPATH=$(GOPATH) go get github.com/philhofer/fwd
+ GOPATH=$(GOPATH) go get github.com/jtolds/gls
+
+fuzz-smat:
+ go test -tags=gofuzz -run=TestGenerateSmatCorpus
+ go-fuzz-build -func FuzzSmat github.com/RoaringBitmap/roaring
+ go-fuzz -bin=./roaring-fuzz.zip -workdir=workdir/ -timeout=200
+
+
+fuzz-stream:
+ go-fuzz-build -func FuzzSerializationStream github.com/RoaringBitmap/roaring
+ go-fuzz -bin=./roaring-fuzz.zip -workdir=workdir/ -timeout=200
+
+
+fuzz-buffer:
+ go-fuzz-build -func FuzzSerializationBuffer github.com/RoaringBitmap/roaring
+ go-fuzz -bin=./roaring-fuzz.zip -workdir=workdir/ -timeout=200
+
+# Remove any build artifact
+clean:
+ GOPATH=$(GOPATH) go clean ./...
+
+# Deletes any intermediate file
+nuke:
+ rm -rf ./target
+ GOPATH=$(GOPATH) go clean -i ./...
+
+
+ser:
+ go generate
+
+cover:
+ go test -coverprofile=coverage.out
+ go tool cover -html=coverage.out
+
+fetch-real-roaring-datasets:
+ # pull github.com/RoaringBitmap/real-roaring-datasets -> testdata/real-roaring-datasets
+ git submodule init
+ git submodule update
diff --git a/vendor/github.com/RoaringBitmap/roaring/README.md b/vendor/github.com/RoaringBitmap/roaring/README.md
index f6705df..a3e766d 100644
--- a/vendor/github.com/RoaringBitmap/roaring/README.md
+++ b/vendor/github.com/RoaringBitmap/roaring/README.md
@@ -1,24 +1,20 @@
-# roaring
-
-[](https://godoc.org/github.com/RoaringBitmap/roaring) [](https://goreportcard.com/report/github.com/RoaringBitmap/roaring)
-
-
-
-
+roaring [](https://travis-ci.org/RoaringBitmap/roaring) [](https://coveralls.io/github/RoaringBitmap/roaring?branch=master) [](https://godoc.org/github.com/RoaringBitmap/roaring) [](https://goreportcard.com/report/github.com/RoaringBitmap/roaring)
=============
This is a go version of the Roaring bitmap data structure.
+
+
Roaring bitmaps are used by several major systems such as [Apache Lucene][lucene] and derivative systems such as [Solr][solr] and
-[Elasticsearch][elasticsearch], [Apache Druid (Incubating)][druid], [LinkedIn Pinot][pinot], [Netflix Atlas][atlas], [Apache Spark][spark], [OpenSearchServer][opensearchserver], [anacrolix/torrent][anacrolix/torrent], [Whoosh][whoosh], [Redpanda](https://github.com/redpanda-data/redpanda), [Pilosa][pilosa], [Microsoft Visual Studio Team Services (VSTS)][vsts], and eBay's [Apache Kylin][kylin]. The YouTube SQL Engine, [Google Procella](https://research.google/pubs/pub48388/), uses Roaring bitmaps for indexing.
+[Elasticsearch][elasticsearch], [Metamarkets' Druid][druid], [LinkedIn Pinot][pinot], [Netflix Atlas][atlas], [Apache Spark][spark], [OpenSearchServer][opensearchserver], [Cloud Torrent][cloudtorrent], [Whoosh][whoosh], [Pilosa][pilosa], [Microsoft Visual Studio Team Services (VSTS)][vsts], and eBay's [Apache Kylin][kylin].
[lucene]: https://lucene.apache.org/
[solr]: https://lucene.apache.org/solr/
[elasticsearch]: https://www.elastic.co/products/elasticsearch
-[druid]: https://druid.apache.org/
+[druid]: http://druid.io/
[spark]: https://spark.apache.org/
[opensearchserver]: http://www.opensearchserver.com
-[anacrolix/torrent]: https://github.com/anacrolix/torrent
+[cloudtorrent]: https://github.com/jpillora/cloud-torrent
[whoosh]: https://bitbucket.org/mchaput/whoosh/wiki/Home
[pilosa]: https://www.pilosa.com/
[kylin]: http://kylin.apache.org/
@@ -32,18 +28,11 @@ Roaring bitmaps are found to work well in many important applications:
The ``roaring`` Go library is used by
-* [anacrolix/torrent]
+* [Cloud Torrent](https://github.com/jpillora/cloud-torrent): a self-hosted remote torrent client
+* [runv](https://github.com/hyperhq/runv): an Hypervisor-based runtime for the Open Containers Initiative
* [InfluxDB](https://www.influxdata.com)
* [Pilosa](https://www.pilosa.com/)
* [Bleve](http://www.blevesearch.com)
-* [Weaviate](https://github.com/weaviate/weaviate)
-* [lindb](https://github.com/lindb/lindb)
-* [Elasticell](https://github.com/deepfabric/elasticell)
-* [SourceGraph](https://github.com/sourcegraph/sourcegraph)
-* [M3](https://github.com/m3db/m3)
-* [trident](https://github.com/NetApp/trident)
-* [Husky](https://www.datadoghq.com/blog/engineering/introducing-husky/)
-* [FrostDB](https://github.com/polarsignals/frostdb)
This library is used in production in several systems, it is part of the [Awesome Go collection](https://awesome-go.com).
@@ -56,108 +45,24 @@ This code is licensed under Apache License, Version 2.0 (ASL2.0).
Copyright 2016-... by the authors.
-When should you use a bitmap?
-===================================
-
-
-Sets are a fundamental abstraction in
-software. They can be implemented in various
-ways, as hash sets, as trees, and so forth.
-In databases and search engines, sets are often an integral
-part of indexes. For example, we may need to maintain a set
-of all documents or rows (represented by numerical identifier)
-that satisfy some property. Besides adding or removing
-elements from the set, we need fast functions
-to compute the intersection, the union, the difference between sets, and so on.
-
-
-To implement a set
-of integers, a particularly appealing strategy is the
-bitmap (also called bitset or bit vector). Using n bits,
-we can represent any set made of the integers from the range
-[0,n): the ith bit is set to one if integer i is present in the set.
-Commodity processors use words of W=32 or W=64 bits. By combining many such words, we can
-support large values of n. Intersections, unions and differences can then be implemented
- as bitwise AND, OR and ANDNOT operations.
-More complicated set functions can also be implemented as bitwise operations.
-
-When the bitset approach is applicable, it can be orders of
-magnitude faster than other possible implementation of a set (e.g., as a hash set)
-while using several times less memory.
-
-However, a bitset, even a compressed one is not always applicable. For example, if
-you have 1000 random-looking integers, then a simple array might be the best representation.
-We refer to this case as the "sparse" scenario.
-
-When should you use compressed bitmaps?
-===================================
-
-An uncompressed BitSet can use a lot of memory. For example, if you take a BitSet
-and set the bit at position 1,000,000 to true and you have just over 100kB. That is over 100kB
-to store the position of one bit. This is wasteful even if you do not care about memory:
-suppose that you need to compute the intersection between this BitSet and another one
-that has a bit at position 1,000,001 to true, then you need to go through all these zeroes,
-whether you like it or not. That can become very wasteful.
-
-This being said, there are definitively cases where attempting to use compressed bitmaps is wasteful.
-For example, if you have a small universe size. E.g., your bitmaps represent sets of integers
-from [0,n) where n is small (e.g., n=64 or n=128). If you can use uncompressed BitSet and
-it does not blow up your memory usage, then compressed bitmaps are probably not useful
-to you. In fact, if you do not need compression, then a BitSet offers remarkable speed.
-
-The sparse scenario is another use case where compressed bitmaps should not be used.
-Keep in mind that random-looking data is usually not compressible. E.g., if you have a small set of
-32-bit random integers, it is not mathematically possible to use far less than 32 bits per integer,
-and attempts at compression can be counterproductive.
-
-How does Roaring compares with the alternatives?
-==================================================
-
-
-Most alternatives to Roaring are part of a larger family of compressed bitmaps that are run-length-encoded
-bitmaps. They identify long runs of 1s or 0s and they represent them with a marker word.
-If you have a local mix of 1s and 0, you use an uncompressed word.
-
-There are many formats in this family:
-
-* Oracle's BBC is an obsolete format at this point: though it may provide good compression,
-it is likely much slower than more recent alternatives due to excessive branching.
-* WAH is a patented variation on BBC that provides better performance.
-* Concise is a variation on the patented WAH. It some specific instances, it can compress
-much better than WAH (up to 2x better), but it is generally slower.
-* EWAH is both free of patent, and it is faster than all the above. On the downside, it
-does not compress quite as well. It is faster because it allows some form of "skipping"
-over uncompressed words. So though none of these formats are great at random access, EWAH
-is better than the alternatives.
-
-
-
-There is a big problem with these formats however that can hurt you badly in some cases: there is no random access. If you want to check whether a given value is present in the set, you have to start from the beginning and "uncompress" the whole thing. This means that if you want to intersect a big set with a large set, you still have to uncompress the whole big set in the worst case...
-
-Roaring solves this problem. It works in the following manner. It divides the data into chunks of 2
16 integers
-(e.g., [0, 2
16), [2
16, 2 x 2
16), ...). Within a chunk, it can use an uncompressed bitmap, a simple list of integers,
-or a list of runs. Whatever format it uses, they all allow you to check for the presence of any one value quickly
-(e.g., with a binary search). The net result is that Roaring can compute many operations much faster than run-length-encoded
-formats like WAH, EWAH, Concise... Maybe surprisingly, Roaring also generally offers better compression ratios.
-
-
-
-
### References
- Daniel Lemire, Owen Kaser, Nathan Kurz, Luca Deri, Chris O'Hara, François Saint-Jacques, Gregory Ssi-Yan-Kai, Roaring Bitmaps: Implementation of an Optimized Software Library, Software: Practice and Experience 48 (4), 2018 [arXiv:1709.07821](https://arxiv.org/abs/1709.07821)
- Samy Chambi, Daniel Lemire, Owen Kaser, Robert Godin,
Better bitmap performance with Roaring bitmaps,
-Software: Practice and Experience 46 (5), 2016.[arXiv:1402.6407](http://arxiv.org/abs/1402.6407) This paper used data from http://lemire.me/data/realroaring2014.html
-- Daniel Lemire, Gregory Ssi-Yan-Kai, Owen Kaser, Consistently faster and smaller compressed bitmaps with Roaring, Software: Practice and Experience 46 (11), 2016. [arXiv:1603.06549](http://arxiv.org/abs/1603.06549)
+Software: Practice and Experience 46 (5), 2016.
+http://arxiv.org/abs/1402.6407 This paper used data from http://lemire.me/data/realroaring2014.html
+- Daniel Lemire, Gregory Ssi-Yan-Kai, Owen Kaser, Consistently faster and smaller compressed bitmaps with Roaring, Software: Practice and Experience 46 (11), 2016. http://arxiv.org/abs/1603.06549
+
### Dependencies
Dependencies are fetched automatically by giving the `-t` flag to `go get`.
they include
- - github.com/bits-and-blooms/bitset
+ - github.com/smartystreets/goconvey/convey
+ - github.com/willf/bitset
- github.com/mschoch/smat
- github.com/glycerine/go-unsnap-stream
- github.com/philhofer/fwd
@@ -169,15 +74,6 @@ Note that the smat library requires Go 1.6 or better.
- go get -t github.com/RoaringBitmap/roaring
-### Instructions for contributors
-
-Using bash or other common shells:
-```
-$ git clone git@github.com:RoaringBitmap/roaring.git
-$ export GO111MODULE=on
-$ go mod tidy
-$ go test -v
-```
### Example
@@ -270,70 +166,10 @@ That is, given a fixed overhead for the universe size (x), Roaring
bitmaps never use more than 2 bytes per integer. You can call
``BoundSerializedSizeInBytes`` for a more precise estimate.
-### 64-bit Roaring
-
-By default, roaring is used to stored unsigned 32-bit integers. However, we also offer
-an extension dedicated to 64-bit integers. It supports roughly the same functions:
-
-```go
-package main
-
-import (
- "fmt"
- "github.com/RoaringBitmap/roaring/roaring64"
- "bytes"
-)
-
-
-func main() {
- // example inspired by https://github.com/fzandona/goroar
- fmt.Println("==roaring64==")
- rb1 := roaring64.BitmapOf(1, 2, 3, 4, 5, 100, 1000)
- fmt.Println(rb1.String())
-
- rb2 := roaring64.BitmapOf(3, 4, 1000)
- fmt.Println(rb2.String())
-
- rb3 := roaring64.New()
- fmt.Println(rb3.String())
-
- fmt.Println("Cardinality: ", rb1.GetCardinality())
-
- fmt.Println("Contains 3? ", rb1.Contains(3))
-
- rb1.And(rb2)
-
- rb3.Add(1)
- rb3.Add(5)
-
- rb3.Or(rb1)
-
-
-
- // prints 1, 3, 4, 5, 1000
- i := rb3.Iterator()
- for i.HasNext() {
- fmt.Println(i.Next())
- }
- fmt.Println()
-
- // next we include an example of serialization
- buf := new(bytes.Buffer)
- rb1.WriteTo(buf) // we omit error handling
- newrb:= roaring64.New()
- newrb.ReadFrom(buf)
- if rb1.Equals(newrb) {
- fmt.Println("I wrote the content to a byte stream and read it back.")
- }
- // you can iterate over bitmaps using ReverseIterator(), Iterator, ManyIterator()
-}
-```
-
-Only the 32-bit roaring format is standard and cross-operable between Java, C++, C and Go. There is no guarantee that the 64-bit versions are compatible.
### Documentation
-Current documentation is available at https://pkg.go.dev/github.com/RoaringBitmap/roaring and https://pkg.go.dev/github.com/RoaringBitmap/roaring/roaring64
+Current documentation is available at http://godoc.org/github.com/RoaringBitmap/roaring
### Goroutine safety
@@ -371,7 +207,7 @@ You can use roaring with gore:
- go get -u github.com/motemen/gore
- Make sure that ``$GOPATH/bin`` is in your ``$PATH``.
-- go get github.com/RoaringBitmap/roaring
+- go get github/RoaringBitmap/roaring
```go
$ gore
@@ -392,14 +228,12 @@ You can help us test further the library with fuzzy testing:
go get github.com/dvyukov/go-fuzz/go-fuzz-build
go test -tags=gofuzz -run=TestGenerateSmatCorpus
go-fuzz-build github.com/RoaringBitmap/roaring
- go-fuzz -bin=./roaring-fuzz.zip -workdir=workdir/ -timeout=200 -func FuzzSmat
+ go-fuzz -bin=./roaring-fuzz.zip -workdir=workdir/ -timeout=200
Let it run, and if the # of crashers is > 0, check out the reports in
the workdir where you should be able to find the panic goroutine stack
traces.
-You may also replace `-func FuzzSmat` by `-func FuzzSerializationBuffer` or `-func FuzzSerializationStream`.
-
### Alternative in Go
There is a Go version wrapping the C/C++ implementation https://github.com/RoaringBitmap/gocroaring
diff --git a/vendor/github.com/RoaringBitmap/roaring/arraycontainer.go b/vendor/github.com/RoaringBitmap/roaring/arraycontainer.go
index 80fa676..8b1a6c8 100644
--- a/vendor/github.com/RoaringBitmap/roaring/arraycontainer.go
+++ b/vendor/github.com/RoaringBitmap/roaring/arraycontainer.go
@@ -4,6 +4,8 @@ import (
"fmt"
)
+//go:generate msgp -unexported
+
type arrayContainer struct {
content []uint16
}
@@ -16,35 +18,13 @@ func (ac *arrayContainer) String() string {
return s + "}"
}
-func (ac *arrayContainer) fillLeastSignificant16bits(x []uint32, i int, mask uint32) int {
- if i < 0 {
- panic("negative index")
- }
- if len(ac.content) == 0 {
- return i
- }
- _ = x[len(ac.content)-1+i]
- _ = ac.content[len(ac.content)-1]
+func (ac *arrayContainer) fillLeastSignificant16bits(x []uint32, i int, mask uint32) {
for k := 0; k < len(ac.content); k++ {
- x[k+i] =
- uint32(ac.content[k]) | mask
+ x[k+i] = uint32(ac.content[k]) | mask
}
- return i + len(ac.content)
}
-func (ac *arrayContainer) iterate(cb func(x uint16) bool) bool {
- iterator := shortIterator{ac.content, 0}
-
- for iterator.hasNext() {
- if !cb(iterator.next()) {
- return false
- }
- }
-
- return true
-}
-
-func (ac *arrayContainer) getShortIterator() shortPeekable {
+func (ac *arrayContainer) getShortIterator() shortIterable {
return &shortIterator{ac.content, 0}
}
@@ -53,7 +33,7 @@ func (ac *arrayContainer) getReverseIterator() shortIterable {
}
func (ac *arrayContainer) getManyIterator() manyIterable {
- return &shortIterator{ac.content, 0}
+ return &manyIterator{ac.content, 0}
}
func (ac *arrayContainer) minimum() uint16 {
@@ -367,17 +347,28 @@ func (ac *arrayContainer) iorArray(value2 *arrayContainer) container {
len1 := value1.getCardinality()
len2 := value2.getCardinality()
maxPossibleCardinality := len1 + len2
- if maxPossibleCardinality > cap(value1.content) {
- // doubling the capacity reduces new slice allocations in the case of
- // repeated calls to iorArray().
- newSize := 2 * maxPossibleCardinality
- // the second check is to handle overly large array containers
- // and should not occur in normal usage,
- // as all array containers should be at most arrayDefaultMaxSize
- if newSize > 2*arrayDefaultMaxSize && maxPossibleCardinality <= 2*arrayDefaultMaxSize {
- newSize = 2 * arrayDefaultMaxSize
+ if maxPossibleCardinality > arrayDefaultMaxSize { // it could be a bitmap!
+ bc := newBitmapContainer()
+ for k := 0; k < len(value2.content); k++ {
+ v := value2.content[k]
+ i := uint(v) >> 6
+ mask := uint64(1) << (v % 64)
+ bc.bitmap[i] |= mask
}
- newcontent := make([]uint16, 0, newSize)
+ for k := 0; k < len(ac.content); k++ {
+ v := ac.content[k]
+ i := uint(v) >> 6
+ mask := uint64(1) << (v % 64)
+ bc.bitmap[i] |= mask
+ }
+ bc.cardinality = int(popcntSlice(bc.bitmap))
+ if bc.cardinality <= arrayDefaultMaxSize {
+ return bc.toArrayContainer()
+ }
+ return bc
+ }
+ if maxPossibleCardinality > cap(value1.content) {
+ newcontent := make([]uint16, 0, maxPossibleCardinality)
copy(newcontent[len2:maxPossibleCardinality], ac.content[0:len1])
ac.content = newcontent
} else {
@@ -385,13 +376,6 @@ func (ac *arrayContainer) iorArray(value2 *arrayContainer) container {
}
nl := union2by2(value1.content[len2:maxPossibleCardinality], value2.content, ac.content)
ac.content = ac.content[:nl] // reslice to match actual used capacity
-
- if nl > arrayDefaultMaxSize {
- // Only converting to a bitmap when arrayDefaultMaxSize
- // is actually exceeded minimizes conversions in the case of repeated
- // calls to iorArray().
- return ac.toBitmapContainer()
- }
return ac
}
@@ -404,19 +388,11 @@ func (ac *arrayContainer) iorBitmap(bc2 *bitmapContainer) container {
}
func (ac *arrayContainer) iorRun16(rc *runContainer16) container {
- runCardinality := rc.getCardinality()
- // heuristic for if the container should maybe be an
- // array container.
- if runCardinality < ac.getCardinality() &&
- runCardinality+ac.getCardinality() < arrayDefaultMaxSize {
- var result container
- result = ac
- for _, run := range rc.iv {
- result = result.iaddRange(int(run.start), int(run.start)+int(run.length)+1)
- }
- return result
- }
- return rc.orArray(ac)
+ bc1 := ac.toBitmapContainer()
+ bc2 := rc.toBitmapContainer()
+ bc1.iorBitmap(bc2)
+ *ac = *newArrayContainerFromBitmap(bc1)
+ return ac
}
func (ac *arrayContainer) lazyIOR(a container) container {
@@ -501,7 +477,7 @@ func (ac *arrayContainer) orArrayCardinality(value2 *arrayContainer) int {
func (ac *arrayContainer) lazyorArray(value2 *arrayContainer) container {
value1 := ac
maxPossibleCardinality := value1.getCardinality() + value2.getCardinality()
- if maxPossibleCardinality > arrayLazyLowerBound { // it could be a bitmap!
+ if maxPossibleCardinality > arrayLazyLowerBound { // it could be a bitmap!^M
bc := newBitmapContainer()
for k := 0; k < len(value2.content); k++ {
v := value2.content[k]
@@ -664,54 +640,10 @@ func (ac *arrayContainer) iandNot(a container) container {
}
func (ac *arrayContainer) iandNotRun16(rc *runContainer16) container {
- // Fast path: if either the array container or the run container is empty, the result is the array.
- if ac.isEmpty() || rc.isEmpty() {
- // Empty
- return ac
- }
- // Fast path: if the run container is full, the result is empty.
- if rc.isFull() {
- ac.content = ac.content[:0]
- return ac
- }
- current_run := 0
- // All values in [start_run, end_end] are part of the run
- start_run := rc.iv[current_run].start
- end_end := start_run + rc.iv[current_run].length
- // We are going to read values in the array at index i, and we are
- // going to write them at index pos. So we do in-place processing.
- // We always have that pos <= i by construction. So we can either
- // overwrite a value just read, or a value that was previous read.
- pos := 0
- i := 0
- for ; i < len(ac.content); i++ {
- if ac.content[i] < start_run {
- // the value in the array appears before the run [start_run, end_end]
- ac.content[pos] = ac.content[i]
- pos++
- } else if ac.content[i] <= end_end {
- // nothing to do, the value is in the array but also in the run.
- } else {
- // We have the value in the array after the run. We cannot tell
- // whether we need to keep it or not. So let us move to another run.
- if current_run+1 < len(rc.iv) {
- current_run++
- start_run = rc.iv[current_run].start
- end_end = start_run + rc.iv[current_run].length
- i-- // retry with the same i
- } else {
- // We have exhausted the number of runs. We can keep the rest of the values
- // from i to len(ac.content) - 1 inclusively.
- break // We are done, the rest of the array will be kept
- }
- }
- }
- for ; i < len(ac.content); i++ {
- ac.content[pos] = ac.content[i]
- pos++
- }
- // We 'shink' the slice.
- ac.content = ac.content[:pos]
+ rcb := rc.toBitmapContainer()
+ acb := ac.toBitmapContainer()
+ acb.iandNotBitmapSurely(rcb)
+ *ac = *(acb.toArrayContainer())
return ac
}
@@ -905,10 +837,6 @@ func (ac *arrayContainer) getCardinality() int {
return len(ac.content)
}
-func (ac *arrayContainer) isEmpty() bool {
- return len(ac.content) == 0
-}
-
func (ac *arrayContainer) rank(x uint16) int {
answer := binarySearch(ac.content, x)
if answer >= 0 {
@@ -936,41 +864,6 @@ func (ac *arrayContainer) loadData(bitmapContainer *bitmapContainer) {
ac.content = make([]uint16, bitmapContainer.cardinality, bitmapContainer.cardinality)
bitmapContainer.fillArray(ac.content)
}
-
-func (ac *arrayContainer) resetTo(a container) {
- switch x := a.(type) {
- case *arrayContainer:
- ac.realloc(len(x.content))
- copy(ac.content, x.content)
-
- case *bitmapContainer:
- ac.realloc(x.cardinality)
- x.fillArray(ac.content)
-
- case *runContainer16:
- card := int(x.getCardinality())
- ac.realloc(card)
- cur := 0
- for _, r := range x.iv {
- for val := r.start; val <= r.last(); val++ {
- ac.content[cur] = val
- cur++
- }
- }
-
- default:
- panic("unsupported container type")
- }
-}
-
-func (ac *arrayContainer) realloc(size int) {
- if cap(ac.content) < size {
- ac.content = make([]uint16, size)
- } else {
- ac.content = ac.content[:size]
- }
-}
-
func newArrayContainer() *arrayContainer {
p := new(arrayContainer)
return p
@@ -1022,10 +915,10 @@ func (ac *arrayContainer) numberOfRuns() (nr int) {
runlen++
} else {
if cur < prev {
- panic("the fundamental arrayContainer assumption of sorted ac.content was broken")
+ panic("then fundamental arrayContainer assumption of sorted ac.content was broken")
}
if cur == prev {
- panic("the fundamental arrayContainer assumption of deduplicated content was broken")
+ panic("then fundamental arrayContainer assumption of deduplicated content was broken")
} else {
nr++
runlen = 0
@@ -1060,42 +953,16 @@ func (ac *arrayContainer) containerType() contype {
return arrayContype
}
-func (ac *arrayContainer) addOffset(x uint16) (container, container) {
- var low, high *arrayContainer
-
- if len(ac.content) == 0 {
- return nil, nil
- }
-
- if y := uint32(ac.content[0]) + uint32(x); highbits(y) == 0 {
- // Some elements will fall into low part, allocate a container.
- // Checking the first one is enough because they are ordered.
- low = &arrayContainer{}
- }
- if y := uint32(ac.content[len(ac.content)-1]) + uint32(x); highbits(y) > 0 {
- // Some elements will fall into high part, allocate a container.
- // Checking the last one is enough because they are ordered.
- high = &arrayContainer{}
- }
-
+func (ac *arrayContainer) addOffset(x uint16) []container {
+ low := &arrayContainer{}
+ high := &arrayContainer{}
for _, val := range ac.content {
y := uint32(val) + uint32(x)
if highbits(y) > 0 {
- // OK, if high == nil then highbits(y) == 0 for all y.
high.content = append(high.content, lowbits(y))
} else {
- // OK, if low == nil then highbits(y) > 0 for all y.
low.content = append(low.content, lowbits(y))
}
}
-
- // Ensure proper nil interface.
- if low == nil {
- return nil, high
- }
- if high == nil {
- return low, nil
- }
-
- return low, high
+ return []container{low, high}
}
diff --git a/vendor/github.com/RoaringBitmap/roaring/arraycontainer_gen.go b/vendor/github.com/RoaringBitmap/roaring/arraycontainer_gen.go
new file mode 100644
index 0000000..6ee670e
--- /dev/null
+++ b/vendor/github.com/RoaringBitmap/roaring/arraycontainer_gen.go
@@ -0,0 +1,134 @@
+package roaring
+
+// NOTE: THIS FILE WAS PRODUCED BY THE
+// MSGP CODE GENERATION TOOL (github.com/tinylib/msgp)
+// DO NOT EDIT
+
+import "github.com/tinylib/msgp/msgp"
+
+// Deprecated: DecodeMsg implements msgp.Decodable
+func (z *arrayContainer) DecodeMsg(dc *msgp.Reader) (err error) {
+ var field []byte
+ _ = field
+ var zbzg uint32
+ zbzg, err = dc.ReadMapHeader()
+ if err != nil {
+ return
+ }
+ for zbzg > 0 {
+ zbzg--
+ field, err = dc.ReadMapKeyPtr()
+ if err != nil {
+ return
+ }
+ switch msgp.UnsafeString(field) {
+ case "content":
+ var zbai uint32
+ zbai, err = dc.ReadArrayHeader()
+ if err != nil {
+ return
+ }
+ if cap(z.content) >= int(zbai) {
+ z.content = (z.content)[:zbai]
+ } else {
+ z.content = make([]uint16, zbai)
+ }
+ for zxvk := range z.content {
+ z.content[zxvk], err = dc.ReadUint16()
+ if err != nil {
+ return
+ }
+ }
+ default:
+ err = dc.Skip()
+ if err != nil {
+ return
+ }
+ }
+ }
+ return
+}
+
+// Deprecated: EncodeMsg implements msgp.Encodable
+func (z *arrayContainer) EncodeMsg(en *msgp.Writer) (err error) {
+ // map header, size 1
+ // write "content"
+ err = en.Append(0x81, 0xa7, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74)
+ if err != nil {
+ return err
+ }
+ err = en.WriteArrayHeader(uint32(len(z.content)))
+ if err != nil {
+ return
+ }
+ for zxvk := range z.content {
+ err = en.WriteUint16(z.content[zxvk])
+ if err != nil {
+ return
+ }
+ }
+ return
+}
+
+// Deprecated: MarshalMsg implements msgp.Marshaler
+func (z *arrayContainer) MarshalMsg(b []byte) (o []byte, err error) {
+ o = msgp.Require(b, z.Msgsize())
+ // map header, size 1
+ // string "content"
+ o = append(o, 0x81, 0xa7, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74)
+ o = msgp.AppendArrayHeader(o, uint32(len(z.content)))
+ for zxvk := range z.content {
+ o = msgp.AppendUint16(o, z.content[zxvk])
+ }
+ return
+}
+
+// Deprecated: UnmarshalMsg implements msgp.Unmarshaler
+func (z *arrayContainer) UnmarshalMsg(bts []byte) (o []byte, err error) {
+ var field []byte
+ _ = field
+ var zcmr uint32
+ zcmr, bts, err = msgp.ReadMapHeaderBytes(bts)
+ if err != nil {
+ return
+ }
+ for zcmr > 0 {
+ zcmr--
+ field, bts, err = msgp.ReadMapKeyZC(bts)
+ if err != nil {
+ return
+ }
+ switch msgp.UnsafeString(field) {
+ case "content":
+ var zajw uint32
+ zajw, bts, err = msgp.ReadArrayHeaderBytes(bts)
+ if err != nil {
+ return
+ }
+ if cap(z.content) >= int(zajw) {
+ z.content = (z.content)[:zajw]
+ } else {
+ z.content = make([]uint16, zajw)
+ }
+ for zxvk := range z.content {
+ z.content[zxvk], bts, err = msgp.ReadUint16Bytes(bts)
+ if err != nil {
+ return
+ }
+ }
+ default:
+ bts, err = msgp.Skip(bts)
+ if err != nil {
+ return
+ }
+ }
+ }
+ o = bts
+ return
+}
+
+// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *arrayContainer) Msgsize() (s int) {
+ s = 1 + 8 + msgp.ArrayHeaderSize + (len(z.content) * (msgp.Uint16Size))
+ return
+}
diff --git a/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer.go b/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer.go
index bf08bfc..038863d 100644
--- a/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer.go
+++ b/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer.go
@@ -5,6 +5,8 @@ import (
"unsafe"
)
+//go:generate msgp -unexported
+
type bitmapContainer struct {
cardinality int
bitmap []uint64
@@ -94,18 +96,6 @@ func (bc *bitmapContainer) maximum() uint16 {
return uint16(0)
}
-func (bc *bitmapContainer) iterate(cb func(x uint16) bool) bool {
- iterator := bitmapContainerShortIterator{bc, bc.NextSetBit(0)}
-
- for iterator.hasNext() {
- if !cb(iterator.next()) {
- return false
- }
- }
-
- return true
-}
-
type bitmapContainerShortIterator struct {
ptr *bitmapContainer
i int
@@ -113,28 +103,18 @@ type bitmapContainerShortIterator struct {
func (bcsi *bitmapContainerShortIterator) next() uint16 {
j := bcsi.i
- bcsi.i = bcsi.ptr.NextSetBit(uint(bcsi.i) + 1)
+ bcsi.i = bcsi.ptr.NextSetBit(bcsi.i + 1)
return uint16(j)
}
func (bcsi *bitmapContainerShortIterator) hasNext() bool {
return bcsi.i >= 0
}
-func (bcsi *bitmapContainerShortIterator) peekNext() uint16 {
- return uint16(bcsi.i)
-}
-
-func (bcsi *bitmapContainerShortIterator) advanceIfNeeded(minval uint16) {
- if bcsi.hasNext() && bcsi.peekNext() < minval {
- bcsi.i = bcsi.ptr.NextSetBit(uint(minval))
- }
-}
-
func newBitmapContainerShortIterator(a *bitmapContainer) *bitmapContainerShortIterator {
return &bitmapContainerShortIterator{a, a.NextSetBit(0)}
}
-func (bc *bitmapContainer) getShortIterator() shortPeekable {
+func (bc *bitmapContainer) getShortIterator() shortIterable {
return newBitmapContainerShortIterator(bc)
}
@@ -201,33 +181,6 @@ func (bcmi *bitmapContainerManyIterator) nextMany(hs uint32, buf []uint32) int {
return n
}
-func (bcmi *bitmapContainerManyIterator) nextMany64(hs uint64, buf []uint64) int {
- n := 0
- base := bcmi.base
- bitset := bcmi.bitset
-
- for n < len(buf) {
- if bitset == 0 {
- base++
- if base >= len(bcmi.ptr.bitmap) {
- bcmi.base = base
- bcmi.bitset = bitset
- return n
- }
- bitset = bcmi.ptr.bitmap[base]
- continue
- }
- t := bitset & -bitset
- buf[n] = uint64(((base * 64) + int(popcount(t-1)))) | hs
- n = n + 1
- bitset ^= t
- }
-
- bcmi.base = base
- bcmi.bitset = bitset
- return n
-}
-
func newBitmapContainerManyIterator(a *bitmapContainer) *bitmapContainerManyIterator {
return &bitmapContainerManyIterator{a, -1, 0}
}
@@ -264,7 +217,7 @@ func bitmapEquals(a, b []uint64) bool {
return true
}
-func (bc *bitmapContainer) fillLeastSignificant16bits(x []uint32, i int, mask uint32) int {
+func (bc *bitmapContainer) fillLeastSignificant16bits(x []uint32, i int, mask uint32) {
// TODO: should be written as optimized assembly
pos := i
base := mask
@@ -278,7 +231,6 @@ func (bc *bitmapContainer) fillLeastSignificant16bits(x []uint32, i int, mask ui
}
base += 64
}
- return pos
}
func (bc *bitmapContainer) equals(o container) bool {
@@ -350,10 +302,6 @@ func (bc *bitmapContainer) getCardinality() int {
return bc.cardinality
}
-func (bc *bitmapContainer) isEmpty() bool {
- return bc.cardinality == 0
-}
-
func (bc *bitmapContainer) clone() container {
ptr := bitmapContainer{bc.cardinality, make([]uint64, len(bc.bitmap))}
copy(ptr.bitmap, bc.bitmap[:])
@@ -888,67 +836,13 @@ func (bc *bitmapContainer) iandNot(a container) container {
}
func (bc *bitmapContainer) iandNotArray(ac *arrayContainer) container {
- if ac.isEmpty() || bc.isEmpty() {
- // Nothing to do.
- return bc
- }
-
- // Word by word, we remove the elements in ac from bc. The approach is to build
- // a mask of the elements to remove, and then apply it to the bitmap.
- wordIdx := uint16(0)
- mask := uint64(0)
- for i, v := range ac.content {
- if v/64 != wordIdx {
- // Flush the current word.
- if i != 0 {
- // We're removing bits that are set in the mask and in the current word.
- // To figure out the cardinality change, we count the number of bits that
- // are set in the mask and in the current word.
- mask &= bc.bitmap[wordIdx]
- bc.bitmap[wordIdx] &= ^mask
- bc.cardinality -= int(popcount(mask))
- }
-
- wordIdx = v / 64
- mask = 0
- }
- mask |= 1 << (v % 64)
- }
-
- // Flush the last word.
- mask &= bc.bitmap[wordIdx]
- bc.bitmap[wordIdx] &= ^mask
- bc.cardinality -= int(popcount(mask))
-
- if bc.getCardinality() <= arrayDefaultMaxSize {
- return bc.toArrayContainer()
- }
- return bc
+ acb := ac.toBitmapContainer()
+ return bc.iandNotBitmapSurely(acb)
}
func (bc *bitmapContainer) iandNotRun16(rc *runContainer16) container {
- if rc.isEmpty() || bc.isEmpty() {
- // Nothing to do.
- return bc
- }
-
- wordRangeStart := rc.iv[0].start / 64
- wordRangeEnd := (rc.iv[len(rc.iv)-1].last()) / 64 // inclusive
-
- cardinalityChange := popcntSlice(bc.bitmap[wordRangeStart : wordRangeEnd+1]) // before cardinality - after cardinality (for word range)
-
- for _, iv := range rc.iv {
- resetBitmapRange(bc.bitmap, int(iv.start), int(iv.last())+1)
- }
-
- cardinalityChange -= popcntSlice(bc.bitmap[wordRangeStart : wordRangeEnd+1])
-
- bc.cardinality -= int(cardinalityChange)
-
- if bc.getCardinality() <= arrayDefaultMaxSize {
- return bc.toArrayContainer()
- }
- return bc
+ rcb := rc.toBitmapContainer()
+ return bc.iandNotBitmapSurely(rcb)
}
func (bc *bitmapContainer) andNotArray(value2 *arrayContainer) container {
@@ -1018,32 +912,6 @@ func (bc *bitmapContainer) loadData(arrayContainer *arrayContainer) {
}
}
-func (bc *bitmapContainer) resetTo(a container) {
- switch x := a.(type) {
- case *arrayContainer:
- fill(bc.bitmap, 0)
- bc.loadData(x)
-
- case *bitmapContainer:
- bc.cardinality = x.cardinality
- copy(bc.bitmap, x.bitmap)
-
- case *runContainer16:
- bc.cardinality = len(x.iv)
- lastEnd := 0
- for _, r := range x.iv {
- bc.cardinality += int(r.length)
- resetBitmapRange(bc.bitmap, lastEnd, int(r.start))
- lastEnd = int(r.start+r.length) + 1
- setBitmapRange(bc.bitmap, int(r.start), lastEnd)
- }
- resetBitmapRange(bc.bitmap, lastEnd, maxCapacity)
-
- default:
- panic("unsupported container type")
- }
-}
-
func (bc *bitmapContainer) toArrayContainer() *arrayContainer {
ac := &arrayContainer{}
ac.loadData(bc)
@@ -1066,23 +934,20 @@ func (bc *bitmapContainer) fillArray(container []uint16) {
}
}
-func (bc *bitmapContainer) NextSetBit(i uint) int {
- var (
- x = i / 64
- length = uint(len(bc.bitmap))
- )
- if x >= length {
+func (bc *bitmapContainer) NextSetBit(i int) int {
+ x := i / 64
+ if x >= len(bc.bitmap) {
return -1
}
w := bc.bitmap[x]
w = w >> uint(i%64)
if w != 0 {
- return int(i) + countTrailingZeros(w)
+ return i + countTrailingZeros(w)
}
x++
- for ; x < length; x++ {
+ for ; x < len(bc.bitmap); x++ {
if bc.bitmap[x] != 0 {
- return int(x*64) + countTrailingZeros(bc.bitmap[x])
+ return (x * 64) + countTrailingZeros(bc.bitmap[x])
}
}
return -1
@@ -1116,6 +981,7 @@ func (bc *bitmapContainer) PrevSetBit(i int) int {
// reference the java implementation
// https://github.com/RoaringBitmap/RoaringBitmap/blob/master/src/main/java/org/roaringbitmap/BitmapContainer.java#L875-L892
+//
func (bc *bitmapContainer) numberOfRuns() int {
if bc.cardinality == 0 {
return 0
@@ -1177,60 +1043,34 @@ func (bc *bitmapContainer) containerType() contype {
return bitmapContype
}
-func (bc *bitmapContainer) addOffset(x uint16) (container, container) {
- var low, high *bitmapContainer
-
- if bc.cardinality == 0 {
- return nil, nil
- }
-
+func (bc *bitmapContainer) addOffset(x uint16) []container {
+ low := newBitmapContainer()
+ high := newBitmapContainer()
b := uint32(x) >> 6
i := uint32(x) % 64
end := uint32(1024) - b
-
- low = newBitmapContainer()
if i == 0 {
copy(low.bitmap[b:], bc.bitmap[:end])
+ copy(high.bitmap[:b], bc.bitmap[end:])
} else {
low.bitmap[b] = bc.bitmap[0] << i
for k := uint32(1); k < end; k++ {
newval := bc.bitmap[k] << i
- newval |= bc.bitmap[k-1] >> (64 - i)
+ if newval == 0 {
+ newval = bc.bitmap[k-1] >> (64 - i)
+ }
low.bitmap[b+k] = newval
}
- }
- low.computeCardinality()
-
- if low.cardinality == bc.cardinality {
- // All elements from bc ended up in low, meaning high will be empty.
- return low, nil
- }
-
- if low.cardinality == 0 {
- // low is empty, let's reuse the container for high.
- high = low
- low = nil
- } else {
- // None of the containers will be empty, so allocate both.
- high = newBitmapContainer()
- }
-
- if i == 0 {
- copy(high.bitmap[:b], bc.bitmap[end:])
- } else {
for k := end; k < 1024; k++ {
newval := bc.bitmap[k] << i
- newval |= bc.bitmap[k-1] >> (64 - i)
+ if newval == 0 {
+ newval = bc.bitmap[k-1] >> (64 - i)
+ }
high.bitmap[k-end] = newval
}
high.bitmap[b] = bc.bitmap[1023] >> (64 - i)
}
+ low.computeCardinality()
high.computeCardinality()
-
- // Ensure proper nil interface.
- if low == nil {
- return nil, high
- }
-
- return low, high
+ return []container{low, high}
}
diff --git a/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer_gen.go b/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer_gen.go
new file mode 100644
index 0000000..9b5a465
--- /dev/null
+++ b/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer_gen.go
@@ -0,0 +1,415 @@
+package roaring
+
+// NOTE: THIS FILE WAS PRODUCED BY THE
+// MSGP CODE GENERATION TOOL (github.com/tinylib/msgp)
+// DO NOT EDIT
+
+import "github.com/tinylib/msgp/msgp"
+
+// Deprecated: DecodeMsg implements msgp.Decodable
+func (z *bitmapContainer) DecodeMsg(dc *msgp.Reader) (err error) {
+ var field []byte
+ _ = field
+ var zbzg uint32
+ zbzg, err = dc.ReadMapHeader()
+ if err != nil {
+ return
+ }
+ for zbzg > 0 {
+ zbzg--
+ field, err = dc.ReadMapKeyPtr()
+ if err != nil {
+ return
+ }
+ switch msgp.UnsafeString(field) {
+ case "cardinality":
+ z.cardinality, err = dc.ReadInt()
+ if err != nil {
+ return
+ }
+ case "bitmap":
+ var zbai uint32
+ zbai, err = dc.ReadArrayHeader()
+ if err != nil {
+ return
+ }
+ if cap(z.bitmap) >= int(zbai) {
+ z.bitmap = (z.bitmap)[:zbai]
+ } else {
+ z.bitmap = make([]uint64, zbai)
+ }
+ for zxvk := range z.bitmap {
+ z.bitmap[zxvk], err = dc.ReadUint64()
+ if err != nil {
+ return
+ }
+ }
+ default:
+ err = dc.Skip()
+ if err != nil {
+ return
+ }
+ }
+ }
+ return
+}
+
+// Deprecated: EncodeMsg implements msgp.Encodable
+func (z *bitmapContainer) EncodeMsg(en *msgp.Writer) (err error) {
+ // map header, size 2
+ // write "cardinality"
+ err = en.Append(0x82, 0xab, 0x63, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x74, 0x79)
+ if err != nil {
+ return err
+ }
+ err = en.WriteInt(z.cardinality)
+ if err != nil {
+ return
+ }
+ // write "bitmap"
+ err = en.Append(0xa6, 0x62, 0x69, 0x74, 0x6d, 0x61, 0x70)
+ if err != nil {
+ return err
+ }
+ err = en.WriteArrayHeader(uint32(len(z.bitmap)))
+ if err != nil {
+ return
+ }
+ for zxvk := range z.bitmap {
+ err = en.WriteUint64(z.bitmap[zxvk])
+ if err != nil {
+ return
+ }
+ }
+ return
+}
+
+// Deprecated: MarshalMsg implements msgp.Marshaler
+func (z *bitmapContainer) MarshalMsg(b []byte) (o []byte, err error) {
+ o = msgp.Require(b, z.Msgsize())
+ // map header, size 2
+ // string "cardinality"
+ o = append(o, 0x82, 0xab, 0x63, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x74, 0x79)
+ o = msgp.AppendInt(o, z.cardinality)
+ // string "bitmap"
+ o = append(o, 0xa6, 0x62, 0x69, 0x74, 0x6d, 0x61, 0x70)
+ o = msgp.AppendArrayHeader(o, uint32(len(z.bitmap)))
+ for zxvk := range z.bitmap {
+ o = msgp.AppendUint64(o, z.bitmap[zxvk])
+ }
+ return
+}
+
+// Deprecated: UnmarshalMsg implements msgp.Unmarshaler
+func (z *bitmapContainer) UnmarshalMsg(bts []byte) (o []byte, err error) {
+ var field []byte
+ _ = field
+ var zcmr uint32
+ zcmr, bts, err = msgp.ReadMapHeaderBytes(bts)
+ if err != nil {
+ return
+ }
+ for zcmr > 0 {
+ zcmr--
+ field, bts, err = msgp.ReadMapKeyZC(bts)
+ if err != nil {
+ return
+ }
+ switch msgp.UnsafeString(field) {
+ case "cardinality":
+ z.cardinality, bts, err = msgp.ReadIntBytes(bts)
+ if err != nil {
+ return
+ }
+ case "bitmap":
+ var zajw uint32
+ zajw, bts, err = msgp.ReadArrayHeaderBytes(bts)
+ if err != nil {
+ return
+ }
+ if cap(z.bitmap) >= int(zajw) {
+ z.bitmap = (z.bitmap)[:zajw]
+ } else {
+ z.bitmap = make([]uint64, zajw)
+ }
+ for zxvk := range z.bitmap {
+ z.bitmap[zxvk], bts, err = msgp.ReadUint64Bytes(bts)
+ if err != nil {
+ return
+ }
+ }
+ default:
+ bts, err = msgp.Skip(bts)
+ if err != nil {
+ return
+ }
+ }
+ }
+ o = bts
+ return
+}
+
+// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *bitmapContainer) Msgsize() (s int) {
+ s = 1 + 12 + msgp.IntSize + 7 + msgp.ArrayHeaderSize + (len(z.bitmap) * (msgp.Uint64Size))
+ return
+}
+
+// Deprecated: DecodeMsg implements msgp.Decodable
+func (z *bitmapContainerShortIterator) DecodeMsg(dc *msgp.Reader) (err error) {
+ var field []byte
+ _ = field
+ var zhct uint32
+ zhct, err = dc.ReadMapHeader()
+ if err != nil {
+ return
+ }
+ for zhct > 0 {
+ zhct--
+ field, err = dc.ReadMapKeyPtr()
+ if err != nil {
+ return
+ }
+ switch msgp.UnsafeString(field) {
+ case "ptr":
+ if dc.IsNil() {
+ err = dc.ReadNil()
+ if err != nil {
+ return
+ }
+ z.ptr = nil
+ } else {
+ if z.ptr == nil {
+ z.ptr = new(bitmapContainer)
+ }
+ var zcua uint32
+ zcua, err = dc.ReadMapHeader()
+ if err != nil {
+ return
+ }
+ for zcua > 0 {
+ zcua--
+ field, err = dc.ReadMapKeyPtr()
+ if err != nil {
+ return
+ }
+ switch msgp.UnsafeString(field) {
+ case "cardinality":
+ z.ptr.cardinality, err = dc.ReadInt()
+ if err != nil {
+ return
+ }
+ case "bitmap":
+ var zxhx uint32
+ zxhx, err = dc.ReadArrayHeader()
+ if err != nil {
+ return
+ }
+ if cap(z.ptr.bitmap) >= int(zxhx) {
+ z.ptr.bitmap = (z.ptr.bitmap)[:zxhx]
+ } else {
+ z.ptr.bitmap = make([]uint64, zxhx)
+ }
+ for zwht := range z.ptr.bitmap {
+ z.ptr.bitmap[zwht], err = dc.ReadUint64()
+ if err != nil {
+ return
+ }
+ }
+ default:
+ err = dc.Skip()
+ if err != nil {
+ return
+ }
+ }
+ }
+ }
+ case "i":
+ z.i, err = dc.ReadInt()
+ if err != nil {
+ return
+ }
+ default:
+ err = dc.Skip()
+ if err != nil {
+ return
+ }
+ }
+ }
+ return
+}
+
+// Deprecated: EncodeMsg implements msgp.Encodable
+func (z *bitmapContainerShortIterator) EncodeMsg(en *msgp.Writer) (err error) {
+ // map header, size 2
+ // write "ptr"
+ err = en.Append(0x82, 0xa3, 0x70, 0x74, 0x72)
+ if err != nil {
+ return err
+ }
+ if z.ptr == nil {
+ err = en.WriteNil()
+ if err != nil {
+ return
+ }
+ } else {
+ // map header, size 2
+ // write "cardinality"
+ err = en.Append(0x82, 0xab, 0x63, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x74, 0x79)
+ if err != nil {
+ return err
+ }
+ err = en.WriteInt(z.ptr.cardinality)
+ if err != nil {
+ return
+ }
+ // write "bitmap"
+ err = en.Append(0xa6, 0x62, 0x69, 0x74, 0x6d, 0x61, 0x70)
+ if err != nil {
+ return err
+ }
+ err = en.WriteArrayHeader(uint32(len(z.ptr.bitmap)))
+ if err != nil {
+ return
+ }
+ for zwht := range z.ptr.bitmap {
+ err = en.WriteUint64(z.ptr.bitmap[zwht])
+ if err != nil {
+ return
+ }
+ }
+ }
+ // write "i"
+ err = en.Append(0xa1, 0x69)
+ if err != nil {
+ return err
+ }
+ err = en.WriteInt(z.i)
+ if err != nil {
+ return
+ }
+ return
+}
+
+// Deprecated: MarshalMsg implements msgp.Marshaler
+func (z *bitmapContainerShortIterator) MarshalMsg(b []byte) (o []byte, err error) {
+ o = msgp.Require(b, z.Msgsize())
+ // map header, size 2
+ // string "ptr"
+ o = append(o, 0x82, 0xa3, 0x70, 0x74, 0x72)
+ if z.ptr == nil {
+ o = msgp.AppendNil(o)
+ } else {
+ // map header, size 2
+ // string "cardinality"
+ o = append(o, 0x82, 0xab, 0x63, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x74, 0x79)
+ o = msgp.AppendInt(o, z.ptr.cardinality)
+ // string "bitmap"
+ o = append(o, 0xa6, 0x62, 0x69, 0x74, 0x6d, 0x61, 0x70)
+ o = msgp.AppendArrayHeader(o, uint32(len(z.ptr.bitmap)))
+ for zwht := range z.ptr.bitmap {
+ o = msgp.AppendUint64(o, z.ptr.bitmap[zwht])
+ }
+ }
+ // string "i"
+ o = append(o, 0xa1, 0x69)
+ o = msgp.AppendInt(o, z.i)
+ return
+}
+
+// Deprecated: UnmarshalMsg implements msgp.Unmarshaler
+func (z *bitmapContainerShortIterator) UnmarshalMsg(bts []byte) (o []byte, err error) {
+ var field []byte
+ _ = field
+ var zlqf uint32
+ zlqf, bts, err = msgp.ReadMapHeaderBytes(bts)
+ if err != nil {
+ return
+ }
+ for zlqf > 0 {
+ zlqf--
+ field, bts, err = msgp.ReadMapKeyZC(bts)
+ if err != nil {
+ return
+ }
+ switch msgp.UnsafeString(field) {
+ case "ptr":
+ if msgp.IsNil(bts) {
+ bts, err = msgp.ReadNilBytes(bts)
+ if err != nil {
+ return
+ }
+ z.ptr = nil
+ } else {
+ if z.ptr == nil {
+ z.ptr = new(bitmapContainer)
+ }
+ var zdaf uint32
+ zdaf, bts, err = msgp.ReadMapHeaderBytes(bts)
+ if err != nil {
+ return
+ }
+ for zdaf > 0 {
+ zdaf--
+ field, bts, err = msgp.ReadMapKeyZC(bts)
+ if err != nil {
+ return
+ }
+ switch msgp.UnsafeString(field) {
+ case "cardinality":
+ z.ptr.cardinality, bts, err = msgp.ReadIntBytes(bts)
+ if err != nil {
+ return
+ }
+ case "bitmap":
+ var zpks uint32
+ zpks, bts, err = msgp.ReadArrayHeaderBytes(bts)
+ if err != nil {
+ return
+ }
+ if cap(z.ptr.bitmap) >= int(zpks) {
+ z.ptr.bitmap = (z.ptr.bitmap)[:zpks]
+ } else {
+ z.ptr.bitmap = make([]uint64, zpks)
+ }
+ for zwht := range z.ptr.bitmap {
+ z.ptr.bitmap[zwht], bts, err = msgp.ReadUint64Bytes(bts)
+ if err != nil {
+ return
+ }
+ }
+ default:
+ bts, err = msgp.Skip(bts)
+ if err != nil {
+ return
+ }
+ }
+ }
+ }
+ case "i":
+ z.i, bts, err = msgp.ReadIntBytes(bts)
+ if err != nil {
+ return
+ }
+ default:
+ bts, err = msgp.Skip(bts)
+ if err != nil {
+ return
+ }
+ }
+ }
+ o = bts
+ return
+}
+
+// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *bitmapContainerShortIterator) Msgsize() (s int) {
+ s = 1 + 4
+ if z.ptr == nil {
+ s += msgp.NilSize
+ } else {
+ s += 1 + 12 + msgp.IntSize + 7 + msgp.ArrayHeaderSize + (len(z.ptr.bitmap) * (msgp.Uint64Size))
+ }
+ s += 2 + msgp.IntSize
+ return
+}
diff --git a/vendor/github.com/RoaringBitmap/roaring/clz.go b/vendor/github.com/RoaringBitmap/roaring/clz.go
index ee0ebc6..bcd80d3 100644
--- a/vendor/github.com/RoaringBitmap/roaring/clz.go
+++ b/vendor/github.com/RoaringBitmap/roaring/clz.go
@@ -1,6 +1,4 @@
-//go:build go1.9
// +build go1.9
-
// "go1.9", from Go version 1.9 onward
// See https://golang.org/pkg/go/build/#hdr-Build_Constraints
diff --git a/vendor/github.com/RoaringBitmap/roaring/clz_compat.go b/vendor/github.com/RoaringBitmap/roaring/clz_compat.go
index 7ee16b4..eeef4de 100644
--- a/vendor/github.com/RoaringBitmap/roaring/clz_compat.go
+++ b/vendor/github.com/RoaringBitmap/roaring/clz_compat.go
@@ -1,4 +1,3 @@
-//go:build !go1.9
// +build !go1.9
package roaring
diff --git a/vendor/github.com/RoaringBitmap/roaring/ctz.go b/vendor/github.com/RoaringBitmap/roaring/ctz.go
index fbcfe91..e399ddd 100644
--- a/vendor/github.com/RoaringBitmap/roaring/ctz.go
+++ b/vendor/github.com/RoaringBitmap/roaring/ctz.go
@@ -1,6 +1,4 @@
-//go:build go1.9
// +build go1.9
-
// "go1.9", from Go version 1.9 onward
// See https://golang.org/pkg/go/build/#hdr-Build_Constraints
diff --git a/vendor/github.com/RoaringBitmap/roaring/ctz_compat.go b/vendor/github.com/RoaringBitmap/roaring/ctz_compat.go
index d01df82..80220e6 100644
--- a/vendor/github.com/RoaringBitmap/roaring/ctz_compat.go
+++ b/vendor/github.com/RoaringBitmap/roaring/ctz_compat.go
@@ -1,4 +1,3 @@
-//go:build !go1.9
// +build !go1.9
package roaring
diff --git a/vendor/github.com/RoaringBitmap/roaring/fastaggregation.go b/vendor/github.com/RoaringBitmap/roaring/fastaggregation.go
index 7d0a92f..762e500 100644
--- a/vendor/github.com/RoaringBitmap/roaring/fastaggregation.go
+++ b/vendor/github.com/RoaringBitmap/roaring/fastaggregation.go
@@ -33,6 +33,15 @@ main:
s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
} else {
c1 := x1.highlowcontainer.getContainerAtIndex(pos1)
+ switch t := c1.(type) {
+ case *arrayContainer:
+ c1 = t.toBitmapContainer()
+ case *runContainer16:
+ if !t.isFull() {
+ c1 = t.toBitmapContainer()
+ }
+ }
+
answer.highlowcontainer.appendContainer(s1, c1.lazyOR(x2.highlowcontainer.getContainerAtIndex(pos2)), false)
pos1++
pos2++
@@ -80,7 +89,18 @@ main:
}
s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
} else {
- c1 := x1.highlowcontainer.getWritableContainerAtIndex(pos1)
+ c1 := x1.highlowcontainer.getContainerAtIndex(pos1)
+ switch t := c1.(type) {
+ case *arrayContainer:
+ c1 = t.toBitmapContainer()
+ case *runContainer16:
+ if !t.isFull() {
+ c1 = t.toBitmapContainer()
+ }
+ case *bitmapContainer:
+ c1 = x1.highlowcontainer.getWritableContainerAtIndex(pos1)
+ }
+
x1.highlowcontainer.containers[pos1] = c1.lazyIOR(x2.highlowcontainer.getContainerAtIndex(pos2))
x1.highlowcontainer.needCopyOnWrite[pos1] = false
pos1++
@@ -121,10 +141,6 @@ func (x1 *Bitmap) repairAfterLazy() {
// FastAnd computes the intersection between many bitmaps quickly
// Compared to the And function, it can take many bitmaps as input, thus saving the trouble
// of manually calling "And" many times.
-//
-// Performance hints: if you have very large and tiny bitmaps,
-// it may be beneficial performance-wise to put a tiny bitmap
-// in first position.
func FastAnd(bitmaps ...*Bitmap) *Bitmap {
if len(bitmaps) == 0 {
return NewBitmap()
@@ -197,117 +213,3 @@ func HeapXor(bitmaps ...*Bitmap) *Bitmap {
}
return heap.Pop(&pq).(*item).value
}
-
-// AndAny provides a result equivalent to x1.And(FastOr(bitmaps)).
-// It's optimized to minimize allocations. It also might be faster than separate calls.
-func (x1 *Bitmap) AndAny(bitmaps ...*Bitmap) {
- if len(bitmaps) == 0 {
- return
- } else if len(bitmaps) == 1 {
- x1.And(bitmaps[0])
- return
- }
-
- type withPos struct {
- bitmap *roaringArray
- pos int
- key uint16
- }
- filters := make([]withPos, 0, len(bitmaps))
-
- for _, b := range bitmaps {
- if b.highlowcontainer.size() > 0 {
- filters = append(filters, withPos{
- bitmap: &b.highlowcontainer,
- pos: 0,
- key: b.highlowcontainer.getKeyAtIndex(0),
- })
- }
- }
-
- basePos := 0
- intersections := 0
- keyContainers := make([]container, 0, len(filters))
- var (
- tmpArray *arrayContainer
- tmpBitmap *bitmapContainer
- minNextKey uint16
- )
-
- for basePos < x1.highlowcontainer.size() && len(filters) > 0 {
- baseKey := x1.highlowcontainer.getKeyAtIndex(basePos)
-
- // accumulate containers for current key, find next minimal key in filters
- // and exclude filters that do not have related values anymore
- i := 0
- maxPossibleOr := 0
- minNextKey = MaxUint16
- for _, f := range filters {
- if f.key < baseKey {
- f.pos = f.bitmap.advanceUntil(baseKey, f.pos)
- if f.pos == f.bitmap.size() {
- continue
- }
- f.key = f.bitmap.getKeyAtIndex(f.pos)
- }
-
- if f.key == baseKey {
- cont := f.bitmap.getContainerAtIndex(f.pos)
- keyContainers = append(keyContainers, cont)
- maxPossibleOr += cont.getCardinality()
-
- f.pos++
- if f.pos == f.bitmap.size() {
- continue
- }
- f.key = f.bitmap.getKeyAtIndex(f.pos)
- }
-
- minNextKey = minOfUint16(minNextKey, f.key)
- filters[i] = f
- i++
- }
- filters = filters[:i]
-
- if len(keyContainers) == 0 {
- basePos = x1.highlowcontainer.advanceUntil(minNextKey, basePos)
- continue
- }
-
- var ored container
-
- if len(keyContainers) == 1 {
- ored = keyContainers[0]
- } else {
- //TODO: special case for run containers?
- if maxPossibleOr > arrayDefaultMaxSize {
- if tmpBitmap == nil {
- tmpBitmap = newBitmapContainer()
- }
- tmpBitmap.resetTo(keyContainers[0])
- ored = tmpBitmap
- } else {
- if tmpArray == nil {
- tmpArray = newArrayContainerCapacity(maxPossibleOr)
- }
- tmpArray.realloc(maxPossibleOr)
- tmpArray.resetTo(keyContainers[0])
- ored = tmpArray
- }
- for _, c := range keyContainers[1:] {
- ored = ored.ior(c)
- }
- }
-
- result := x1.highlowcontainer.getWritableContainerAtIndex(basePos).iand(ored)
- if !result.isEmpty() {
- x1.highlowcontainer.replaceKeyAndContainerAtIndex(intersections, baseKey, result, false)
- intersections++
- }
-
- keyContainers = keyContainers[:0]
- basePos = x1.highlowcontainer.advanceUntil(minNextKey, basePos)
- }
-
- x1.highlowcontainer.resize(intersections)
-}
diff --git a/vendor/github.com/RoaringBitmap/roaring/go.mod b/vendor/github.com/RoaringBitmap/roaring/go.mod
new file mode 100644
index 0000000..95406c9
--- /dev/null
+++ b/vendor/github.com/RoaringBitmap/roaring/go.mod
@@ -0,0 +1,15 @@
+module github.com/RoaringBitmap/roaring
+
+go 1.12
+
+require (
+ github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2
+ github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493 // indirect
+ github.com/golang/snappy v0.0.1 // indirect
+ github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae
+ github.com/philhofer/fwd v1.0.0 // indirect
+ github.com/smartystreets/goconvey v0.0.0-20190306220146-200a235640ff
+ github.com/stretchr/testify v1.3.0
+ github.com/tinylib/msgp v1.1.0
+ github.com/willf/bitset v1.1.10
+)
diff --git a/vendor/github.com/RoaringBitmap/roaring/go.sum b/vendor/github.com/RoaringBitmap/roaring/go.sum
new file mode 100644
index 0000000..3dcbb91
--- /dev/null
+++ b/vendor/github.com/RoaringBitmap/roaring/go.sum
@@ -0,0 +1,29 @@
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2 h1:Ujru1hufTHVb++eG6OuNDKMxZnGIvF6o/u8q/8h2+I4=
+github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE=
+github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493 h1:OTanQnFt0bi5iLFSdbEVA/idR6Q2WhCm+deb7ir2CcM=
+github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae h1:VeRdUYdCw49yizlSbMEn2SZ+gT+3IUKx8BqxyQdz+BY=
+github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg=
+github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ=
+github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v0.0.0-20190306220146-200a235640ff h1:86HlEv0yBCry9syNuylzqznKXDK11p6D0DT596yNMys=
+github.com/smartystreets/goconvey v0.0.0-20190306220146-200a235640ff/go.mod h1:KSQcGKpxUMHk3nbYzs/tIBAM2iDooCn0BmttHOJEbLs=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/tinylib/msgp v1.1.0 h1:9fQd+ICuRIu/ue4vxJZu6/LzxN0HwMds2nq/0cFvxHU=
+github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
+github.com/willf/bitset v1.1.10 h1:NotGKqX0KwQ72NUzqrjZq5ipPNDQex9lo3WpaS8L2sc=
+github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
diff --git a/vendor/github.com/RoaringBitmap/roaring/internal/byte_input.go b/vendor/github.com/RoaringBitmap/roaring/internal/byte_input.go
deleted file mode 100644
index d5ebb91..0000000
--- a/vendor/github.com/RoaringBitmap/roaring/internal/byte_input.go
+++ /dev/null
@@ -1,215 +0,0 @@
-package internal
-
-import (
- "encoding/binary"
- "io"
-)
-
-// ByteInput typed interface around io.Reader or raw bytes
-type ByteInput interface {
- // Next returns a slice containing the next n bytes from the buffer,
- // advancing the buffer as if the bytes had been returned by Read.
- Next(n int) ([]byte, error)
- // NextReturnsSafeSlice returns true if Next() returns a safe slice as opposed
- // to a slice that points to an underlying buffer possibly owned by another system.
- // When NextReturnsSafeSlice returns false, the result from Next() should be copied
- // before it is modified (i.e., it is immutable).
- NextReturnsSafeSlice() bool
- // ReadUInt32 reads uint32 with LittleEndian order
- ReadUInt32() (uint32, error)
- // ReadUInt16 reads uint16 with LittleEndian order
- ReadUInt16() (uint16, error)
- // GetReadBytes returns read bytes
- GetReadBytes() int64
- // SkipBytes skips exactly n bytes
- SkipBytes(n int) error
-}
-
-// NewByteInputFromReader creates reader wrapper
-func NewByteInputFromReader(reader io.Reader) ByteInput {
- return &ByteInputAdapter{
- r: reader,
- readBytes: 0,
- }
-}
-
-// NewByteInput creates raw bytes wrapper
-func NewByteInput(buf []byte) ByteInput {
- return &ByteBuffer{
- buf: buf,
- off: 0,
- }
-}
-
-// ByteBuffer raw bytes wrapper
-type ByteBuffer struct {
- buf []byte
- off int
-}
-
-// NewByteBuffer creates a new ByteBuffer.
-func NewByteBuffer(buf []byte) *ByteBuffer {
- return &ByteBuffer{
- buf: buf,
- }
-}
-
-var _ io.Reader = (*ByteBuffer)(nil)
-
-// Read implements io.Reader.
-func (b *ByteBuffer) Read(p []byte) (int, error) {
- data, err := b.Next(len(p))
- if err != nil {
- return 0, err
- }
- copy(p, data)
- return len(data), nil
-}
-
-// Next returns a slice containing the next n bytes from the reader
-// If there are fewer bytes than the given n, io.ErrUnexpectedEOF will be returned
-func (b *ByteBuffer) Next(n int) ([]byte, error) {
- m := len(b.buf) - b.off
-
- if n > m {
- return nil, io.ErrUnexpectedEOF
- }
-
- data := b.buf[b.off : b.off+n]
- b.off += n
-
- return data, nil
-}
-
-// NextReturnsSafeSlice returns false since ByteBuffer might hold
-// an array owned by some other systems.
-func (b *ByteBuffer) NextReturnsSafeSlice() bool {
- return false
-}
-
-// ReadUInt32 reads uint32 with LittleEndian order
-func (b *ByteBuffer) ReadUInt32() (uint32, error) {
- if len(b.buf)-b.off < 4 {
- return 0, io.ErrUnexpectedEOF
- }
-
- v := binary.LittleEndian.Uint32(b.buf[b.off:])
- b.off += 4
-
- return v, nil
-}
-
-// ReadUInt16 reads uint16 with LittleEndian order
-func (b *ByteBuffer) ReadUInt16() (uint16, error) {
- if len(b.buf)-b.off < 2 {
- return 0, io.ErrUnexpectedEOF
- }
-
- v := binary.LittleEndian.Uint16(b.buf[b.off:])
- b.off += 2
-
- return v, nil
-}
-
-// GetReadBytes returns read bytes
-func (b *ByteBuffer) GetReadBytes() int64 {
- return int64(b.off)
-}
-
-// SkipBytes skips exactly n bytes
-func (b *ByteBuffer) SkipBytes(n int) error {
- m := len(b.buf) - b.off
-
- if n > m {
- return io.ErrUnexpectedEOF
- }
-
- b.off += n
-
- return nil
-}
-
-// Reset resets the given buffer with a new byte slice
-func (b *ByteBuffer) Reset(buf []byte) {
- b.buf = buf
- b.off = 0
-}
-
-// ByteInputAdapter reader wrapper
-type ByteInputAdapter struct {
- r io.Reader
- readBytes int
- buf [4]byte
-}
-
-var _ io.Reader = (*ByteInputAdapter)(nil)
-
-// Read implements io.Reader.
-func (b *ByteInputAdapter) Read(buf []byte) (int, error) {
- m, err := io.ReadAtLeast(b.r, buf, len(buf))
- b.readBytes += m
-
- if err != nil {
- return 0, err
- }
-
- return m, nil
-}
-
-// Next returns a slice containing the next n bytes from the buffer,
-// advancing the buffer as if the bytes had been returned by Read.
-func (b *ByteInputAdapter) Next(n int) ([]byte, error) {
- buf := make([]byte, n)
- _, err := b.Read(buf)
-
- if err != nil {
- return nil, err
- }
- return buf, nil
-}
-
-// NextReturnsSafeSlice returns true since ByteInputAdapter always returns a slice
-// allocated with make([]byte, ...)
-func (b *ByteInputAdapter) NextReturnsSafeSlice() bool {
- return true
-}
-
-// ReadUInt32 reads uint32 with LittleEndian order
-func (b *ByteInputAdapter) ReadUInt32() (uint32, error) {
- buf := b.buf[:4]
- _, err := b.Read(buf)
- if err != nil {
- return 0, err
- }
-
- return binary.LittleEndian.Uint32(buf), nil
-}
-
-// ReadUInt16 reads uint16 with LittleEndian order
-func (b *ByteInputAdapter) ReadUInt16() (uint16, error) {
- buf := b.buf[:2]
- _, err := b.Read(buf)
- if err != nil {
- return 0, err
- }
-
- return binary.LittleEndian.Uint16(buf), nil
-}
-
-// GetReadBytes returns read bytes
-func (b *ByteInputAdapter) GetReadBytes() int64 {
- return int64(b.readBytes)
-}
-
-// SkipBytes skips exactly n bytes
-func (b *ByteInputAdapter) SkipBytes(n int) error {
- _, err := b.Next(n)
-
- return err
-}
-
-// Reset resets the given buffer with a new stream
-func (b *ByteInputAdapter) Reset(stream io.Reader) {
- b.r = stream
- b.readBytes = 0
-}
diff --git a/vendor/github.com/RoaringBitmap/roaring/internal/pools.go b/vendor/github.com/RoaringBitmap/roaring/internal/pools.go
deleted file mode 100644
index d258356..0000000
--- a/vendor/github.com/RoaringBitmap/roaring/internal/pools.go
+++ /dev/null
@@ -1,21 +0,0 @@
-package internal
-
-import (
- "sync"
-)
-
-var (
- // ByteInputAdapterPool shared pool
- ByteInputAdapterPool = sync.Pool{
- New: func() interface{} {
- return &ByteInputAdapter{}
- },
- }
-
- // ByteBufferPool shared pool
- ByteBufferPool = sync.Pool{
- New: func() interface{} {
- return &ByteBuffer{}
- },
- }
-)
diff --git a/vendor/github.com/RoaringBitmap/roaring/manyiterator.go b/vendor/github.com/RoaringBitmap/roaring/manyiterator.go
index eaa5b79..b4f630a 100644
--- a/vendor/github.com/RoaringBitmap/roaring/manyiterator.go
+++ b/vendor/github.com/RoaringBitmap/roaring/manyiterator.go
@@ -2,10 +2,14 @@ package roaring
type manyIterable interface {
nextMany(hs uint32, buf []uint32) int
- nextMany64(hs uint64, buf []uint64) int
}
-func (si *shortIterator) nextMany(hs uint32, buf []uint32) int {
+type manyIterator struct {
+ slice []uint16
+ loc int
+}
+
+func (si *manyIterator) nextMany(hs uint32, buf []uint32) int {
n := 0
l := si.loc
s := si.slice
@@ -17,16 +21,3 @@ func (si *shortIterator) nextMany(hs uint32, buf []uint32) int {
si.loc = l
return n
}
-
-func (si *shortIterator) nextMany64(hs uint64, buf []uint64) int {
- n := 0
- l := si.loc
- s := si.slice
- for n < len(buf) && l < len(s) {
- buf[n] = uint64(s[l]) | hs
- l++
- n++
- }
- si.loc = l
- return n
-}
diff --git a/vendor/github.com/RoaringBitmap/roaring/parallel.go b/vendor/github.com/RoaringBitmap/roaring/parallel.go
index 9208e3e..2af1aed 100644
--- a/vendor/github.com/RoaringBitmap/roaring/parallel.go
+++ b/vendor/github.com/RoaringBitmap/roaring/parallel.go
@@ -166,6 +166,7 @@ func appenderRoutine(bitmapChan chan<- *Bitmap, resultChan <-chan keyedContainer
make([]container, 0, expectedKeys),
make([]bool, 0, expectedKeys),
false,
+ nil,
},
}
for i := range keys {
@@ -285,14 +286,14 @@ func ParAnd(parallelism int, bitmaps ...*Bitmap) *Bitmap {
for input := range inputChan {
c := input.containers[0].and(input.containers[1])
for _, next := range input.containers[2:] {
- if c.isEmpty() {
+ if c.getCardinality() == 0 {
break
}
c = c.iand(next)
}
// Send a nil explicitly if the result of the intersection is an empty container
- if c.isEmpty() {
+ if c.getCardinality() == 0 {
c = nil
}
@@ -354,10 +355,10 @@ func ParOr(parallelism int, bitmaps ...*Bitmap) *Bitmap {
if lKey == MaxUint16 && hKey == 0 {
return New()
} else if len(bitmaps) == 1 {
- return bitmaps[0].Clone()
+ return bitmaps[0]
}
- keyRange := int(hKey) - int(lKey) + 1
+ keyRange := hKey - lKey + 1
if keyRange == 1 {
// revert to FastOr. Since the key range is 0
// no container-level aggregation parallelism is achievable
diff --git a/vendor/github.com/RoaringBitmap/roaring/popcnt.go b/vendor/github.com/RoaringBitmap/roaring/popcnt.go
index b4980aa..9d99508 100644
--- a/vendor/github.com/RoaringBitmap/roaring/popcnt.go
+++ b/vendor/github.com/RoaringBitmap/roaring/popcnt.go
@@ -1,6 +1,4 @@
-//go:build go1.9
// +build go1.9
-
// "go1.9", from Go version 1.9 onward
// See https://golang.org/pkg/go/build/#hdr-Build_Constraints
diff --git a/vendor/github.com/RoaringBitmap/roaring/popcnt_asm.go b/vendor/github.com/RoaringBitmap/roaring/popcnt_asm.go
index ba2dac9..882d7f4 100644
--- a/vendor/github.com/RoaringBitmap/roaring/popcnt_asm.go
+++ b/vendor/github.com/RoaringBitmap/roaring/popcnt_asm.go
@@ -1,4 +1,3 @@
-//go:build amd64 && !appengine && !go1.9
// +build amd64,!appengine,!go1.9
package roaring
diff --git a/vendor/github.com/RoaringBitmap/roaring/popcnt_compat.go b/vendor/github.com/RoaringBitmap/roaring/popcnt_compat.go
index 5933e52..7ae82d4 100644
--- a/vendor/github.com/RoaringBitmap/roaring/popcnt_compat.go
+++ b/vendor/github.com/RoaringBitmap/roaring/popcnt_compat.go
@@ -1,4 +1,3 @@
-//go:build !go1.9
// +build !go1.9
package roaring
diff --git a/vendor/github.com/RoaringBitmap/roaring/popcnt_generic.go b/vendor/github.com/RoaringBitmap/roaring/popcnt_generic.go
index 4ae6d5a..edf2083 100644
--- a/vendor/github.com/RoaringBitmap/roaring/popcnt_generic.go
+++ b/vendor/github.com/RoaringBitmap/roaring/popcnt_generic.go
@@ -1,4 +1,3 @@
-//go:build !amd64 || appengine || go1.9
// +build !amd64 appengine go1.9
package roaring
diff --git a/vendor/github.com/RoaringBitmap/roaring/roaring.go b/vendor/github.com/RoaringBitmap/roaring/roaring.go
index a31cdbd..02daef9 100644
--- a/vendor/github.com/RoaringBitmap/roaring/roaring.go
+++ b/vendor/github.com/RoaringBitmap/roaring/roaring.go
@@ -6,14 +6,12 @@
package roaring
import (
+ "bufio"
"bytes"
"encoding/base64"
"fmt"
"io"
"strconv"
-
- "github.com/RoaringBitmap/roaring/internal"
- "github.com/bits-and-blooms/bitset"
)
// Bitmap represents a compressed bitmap where you can add integers.
@@ -54,263 +52,23 @@ func (rb *Bitmap) ToBytes() ([]byte, error) {
return rb.highlowcontainer.toBytes()
}
-const wordSize = uint64(64)
-const log2WordSize = uint64(6)
-const capacity = ^uint64(0)
-const bitmapContainerSize = (1 << 16) / 64 // bitmap size in words
-
-// DenseSize returns the size of the bitmap when stored as a dense bitmap.
-func (rb *Bitmap) DenseSize() uint64 {
- if rb.highlowcontainer.size() == 0 {
- return 0
- }
-
- maximum := 1 + uint64(rb.Maximum())
- if maximum > (capacity - wordSize + 1) {
- return uint64(capacity >> log2WordSize)
- }
-
- return uint64((maximum + (wordSize - 1)) >> log2WordSize)
-}
-
-// ToDense returns a slice of uint64s representing the bitmap as a dense bitmap.
-// Useful to convert a roaring bitmap to a format that can be used by other libraries
-// like https://github.com/bits-and-blooms/bitset or https://github.com/kelindar/bitmap
-func (rb *Bitmap) ToDense() []uint64 {
- sz := rb.DenseSize()
- if sz == 0 {
- return nil
- }
-
- bitmap := make([]uint64, sz)
- rb.WriteDenseTo(bitmap)
- return bitmap
-}
-
-// FromDense creates a bitmap from a slice of uint64s representing the bitmap as a dense bitmap.
-// Useful to convert bitmaps from libraries like https://github.com/bits-and-blooms/bitset or
-// https://github.com/kelindar/bitmap into roaring bitmaps fast and with convenience.
-//
-// This function will not create any run containers, only array and bitmap containers. It's up to
-// the caller to call RunOptimize if they want to further compress the runs of consecutive values.
-//
-// When doCopy is true, the bitmap is copied into a new slice for each bitmap container.
-// This is useful when the bitmap is going to be modified after this function returns or if it's
-// undesirable to hold references to large bitmaps which the GC would not be able to collect.
-// One copy can still happen even when doCopy is false if the bitmap length is not divisible
-// by bitmapContainerSize.
-//
-// See also FromBitSet.
-func FromDense(bitmap []uint64, doCopy bool) *Bitmap {
- sz := (len(bitmap) + bitmapContainerSize - 1) / bitmapContainerSize // round up
- rb := &Bitmap{
- highlowcontainer: roaringArray{
- containers: make([]container, 0, sz),
- keys: make([]uint16, 0, sz),
- needCopyOnWrite: make([]bool, 0, sz),
- },
- }
- rb.FromDense(bitmap, doCopy)
- return rb
-}
-
-// FromDense unmarshalls from a slice of uint64s representing the bitmap as a dense bitmap.
-// Useful to convert bitmaps from libraries like https://github.com/bits-and-blooms/bitset or
-// https://github.com/kelindar/bitmap into roaring bitmaps fast and with convenience.
-// Callers are responsible for ensuring that the bitmap is empty before calling this function.
-//
-// This function will not create any run containers, only array and bitmap containers. It is up to
-// the caller to call RunOptimize if they want to further compress the runs of consecutive values.
-//
-// When doCopy is true, the bitmap is copied into a new slice for each bitmap container.
-// This is useful when the bitmap is going to be modified after this function returns or if it's
-// undesirable to hold references to large bitmaps which the GC would not be able to collect.
-// One copy can still happen even when doCopy is false if the bitmap length is not divisible
-// by bitmapContainerSize.
-//
-// See FromBitSet.
-func (rb *Bitmap) FromDense(bitmap []uint64, doCopy bool) {
- if len(bitmap) == 0 {
- return
- }
-
- var k uint16
- const size = bitmapContainerSize
-
- for len(bitmap) > 0 {
- hi := size
- if len(bitmap) < size {
- hi = len(bitmap)
- }
-
- words := bitmap[:hi]
- count := int(popcntSlice(words))
-
- switch {
- case count > arrayDefaultMaxSize:
- c := &bitmapContainer{cardinality: count, bitmap: words}
- cow := true
-
- if doCopy || len(words) < size {
- c.bitmap = make([]uint64, size)
- copy(c.bitmap, words)
- cow = false
- }
-
- rb.highlowcontainer.appendContainer(k, c, cow)
-
- case count > 0:
- c := &arrayContainer{content: make([]uint16, count)}
- var pos, base int
- for _, w := range words {
- for w != 0 {
- t := w & -w
- c.content[pos] = uint16(base + int(popcount(t-1)))
- pos++
- w ^= t
- }
- base += 64
- }
- rb.highlowcontainer.appendContainer(k, c, false)
- }
-
- bitmap = bitmap[hi:]
- k++
- }
-}
-
-// WriteDenseTo writes to a slice of uint64s representing the bitmap as a dense bitmap.
-// Callers are responsible for allocating enough space in the bitmap using DenseSize.
-// Useful to convert a roaring bitmap to a format that can be used by other libraries
-// like https://github.com/bits-and-blooms/bitset or https://github.com/kelindar/bitmap
-func (rb *Bitmap) WriteDenseTo(bitmap []uint64) {
- for i, ct := range rb.highlowcontainer.containers {
- hb := uint32(rb.highlowcontainer.keys[i]) << 16
-
- switch c := ct.(type) {
- case *arrayContainer:
- for _, x := range c.content {
- n := int(hb | uint32(x))
- bitmap[n>>log2WordSize] |= uint64(1) << uint(x%64)
- }
-
- case *bitmapContainer:
- copy(bitmap[int(hb)>>log2WordSize:], c.bitmap)
-
- case *runContainer16:
- for j := range c.iv {
- start := uint32(c.iv[j].start)
- end := start + uint32(c.iv[j].length) + 1
- lo := int(hb|start) >> log2WordSize
- hi := int(hb|(end-1)) >> log2WordSize
-
- if lo == hi {
- bitmap[lo] |= (^uint64(0) << uint(start%64)) &
- (^uint64(0) >> (uint(-end) % 64))
- continue
- }
-
- bitmap[lo] |= ^uint64(0) << uint(start%64)
- for n := lo + 1; n < hi; n++ {
- bitmap[n] = ^uint64(0)
- }
- bitmap[hi] |= ^uint64(0) >> (uint(-end) % 64)
- }
- default:
- panic("unsupported container type")
- }
- }
-}
-
-// Checksum computes a hash (currently FNV-1a) for a bitmap that is suitable for
-// using bitmaps as elements in hash sets or as keys in hash maps, as well as
-// generally quicker comparisons.
-// The implementation is biased towards efficiency in little endian machines, so
-// expect some extra CPU cycles and memory to be used if your machine is big endian.
-// Likewise, do not use this to verify integrity unless you are certain you will load
-// the bitmap on a machine with the same endianess used to create it. (Thankfully
-// very few people use big endian machines these days.)
-func (rb *Bitmap) Checksum() uint64 {
- const (
- offset = 14695981039346656037
- prime = 1099511628211
- )
-
- var bytes []byte
-
- hash := uint64(offset)
-
- bytes = uint16SliceAsByteSlice(rb.highlowcontainer.keys)
-
- for _, b := range bytes {
- hash ^= uint64(b)
- hash *= prime
- }
-
- for _, c := range rb.highlowcontainer.containers {
- // 0 separator
- hash ^= 0
- hash *= prime
-
- switch c := c.(type) {
- case *bitmapContainer:
- bytes = uint64SliceAsByteSlice(c.bitmap)
- case *arrayContainer:
- bytes = uint16SliceAsByteSlice(c.content)
- case *runContainer16:
- bytes = interval16SliceAsByteSlice(c.iv)
- default:
- panic("invalid container type")
- }
-
- if len(bytes) == 0 {
- panic("empty containers are not supported")
- }
-
- for _, b := range bytes {
- hash ^= uint64(b)
- hash *= prime
- }
- }
-
- return hash
-}
-
-// FromUnsafeBytes reads a serialized version of this bitmap from the byte buffer without copy.
-// It is the caller's responsibility to ensure that the input data is not modified and remains valid for the entire lifetime of this bitmap.
-// This method avoids small allocations but holds references to the input data buffer. It is GC-friendly, but it may consume more memory eventually.
-// The containers in the resulting bitmap are immutable containers tied to the provided byte array and they rely on
-// copy-on-write which means that modifying them creates copies. Thus FromUnsafeBytes is more likely to be appropriate for read-only use cases,
-// when the resulting bitmap can be considered immutable.
-//
-// See also the FromBuffer function.
-// See https://github.com/RoaringBitmap/roaring/pull/395 for more details.
-func (rb *Bitmap) FromUnsafeBytes(data []byte, cookieHeader ...byte) (p int64, err error) {
- stream := internal.NewByteBuffer(data)
- return rb.ReadFrom(stream)
+// Deprecated: WriteToMsgpack writes a msgpack2/snappy-streaming compressed serialized
+// version of this bitmap to stream. The format is not
+// compatible with the WriteTo() format, and is
+// experimental: it may produce smaller on disk
+// footprint and/or be faster to read, depending
+// on your content. Currently only the Go roaring
+// implementation supports this format.
+func (rb *Bitmap) WriteToMsgpack(stream io.Writer) (int64, error) {
+ return 0, rb.highlowcontainer.writeToMsgpack(stream)
}
// ReadFrom reads a serialized version of this bitmap from stream.
// The format is compatible with other RoaringBitmap
// implementations (Java, C) and is documented here:
// https://github.com/RoaringBitmap/RoaringFormatSpec
-// Since io.Reader is regarded as a stream and cannot be read twice.
-// So add cookieHeader to accept the 4-byte data that has been read in roaring64.ReadFrom.
-// It is not necessary to pass cookieHeader when call roaring.ReadFrom to read the roaring32 data directly.
-func (rb *Bitmap) ReadFrom(reader io.Reader, cookieHeader ...byte) (p int64, err error) {
- stream, ok := reader.(internal.ByteInput)
- if !ok {
- byteInputAdapter := internal.ByteInputAdapterPool.Get().(*internal.ByteInputAdapter)
- byteInputAdapter.Reset(reader)
- stream = byteInputAdapter
- }
-
- p, err = rb.highlowcontainer.readFrom(stream, cookieHeader...)
-
- if !ok {
- internal.ByteInputAdapterPool.Put(stream.(*internal.ByteInputAdapter))
- }
- return
+func (rb *Bitmap) ReadFrom(stream io.Reader) (int64, error) {
+ return rb.highlowcontainer.readFrom(stream)
}
// FromBuffer creates a bitmap from its serialized version stored in buffer
@@ -329,25 +87,8 @@ func (rb *Bitmap) ReadFrom(reader io.Reader, cookieHeader ...byte) (p int64, err
// You should *not* change the copy-on-write status of the resulting
// bitmaps (SetCopyOnWrite).
//
-// Thus FromBuffer is more likely to be appropriate for read-only use cases,
-// when the resulting bitmap can be considered immutable.
-//
-// If buf becomes unavailable, then a bitmap created with
-// FromBuffer would be effectively broken. Furthermore, any
-// bitmap derived from this bitmap (e.g., via Or, And) might
-// also be broken. Thus, before making buf unavailable, you should
-// call CloneCopyOnWriteContainers on all such bitmaps.
-//
-// See also the FromUnsafeBytes function which can have better performance
-// in some cases.
-func (rb *Bitmap) FromBuffer(buf []byte) (p int64, err error) {
- stream := internal.ByteBufferPool.Get().(*internal.ByteBuffer)
- stream.Reset(buf)
-
- p, err = rb.highlowcontainer.readFrom(stream)
- internal.ByteBufferPool.Put(stream)
-
- return
+func (rb *Bitmap) FromBuffer(buf []byte) (int64, error) {
+ return rb.highlowcontainer.fromBuffer(buf)
}
// RunOptimize attempts to further compress the runs of consecutive values found in the bitmap
@@ -360,16 +101,38 @@ func (rb *Bitmap) HasRunCompression() bool {
return rb.highlowcontainer.hasRunCompression()
}
+// Deprecated: ReadFromMsgpack reads a msgpack2/snappy-streaming serialized
+// version of this bitmap from stream. The format is
+// expected is that written by the WriteToMsgpack()
+// call; see additional notes there.
+func (rb *Bitmap) ReadFromMsgpack(stream io.Reader) (int64, error) {
+ return 0, rb.highlowcontainer.readFromMsgpack(stream)
+}
+
// MarshalBinary implements the encoding.BinaryMarshaler interface for the bitmap
-// (same as ToBytes)
func (rb *Bitmap) MarshalBinary() ([]byte, error) {
- return rb.ToBytes()
+ var buf bytes.Buffer
+ writer := bufio.NewWriter(&buf)
+ _, err := rb.WriteTo(writer)
+ if err != nil {
+ return nil, err
+ }
+ err = writer.Flush()
+ if err != nil {
+ return nil, err
+ }
+ return buf.Bytes(), nil
}
// UnmarshalBinary implements the encoding.BinaryUnmarshaler interface for the bitmap
func (rb *Bitmap) UnmarshalBinary(data []byte) error {
- r := bytes.NewReader(data)
- _, err := rb.ReadFrom(r)
+ var buf bytes.Buffer
+ _, err := buf.Write(data)
+ if err != nil {
+ return err
+ }
+ reader := bufio.NewReader(&buf)
+ _, err = rb.ReadFrom(reader)
return err
}
@@ -389,16 +152,6 @@ func (rb *Bitmap) Clear() {
rb.highlowcontainer.clear()
}
-// ToBitSet copies the content of the RoaringBitmap into a bitset.BitSet instance
-func (rb *Bitmap) ToBitSet() *bitset.BitSet {
- return bitset.From(rb.ToDense())
-}
-
-// FromBitSet creates a new RoaringBitmap from a bitset.BitSet instance
-func FromBitSet(bitset *bitset.BitSet) *Bitmap {
- return FromDense(bitset.Bytes(), false)
-}
-
// ToArray creates a new slice containing all of the integers stored in the Bitmap in sorted order
func (rb *Bitmap) ToArray() []uint32 {
array := make([]uint32, rb.GetCardinality())
@@ -409,7 +162,8 @@ func (rb *Bitmap) ToArray() []uint32 {
hs := uint32(rb.highlowcontainer.getKeyAtIndex(pos)) << 16
c := rb.highlowcontainer.getContainerAtIndex(pos)
pos++
- pos2 = c.fillLeastSignificant16bits(array, pos2, hs)
+ c.fillLeastSignificant16bits(array, pos2, hs)
+ pos2 += c.getCardinality()
}
return array
}
@@ -438,7 +192,7 @@ func BoundSerializedSizeInBytes(cardinality uint64, universeSize uint64) uint64
contnbr := (universeSize + uint64(65535)) / uint64(65536)
if contnbr > cardinality {
contnbr = cardinality
- // we cannot have more containers than we have values
+ // we can't have more containers than we have values
}
headermax := 8*contnbr + 4
if 4 > (contnbr+7)/8 {
@@ -461,29 +215,11 @@ type IntIterable interface {
Next() uint32
}
-// IntPeekable allows you to look at the next value without advancing and
-// advance as long as the next value is smaller than minval
-type IntPeekable interface {
- IntIterable
- // PeekNext peeks the next value without advancing the iterator
- PeekNext() uint32
- // AdvanceIfNeeded advances as long as the next value is smaller than minval
- AdvanceIfNeeded(minval uint32)
-}
-
type intIterator struct {
pos int
hs uint32
- iter shortPeekable
+ iter shortIterable
highlowcontainer *roaringArray
-
- // These embedded iterators per container type help reduce load in the GC.
- // This way, instead of making up-to 64k allocations per full iteration
- // we get a single allocation and simply reinitialize the appropriate
- // iterator and point to it in the generic `iter` member on each key bound.
- shortIter shortIterator
- runIter runIterator16
- bitmapIter bitmapContainerShortIterator
}
// HasNext returns true if there are more integers to iterate over
@@ -493,19 +229,8 @@ func (ii *intIterator) HasNext() bool {
func (ii *intIterator) init() {
if ii.highlowcontainer.size() > ii.pos {
+ ii.iter = ii.highlowcontainer.getContainerAtIndex(ii.pos).getShortIterator()
ii.hs = uint32(ii.highlowcontainer.getKeyAtIndex(ii.pos)) << 16
- c := ii.highlowcontainer.getContainerAtIndex(ii.pos)
- switch t := c.(type) {
- case *arrayContainer:
- ii.shortIter = shortIterator{t.content, 0}
- ii.iter = &ii.shortIter
- case *runContainer16:
- ii.runIter = runIterator16{rc: t, curIndex: 0, curPosInIndex: 0}
- ii.iter = &ii.runIter
- case *bitmapContainer:
- ii.bitmapIter = bitmapContainerShortIterator{t, t.NextSetBit(0)}
- ii.iter = &ii.bitmapIter
- }
}
}
@@ -519,40 +244,12 @@ func (ii *intIterator) Next() uint32 {
return x
}
-// PeekNext peeks the next value without advancing the iterator
-func (ii *intIterator) PeekNext() uint32 {
- return uint32(ii.iter.peekNext()&maxLowBit) | ii.hs
-}
-
-// AdvanceIfNeeded advances as long as the next value is smaller than minval
-func (ii *intIterator) AdvanceIfNeeded(minval uint32) {
- to := minval & 0xffff0000
-
- for ii.HasNext() && ii.hs < to {
- ii.pos++
- ii.init()
- }
-
- if ii.HasNext() && ii.hs == to {
- ii.iter.advanceIfNeeded(lowbits(minval))
-
- if !ii.iter.hasNext() {
- ii.pos++
- ii.init()
- }
- }
-}
-
-// IntIterator is meant to allow you to iterate through the values of a bitmap, see Initialize(a *Bitmap)
-type IntIterator = intIterator
-
-// Initialize configures the existing iterator so that it can iterate through the values of
-// the provided bitmap.
-// The iteration results are undefined if the bitmap is modified (e.g., with Add or Remove).
-func (ii *intIterator) Initialize(a *Bitmap) {
- ii.pos = 0
- ii.highlowcontainer = &a.highlowcontainer
- ii.init()
+func newIntIterator(a *Bitmap) *intIterator {
+ p := new(intIterator)
+ p.pos = 0
+ p.highlowcontainer = &a.highlowcontainer
+ p.init()
+ return p
}
type intReverseIterator struct {
@@ -560,10 +257,6 @@ type intReverseIterator struct {
hs uint32
iter shortIterable
highlowcontainer *roaringArray
-
- shortIter reverseIterator
- runIter runReverseIterator16
- bitmapIter reverseBitmapContainerShortIterator
}
// HasNext returns true if there are more integers to iterate over
@@ -573,30 +266,8 @@ func (ii *intReverseIterator) HasNext() bool {
func (ii *intReverseIterator) init() {
if ii.pos >= 0 {
+ ii.iter = ii.highlowcontainer.getContainerAtIndex(ii.pos).getReverseIterator()
ii.hs = uint32(ii.highlowcontainer.getKeyAtIndex(ii.pos)) << 16
- c := ii.highlowcontainer.getContainerAtIndex(ii.pos)
- switch t := c.(type) {
- case *arrayContainer:
- ii.shortIter = reverseIterator{t.content, len(t.content) - 1}
- ii.iter = &ii.shortIter
- case *runContainer16:
- index := int(len(t.iv)) - 1
- pos := uint16(0)
-
- if index >= 0 {
- pos = t.iv[index].length
- }
-
- ii.runIter = runReverseIterator16{rc: t, curIndex: index, curPosInIndex: pos}
- ii.iter = &ii.runIter
- case *bitmapContainer:
- pos := -1
- if t.cardinality > 0 {
- pos = int(t.maximum())
- }
- ii.bitmapIter = reverseBitmapContainerShortIterator{t, pos}
- ii.iter = &ii.bitmapIter
- }
} else {
ii.iter = nil
}
@@ -612,24 +283,18 @@ func (ii *intReverseIterator) Next() uint32 {
return x
}
-// IntReverseIterator is meant to allow you to iterate through the values of a bitmap, see Initialize(a *Bitmap)
-type IntReverseIterator = intReverseIterator
-
-// Initialize configures the existing iterator so that it can iterate through the values of
-// the provided bitmap.
-// The iteration results are undefined if the bitmap is modified (e.g., with Add or Remove).
-func (ii *intReverseIterator) Initialize(a *Bitmap) {
- ii.highlowcontainer = &a.highlowcontainer
- ii.pos = a.highlowcontainer.size() - 1
- ii.init()
+func newIntReverseIterator(a *Bitmap) *intReverseIterator {
+ p := new(intReverseIterator)
+ p.highlowcontainer = &a.highlowcontainer
+ p.pos = a.highlowcontainer.size() - 1
+ p.init()
+ return p
}
// ManyIntIterable allows you to iterate over the values in a Bitmap
type ManyIntIterable interface {
- // NextMany fills buf up with values, returns how many values were returned
- NextMany(buf []uint32) int
- // NextMany64 fills up buf with 64 bit values, uses hs as a mask (OR), returns how many values were returned
- NextMany64(hs uint64, buf []uint64) int
+ // pass in a buffer to fill up with values, returns how many values were returned
+ NextMany([]uint32) int
}
type manyIntIterator struct {
@@ -637,27 +302,12 @@ type manyIntIterator struct {
hs uint32
iter manyIterable
highlowcontainer *roaringArray
-
- shortIter shortIterator
- runIter runIterator16
- bitmapIter bitmapContainerManyIterator
}
func (ii *manyIntIterator) init() {
if ii.highlowcontainer.size() > ii.pos {
+ ii.iter = ii.highlowcontainer.getContainerAtIndex(ii.pos).getManyIterator()
ii.hs = uint32(ii.highlowcontainer.getKeyAtIndex(ii.pos)) << 16
- c := ii.highlowcontainer.getContainerAtIndex(ii.pos)
- switch t := c.(type) {
- case *arrayContainer:
- ii.shortIter = shortIterator{t.content, 0}
- ii.iter = &ii.shortIter
- case *runContainer16:
- ii.runIter = runIterator16{rc: t, curIndex: 0, curPosInIndex: 0}
- ii.iter = &ii.runIter
- case *bitmapContainer:
- ii.bitmapIter = bitmapContainerManyIterator{t, -1, 0}
- ii.iter = &ii.bitmapIter
- }
} else {
ii.iter = nil
}
@@ -680,35 +330,12 @@ func (ii *manyIntIterator) NextMany(buf []uint32) int {
return n
}
-func (ii *manyIntIterator) NextMany64(hs64 uint64, buf []uint64) int {
- n := 0
- for n < len(buf) {
- if ii.iter == nil {
- break
- }
-
- hs := uint64(ii.hs) | hs64
- moreN := ii.iter.nextMany64(hs, buf[n:])
- n += moreN
- if moreN == 0 {
- ii.pos = ii.pos + 1
- ii.init()
- }
- }
-
- return n
-}
-
-// ManyIntIterator is meant to allow you to iterate through the values of a bitmap, see Initialize(a *Bitmap)
-type ManyIntIterator = manyIntIterator
-
-// Initialize configures the existing iterator so that it can iterate through the values of
-// the provided bitmap.
-// The iteration results are undefined if the bitmap is modified (e.g., with Add or Remove).
-func (ii *manyIntIterator) Initialize(a *Bitmap) {
- ii.pos = 0
- ii.highlowcontainer = &a.highlowcontainer
- ii.init()
+func newManyIntIterator(a *Bitmap) *manyIntIterator {
+ p := new(manyIntIterator)
+ p.pos = 0
+ p.highlowcontainer = &a.highlowcontainer
+ p.init()
+ return p
}
// String creates a string representation of the Bitmap
@@ -737,60 +364,22 @@ func (rb *Bitmap) String() string {
return buffer.String()
}
-// Iterate iterates over the bitmap, calling the given callback with each value in the bitmap. If the callback returns
-// false, the iteration is halted.
-// The iteration results are undefined if the bitmap is modified (e.g., with Add or Remove).
-// There is no guarantee as to what order the values will be iterated.
-func (rb *Bitmap) Iterate(cb func(x uint32) bool) {
- for i := 0; i < rb.highlowcontainer.size(); i++ {
- hs := uint32(rb.highlowcontainer.getKeyAtIndex(i)) << 16
- c := rb.highlowcontainer.getContainerAtIndex(i)
-
- var shouldContinue bool
- // This is hacky but it avoids allocations from invoking an interface method with a closure
- switch t := c.(type) {
- case *arrayContainer:
- shouldContinue = t.iterate(func(x uint16) bool {
- return cb(uint32(x) | hs)
- })
- case *runContainer16:
- shouldContinue = t.iterate(func(x uint16) bool {
- return cb(uint32(x) | hs)
- })
- case *bitmapContainer:
- shouldContinue = t.iterate(func(x uint16) bool {
- return cb(uint32(x) | hs)
- })
- }
-
- if !shouldContinue {
- break
- }
- }
-}
-
-// Iterator creates a new IntPeekable to iterate over the integers contained in the bitmap, in sorted order;
+// Iterator creates a new IntIterable to iterate over the integers contained in the bitmap, in sorted order;
// the iterator becomes invalid if the bitmap is modified (e.g., with Add or Remove).
-func (rb *Bitmap) Iterator() IntPeekable {
- p := new(intIterator)
- p.Initialize(rb)
- return p
+func (rb *Bitmap) Iterator() IntIterable {
+ return newIntIterator(rb)
}
// ReverseIterator creates a new IntIterable to iterate over the integers contained in the bitmap, in sorted order;
// the iterator becomes invalid if the bitmap is modified (e.g., with Add or Remove).
func (rb *Bitmap) ReverseIterator() IntIterable {
- p := new(intReverseIterator)
- p.Initialize(rb)
- return p
+ return newIntReverseIterator(rb)
}
// ManyIterator creates a new ManyIntIterable to iterate over the integers contained in the bitmap, in sorted order;
// the iterator becomes invalid if the bitmap is modified (e.g., with Add or Remove).
func (rb *Bitmap) ManyIterator() ManyIntIterable {
- p := new(manyIntIterator)
- p.Initialize(rb)
- return p
+ return newManyIntIterator(rb)
}
// Clone creates a copy of the Bitmap
@@ -802,17 +391,11 @@ func (rb *Bitmap) Clone() *Bitmap {
// Minimum get the smallest value stored in this roaring bitmap, assumes that it is not empty
func (rb *Bitmap) Minimum() uint32 {
- if len(rb.highlowcontainer.containers) == 0 {
- panic("Empty bitmap")
- }
return uint32(rb.highlowcontainer.containers[0].minimum()) | (uint32(rb.highlowcontainer.keys[0]) << 16)
}
// Maximum get the largest value stored in this roaring bitmap, assumes that it is not empty
func (rb *Bitmap) Maximum() uint32 {
- if len(rb.highlowcontainer.containers) == 0 {
- panic("Empty bitmap")
- }
lastindex := len(rb.highlowcontainer.containers) - 1
return uint32(rb.highlowcontainer.containers[lastindex].maximum()) | (uint32(rb.highlowcontainer.keys[lastindex]) << 16)
}
@@ -840,76 +423,41 @@ func (rb *Bitmap) Equals(o interface{}) bool {
// AddOffset adds the value 'offset' to each and every value in a bitmap, generating a new bitmap in the process
func AddOffset(x *Bitmap, offset uint32) (answer *Bitmap) {
- return AddOffset64(x, int64(offset))
-}
-
-// AddOffset64 adds the value 'offset' to each and every value in a bitmap, generating a new bitmap in the process
-// If offset + element is outside of the range [0,2^32), that the element will be dropped
-func AddOffset64(x *Bitmap, offset int64) (answer *Bitmap) {
- // we need "offset" to be a long because we want to support values
- // between -0xFFFFFFFF up to +-0xFFFFFFFF
- var containerOffset64 int64
-
- if offset < 0 {
- containerOffset64 = (offset - (1 << 16) + 1) / (1 << 16)
- } else {
- containerOffset64 = offset >> 16
- }
-
- answer = New()
-
- if containerOffset64 >= (1<<16) || containerOffset64 < -(1<<16) {
- return answer
- }
-
- containerOffset := int32(containerOffset64)
- inOffset := (uint16)(offset - containerOffset64*(1<<16))
-
+ containerOffset := highbits(offset)
+ inOffset := lowbits(offset)
if inOffset == 0 {
- for pos := 0; pos < x.highlowcontainer.size(); pos++ {
- key := int32(x.highlowcontainer.getKeyAtIndex(pos))
+ answer = x.Clone()
+ for pos := 0; pos < answer.highlowcontainer.size(); pos++ {
+ key := answer.highlowcontainer.getKeyAtIndex(pos)
key += containerOffset
-
- if key >= 0 && key <= MaxUint16 {
- c := x.highlowcontainer.getContainerAtIndex(pos).clone()
- answer.highlowcontainer.appendContainer(uint16(key), c, false)
- }
+ answer.highlowcontainer.keys[pos] = key
}
} else {
+ answer = New()
for pos := 0; pos < x.highlowcontainer.size(); pos++ {
- key := int32(x.highlowcontainer.getKeyAtIndex(pos))
+ key := x.highlowcontainer.getKeyAtIndex(pos)
key += containerOffset
-
- if key+1 < 0 || key > MaxUint16 {
- continue
- }
-
c := x.highlowcontainer.getContainerAtIndex(pos)
- lo, hi := c.addOffset(inOffset)
-
- if lo != nil && key >= 0 {
+ offsetted := c.addOffset(inOffset)
+ if offsetted[0].getCardinality() > 0 {
curSize := answer.highlowcontainer.size()
- lastkey := int32(0)
-
+ lastkey := uint16(0)
if curSize > 0 {
- lastkey = int32(answer.highlowcontainer.getKeyAtIndex(curSize - 1))
+ lastkey = answer.highlowcontainer.getKeyAtIndex(curSize - 1)
}
-
if curSize > 0 && lastkey == key {
prev := answer.highlowcontainer.getContainerAtIndex(curSize - 1)
- orresult := prev.ior(lo)
- answer.highlowcontainer.setContainerAtIndex(curSize-1, orresult)
+ orrseult := prev.ior(offsetted[0])
+ answer.highlowcontainer.setContainerAtIndex(curSize-1, orrseult)
} else {
- answer.highlowcontainer.appendContainer(uint16(key), lo, false)
+ answer.highlowcontainer.appendContainer(key, offsetted[0], false)
}
}
-
- if hi != nil && key+1 <= MaxUint16 {
- answer.highlowcontainer.appendContainer(uint16(key+1), hi, false)
+ if offsetted[1].getCardinality() > 0 {
+ answer.highlowcontainer.appendContainer(key+1, offsetted[1], false)
}
}
}
-
return answer
}
@@ -975,13 +523,13 @@ func (rb *Bitmap) Remove(x uint32) {
if i >= 0 {
c := rb.highlowcontainer.getWritableContainerAtIndex(i).iremoveReturnMinimized(lowbits(x))
rb.highlowcontainer.setContainerAtIndex(i, c)
- if rb.highlowcontainer.getContainerAtIndex(i).isEmpty() {
+ if rb.highlowcontainer.getContainerAtIndex(i).getCardinality() == 0 {
rb.highlowcontainer.removeAtIndex(i)
}
}
}
-// CheckedRemove removes the integer x from the bitmap and return true if the integer was effectively removed (and false if the integer was not present)
+// CheckedRemove removes the integer x from the bitmap and return true if the integer was effectively remove (and false if the integer was not present)
func (rb *Bitmap) CheckedRemove(x uint32) bool {
// TODO: add unit tests for this method
hb := highbits(x)
@@ -991,7 +539,7 @@ func (rb *Bitmap) CheckedRemove(x uint32) bool {
oldcard := C.getCardinality()
C = C.iremoveReturnMinimized(lowbits(x))
rb.highlowcontainer.setContainerAtIndex(i, C)
- if rb.highlowcontainer.getContainerAtIndex(i).isEmpty() {
+ if rb.highlowcontainer.getContainerAtIndex(i).getCardinality() == 0 {
rb.highlowcontainer.removeAtIndex(i)
return true
}
@@ -1015,10 +563,7 @@ func (rb *Bitmap) GetCardinality() uint64 {
return size
}
-// Rank returns the number of integers that are smaller or equal to x (Rank(infinity) would be GetCardinality()).
-// If you pass the smallest value, you get the value 1. If you pass a value that is smaller than the smallest
-// value, you get 0. Note that this function differs in convention from the Select function since it
-// return 1 and not 0 on the smallest value.
+// Rank returns the number of integers that are smaller or equal to x (Rank(infinity) would be GetCardinality())
func (rb *Bitmap) Rank(x uint32) uint64 {
size := uint64(0)
for i := 0; i < rb.highlowcontainer.size(); i++ {
@@ -1035,22 +580,23 @@ func (rb *Bitmap) Rank(x uint32) uint64 {
return size
}
-// Select returns the xth integer in the bitmap. If you pass 0, you get
-// the smallest element. Note that this function differs in convention from
-// the Rank function which returns 1 on the smallest value.
+// Select returns the xth integer in the bitmap
func (rb *Bitmap) Select(x uint32) (uint32, error) {
+ if rb.GetCardinality() <= uint64(x) {
+ return 0, fmt.Errorf("can't find %dth integer in a bitmap with only %d items", x, rb.GetCardinality())
+ }
+
remaining := x
for i := 0; i < rb.highlowcontainer.size(); i++ {
c := rb.highlowcontainer.getContainerAtIndex(i)
- card := uint32(c.getCardinality())
- if remaining >= card {
- remaining -= card
+ if remaining >= uint32(c.getCardinality()) {
+ remaining -= uint32(c.getCardinality())
} else {
key := rb.highlowcontainer.getKeyAtIndex(i)
return uint32(key)<<16 + uint32(c.selectInt(uint16(remaining))), nil
}
}
- return 0, fmt.Errorf("cannot find %dth integer in a bitmap with only %d items", x, rb.GetCardinality())
+ return 0, fmt.Errorf("can't find %dth integer in a bitmap with only %d items", x, rb.GetCardinality())
}
// And computes the intersection between two bitmaps and stores the result in the current bitmap
@@ -1071,7 +617,7 @@ main:
c1 := rb.highlowcontainer.getWritableContainerAtIndex(pos1)
c2 := x2.highlowcontainer.getContainerAtIndex(pos2)
diff := c1.iand(c2)
- if !diff.isEmpty() {
+ if diff.getCardinality() > 0 {
rb.highlowcontainer.replaceKeyAndContainerAtIndex(intersectionsize, s1, diff, false)
intersectionsize++
}
@@ -1202,28 +748,6 @@ main:
return answer
}
-// IntersectsWithInterval checks whether a bitmap 'rb' and an open interval '[x,y)' intersect.
-func (rb *Bitmap) IntersectsWithInterval(x, y uint64) bool {
- if x >= y {
- return false
- }
- if x > MaxUint32 {
- return false
- }
-
- it := intIterator{}
- it.Initialize(rb)
- it.AdvanceIfNeeded(uint32(x))
- if !it.HasNext() {
- return false
- }
- if uint64(it.Next()) >= y {
- return false
- }
-
- return true
-}
-
// Intersects checks whether two bitmap intersects, bitmaps are not modified
func (rb *Bitmap) Intersects(x2 *Bitmap) bool {
pos1 := 0
@@ -1295,7 +819,7 @@ func (rb *Bitmap) Xor(x2 *Bitmap) {
} else {
// TODO: couple be computed in-place for reduced memory usage
c := rb.highlowcontainer.getContainerAtIndex(pos1).xor(x2.highlowcontainer.getContainerAtIndex(pos2))
- if !c.isEmpty() {
+ if c.getCardinality() > 0 {
rb.highlowcontainer.setContainerAtIndex(pos1, c)
pos1++
} else {
@@ -1341,7 +865,7 @@ main:
}
s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
} else {
- rb.highlowcontainer.replaceKeyAndContainerAtIndex(pos1, s1, rb.highlowcontainer.getUnionedWritableContainer(pos1, x2.highlowcontainer.getContainerAtIndex(pos2)), false)
+ rb.highlowcontainer.replaceKeyAndContainerAtIndex(pos1, s1, rb.highlowcontainer.getWritableContainerAtIndex(pos1).ior(x2.highlowcontainer.getContainerAtIndex(pos2)), false)
pos1++
pos2++
if (pos1 == length1) || (pos2 == length2) {
@@ -1375,7 +899,7 @@ main:
c1 := rb.highlowcontainer.getWritableContainerAtIndex(pos1)
c2 := x2.highlowcontainer.getContainerAtIndex(pos2)
diff := c1.iandNot(c2)
- if !diff.isEmpty() {
+ if diff.getCardinality() > 0 {
rb.highlowcontainer.replaceKeyAndContainerAtIndex(intersectionsize, s1, diff, false)
intersectionsize++
}
@@ -1484,7 +1008,7 @@ main:
C := x1.highlowcontainer.getContainerAtIndex(pos1)
C = C.and(x2.highlowcontainer.getContainerAtIndex(pos2))
- if !C.isEmpty() {
+ if C.getCardinality() > 0 {
answer.highlowcontainer.appendContainer(s1, C, false)
}
pos1++
@@ -1531,7 +1055,7 @@ func Xor(x1, x2 *Bitmap) *Bitmap {
pos2++
} else {
c := x1.highlowcontainer.getContainerAtIndex(pos1).xor(x2.highlowcontainer.getContainerAtIndex(pos2))
- if !c.isEmpty() {
+ if c.getCardinality() > 0 {
answer.highlowcontainer.appendContainer(s1, c, false)
}
pos1++
@@ -1574,7 +1098,7 @@ main:
c1 := x1.highlowcontainer.getContainerAtIndex(pos1)
c2 := x2.highlowcontainer.getContainerAtIndex(pos2)
diff := c1.andNot(c2)
- if !diff.isEmpty() {
+ if diff.getCardinality() > 0 {
answer.highlowcontainer.appendContainer(s1, diff, false)
}
pos1++
@@ -1664,7 +1188,7 @@ func (rb *Bitmap) Flip(rangeStart, rangeEnd uint64) {
if i >= 0 {
c := rb.highlowcontainer.getWritableContainerAtIndex(i).inot(int(containerStart), int(containerLast)+1)
- if !c.isEmpty() {
+ if c.getCardinality() > 0 {
rb.highlowcontainer.setContainerAtIndex(i, c)
} else {
rb.highlowcontainer.removeAtIndex(i)
@@ -1745,7 +1269,7 @@ func (rb *Bitmap) RemoveRange(rangeStart, rangeEnd uint64) {
return
}
c := rb.highlowcontainer.getWritableContainerAtIndex(i).iremoveRange(int(lbStart), int(lbLast+1))
- if !c.isEmpty() {
+ if c.getCardinality() > 0 {
rb.highlowcontainer.setContainerAtIndex(i, c)
} else {
rb.highlowcontainer.removeAtIndex(i)
@@ -1758,7 +1282,7 @@ func (rb *Bitmap) RemoveRange(rangeStart, rangeEnd uint64) {
if ifirst >= 0 {
if lbStart != 0 {
c := rb.highlowcontainer.getWritableContainerAtIndex(ifirst).iremoveRange(int(lbStart), int(max+1))
- if !c.isEmpty() {
+ if c.getCardinality() > 0 {
rb.highlowcontainer.setContainerAtIndex(ifirst, c)
ifirst++
}
@@ -1769,7 +1293,7 @@ func (rb *Bitmap) RemoveRange(rangeStart, rangeEnd uint64) {
if ilast >= 0 {
if lbLast != max {
c := rb.highlowcontainer.getWritableContainerAtIndex(ilast).iremoveRange(int(0), int(lbLast+1))
- if !c.isEmpty() {
+ if c.getCardinality() > 0 {
rb.highlowcontainer.setContainerAtIndex(ilast, c)
} else {
ilast++
@@ -1825,7 +1349,7 @@ func Flip(bm *Bitmap, rangeStart, rangeEnd uint64) *Bitmap {
if i >= 0 {
c := bm.highlowcontainer.getContainerAtIndex(i).not(int(containerStart), int(containerLast)+1)
- if !c.isEmpty() {
+ if c.getCardinality() > 0 {
answer.highlowcontainer.insertNewKeyValueAt(-j-1, uint16(hb), c)
}
@@ -1854,21 +1378,6 @@ func (rb *Bitmap) GetCopyOnWrite() (val bool) {
return rb.highlowcontainer.copyOnWrite
}
-// CloneCopyOnWriteContainers clones all containers which have
-// needCopyOnWrite set to true.
-// This can be used to make sure it is safe to munmap a []byte
-// that the roaring array may still have a reference to, after
-// calling FromBuffer.
-// More generally this function is useful if you call FromBuffer
-// to construct a bitmap with a backing array buf
-// and then later discard the buf array. Note that you should call
-// CloneCopyOnWriteContainers on all bitmaps that were derived
-// from the 'FromBuffer' bitmap since they map have dependencies
-// on the buf array as well.
-func (rb *Bitmap) CloneCopyOnWriteContainers() {
- rb.highlowcontainer.cloneCopyOnWriteContainers()
-}
-
// FlipInt calls Flip after casting the parameters (convenience method)
func FlipInt(bm *Bitmap, rangeStart, rangeEnd int) *Bitmap {
return Flip(bm, uint64(rangeStart), uint64(rangeEnd))
diff --git a/vendor/github.com/RoaringBitmap/roaring/roaringarray.go b/vendor/github.com/RoaringBitmap/roaring/roaringarray.go
index 079195d..3f77759 100644
--- a/vendor/github.com/RoaringBitmap/roaring/roaringarray.go
+++ b/vendor/github.com/RoaringBitmap/roaring/roaringarray.go
@@ -5,15 +5,16 @@ import (
"encoding/binary"
"fmt"
"io"
+ "io/ioutil"
- "github.com/RoaringBitmap/roaring/internal"
+ snappy "github.com/glycerine/go-unsnap-stream"
+ "github.com/tinylib/msgp/msgp"
)
+//go:generate msgp -unexported
+
type container interface {
- // addOffset returns the (low, high) parts of the shifted container.
- // Whenever one of them would be empty, nil will be returned instead to
- // avoid unnecessary allocations.
- addOffset(uint16) (container, container)
+ addOffset(uint16) []container
clone() container
and(container) container
@@ -21,7 +22,6 @@ type container interface {
iand(container) container // i stands for inplace
andNot(container) container
iandNot(container) container // i stands for inplace
- isEmpty() bool
getCardinality() int
// rank returns the number of integers that are
// smaller or equal to x. rank(infinity) would be getCardinality().
@@ -39,8 +39,7 @@ type container interface {
not(start, final int) container // range is [firstOfRange,lastOfRange)
inot(firstOfRange, endx int) container // i stands for inplace, range is [firstOfRange,endx)
xor(r container) container
- getShortIterator() shortPeekable
- iterate(cb func(x uint16) bool) bool
+ getShortIterator() shortIterable
getReverseIterator() shortIterable
getManyIterator() manyIterable
contains(i uint16) bool
@@ -52,7 +51,7 @@ type container interface {
// any of the implementations.
equals(r container) bool
- fillLeastSignificant16bits(array []uint32, i int, mask uint32) int
+ fillLeastSignificant16bits(array []uint32, i int, mask uint32)
or(r container) container
orCardinality(r container) int
isFull() bool
@@ -65,6 +64,7 @@ type container interface {
iremoveRange(start, final int) container // i stands for inplace, range is [firstOfRange,lastOfRange)
selectInt(x uint16) int // selectInt returns the xth integer in the container
serializedSizeInBytes() int
+ readFrom(io.Reader) (int, error)
writeTo(io.Writer) (int, error)
numberOfRuns() int
@@ -104,6 +104,18 @@ type roaringArray struct {
containers []container `msg:"-"` // don't try to serialize directly.
needCopyOnWrite []bool
copyOnWrite bool
+
+ // conserz is used at serialization time
+ // to serialize containers. Otherwise empty.
+ conserz []containerSerz
+}
+
+// containerSerz facilitates serializing container (tricky to
+// serialize because it is an interface) by providing a
+// light wrapper with a type identifier.
+type containerSerz struct {
+ t contype `msg:"t"` // type
+ r msgp.Raw `msg:"r"` // Raw msgpack of the actual container type
}
func newRoaringArray() *roaringArray {
@@ -113,10 +125,9 @@ func newRoaringArray() *roaringArray {
// runOptimize compresses the element containers to minimize space consumed.
// Q: how does this interact with copyOnWrite and needCopyOnWrite?
// A: since we aren't changing the logical content, just the representation,
-//
-// we don't bother to check the needCopyOnWrite bits. We replace
-// (possibly all) elements of ra.containers in-place with space
-// optimized versions.
+// we don't bother to check the needCopyOnWrite bits. We replace
+// (possibly all) elements of ra.containers in-place with space
+// optimized versions.
func (ra *roaringArray) runOptimize() {
for i := range ra.containers {
ra.containers[i] = ra.containers[i].toEfficientContainer()
@@ -236,6 +247,7 @@ func (ra *roaringArray) resize(newsize int) {
func (ra *roaringArray) clear() {
ra.resize(0)
ra.copyOnWrite = false
+ ra.conserz = nil
}
func (ra *roaringArray) clone() *roaringArray {
@@ -271,18 +283,6 @@ func (ra *roaringArray) clone() *roaringArray {
return &sa
}
-// clone all containers which have needCopyOnWrite set to true
-// This can be used to make sure it is safe to munmap a []byte
-// that the roaring array may still have a reference to.
-func (ra *roaringArray) cloneCopyOnWriteContainers() {
- for i, needCopyOnWrite := range ra.needCopyOnWrite {
- if needCopyOnWrite {
- ra.containers[i] = ra.containers[i].clone()
- ra.needCopyOnWrite[i] = false
- }
- }
-}
-
// unused function:
//func (ra *roaringArray) containsKey(x uint16) bool {
// return (ra.binarySearch(0, int64(len(ra.keys)), x) >= 0)
@@ -317,17 +317,6 @@ func (ra *roaringArray) getFastContainerAtIndex(i int, needsWriteable bool) cont
return c
}
-// getUnionedWritableContainer switches behavior for in-place Or
-// depending on whether the container requires a copy on write.
-// If it does using the non-inplace or() method leads to fewer allocations.
-func (ra *roaringArray) getUnionedWritableContainer(pos int, other container) container {
- if ra.needCopyOnWrite[pos] {
- return ra.getContainerAtIndex(pos).or(other)
- }
- return ra.getContainerAtIndex(pos).ior(other)
-
-}
-
func (ra *roaringArray) getWritableContainerAtIndex(i int) container {
if ra.needCopyOnWrite[i] {
ra.containers[i] = ra.containers[i].clone()
@@ -467,7 +456,9 @@ func (ra *roaringArray) serializedSizeInBytes() uint64 {
return answer
}
+//
// spec: https://github.com/RoaringBitmap/RoaringFormatSpec
+//
func (ra *roaringArray) writeTo(w io.Writer) (n int64, err error) {
hasRun := ra.hasRunCompression()
isRunSizeInBytes := 0
@@ -488,15 +479,20 @@ func (ra *roaringArray) writeTo(w io.Writer) (n int64, err error) {
nw += 2
binary.LittleEndian.PutUint16(buf[2:], uint16(len(ra.keys)-1))
nw += 2
- // compute isRun bitmap without temporary allocation
- var runbitmapslice = buf[nw : nw+isRunSizeInBytes]
+
+ // compute isRun bitmap
+ var ir []byte
+
+ isRun := newBitmapContainer()
for i, c := range ra.containers {
switch c.(type) {
case *runContainer16:
- runbitmapslice[i/8] |= 1 << (uint(i) % 8)
+ isRun.iadd(uint16(i))
}
}
- nw += isRunSizeInBytes
+ // convert to little endian
+ ir = isRun.asLittleEndianByteSlice()[:isRunSizeInBytes]
+ nw += copy(buf[nw:], ir)
} else {
binary.LittleEndian.PutUint32(buf[0:], uint32(serialCookieNoRunContainer))
nw += 4
@@ -544,69 +540,59 @@ func (ra *roaringArray) writeTo(w io.Writer) (n int64, err error) {
return n, nil
}
+//
// spec: https://github.com/RoaringBitmap/RoaringFormatSpec
+//
func (ra *roaringArray) toBytes() ([]byte, error) {
var buf bytes.Buffer
_, err := ra.writeTo(&buf)
return buf.Bytes(), err
}
-// Reads a serialized roaringArray from a byte slice.
-func (ra *roaringArray) readFrom(stream internal.ByteInput, cookieHeader ...byte) (int64, error) {
- var cookie uint32
- var err error
- if len(cookieHeader) > 0 && len(cookieHeader) != 4 {
- return int64(len(cookieHeader)), fmt.Errorf("error in roaringArray.readFrom: could not read initial cookie: incorrect size of cookie header")
+func (ra *roaringArray) fromBuffer(buf []byte) (int64, error) {
+ pos := 0
+ if len(buf) < 8 {
+ return 0, fmt.Errorf("buffer too small, expecting at least 8 bytes, was %d", len(buf))
}
- if len(cookieHeader) == 4 {
- cookie = binary.LittleEndian.Uint32(cookieHeader)
- } else {
- cookie, err = stream.ReadUInt32()
- if err != nil {
- return stream.GetReadBytes(), fmt.Errorf("error in roaringArray.readFrom: could not read initial cookie: %s", err)
- }
- }
- // If NextReturnsSafeSlice is false, then willNeedCopyOnWrite should be true
- willNeedCopyOnWrite := !stream.NextReturnsSafeSlice()
- var size uint32
+ cookie := binary.LittleEndian.Uint32(buf)
+ pos += 4
+ var size uint32 // number of containers
+ haveRunContainers := false
var isRunBitmap []byte
+ // cookie header
if cookie&0x0000FFFF == serialCookie {
- size = uint32(cookie>>16 + 1)
+ haveRunContainers = true
+ size = uint32(uint16(cookie>>16) + 1) // number of containers
+
// create is-run-container bitmap
isRunBitmapSize := (int(size) + 7) / 8
- isRunBitmap, err = stream.Next(isRunBitmapSize)
-
- if err != nil {
- return stream.GetReadBytes(), fmt.Errorf("malformed bitmap, failed to read is-run bitmap, got: %s", err)
+ if pos+isRunBitmapSize > len(buf) {
+ return 0, fmt.Errorf("malformed bitmap, is-run bitmap overruns buffer at %d", pos+isRunBitmapSize)
}
+
+ isRunBitmap = buf[pos : pos+isRunBitmapSize]
+ pos += isRunBitmapSize
} else if cookie == serialCookieNoRunContainer {
- size, err = stream.ReadUInt32()
- if err != nil {
- return stream.GetReadBytes(), fmt.Errorf("malformed bitmap, failed to read a bitmap size: %s", err)
- }
+ size = binary.LittleEndian.Uint32(buf[pos:])
+ pos += 4
} else {
- return stream.GetReadBytes(), fmt.Errorf("error in roaringArray.readFrom: did not find expected serialCookie in header")
+ return 0, fmt.Errorf("error in roaringArray.readFrom: did not find expected serialCookie in header")
}
-
if size > (1 << 16) {
- return stream.GetReadBytes(), fmt.Errorf("it is logically impossible to have more than (1<<16) containers")
+ return 0, fmt.Errorf("It is logically impossible to have more than (1<<16) containers.")
}
-
// descriptive header
- buf, err := stream.Next(2 * 2 * int(size))
-
- if err != nil {
- return stream.GetReadBytes(), fmt.Errorf("failed to read descriptive header: %s", err)
+ // keycard - is {key, cardinality} tuple slice
+ if pos+2*2*int(size) > len(buf) {
+ return 0, fmt.Errorf("malfomred bitmap, key-cardinality slice overruns buffer at %d", pos+2*2*int(size))
}
+ keycard := byteSliceAsUint16Slice(buf[pos : pos+2*2*int(size)])
+ pos += 2 * 2 * int(size)
- keycard := byteSliceAsUint16Slice(buf)
-
- if isRunBitmap == nil || size >= noOffsetThreshold {
- if err := stream.SkipBytes(int(size) * 4); err != nil {
- return stream.GetReadBytes(), fmt.Errorf("failed to skip bytes: %s", err)
- }
+ if !haveRunContainers || size >= noOffsetThreshold {
+ pos += 4 * int(size)
}
// Allocate slices upfront as number of containers is known
@@ -615,13 +601,11 @@ func (ra *roaringArray) readFrom(stream internal.ByteInput, cookieHeader ...byte
} else {
ra.containers = make([]container, size)
}
-
if cap(ra.keys) >= int(size) {
ra.keys = ra.keys[:size]
} else {
ra.keys = make([]uint16, size)
}
-
if cap(ra.needCopyOnWrite) >= int(size) {
ra.needCopyOnWrite = ra.needCopyOnWrite[:size]
} else {
@@ -629,61 +613,129 @@ func (ra *roaringArray) readFrom(stream internal.ByteInput, cookieHeader ...byte
}
for i := uint32(0); i < size; i++ {
- key := keycard[2*i]
+ key := uint16(keycard[2*i])
card := int(keycard[2*i+1]) + 1
ra.keys[i] = key
- ra.needCopyOnWrite[i] = willNeedCopyOnWrite
+ ra.needCopyOnWrite[i] = true
- if isRunBitmap != nil && isRunBitmap[i/8]&(1<<(i%8)) != 0 {
+ if haveRunContainers && isRunBitmap[i/8]&(1<<(i%8)) != 0 {
// run container
- nr, err := stream.ReadUInt16()
-
- if err != nil {
- return 0, fmt.Errorf("failed to read runtime container size: %s", err)
+ nr := binary.LittleEndian.Uint16(buf[pos:])
+ pos += 2
+ if pos+int(nr)*4 > len(buf) {
+ return 0, fmt.Errorf("malformed bitmap, a run container overruns buffer at %d:%d", pos, pos+int(nr)*4)
}
-
- buf, err := stream.Next(int(nr) * 4)
-
- if err != nil {
- return stream.GetReadBytes(), fmt.Errorf("failed to read runtime container content: %s", err)
- }
-
nb := runContainer16{
- iv: byteSliceAsInterval16Slice(buf),
+ iv: byteSliceAsInterval16Slice(buf[pos : pos+int(nr)*4]),
+ card: int64(card),
}
-
+ pos += int(nr) * 4
ra.containers[i] = &nb
} else if card > arrayDefaultMaxSize {
// bitmap container
- buf, err := stream.Next(arrayDefaultMaxSize * 2)
-
- if err != nil {
- return stream.GetReadBytes(), fmt.Errorf("failed to read bitmap container: %s", err)
- }
-
nb := bitmapContainer{
cardinality: card,
- bitmap: byteSliceAsUint64Slice(buf),
+ bitmap: byteSliceAsUint64Slice(buf[pos : pos+arrayDefaultMaxSize*2]),
}
-
+ pos += arrayDefaultMaxSize * 2
ra.containers[i] = &nb
} else {
// array container
- buf, err := stream.Next(card * 2)
-
- if err != nil {
- return stream.GetReadBytes(), fmt.Errorf("failed to read array container: %s", err)
- }
-
nb := arrayContainer{
- byteSliceAsUint16Slice(buf),
+ byteSliceAsUint16Slice(buf[pos : pos+card*2]),
}
-
+ pos += card * 2
ra.containers[i] = &nb
}
}
- return stream.GetReadBytes(), nil
+ return int64(pos), nil
+}
+
+func (ra *roaringArray) readFrom(stream io.Reader) (int64, error) {
+ pos := 0
+ var cookie uint32
+ err := binary.Read(stream, binary.LittleEndian, &cookie)
+ if err != nil {
+ return 0, fmt.Errorf("error in roaringArray.readFrom: could not read initial cookie: %s", err)
+ }
+ pos += 4
+ var size uint32
+ haveRunContainers := false
+ var isRun *bitmapContainer
+ if cookie&0x0000FFFF == serialCookie {
+ haveRunContainers = true
+ size = uint32(uint16(cookie>>16) + 1)
+ bytesToRead := (int(size) + 7) / 8
+ numwords := (bytesToRead + 7) / 8
+ by := make([]byte, bytesToRead, numwords*8)
+ nr, err := io.ReadFull(stream, by)
+ if err != nil {
+ return 8 + int64(nr), fmt.Errorf("error in readFrom: could not read the "+
+ "runContainer bit flags of length %v bytes: %v", bytesToRead, err)
+ }
+ pos += bytesToRead
+ by = by[:cap(by)]
+ isRun = newBitmapContainer()
+ for i := 0; i < numwords; i++ {
+ isRun.bitmap[i] = binary.LittleEndian.Uint64(by)
+ by = by[8:]
+ }
+ } else if cookie == serialCookieNoRunContainer {
+ err = binary.Read(stream, binary.LittleEndian, &size)
+ if err != nil {
+ return 0, fmt.Errorf("error in roaringArray.readFrom: when reading size, got: %s", err)
+ }
+ pos += 4
+ } else {
+ return 0, fmt.Errorf("error in roaringArray.readFrom: did not find expected serialCookie in header")
+ }
+ if size > (1 << 16) {
+ return 0, fmt.Errorf("It is logically impossible to have more than (1<<16) containers.")
+ }
+ // descriptive header
+ keycard := make([]uint16, 2*size, 2*size)
+ err = binary.Read(stream, binary.LittleEndian, keycard)
+ if err != nil {
+ return 0, err
+ }
+ pos += 2 * 2 * int(size)
+ // offset header
+ if !haveRunContainers || size >= noOffsetThreshold {
+ io.CopyN(ioutil.Discard, stream, 4*int64(size)) // we never skip ahead so this data can be ignored
+ pos += 4 * int(size)
+ }
+ for i := uint32(0); i < size; i++ {
+ key := int(keycard[2*i])
+ card := int(keycard[2*i+1]) + 1
+ if haveRunContainers && isRun.contains(uint16(i)) {
+ nb := newRunContainer16()
+ nr, err := nb.readFrom(stream)
+ if err != nil {
+ return 0, err
+ }
+ pos += nr
+ ra.appendContainer(uint16(key), nb, false)
+ } else if card > arrayDefaultMaxSize {
+ nb := newBitmapContainer()
+ nr, err := nb.readFrom(stream)
+ if err != nil {
+ return 0, err
+ }
+ nb.cardinality = card
+ pos += nr
+ ra.appendContainer(keycard[2*i], nb, false)
+ } else {
+ nb := newArrayContainerSize(card)
+ nr, err := nb.readFrom(stream)
+ if err != nil {
+ return 0, err
+ }
+ pos += nr
+ ra.appendContainer(keycard[2*i], nb, false)
+ }
+ }
+ return int64(pos), nil
}
func (ra *roaringArray) hasRunCompression() bool {
@@ -696,6 +748,84 @@ func (ra *roaringArray) hasRunCompression() bool {
return false
}
+func (ra *roaringArray) writeToMsgpack(stream io.Writer) error {
+
+ ra.conserz = make([]containerSerz, len(ra.containers))
+ for i, v := range ra.containers {
+ switch cn := v.(type) {
+ case *bitmapContainer:
+ bts, err := cn.MarshalMsg(nil)
+ if err != nil {
+ return err
+ }
+ ra.conserz[i].t = bitmapContype
+ ra.conserz[i].r = bts
+ case *arrayContainer:
+ bts, err := cn.MarshalMsg(nil)
+ if err != nil {
+ return err
+ }
+ ra.conserz[i].t = arrayContype
+ ra.conserz[i].r = bts
+ case *runContainer16:
+ bts, err := cn.MarshalMsg(nil)
+ if err != nil {
+ return err
+ }
+ ra.conserz[i].t = run16Contype
+ ra.conserz[i].r = bts
+ default:
+ panic(fmt.Errorf("Unrecognized container implementation: %T", cn))
+ }
+ }
+ w := snappy.NewWriter(stream)
+ err := msgp.Encode(w, ra)
+ ra.conserz = nil
+ return err
+}
+
+func (ra *roaringArray) readFromMsgpack(stream io.Reader) error {
+ r := snappy.NewReader(stream)
+ err := msgp.Decode(r, ra)
+ if err != nil {
+ return err
+ }
+
+ if len(ra.containers) != len(ra.keys) {
+ ra.containers = make([]container, len(ra.keys))
+ }
+
+ for i, v := range ra.conserz {
+ switch v.t {
+ case bitmapContype:
+ c := &bitmapContainer{}
+ _, err = c.UnmarshalMsg(v.r)
+ if err != nil {
+ return err
+ }
+ ra.containers[i] = c
+ case arrayContype:
+ c := &arrayContainer{}
+ _, err = c.UnmarshalMsg(v.r)
+ if err != nil {
+ return err
+ }
+ ra.containers[i] = c
+ case run16Contype:
+ c := &runContainer16{}
+ _, err = c.UnmarshalMsg(v.r)
+ if err != nil {
+ return err
+ }
+ ra.containers[i] = c
+ default:
+ return fmt.Errorf("unrecognized contype serialization code: '%v'", v.t)
+ }
+ }
+ ra.conserz = nil
+ return nil
+}
+
func (ra *roaringArray) advanceUntil(min uint16, pos int) int {
lower := pos + 1
diff --git a/vendor/github.com/RoaringBitmap/roaring/roaringarray_gen.go b/vendor/github.com/RoaringBitmap/roaring/roaringarray_gen.go
new file mode 100644
index 0000000..dcd7187
--- /dev/null
+++ b/vendor/github.com/RoaringBitmap/roaring/roaringarray_gen.go
@@ -0,0 +1,529 @@
+package roaring
+
+// NOTE: THIS FILE WAS PRODUCED BY THE
+// MSGP CODE GENERATION TOOL (github.com/tinylib/msgp)
+// DO NOT EDIT
+
+import (
+ "github.com/tinylib/msgp/msgp"
+)
+
+// Deprecated: DecodeMsg implements msgp.Decodable
+func (z *containerSerz) DecodeMsg(dc *msgp.Reader) (err error) {
+ var field []byte
+ _ = field
+ var zxvk uint32
+ zxvk, err = dc.ReadMapHeader()
+ if err != nil {
+ return
+ }
+ for zxvk > 0 {
+ zxvk--
+ field, err = dc.ReadMapKeyPtr()
+ if err != nil {
+ return
+ }
+ switch msgp.UnsafeString(field) {
+ case "t":
+ {
+ var zbzg uint8
+ zbzg, err = dc.ReadUint8()
+ z.t = contype(zbzg)
+ }
+ if err != nil {
+ return
+ }
+ case "r":
+ err = z.r.DecodeMsg(dc)
+ if err != nil {
+ return
+ }
+ default:
+ err = dc.Skip()
+ if err != nil {
+ return
+ }
+ }
+ }
+ return
+}
+
+// Deprecated: EncodeMsg implements msgp.Encodable
+func (z *containerSerz) EncodeMsg(en *msgp.Writer) (err error) {
+ // map header, size 2
+ // write "t"
+ err = en.Append(0x82, 0xa1, 0x74)
+ if err != nil {
+ return err
+ }
+ err = en.WriteUint8(uint8(z.t))
+ if err != nil {
+ return
+ }
+ // write "r"
+ err = en.Append(0xa1, 0x72)
+ if err != nil {
+ return err
+ }
+ err = z.r.EncodeMsg(en)
+ if err != nil {
+ return
+ }
+ return
+}
+
+// Deprecated: MarshalMsg implements msgp.Marshaler
+func (z *containerSerz) MarshalMsg(b []byte) (o []byte, err error) {
+ o = msgp.Require(b, z.Msgsize())
+ // map header, size 2
+ // string "t"
+ o = append(o, 0x82, 0xa1, 0x74)
+ o = msgp.AppendUint8(o, uint8(z.t))
+ // string "r"
+ o = append(o, 0xa1, 0x72)
+ o, err = z.r.MarshalMsg(o)
+ if err != nil {
+ return
+ }
+ return
+}
+
+// Deprecated: UnmarshalMsg implements msgp.Unmarshaler
+func (z *containerSerz) UnmarshalMsg(bts []byte) (o []byte, err error) {
+ var field []byte
+ _ = field
+ var zbai uint32
+ zbai, bts, err = msgp.ReadMapHeaderBytes(bts)
+ if err != nil {
+ return
+ }
+ for zbai > 0 {
+ zbai--
+ field, bts, err = msgp.ReadMapKeyZC(bts)
+ if err != nil {
+ return
+ }
+ switch msgp.UnsafeString(field) {
+ case "t":
+ {
+ var zcmr uint8
+ zcmr, bts, err = msgp.ReadUint8Bytes(bts)
+ z.t = contype(zcmr)
+ }
+ if err != nil {
+ return
+ }
+ case "r":
+ bts, err = z.r.UnmarshalMsg(bts)
+ if err != nil {
+ return
+ }
+ default:
+ bts, err = msgp.Skip(bts)
+ if err != nil {
+ return
+ }
+ }
+ }
+ o = bts
+ return
+}
+
+// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *containerSerz) Msgsize() (s int) {
+ s = 1 + 2 + msgp.Uint8Size + 2 + z.r.Msgsize()
+ return
+}
+
+// Deprecated: DecodeMsg implements msgp.Decodable
+func (z *contype) DecodeMsg(dc *msgp.Reader) (err error) {
+ {
+ var zajw uint8
+ zajw, err = dc.ReadUint8()
+ (*z) = contype(zajw)
+ }
+ if err != nil {
+ return
+ }
+ return
+}
+
+// Deprecated: EncodeMsg implements msgp.Encodable
+func (z contype) EncodeMsg(en *msgp.Writer) (err error) {
+ err = en.WriteUint8(uint8(z))
+ if err != nil {
+ return
+ }
+ return
+}
+
+// Deprecated: MarshalMsg implements msgp.Marshaler
+func (z contype) MarshalMsg(b []byte) (o []byte, err error) {
+ o = msgp.Require(b, z.Msgsize())
+ o = msgp.AppendUint8(o, uint8(z))
+ return
+}
+
+// Deprecated: UnmarshalMsg implements msgp.Unmarshaler
+func (z *contype) UnmarshalMsg(bts []byte) (o []byte, err error) {
+ {
+ var zwht uint8
+ zwht, bts, err = msgp.ReadUint8Bytes(bts)
+ (*z) = contype(zwht)
+ }
+ if err != nil {
+ return
+ }
+ o = bts
+ return
+}
+
+// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z contype) Msgsize() (s int) {
+ s = msgp.Uint8Size
+ return
+}
+
+// Deprecated: DecodeMsg implements msgp.Decodable
+func (z *roaringArray) DecodeMsg(dc *msgp.Reader) (err error) {
+ var field []byte
+ _ = field
+ var zlqf uint32
+ zlqf, err = dc.ReadMapHeader()
+ if err != nil {
+ return
+ }
+ for zlqf > 0 {
+ zlqf--
+ field, err = dc.ReadMapKeyPtr()
+ if err != nil {
+ return
+ }
+ switch msgp.UnsafeString(field) {
+ case "keys":
+ var zdaf uint32
+ zdaf, err = dc.ReadArrayHeader()
+ if err != nil {
+ return
+ }
+ if cap(z.keys) >= int(zdaf) {
+ z.keys = (z.keys)[:zdaf]
+ } else {
+ z.keys = make([]uint16, zdaf)
+ }
+ for zhct := range z.keys {
+ z.keys[zhct], err = dc.ReadUint16()
+ if err != nil {
+ return
+ }
+ }
+ case "needCopyOnWrite":
+ var zpks uint32
+ zpks, err = dc.ReadArrayHeader()
+ if err != nil {
+ return
+ }
+ if cap(z.needCopyOnWrite) >= int(zpks) {
+ z.needCopyOnWrite = (z.needCopyOnWrite)[:zpks]
+ } else {
+ z.needCopyOnWrite = make([]bool, zpks)
+ }
+ for zcua := range z.needCopyOnWrite {
+ z.needCopyOnWrite[zcua], err = dc.ReadBool()
+ if err != nil {
+ return
+ }
+ }
+ case "copyOnWrite":
+ z.copyOnWrite, err = dc.ReadBool()
+ if err != nil {
+ return
+ }
+ case "conserz":
+ var zjfb uint32
+ zjfb, err = dc.ReadArrayHeader()
+ if err != nil {
+ return
+ }
+ if cap(z.conserz) >= int(zjfb) {
+ z.conserz = (z.conserz)[:zjfb]
+ } else {
+ z.conserz = make([]containerSerz, zjfb)
+ }
+ for zxhx := range z.conserz {
+ var zcxo uint32
+ zcxo, err = dc.ReadMapHeader()
+ if err != nil {
+ return
+ }
+ for zcxo > 0 {
+ zcxo--
+ field, err = dc.ReadMapKeyPtr()
+ if err != nil {
+ return
+ }
+ switch msgp.UnsafeString(field) {
+ case "t":
+ {
+ var zeff uint8
+ zeff, err = dc.ReadUint8()
+ z.conserz[zxhx].t = contype(zeff)
+ }
+ if err != nil {
+ return
+ }
+ case "r":
+ err = z.conserz[zxhx].r.DecodeMsg(dc)
+ if err != nil {
+ return
+ }
+ default:
+ err = dc.Skip()
+ if err != nil {
+ return
+ }
+ }
+ }
+ }
+ default:
+ err = dc.Skip()
+ if err != nil {
+ return
+ }
+ }
+ }
+ return
+}
+
+// Deprecated: EncodeMsg implements msgp.Encodable
+func (z *roaringArray) EncodeMsg(en *msgp.Writer) (err error) {
+ // map header, size 4
+ // write "keys"
+ err = en.Append(0x84, 0xa4, 0x6b, 0x65, 0x79, 0x73)
+ if err != nil {
+ return err
+ }
+ err = en.WriteArrayHeader(uint32(len(z.keys)))
+ if err != nil {
+ return
+ }
+ for zhct := range z.keys {
+ err = en.WriteUint16(z.keys[zhct])
+ if err != nil {
+ return
+ }
+ }
+ // write "needCopyOnWrite"
+ err = en.Append(0xaf, 0x6e, 0x65, 0x65, 0x64, 0x43, 0x6f, 0x70, 0x79, 0x4f, 0x6e, 0x57, 0x72, 0x69, 0x74, 0x65)
+ if err != nil {
+ return err
+ }
+ err = en.WriteArrayHeader(uint32(len(z.needCopyOnWrite)))
+ if err != nil {
+ return
+ }
+ for zcua := range z.needCopyOnWrite {
+ err = en.WriteBool(z.needCopyOnWrite[zcua])
+ if err != nil {
+ return
+ }
+ }
+ // write "copyOnWrite"
+ err = en.Append(0xab, 0x63, 0x6f, 0x70, 0x79, 0x4f, 0x6e, 0x57, 0x72, 0x69, 0x74, 0x65)
+ if err != nil {
+ return err
+ }
+ err = en.WriteBool(z.copyOnWrite)
+ if err != nil {
+ return
+ }
+ // write "conserz"
+ err = en.Append(0xa7, 0x63, 0x6f, 0x6e, 0x73, 0x65, 0x72, 0x7a)
+ if err != nil {
+ return err
+ }
+ err = en.WriteArrayHeader(uint32(len(z.conserz)))
+ if err != nil {
+ return
+ }
+ for zxhx := range z.conserz {
+ // map header, size 2
+ // write "t"
+ err = en.Append(0x82, 0xa1, 0x74)
+ if err != nil {
+ return err
+ }
+ err = en.WriteUint8(uint8(z.conserz[zxhx].t))
+ if err != nil {
+ return
+ }
+ // write "r"
+ err = en.Append(0xa1, 0x72)
+ if err != nil {
+ return err
+ }
+ err = z.conserz[zxhx].r.EncodeMsg(en)
+ if err != nil {
+ return
+ }
+ }
+ return
+}
+
+// Deprecated: MarshalMsg implements msgp.Marshaler
+func (z *roaringArray) MarshalMsg(b []byte) (o []byte, err error) {
+ o = msgp.Require(b, z.Msgsize())
+ // map header, size 4
+ // string "keys"
+ o = append(o, 0x84, 0xa4, 0x6b, 0x65, 0x79, 0x73)
+ o = msgp.AppendArrayHeader(o, uint32(len(z.keys)))
+ for zhct := range z.keys {
+ o = msgp.AppendUint16(o, z.keys[zhct])
+ }
+ // string "needCopyOnWrite"
+ o = append(o, 0xaf, 0x6e, 0x65, 0x65, 0x64, 0x43, 0x6f, 0x70, 0x79, 0x4f, 0x6e, 0x57, 0x72, 0x69, 0x74, 0x65)
+ o = msgp.AppendArrayHeader(o, uint32(len(z.needCopyOnWrite)))
+ for zcua := range z.needCopyOnWrite {
+ o = msgp.AppendBool(o, z.needCopyOnWrite[zcua])
+ }
+ // string "copyOnWrite"
+ o = append(o, 0xab, 0x63, 0x6f, 0x70, 0x79, 0x4f, 0x6e, 0x57, 0x72, 0x69, 0x74, 0x65)
+ o = msgp.AppendBool(o, z.copyOnWrite)
+ // string "conserz"
+ o = append(o, 0xa7, 0x63, 0x6f, 0x6e, 0x73, 0x65, 0x72, 0x7a)
+ o = msgp.AppendArrayHeader(o, uint32(len(z.conserz)))
+ for zxhx := range z.conserz {
+ // map header, size 2
+ // string "t"
+ o = append(o, 0x82, 0xa1, 0x74)
+ o = msgp.AppendUint8(o, uint8(z.conserz[zxhx].t))
+ // string "r"
+ o = append(o, 0xa1, 0x72)
+ o, err = z.conserz[zxhx].r.MarshalMsg(o)
+ if err != nil {
+ return
+ }
+ }
+ return
+}
+
+// Deprecated: UnmarshalMsg implements msgp.Unmarshaler
+func (z *roaringArray) UnmarshalMsg(bts []byte) (o []byte, err error) {
+ var field []byte
+ _ = field
+ var zrsw uint32
+ zrsw, bts, err = msgp.ReadMapHeaderBytes(bts)
+ if err != nil {
+ return
+ }
+ for zrsw > 0 {
+ zrsw--
+ field, bts, err = msgp.ReadMapKeyZC(bts)
+ if err != nil {
+ return
+ }
+ switch msgp.UnsafeString(field) {
+ case "keys":
+ var zxpk uint32
+ zxpk, bts, err = msgp.ReadArrayHeaderBytes(bts)
+ if err != nil {
+ return
+ }
+ if cap(z.keys) >= int(zxpk) {
+ z.keys = (z.keys)[:zxpk]
+ } else {
+ z.keys = make([]uint16, zxpk)
+ }
+ for zhct := range z.keys {
+ z.keys[zhct], bts, err = msgp.ReadUint16Bytes(bts)
+ if err != nil {
+ return
+ }
+ }
+ case "needCopyOnWrite":
+ var zdnj uint32
+ zdnj, bts, err = msgp.ReadArrayHeaderBytes(bts)
+ if err != nil {
+ return
+ }
+ if cap(z.needCopyOnWrite) >= int(zdnj) {
+ z.needCopyOnWrite = (z.needCopyOnWrite)[:zdnj]
+ } else {
+ z.needCopyOnWrite = make([]bool, zdnj)
+ }
+ for zcua := range z.needCopyOnWrite {
+ z.needCopyOnWrite[zcua], bts, err = msgp.ReadBoolBytes(bts)
+ if err != nil {
+ return
+ }
+ }
+ case "copyOnWrite":
+ z.copyOnWrite, bts, err = msgp.ReadBoolBytes(bts)
+ if err != nil {
+ return
+ }
+ case "conserz":
+ var zobc uint32
+ zobc, bts, err = msgp.ReadArrayHeaderBytes(bts)
+ if err != nil {
+ return
+ }
+ if cap(z.conserz) >= int(zobc) {
+ z.conserz = (z.conserz)[:zobc]
+ } else {
+ z.conserz = make([]containerSerz, zobc)
+ }
+ for zxhx := range z.conserz {
+ var zsnv uint32
+ zsnv, bts, err = msgp.ReadMapHeaderBytes(bts)
+ if err != nil {
+ return
+ }
+ for zsnv > 0 {
+ zsnv--
+ field, bts, err = msgp.ReadMapKeyZC(bts)
+ if err != nil {
+ return
+ }
+ switch msgp.UnsafeString(field) {
+ case "t":
+ {
+ var zkgt uint8
+ zkgt, bts, err = msgp.ReadUint8Bytes(bts)
+ z.conserz[zxhx].t = contype(zkgt)
+ }
+ if err != nil {
+ return
+ }
+ case "r":
+ bts, err = z.conserz[zxhx].r.UnmarshalMsg(bts)
+ if err != nil {
+ return
+ }
+ default:
+ bts, err = msgp.Skip(bts)
+ if err != nil {
+ return
+ }
+ }
+ }
+ }
+ default:
+ bts, err = msgp.Skip(bts)
+ if err != nil {
+ return
+ }
+ }
+ }
+ o = bts
+ return
+}
+
+// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *roaringArray) Msgsize() (s int) {
+ s = 1 + 5 + msgp.ArrayHeaderSize + (len(z.keys) * (msgp.Uint16Size)) + 16 + msgp.ArrayHeaderSize + (len(z.needCopyOnWrite) * (msgp.BoolSize)) + 12 + msgp.BoolSize + 8 + msgp.ArrayHeaderSize
+ for zxhx := range z.conserz {
+ s += 1 + 2 + msgp.Uint8Size + 2 + z.conserz[zxhx].r.Msgsize()
+ }
+ return
+}
diff --git a/vendor/github.com/RoaringBitmap/roaring/runcontainer.go b/vendor/github.com/RoaringBitmap/roaring/runcontainer.go
index 7098ba2..eed894b 100644
--- a/vendor/github.com/RoaringBitmap/roaring/runcontainer.go
+++ b/vendor/github.com/RoaringBitmap/roaring/runcontainer.go
@@ -44,11 +44,16 @@ import (
"unsafe"
)
+//go:generate msgp -unexported
+
// runContainer16 does run-length encoding of sets of
// uint16 integers.
type runContainer16 struct {
- // iv is a slice of sorted, non-overlapping, non-adjacent intervals.
- iv []interval16
+ iv []interval16
+ card int64
+
+ // avoid allocation during search
+ myOpts searchOptions `msg:"-"`
}
// interval16 is the internal to runContainer16
@@ -71,8 +76,8 @@ func newInterval16Range(start, last uint16) interval16 {
}
// runlen returns the count of integers in the interval.
-func (iv interval16) runlen() int {
- return int(iv.length) + 1
+func (iv interval16) runlen() int64 {
+ return int64(iv.length) + 1
}
func (iv interval16) last() uint16 {
@@ -115,6 +120,8 @@ func (p uint16Slice) Less(i, j int) bool { return p[i] < p[j] }
// Swap swaps elements i and j.
func (p uint16Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
+//msgp:ignore addHelper
+
// addHelper helps build a runContainer16.
type addHelper16 struct {
runstart uint16
@@ -194,6 +201,7 @@ func newRunContainer16FromVals(alreadySorted bool, vals ...uint16) *runContainer
ah.storeIval(ah.runstart, ah.runlen)
}
rc.iv = ah.m
+ rc.card = int64(ah.actuallyAdded)
return rc
}
@@ -254,8 +262,10 @@ func newRunContainer16FromBitmapContainer(bc *bitmapContainer) *runContainer16 {
}
+//
// newRunContainer16FromArray populates a new
// runContainer16 from the contents of arr.
+//
func newRunContainer16FromArray(arr *arrayContainer) *runContainer16 {
// keep this in sync with newRunContainer16FromVals above
@@ -281,6 +291,7 @@ func newRunContainer16FromArray(arr *arrayContainer) *runContainer16 {
ah.storeIval(ah.runstart, ah.runlen)
}
rc.iv = ah.m
+ rc.card = int64(ah.actuallyAdded)
return rc
}
@@ -297,6 +308,7 @@ func (rc *runContainer16) set(alreadySorted bool, vals ...uint16) {
rc2 := newRunContainer16FromVals(alreadySorted, vals...)
un := rc.union(rc2)
rc.iv = un.iv
+ rc.card = 0
}
// canMerge returns true iff the intervals
@@ -304,10 +316,10 @@ func (rc *runContainer16) set(alreadySorted bool, vals ...uint16) {
// contiguous and so can be merged into
// a single interval.
func canMerge16(a, b interval16) bool {
- if int(a.last())+1 < int(b.start) {
+ if int64(a.last())+1 < int64(b.start) {
return false
}
- return int(b.last())+1 >= int(a.start)
+ return int64(b.last())+1 >= int64(a.start)
}
// haveOverlap differs from canMerge in that
@@ -316,10 +328,10 @@ func canMerge16(a, b interval16) bool {
// it would be the empty set, and we return
// false).
func haveOverlap16(a, b interval16) bool {
- if int(a.last())+1 <= int(b.start) {
+ if int64(a.last())+1 <= int64(b.start) {
return false
}
- return int(b.last())+1 > int(a.start)
+ return int64(b.last())+1 > int64(a.start)
}
// mergeInterval16s joins a and b into a
@@ -380,11 +392,11 @@ func (rc *runContainer16) union(b *runContainer16) *runContainer16 {
var m []interval16
- alim := int(len(rc.iv))
- blim := int(len(b.iv))
+ alim := int64(len(rc.iv))
+ blim := int64(len(b.iv))
- var na int // next from a
- var nb int // next from b
+ var na int64 // next from a
+ var nb int64 // next from b
// merged holds the current merge output, which might
// get additional merges before being appended to m.
@@ -404,12 +416,12 @@ func (rc *runContainer16) union(b *runContainer16) *runContainer16 {
mergedUpdated := false
if canMerge16(cura, merged) {
merged = mergeInterval16s(cura, merged)
- na = rc.indexOfIntervalAtOrAfter(int(merged.last())+1, na+1)
+ na = rc.indexOfIntervalAtOrAfter(int64(merged.last())+1, na+1)
mergedUpdated = true
}
if canMerge16(curb, merged) {
merged = mergeInterval16s(curb, merged)
- nb = b.indexOfIntervalAtOrAfter(int(merged.last())+1, nb+1)
+ nb = b.indexOfIntervalAtOrAfter(int64(merged.last())+1, nb+1)
mergedUpdated = true
}
if !mergedUpdated {
@@ -432,8 +444,8 @@ func (rc *runContainer16) union(b *runContainer16) *runContainer16 {
} else {
merged = mergeInterval16s(cura, curb)
mergedUsed = true
- na = rc.indexOfIntervalAtOrAfter(int(merged.last())+1, na+1)
- nb = b.indexOfIntervalAtOrAfter(int(merged.last())+1, nb+1)
+ na = rc.indexOfIntervalAtOrAfter(int64(merged.last())+1, na+1)
+ nb = b.indexOfIntervalAtOrAfter(int64(merged.last())+1, nb+1)
}
}
}
@@ -452,7 +464,7 @@ func (rc *runContainer16) union(b *runContainer16) *runContainer16 {
cura = rc.iv[na]
if canMerge16(cura, merged) {
merged = mergeInterval16s(cura, merged)
- na = rc.indexOfIntervalAtOrAfter(int(merged.last())+1, na+1)
+ na = rc.indexOfIntervalAtOrAfter(int64(merged.last())+1, na+1)
} else {
break aAdds
}
@@ -466,7 +478,7 @@ func (rc *runContainer16) union(b *runContainer16) *runContainer16 {
curb = b.iv[nb]
if canMerge16(curb, merged) {
merged = mergeInterval16s(curb, merged)
- nb = b.indexOfIntervalAtOrAfter(int(merged.last())+1, nb+1)
+ nb = b.indexOfIntervalAtOrAfter(int64(merged.last())+1, nb+1)
} else {
break bAdds
}
@@ -488,17 +500,17 @@ func (rc *runContainer16) union(b *runContainer16) *runContainer16 {
}
// unionCardinality returns the cardinality of the merger of two runContainer16s, the union of rc and b.
-func (rc *runContainer16) unionCardinality(b *runContainer16) uint {
+func (rc *runContainer16) unionCardinality(b *runContainer16) uint64 {
// rc is also known as 'a' here, but golint insisted we
// call it rc for consistency with the rest of the methods.
- answer := uint(0)
+ answer := uint64(0)
- alim := int(len(rc.iv))
- blim := int(len(b.iv))
+ alim := int64(len(rc.iv))
+ blim := int64(len(b.iv))
- var na int // next from a
- var nb int // next from b
+ var na int64 // next from a
+ var nb int64 // next from b
// merged holds the current merge output, which might
// get additional merges before being appended to m.
@@ -518,18 +530,18 @@ func (rc *runContainer16) unionCardinality(b *runContainer16) uint {
mergedUpdated := false
if canMerge16(cura, merged) {
merged = mergeInterval16s(cura, merged)
- na = rc.indexOfIntervalAtOrAfter(int(merged.last())+1, na+1)
+ na = rc.indexOfIntervalAtOrAfter(int64(merged.last())+1, na+1)
mergedUpdated = true
}
if canMerge16(curb, merged) {
merged = mergeInterval16s(curb, merged)
- nb = b.indexOfIntervalAtOrAfter(int(merged.last())+1, nb+1)
+ nb = b.indexOfIntervalAtOrAfter(int64(merged.last())+1, nb+1)
mergedUpdated = true
}
if !mergedUpdated {
// we know that merged is disjoint from cura and curb
//m = append(m, merged)
- answer += uint(merged.last()) - uint(merged.start) + 1
+ answer += uint64(merged.last()) - uint64(merged.start) + 1
mergedUsed = false
}
continue
@@ -538,19 +550,19 @@ func (rc *runContainer16) unionCardinality(b *runContainer16) uint {
// !mergedUsed
if !canMerge16(cura, curb) {
if cura.start < curb.start {
- answer += uint(cura.last()) - uint(cura.start) + 1
+ answer += uint64(cura.last()) - uint64(cura.start) + 1
//m = append(m, cura)
na++
} else {
- answer += uint(curb.last()) - uint(curb.start) + 1
+ answer += uint64(curb.last()) - uint64(curb.start) + 1
//m = append(m, curb)
nb++
}
} else {
merged = mergeInterval16s(cura, curb)
mergedUsed = true
- na = rc.indexOfIntervalAtOrAfter(int(merged.last())+1, na+1)
- nb = b.indexOfIntervalAtOrAfter(int(merged.last())+1, nb+1)
+ na = rc.indexOfIntervalAtOrAfter(int64(merged.last())+1, na+1)
+ nb = b.indexOfIntervalAtOrAfter(int64(merged.last())+1, nb+1)
}
}
}
@@ -569,7 +581,7 @@ func (rc *runContainer16) unionCardinality(b *runContainer16) uint {
cura = rc.iv[na]
if canMerge16(cura, merged) {
merged = mergeInterval16s(cura, merged)
- na = rc.indexOfIntervalAtOrAfter(int(merged.last())+1, na+1)
+ na = rc.indexOfIntervalAtOrAfter(int64(merged.last())+1, na+1)
} else {
break aAdds
}
@@ -583,7 +595,7 @@ func (rc *runContainer16) unionCardinality(b *runContainer16) uint {
curb = b.iv[nb]
if canMerge16(curb, merged) {
merged = mergeInterval16s(curb, merged)
- nb = b.indexOfIntervalAtOrAfter(int(merged.last())+1, nb+1)
+ nb = b.indexOfIntervalAtOrAfter(int64(merged.last())+1, nb+1)
} else {
break bAdds
}
@@ -592,20 +604,23 @@ func (rc *runContainer16) unionCardinality(b *runContainer16) uint {
}
//m = append(m, merged)
- answer += uint(merged.last()) - uint(merged.start) + 1
+ answer += uint64(merged.last()) - uint64(merged.start) + 1
}
for _, r := range rc.iv[na:] {
- answer += uint(r.last()) - uint(r.start) + 1
+ answer += uint64(r.last()) - uint64(r.start) + 1
}
for _, r := range b.iv[nb:] {
- answer += uint(r.last()) - uint(r.start) + 1
+ answer += uint64(r.last()) - uint64(r.start) + 1
}
return answer
}
// indexOfIntervalAtOrAfter is a helper for union.
-func (rc *runContainer16) indexOfIntervalAtOrAfter(key int, startIndex int) int {
- w, already, _ := rc.searchRange(key, startIndex, 0)
+func (rc *runContainer16) indexOfIntervalAtOrAfter(key int64, startIndex int64) int64 {
+ rc.myOpts.startIndex = startIndex
+ rc.myOpts.endxIndex = 0
+
+ w, already, _ := rc.search(key, &rc.myOpts)
if already {
return w
}
@@ -617,8 +632,8 @@ func (rc *runContainer16) indexOfIntervalAtOrAfter(key int, startIndex int) int
func (rc *runContainer16) intersect(b *runContainer16) *runContainer16 {
a := rc
- numa := int(len(a.iv))
- numb := int(len(b.iv))
+ numa := int64(len(a.iv))
+ numb := int64(len(b.iv))
res := &runContainer16{}
if numa == 0 || numb == 0 {
return res
@@ -632,21 +647,21 @@ func (rc *runContainer16) intersect(b *runContainer16) *runContainer16 {
var output []interval16
- var acuri int
- var bcuri int
+ var acuri int64
+ var bcuri int64
- astart := int(a.iv[acuri].start)
- bstart := int(b.iv[bcuri].start)
+ astart := int64(a.iv[acuri].start)
+ bstart := int64(b.iv[bcuri].start)
var intersection interval16
- var leftoverstart int
+ var leftoverstart int64
var isOverlap, isLeftoverA, isLeftoverB bool
var done bool
toploop:
for acuri < numa && bcuri < numb {
isOverlap, isLeftoverA, isLeftoverB, leftoverstart, intersection =
- intersectWithLeftover16(astart, int(a.iv[acuri].last()), bstart, int(b.iv[bcuri].last()))
+ intersectWithLeftover16(astart, int64(a.iv[acuri].last()), bstart, int64(b.iv[bcuri].last()))
if !isOverlap {
switch {
@@ -655,14 +670,17 @@ toploop:
if done {
break toploop
}
- astart = int(a.iv[acuri].start)
+ astart = int64(a.iv[acuri].start)
case astart > bstart:
bcuri, done = b.findNextIntervalThatIntersectsStartingFrom(bcuri+1, astart)
if done {
break toploop
}
- bstart = int(b.iv[bcuri].start)
+ bstart = int64(b.iv[bcuri].start)
+
+ //default:
+ // panic("impossible that astart == bstart, since !isOverlap")
}
} else {
@@ -677,7 +695,7 @@ toploop:
if bcuri >= numb {
break toploop
}
- bstart = int(b.iv[bcuri].start)
+ bstart = int64(b.iv[bcuri].start)
case isLeftoverB:
// note that we change bstart without advancing bcuri,
// since we need to capture any 2ndary intersections with b.iv[bcuri]
@@ -686,23 +704,27 @@ toploop:
if acuri >= numa {
break toploop
}
- astart = int(a.iv[acuri].start)
+ astart = int64(a.iv[acuri].start)
default:
// neither had leftover, both completely consumed
+ // optionally, assert for sanity:
+ //if a.iv[acuri].endx != b.iv[bcuri].endx {
+ // panic("huh? should only be possible that endx agree now!")
+ //}
// advance to next a interval
acuri++
if acuri >= numa {
break toploop
}
- astart = int(a.iv[acuri].start)
+ astart = int64(a.iv[acuri].start)
// advance to next b interval
bcuri++
if bcuri >= numb {
break toploop
}
- bstart = int(b.iv[bcuri].start)
+ bstart = int64(b.iv[bcuri].start)
}
}
} // end for toploop
@@ -717,12 +739,12 @@ toploop:
// intersectCardinality returns the cardinality of the
// intersection of rc (also known as 'a') and b.
-func (rc *runContainer16) intersectCardinality(b *runContainer16) int {
- answer := int(0)
+func (rc *runContainer16) intersectCardinality(b *runContainer16) int64 {
+ answer := int64(0)
a := rc
- numa := int(len(a.iv))
- numb := int(len(b.iv))
+ numa := int64(len(a.iv))
+ numb := int64(len(b.iv))
if numa == 0 || numb == 0 {
return 0
}
@@ -733,14 +755,14 @@ func (rc *runContainer16) intersectCardinality(b *runContainer16) int {
}
}
- var acuri int
- var bcuri int
+ var acuri int64
+ var bcuri int64
- astart := int(a.iv[acuri].start)
- bstart := int(b.iv[bcuri].start)
+ astart := int64(a.iv[acuri].start)
+ bstart := int64(b.iv[bcuri].start)
var intersection interval16
- var leftoverstart int
+ var leftoverstart int64
var isOverlap, isLeftoverA, isLeftoverB bool
var done bool
pass := 0
@@ -749,7 +771,7 @@ toploop:
pass++
isOverlap, isLeftoverA, isLeftoverB, leftoverstart, intersection =
- intersectWithLeftover16(astart, int(a.iv[acuri].last()), bstart, int(b.iv[bcuri].last()))
+ intersectWithLeftover16(astart, int64(a.iv[acuri].last()), bstart, int64(b.iv[bcuri].last()))
if !isOverlap {
switch {
@@ -758,19 +780,22 @@ toploop:
if done {
break toploop
}
- astart = int(a.iv[acuri].start)
+ astart = int64(a.iv[acuri].start)
case astart > bstart:
bcuri, done = b.findNextIntervalThatIntersectsStartingFrom(bcuri+1, astart)
if done {
break toploop
}
- bstart = int(b.iv[bcuri].start)
+ bstart = int64(b.iv[bcuri].start)
+
+ //default:
+ // panic("impossible that astart == bstart, since !isOverlap")
}
} else {
// isOverlap
- answer += int(intersection.last()) - int(intersection.start) + 1
+ answer += int64(intersection.last()) - int64(intersection.start) + 1
switch {
case isLeftoverA:
// note that we change astart without advancing acuri,
@@ -780,7 +805,7 @@ toploop:
if bcuri >= numb {
break toploop
}
- bstart = int(b.iv[bcuri].start)
+ bstart = int64(b.iv[bcuri].start)
case isLeftoverB:
// note that we change bstart without advancing bcuri,
// since we need to capture any 2ndary intersections with b.iv[bcuri]
@@ -789,23 +814,27 @@ toploop:
if acuri >= numa {
break toploop
}
- astart = int(a.iv[acuri].start)
+ astart = int64(a.iv[acuri].start)
default:
// neither had leftover, both completely consumed
+ // optionally, assert for sanity:
+ //if a.iv[acuri].endx != b.iv[bcuri].endx {
+ // panic("huh? should only be possible that endx agree now!")
+ //}
// advance to next a interval
acuri++
if acuri >= numa {
break toploop
}
- astart = int(a.iv[acuri].start)
+ astart = int64(a.iv[acuri].start)
// advance to next b interval
bcuri++
if bcuri >= numb {
break toploop
}
- bstart = int(b.iv[bcuri].start)
+ bstart = int64(b.iv[bcuri].start)
}
}
} // end for toploop
@@ -815,7 +844,7 @@ toploop:
// get returns true iff key is in the container.
func (rc *runContainer16) contains(key uint16) bool {
- _, in, _ := rc.search(int(key))
+ _, in, _ := rc.search(int64(key), nil)
return in
}
@@ -824,7 +853,22 @@ func (rc *runContainer16) numIntervals() int {
return len(rc.iv)
}
-// searchRange returns alreadyPresent to indicate if the
+// searchOptions allows us to accelerate search with
+// prior knowledge of (mostly lower) bounds. This is used by Union
+// and Intersect.
+type searchOptions struct {
+ // start here instead of at 0
+ startIndex int64
+
+ // upper bound instead of len(rc.iv);
+ // endxIndex == 0 means ignore the bound and use
+ // endxIndex == n ==len(rc.iv) which is also
+ // naturally the default for search()
+ // when opt = nil.
+ endxIndex int64
+}
+
+// search returns alreadyPresent to indicate if the
// key is already in one of our interval16s.
//
// If key is alreadyPresent, then whichInterval16 tells
@@ -833,30 +877,39 @@ func (rc *runContainer16) numIntervals() int {
// If key is not already present, then whichInterval16 is
// set as follows:
//
-// a) whichInterval16 == len(rc.iv)-1 if key is beyond our
-// last interval16 in rc.iv;
+// a) whichInterval16 == len(rc.iv)-1 if key is beyond our
+// last interval16 in rc.iv;
//
-// b) whichInterval16 == -1 if key is before our first
-// interval16 in rc.iv;
+// b) whichInterval16 == -1 if key is before our first
+// interval16 in rc.iv;
//
-// c) whichInterval16 is set to the minimum index of rc.iv
-// which comes strictly before the key;
-// so rc.iv[whichInterval16].last < key,
-// and if whichInterval16+1 exists, then key < rc.iv[whichInterval16+1].start
-// (Note that whichInterval16+1 won't exist when
-// whichInterval16 is the last interval.)
+// c) whichInterval16 is set to the minimum index of rc.iv
+// which comes strictly before the key;
+// so rc.iv[whichInterval16].last < key,
+// and if whichInterval16+1 exists, then key < rc.iv[whichInterval16+1].start
+// (Note that whichInterval16+1 won't exist when
+// whichInterval16 is the last interval.)
//
// runContainer16.search always returns whichInterval16 < len(rc.iv).
//
-// The search space is from startIndex to endxIndex. If endxIndex is set to zero, then there
-// no upper bound.
-func (rc *runContainer16) searchRange(key int, startIndex int, endxIndex int) (whichInterval16 int, alreadyPresent bool, numCompares int) {
- n := int(len(rc.iv))
+// If not nil, opts can be used to further restrict
+// the search space.
+//
+func (rc *runContainer16) search(key int64, opts *searchOptions) (whichInterval16 int64, alreadyPresent bool, numCompares int) {
+ n := int64(len(rc.iv))
if n == 0 {
return -1, false, 0
}
- if endxIndex == 0 {
- endxIndex = n
+
+ startIndex := int64(0)
+ endxIndex := n
+ if opts != nil {
+ startIndex = opts.startIndex
+
+ // let endxIndex == 0 mean no effect
+ if opts.endxIndex > 0 {
+ endxIndex = opts.endxIndex
+ }
}
// sort.Search returns the smallest index i
@@ -874,7 +927,7 @@ func (rc *runContainer16) searchRange(key int, startIndex int, endxIndex int) (w
h := i + (j-i)/2 // avoid overflow when computing h as the bisector
// i <= h < j
numCompares++
- if !(key < int(rc.iv[h].start)) {
+ if !(key < int64(rc.iv[h].start)) {
i = h + 1
} else {
j = h
@@ -894,7 +947,7 @@ func (rc *runContainer16) searchRange(key int, startIndex int, endxIndex int) (w
if below == n {
// all falses => key is >= start of all interval16s
// ... so does it belong to the last interval16?
- if key < int(rc.iv[n-1].last())+1 {
+ if key < int64(rc.iv[n-1].last())+1 {
// yes, it belongs to the last interval16
alreadyPresent = true
return
@@ -915,7 +968,7 @@ func (rc *runContainer16) searchRange(key int, startIndex int, endxIndex int) (w
// key is < rc.iv[below].start
// is key in below-1 interval16?
- if key >= int(rc.iv[below-1].start) && key < int(rc.iv[below-1].last())+1 {
+ if key >= int64(rc.iv[below-1].start) && key < int64(rc.iv[below-1].last())+1 {
// yes, it is. key is in below-1 interval16.
alreadyPresent = true
return
@@ -926,54 +979,28 @@ func (rc *runContainer16) searchRange(key int, startIndex int, endxIndex int) (w
return
}
-// search returns alreadyPresent to indicate if the
-// key is already in one of our interval16s.
-//
-// If key is alreadyPresent, then whichInterval16 tells
-// you where.
-//
-// If key is not already present, then whichInterval16 is
-// set as follows:
-//
-// a) whichInterval16 == len(rc.iv)-1 if key is beyond our
-// last interval16 in rc.iv;
-//
-// b) whichInterval16 == -1 if key is before our first
-// interval16 in rc.iv;
-//
-// c) whichInterval16 is set to the minimum index of rc.iv
-// which comes strictly before the key;
-// so rc.iv[whichInterval16].last < key,
-// and if whichInterval16+1 exists, then key < rc.iv[whichInterval16+1].start
-// (Note that whichInterval16+1 won't exist when
-// whichInterval16 is the last interval.)
-//
-// runContainer16.search always returns whichInterval16 < len(rc.iv).
-func (rc *runContainer16) search(key int) (whichInterval16 int, alreadyPresent bool, numCompares int) {
- return rc.searchRange(key, 0, 0)
-}
-
-// getCardinality returns the count of the integers stored in the
-// runContainer16. The running complexity depends on the size
-// of the container.
-func (rc *runContainer16) getCardinality() int {
+// cardinality returns the count of the integers stored in the
+// runContainer16.
+func (rc *runContainer16) cardinality() int64 {
+ if len(rc.iv) == 0 {
+ rc.card = 0
+ return 0
+ }
+ if rc.card > 0 {
+ return rc.card // already cached
+ }
// have to compute it
- n := 0
+ var n int64
for _, p := range rc.iv {
n += p.runlen()
}
+ rc.card = n // cache it
return n
}
-// isEmpty returns true if the container is empty.
-// It runs in constant time.
-func (rc *runContainer16) isEmpty() bool {
- return len(rc.iv) == 0
-}
-
// AsSlice decompresses the contents into a []uint16 slice.
func (rc *runContainer16) AsSlice() []uint16 {
- s := make([]uint16, rc.getCardinality())
+ s := make([]uint16, rc.cardinality())
j := 0
for _, p := range rc.iv {
for i := p.start; i <= p.last(); i++ {
@@ -991,6 +1018,7 @@ func newRunContainer16() *runContainer16 {
// newRunContainer16CopyIv creates a run container, initializing
// with a copy of the supplied iv slice.
+//
func newRunContainer16CopyIv(iv []interval16) *runContainer16 {
rc := &runContainer16{
iv: make([]interval16, len(iv)),
@@ -1007,6 +1035,7 @@ func (rc *runContainer16) Clone() *runContainer16 {
// newRunContainer16TakeOwnership returns a new runContainer16
// backed by the provided iv slice, which we will
// assume exclusive control over from now on.
+//
func newRunContainer16TakeOwnership(iv []interval16) *runContainer16 {
rc := &runContainer16{
iv: iv,
@@ -1041,15 +1070,19 @@ func (rc *runContainer16) Add(k uint16) (wasNew bool) {
// but note that some unit tests use this method to build up test
// runcontainers without calling runOptimize
- k64 := int(k)
+ k64 := int64(k)
- index, present, _ := rc.search(k64)
+ index, present, _ := rc.search(k64, nil)
if present {
return // already there
}
wasNew = true
- n := int(len(rc.iv))
+ // increment card if it is cached already
+ if rc.card > 0 {
+ rc.card++
+ }
+ n := int64(len(rc.iv))
if index == -1 {
// we may need to extend the first run
if n > 0 {
@@ -1066,7 +1099,7 @@ func (rc *runContainer16) Add(k uint16) (wasNew bool) {
// are we off the end? handle both index == n and index == n-1:
if index >= n-1 {
- if int(rc.iv[n-1].last())+1 == k64 {
+ if int64(rc.iv[n-1].last())+1 == k64 {
rc.iv[n-1].length++
return
}
@@ -1085,7 +1118,7 @@ func (rc *runContainer16) Add(k uint16) (wasNew bool) {
right := index + 1
// are we fusing left and right by adding k?
- if int(rc.iv[left].last())+1 == k64 && int(rc.iv[right].start) == k64+1 {
+ if int64(rc.iv[left].last())+1 == k64 && int64(rc.iv[right].start) == k64+1 {
// fuse into left
rc.iv[left].length = rc.iv[right].last() - rc.iv[left].start
// remove redundant right
@@ -1094,14 +1127,14 @@ func (rc *runContainer16) Add(k uint16) (wasNew bool) {
}
// are we an addition to left?
- if int(rc.iv[left].last())+1 == k64 {
+ if int64(rc.iv[left].last())+1 == k64 {
// yes
rc.iv[left].length++
return
}
// are we an addition to right?
- if int(rc.iv[right].start) == k64+1 {
+ if int64(rc.iv[right].start) == k64+1 {
// yes
rc.iv[right].start = k
rc.iv[right].length++
@@ -1114,237 +1147,238 @@ func (rc *runContainer16) Add(k uint16) (wasNew bool) {
return
}
-// runIterator16 advice: you must call hasNext()
-// before calling next()/peekNext() to insure there are contents.
+//msgp:ignore runIterator
+
+// runIterator16 advice: you must call Next() at least once
+// before calling Cur(); and you should call HasNext()
+// before calling Next() to insure there are contents.
type runIterator16 struct {
rc *runContainer16
- curIndex int
+ curIndex int64
curPosInIndex uint16
+ curSeq int64
}
// newRunIterator16 returns a new empty run container.
func (rc *runContainer16) newRunIterator16() *runIterator16 {
- return &runIterator16{rc: rc, curIndex: 0, curPosInIndex: 0}
+ return &runIterator16{rc: rc, curIndex: -1}
}
-func (rc *runContainer16) iterate(cb func(x uint16) bool) bool {
- iterator := runIterator16{rc, 0, 0}
-
- for iterator.hasNext() {
- if !cb(iterator.next()) {
- return false
- }
- }
-
- return true
-}
-
-// hasNext returns false if calling next will panic. It
+// HasNext returns false if calling Next will panic. It
// returns true when there is at least one more value
// available in the iteration sequence.
func (ri *runIterator16) hasNext() bool {
- return int(len(ri.rc.iv)) > ri.curIndex+1 ||
- (int(len(ri.rc.iv)) == ri.curIndex+1 && ri.rc.iv[ri.curIndex].length >= ri.curPosInIndex)
-}
-
-// next returns the next value in the iteration sequence.
-func (ri *runIterator16) next() uint16 {
- next := ri.rc.iv[ri.curIndex].start + ri.curPosInIndex
-
- if ri.curPosInIndex == ri.rc.iv[ri.curIndex].length {
- ri.curPosInIndex = 0
- ri.curIndex++
- } else {
- ri.curPosInIndex++
+ if len(ri.rc.iv) == 0 {
+ return false
}
-
- return next
+ if ri.curIndex == -1 {
+ return true
+ }
+ return ri.curSeq+1 < ri.rc.cardinality()
}
-// peekNext returns the next value in the iteration sequence without advancing the iterator
-func (ri *runIterator16) peekNext() uint16 {
+// cur returns the current value pointed to by the iterator.
+func (ri *runIterator16) cur() uint16 {
return ri.rc.iv[ri.curIndex].start + ri.curPosInIndex
}
-// advanceIfNeeded advances as long as the next value is smaller than minval
-func (ri *runIterator16) advanceIfNeeded(minval uint16) {
- if !ri.hasNext() || ri.peekNext() >= minval {
- return
+// Next returns the next value in the iteration sequence.
+func (ri *runIterator16) next() uint16 {
+ if !ri.hasNext() {
+ panic("no Next available")
}
-
- // interval cannot be -1 because of minval > peekNext
- interval, isPresent, _ := ri.rc.searchRange(int(minval), ri.curIndex, int(len(ri.rc.iv)))
-
- // if the minval is present, set the curPosIndex at the right position
- if isPresent {
- ri.curIndex = interval
- ri.curPosInIndex = minval - ri.rc.iv[ri.curIndex].start
+ if ri.curIndex >= int64(len(ri.rc.iv)) {
+ panic("runIterator.Next() going beyond what is available")
+ }
+ if ri.curIndex == -1 {
+ // first time is special
+ ri.curIndex = 0
} else {
- // otherwise interval is set to to the minimum index of rc.iv
- // which comes strictly before the key, that's why we set the next interval
- ri.curIndex = interval + 1
- ri.curPosInIndex = 0
+ ri.curPosInIndex++
+ if int64(ri.rc.iv[ri.curIndex].start)+int64(ri.curPosInIndex) == int64(ri.rc.iv[ri.curIndex].last())+1 {
+ ri.curPosInIndex = 0
+ ri.curIndex++
+ }
+ ri.curSeq++
}
+ return ri.cur()
}
-// runReverseIterator16 advice: you must call hasNext()
+// remove removes the element that the iterator
+// is on from the run container. You can use
+// Cur if you want to double check what is about
+// to be deleted.
+func (ri *runIterator16) remove() uint16 {
+ n := ri.rc.cardinality()
+ if n == 0 {
+ panic("runIterator.Remove called on empty runContainer16")
+ }
+ cur := ri.cur()
+
+ ri.rc.deleteAt(&ri.curIndex, &ri.curPosInIndex, &ri.curSeq)
+ return cur
+}
+
+// runReverseIterator16 advice: you must call next() at least once
+// before calling cur(); and you should call hasNext()
// before calling next() to insure there are contents.
type runReverseIterator16 struct {
rc *runContainer16
- curIndex int // index into rc.iv
+ curIndex int64 // index into rc.iv
curPosInIndex uint16 // offset in rc.iv[curIndex]
+ curSeq int64 // 0->cardinality, performance optimization in hasNext()
}
// newRunReverseIterator16 returns a new empty run iterator.
func (rc *runContainer16) newRunReverseIterator16() *runReverseIterator16 {
- index := int(len(rc.iv)) - 1
- pos := uint16(0)
-
- if index >= 0 {
- pos = rc.iv[index].length
- }
-
- return &runReverseIterator16{
- rc: rc,
- curIndex: index,
- curPosInIndex: pos,
- }
+ return &runReverseIterator16{rc: rc, curIndex: -2}
}
// hasNext returns false if calling next will panic. It
// returns true when there is at least one more value
// available in the iteration sequence.
func (ri *runReverseIterator16) hasNext() bool {
- return ri.curIndex > 0 || ri.curIndex == 0 && ri.curPosInIndex >= 0
+ if len(ri.rc.iv) == 0 {
+ return false
+ }
+ if ri.curIndex == -2 {
+ return true
+ }
+ return ri.rc.cardinality()-ri.curSeq > 1
+}
+
+// cur returns the current value pointed to by the iterator.
+func (ri *runReverseIterator16) cur() uint16 {
+ return ri.rc.iv[ri.curIndex].start + ri.curPosInIndex
}
// next returns the next value in the iteration sequence.
func (ri *runReverseIterator16) next() uint16 {
- next := ri.rc.iv[ri.curIndex].start + ri.curPosInIndex
-
- if ri.curPosInIndex > 0 {
- ri.curPosInIndex--
+ if !ri.hasNext() {
+ panic("no next available")
+ }
+ if ri.curIndex == -1 {
+ panic("runReverseIterator.next() going beyond what is available")
+ }
+ if ri.curIndex == -2 {
+ // first time is special
+ ri.curIndex = int64(len(ri.rc.iv)) - 1
+ ri.curPosInIndex = ri.rc.iv[ri.curIndex].length
} else {
- ri.curIndex--
-
- if ri.curIndex >= 0 {
+ if ri.curPosInIndex > 0 {
+ ri.curPosInIndex--
+ } else {
+ ri.curIndex--
ri.curPosInIndex = ri.rc.iv[ri.curIndex].length
}
+ ri.curSeq++
}
-
- return next
+ return ri.cur()
}
-func (rc *runContainer16) newManyRunIterator16() *runIterator16 {
- return rc.newRunIterator16()
+// remove removes the element that the iterator
+// is on from the run container. You can use
+// cur if you want to double check what is about
+// to be deleted.
+func (ri *runReverseIterator16) remove() uint16 {
+ n := ri.rc.cardinality()
+ if n == 0 {
+ panic("runReverseIterator.Remove called on empty runContainer16")
+ }
+ cur := ri.cur()
+
+ ri.rc.deleteAt(&ri.curIndex, &ri.curPosInIndex, &ri.curSeq)
+ return cur
+}
+
+type manyRunIterator16 struct {
+ rc *runContainer16
+ curIndex int64
+ curPosInIndex uint16
+ curSeq int64
+}
+
+func (rc *runContainer16) newManyRunIterator16() *manyRunIterator16 {
+ return &manyRunIterator16{rc: rc, curIndex: -1}
+}
+
+func (ri *manyRunIterator16) hasNext() bool {
+ if len(ri.rc.iv) == 0 {
+ return false
+ }
+ if ri.curIndex == -1 {
+ return true
+ }
+ return ri.curSeq+1 < ri.rc.cardinality()
}
// hs are the high bits to include to avoid needing to reiterate over the buffer in NextMany
-func (ri *runIterator16) nextMany(hs uint32, buf []uint32) int {
+func (ri *manyRunIterator16) nextMany(hs uint32, buf []uint32) int {
n := 0
-
if !ri.hasNext() {
return n
}
-
// start and end are inclusive
for n < len(buf) {
- moreVals := 0
-
- if ri.rc.iv[ri.curIndex].length >= ri.curPosInIndex {
- // add as many as you can from this seq
- moreVals = minOfInt(int(ri.rc.iv[ri.curIndex].length-ri.curPosInIndex)+1, len(buf)-n)
- base := uint32(ri.rc.iv[ri.curIndex].start+ri.curPosInIndex) | hs
-
- // allows BCE
- buf2 := buf[n : n+moreVals]
- for i := range buf2 {
- buf2[i] = base + uint32(i)
- }
-
- // update values
- n += moreVals
- }
-
- if moreVals+int(ri.curPosInIndex) > int(ri.rc.iv[ri.curIndex].length) {
+ if ri.curIndex == -1 || int(ri.rc.iv[ri.curIndex].length-ri.curPosInIndex) <= 0 {
ri.curPosInIndex = 0
ri.curIndex++
-
- if ri.curIndex == int(len(ri.rc.iv)) {
+ if ri.curIndex == int64(len(ri.rc.iv)) {
break
}
- } else {
- ri.curPosInIndex += uint16(moreVals) //moreVals always fits in uint16
- }
- }
-
- return n
-}
-
-func (ri *runIterator16) nextMany64(hs uint64, buf []uint64) int {
- n := 0
-
- if !ri.hasNext() {
- return n
- }
-
- // start and end are inclusive
- for n < len(buf) {
- moreVals := 0
-
- if ri.rc.iv[ri.curIndex].length >= ri.curPosInIndex {
- // add as many as you can from this seq
- moreVals = minOfInt(int(ri.rc.iv[ri.curIndex].length-ri.curPosInIndex)+1, len(buf)-n)
- base := uint64(ri.rc.iv[ri.curIndex].start+ri.curPosInIndex) | hs
-
- // allows BCE
- buf2 := buf[n : n+moreVals]
- for i := range buf2 {
- buf2[i] = base + uint64(i)
+ buf[n] = uint32(ri.rc.iv[ri.curIndex].start) | hs
+ if ri.curIndex != 0 {
+ ri.curSeq++
}
+ n++
+ // not strictly necessarily due to len(buf)-n min check, but saves some work
+ continue
+ }
+ // add as many as you can from this seq
+ moreVals := minOfInt(int(ri.rc.iv[ri.curIndex].length-ri.curPosInIndex), len(buf)-n)
- // update values
- n += moreVals
+ base := uint32(ri.rc.iv[ri.curIndex].start+ri.curPosInIndex+1) | hs
+
+ // allows BCE
+ buf2 := buf[n : n+moreVals]
+ for i := range buf2 {
+ buf2[i] = base + uint32(i)
}
- if moreVals+int(ri.curPosInIndex) > int(ri.rc.iv[ri.curIndex].length) {
- ri.curPosInIndex = 0
- ri.curIndex++
-
- if ri.curIndex == int(len(ri.rc.iv)) {
- break
- }
- } else {
- ri.curPosInIndex += uint16(moreVals) //moreVals always fits in uint16
- }
+ // update values
+ ri.curPosInIndex += uint16(moreVals) //moreVals always fits in uint16
+ ri.curSeq += int64(moreVals)
+ n += moreVals
}
-
return n
}
// remove removes key from the container.
func (rc *runContainer16) removeKey(key uint16) (wasPresent bool) {
- var index int
- index, wasPresent, _ = rc.search(int(key))
+ var index int64
+ var curSeq int64
+ index, wasPresent, _ = rc.search(int64(key), nil)
if !wasPresent {
return // already removed, nothing to do.
}
pos := key - rc.iv[index].start
- rc.deleteAt(&index, &pos)
+ rc.deleteAt(&index, &pos, &curSeq)
return
}
// internal helper functions
-func (rc *runContainer16) deleteAt(curIndex *int, curPosInIndex *uint16) {
+func (rc *runContainer16) deleteAt(curIndex *int64, curPosInIndex *uint16, curSeq *int64) {
+ rc.card--
+ *curSeq--
ci := *curIndex
pos := *curPosInIndex
// are we first, last, or in the middle of our interval16?
switch {
case pos == 0:
- if int(rc.iv[ci].length) == 0 {
+ if int64(rc.iv[ci].length) == 0 {
// our interval disappears
rc.iv = append(rc.iv[:ci], rc.iv[ci+1:]...)
// curIndex stays the same, since the delete did
@@ -1365,8 +1399,8 @@ func (rc *runContainer16) deleteAt(curIndex *int, curPosInIndex *uint16) {
// split into two, adding an interval16
new0 := newInterval16Range(rc.iv[ci].start, rc.iv[ci].start+*curPosInIndex-1)
- new1start := int(rc.iv[ci].start+*curPosInIndex) + 1
- if new1start > int(MaxUint16) {
+ new1start := int64(rc.iv[ci].start+*curPosInIndex) + 1
+ if new1start > int64(MaxUint16) {
panic("overflow?!?!")
}
new1 := newInterval16Range(uint16(new1start), rc.iv[ci].last())
@@ -1379,14 +1413,14 @@ func (rc *runContainer16) deleteAt(curIndex *int, curPosInIndex *uint16) {
}
-func have4Overlap16(astart, alast, bstart, blast int) bool {
+func have4Overlap16(astart, alast, bstart, blast int64) bool {
if alast+1 <= bstart {
return false
}
return blast+1 > astart
}
-func intersectWithLeftover16(astart, alast, bstart, blast int) (isOverlap, isLeftoverA, isLeftoverB bool, leftoverstart int, intersection interval16) {
+func intersectWithLeftover16(astart, alast, bstart, blast int64) (isOverlap, isLeftoverA, isLeftoverB bool, leftoverstart int64, intersection interval16) {
if !have4Overlap16(astart, alast, bstart, blast) {
return
}
@@ -1416,13 +1450,17 @@ func intersectWithLeftover16(astart, alast, bstart, blast int) (isOverlap, isLef
return
}
-func (rc *runContainer16) findNextIntervalThatIntersectsStartingFrom(startIndex int, key int) (index int, done bool) {
- w, _, _ := rc.searchRange(key, startIndex, 0)
+func (rc *runContainer16) findNextIntervalThatIntersectsStartingFrom(startIndex int64, key int64) (index int64, done bool) {
+
+ rc.myOpts.startIndex = startIndex
+ rc.myOpts.endxIndex = 0
+
+ w, _, _ := rc.search(key, &rc.myOpts)
// rc.search always returns w < len(rc.iv)
if w < startIndex {
// not found and comes before lower bound startIndex,
// so just use the lower bound.
- if startIndex == int(len(rc.iv)) {
+ if startIndex == int64(len(rc.iv)) {
// also this bump up means that we are done
return startIndex, true
}
@@ -1440,6 +1478,25 @@ func sliceToString16(m []interval16) string {
return s
}
+// selectInt16 returns the j-th value in the container.
+// We panic of j is out of bounds.
+func (rc *runContainer16) selectInt16(j uint16) int {
+ n := rc.cardinality()
+ if int64(j) > n {
+ panic(fmt.Sprintf("Cannot select %v since Cardinality is %v", j, n))
+ }
+
+ var offset int64
+ for k := range rc.iv {
+ nextOffset := offset + rc.iv[k].runlen() + 1
+ if nextOffset > int64(j) {
+ return int(int64(rc.iv[k].start) + (int64(j) - offset))
+ }
+ offset = nextOffset
+ }
+ panic(fmt.Sprintf("Cannot select %v since Cardinality is %v", j, n))
+}
+
// helper for invert
func (rc *runContainer16) invertlastInterval(origin uint16, lastIdx int) []interval16 {
cur := rc.iv[lastIdx]
@@ -1471,7 +1528,7 @@ func (rc *runContainer16) invert() *runContainer16 {
case 1:
return &runContainer16{iv: rc.invertlastInterval(0, 0)}
}
- var invstart int
+ var invstart int64
ult := ni - 1
for i, cur := range rc.iv {
if i == ult {
@@ -1490,7 +1547,7 @@ func (rc *runContainer16) invert() *runContainer16 {
if cur.start > 0 {
m = append(m, newInterval16Range(uint16(invstart), cur.start-1))
}
- invstart = int(cur.last() + 1)
+ invstart = int64(cur.last() + 1)
}
return &runContainer16{iv: m}
}
@@ -1503,7 +1560,7 @@ func (iv interval16) isSuperSetOf(b interval16) bool {
return iv.start <= b.start && b.last() <= iv.last()
}
-func (iv interval16) subtractInterval(del interval16) (left []interval16, delcount int) {
+func (iv interval16) subtractInterval(del interval16) (left []interval16, delcount int64) {
isect, isEmpty := intersectInterval16s(iv, del)
if isEmpty {
@@ -1528,7 +1585,7 @@ func (iv interval16) subtractInterval(del interval16) (left []interval16, delcou
func (rc *runContainer16) isubtract(del interval16) {
origiv := make([]interval16, len(rc.iv))
copy(origiv, rc.iv)
- n := int(len(rc.iv))
+ n := int64(len(rc.iv))
if n == 0 {
return // already done.
}
@@ -1539,8 +1596,9 @@ func (rc *runContainer16) isubtract(del interval16) {
}
// INVAR there is some intersection between rc and del
- istart, startAlready, _ := rc.search(int(del.start))
- ilast, lastAlready, _ := rc.search(int(del.last()))
+ istart, startAlready, _ := rc.search(int64(del.start), nil)
+ ilast, lastAlready, _ := rc.search(int64(del.last()), nil)
+ rc.card = -1
if istart == -1 {
if ilast == n-1 && !lastAlready {
rc.iv = nil
@@ -1555,8 +1613,8 @@ func (rc *runContainer16) isubtract(del interval16) {
// would overwrite values in iv b/c res0 can have len 2. so
// write to origiv instead.
lost := 1 + ilast - istart
- changeSize := int(len(res0)) - lost
- newSize := int(len(rc.iv)) + changeSize
+ changeSize := int64(len(res0)) - lost
+ newSize := int64(len(rc.iv)) + changeSize
// rc.iv = append(pre, caboose...)
// return
@@ -1564,19 +1622,19 @@ func (rc *runContainer16) isubtract(del interval16) {
if ilast != istart {
res1, _ := rc.iv[ilast].subtractInterval(del)
res0 = append(res0, res1...)
- changeSize = int(len(res0)) - lost
- newSize = int(len(rc.iv)) + changeSize
+ changeSize = int64(len(res0)) - lost
+ newSize = int64(len(rc.iv)) + changeSize
}
switch {
case changeSize < 0:
// shrink
- copy(rc.iv[istart+int(len(res0)):], rc.iv[ilast+1:])
- copy(rc.iv[istart:istart+int(len(res0))], res0)
+ copy(rc.iv[istart+int64(len(res0)):], rc.iv[ilast+1:])
+ copy(rc.iv[istart:istart+int64(len(res0))], res0)
rc.iv = rc.iv[:newSize]
return
case changeSize == 0:
// stay the same
- copy(rc.iv[istart:istart+int(len(res0))], res0)
+ copy(rc.iv[istart:istart+int64(len(res0))], res0)
return
default:
// changeSize > 0 is only possible when ilast == istart.
@@ -1633,7 +1691,7 @@ func (rc *runContainer16) isubtract(del interval16) {
// INVAR: ilast < n-1
lost := ilast - istart
changeSize := -lost
- newSize := int(len(rc.iv)) + changeSize
+ newSize := int64(len(rc.iv)) + changeSize
if changeSize != 0 {
copy(rc.iv[ilast+1+changeSize:], rc.iv[ilast+1:])
}
@@ -1650,8 +1708,8 @@ func (rc *runContainer16) isubtract(del interval16) {
rc.iv[istart] = res0[0]
}
lost := 1 + (ilast - istart)
- changeSize := int(len(res0)) - lost
- newSize := int(len(rc.iv)) + changeSize
+ changeSize := int64(len(res0)) - lost
+ newSize := int64(len(rc.iv)) + changeSize
if changeSize != 0 {
copy(rc.iv[ilast+1+changeSize:], rc.iv[ilast+1:])
}
@@ -1662,8 +1720,8 @@ func (rc *runContainer16) isubtract(del interval16) {
// we can only shrink or stay the same size
res1, _ := rc.iv[ilast].subtractInterval(del)
lost := ilast - istart
- changeSize := int(len(res1)) - lost
- newSize := int(len(rc.iv)) + changeSize
+ changeSize := int64(len(res1)) - lost
+ newSize := int64(len(rc.iv)) + changeSize
if changeSize != 0 {
// move the tail first to make room for res1
copy(rc.iv[ilast+1+changeSize:], rc.iv[ilast+1:])
@@ -1867,6 +1925,8 @@ func (rc *runContainer16) iand(a container) container {
}
func (rc *runContainer16) inplaceIntersect(rc2 *runContainer16) container {
+ // TODO: optimize by doing less allocation, possibly?
+ // sect will be new
sect := rc.intersect(rc2)
*rc = *sect
return rc
@@ -1920,21 +1980,20 @@ func (rc *runContainer16) andNot(a container) container {
panic("unsupported container type")
}
-func (rc *runContainer16) fillLeastSignificant16bits(x []uint32, i int, mask uint32) int {
- k := i
- var val int
+func (rc *runContainer16) fillLeastSignificant16bits(x []uint32, i int, mask uint32) {
+ k := 0
+ var val int64
for _, p := range rc.iv {
n := p.runlen()
- for j := int(0); j < n; j++ {
- val = int(p.start) + j
- x[k] = uint32(val) | mask
+ for j := int64(0); j < n; j++ {
+ val = int64(p.start) + j
+ x[k+i] = uint32(val) | mask
k++
}
}
- return k
}
-func (rc *runContainer16) getShortIterator() shortPeekable {
+func (rc *runContainer16) getShortIterator() shortIterable {
return rc.newRunIterator16()
}
@@ -1950,11 +2009,8 @@ func (rc *runContainer16) getManyIterator() manyIterable {
// is still abe to express 2^16 because it is an int not an uint16.
func (rc *runContainer16) iaddRange(firstOfRange, endx int) container {
- if firstOfRange > endx {
- panic(fmt.Sprintf("invalid %v = endx > firstOfRange", endx))
- }
- if firstOfRange == endx {
- return rc
+ if firstOfRange >= endx {
+ panic(fmt.Sprintf("invalid %v = endx >= firstOfRange", endx))
}
addme := newRunContainer16TakeOwnership([]interval16{
{
@@ -1968,13 +2024,10 @@ func (rc *runContainer16) iaddRange(firstOfRange, endx int) container {
// remove the values in the range [firstOfRange,endx)
func (rc *runContainer16) iremoveRange(firstOfRange, endx int) container {
- if firstOfRange > endx {
+ if firstOfRange >= endx {
panic(fmt.Sprintf("request to iremove empty set [%v, %v),"+
" nothing to do.", firstOfRange, endx))
- }
- // empty removal
- if firstOfRange == endx {
- return rc
+ //return rc
}
x := newInterval16Range(uint16(firstOfRange), uint16(endx-1))
rc.isubtract(x)
@@ -1983,8 +2036,8 @@ func (rc *runContainer16) iremoveRange(firstOfRange, endx int) container {
// not flip the values in the range [firstOfRange,endx)
func (rc *runContainer16) not(firstOfRange, endx int) container {
- if firstOfRange > endx {
- panic(fmt.Sprintf("invalid %v = endx > firstOfRange = %v", endx, firstOfRange))
+ if firstOfRange >= endx {
+ panic(fmt.Sprintf("invalid %v = endx >= firstOfRange = %v", endx, firstOfRange))
}
return rc.Not(firstOfRange, endx)
@@ -2001,10 +2054,11 @@ func (rc *runContainer16) not(firstOfRange, endx int) container {
// Current routine is correct but
// makes 2 more passes through the arrays than should be
// strictly necessary. Measure both ways though--this may not matter.
+//
func (rc *runContainer16) Not(firstOfRange, endx int) *runContainer16 {
- if firstOfRange > endx {
- panic(fmt.Sprintf("invalid %v = endx > firstOfRange == %v", endx, firstOfRange))
+ if firstOfRange >= endx {
+ panic(fmt.Sprintf("invalid %v = endx >= firstOfRange == %v", endx, firstOfRange))
}
if firstOfRange >= endx {
@@ -2142,21 +2196,9 @@ func (rc *runContainer16) orBitmapContainerCardinality(bc *bitmapContainer) int
// orArray finds the union of rc and ac.
func (rc *runContainer16) orArray(ac *arrayContainer) container {
- if ac.isEmpty() {
- return rc.clone()
- }
- if rc.isEmpty() {
- return ac.clone()
- }
- intervals, cardMinusOne := runArrayUnionToRuns(rc, ac)
- result := newRunContainer16TakeOwnership(intervals)
- if len(intervals) >= 2048 && cardMinusOne >= arrayDefaultMaxSize {
- return newBitmapContainerFromRun(result)
- }
- if len(intervals)*2 > 1+int(cardMinusOne) {
- return result.toArrayContainer()
- }
- return result
+ bc1 := newBitmapContainerFromRun(rc)
+ bc2 := ac.toBitmapContainer()
+ return bc1.orBitmap(bc2)
}
// orArray finds the union of rc and ac.
@@ -2181,8 +2223,8 @@ func (rc *runContainer16) ior(a container) container {
func (rc *runContainer16) inplaceUnion(rc2 *runContainer16) container {
for _, p := range rc2.iv {
- last := int(p.last())
- for i := int(p.start); i <= last; i++ {
+ last := int64(p.last())
+ for i := int64(p.start); i <= last; i++ {
rc.Add(uint16(i))
}
}
@@ -2199,88 +2241,13 @@ func (rc *runContainer16) iorBitmapContainer(bc *bitmapContainer) container {
}
func (rc *runContainer16) iorArray(ac *arrayContainer) container {
- if rc.isEmpty() {
- return ac.clone()
- }
- if ac.isEmpty() {
- return rc
- }
- var cardMinusOne uint16
- //TODO: perform the union algorithm in-place using rc.iv
- // this can be done with methods like the in-place array container union
- // but maybe lazily moving the remaining elements back.
- rc.iv, cardMinusOne = runArrayUnionToRuns(rc, ac)
- if len(rc.iv) >= 2048 && cardMinusOne >= arrayDefaultMaxSize {
- return newBitmapContainerFromRun(rc)
- }
- if len(rc.iv)*2 > 1+int(cardMinusOne) {
- return rc.toArrayContainer()
+ it := ac.getShortIterator()
+ for it.hasNext() {
+ rc.Add(it.next())
}
return rc
}
-func runArrayUnionToRuns(rc *runContainer16, ac *arrayContainer) ([]interval16, uint16) {
- pos1 := 0
- pos2 := 0
- length1 := len(ac.content)
- length2 := len(rc.iv)
- target := make([]interval16, 0, len(rc.iv))
- // have to find the first range
- // options are
- // 1. from array container
- // 2. from run container
- var previousInterval interval16
- var cardMinusOne uint16
- if ac.content[0] < rc.iv[0].start {
- previousInterval.start = ac.content[0]
- previousInterval.length = 0
- pos1++
- } else {
- previousInterval.start = rc.iv[0].start
- previousInterval.length = rc.iv[0].length
- pos2++
- }
-
- for pos1 < length1 || pos2 < length2 {
- if pos1 < length1 {
- s1 := ac.content[pos1]
- if s1 <= previousInterval.start+previousInterval.length {
- pos1++
- continue
- }
- if previousInterval.last() < MaxUint16 && previousInterval.last()+1 == s1 {
- previousInterval.length++
- pos1++
- continue
- }
- }
- if pos2 < length2 {
- range2 := rc.iv[pos2]
- if range2.start <= previousInterval.last() || range2.start > 0 && range2.start-1 == previousInterval.last() {
- pos2++
- if previousInterval.last() < range2.last() {
- previousInterval.length = range2.last() - previousInterval.start
- }
- continue
- }
- }
- cardMinusOne += previousInterval.length + 1
- target = append(target, previousInterval)
- if pos2 == length2 || pos1 < length1 && ac.content[pos1] < rc.iv[pos2].start {
- previousInterval.start = ac.content[pos1]
- previousInterval.length = 0
- pos1++
- } else {
- previousInterval = rc.iv[pos2]
- pos2++
- }
- }
- cardMinusOne += previousInterval.length
- target = append(target, previousInterval)
-
- return target, cardMinusOne
-}
-
// lazyIOR is described (not yet implemented) in
// this nice note from @lemire on
// https://github.com/RoaringBitmap/roaring/pull/70#issuecomment-263613737
@@ -2323,6 +2290,7 @@ func runArrayUnionToRuns(rc *runContainer16, ac *arrayContainer) ([]interval16,
// the backing array, and then you write
// the answer at the beginning. What this
// trick does is minimize memory allocations.
+//
func (rc *runContainer16) lazyIOR(a container) container {
// not lazy at the moment
return rc.ior(a)
@@ -2335,9 +2303,9 @@ func (rc *runContainer16) lazyOR(a container) container {
}
func (rc *runContainer16) intersects(a container) bool {
- // TODO: optimize by doing inplace/less allocation
+ // TODO: optimize by doing inplace/less allocation, possibly?
isect := rc.and(a)
- return !isect.isEmpty()
+ return isect.getCardinality() > 0
}
func (rc *runContainer16) xor(a container) container {
@@ -2366,51 +2334,44 @@ func (rc *runContainer16) iandNot(a container) container {
// flip the values in the range [firstOfRange,endx)
func (rc *runContainer16) inot(firstOfRange, endx int) container {
- if firstOfRange > endx {
- panic(fmt.Sprintf("invalid %v = endx > firstOfRange = %v", endx, firstOfRange))
- }
- if firstOfRange > endx {
- return rc
+ if firstOfRange >= endx {
+ panic(fmt.Sprintf("invalid %v = endx >= firstOfRange = %v", endx, firstOfRange))
}
// TODO: minimize copies, do it all inplace; not() makes a copy.
rc = rc.Not(firstOfRange, endx)
return rc
}
+func (rc *runContainer16) getCardinality() int {
+ return int(rc.cardinality())
+}
+
func (rc *runContainer16) rank(x uint16) int {
- n := int(len(rc.iv))
- xx := int(x)
- w, already, _ := rc.search(xx)
+ n := int64(len(rc.iv))
+ xx := int64(x)
+ w, already, _ := rc.search(xx, nil)
if w < 0 {
return 0
}
if !already && w == n-1 {
return rc.getCardinality()
}
- var rnk int
+ var rnk int64
if !already {
- for i := int(0); i <= w; i++ {
+ for i := int64(0); i <= w; i++ {
rnk += rc.iv[i].runlen()
}
return int(rnk)
}
- for i := int(0); i < w; i++ {
+ for i := int64(0); i < w; i++ {
rnk += rc.iv[i].runlen()
}
- rnk += int(x-rc.iv[w].start) + 1
+ rnk += int64(x-rc.iv[w].start) + 1
return int(rnk)
}
func (rc *runContainer16) selectInt(x uint16) int {
- var offset int
- for k := range rc.iv {
- nextOffset := offset + rc.iv[k].runlen()
- if nextOffset > int(x) {
- return int(int(rc.iv[k].start) + (int(x) - offset))
- }
- offset = nextOffset
- }
- panic("cannot select x")
+ return rc.selectInt16(x)
}
func (rc *runContainer16) andNotRunContainer16(b *runContainer16) container {
@@ -2488,9 +2449,11 @@ func (rc *runContainer16) xorBitmap(bc *bitmapContainer) container {
// convert to bitmap or array *if needed*
func (rc *runContainer16) toEfficientContainer() container {
+
+ // runContainer16SerializedSizeInBytes(numRuns)
sizeAsRunContainer := rc.getSizeInBytes()
sizeAsBitmapContainer := bitmapContainerSizeInBytes()
- card := rc.getCardinality()
+ card := int(rc.cardinality())
sizeAsArrayContainer := arrayContainerSizeInBytes(card)
if sizeAsRunContainer <= minOfInt(sizeAsBitmapContainer, sizeAsArrayContainer) {
return rc
@@ -2575,27 +2538,9 @@ func (rc *runContainer16) serializedSizeInBytes() int {
return 2 + len(rc.iv)*4
}
-func (rc *runContainer16) addOffset(x uint16) (container, container) {
- var low, high *runContainer16
-
- if len(rc.iv) == 0 {
- return nil, nil
- }
-
- first := uint32(rc.iv[0].start) + uint32(x)
- if highbits(first) == 0 {
- // Some elements will fall into low part, allocate a container.
- // Checking the first one is enough because they are ordered.
- low = newRunContainer16()
- }
- last := uint32(rc.iv[len(rc.iv)-1].start)
- last += uint32(rc.iv[len(rc.iv)-1].length)
- last += uint32(x)
- if highbits(last) > 0 {
- // Some elements will fall into high part, allocate a container.
- // Checking the last one is enough because they are ordered.
- high = newRunContainer16()
- }
+func (rc *runContainer16) addOffset(x uint16) []container {
+ low := newRunContainer16()
+ high := newRunContainer16()
for _, iv := range rc.iv {
val := int(iv.start) + int(x)
@@ -2611,14 +2556,5 @@ func (rc *runContainer16) addOffset(x uint16) (container, container) {
high.iv = append(high.iv, interval16{uint16(val & 0xffff), iv.length})
}
}
-
- // Ensure proper nil interface.
- if low == nil {
- return nil, high
- }
- if high == nil {
- return low, nil
- }
-
- return low, high
+ return []container{low, high}
}
diff --git a/vendor/github.com/RoaringBitmap/roaring/runcontainer_gen.go b/vendor/github.com/RoaringBitmap/roaring/runcontainer_gen.go
new file mode 100644
index 0000000..cb91052
--- /dev/null
+++ b/vendor/github.com/RoaringBitmap/roaring/runcontainer_gen.go
@@ -0,0 +1,1126 @@
+package roaring
+
+// NOTE: THIS FILE WAS PRODUCED BY THE
+// MSGP CODE GENERATION TOOL (github.com/tinylib/msgp)
+// DO NOT EDIT
+
+import "github.com/tinylib/msgp/msgp"
+
+// Deprecated: DecodeMsg implements msgp.Decodable
+func (z *addHelper16) DecodeMsg(dc *msgp.Reader) (err error) {
+ var field []byte
+ _ = field
+ var zbai uint32
+ zbai, err = dc.ReadMapHeader()
+ if err != nil {
+ return
+ }
+ for zbai > 0 {
+ zbai--
+ field, err = dc.ReadMapKeyPtr()
+ if err != nil {
+ return
+ }
+ switch msgp.UnsafeString(field) {
+ case "runstart":
+ z.runstart, err = dc.ReadUint16()
+ if err != nil {
+ return
+ }
+ case "runlen":
+ z.runlen, err = dc.ReadUint16()
+ if err != nil {
+ return
+ }
+ case "actuallyAdded":
+ z.actuallyAdded, err = dc.ReadUint16()
+ if err != nil {
+ return
+ }
+ case "m":
+ var zcmr uint32
+ zcmr, err = dc.ReadArrayHeader()
+ if err != nil {
+ return
+ }
+ if cap(z.m) >= int(zcmr) {
+ z.m = (z.m)[:zcmr]
+ } else {
+ z.m = make([]interval16, zcmr)
+ }
+ for zxvk := range z.m {
+ var zajw uint32
+ zajw, err = dc.ReadMapHeader()
+ if err != nil {
+ return
+ }
+ for zajw > 0 {
+ zajw--
+ field, err = dc.ReadMapKeyPtr()
+ if err != nil {
+ return
+ }
+ switch msgp.UnsafeString(field) {
+ case "start":
+ z.m[zxvk].start, err = dc.ReadUint16()
+ if err != nil {
+ return
+ }
+ case "last":
+ z.m[zxvk].length, err = dc.ReadUint16()
+ z.m[zxvk].length -= z.m[zxvk].start
+ if err != nil {
+ return
+ }
+ default:
+ err = dc.Skip()
+ if err != nil {
+ return
+ }
+ }
+ }
+ }
+ case "rc":
+ if dc.IsNil() {
+ err = dc.ReadNil()
+ if err != nil {
+ return
+ }
+ z.rc = nil
+ } else {
+ if z.rc == nil {
+ z.rc = new(runContainer16)
+ }
+ var zwht uint32
+ zwht, err = dc.ReadMapHeader()
+ if err != nil {
+ return
+ }
+ for zwht > 0 {
+ zwht--
+ field, err = dc.ReadMapKeyPtr()
+ if err != nil {
+ return
+ }
+ switch msgp.UnsafeString(field) {
+ case "iv":
+ var zhct uint32
+ zhct, err = dc.ReadArrayHeader()
+ if err != nil {
+ return
+ }
+ if cap(z.rc.iv) >= int(zhct) {
+ z.rc.iv = (z.rc.iv)[:zhct]
+ } else {
+ z.rc.iv = make([]interval16, zhct)
+ }
+ for zbzg := range z.rc.iv {
+ var zcua uint32
+ zcua, err = dc.ReadMapHeader()
+ if err != nil {
+ return
+ }
+ for zcua > 0 {
+ zcua--
+ field, err = dc.ReadMapKeyPtr()
+ if err != nil {
+ return
+ }
+ switch msgp.UnsafeString(field) {
+ case "start":
+ z.rc.iv[zbzg].start, err = dc.ReadUint16()
+ if err != nil {
+ return
+ }
+ case "last":
+ z.rc.iv[zbzg].length, err = dc.ReadUint16()
+ z.rc.iv[zbzg].length -= z.rc.iv[zbzg].start
+ if err != nil {
+ return
+ }
+ default:
+ err = dc.Skip()
+ if err != nil {
+ return
+ }
+ }
+ }
+ }
+ case "card":
+ z.rc.card, err = dc.ReadInt64()
+ if err != nil {
+ return
+ }
+ default:
+ err = dc.Skip()
+ if err != nil {
+ return
+ }
+ }
+ }
+ }
+ default:
+ err = dc.Skip()
+ if err != nil {
+ return
+ }
+ }
+ }
+ return
+}
+
+// Deprecated: EncodeMsg implements msgp.Encodable
+func (z *addHelper16) EncodeMsg(en *msgp.Writer) (err error) {
+ // map header, size 5
+ // write "runstart"
+ err = en.Append(0x85, 0xa8, 0x72, 0x75, 0x6e, 0x73, 0x74, 0x61, 0x72, 0x74)
+ if err != nil {
+ return err
+ }
+ err = en.WriteUint16(z.runstart)
+ if err != nil {
+ return
+ }
+ // write "runlen"
+ err = en.Append(0xa6, 0x72, 0x75, 0x6e, 0x6c, 0x65, 0x6e)
+ if err != nil {
+ return err
+ }
+ err = en.WriteUint16(z.runlen)
+ if err != nil {
+ return
+ }
+ // write "actuallyAdded"
+ err = en.Append(0xad, 0x61, 0x63, 0x74, 0x75, 0x61, 0x6c, 0x6c, 0x79, 0x41, 0x64, 0x64, 0x65, 0x64)
+ if err != nil {
+ return err
+ }
+ err = en.WriteUint16(z.actuallyAdded)
+ if err != nil {
+ return
+ }
+ // write "m"
+ err = en.Append(0xa1, 0x6d)
+ if err != nil {
+ return err
+ }
+ err = en.WriteArrayHeader(uint32(len(z.m)))
+ if err != nil {
+ return
+ }
+ for zxvk := range z.m {
+ // map header, size 2
+ // write "start"
+ err = en.Append(0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74)
+ if err != nil {
+ return err
+ }
+ err = en.WriteUint16(z.m[zxvk].start)
+ if err != nil {
+ return
+ }
+ // write "last"
+ err = en.Append(0xa4, 0x6c, 0x61, 0x73, 0x74)
+ if err != nil {
+ return err
+ }
+ err = en.WriteUint16(z.m[zxvk].last())
+ if err != nil {
+ return
+ }
+ }
+ // write "rc"
+ err = en.Append(0xa2, 0x72, 0x63)
+ if err != nil {
+ return err
+ }
+ if z.rc == nil {
+ err = en.WriteNil()
+ if err != nil {
+ return
+ }
+ } else {
+ // map header, size 2
+ // write "iv"
+ err = en.Append(0x82, 0xa2, 0x69, 0x76)
+ if err != nil {
+ return err
+ }
+ err = en.WriteArrayHeader(uint32(len(z.rc.iv)))
+ if err != nil {
+ return
+ }
+ for zbzg := range z.rc.iv {
+ // map header, size 2
+ // write "start"
+ err = en.Append(0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74)
+ if err != nil {
+ return err
+ }
+ err = en.WriteUint16(z.rc.iv[zbzg].start)
+ if err != nil {
+ return
+ }
+ // write "last"
+ err = en.Append(0xa4, 0x6c, 0x61, 0x73, 0x74)
+ if err != nil {
+ return err
+ }
+ err = en.WriteUint16(z.rc.iv[zbzg].last())
+ if err != nil {
+ return
+ }
+ }
+ // write "card"
+ err = en.Append(0xa4, 0x63, 0x61, 0x72, 0x64)
+ if err != nil {
+ return err
+ }
+ err = en.WriteInt64(z.rc.card)
+ if err != nil {
+ return
+ }
+ }
+ return
+}
+
+// Deprecated: MarshalMsg implements msgp.Marshaler
+func (z *addHelper16) MarshalMsg(b []byte) (o []byte, err error) {
+ o = msgp.Require(b, z.Msgsize())
+ // map header, size 5
+ // string "runstart"
+ o = append(o, 0x85, 0xa8, 0x72, 0x75, 0x6e, 0x73, 0x74, 0x61, 0x72, 0x74)
+ o = msgp.AppendUint16(o, z.runstart)
+ // string "runlen"
+ o = append(o, 0xa6, 0x72, 0x75, 0x6e, 0x6c, 0x65, 0x6e)
+ o = msgp.AppendUint16(o, z.runlen)
+ // string "actuallyAdded"
+ o = append(o, 0xad, 0x61, 0x63, 0x74, 0x75, 0x61, 0x6c, 0x6c, 0x79, 0x41, 0x64, 0x64, 0x65, 0x64)
+ o = msgp.AppendUint16(o, z.actuallyAdded)
+ // string "m"
+ o = append(o, 0xa1, 0x6d)
+ o = msgp.AppendArrayHeader(o, uint32(len(z.m)))
+ for zxvk := range z.m {
+ // map header, size 2
+ // string "start"
+ o = append(o, 0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74)
+ o = msgp.AppendUint16(o, z.m[zxvk].start)
+ // string "last"
+ o = append(o, 0xa4, 0x6c, 0x61, 0x73, 0x74)
+ o = msgp.AppendUint16(o, z.m[zxvk].last())
+ }
+ // string "rc"
+ o = append(o, 0xa2, 0x72, 0x63)
+ if z.rc == nil {
+ o = msgp.AppendNil(o)
+ } else {
+ // map header, size 2
+ // string "iv"
+ o = append(o, 0x82, 0xa2, 0x69, 0x76)
+ o = msgp.AppendArrayHeader(o, uint32(len(z.rc.iv)))
+ for zbzg := range z.rc.iv {
+ // map header, size 2
+ // string "start"
+ o = append(o, 0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74)
+ o = msgp.AppendUint16(o, z.rc.iv[zbzg].start)
+ // string "last"
+ o = append(o, 0xa4, 0x6c, 0x61, 0x73, 0x74)
+ o = msgp.AppendUint16(o, z.rc.iv[zbzg].last())
+ }
+ // string "card"
+ o = append(o, 0xa4, 0x63, 0x61, 0x72, 0x64)
+ o = msgp.AppendInt64(o, z.rc.card)
+ }
+ return
+}
+
+// Deprecated: UnmarshalMsg implements msgp.Unmarshaler
+func (z *addHelper16) UnmarshalMsg(bts []byte) (o []byte, err error) {
+ var field []byte
+ _ = field
+ var zxhx uint32
+ zxhx, bts, err = msgp.ReadMapHeaderBytes(bts)
+ if err != nil {
+ return
+ }
+ for zxhx > 0 {
+ zxhx--
+ field, bts, err = msgp.ReadMapKeyZC(bts)
+ if err != nil {
+ return
+ }
+ switch msgp.UnsafeString(field) {
+ case "runstart":
+ z.runstart, bts, err = msgp.ReadUint16Bytes(bts)
+ if err != nil {
+ return
+ }
+ case "runlen":
+ z.runlen, bts, err = msgp.ReadUint16Bytes(bts)
+ if err != nil {
+ return
+ }
+ case "actuallyAdded":
+ z.actuallyAdded, bts, err = msgp.ReadUint16Bytes(bts)
+ if err != nil {
+ return
+ }
+ case "m":
+ var zlqf uint32
+ zlqf, bts, err = msgp.ReadArrayHeaderBytes(bts)
+ if err != nil {
+ return
+ }
+ if cap(z.m) >= int(zlqf) {
+ z.m = (z.m)[:zlqf]
+ } else {
+ z.m = make([]interval16, zlqf)
+ }
+ for zxvk := range z.m {
+ var zdaf uint32
+ zdaf, bts, err = msgp.ReadMapHeaderBytes(bts)
+ if err != nil {
+ return
+ }
+ for zdaf > 0 {
+ zdaf--
+ field, bts, err = msgp.ReadMapKeyZC(bts)
+ if err != nil {
+ return
+ }
+ switch msgp.UnsafeString(field) {
+ case "start":
+ z.m[zxvk].start, bts, err = msgp.ReadUint16Bytes(bts)
+ if err != nil {
+ return
+ }
+ case "last":
+ z.m[zxvk].length, bts, err = msgp.ReadUint16Bytes(bts)
+ z.m[zxvk].length -= z.m[zxvk].start
+ if err != nil {
+ return
+ }
+ default:
+ bts, err = msgp.Skip(bts)
+ if err != nil {
+ return
+ }
+ }
+ }
+ }
+ case "rc":
+ if msgp.IsNil(bts) {
+ bts, err = msgp.ReadNilBytes(bts)
+ if err != nil {
+ return
+ }
+ z.rc = nil
+ } else {
+ if z.rc == nil {
+ z.rc = new(runContainer16)
+ }
+ var zpks uint32
+ zpks, bts, err = msgp.ReadMapHeaderBytes(bts)
+ if err != nil {
+ return
+ }
+ for zpks > 0 {
+ zpks--
+ field, bts, err = msgp.ReadMapKeyZC(bts)
+ if err != nil {
+ return
+ }
+ switch msgp.UnsafeString(field) {
+ case "iv":
+ var zjfb uint32
+ zjfb, bts, err = msgp.ReadArrayHeaderBytes(bts)
+ if err != nil {
+ return
+ }
+ if cap(z.rc.iv) >= int(zjfb) {
+ z.rc.iv = (z.rc.iv)[:zjfb]
+ } else {
+ z.rc.iv = make([]interval16, zjfb)
+ }
+ for zbzg := range z.rc.iv {
+ var zcxo uint32
+ zcxo, bts, err = msgp.ReadMapHeaderBytes(bts)
+ if err != nil {
+ return
+ }
+ for zcxo > 0 {
+ zcxo--
+ field, bts, err = msgp.ReadMapKeyZC(bts)
+ if err != nil {
+ return
+ }
+ switch msgp.UnsafeString(field) {
+ case "start":
+ z.rc.iv[zbzg].start, bts, err = msgp.ReadUint16Bytes(bts)
+ if err != nil {
+ return
+ }
+ case "last":
+ z.rc.iv[zbzg].length, bts, err = msgp.ReadUint16Bytes(bts)
+ z.rc.iv[zbzg].length -= z.rc.iv[zbzg].start
+ if err != nil {
+ return
+ }
+ default:
+ bts, err = msgp.Skip(bts)
+ if err != nil {
+ return
+ }
+ }
+ }
+ }
+ case "card":
+ z.rc.card, bts, err = msgp.ReadInt64Bytes(bts)
+ if err != nil {
+ return
+ }
+ default:
+ bts, err = msgp.Skip(bts)
+ if err != nil {
+ return
+ }
+ }
+ }
+ }
+ default:
+ bts, err = msgp.Skip(bts)
+ if err != nil {
+ return
+ }
+ }
+ }
+ o = bts
+ return
+}
+
+// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *addHelper16) Msgsize() (s int) {
+ s = 1 + 9 + msgp.Uint16Size + 7 + msgp.Uint16Size + 14 + msgp.Uint16Size + 2 + msgp.ArrayHeaderSize + (len(z.m) * (12 + msgp.Uint16Size + msgp.Uint16Size)) + 3
+ if z.rc == nil {
+ s += msgp.NilSize
+ } else {
+ s += 1 + 3 + msgp.ArrayHeaderSize + (len(z.rc.iv) * (12 + msgp.Uint16Size + msgp.Uint16Size)) + 5 + msgp.Int64Size
+ }
+ return
+}
+
+// Deprecated: DecodeMsg implements msgp.Decodable
+func (z *interval16) DecodeMsg(dc *msgp.Reader) (err error) {
+ var field []byte
+ _ = field
+ var zeff uint32
+ zeff, err = dc.ReadMapHeader()
+ if err != nil {
+ return
+ }
+ for zeff > 0 {
+ zeff--
+ field, err = dc.ReadMapKeyPtr()
+ if err != nil {
+ return
+ }
+ switch msgp.UnsafeString(field) {
+ case "start":
+ z.start, err = dc.ReadUint16()
+ if err != nil {
+ return
+ }
+ case "last":
+ z.length, err = dc.ReadUint16()
+ z.length = -z.start
+ if err != nil {
+ return
+ }
+ default:
+ err = dc.Skip()
+ if err != nil {
+ return
+ }
+ }
+ }
+ return
+}
+
+// Deprecated: EncodeMsg implements msgp.Encodable
+func (z interval16) EncodeMsg(en *msgp.Writer) (err error) {
+ // map header, size 2
+ // write "start"
+ err = en.Append(0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74)
+ if err != nil {
+ return err
+ }
+ err = en.WriteUint16(z.start)
+ if err != nil {
+ return
+ }
+ // write "last"
+ err = en.Append(0xa4, 0x6c, 0x61, 0x73, 0x74)
+ if err != nil {
+ return err
+ }
+ err = en.WriteUint16(z.last())
+ if err != nil {
+ return
+ }
+ return
+}
+
+// Deprecated: MarshalMsg implements msgp.Marshaler
+func (z interval16) MarshalMsg(b []byte) (o []byte, err error) {
+ o = msgp.Require(b, z.Msgsize())
+ // map header, size 2
+ // string "start"
+ o = append(o, 0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74)
+ o = msgp.AppendUint16(o, z.start)
+ // string "last"
+ o = append(o, 0xa4, 0x6c, 0x61, 0x73, 0x74)
+ o = msgp.AppendUint16(o, z.last())
+ return
+}
+
+// Deprecated: UnmarshalMsg implements msgp.Unmarshaler
+func (z *interval16) UnmarshalMsg(bts []byte) (o []byte, err error) {
+ var field []byte
+ _ = field
+ var zrsw uint32
+ zrsw, bts, err = msgp.ReadMapHeaderBytes(bts)
+ if err != nil {
+ return
+ }
+ for zrsw > 0 {
+ zrsw--
+ field, bts, err = msgp.ReadMapKeyZC(bts)
+ if err != nil {
+ return
+ }
+ switch msgp.UnsafeString(field) {
+ case "start":
+ z.start, bts, err = msgp.ReadUint16Bytes(bts)
+ if err != nil {
+ return
+ }
+ case "last":
+ z.length, bts, err = msgp.ReadUint16Bytes(bts)
+ z.length -= z.start
+ if err != nil {
+ return
+ }
+ default:
+ bts, err = msgp.Skip(bts)
+ if err != nil {
+ return
+ }
+ }
+ }
+ o = bts
+ return
+}
+
+// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z interval16) Msgsize() (s int) {
+ s = 1 + 6 + msgp.Uint16Size + 5 + msgp.Uint16Size
+ return
+}
+
+// Deprecated: DecodeMsg implements msgp.Decodable
+func (z *runContainer16) DecodeMsg(dc *msgp.Reader) (err error) {
+ var field []byte
+ _ = field
+ var zdnj uint32
+ zdnj, err = dc.ReadMapHeader()
+ if err != nil {
+ return
+ }
+ for zdnj > 0 {
+ zdnj--
+ field, err = dc.ReadMapKeyPtr()
+ if err != nil {
+ return
+ }
+ switch msgp.UnsafeString(field) {
+ case "iv":
+ var zobc uint32
+ zobc, err = dc.ReadArrayHeader()
+ if err != nil {
+ return
+ }
+ if cap(z.iv) >= int(zobc) {
+ z.iv = (z.iv)[:zobc]
+ } else {
+ z.iv = make([]interval16, zobc)
+ }
+ for zxpk := range z.iv {
+ var zsnv uint32
+ zsnv, err = dc.ReadMapHeader()
+ if err != nil {
+ return
+ }
+ for zsnv > 0 {
+ zsnv--
+ field, err = dc.ReadMapKeyPtr()
+ if err != nil {
+ return
+ }
+ switch msgp.UnsafeString(field) {
+ case "start":
+ z.iv[zxpk].start, err = dc.ReadUint16()
+ if err != nil {
+ return
+ }
+ case "last":
+ z.iv[zxpk].length, err = dc.ReadUint16()
+ z.iv[zxpk].length -= z.iv[zxpk].start
+ if err != nil {
+ return
+ }
+ default:
+ err = dc.Skip()
+ if err != nil {
+ return
+ }
+ }
+ }
+ }
+ case "card":
+ z.card, err = dc.ReadInt64()
+ if err != nil {
+ return
+ }
+ default:
+ err = dc.Skip()
+ if err != nil {
+ return
+ }
+ }
+ }
+ return
+}
+
+// Deprecated: EncodeMsg implements msgp.Encodable
+func (z *runContainer16) EncodeMsg(en *msgp.Writer) (err error) {
+ // map header, size 2
+ // write "iv"
+ err = en.Append(0x82, 0xa2, 0x69, 0x76)
+ if err != nil {
+ return err
+ }
+ err = en.WriteArrayHeader(uint32(len(z.iv)))
+ if err != nil {
+ return
+ }
+ for zxpk := range z.iv {
+ // map header, size 2
+ // write "start"
+ err = en.Append(0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74)
+ if err != nil {
+ return err
+ }
+ err = en.WriteUint16(z.iv[zxpk].start)
+ if err != nil {
+ return
+ }
+ // write "last"
+ err = en.Append(0xa4, 0x6c, 0x61, 0x73, 0x74)
+ if err != nil {
+ return err
+ }
+ err = en.WriteUint16(z.iv[zxpk].last())
+ if err != nil {
+ return
+ }
+ }
+ // write "card"
+ err = en.Append(0xa4, 0x63, 0x61, 0x72, 0x64)
+ if err != nil {
+ return err
+ }
+ err = en.WriteInt64(z.card)
+ if err != nil {
+ return
+ }
+ return
+}
+
+// Deprecated: MarshalMsg implements msgp.Marshaler
+func (z *runContainer16) MarshalMsg(b []byte) (o []byte, err error) {
+ o = msgp.Require(b, z.Msgsize())
+ // map header, size 2
+ // string "iv"
+ o = append(o, 0x82, 0xa2, 0x69, 0x76)
+ o = msgp.AppendArrayHeader(o, uint32(len(z.iv)))
+ for zxpk := range z.iv {
+ // map header, size 2
+ // string "start"
+ o = append(o, 0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74)
+ o = msgp.AppendUint16(o, z.iv[zxpk].start)
+ // string "last"
+ o = append(o, 0xa4, 0x6c, 0x61, 0x73, 0x74)
+ o = msgp.AppendUint16(o, z.iv[zxpk].last())
+ }
+ // string "card"
+ o = append(o, 0xa4, 0x63, 0x61, 0x72, 0x64)
+ o = msgp.AppendInt64(o, z.card)
+ return
+}
+
+// Deprecated: UnmarshalMsg implements msgp.Unmarshaler
+func (z *runContainer16) UnmarshalMsg(bts []byte) (o []byte, err error) {
+ var field []byte
+ _ = field
+ var zkgt uint32
+ zkgt, bts, err = msgp.ReadMapHeaderBytes(bts)
+ if err != nil {
+ return
+ }
+ for zkgt > 0 {
+ zkgt--
+ field, bts, err = msgp.ReadMapKeyZC(bts)
+ if err != nil {
+ return
+ }
+ switch msgp.UnsafeString(field) {
+ case "iv":
+ var zema uint32
+ zema, bts, err = msgp.ReadArrayHeaderBytes(bts)
+ if err != nil {
+ return
+ }
+ if cap(z.iv) >= int(zema) {
+ z.iv = (z.iv)[:zema]
+ } else {
+ z.iv = make([]interval16, zema)
+ }
+ for zxpk := range z.iv {
+ var zpez uint32
+ zpez, bts, err = msgp.ReadMapHeaderBytes(bts)
+ if err != nil {
+ return
+ }
+ for zpez > 0 {
+ zpez--
+ field, bts, err = msgp.ReadMapKeyZC(bts)
+ if err != nil {
+ return
+ }
+ switch msgp.UnsafeString(field) {
+ case "start":
+ z.iv[zxpk].start, bts, err = msgp.ReadUint16Bytes(bts)
+ if err != nil {
+ return
+ }
+ case "last":
+ z.iv[zxpk].length, bts, err = msgp.ReadUint16Bytes(bts)
+ z.iv[zxpk].length -= z.iv[zxpk].start
+ if err != nil {
+ return
+ }
+ default:
+ bts, err = msgp.Skip(bts)
+ if err != nil {
+ return
+ }
+ }
+ }
+ }
+ case "card":
+ z.card, bts, err = msgp.ReadInt64Bytes(bts)
+ if err != nil {
+ return
+ }
+ default:
+ bts, err = msgp.Skip(bts)
+ if err != nil {
+ return
+ }
+ }
+ }
+ o = bts
+ return
+}
+
+// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *runContainer16) Msgsize() (s int) {
+ s = 1 + 3 + msgp.ArrayHeaderSize + (len(z.iv) * (12 + msgp.Uint16Size + msgp.Uint16Size)) + 5 + msgp.Int64Size
+ return
+}
+
+// Deprecated: DecodeMsg implements msgp.Decodable
+func (z *runIterator16) DecodeMsg(dc *msgp.Reader) (err error) {
+ var field []byte
+ _ = field
+ var zqke uint32
+ zqke, err = dc.ReadMapHeader()
+ if err != nil {
+ return
+ }
+ for zqke > 0 {
+ zqke--
+ field, err = dc.ReadMapKeyPtr()
+ if err != nil {
+ return
+ }
+ switch msgp.UnsafeString(field) {
+ case "rc":
+ if dc.IsNil() {
+ err = dc.ReadNil()
+ if err != nil {
+ return
+ }
+ z.rc = nil
+ } else {
+ if z.rc == nil {
+ z.rc = new(runContainer16)
+ }
+ err = z.rc.DecodeMsg(dc)
+ if err != nil {
+ return
+ }
+ }
+ case "curIndex":
+ z.curIndex, err = dc.ReadInt64()
+ if err != nil {
+ return
+ }
+ case "curPosInIndex":
+ z.curPosInIndex, err = dc.ReadUint16()
+ if err != nil {
+ return
+ }
+ case "curSeq":
+ z.curSeq, err = dc.ReadInt64()
+ if err != nil {
+ return
+ }
+ default:
+ err = dc.Skip()
+ if err != nil {
+ return
+ }
+ }
+ }
+ return
+}
+
+// Deprecated: EncodeMsg implements msgp.Encodable
+func (z *runIterator16) EncodeMsg(en *msgp.Writer) (err error) {
+ // map header, size 4
+ // write "rc"
+ err = en.Append(0x84, 0xa2, 0x72, 0x63)
+ if err != nil {
+ return err
+ }
+ if z.rc == nil {
+ err = en.WriteNil()
+ if err != nil {
+ return
+ }
+ } else {
+ err = z.rc.EncodeMsg(en)
+ if err != nil {
+ return
+ }
+ }
+ // write "curIndex"
+ err = en.Append(0xa8, 0x63, 0x75, 0x72, 0x49, 0x6e, 0x64, 0x65, 0x78)
+ if err != nil {
+ return err
+ }
+ err = en.WriteInt64(z.curIndex)
+ if err != nil {
+ return
+ }
+ // write "curPosInIndex"
+ err = en.Append(0xad, 0x63, 0x75, 0x72, 0x50, 0x6f, 0x73, 0x49, 0x6e, 0x49, 0x6e, 0x64, 0x65, 0x78)
+ if err != nil {
+ return err
+ }
+ err = en.WriteUint16(z.curPosInIndex)
+ if err != nil {
+ return
+ }
+ // write "curSeq"
+ err = en.Append(0xa6, 0x63, 0x75, 0x72, 0x53, 0x65, 0x71)
+ if err != nil {
+ return err
+ }
+ err = en.WriteInt64(z.curSeq)
+ if err != nil {
+ return
+ }
+ return
+}
+
+// Deprecated: MarshalMsg implements msgp.Marshaler
+func (z *runIterator16) MarshalMsg(b []byte) (o []byte, err error) {
+ o = msgp.Require(b, z.Msgsize())
+ // map header, size 4
+ // string "rc"
+ o = append(o, 0x84, 0xa2, 0x72, 0x63)
+ if z.rc == nil {
+ o = msgp.AppendNil(o)
+ } else {
+ o, err = z.rc.MarshalMsg(o)
+ if err != nil {
+ return
+ }
+ }
+ // string "curIndex"
+ o = append(o, 0xa8, 0x63, 0x75, 0x72, 0x49, 0x6e, 0x64, 0x65, 0x78)
+ o = msgp.AppendInt64(o, z.curIndex)
+ // string "curPosInIndex"
+ o = append(o, 0xad, 0x63, 0x75, 0x72, 0x50, 0x6f, 0x73, 0x49, 0x6e, 0x49, 0x6e, 0x64, 0x65, 0x78)
+ o = msgp.AppendUint16(o, z.curPosInIndex)
+ // string "curSeq"
+ o = append(o, 0xa6, 0x63, 0x75, 0x72, 0x53, 0x65, 0x71)
+ o = msgp.AppendInt64(o, z.curSeq)
+ return
+}
+
+// Deprecated: UnmarshalMsg implements msgp.Unmarshaler
+func (z *runIterator16) UnmarshalMsg(bts []byte) (o []byte, err error) {
+ var field []byte
+ _ = field
+ var zqyh uint32
+ zqyh, bts, err = msgp.ReadMapHeaderBytes(bts)
+ if err != nil {
+ return
+ }
+ for zqyh > 0 {
+ zqyh--
+ field, bts, err = msgp.ReadMapKeyZC(bts)
+ if err != nil {
+ return
+ }
+ switch msgp.UnsafeString(field) {
+ case "rc":
+ if msgp.IsNil(bts) {
+ bts, err = msgp.ReadNilBytes(bts)
+ if err != nil {
+ return
+ }
+ z.rc = nil
+ } else {
+ if z.rc == nil {
+ z.rc = new(runContainer16)
+ }
+ bts, err = z.rc.UnmarshalMsg(bts)
+ if err != nil {
+ return
+ }
+ }
+ case "curIndex":
+ z.curIndex, bts, err = msgp.ReadInt64Bytes(bts)
+ if err != nil {
+ return
+ }
+ case "curPosInIndex":
+ z.curPosInIndex, bts, err = msgp.ReadUint16Bytes(bts)
+ if err != nil {
+ return
+ }
+ case "curSeq":
+ z.curSeq, bts, err = msgp.ReadInt64Bytes(bts)
+ if err != nil {
+ return
+ }
+ default:
+ bts, err = msgp.Skip(bts)
+ if err != nil {
+ return
+ }
+ }
+ }
+ o = bts
+ return
+}
+
+// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *runIterator16) Msgsize() (s int) {
+ s = 1 + 3
+ if z.rc == nil {
+ s += msgp.NilSize
+ } else {
+ s += z.rc.Msgsize()
+ }
+ s += 9 + msgp.Int64Size + 14 + msgp.Uint16Size + 7 + msgp.Int64Size
+ return
+}
+
+// Deprecated: DecodeMsg implements msgp.Decodable
+func (z *uint16Slice) DecodeMsg(dc *msgp.Reader) (err error) {
+ var zjpj uint32
+ zjpj, err = dc.ReadArrayHeader()
+ if err != nil {
+ return
+ }
+ if cap((*z)) >= int(zjpj) {
+ (*z) = (*z)[:zjpj]
+ } else {
+ (*z) = make(uint16Slice, zjpj)
+ }
+ for zywj := range *z {
+ (*z)[zywj], err = dc.ReadUint16()
+ if err != nil {
+ return
+ }
+ }
+ return
+}
+
+// Deprecated: EncodeMsg implements msgp.Encodable
+func (z uint16Slice) EncodeMsg(en *msgp.Writer) (err error) {
+ err = en.WriteArrayHeader(uint32(len(z)))
+ if err != nil {
+ return
+ }
+ for zzpf := range z {
+ err = en.WriteUint16(z[zzpf])
+ if err != nil {
+ return
+ }
+ }
+ return
+}
+
+// Deprecated: MarshalMsg implements msgp.Marshaler
+func (z uint16Slice) MarshalMsg(b []byte) (o []byte, err error) {
+ o = msgp.Require(b, z.Msgsize())
+ o = msgp.AppendArrayHeader(o, uint32(len(z)))
+ for zzpf := range z {
+ o = msgp.AppendUint16(o, z[zzpf])
+ }
+ return
+}
+
+// Deprecated: UnmarshalMsg implements msgp.Unmarshaler
+func (z *uint16Slice) UnmarshalMsg(bts []byte) (o []byte, err error) {
+ var zgmo uint32
+ zgmo, bts, err = msgp.ReadArrayHeaderBytes(bts)
+ if err != nil {
+ return
+ }
+ if cap((*z)) >= int(zgmo) {
+ (*z) = (*z)[:zgmo]
+ } else {
+ (*z) = make(uint16Slice, zgmo)
+ }
+ for zrfe := range *z {
+ (*z)[zrfe], bts, err = msgp.ReadUint16Bytes(bts)
+ if err != nil {
+ return
+ }
+ }
+ o = bts
+ return
+}
+
+// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z uint16Slice) Msgsize() (s int) {
+ s = msgp.ArrayHeaderSize + (len(z) * (msgp.Uint16Size))
+ return
+}
diff --git a/vendor/github.com/RoaringBitmap/roaring/serialization.go b/vendor/github.com/RoaringBitmap/roaring/serialization.go
index dbfecc8..98fef3c 100644
--- a/vendor/github.com/RoaringBitmap/roaring/serialization.go
+++ b/vendor/github.com/RoaringBitmap/roaring/serialization.go
@@ -2,11 +2,16 @@ package roaring
import (
"encoding/binary"
+ "errors"
+ "fmt"
"io"
+
+ "github.com/tinylib/msgp/msgp"
)
// writeTo for runContainer16 follows this
// spec: https://github.com/RoaringBitmap/RoaringFormatSpec
+//
func (b *runContainer16) writeTo(stream io.Writer) (int, error) {
buf := make([]byte, 2+4*len(b.iv))
binary.LittleEndian.PutUint16(buf[0:], uint16(len(b.iv)))
@@ -16,3 +21,50 @@ func (b *runContainer16) writeTo(stream io.Writer) (int, error) {
}
return stream.Write(buf)
}
+
+func (b *runContainer16) writeToMsgpack(stream io.Writer) (int, error) {
+ bts, err := b.MarshalMsg(nil)
+ if err != nil {
+ return 0, err
+ }
+ return stream.Write(bts)
+}
+
+func (b *runContainer16) readFromMsgpack(stream io.Reader) (int, error) {
+ err := msgp.Decode(stream, b)
+ return 0, err
+}
+
+var errCorruptedStream = errors.New("insufficient/odd number of stored bytes, corrupted stream detected")
+
+func (b *runContainer16) readFrom(stream io.Reader) (int, error) {
+ b.iv = b.iv[:0]
+ b.card = 0
+ var numRuns uint16
+ err := binary.Read(stream, binary.LittleEndian, &numRuns)
+ if err != nil {
+ return 0, err
+ }
+ nr := int(numRuns)
+ encRun := make([]uint16, 2*nr)
+ by := make([]byte, 4*nr)
+ err = binary.Read(stream, binary.LittleEndian, &by)
+ if err != nil {
+ return 0, err
+ }
+ for i := range encRun {
+ if len(by) < 2 {
+ return 0, errCorruptedStream
+ }
+ encRun[i] = binary.LittleEndian.Uint16(by)
+ by = by[2:]
+ }
+ for i := 0; i < nr; i++ {
+ if i > 0 && b.iv[i-1].last() >= encRun[i*2] {
+ return 0, fmt.Errorf("error: stored runContainer had runs that were not in sorted order!! (b.iv[i-1=%v].last = %v >= encRun[i=%v] = %v)", i-1, b.iv[i-1].last(), i, encRun[i*2])
+ }
+ b.iv = append(b.iv, interval16{start: encRun[i*2], length: encRun[i*2+1]})
+ b.card += int64(encRun[i*2+1]) + 1
+ }
+ return 0, err
+}
diff --git a/vendor/github.com/RoaringBitmap/roaring/serialization_generic.go b/vendor/github.com/RoaringBitmap/roaring/serialization_generic.go
index 7e1f180..f4805ca 100644
--- a/vendor/github.com/RoaringBitmap/roaring/serialization_generic.go
+++ b/vendor/github.com/RoaringBitmap/roaring/serialization_generic.go
@@ -1,5 +1,4 @@
-//go:build (!amd64 && !386 && !arm && !arm64 && !ppc64le && !mipsle && !mips64le && !mips64p32le && !wasm) || appengine
-// +build !amd64,!386,!arm,!arm64,!ppc64le,!mipsle,!mips64le,!mips64p32le,!wasm appengine
+// +build !amd64,!386 appengine
package roaring
@@ -75,27 +74,6 @@ func uint64SliceAsByteSlice(slice []uint64) []byte {
return by
}
-func uint16SliceAsByteSlice(slice []uint16) []byte {
- by := make([]byte, len(slice)*2)
-
- for i, v := range slice {
- binary.LittleEndian.PutUint16(by[i*2:], v)
- }
-
- return by
-}
-
-func interval16SliceAsByteSlice(slice []interval16) []byte {
- by := make([]byte, len(slice)*4)
-
- for i, v := range slice {
- binary.LittleEndian.PutUint16(by[i*2:], v.start)
- binary.LittleEndian.PutUint16(by[i*2+2:], v.length)
- }
-
- return by
-}
-
func byteSliceAsUint16Slice(slice []byte) []uint16 {
if len(slice)%2 != 0 {
panic("Slice size should be divisible by 2")
diff --git a/vendor/github.com/RoaringBitmap/roaring/serialization_littleendian.go b/vendor/github.com/RoaringBitmap/roaring/serialization_littleendian.go
index 6e3a5d5..ba5b753 100644
--- a/vendor/github.com/RoaringBitmap/roaring/serialization_littleendian.go
+++ b/vendor/github.com/RoaringBitmap/roaring/serialization_littleendian.go
@@ -1,14 +1,11 @@
-//go:build (386 && !appengine) || (amd64 && !appengine) || (arm && !appengine) || (arm64 && !appengine) || (ppc64le && !appengine) || (mipsle && !appengine) || (mips64le && !appengine) || (mips64p32le && !appengine) || (wasm && !appengine)
-// +build 386,!appengine amd64,!appengine arm,!appengine arm64,!appengine ppc64le,!appengine mipsle,!appengine mips64le,!appengine mips64p32le,!appengine wasm,!appengine
+// +build 386 amd64,!appengine
package roaring
import (
- "encoding/binary"
"errors"
"io"
"reflect"
- "runtime"
"unsafe"
)
@@ -25,6 +22,22 @@ func (bc *bitmapContainer) writeTo(stream io.Writer) (int, error) {
return stream.Write(buf)
}
+// readFrom reads an arrayContainer from stream.
+// PRE-REQUISITE: you must size the arrayContainer correctly (allocate b.content)
+// *before* you call readFrom. We can't guess the size in the stream
+// by this point.
+func (ac *arrayContainer) readFrom(stream io.Reader) (int, error) {
+ buf := uint16SliceAsByteSlice(ac.content)
+ return io.ReadFull(stream, buf)
+}
+
+func (bc *bitmapContainer) readFrom(stream io.Reader) (int, error) {
+ buf := uint64SliceAsByteSlice(bc.bitmap)
+ n, err := io.ReadFull(stream, buf)
+ bc.computeCardinality()
+ return n, err
+}
+
func uint64SliceAsByteSlice(slice []uint64) []byte {
// make a new slice header
header := *(*reflect.SliceHeader)(unsafe.Pointer(&slice))
@@ -33,12 +46,8 @@ func uint64SliceAsByteSlice(slice []uint64) []byte {
header.Len *= 8
header.Cap *= 8
- // instantiate result and use KeepAlive so data isn't unmapped.
- result := *(*[]byte)(unsafe.Pointer(&header))
- runtime.KeepAlive(&slice)
-
// return it
- return result
+ return *(*[]byte)(unsafe.Pointer(&header))
}
func uint16SliceAsByteSlice(slice []uint16) []byte {
@@ -49,28 +58,8 @@ func uint16SliceAsByteSlice(slice []uint16) []byte {
header.Len *= 2
header.Cap *= 2
- // instantiate result and use KeepAlive so data isn't unmapped.
- result := *(*[]byte)(unsafe.Pointer(&header))
- runtime.KeepAlive(&slice)
-
// return it
- return result
-}
-
-func interval16SliceAsByteSlice(slice []interval16) []byte {
- // make a new slice header
- header := *(*reflect.SliceHeader)(unsafe.Pointer(&slice))
-
- // update its capacity and length
- header.Len *= 4
- header.Cap *= 4
-
- // instantiate result and use KeepAlive so data isn't unmapped.
- result := *(*[]byte)(unsafe.Pointer(&header))
- runtime.KeepAlive(&slice)
-
- // return it
- return result
+ return *(*[]byte)(unsafe.Pointer(&header))
}
func (bc *bitmapContainer) asLittleEndianByteSlice() []byte {
@@ -79,584 +68,50 @@ func (bc *bitmapContainer) asLittleEndianByteSlice() []byte {
// Deserialization code follows
-// //
-// These methods (byteSliceAsUint16Slice,...) do not make copies,
-// they are pointer-based (unsafe). The caller is responsible to
-// ensure that the input slice does not get garbage collected, deleted
-// or modified while you hold the returned slince.
-// //
-func byteSliceAsUint16Slice(slice []byte) (result []uint16) { // here we create a new slice holder
+func byteSliceAsUint16Slice(slice []byte) []uint16 {
if len(slice)%2 != 0 {
panic("Slice size should be divisible by 2")
}
- // reference: https://go101.org/article/unsafe.html
// make a new slice header
- bHeader := (*reflect.SliceHeader)(unsafe.Pointer(&slice))
- rHeader := (*reflect.SliceHeader)(unsafe.Pointer(&result))
+ header := *(*reflect.SliceHeader)(unsafe.Pointer(&slice))
- // transfer the data from the given slice to a new variable (our result)
- rHeader.Data = bHeader.Data
- rHeader.Len = bHeader.Len / 2
- rHeader.Cap = bHeader.Cap / 2
+ // update its capacity and length
+ header.Len /= 2
+ header.Cap /= 2
- // instantiate result and use KeepAlive so data isn't unmapped.
- runtime.KeepAlive(&slice) // it is still crucial, GC can free it)
-
- // return result
- return
+ // return it
+ return *(*[]uint16)(unsafe.Pointer(&header))
}
-func byteSliceAsUint64Slice(slice []byte) (result []uint64) {
+func byteSliceAsUint64Slice(slice []byte) []uint64 {
if len(slice)%8 != 0 {
panic("Slice size should be divisible by 8")
}
- // reference: https://go101.org/article/unsafe.html
// make a new slice header
- bHeader := (*reflect.SliceHeader)(unsafe.Pointer(&slice))
- rHeader := (*reflect.SliceHeader)(unsafe.Pointer(&result))
+ header := *(*reflect.SliceHeader)(unsafe.Pointer(&slice))
- // transfer the data from the given slice to a new variable (our result)
- rHeader.Data = bHeader.Data
- rHeader.Len = bHeader.Len / 8
- rHeader.Cap = bHeader.Cap / 8
+ // update its capacity and length
+ header.Len /= 8
+ header.Cap /= 8
- // instantiate result and use KeepAlive so data isn't unmapped.
- runtime.KeepAlive(&slice) // it is still crucial, GC can free it)
-
- // return result
- return
+ // return it
+ return *(*[]uint64)(unsafe.Pointer(&header))
}
-func byteSliceAsInterval16Slice(slice []byte) (result []interval16) {
+func byteSliceAsInterval16Slice(slice []byte) []interval16 {
if len(slice)%4 != 0 {
panic("Slice size should be divisible by 4")
}
- // reference: https://go101.org/article/unsafe.html
// make a new slice header
- bHeader := (*reflect.SliceHeader)(unsafe.Pointer(&slice))
- rHeader := (*reflect.SliceHeader)(unsafe.Pointer(&result))
+ header := *(*reflect.SliceHeader)(unsafe.Pointer(&slice))
- // transfer the data from the given slice to a new variable (our result)
- rHeader.Data = bHeader.Data
- rHeader.Len = bHeader.Len / 4
- rHeader.Cap = bHeader.Cap / 4
+ // update its capacity and length
+ header.Len /= 4
+ header.Cap /= 4
- // instantiate result and use KeepAlive so data isn't unmapped.
- runtime.KeepAlive(&slice) // it is still crucial, GC can free it)
-
- // return result
- return
-}
-
-func byteSliceAsContainerSlice(slice []byte) (result []container) {
- var c container
- containerSize := int(unsafe.Sizeof(c))
-
- if len(slice)%containerSize != 0 {
- panic("Slice size should be divisible by unsafe.Sizeof(container)")
- }
- // reference: https://go101.org/article/unsafe.html
-
- // make a new slice header
- bHeader := (*reflect.SliceHeader)(unsafe.Pointer(&slice))
- rHeader := (*reflect.SliceHeader)(unsafe.Pointer(&result))
-
- // transfer the data from the given slice to a new variable (our result)
- rHeader.Data = bHeader.Data
- rHeader.Len = bHeader.Len / containerSize
- rHeader.Cap = bHeader.Cap / containerSize
-
- // instantiate result and use KeepAlive so data isn't unmapped.
- runtime.KeepAlive(&slice) // it is still crucial, GC can free it)
-
- // return result
- return
-}
-
-func byteSliceAsBitsetSlice(slice []byte) (result []bitmapContainer) {
- bitsetSize := int(unsafe.Sizeof(bitmapContainer{}))
- if len(slice)%bitsetSize != 0 {
- panic("Slice size should be divisible by unsafe.Sizeof(bitmapContainer)")
- }
- // reference: https://go101.org/article/unsafe.html
-
- // make a new slice header
- bHeader := (*reflect.SliceHeader)(unsafe.Pointer(&slice))
- rHeader := (*reflect.SliceHeader)(unsafe.Pointer(&result))
-
- // transfer the data from the given slice to a new variable (our result)
- rHeader.Data = bHeader.Data
- rHeader.Len = bHeader.Len / bitsetSize
- rHeader.Cap = bHeader.Cap / bitsetSize
-
- // instantiate result and use KeepAlive so data isn't unmapped.
- runtime.KeepAlive(&slice) // it is still crucial, GC can free it)
-
- // return result
- return
-}
-
-func byteSliceAsArraySlice(slice []byte) (result []arrayContainer) {
- arraySize := int(unsafe.Sizeof(arrayContainer{}))
- if len(slice)%arraySize != 0 {
- panic("Slice size should be divisible by unsafe.Sizeof(arrayContainer)")
- }
- // reference: https://go101.org/article/unsafe.html
-
- // make a new slice header
- bHeader := (*reflect.SliceHeader)(unsafe.Pointer(&slice))
- rHeader := (*reflect.SliceHeader)(unsafe.Pointer(&result))
-
- // transfer the data from the given slice to a new variable (our result)
- rHeader.Data = bHeader.Data
- rHeader.Len = bHeader.Len / arraySize
- rHeader.Cap = bHeader.Cap / arraySize
-
- // instantiate result and use KeepAlive so data isn't unmapped.
- runtime.KeepAlive(&slice) // it is still crucial, GC can free it)
-
- // return result
- return
-}
-
-func byteSliceAsRun16Slice(slice []byte) (result []runContainer16) {
- run16Size := int(unsafe.Sizeof(runContainer16{}))
- if len(slice)%run16Size != 0 {
- panic("Slice size should be divisible by unsafe.Sizeof(runContainer16)")
- }
- // reference: https://go101.org/article/unsafe.html
-
- // make a new slice header
- bHeader := (*reflect.SliceHeader)(unsafe.Pointer(&slice))
- rHeader := (*reflect.SliceHeader)(unsafe.Pointer(&result))
-
- // transfer the data from the given slice to a new variable (our result)
- rHeader.Data = bHeader.Data
- rHeader.Len = bHeader.Len / run16Size
- rHeader.Cap = bHeader.Cap / run16Size
-
- // instantiate result and use KeepAlive so data isn't unmapped.
- runtime.KeepAlive(&slice) // it is still crucial, GC can free it)
-
- // return result
- return
-}
-
-func byteSliceAsBoolSlice(slice []byte) (result []bool) {
- boolSize := int(unsafe.Sizeof(true))
- if len(slice)%boolSize != 0 {
- panic("Slice size should be divisible by unsafe.Sizeof(bool)")
- }
- // reference: https://go101.org/article/unsafe.html
-
- // make a new slice header
- bHeader := (*reflect.SliceHeader)(unsafe.Pointer(&slice))
- rHeader := (*reflect.SliceHeader)(unsafe.Pointer(&result))
-
- // transfer the data from the given slice to a new variable (our result)
- rHeader.Data = bHeader.Data
- rHeader.Len = bHeader.Len / boolSize
- rHeader.Cap = bHeader.Cap / boolSize
-
- // instantiate result and use KeepAlive so data isn't unmapped.
- runtime.KeepAlive(&slice) // it is still crucial, GC can free it)
-
- // return result
- return
-}
-
-// FrozenView creates a static view of a serialized bitmap stored in buf.
-// It uses CRoaring's frozen bitmap format.
-//
-// The format specification is available here:
-// https://github.com/RoaringBitmap/CRoaring/blob/2c867e9f9c9e2a3a7032791f94c4c7ae3013f6e0/src/roaring.c#L2756-L2783
-//
-// The provided byte array (buf) is expected to be a constant.
-// The function makes the best effort attempt not to copy data.
-// Only little endian is supported. The function will err if it detects a big
-// endian serialized file.
-// You should take care not to modify buff as it will likely result in
-// unexpected program behavior.
-// If said buffer comes from a memory map, it's advisable to give it read
-// only permissions, either at creation or by calling Mprotect from the
-// golang.org/x/sys/unix package.
-//
-// Resulting bitmaps are effectively immutable in the following sense:
-// a copy-on-write marker is used so that when you modify the resulting
-// bitmap, copies of selected data (containers) are made.
-// You should *not* change the copy-on-write status of the resulting
-// bitmaps (SetCopyOnWrite).
-//
-// If buf becomes unavailable, then a bitmap created with
-// FromBuffer would be effectively broken. Furthermore, any
-// bitmap derived from this bitmap (e.g., via Or, And) might
-// also be broken. Thus, before making buf unavailable, you should
-// call CloneCopyOnWriteContainers on all such bitmaps.
-func (rb *Bitmap) FrozenView(buf []byte) error {
- return rb.highlowcontainer.frozenView(buf)
-}
-
-/* Verbatim specification from CRoaring.
- *
- * FROZEN SERIALIZATION FORMAT DESCRIPTION
- *
- * -- (beginning must be aligned by 32 bytes) --
- *
uint64_t[BITSET_CONTAINER_SIZE_IN_WORDS * num_bitset_containers]
- * rle16_t[total number of rle elements in all run containers]
- * uint16_t[total number of array elements in all array containers]
- * uint16_t[num_containers]
- * uint16_t[num_containers]
- * uint8_t[num_containers]
- * uint32_t
- *
- * is a 4-byte value which is a bit union of frozenCookie (15 bits)
- * and the number of containers (17 bits).
- *
- * stores number of elements for every container.
- * Its meaning depends on container type.
- * For array and bitset containers, this value is the container cardinality minus one.
- * For run container, it is the number of rle_t elements (n_runs).
- *
- * ,, are flat arrays of elements of
- * all containers of respective type.
- *
- * <*_data> and are kept close together because they are not accessed
- * during deserilization. This may reduce IO in case of large mmaped bitmaps.
- * All members have their native alignments during deserilization except