diff --git a/templates/userTotp.html b/pkg/server/templates/userTotp.html
similarity index 100%
rename from templates/userTotp.html
rename to pkg/server/templates/userTotp.html
diff --git a/pkg/server/webdav.go b/pkg/server/webdav.go
index 3d8e260..0fdf5b5 100644
--- a/pkg/server/webdav.go
+++ b/pkg/server/webdav.go
@@ -1,14 +1,27 @@
+// Copyright (C) 2022 Marius Schellenberger
+
package server
import (
- "git.giftfish.de/ston1th/docstore/pkg/log"
"net/http"
+
+ "github.com/go-logr/logr"
)
-func webdavLogger(r *http.Request, err error) {
- if err != nil {
- log.Printf("webdav: %s %s %s error: %s\n", r.RemoteAddr, r.Method, r.URL, err)
- return
+func webdavLogger(log logr.Logger) func(r *http.Request, err error) {
+ return func(r *http.Request, err error) {
+ if err != nil {
+ log.Error(err, "access",
+ "client", r.RemoteAddr,
+ "method", r.Method,
+ "uri", r.URL.Path,
+ )
+ return
+ }
+ log.Info("access",
+ "client", r.RemoteAddr,
+ "method", r.Method,
+ "uri", r.URL.Path,
+ )
}
- log.Printf("webdav: %s %s %s\n", r.RemoteAddr, r.Method, r.URL)
}
diff --git a/pkg/server/xsrf.go b/pkg/server/xsrf.go
index 07b7090..feae116 100644
--- a/pkg/server/xsrf.go
+++ b/pkg/server/xsrf.go
@@ -1,28 +1,30 @@
-// Copyright (C) 2019 Marius Schellenberger
+// Copyright (C) 2021 Marius Schellenberger
package server
import (
+ "crypto/hmac"
"crypto/rand"
- "crypto/subtle"
+ "crypto/sha256"
"encoding/base64"
- "git.giftfish.de/ston1th/docstore/pkg/log"
- "git.giftfish.de/ston1th/jwt/v3"
)
-const keySize = jwt.KeySize / 2
+const (
+ keySize = 16
+ fullSize = keySize * 2
+)
func genKey(length int) (bytes []byte) {
bytes = make([]byte, length)
- if _, err := rand.Reader.Read(bytes); err != nil {
- log.Println("genKey:", err)
- }
+ rand.Read(bytes)
return
}
func newXsrf(secret []byte) string {
+ mac := hmac.New(sha256.New, secret)
rnd := genKey(keySize)
- return base64.RawURLEncoding.EncodeToString(append(rnd, xor(rnd, secret)...))
+ mac.Write(rnd)
+ return base64.RawURLEncoding.EncodeToString(append(rnd, mac.Sum(nil)[:keySize]...))
}
func checkXsrf(xsrf string, secret []byte) bool {
@@ -30,20 +32,10 @@ func checkXsrf(xsrf string, secret []byte) bool {
if err != nil {
return false
}
- if len(t) != jwt.KeySize {
+ if len(t) != fullSize {
return false
}
- return subtle.ConstantTimeCompare(secret, xor(t[keySize:], t[:keySize])) == 1
-}
-
-func xor(a, b []byte) (c []byte) {
- n := len(a)
- if len(b) < n {
- return nil
- }
- c = make([]byte, n)
- for i := 0; i < n; i++ {
- c[i] = a[i] ^ b[i]
- }
- return
+ mac := hmac.New(sha256.New, secret)
+ mac.Write(t[:keySize])
+ return hmac.Equal(t[keySize:], mac.Sum(nil)[:keySize])
}
diff --git a/pkg/server/xsrf_test.go b/pkg/server/xsrf_test.go
index fd6944b..d611a85 100644
--- a/pkg/server/xsrf_test.go
+++ b/pkg/server/xsrf_test.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019 Marius Schellenberger
+// Copyright (C) 2021 Marius Schellenberger
package server
diff --git a/pkg/store/boltstore.go b/pkg/store/boltstore.go
index 0c36c51..5251785 100644
--- a/pkg/store/boltstore.go
+++ b/pkg/store/boltstore.go
@@ -1,16 +1,16 @@
-// Copyright (C) 2019 Marius Schellenberger
+// Copyright (C) 2021 Marius Schellenberger
package store
import (
"encoding/json"
- "github.com/boltdb/bolt"
+ bolt "go.etcd.io/bbolt"
"io"
)
const (
defaultBoltBucket = "default"
- fileMode = 0640
+ fileMode = 0o640
)
type BoltStore struct {
@@ -26,7 +26,7 @@ func NewBoltStore(file string, m Marshaler) (Store, error) {
if err != nil {
return nil, err
}
- if err = db.Update(func(tx *bolt.Tx) (err error) {
+ if err := db.Update(func(tx *bolt.Tx) (err error) {
_, err = tx.CreateBucketIfNotExists([]byte(defaultBoltBucket))
return
}); err != nil {
@@ -84,19 +84,6 @@ func (bs *BoltStore) Restore(r io.Reader) (err error) {
})
}
-func (bs *BoltStore) Tx(writeable bool, f func(tx KVStore) error) error {
- tx, err := db.db.Begin(writeable)
- if err != nil {
- return err
- }
- err = f(&BoltStoreTx{tx, bs})
- if err != nil {
- tx.Rollback()
- return err
- }
- return tx.Commit()
-}
-
func (bs *BoltStore) Get(key string, v interface{}) (err error) {
err = bs.db.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(defaultBoltBucket)).Get([]byte(key))
@@ -133,23 +120,12 @@ func (bs *BoltStore) ForEach(f func(string, []byte) error) error {
}
func (bs *BoltStore) ForEachPrefix(prefix string, f func(string, []byte) error) error {
- return bs.db.View(func(tx *bolt.Tx) error {
- c := tx.Bucket([]byte(defaultBoltBucket)).Cursor()
- p := []byte(prefix)
- for k, v := c.Seek(p); k != nil && bytes.HasPrefix(k, p); k, v = c.Next() {
- err := f(trimPrefix(k, p), v)
- if err != nil {
- return err
- }
+ return bs.ForEach(func(k string, v []byte) error {
+ if trim, ok := hasTrimPrefix(k, prefix); ok {
+ return f(trim, v)
}
return nil
})
- //return bs.ForEach(func(k string, v []byte) error {
- // if trim, ok := hasTrimPrefix(k, prefix); ok {
- // return f(trim, v)
- // }
- // return nil
- //})
}
func (bs *BoltStore) Delete(key string) error {
@@ -161,52 +137,3 @@ func (bs *BoltStore) Delete(key string) error {
func (bs *BoltStore) Close() error {
return bs.db.Close()
}
-
-type BoltStoreTx struct {
- tx *bolt.Tx
- bs *BoltStore
-}
-
-func (tx *BoltStoreTx) Get(key string, v interface{}) (err error) {
- b := tx.tx.Bucket([]byte(defaultBoltBucket)).Get([]byte(key))
- if b == nil {
- return ErrKeyNotFound
- }
- if v != nil {
- return tx.bs.Unmarshal(b, v)
- }
- return nil
-}
-
-func (tx *BoltStoreTx) Set(key string, v interface{}) error {
- var b []byte
- if v != nil {
- b, err = tx.bs.Marshal(v)
- if err != nil {
- return
- }
- }
- return tx.tx.Bucket([]byte(defaultBoltBucket)).Put([]byte(key), b)
-}
-
-func (tx *BoltStoreTx) ForEach(f func(string, []byte) error) error {
- return tx.tx.Bucket([]byte(defaultBoltBucket)).ForEach(func(k, v []byte) error {
- return f(string(k), v)
- })
-}
-
-func (tx *BoltStoreTx) ForEachPrefix(prefix string, f func(string, []byte) error) error {
- c := tx.tx.Bucket([]byte(defaultBoltBucket)).Cursor()
- p := []byte(prefix)
- for k, v := c.Seek(p); k != nil && bytes.HasPrefix(k, p); k, v = c.Next() {
- err := f(trimPrefix(k, p), v)
- if err != nil {
- return err
- }
- }
- return nil
-}
-
-func (tx *BoltStoreTx) Delete(key string) error {
- return tx.tx.Bucket([]byte(defaultBoltBucket)).Delete([]byte(key))
-}
diff --git a/pkg/store/boltstore_test.go b/pkg/store/boltstore_test.go
index 1f2066f..119b006 100644
--- a/pkg/store/boltstore_test.go
+++ b/pkg/store/boltstore_test.go
@@ -1,11 +1,10 @@
-// Copyright (C) 2019 Marius Schellenberger
+// Copyright (C) 2021 Marius Schellenberger
package store
import (
"bytes"
"errors"
- "io/ioutil"
"os"
"path/filepath"
"testing"
@@ -20,7 +19,7 @@ func TestBoltStore(t *testing.T) {
{"123"},
{"hello"},
}
- dir, err := ioutil.TempDir("", "")
+ dir, err := os.MkdirTemp("", "")
if err != nil {
t.Fatal(err)
}
diff --git a/pkg/store/dump.go b/pkg/store/dump.go
index f7b525e..ce47054 100644
--- a/pkg/store/dump.go
+++ b/pkg/store/dump.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019 Marius Schellenberger
+// Copyright (C) 2021 Marius Schellenberger
package store
diff --git a/pkg/store/gob.go b/pkg/store/gob.go
index 9dfdff6..f101705 100644
--- a/pkg/store/gob.go
+++ b/pkg/store/gob.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019 Marius Schellenberger
+// Copyright (C) 2021 Marius Schellenberger
package store
diff --git a/pkg/store/helper.go b/pkg/store/helper.go
index 5d956b0..bc304eb 100644
--- a/pkg/store/helper.go
+++ b/pkg/store/helper.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019 Marius Schellenberger
+// Copyright (C) 2021 Marius Schellenberger
package store
@@ -9,8 +9,3 @@ func hasTrimPrefix(s, prefix string) (string, bool) {
}
return s, false
}
-
-// Parts taken from bytes.TrimPrefix
-func trimPrefix(b, prefix []byte) string {
- return string(b[len(prefix):])
-}
diff --git a/pkg/store/kvstore.go b/pkg/store/kvstore.go
deleted file mode 100644
index c17a4fd..0000000
--- a/pkg/store/kvstore.go
+++ /dev/null
@@ -1,11 +0,0 @@
-// Copyright (C) 2019 Marius Schellenberger
-
-package store
-
-type KVStore interface {
- Get(string, interface{}) error
- Set(string, interface{}) error
- ForEach(func(string, []byte) error) error
- ForEachPrefix(string, func(string, []byte) error) error
- Delete(string) error
-}
diff --git a/pkg/store/marshal.go b/pkg/store/marshal.go
index ce68115..b31bf6c 100644
--- a/pkg/store/marshal.go
+++ b/pkg/store/marshal.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019 Marius Schellenberger
+// Copyright (C) 2021 Marius Schellenberger
package store
diff --git a/pkg/store/store.go b/pkg/store/store.go
index 9421c86..97ab31f 100644
--- a/pkg/store/store.go
+++ b/pkg/store/store.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2019 Marius Schellenberger
+// Copyright (C) 2021 Marius Schellenberger
package store
@@ -14,7 +14,10 @@ type Store interface {
Marshaler
Dumper
Restorer
- KVStore
- Tx(writeable bool, f func(tx KVStore) error) error
+ Get(string, interface{}) error
+ Set(string, interface{}) error
+ ForEach(func(string, []byte) error) error
+ ForEachPrefix(string, func(string, []byte) error) error
+ Delete(string) error
Close() error
}
diff --git a/scripts/install_openbsd.sh b/scripts/install_openbsd.sh
index 70027f8..968184c 100755
--- a/scripts/install_openbsd.sh
+++ b/scripts/install_openbsd.sh
@@ -5,12 +5,18 @@ pkg_add poppler-utils tesseract tesseract-deu tesseract-eng
# rc script
cat <<'EOF'> /etc/rc.d/docstore
-#!/bin/sh
+#!/bin/ksh
daemon="/usr/local/sbin/docstore"
+daemon_user="docstore"
+daemon_logger="daemon.info"
. /etc/rc.d/rc.subr
+pexp=".*${daemon}.*"
+rc_bg=YES
+rc_reload=NO
+
rc_cmd $1
EOF
chmod 555 /etc/rc.d/docstore
diff --git a/sri.sh b/sri.sh
index e71b9e1..9f6e924 100755
--- a/sri.sh
+++ b/sri.sh
@@ -1,8 +1,8 @@
#!/bin/bash
-# Copyright (C) 2019 Marius Schellenberger
+# Copyright (C) 2021 Marius Schellenberger
HASH="sha256"
-DIR="templates"
+DIR="pkg/server/static"
FILES="*.css *.js"
BINS="*.ico"
echo "SRI (subresource integrity) hashes"
diff --git a/templates.sh b/templates.sh
deleted file mode 100755
index e87d338..0000000
--- a/templates.sh
+++ /dev/null
@@ -1,111 +0,0 @@
-#!/bin/bash
-# Copyright (C) 2019 Marius Schellenberger
-
-# this file generates Go source code for the gowiki
-# it contains html and css and uses the contents
-# of the templates directory
-
-templates_go="pkg/server/templates.go"
-echo "Generating ${templates_go}"
-
-if [ ! -d templates ]; then
- echo "error: templates directory not found" 1>&2
- exit 2
-fi
-
-cat << EOF > ${templates_go}
-// Copyright (C) 2019 Marius Schellenberger
-
-// This file is auto-generated by templates.sh
-// using the contents of the templates directory
-
-package server
-
-import (
- "encoding/base64"
- "errors"
- "git.giftfish.de/ston1th/docstore/pkg/log"
- "html/template"
-)
-
-EOF
-
-echo "const (" >> ${templates_go}
-
-for f in $(ls templates/*.html); do
- fname=${f/templates\//}
- fname=${fname/.html/}
- echo " ${fname} = \`$(cat ${f})\`" >> ${templates_go}
-done
-
-for f in $(ls templates/*.css); do
- fname=${f/templates\//}
- fname=${fname/.css/CSS}
- echo " ${fname} = \`$(cat ${f})\`" >> ${templates_go}
-done
-
-echo " favicon = \"$(base64 -w 0 templates/favicon.ico)\"" >> ${templates_go}
-
-cat << EOF >> ${templates_go}
-)
-
-func (s *HTTPServer) loadTemplates() {
- var fatal bool
- s.templ = make(map[string]*template.Template)
- s.res = make(map[string][]byte)
- parse := func(html ...string) (temp *template.Template) {
- var err error
- temp = template.New("")
- for _, s := range html {
- temp, err = temp.Parse(s)
- if err != nil {
- log.Println("parse:", err)
- fatal = true
- return
- }
- }
- return
- }
-
- // resources
- s.res["bootstrap.css"] = []byte(bootstrapCSS)
- s.res["custom.css"] = []byte(customCSS)
- fav, err := base64.StdEncoding.DecodeString(favicon)
- if err != nil {
- log.Println("favicon:", err)
- fatal = true
- }
- s.res["favicon.ico"] = fav
-
- // register
- s.templ["registerHandler"] = parse(index, menu, register)
-
- s.templ["dirHandler"] = parse(index, menu, dir)
- s.templ["fileHandler"] = parse(index, menu, file)
- s.templ["uploadHandler"] = parse(index, menu, upload)
- s.templ["newDirHandler"] = parse(index, menu, newDir)
- s.templ["moveHandler"] = parse(index, menu, move)
- s.templ["enoentHandler"] = parse(index, menu, enoent)
- s.templ["deleteHandler"] = parse(index, menu, delete)
- s.templ["notFoundHandler"] = parse(index, menu, notFound)
- s.templ["forbiddenHandler"] = parse(index, menu, forbidden)
- s.templ["iseHandler"] = parse(index, menu, ise)
- s.templ["loginHandler"] = parse(index, menu, login)
- s.templ["loginTotpHandler"] = parse(index, menu, loginTotp)
- s.templ["searchHandler"] = parse(index, menu, search)
- s.templ["userEditHandler"] = parse(index, menu, userEdit)
- // totp
- s.templ["userTotpHandler"] = parse(index, menu, userTotp)
- // tags
- s.templ["tagsHandler"] = parse(index, menu, tagsList)
- s.templ["tagsNewHandler"] = parse(index, menu, tagsNew)
- s.templ["tagsEditHandler"] = parse(index, menu, tagsEdit)
- s.templ["tagsDelHandler"] = parse(index, menu, tagsDel)
- // logs
- s.templ["logsHandler"] = parse(index, menu, logs)
-
- if fatal {
- log.Fatal("parse: ", errors.New("parsing templates failed"))
- }
-}
-EOF
diff --git a/vendor/git.giftfish.de/ston1th/authdav/LICENSE b/vendor/git.giftfish.de/ston1th/authdav/LICENSE
new file mode 100644
index 0000000..45785b1
--- /dev/null
+++ b/vendor/git.giftfish.de/ston1th/authdav/LICENSE
@@ -0,0 +1,24 @@
+Copyright (C) 2019 Marius Schellenberger
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ * The names of the authors and/or contributors may not be used to
+ endorse or promote products derived from this software without
+ specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL ston1th BE LIABLE FOR ANY
+DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/vendor/git.giftfish.de/ston1th/authdav/README.md b/vendor/git.giftfish.de/ston1th/authdav/README.md
new file mode 100644
index 0000000..aca18a0
--- /dev/null
+++ b/vendor/git.giftfish.de/ston1th/authdav/README.md
@@ -0,0 +1,59 @@
+# authdav - a x/net/webdav wrapper
+
+Wrappers:
+* HTTP Basic Auth wrapper for `webdav.Handler`
+* write only once FileSystem wrapper for `webdav.FileSystem`
+
+## Usage
+
+### HTTP Basic Auth
+
+```
+package main
+
+import (
+ "git.giftfish.de/ston1th/authdav"
+ "golang.org/x/net/webdav"
+ "log"
+ "net/http"
+)
+
+type exampleAuthenticator struct {}
+
+func (exampleAuthenticator) BasicAuth(user, password string) bool {
+ if user == "user" && password == "password" {
+ return true
+ }
+ return false
+}
+
+func main() {
+ auth := exampleAuthenticator{}
+ handler := authdav.NewWebdavBasicAuth("", webdav.Dir("."), nil, nil, auth, "webdav with basic auth")
+ log.Fatal(http.ListenAndServe(":8080", handler))
+}
+```
+
+### Write Only
+
+```
+package main
+
+import (
+ "git.giftfish.de/ston1th/authdav"
+ "golang.org/x/net/webdav"
+ "log"
+ "net/http"
+)
+
+func main() {
+ fs := authdav.NewWriteOnlyOnceFileSystem(webdav.Dir("."))
+ handler := &webdav.Handler{
+ Prefix: "",
+ FileSystem: fs,
+ LockSystem: webdav.NewMemLS(),
+ Logger: nil,
+ }
+ log.Fatal(http.ListenAndServe(":8080", handler))
+}
+```
diff --git a/vendor/git.giftfish.de/ston1th/authdav/authdav.go b/vendor/git.giftfish.de/ston1th/authdav/authdav.go
new file mode 100644
index 0000000..268bc4a
--- /dev/null
+++ b/vendor/git.giftfish.de/ston1th/authdav/authdav.go
@@ -0,0 +1,57 @@
+// Copyright (C) 2019 Marius Schellenberger
+
+package authdav
+
+import (
+ "golang.org/x/net/webdav"
+ "net/http"
+)
+
+// DefaultRealm is the HTTP Basic Auth default realm
+const DefaultRealm = "WebDav"
+
+// Authenticator is the interface for Webdav HTTP Basic Auth
+type Authenticator interface {
+ BasicAuth(username string, password string) bool
+}
+
+// WebdavBasicAuth is a WebDav wrapper for HTTP Basic Auth and implements the http.Handler interface
+type WebdavBasicAuth struct {
+ dav *webdav.Handler
+ auth Authenticator
+ realm string
+}
+
+// NewWebdavBasicAuth returns a new WebdavBasicAuth wrapper
+// prefix can be empty
+// ls and log can be nil
+// an empty authenticator disables HTTP Basic Auth
+// realm can be empty and defaults to authdav.DefaultRealm
+func NewWebdavBasicAuth(prefix string, fs webdav.FileSystem, ls webdav.LockSystem, log func(*http.Request, error), auth Authenticator, realm string) *WebdavBasicAuth {
+ if ls == nil {
+ ls = webdav.NewMemLS()
+ }
+ if realm == "" {
+ realm = DefaultRealm
+ }
+ dav := &webdav.Handler{
+ Prefix: prefix,
+ FileSystem: fs,
+ LockSystem: ls,
+ Logger: log,
+ }
+ return &WebdavBasicAuth{dav, auth, `Basic realm="` + realm + `"`}
+}
+
+func (h *WebdavBasicAuth) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+ if h.auth != nil {
+ u, p, ok := r.BasicAuth()
+ if !ok || !h.auth.BasicAuth(u, p) {
+ w.Header().Set("WWW-Authenticate", h.realm)
+ w.WriteHeader(http.StatusUnauthorized)
+ w.Write([]byte("Unauthorised\n"))
+ return
+ }
+ }
+ h.dav.ServeHTTP(w, r)
+}
diff --git a/vendor/git.giftfish.de/ston1th/authdav/file.go b/vendor/git.giftfish.de/ston1th/authdav/file.go
new file mode 100644
index 0000000..75e2364
--- /dev/null
+++ b/vendor/git.giftfish.de/ston1th/authdav/file.go
@@ -0,0 +1,46 @@
+// Copyright (C) 2019 Marius Schellenberger
+
+package authdav
+
+import (
+ "golang.org/x/net/webdav"
+ "io"
+ "os"
+)
+
+// File is a wrapper for webdav.File providing write only functionality
+type File struct {
+ f webdav.File
+}
+
+func (f *File) Close() error {
+ return f.f.Close()
+}
+
+func (f *File) Read(p []byte) (int, error) {
+ return 0, io.EOF
+}
+
+func (f *File) Seek(offset int64, whence int) (int64, error) {
+ return 0, nil
+}
+
+func (f *File) Readdir(count int) (fi []os.FileInfo, err error) {
+ ffi, err := f.f.Readdir(count)
+ for _, f := range ffi {
+ fi = append(fi, &FileInfo{f})
+ }
+ return
+}
+
+func (f *File) Stat() (os.FileInfo, error) {
+ fi, err := f.f.Stat()
+ if err != nil {
+ return nil, err
+ }
+ return &FileInfo{fi}, nil
+}
+
+func (f *File) Write(p []byte) (int, error) {
+ return f.f.Write(p)
+}
diff --git a/vendor/git.giftfish.de/ston1th/authdav/fileinfo.go b/vendor/git.giftfish.de/ston1th/authdav/fileinfo.go
new file mode 100644
index 0000000..3b4589f
--- /dev/null
+++ b/vendor/git.giftfish.de/ston1th/authdav/fileinfo.go
@@ -0,0 +1,44 @@
+// Copyright (C) 2019 Marius Schellenberger
+
+package authdav
+
+import (
+ "os"
+ "time"
+)
+
+// FileInfo is a wrapper for os.FileInfo which always raturns a zero length for regular files
+type FileInfo struct {
+ fi os.FileInfo
+}
+
+func (fi *FileInfo) Name() string {
+ return fi.fi.Name()
+}
+
+func (fi *FileInfo) RealSize() int64 {
+ return fi.fi.Size()
+}
+
+func (fi *FileInfo) Size() int64 {
+ if fi.Mode().IsRegular() {
+ return 0
+ }
+ return fi.fi.Size()
+}
+
+func (fi *FileInfo) Mode() os.FileMode {
+ return fi.fi.Mode()
+}
+
+func (fi *FileInfo) ModTime() time.Time {
+ return fi.fi.ModTime()
+}
+
+func (fi *FileInfo) IsDir() bool {
+ return fi.fi.IsDir()
+}
+
+func (fi *FileInfo) Sys() interface{} {
+ return fi.fi.Sys()
+}
diff --git a/vendor/git.giftfish.de/ston1th/authdav/filesystem.go b/vendor/git.giftfish.de/ston1th/authdav/filesystem.go
new file mode 100644
index 0000000..447b7ef
--- /dev/null
+++ b/vendor/git.giftfish.de/ston1th/authdav/filesystem.go
@@ -0,0 +1,79 @@
+// Copyright (C) 2019 Marius Schellenberger
+
+package authdav
+
+import (
+ "context"
+ "errors"
+ "golang.org/x/net/webdav"
+ "os"
+)
+
+// ErrWonly indicates forbidden operation
+var ErrWonly = errors.New("authdav: only file write once operations permitted")
+
+const writeFlag = os.O_RDWR | os.O_CREATE | os.O_TRUNC
+
+// WriteOnlyOnceFileSystem is a file write only wrapper for a webdav.FileSystem
+type WriteOnlyOnceFileSystem struct {
+ Filters []Filter
+ fs webdav.FileSystem
+}
+
+// NewWriteOnlyOnceFileSystem returns a new WriteOnlyOnceFileSystem wrapping a webdav.FileSystem
+func NewWriteOnlyOnceFileSystem(fs webdav.FileSystem) *WriteOnlyOnceFileSystem {
+ return &WriteOnlyOnceFileSystem{fs: fs}
+}
+
+// Mkdir is not permitted
+func (fs *WriteOnlyOnceFileSystem) Mkdir(ctx context.Context, name string, perm os.FileMode) error {
+ return ErrWonly
+}
+
+// OpenFile allows write only once operations
+func (fs *WriteOnlyOnceFileSystem) OpenFile(ctx context.Context, name string, flag int, perm os.FileMode) (webdav.File, error) {
+ fi, err := fs.Stat(ctx, name)
+ if err == nil && fi.Mode().IsRegular() {
+ f, ok := fi.(*FileInfo)
+ if ok && flag == writeFlag && f.RealSize() > 0 {
+ return nil, ErrWonly
+ }
+ }
+ for _, filter := range fs.Filters {
+ err = filter.Filter(name)
+ if err != nil {
+ return nil, err
+ }
+ }
+ f, err := fs.fs.OpenFile(ctx, name, flag, perm)
+ if err != nil {
+ return nil, err
+ }
+ return &File{f}, nil
+}
+
+// RemoveAll only permits deleting empty files
+func (fs *WriteOnlyOnceFileSystem) RemoveAll(ctx context.Context, name string) error {
+ fi, err := fs.fs.Stat(ctx, name)
+ if err != nil {
+ return err
+ }
+ if fi.Mode().IsRegular() && fi.Size() == 0 {
+ return fs.fs.RemoveAll(ctx, name)
+ }
+ return ErrWonly
+}
+
+// Rename is not permitted
+func (fs *WriteOnlyOnceFileSystem) Rename(ctx context.Context, oldName, newName string) error {
+ return ErrWonly
+}
+
+// Stat returns os.FileInfo wrapped in FileInfo
+func (fs *WriteOnlyOnceFileSystem) Stat(ctx context.Context, name string) (os.FileInfo, error) {
+ fi, err := fs.fs.Stat(ctx, name)
+ if err != nil {
+ return nil, err
+ }
+ return &FileInfo{fi}, nil
+}
diff --git a/vendor/git.giftfish.de/ston1th/authdav/filter.go b/vendor/git.giftfish.de/ston1th/authdav/filter.go
new file mode 100644
index 0000000..8b039a6
--- /dev/null
+++ b/vendor/git.giftfish.de/ston1th/authdav/filter.go
@@ -0,0 +1,50 @@
+// Copyright (C) 2019 Marius Schellenberger
+
+package authdav
+
+import (
+ "errors"
+ "path/filepath"
+ "strings"
+)
+
+// Matcher is the matcher function type
+type Matcher func(path string) bool
+
+// Filter defines an interface to filter paths or files
+type Filter interface {
+ Filter(path string) error
+}
+
+// ErrMacOSFilter error returned by the MacOSFilter
+var ErrMacOSFilter = errors.New("authdav: MacOS filter: access denied")
+
+// MacOSFilter implements a filter for MacOS specific files
+type MacOSFilter struct {
+ match []Matcher
+}
+
+// NewMacOSFilter returns a new MacOSFilter
+func NewMacOSFilter() *MacOSFilter {
+ return &MacOSFilter{
+ match: []Matcher{
+ func(p string) bool {
+ return p == ".DS_Store"
+ },
+ func(p string) bool {
+ return strings.HasPrefix(p, "._")
+ },
+ },
+ }
+}
+
+// Filter implements the Filter interface
+func (f *MacOSFilter) Filter(path string) error {
+ _, file := filepath.Split(path)
+ for _, m := range f.match {
+ if m(file) {
+ return ErrMacOSFilter
+ }
+ }
+ return nil
+}
diff --git a/vendor/git.giftfish.de/ston1th/godrop/v2/LICENSE b/vendor/git.giftfish.de/ston1th/godrop/v2/LICENSE
index 2bb060c..aa39852 100644
--- a/vendor/git.giftfish.de/ston1th/godrop/v2/LICENSE
+++ b/vendor/git.giftfish.de/ston1th/godrop/v2/LICENSE
@@ -1,4 +1,4 @@
-Copyright (C) 2017 Marius Schellenberger
+Copyright (C) 2022 Marius Schellenberger
All rights reserved.
Redistribution and use in source and binary forms, with or without
diff --git a/vendor/git.giftfish.de/ston1th/godrop/v2/go.mod b/vendor/git.giftfish.de/ston1th/godrop/v2/go.mod
deleted file mode 100644
index a41c9af..0000000
--- a/vendor/git.giftfish.de/ston1th/godrop/v2/go.mod
+++ /dev/null
@@ -1,3 +0,0 @@
-module git.giftfish.de/ston1th/godrop/v2
-
-require golang.org/x/sys v0.0.0-20181026144532-2772b66316d2 // indirect
diff --git a/vendor/git.giftfish.de/ston1th/godrop/v2/go.sum b/vendor/git.giftfish.de/ston1th/godrop/v2/go.sum
deleted file mode 100644
index 36d0abd..0000000
--- a/vendor/git.giftfish.de/ston1th/godrop/v2/go.sum
+++ /dev/null
@@ -1,2 +0,0 @@
-golang.org/x/sys v0.0.0-20181026144532-2772b66316d2 h1:W7CqTdBJ1CmxLKe7LptKDnBYV6PHrVLiGnoyBjaG/JQ=
-golang.org/x/sys v0.0.0-20181026144532-2772b66316d2/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
diff --git a/vendor/git.giftfish.de/ston1th/godrop/v2/godrop.go b/vendor/git.giftfish.de/ston1th/godrop/v2/godrop.go
index 55f39ec..37c7e75 100644
--- a/vendor/git.giftfish.de/ston1th/godrop/v2/godrop.go
+++ b/vendor/git.giftfish.de/ston1th/godrop/v2/godrop.go
@@ -1,5 +1,6 @@
-// Copyright (C) 2018 Marius Schellenberger
+// Copyright (C) 2022 Marius Schellenberger
+//go:build go1.11
// +build go1.11
// Package godrop provides a simple library to drop privileges on Linux and OpenBSD.
diff --git a/vendor/git.giftfish.de/ston1th/godrop/v2/lookup.go b/vendor/git.giftfish.de/ston1th/godrop/v2/lookup.go
index 2d0e1a9..725dc82 100644
--- a/vendor/git.giftfish.de/ston1th/godrop/v2/lookup.go
+++ b/vendor/git.giftfish.de/ston1th/godrop/v2/lookup.go
@@ -1,5 +1,6 @@
-// Copyright (C) 2018 Marius Schellenberger
+// Copyright (C) 2022 Marius Schellenberger
+//go:build go1.11
// +build go1.11
package godrop
diff --git a/vendor/git.giftfish.de/ston1th/godrop/v2/pledge.go b/vendor/git.giftfish.de/ston1th/godrop/v2/pledge.go
index d0e361e..8c97aba 100644
--- a/vendor/git.giftfish.de/ston1th/godrop/v2/pledge.go
+++ b/vendor/git.giftfish.de/ston1th/godrop/v2/pledge.go
@@ -1,5 +1,6 @@
-// Copyright (C) 2018 Marius Schellenberger
+// Copyright (C) 2022 Marius Schellenberger
+//go:build !openbsd
// +build !openbsd
package godrop
diff --git a/vendor/git.giftfish.de/ston1th/godrop/v2/pledge_openbsd.go b/vendor/git.giftfish.de/ston1th/godrop/v2/pledge_openbsd.go
index 7369b03..0af656a 100644
--- a/vendor/git.giftfish.de/ston1th/godrop/v2/pledge_openbsd.go
+++ b/vendor/git.giftfish.de/ston1th/godrop/v2/pledge_openbsd.go
@@ -1,5 +1,6 @@
-// Copyright (C) 2018 Marius Schellenberger
+// Copyright (C) 2022 Marius Schellenberger
+//go:build openbsd
// +build openbsd
package godrop
diff --git a/vendor/git.giftfish.de/ston1th/godrop/v2/unveil.go b/vendor/git.giftfish.de/ston1th/godrop/v2/unveil.go
index 06c8c90..412851b 100644
--- a/vendor/git.giftfish.de/ston1th/godrop/v2/unveil.go
+++ b/vendor/git.giftfish.de/ston1th/godrop/v2/unveil.go
@@ -1,5 +1,6 @@
-// Copyright (C) 2018 Marius Schellenberger
+// Copyright (C) 2022 Marius Schellenberger
+//go:build !openbsd
// +build !openbsd
package godrop
diff --git a/vendor/git.giftfish.de/ston1th/godrop/v2/unveil_openbsd.go b/vendor/git.giftfish.de/ston1th/godrop/v2/unveil_openbsd.go
index f32d630..f1e4bbc 100644
--- a/vendor/git.giftfish.de/ston1th/godrop/v2/unveil_openbsd.go
+++ b/vendor/git.giftfish.de/ston1th/godrop/v2/unveil_openbsd.go
@@ -1,5 +1,6 @@
-// Copyright (C) 2018 Marius Schellenberger
+// Copyright (C) 2022 Marius Schellenberger
+//go:build openbsd
// +build openbsd
package godrop
diff --git a/vendor/git.giftfish.de/ston1th/jwt/v3/LICENSE b/vendor/git.giftfish.de/ston1th/jwt/v3/LICENSE
index 115e96e..74cee2b 100644
--- a/vendor/git.giftfish.de/ston1th/jwt/v3/LICENSE
+++ b/vendor/git.giftfish.de/ston1th/jwt/v3/LICENSE
@@ -1,4 +1,4 @@
-Copyright (C) 2018 Marius Schellenberger
+Copyright (C) 2025 Marius Schellenberger
All rights reserved.
Redistribution and use in source and binary forms, with or without
diff --git a/vendor/git.giftfish.de/ston1th/jwt/v3/blacklist.go b/vendor/git.giftfish.de/ston1th/jwt/v3/blacklist.go
index b60a928..180c5fd 100644
--- a/vendor/git.giftfish.de/ston1th/jwt/v3/blacklist.go
+++ b/vendor/git.giftfish.de/ston1th/jwt/v3/blacklist.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2018 Marius Schellenberger
+// Copyright (C) 2025 Marius Schellenberger
package jwt
diff --git a/vendor/git.giftfish.de/ston1th/jwt/v3/claims.go b/vendor/git.giftfish.de/ston1th/jwt/v3/claims.go
index 10cbcfe..a5f4364 100644
--- a/vendor/git.giftfish.de/ston1th/jwt/v3/claims.go
+++ b/vendor/git.giftfish.de/ston1th/jwt/v3/claims.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2018 Marius Schellenberger
+// Copyright (C) 2025 Marius Schellenberger
package jwt
@@ -30,37 +30,28 @@ func (c Claims) GetBool(key string) (b bool) {
// GetInt returns an int from the claims map
func (c Claims) GetInt(key string) (i int) {
- if f, ok := c.getFloat64(key); ok {
- return int(f)
- }
- if v, ok := c.Get(key); ok && v != nil {
- i, _ = v.(int)
- }
+ i = int(c.GetInt64(key))
return
}
// GetInt64 returns an int64 from the claims map
func (c Claims) GetInt64(key string) (i int64) {
- if f, ok := c.getFloat64(key); ok {
- return int64(f)
- }
if v, ok := c.Get(key); ok && v != nil {
- i, _ = v.(int64)
+ switch val := v.(type) {
+ case int64:
+ i = val
+ case float64:
+ i = int64(val)
+ }
}
return
}
// getFloat64 returns a float64 and ok from the claims map
-func (c Claims) getFloat64(key string) (f float64, fok bool) {
- if v, ok := c.Get(key); ok && v != nil {
- f, fok = v.(float64)
- }
- return
-}
-
-// GetFloat64 returns a float64 from the claims map
func (c Claims) GetFloat64(key string) (f float64) {
- f, _ = c.getFloat64(key)
+ if v, ok := c.Get(key); ok && v != nil {
+ f, _ = v.(float64)
+ }
return
}
diff --git a/vendor/git.giftfish.de/ston1th/jwt/v3/encoding.go b/vendor/git.giftfish.de/ston1th/jwt/v3/encoding.go
index 73b81cf..3693f5f 100644
--- a/vendor/git.giftfish.de/ston1th/jwt/v3/encoding.go
+++ b/vendor/git.giftfish.de/ston1th/jwt/v3/encoding.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2018 Marius Schellenberger
+// Copyright (C) 2025 Marius Schellenberger
package jwt
diff --git a/vendor/git.giftfish.de/ston1th/jwt/v3/go.mod b/vendor/git.giftfish.de/ston1th/jwt/v3/go.mod
deleted file mode 100644
index 6752b42..0000000
--- a/vendor/git.giftfish.de/ston1th/jwt/v3/go.mod
+++ /dev/null
@@ -1 +0,0 @@
-module git.giftfish.de/ston1th/jwt/v3
diff --git a/vendor/git.giftfish.de/ston1th/jwt/v3/hash.go b/vendor/git.giftfish.de/ston1th/jwt/v3/hash.go
index 2c224ab..e47c0cc 100644
--- a/vendor/git.giftfish.de/ston1th/jwt/v3/hash.go
+++ b/vendor/git.giftfish.de/ston1th/jwt/v3/hash.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2018 Marius Schellenberger
+// Copyright (C) 2025 Marius Schellenberger
package jwt
@@ -14,19 +14,25 @@ const (
HS512Name = "HS512"
)
-// ParseHash returns the Hash type equal to the input string
-func ParseHash(alg string) Hash {
+// NewHash returns the Hash type equal to the input string
+func NewHash(alg string) Hash {
switch alg {
case HS256Name:
- return NewHS256()
+ return hs256
case HS384Name:
- return NewHS384()
+ return hs384
case HS512Name:
- return NewHS512()
+ return hs512
}
return nil
}
+var (
+ hs256 = HS256{}
+ hs384 = HS384{}
+ hs512 = HS512{}
+)
+
// Hash is the hashsum interface for signing the jwt
type Hash interface {
Hash() hash.Hash
@@ -36,11 +42,6 @@ type Hash interface {
// HS256 implements the Hash interface with SHA256
type HS256 struct{}
-// NewHS256 returns a new HS256 instance
-func NewHS256() Hash {
- return HS256{}
-}
-
// Alg returns the algorithm name "HS256"
func (HS256) Alg() string {
return HS256Name
@@ -54,11 +55,6 @@ func (HS256) Hash() hash.Hash {
// HS384 implements the Hash interface with SHA384
type HS384 struct{}
-// NewHS384 returns a new HS384 instance
-func NewHS384() Hash {
- return HS384{}
-}
-
// Alg returns the algorithm name "HS384"
func (HS384) Alg() string {
return HS384Name
@@ -72,11 +68,6 @@ func (HS384) Hash() hash.Hash {
// HS512 implements the Hash interface with SHA512
type HS512 struct{}
-// NewHS512 returns a new HS512 instance
-func NewHS512() Hash {
- return HS512{}
-}
-
// Alg returns the algorithm name "HS512"
func (HS512) Alg() string {
return HS512Name
diff --git a/vendor/git.giftfish.de/ston1th/jwt/v3/jwt.go b/vendor/git.giftfish.de/ston1th/jwt/v3/jwt.go
index 44f82ec..f9ce6af 100644
--- a/vendor/git.giftfish.de/ston1th/jwt/v3/jwt.go
+++ b/vendor/git.giftfish.de/ston1th/jwt/v3/jwt.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2018 Marius Schellenberger
+// Copyright (C) 2025 Marius Schellenberger
// Package jwt provides a easy to use JSON Web Token and blacklisting library
package jwt
@@ -31,6 +31,8 @@ const (
ExpClaim = "exp"
// NbfClaim is the nbf claim name
NbfClaim = "nbf"
+ // NonceClaim
+ NonceClaim = "nonce"
// TokenSeparator is the tokens separator char
TokenSeparator = "."
)
@@ -57,14 +59,39 @@ var (
ErrInvalidKeySize = errors.New(jwtErr + "invalid secret key size")
)
+type JWTOption func(*JWT)
+
+func WithExpiry(expiry time.Duration) JWTOption {
+ return func(jwt *JWT) {
+ jwt.expiry = expiry
+ }
+}
+
+func WithBlacklist(blacklist Blacklist) JWTOption {
+ return func(jwt *JWT) {
+ jwt.blacklist = blacklist
+ }
+}
+func WithSecret(secret io.Reader) JWTOption {
+ return func(jwt *JWT) {
+ jwt.secretReader = secret
+ }
+}
+func WithNonce() JWTOption {
+ return func(jwt *JWT) {
+ jwt.nonce = true
+ }
+}
+
// JWT represents the JSON Web Token signing and blacklisting infrastructure
type JWT struct {
- key []byte
- expiry time.Duration
-
- blacklist Blacklist
- stopOnce sync.Once
- done chan struct{}
+ secretReader io.Reader
+ key []byte
+ expiry time.Duration
+ blacklist Blacklist
+ nonce bool
+ stopOnce sync.Once
+ done chan struct{}
}
// New returns a new JWT object with the given expiry timeout.
@@ -73,14 +100,18 @@ type JWT struct {
// If secret is nil the DefaultSecretReader is used.
// If blacklisting is enabled, the JWT object leaks a goroutine to garbage-collect expired blacklisted tokens.
// Call the Stop() method to exit the goroutine.
-func New(expiry time.Duration, blacklist Blacklist, secret io.Reader) (*JWT, error) {
- if expiry <= 0 {
- expiry = DefaultExpiry
+func New(options ...JWTOption) (*JWT, error) {
+ jwt := &JWT{}
+ for _, option := range options {
+ option(jwt)
}
- if secret == nil {
- secret = DefaultSecretReader
+ if jwt.expiry <= 0 {
+ jwt.expiry = DefaultExpiry
}
- secret = io.LimitReader(secret, KeySize)
+ if jwt.secretReader == nil {
+ jwt.secretReader = DefaultSecretReader
+ }
+ secret := io.LimitReader(jwt.secretReader, KeySize)
key := make([]byte, KeySize)
i, err := secret.Read(key)
if err != nil {
@@ -89,12 +120,8 @@ func New(expiry time.Duration, blacklist Blacklist, secret io.Reader) (*JWT, err
if i < KeySize {
return nil, ErrInvalidKeySize
}
- jwt := &JWT{
- key: key,
- expiry: expiry,
- blacklist: blacklist,
- }
- if blacklist != nil {
+ jwt.key = key
+ if jwt.blacklist != nil {
jwt.done = make(chan struct{})
go jwt.clean()
}
@@ -128,7 +155,7 @@ func (jwt *JWT) Invalidate(t *Token) error {
if t.Header.GetString(TypClaim) != Typ {
return ErrNoJWT
}
- h := ParseHash(t.Header.GetString(AlgClaim))
+ h := NewHash(t.Header.GetString(AlgClaim))
if h == nil {
return ErrUnsupportedAlg
}
@@ -190,7 +217,7 @@ func (jwt *JWT) Sign(t *Token) (err error) {
if t.Header.GetString(TypClaim) != Typ {
return ErrNoJWT
}
- h := ParseHash(t.Header.GetString(AlgClaim))
+ h := NewHash(t.Header.GetString(AlgClaim))
if h == nil {
return ErrUnsupportedAlg
}
@@ -201,6 +228,9 @@ func (jwt *JWT) Sign(t *Token) (err error) {
if _, ok := t.Claims.Get(NbfClaim); !ok {
t.Claims.Set(NbfClaim, NewNbf(now))
}
+ if jwt.nonce {
+ t.Claims.Set(NonceClaim, enc.EncodeToString(key16()))
+ }
head, err := json.Marshal(t.Header)
if err != nil {
return
@@ -230,7 +260,7 @@ func (jwt *JWT) Verify(t *Token) error {
if t.Header.GetString(TypClaim) != Typ {
return ErrNoJWT
}
- h := ParseHash(t.Header.GetString(AlgClaim))
+ h := NewHash(t.Header.GetString(AlgClaim))
if h == nil {
return ErrUnsupportedAlg
}
diff --git a/vendor/git.giftfish.de/ston1th/jwt/v3/rand.go b/vendor/git.giftfish.de/ston1th/jwt/v3/rand.go
new file mode 100644
index 0000000..bd1b191
--- /dev/null
+++ b/vendor/git.giftfish.de/ston1th/jwt/v3/rand.go
@@ -0,0 +1,11 @@
+// Copyright (C) 2025 Marius Schellenberger
+
+package jwt
+
+import "crypto/rand"
+
+func key16() []byte {
+ b := make([]byte, 16)
+ rand.Read(b)
+ return b
+}
diff --git a/vendor/git.giftfish.de/ston1th/jwt/v3/time.go b/vendor/git.giftfish.de/ston1th/jwt/v3/time.go
index 71b10ca..77e151f 100644
--- a/vendor/git.giftfish.de/ston1th/jwt/v3/time.go
+++ b/vendor/git.giftfish.de/ston1th/jwt/v3/time.go
@@ -1,3 +1,5 @@
+// Copyright (C) 2025 Marius Schellenberger
+
package jwt
import "time"
diff --git a/vendor/git.giftfish.de/ston1th/jwt/v3/token.go b/vendor/git.giftfish.de/ston1th/jwt/v3/token.go
index a1f1dd1..772415c 100644
--- a/vendor/git.giftfish.de/ston1th/jwt/v3/token.go
+++ b/vendor/git.giftfish.de/ston1th/jwt/v3/token.go
@@ -1,4 +1,4 @@
-// Copyright (C) 2018 Marius Schellenberger
+// Copyright (C) 2025 Marius Schellenberger
package jwt
@@ -45,7 +45,7 @@ func NewToken(claims Claims, hash Hash) *Token {
claims = make(Claims)
}
if hash == nil {
- hash = NewHS256()
+ hash = NewHash(HS256Name)
}
return &Token{
Header: Claims{
diff --git a/vendor/github.com/RoaringBitmap/roaring/.drone.yml b/vendor/github.com/RoaringBitmap/roaring/.drone.yml
new file mode 100644
index 0000000..7936bfe
--- /dev/null
+++ b/vendor/github.com/RoaringBitmap/roaring/.drone.yml
@@ -0,0 +1,19 @@
+kind: pipeline
+name: default
+
+workspace:
+ base: /go
+ path: src/github.com/RoaringBitmap/roaring
+
+steps:
+- name: test
+ image: golang
+ commands:
+ - go get -t
+ - go test
+ - go build -tags appengine
+ - go test -tags appengine
+ - GOARCH=386 go build
+ - GOARCH=386 go test
+ - GOARCH=arm go build
+ - GOARCH=arm64 go build
diff --git a/vendor/github.com/RoaringBitmap/roaring/.gitignore b/vendor/github.com/RoaringBitmap/roaring/.gitignore
index b7943ab..851f323 100644
--- a/vendor/github.com/RoaringBitmap/roaring/.gitignore
+++ b/vendor/github.com/RoaringBitmap/roaring/.gitignore
@@ -3,4 +3,3 @@ roaring-fuzz.zip
workdir
coverage.out
testdata/all3.classic
-testdata/all3.msgp.snappy
diff --git a/vendor/github.com/RoaringBitmap/roaring/.travis.yml b/vendor/github.com/RoaringBitmap/roaring/.travis.yml
deleted file mode 100644
index 32ceaa6..0000000
--- a/vendor/github.com/RoaringBitmap/roaring/.travis.yml
+++ /dev/null
@@ -1,32 +0,0 @@
-language: go
-sudo: false
-install:
-- go get -t github.com/RoaringBitmap/roaring
-- go get -t golang.org/x/tools/cmd/cover
-- go get -t github.com/mattn/goveralls
-- go get -t github.com/mschoch/smat
-notifications:
- email: false
-go:
-- "1.7.x"
-- "1.8.x"
-- "1.9.x"
-- "1.10.x"
-- "1.11.x"
-- "1.12.x"
-- tip
-
-# whitelist
-branches:
- only:
- - master
-script:
-- goveralls -v -service travis-ci -ignore arraycontainer_gen.go,bitmapcontainer_gen.go,rle16_gen.go,rle_gen.go,roaringarray_gen.go,rle.go || go test
-- go test -race -run TestConcurrent*
-- GOARCH=arm64 go build
-- GOARCH=386 go build
-- GOARCH=386 go test
-- GOARCH=arm go build
-matrix:
- allow_failures:
- - go: tip
diff --git a/vendor/github.com/RoaringBitmap/roaring/CONTRIBUTORS b/vendor/github.com/RoaringBitmap/roaring/CONTRIBUTORS
index 4a8bda2..1a8da9c 100644
--- a/vendor/github.com/RoaringBitmap/roaring/CONTRIBUTORS
+++ b/vendor/github.com/RoaringBitmap/roaring/CONTRIBUTORS
@@ -12,4 +12,7 @@ Vali Malinoiu (@0x4139),
Forud Ghafouri (@fzerorubigd),
Joe Nall (@joenall),
(@fredim),
-Edd Robinson (@e-dard)
+Edd Robinson (@e-dard),
+Alexander Petrov (@alldroll),
+Guy Molinari (@guymolinari),
+Ling Jin (@JinLingChristopher)
diff --git a/vendor/github.com/RoaringBitmap/roaring/Makefile b/vendor/github.com/RoaringBitmap/roaring/Makefile
deleted file mode 100644
index c489376..0000000
--- a/vendor/github.com/RoaringBitmap/roaring/Makefile
+++ /dev/null
@@ -1,111 +0,0 @@
-.PHONY: help all test format fmtcheck vet lint qa deps clean nuke ser fetch-real-roaring-datasets
-
-
-
-
-
-
-
-
-# Display general help about this command
-help:
- @echo ""
- @echo "The following commands are available:"
- @echo ""
- @echo " make qa : Run all the tests"
- @echo " make test : Run the unit tests"
- @echo ""
- @echo " make format : Format the source code"
- @echo " make fmtcheck : Check if the source code has been formatted"
- @echo " make vet : Check for suspicious constructs"
- @echo " make lint : Check for style errors"
- @echo ""
- @echo " make deps : Get the dependencies"
- @echo " make clean : Remove any build artifact"
- @echo " make nuke : Deletes any intermediate file"
- @echo ""
- @echo " make fuzz-smat : Fuzzy testing with smat"
- @echo " make fuzz-stream : Fuzzy testing with stream deserialization"
- @echo " make fuzz-buffer : Fuzzy testing with buffer deserialization"
- @echo ""
-
-# Alias for help target
-all: help
-test:
- go test
- go test -race -run TestConcurrent*
-# Format the source code
-format:
- @find ./ -type f -name "*.go" -exec gofmt -w {} \;
-
-# Check if the source code has been formatted
-fmtcheck:
- @mkdir -p target
- @find ./ -type f -name "*.go" -exec gofmt -d {} \; | tee target/format.diff
- @test ! -s target/format.diff || { echo "ERROR: the source code has not been formatted - please use 'make format' or 'gofmt'"; exit 1; }
-
-# Check for syntax errors
-vet:
- GOPATH=$(GOPATH) go vet ./...
-
-# Check for style errors
-lint:
- GOPATH=$(GOPATH) PATH=$(GOPATH)/bin:$(PATH) golint ./...
-
-
-
-
-
-# Alias to run all quality-assurance checks
-qa: fmtcheck test vet lint
-
-# --- INSTALL ---
-
-# Get the dependencies
-deps:
- GOPATH=$(GOPATH) go get github.com/smartystreets/goconvey/convey
- GOPATH=$(GOPATH) go get github.com/willf/bitset
- GOPATH=$(GOPATH) go get github.com/golang/lint/golint
- GOPATH=$(GOPATH) go get github.com/mschoch/smat
- GOPATH=$(GOPATH) go get github.com/dvyukov/go-fuzz/go-fuzz
- GOPATH=$(GOPATH) go get github.com/dvyukov/go-fuzz/go-fuzz-build
- GOPATH=$(GOPATH) go get github.com/glycerine/go-unsnap-stream
- GOPATH=$(GOPATH) go get github.com/philhofer/fwd
- GOPATH=$(GOPATH) go get github.com/jtolds/gls
-
-fuzz-smat:
- go test -tags=gofuzz -run=TestGenerateSmatCorpus
- go-fuzz-build -func FuzzSmat github.com/RoaringBitmap/roaring
- go-fuzz -bin=./roaring-fuzz.zip -workdir=workdir/ -timeout=200
-
-
-fuzz-stream:
- go-fuzz-build -func FuzzSerializationStream github.com/RoaringBitmap/roaring
- go-fuzz -bin=./roaring-fuzz.zip -workdir=workdir/ -timeout=200
-
-
-fuzz-buffer:
- go-fuzz-build -func FuzzSerializationBuffer github.com/RoaringBitmap/roaring
- go-fuzz -bin=./roaring-fuzz.zip -workdir=workdir/ -timeout=200
-
-# Remove any build artifact
-clean:
- GOPATH=$(GOPATH) go clean ./...
-
-# Deletes any intermediate file
-nuke:
- rm -rf ./target
- GOPATH=$(GOPATH) go clean -i ./...
-
-
-ser:
- go generate
-
-cover:
- go test -coverprofile=coverage.out
- go tool cover -html=coverage.out
-
-fetch-real-roaring-datasets:
- # pull github.com/RoaringBitmap/real-roaring-datasets -> testdata/real-roaring-datasets
- git submodule init
- git submodule update
diff --git a/vendor/github.com/RoaringBitmap/roaring/README.md b/vendor/github.com/RoaringBitmap/roaring/README.md
index a3e766d..f6705df 100644
--- a/vendor/github.com/RoaringBitmap/roaring/README.md
+++ b/vendor/github.com/RoaringBitmap/roaring/README.md
@@ -1,20 +1,24 @@
-roaring [](https://travis-ci.org/RoaringBitmap/roaring) [](https://coveralls.io/github/RoaringBitmap/roaring?branch=master) [](https://godoc.org/github.com/RoaringBitmap/roaring) [](https://goreportcard.com/report/github.com/RoaringBitmap/roaring)
+# roaring
+
+[](https://godoc.org/github.com/RoaringBitmap/roaring) [](https://goreportcard.com/report/github.com/RoaringBitmap/roaring)
+
+
+
+
=============
This is a go version of the Roaring bitmap data structure.
-
-
Roaring bitmaps are used by several major systems such as [Apache Lucene][lucene] and derivative systems such as [Solr][solr] and
-[Elasticsearch][elasticsearch], [Metamarkets' Druid][druid], [LinkedIn Pinot][pinot], [Netflix Atlas][atlas], [Apache Spark][spark], [OpenSearchServer][opensearchserver], [Cloud Torrent][cloudtorrent], [Whoosh][whoosh], [Pilosa][pilosa], [Microsoft Visual Studio Team Services (VSTS)][vsts], and eBay's [Apache Kylin][kylin].
+[Elasticsearch][elasticsearch], [Apache Druid (Incubating)][druid], [LinkedIn Pinot][pinot], [Netflix Atlas][atlas], [Apache Spark][spark], [OpenSearchServer][opensearchserver], [anacrolix/torrent][anacrolix/torrent], [Whoosh][whoosh], [Redpanda](https://github.com/redpanda-data/redpanda), [Pilosa][pilosa], [Microsoft Visual Studio Team Services (VSTS)][vsts], and eBay's [Apache Kylin][kylin]. The YouTube SQL Engine, [Google Procella](https://research.google/pubs/pub48388/), uses Roaring bitmaps for indexing.
[lucene]: https://lucene.apache.org/
[solr]: https://lucene.apache.org/solr/
[elasticsearch]: https://www.elastic.co/products/elasticsearch
-[druid]: http://druid.io/
+[druid]: https://druid.apache.org/
[spark]: https://spark.apache.org/
[opensearchserver]: http://www.opensearchserver.com
-[cloudtorrent]: https://github.com/jpillora/cloud-torrent
+[anacrolix/torrent]: https://github.com/anacrolix/torrent
[whoosh]: https://bitbucket.org/mchaput/whoosh/wiki/Home
[pilosa]: https://www.pilosa.com/
[kylin]: http://kylin.apache.org/
@@ -28,11 +32,18 @@ Roaring bitmaps are found to work well in many important applications:
The ``roaring`` Go library is used by
-* [Cloud Torrent](https://github.com/jpillora/cloud-torrent): a self-hosted remote torrent client
-* [runv](https://github.com/hyperhq/runv): an Hypervisor-based runtime for the Open Containers Initiative
+* [anacrolix/torrent]
* [InfluxDB](https://www.influxdata.com)
* [Pilosa](https://www.pilosa.com/)
* [Bleve](http://www.blevesearch.com)
+* [Weaviate](https://github.com/weaviate/weaviate)
+* [lindb](https://github.com/lindb/lindb)
+* [Elasticell](https://github.com/deepfabric/elasticell)
+* [SourceGraph](https://github.com/sourcegraph/sourcegraph)
+* [M3](https://github.com/m3db/m3)
+* [trident](https://github.com/NetApp/trident)
+* [Husky](https://www.datadoghq.com/blog/engineering/introducing-husky/)
+* [FrostDB](https://github.com/polarsignals/frostdb)
This library is used in production in several systems, it is part of the [Awesome Go collection](https://awesome-go.com).
@@ -45,24 +56,108 @@ This code is licensed under Apache License, Version 2.0 (ASL2.0).
Copyright 2016-... by the authors.
+When should you use a bitmap?
+===================================
+
+
+Sets are a fundamental abstraction in
+software. They can be implemented in various
+ways, as hash sets, as trees, and so forth.
+In databases and search engines, sets are often an integral
+part of indexes. For example, we may need to maintain a set
+of all documents or rows (represented by numerical identifier)
+that satisfy some property. Besides adding or removing
+elements from the set, we need fast functions
+to compute the intersection, the union, the difference between sets, and so on.
+
+
+To implement a set
+of integers, a particularly appealing strategy is the
+bitmap (also called bitset or bit vector). Using n bits,
+we can represent any set made of the integers from the range
+[0,n): the ith bit is set to one if integer i is present in the set.
+Commodity processors use words of W=32 or W=64 bits. By combining many such words, we can
+support large values of n. Intersections, unions and differences can then be implemented
+ as bitwise AND, OR and ANDNOT operations.
+More complicated set functions can also be implemented as bitwise operations.
+
+When the bitset approach is applicable, it can be orders of
+magnitude faster than other possible implementation of a set (e.g., as a hash set)
+while using several times less memory.
+
+However, a bitset, even a compressed one is not always applicable. For example, if
+you have 1000 random-looking integers, then a simple array might be the best representation.
+We refer to this case as the "sparse" scenario.
+
+When should you use compressed bitmaps?
+===================================
+
+An uncompressed BitSet can use a lot of memory. For example, if you take a BitSet
+and set the bit at position 1,000,000 to true and you have just over 100kB. That is over 100kB
+to store the position of one bit. This is wasteful even if you do not care about memory:
+suppose that you need to compute the intersection between this BitSet and another one
+that has a bit at position 1,000,001 to true, then you need to go through all these zeroes,
+whether you like it or not. That can become very wasteful.
+
+This being said, there are definitively cases where attempting to use compressed bitmaps is wasteful.
+For example, if you have a small universe size. E.g., your bitmaps represent sets of integers
+from [0,n) where n is small (e.g., n=64 or n=128). If you can use uncompressed BitSet and
+it does not blow up your memory usage, then compressed bitmaps are probably not useful
+to you. In fact, if you do not need compression, then a BitSet offers remarkable speed.
+
+The sparse scenario is another use case where compressed bitmaps should not be used.
+Keep in mind that random-looking data is usually not compressible. E.g., if you have a small set of
+32-bit random integers, it is not mathematically possible to use far less than 32 bits per integer,
+and attempts at compression can be counterproductive.
+
+How does Roaring compares with the alternatives?
+==================================================
+
+
+Most alternatives to Roaring are part of a larger family of compressed bitmaps that are run-length-encoded
+bitmaps. They identify long runs of 1s or 0s and they represent them with a marker word.
+If you have a local mix of 1s and 0, you use an uncompressed word.
+
+There are many formats in this family:
+
+* Oracle's BBC is an obsolete format at this point: though it may provide good compression,
+it is likely much slower than more recent alternatives due to excessive branching.
+* WAH is a patented variation on BBC that provides better performance.
+* Concise is a variation on the patented WAH. It some specific instances, it can compress
+much better than WAH (up to 2x better), but it is generally slower.
+* EWAH is both free of patent, and it is faster than all the above. On the downside, it
+does not compress quite as well. It is faster because it allows some form of "skipping"
+over uncompressed words. So though none of these formats are great at random access, EWAH
+is better than the alternatives.
+
+
+
+There is a big problem with these formats however that can hurt you badly in some cases: there is no random access. If you want to check whether a given value is present in the set, you have to start from the beginning and "uncompress" the whole thing. This means that if you want to intersect a big set with a large set, you still have to uncompress the whole big set in the worst case...
+
+Roaring solves this problem. It works in the following manner. It divides the data into chunks of 2
16 integers
+(e.g., [0, 2
16), [2
16, 2 x 2
16), ...). Within a chunk, it can use an uncompressed bitmap, a simple list of integers,
+or a list of runs. Whatever format it uses, they all allow you to check for the presence of any one value quickly
+(e.g., with a binary search). The net result is that Roaring can compute many operations much faster than run-length-encoded
+formats like WAH, EWAH, Concise... Maybe surprisingly, Roaring also generally offers better compression ratios.
+
+
+
+
### References
- Daniel Lemire, Owen Kaser, Nathan Kurz, Luca Deri, Chris O'Hara, François Saint-Jacques, Gregory Ssi-Yan-Kai, Roaring Bitmaps: Implementation of an Optimized Software Library, Software: Practice and Experience 48 (4), 2018 [arXiv:1709.07821](https://arxiv.org/abs/1709.07821)
- Samy Chambi, Daniel Lemire, Owen Kaser, Robert Godin,
Better bitmap performance with Roaring bitmaps,
-Software: Practice and Experience 46 (5), 2016.
-http://arxiv.org/abs/1402.6407 This paper used data from http://lemire.me/data/realroaring2014.html
-- Daniel Lemire, Gregory Ssi-Yan-Kai, Owen Kaser, Consistently faster and smaller compressed bitmaps with Roaring, Software: Practice and Experience 46 (11), 2016. http://arxiv.org/abs/1603.06549
-
+Software: Practice and Experience 46 (5), 2016.[arXiv:1402.6407](http://arxiv.org/abs/1402.6407) This paper used data from http://lemire.me/data/realroaring2014.html
+- Daniel Lemire, Gregory Ssi-Yan-Kai, Owen Kaser, Consistently faster and smaller compressed bitmaps with Roaring, Software: Practice and Experience 46 (11), 2016. [arXiv:1603.06549](http://arxiv.org/abs/1603.06549)
### Dependencies
Dependencies are fetched automatically by giving the `-t` flag to `go get`.
they include
- - github.com/smartystreets/goconvey/convey
- - github.com/willf/bitset
+ - github.com/bits-and-blooms/bitset
- github.com/mschoch/smat
- github.com/glycerine/go-unsnap-stream
- github.com/philhofer/fwd
@@ -74,6 +169,15 @@ Note that the smat library requires Go 1.6 or better.
- go get -t github.com/RoaringBitmap/roaring
+### Instructions for contributors
+
+Using bash or other common shells:
+```
+$ git clone git@github.com:RoaringBitmap/roaring.git
+$ export GO111MODULE=on
+$ go mod tidy
+$ go test -v
+```
### Example
@@ -166,10 +270,70 @@ That is, given a fixed overhead for the universe size (x), Roaring
bitmaps never use more than 2 bytes per integer. You can call
``BoundSerializedSizeInBytes`` for a more precise estimate.
+### 64-bit Roaring
+
+By default, roaring is used to stored unsigned 32-bit integers. However, we also offer
+an extension dedicated to 64-bit integers. It supports roughly the same functions:
+
+```go
+package main
+
+import (
+ "fmt"
+ "github.com/RoaringBitmap/roaring/roaring64"
+ "bytes"
+)
+
+
+func main() {
+ // example inspired by https://github.com/fzandona/goroar
+ fmt.Println("==roaring64==")
+ rb1 := roaring64.BitmapOf(1, 2, 3, 4, 5, 100, 1000)
+ fmt.Println(rb1.String())
+
+ rb2 := roaring64.BitmapOf(3, 4, 1000)
+ fmt.Println(rb2.String())
+
+ rb3 := roaring64.New()
+ fmt.Println(rb3.String())
+
+ fmt.Println("Cardinality: ", rb1.GetCardinality())
+
+ fmt.Println("Contains 3? ", rb1.Contains(3))
+
+ rb1.And(rb2)
+
+ rb3.Add(1)
+ rb3.Add(5)
+
+ rb3.Or(rb1)
+
+
+
+ // prints 1, 3, 4, 5, 1000
+ i := rb3.Iterator()
+ for i.HasNext() {
+ fmt.Println(i.Next())
+ }
+ fmt.Println()
+
+ // next we include an example of serialization
+ buf := new(bytes.Buffer)
+ rb1.WriteTo(buf) // we omit error handling
+ newrb:= roaring64.New()
+ newrb.ReadFrom(buf)
+ if rb1.Equals(newrb) {
+ fmt.Println("I wrote the content to a byte stream and read it back.")
+ }
+ // you can iterate over bitmaps using ReverseIterator(), Iterator, ManyIterator()
+}
+```
+
+Only the 32-bit roaring format is standard and cross-operable between Java, C++, C and Go. There is no guarantee that the 64-bit versions are compatible.
### Documentation
-Current documentation is available at http://godoc.org/github.com/RoaringBitmap/roaring
+Current documentation is available at https://pkg.go.dev/github.com/RoaringBitmap/roaring and https://pkg.go.dev/github.com/RoaringBitmap/roaring/roaring64
### Goroutine safety
@@ -207,7 +371,7 @@ You can use roaring with gore:
- go get -u github.com/motemen/gore
- Make sure that ``$GOPATH/bin`` is in your ``$PATH``.
-- go get github/RoaringBitmap/roaring
+- go get github.com/RoaringBitmap/roaring
```go
$ gore
@@ -228,12 +392,14 @@ You can help us test further the library with fuzzy testing:
go get github.com/dvyukov/go-fuzz/go-fuzz-build
go test -tags=gofuzz -run=TestGenerateSmatCorpus
go-fuzz-build github.com/RoaringBitmap/roaring
- go-fuzz -bin=./roaring-fuzz.zip -workdir=workdir/ -timeout=200
+ go-fuzz -bin=./roaring-fuzz.zip -workdir=workdir/ -timeout=200 -func FuzzSmat
Let it run, and if the # of crashers is > 0, check out the reports in
the workdir where you should be able to find the panic goroutine stack
traces.
+You may also replace `-func FuzzSmat` by `-func FuzzSerializationBuffer` or `-func FuzzSerializationStream`.
+
### Alternative in Go
There is a Go version wrapping the C/C++ implementation https://github.com/RoaringBitmap/gocroaring
diff --git a/vendor/github.com/RoaringBitmap/roaring/arraycontainer.go b/vendor/github.com/RoaringBitmap/roaring/arraycontainer.go
index 8b1a6c8..80fa676 100644
--- a/vendor/github.com/RoaringBitmap/roaring/arraycontainer.go
+++ b/vendor/github.com/RoaringBitmap/roaring/arraycontainer.go
@@ -4,8 +4,6 @@ import (
"fmt"
)
-//go:generate msgp -unexported
-
type arrayContainer struct {
content []uint16
}
@@ -18,13 +16,35 @@ func (ac *arrayContainer) String() string {
return s + "}"
}
-func (ac *arrayContainer) fillLeastSignificant16bits(x []uint32, i int, mask uint32) {
- for k := 0; k < len(ac.content); k++ {
- x[k+i] = uint32(ac.content[k]) | mask
+func (ac *arrayContainer) fillLeastSignificant16bits(x []uint32, i int, mask uint32) int {
+ if i < 0 {
+ panic("negative index")
}
+ if len(ac.content) == 0 {
+ return i
+ }
+ _ = x[len(ac.content)-1+i]
+ _ = ac.content[len(ac.content)-1]
+ for k := 0; k < len(ac.content); k++ {
+ x[k+i] =
+ uint32(ac.content[k]) | mask
+ }
+ return i + len(ac.content)
}
-func (ac *arrayContainer) getShortIterator() shortIterable {
+func (ac *arrayContainer) iterate(cb func(x uint16) bool) bool {
+ iterator := shortIterator{ac.content, 0}
+
+ for iterator.hasNext() {
+ if !cb(iterator.next()) {
+ return false
+ }
+ }
+
+ return true
+}
+
+func (ac *arrayContainer) getShortIterator() shortPeekable {
return &shortIterator{ac.content, 0}
}
@@ -33,7 +53,7 @@ func (ac *arrayContainer) getReverseIterator() shortIterable {
}
func (ac *arrayContainer) getManyIterator() manyIterable {
- return &manyIterator{ac.content, 0}
+ return &shortIterator{ac.content, 0}
}
func (ac *arrayContainer) minimum() uint16 {
@@ -347,28 +367,17 @@ func (ac *arrayContainer) iorArray(value2 *arrayContainer) container {
len1 := value1.getCardinality()
len2 := value2.getCardinality()
maxPossibleCardinality := len1 + len2
- if maxPossibleCardinality > arrayDefaultMaxSize { // it could be a bitmap!
- bc := newBitmapContainer()
- for k := 0; k < len(value2.content); k++ {
- v := value2.content[k]
- i := uint(v) >> 6
- mask := uint64(1) << (v % 64)
- bc.bitmap[i] |= mask
- }
- for k := 0; k < len(ac.content); k++ {
- v := ac.content[k]
- i := uint(v) >> 6
- mask := uint64(1) << (v % 64)
- bc.bitmap[i] |= mask
- }
- bc.cardinality = int(popcntSlice(bc.bitmap))
- if bc.cardinality <= arrayDefaultMaxSize {
- return bc.toArrayContainer()
- }
- return bc
- }
if maxPossibleCardinality > cap(value1.content) {
- newcontent := make([]uint16, 0, maxPossibleCardinality)
+ // doubling the capacity reduces new slice allocations in the case of
+ // repeated calls to iorArray().
+ newSize := 2 * maxPossibleCardinality
+ // the second check is to handle overly large array containers
+ // and should not occur in normal usage,
+ // as all array containers should be at most arrayDefaultMaxSize
+ if newSize > 2*arrayDefaultMaxSize && maxPossibleCardinality <= 2*arrayDefaultMaxSize {
+ newSize = 2 * arrayDefaultMaxSize
+ }
+ newcontent := make([]uint16, 0, newSize)
copy(newcontent[len2:maxPossibleCardinality], ac.content[0:len1])
ac.content = newcontent
} else {
@@ -376,6 +385,13 @@ func (ac *arrayContainer) iorArray(value2 *arrayContainer) container {
}
nl := union2by2(value1.content[len2:maxPossibleCardinality], value2.content, ac.content)
ac.content = ac.content[:nl] // reslice to match actual used capacity
+
+ if nl > arrayDefaultMaxSize {
+ // Only converting to a bitmap when arrayDefaultMaxSize
+ // is actually exceeded minimizes conversions in the case of repeated
+ // calls to iorArray().
+ return ac.toBitmapContainer()
+ }
return ac
}
@@ -388,11 +404,19 @@ func (ac *arrayContainer) iorBitmap(bc2 *bitmapContainer) container {
}
func (ac *arrayContainer) iorRun16(rc *runContainer16) container {
- bc1 := ac.toBitmapContainer()
- bc2 := rc.toBitmapContainer()
- bc1.iorBitmap(bc2)
- *ac = *newArrayContainerFromBitmap(bc1)
- return ac
+ runCardinality := rc.getCardinality()
+ // heuristic for if the container should maybe be an
+ // array container.
+ if runCardinality < ac.getCardinality() &&
+ runCardinality+ac.getCardinality() < arrayDefaultMaxSize {
+ var result container
+ result = ac
+ for _, run := range rc.iv {
+ result = result.iaddRange(int(run.start), int(run.start)+int(run.length)+1)
+ }
+ return result
+ }
+ return rc.orArray(ac)
}
func (ac *arrayContainer) lazyIOR(a container) container {
@@ -477,7 +501,7 @@ func (ac *arrayContainer) orArrayCardinality(value2 *arrayContainer) int {
func (ac *arrayContainer) lazyorArray(value2 *arrayContainer) container {
value1 := ac
maxPossibleCardinality := value1.getCardinality() + value2.getCardinality()
- if maxPossibleCardinality > arrayLazyLowerBound { // it could be a bitmap!^M
+ if maxPossibleCardinality > arrayLazyLowerBound { // it could be a bitmap!
bc := newBitmapContainer()
for k := 0; k < len(value2.content); k++ {
v := value2.content[k]
@@ -640,10 +664,54 @@ func (ac *arrayContainer) iandNot(a container) container {
}
func (ac *arrayContainer) iandNotRun16(rc *runContainer16) container {
- rcb := rc.toBitmapContainer()
- acb := ac.toBitmapContainer()
- acb.iandNotBitmapSurely(rcb)
- *ac = *(acb.toArrayContainer())
+ // Fast path: if either the array container or the run container is empty, the result is the array.
+ if ac.isEmpty() || rc.isEmpty() {
+ // Empty
+ return ac
+ }
+ // Fast path: if the run container is full, the result is empty.
+ if rc.isFull() {
+ ac.content = ac.content[:0]
+ return ac
+ }
+ current_run := 0
+ // All values in [start_run, end_end] are part of the run
+ start_run := rc.iv[current_run].start
+ end_end := start_run + rc.iv[current_run].length
+ // We are going to read values in the array at index i, and we are
+ // going to write them at index pos. So we do in-place processing.
+ // We always have that pos <= i by construction. So we can either
+ // overwrite a value just read, or a value that was previous read.
+ pos := 0
+ i := 0
+ for ; i < len(ac.content); i++ {
+ if ac.content[i] < start_run {
+ // the value in the array appears before the run [start_run, end_end]
+ ac.content[pos] = ac.content[i]
+ pos++
+ } else if ac.content[i] <= end_end {
+ // nothing to do, the value is in the array but also in the run.
+ } else {
+ // We have the value in the array after the run. We cannot tell
+ // whether we need to keep it or not. So let us move to another run.
+ if current_run+1 < len(rc.iv) {
+ current_run++
+ start_run = rc.iv[current_run].start
+ end_end = start_run + rc.iv[current_run].length
+ i-- // retry with the same i
+ } else {
+ // We have exhausted the number of runs. We can keep the rest of the values
+ // from i to len(ac.content) - 1 inclusively.
+ break // We are done, the rest of the array will be kept
+ }
+ }
+ }
+ for ; i < len(ac.content); i++ {
+ ac.content[pos] = ac.content[i]
+ pos++
+ }
+ // We 'shink' the slice.
+ ac.content = ac.content[:pos]
return ac
}
@@ -837,6 +905,10 @@ func (ac *arrayContainer) getCardinality() int {
return len(ac.content)
}
+func (ac *arrayContainer) isEmpty() bool {
+ return len(ac.content) == 0
+}
+
func (ac *arrayContainer) rank(x uint16) int {
answer := binarySearch(ac.content, x)
if answer >= 0 {
@@ -864,6 +936,41 @@ func (ac *arrayContainer) loadData(bitmapContainer *bitmapContainer) {
ac.content = make([]uint16, bitmapContainer.cardinality, bitmapContainer.cardinality)
bitmapContainer.fillArray(ac.content)
}
+
+func (ac *arrayContainer) resetTo(a container) {
+ switch x := a.(type) {
+ case *arrayContainer:
+ ac.realloc(len(x.content))
+ copy(ac.content, x.content)
+
+ case *bitmapContainer:
+ ac.realloc(x.cardinality)
+ x.fillArray(ac.content)
+
+ case *runContainer16:
+ card := int(x.getCardinality())
+ ac.realloc(card)
+ cur := 0
+ for _, r := range x.iv {
+ for val := r.start; val <= r.last(); val++ {
+ ac.content[cur] = val
+ cur++
+ }
+ }
+
+ default:
+ panic("unsupported container type")
+ }
+}
+
+func (ac *arrayContainer) realloc(size int) {
+ if cap(ac.content) < size {
+ ac.content = make([]uint16, size)
+ } else {
+ ac.content = ac.content[:size]
+ }
+}
+
func newArrayContainer() *arrayContainer {
p := new(arrayContainer)
return p
@@ -915,10 +1022,10 @@ func (ac *arrayContainer) numberOfRuns() (nr int) {
runlen++
} else {
if cur < prev {
- panic("then fundamental arrayContainer assumption of sorted ac.content was broken")
+ panic("the fundamental arrayContainer assumption of sorted ac.content was broken")
}
if cur == prev {
- panic("then fundamental arrayContainer assumption of deduplicated content was broken")
+ panic("the fundamental arrayContainer assumption of deduplicated content was broken")
} else {
nr++
runlen = 0
@@ -953,16 +1060,42 @@ func (ac *arrayContainer) containerType() contype {
return arrayContype
}
-func (ac *arrayContainer) addOffset(x uint16) []container {
- low := &arrayContainer{}
- high := &arrayContainer{}
+func (ac *arrayContainer) addOffset(x uint16) (container, container) {
+ var low, high *arrayContainer
+
+ if len(ac.content) == 0 {
+ return nil, nil
+ }
+
+ if y := uint32(ac.content[0]) + uint32(x); highbits(y) == 0 {
+ // Some elements will fall into low part, allocate a container.
+ // Checking the first one is enough because they are ordered.
+ low = &arrayContainer{}
+ }
+ if y := uint32(ac.content[len(ac.content)-1]) + uint32(x); highbits(y) > 0 {
+ // Some elements will fall into high part, allocate a container.
+ // Checking the last one is enough because they are ordered.
+ high = &arrayContainer{}
+ }
+
for _, val := range ac.content {
y := uint32(val) + uint32(x)
if highbits(y) > 0 {
+ // OK, if high == nil then highbits(y) == 0 for all y.
high.content = append(high.content, lowbits(y))
} else {
+ // OK, if low == nil then highbits(y) > 0 for all y.
low.content = append(low.content, lowbits(y))
}
}
- return []container{low, high}
+
+ // Ensure proper nil interface.
+ if low == nil {
+ return nil, high
+ }
+ if high == nil {
+ return low, nil
+ }
+
+ return low, high
}
diff --git a/vendor/github.com/RoaringBitmap/roaring/arraycontainer_gen.go b/vendor/github.com/RoaringBitmap/roaring/arraycontainer_gen.go
deleted file mode 100644
index 6ee670e..0000000
--- a/vendor/github.com/RoaringBitmap/roaring/arraycontainer_gen.go
+++ /dev/null
@@ -1,134 +0,0 @@
-package roaring
-
-// NOTE: THIS FILE WAS PRODUCED BY THE
-// MSGP CODE GENERATION TOOL (github.com/tinylib/msgp)
-// DO NOT EDIT
-
-import "github.com/tinylib/msgp/msgp"
-
-// Deprecated: DecodeMsg implements msgp.Decodable
-func (z *arrayContainer) DecodeMsg(dc *msgp.Reader) (err error) {
- var field []byte
- _ = field
- var zbzg uint32
- zbzg, err = dc.ReadMapHeader()
- if err != nil {
- return
- }
- for zbzg > 0 {
- zbzg--
- field, err = dc.ReadMapKeyPtr()
- if err != nil {
- return
- }
- switch msgp.UnsafeString(field) {
- case "content":
- var zbai uint32
- zbai, err = dc.ReadArrayHeader()
- if err != nil {
- return
- }
- if cap(z.content) >= int(zbai) {
- z.content = (z.content)[:zbai]
- } else {
- z.content = make([]uint16, zbai)
- }
- for zxvk := range z.content {
- z.content[zxvk], err = dc.ReadUint16()
- if err != nil {
- return
- }
- }
- default:
- err = dc.Skip()
- if err != nil {
- return
- }
- }
- }
- return
-}
-
-// Deprecated: EncodeMsg implements msgp.Encodable
-func (z *arrayContainer) EncodeMsg(en *msgp.Writer) (err error) {
- // map header, size 1
- // write "content"
- err = en.Append(0x81, 0xa7, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74)
- if err != nil {
- return err
- }
- err = en.WriteArrayHeader(uint32(len(z.content)))
- if err != nil {
- return
- }
- for zxvk := range z.content {
- err = en.WriteUint16(z.content[zxvk])
- if err != nil {
- return
- }
- }
- return
-}
-
-// Deprecated: MarshalMsg implements msgp.Marshaler
-func (z *arrayContainer) MarshalMsg(b []byte) (o []byte, err error) {
- o = msgp.Require(b, z.Msgsize())
- // map header, size 1
- // string "content"
- o = append(o, 0x81, 0xa7, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74)
- o = msgp.AppendArrayHeader(o, uint32(len(z.content)))
- for zxvk := range z.content {
- o = msgp.AppendUint16(o, z.content[zxvk])
- }
- return
-}
-
-// Deprecated: UnmarshalMsg implements msgp.Unmarshaler
-func (z *arrayContainer) UnmarshalMsg(bts []byte) (o []byte, err error) {
- var field []byte
- _ = field
- var zcmr uint32
- zcmr, bts, err = msgp.ReadMapHeaderBytes(bts)
- if err != nil {
- return
- }
- for zcmr > 0 {
- zcmr--
- field, bts, err = msgp.ReadMapKeyZC(bts)
- if err != nil {
- return
- }
- switch msgp.UnsafeString(field) {
- case "content":
- var zajw uint32
- zajw, bts, err = msgp.ReadArrayHeaderBytes(bts)
- if err != nil {
- return
- }
- if cap(z.content) >= int(zajw) {
- z.content = (z.content)[:zajw]
- } else {
- z.content = make([]uint16, zajw)
- }
- for zxvk := range z.content {
- z.content[zxvk], bts, err = msgp.ReadUint16Bytes(bts)
- if err != nil {
- return
- }
- }
- default:
- bts, err = msgp.Skip(bts)
- if err != nil {
- return
- }
- }
- }
- o = bts
- return
-}
-
-// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (z *arrayContainer) Msgsize() (s int) {
- s = 1 + 8 + msgp.ArrayHeaderSize + (len(z.content) * (msgp.Uint16Size))
- return
-}
diff --git a/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer.go b/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer.go
index 038863d..bf08bfc 100644
--- a/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer.go
+++ b/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer.go
@@ -5,8 +5,6 @@ import (
"unsafe"
)
-//go:generate msgp -unexported
-
type bitmapContainer struct {
cardinality int
bitmap []uint64
@@ -96,6 +94,18 @@ func (bc *bitmapContainer) maximum() uint16 {
return uint16(0)
}
+func (bc *bitmapContainer) iterate(cb func(x uint16) bool) bool {
+ iterator := bitmapContainerShortIterator{bc, bc.NextSetBit(0)}
+
+ for iterator.hasNext() {
+ if !cb(iterator.next()) {
+ return false
+ }
+ }
+
+ return true
+}
+
type bitmapContainerShortIterator struct {
ptr *bitmapContainer
i int
@@ -103,18 +113,28 @@ type bitmapContainerShortIterator struct {
func (bcsi *bitmapContainerShortIterator) next() uint16 {
j := bcsi.i
- bcsi.i = bcsi.ptr.NextSetBit(bcsi.i + 1)
+ bcsi.i = bcsi.ptr.NextSetBit(uint(bcsi.i) + 1)
return uint16(j)
}
func (bcsi *bitmapContainerShortIterator) hasNext() bool {
return bcsi.i >= 0
}
+func (bcsi *bitmapContainerShortIterator) peekNext() uint16 {
+ return uint16(bcsi.i)
+}
+
+func (bcsi *bitmapContainerShortIterator) advanceIfNeeded(minval uint16) {
+ if bcsi.hasNext() && bcsi.peekNext() < minval {
+ bcsi.i = bcsi.ptr.NextSetBit(uint(minval))
+ }
+}
+
func newBitmapContainerShortIterator(a *bitmapContainer) *bitmapContainerShortIterator {
return &bitmapContainerShortIterator{a, a.NextSetBit(0)}
}
-func (bc *bitmapContainer) getShortIterator() shortIterable {
+func (bc *bitmapContainer) getShortIterator() shortPeekable {
return newBitmapContainerShortIterator(bc)
}
@@ -181,6 +201,33 @@ func (bcmi *bitmapContainerManyIterator) nextMany(hs uint32, buf []uint32) int {
return n
}
+func (bcmi *bitmapContainerManyIterator) nextMany64(hs uint64, buf []uint64) int {
+ n := 0
+ base := bcmi.base
+ bitset := bcmi.bitset
+
+ for n < len(buf) {
+ if bitset == 0 {
+ base++
+ if base >= len(bcmi.ptr.bitmap) {
+ bcmi.base = base
+ bcmi.bitset = bitset
+ return n
+ }
+ bitset = bcmi.ptr.bitmap[base]
+ continue
+ }
+ t := bitset & -bitset
+ buf[n] = uint64(((base * 64) + int(popcount(t-1)))) | hs
+ n = n + 1
+ bitset ^= t
+ }
+
+ bcmi.base = base
+ bcmi.bitset = bitset
+ return n
+}
+
func newBitmapContainerManyIterator(a *bitmapContainer) *bitmapContainerManyIterator {
return &bitmapContainerManyIterator{a, -1, 0}
}
@@ -217,7 +264,7 @@ func bitmapEquals(a, b []uint64) bool {
return true
}
-func (bc *bitmapContainer) fillLeastSignificant16bits(x []uint32, i int, mask uint32) {
+func (bc *bitmapContainer) fillLeastSignificant16bits(x []uint32, i int, mask uint32) int {
// TODO: should be written as optimized assembly
pos := i
base := mask
@@ -231,6 +278,7 @@ func (bc *bitmapContainer) fillLeastSignificant16bits(x []uint32, i int, mask ui
}
base += 64
}
+ return pos
}
func (bc *bitmapContainer) equals(o container) bool {
@@ -302,6 +350,10 @@ func (bc *bitmapContainer) getCardinality() int {
return bc.cardinality
}
+func (bc *bitmapContainer) isEmpty() bool {
+ return bc.cardinality == 0
+}
+
func (bc *bitmapContainer) clone() container {
ptr := bitmapContainer{bc.cardinality, make([]uint64, len(bc.bitmap))}
copy(ptr.bitmap, bc.bitmap[:])
@@ -836,13 +888,67 @@ func (bc *bitmapContainer) iandNot(a container) container {
}
func (bc *bitmapContainer) iandNotArray(ac *arrayContainer) container {
- acb := ac.toBitmapContainer()
- return bc.iandNotBitmapSurely(acb)
+ if ac.isEmpty() || bc.isEmpty() {
+ // Nothing to do.
+ return bc
+ }
+
+ // Word by word, we remove the elements in ac from bc. The approach is to build
+ // a mask of the elements to remove, and then apply it to the bitmap.
+ wordIdx := uint16(0)
+ mask := uint64(0)
+ for i, v := range ac.content {
+ if v/64 != wordIdx {
+ // Flush the current word.
+ if i != 0 {
+ // We're removing bits that are set in the mask and in the current word.
+ // To figure out the cardinality change, we count the number of bits that
+ // are set in the mask and in the current word.
+ mask &= bc.bitmap[wordIdx]
+ bc.bitmap[wordIdx] &= ^mask
+ bc.cardinality -= int(popcount(mask))
+ }
+
+ wordIdx = v / 64
+ mask = 0
+ }
+ mask |= 1 << (v % 64)
+ }
+
+ // Flush the last word.
+ mask &= bc.bitmap[wordIdx]
+ bc.bitmap[wordIdx] &= ^mask
+ bc.cardinality -= int(popcount(mask))
+
+ if bc.getCardinality() <= arrayDefaultMaxSize {
+ return bc.toArrayContainer()
+ }
+ return bc
}
func (bc *bitmapContainer) iandNotRun16(rc *runContainer16) container {
- rcb := rc.toBitmapContainer()
- return bc.iandNotBitmapSurely(rcb)
+ if rc.isEmpty() || bc.isEmpty() {
+ // Nothing to do.
+ return bc
+ }
+
+ wordRangeStart := rc.iv[0].start / 64
+ wordRangeEnd := (rc.iv[len(rc.iv)-1].last()) / 64 // inclusive
+
+ cardinalityChange := popcntSlice(bc.bitmap[wordRangeStart : wordRangeEnd+1]) // before cardinality - after cardinality (for word range)
+
+ for _, iv := range rc.iv {
+ resetBitmapRange(bc.bitmap, int(iv.start), int(iv.last())+1)
+ }
+
+ cardinalityChange -= popcntSlice(bc.bitmap[wordRangeStart : wordRangeEnd+1])
+
+ bc.cardinality -= int(cardinalityChange)
+
+ if bc.getCardinality() <= arrayDefaultMaxSize {
+ return bc.toArrayContainer()
+ }
+ return bc
}
func (bc *bitmapContainer) andNotArray(value2 *arrayContainer) container {
@@ -912,6 +1018,32 @@ func (bc *bitmapContainer) loadData(arrayContainer *arrayContainer) {
}
}
+func (bc *bitmapContainer) resetTo(a container) {
+ switch x := a.(type) {
+ case *arrayContainer:
+ fill(bc.bitmap, 0)
+ bc.loadData(x)
+
+ case *bitmapContainer:
+ bc.cardinality = x.cardinality
+ copy(bc.bitmap, x.bitmap)
+
+ case *runContainer16:
+ bc.cardinality = len(x.iv)
+ lastEnd := 0
+ for _, r := range x.iv {
+ bc.cardinality += int(r.length)
+ resetBitmapRange(bc.bitmap, lastEnd, int(r.start))
+ lastEnd = int(r.start+r.length) + 1
+ setBitmapRange(bc.bitmap, int(r.start), lastEnd)
+ }
+ resetBitmapRange(bc.bitmap, lastEnd, maxCapacity)
+
+ default:
+ panic("unsupported container type")
+ }
+}
+
func (bc *bitmapContainer) toArrayContainer() *arrayContainer {
ac := &arrayContainer{}
ac.loadData(bc)
@@ -934,20 +1066,23 @@ func (bc *bitmapContainer) fillArray(container []uint16) {
}
}
-func (bc *bitmapContainer) NextSetBit(i int) int {
- x := i / 64
- if x >= len(bc.bitmap) {
+func (bc *bitmapContainer) NextSetBit(i uint) int {
+ var (
+ x = i / 64
+ length = uint(len(bc.bitmap))
+ )
+ if x >= length {
return -1
}
w := bc.bitmap[x]
w = w >> uint(i%64)
if w != 0 {
- return i + countTrailingZeros(w)
+ return int(i) + countTrailingZeros(w)
}
x++
- for ; x < len(bc.bitmap); x++ {
+ for ; x < length; x++ {
if bc.bitmap[x] != 0 {
- return (x * 64) + countTrailingZeros(bc.bitmap[x])
+ return int(x*64) + countTrailingZeros(bc.bitmap[x])
}
}
return -1
@@ -981,7 +1116,6 @@ func (bc *bitmapContainer) PrevSetBit(i int) int {
// reference the java implementation
// https://github.com/RoaringBitmap/RoaringBitmap/blob/master/src/main/java/org/roaringbitmap/BitmapContainer.java#L875-L892
-//
func (bc *bitmapContainer) numberOfRuns() int {
if bc.cardinality == 0 {
return 0
@@ -1043,34 +1177,60 @@ func (bc *bitmapContainer) containerType() contype {
return bitmapContype
}
-func (bc *bitmapContainer) addOffset(x uint16) []container {
- low := newBitmapContainer()
- high := newBitmapContainer()
+func (bc *bitmapContainer) addOffset(x uint16) (container, container) {
+ var low, high *bitmapContainer
+
+ if bc.cardinality == 0 {
+ return nil, nil
+ }
+
b := uint32(x) >> 6
i := uint32(x) % 64
end := uint32(1024) - b
+
+ low = newBitmapContainer()
if i == 0 {
copy(low.bitmap[b:], bc.bitmap[:end])
- copy(high.bitmap[:b], bc.bitmap[end:])
} else {
low.bitmap[b] = bc.bitmap[0] << i
for k := uint32(1); k < end; k++ {
newval := bc.bitmap[k] << i
- if newval == 0 {
- newval = bc.bitmap[k-1] >> (64 - i)
- }
+ newval |= bc.bitmap[k-1] >> (64 - i)
low.bitmap[b+k] = newval
}
+ }
+ low.computeCardinality()
+
+ if low.cardinality == bc.cardinality {
+ // All elements from bc ended up in low, meaning high will be empty.
+ return low, nil
+ }
+
+ if low.cardinality == 0 {
+ // low is empty, let's reuse the container for high.
+ high = low
+ low = nil
+ } else {
+ // None of the containers will be empty, so allocate both.
+ high = newBitmapContainer()
+ }
+
+ if i == 0 {
+ copy(high.bitmap[:b], bc.bitmap[end:])
+ } else {
for k := end; k < 1024; k++ {
newval := bc.bitmap[k] << i
- if newval == 0 {
- newval = bc.bitmap[k-1] >> (64 - i)
- }
+ newval |= bc.bitmap[k-1] >> (64 - i)
high.bitmap[k-end] = newval
}
high.bitmap[b] = bc.bitmap[1023] >> (64 - i)
}
- low.computeCardinality()
high.computeCardinality()
- return []container{low, high}
+
+ // Ensure proper nil interface.
+ if low == nil {
+ return nil, high
+ }
+
+ return low, high
}
diff --git a/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer_gen.go b/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer_gen.go
deleted file mode 100644
index 9b5a465..0000000
--- a/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer_gen.go
+++ /dev/null
@@ -1,415 +0,0 @@
-package roaring
-
-// NOTE: THIS FILE WAS PRODUCED BY THE
-// MSGP CODE GENERATION TOOL (github.com/tinylib/msgp)
-// DO NOT EDIT
-
-import "github.com/tinylib/msgp/msgp"
-
-// Deprecated: DecodeMsg implements msgp.Decodable
-func (z *bitmapContainer) DecodeMsg(dc *msgp.Reader) (err error) {
- var field []byte
- _ = field
- var zbzg uint32
- zbzg, err = dc.ReadMapHeader()
- if err != nil {
- return
- }
- for zbzg > 0 {
- zbzg--
- field, err = dc.ReadMapKeyPtr()
- if err != nil {
- return
- }
- switch msgp.UnsafeString(field) {
- case "cardinality":
- z.cardinality, err = dc.ReadInt()
- if err != nil {
- return
- }
- case "bitmap":
- var zbai uint32
- zbai, err = dc.ReadArrayHeader()
- if err != nil {
- return
- }
- if cap(z.bitmap) >= int(zbai) {
- z.bitmap = (z.bitmap)[:zbai]
- } else {
- z.bitmap = make([]uint64, zbai)
- }
- for zxvk := range z.bitmap {
- z.bitmap[zxvk], err = dc.ReadUint64()
- if err != nil {
- return
- }
- }
- default:
- err = dc.Skip()
- if err != nil {
- return
- }
- }
- }
- return
-}
-
-// Deprecated: EncodeMsg implements msgp.Encodable
-func (z *bitmapContainer) EncodeMsg(en *msgp.Writer) (err error) {
- // map header, size 2
- // write "cardinality"
- err = en.Append(0x82, 0xab, 0x63, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x74, 0x79)
- if err != nil {
- return err
- }
- err = en.WriteInt(z.cardinality)
- if err != nil {
- return
- }
- // write "bitmap"
- err = en.Append(0xa6, 0x62, 0x69, 0x74, 0x6d, 0x61, 0x70)
- if err != nil {
- return err
- }
- err = en.WriteArrayHeader(uint32(len(z.bitmap)))
- if err != nil {
- return
- }
- for zxvk := range z.bitmap {
- err = en.WriteUint64(z.bitmap[zxvk])
- if err != nil {
- return
- }
- }
- return
-}
-
-// Deprecated: MarshalMsg implements msgp.Marshaler
-func (z *bitmapContainer) MarshalMsg(b []byte) (o []byte, err error) {
- o = msgp.Require(b, z.Msgsize())
- // map header, size 2
- // string "cardinality"
- o = append(o, 0x82, 0xab, 0x63, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x74, 0x79)
- o = msgp.AppendInt(o, z.cardinality)
- // string "bitmap"
- o = append(o, 0xa6, 0x62, 0x69, 0x74, 0x6d, 0x61, 0x70)
- o = msgp.AppendArrayHeader(o, uint32(len(z.bitmap)))
- for zxvk := range z.bitmap {
- o = msgp.AppendUint64(o, z.bitmap[zxvk])
- }
- return
-}
-
-// Deprecated: UnmarshalMsg implements msgp.Unmarshaler
-func (z *bitmapContainer) UnmarshalMsg(bts []byte) (o []byte, err error) {
- var field []byte
- _ = field
- var zcmr uint32
- zcmr, bts, err = msgp.ReadMapHeaderBytes(bts)
- if err != nil {
- return
- }
- for zcmr > 0 {
- zcmr--
- field, bts, err = msgp.ReadMapKeyZC(bts)
- if err != nil {
- return
- }
- switch msgp.UnsafeString(field) {
- case "cardinality":
- z.cardinality, bts, err = msgp.ReadIntBytes(bts)
- if err != nil {
- return
- }
- case "bitmap":
- var zajw uint32
- zajw, bts, err = msgp.ReadArrayHeaderBytes(bts)
- if err != nil {
- return
- }
- if cap(z.bitmap) >= int(zajw) {
- z.bitmap = (z.bitmap)[:zajw]
- } else {
- z.bitmap = make([]uint64, zajw)
- }
- for zxvk := range z.bitmap {
- z.bitmap[zxvk], bts, err = msgp.ReadUint64Bytes(bts)
- if err != nil {
- return
- }
- }
- default:
- bts, err = msgp.Skip(bts)
- if err != nil {
- return
- }
- }
- }
- o = bts
- return
-}
-
-// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (z *bitmapContainer) Msgsize() (s int) {
- s = 1 + 12 + msgp.IntSize + 7 + msgp.ArrayHeaderSize + (len(z.bitmap) * (msgp.Uint64Size))
- return
-}
-
-// Deprecated: DecodeMsg implements msgp.Decodable
-func (z *bitmapContainerShortIterator) DecodeMsg(dc *msgp.Reader) (err error) {
- var field []byte
- _ = field
- var zhct uint32
- zhct, err = dc.ReadMapHeader()
- if err != nil {
- return
- }
- for zhct > 0 {
- zhct--
- field, err = dc.ReadMapKeyPtr()
- if err != nil {
- return
- }
- switch msgp.UnsafeString(field) {
- case "ptr":
- if dc.IsNil() {
- err = dc.ReadNil()
- if err != nil {
- return
- }
- z.ptr = nil
- } else {
- if z.ptr == nil {
- z.ptr = new(bitmapContainer)
- }
- var zcua uint32
- zcua, err = dc.ReadMapHeader()
- if err != nil {
- return
- }
- for zcua > 0 {
- zcua--
- field, err = dc.ReadMapKeyPtr()
- if err != nil {
- return
- }
- switch msgp.UnsafeString(field) {
- case "cardinality":
- z.ptr.cardinality, err = dc.ReadInt()
- if err != nil {
- return
- }
- case "bitmap":
- var zxhx uint32
- zxhx, err = dc.ReadArrayHeader()
- if err != nil {
- return
- }
- if cap(z.ptr.bitmap) >= int(zxhx) {
- z.ptr.bitmap = (z.ptr.bitmap)[:zxhx]
- } else {
- z.ptr.bitmap = make([]uint64, zxhx)
- }
- for zwht := range z.ptr.bitmap {
- z.ptr.bitmap[zwht], err = dc.ReadUint64()
- if err != nil {
- return
- }
- }
- default:
- err = dc.Skip()
- if err != nil {
- return
- }
- }
- }
- }
- case "i":
- z.i, err = dc.ReadInt()
- if err != nil {
- return
- }
- default:
- err = dc.Skip()
- if err != nil {
- return
- }
- }
- }
- return
-}
-
-// Deprecated: EncodeMsg implements msgp.Encodable
-func (z *bitmapContainerShortIterator) EncodeMsg(en *msgp.Writer) (err error) {
- // map header, size 2
- // write "ptr"
- err = en.Append(0x82, 0xa3, 0x70, 0x74, 0x72)
- if err != nil {
- return err
- }
- if z.ptr == nil {
- err = en.WriteNil()
- if err != nil {
- return
- }
- } else {
- // map header, size 2
- // write "cardinality"
- err = en.Append(0x82, 0xab, 0x63, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x74, 0x79)
- if err != nil {
- return err
- }
- err = en.WriteInt(z.ptr.cardinality)
- if err != nil {
- return
- }
- // write "bitmap"
- err = en.Append(0xa6, 0x62, 0x69, 0x74, 0x6d, 0x61, 0x70)
- if err != nil {
- return err
- }
- err = en.WriteArrayHeader(uint32(len(z.ptr.bitmap)))
- if err != nil {
- return
- }
- for zwht := range z.ptr.bitmap {
- err = en.WriteUint64(z.ptr.bitmap[zwht])
- if err != nil {
- return
- }
- }
- }
- // write "i"
- err = en.Append(0xa1, 0x69)
- if err != nil {
- return err
- }
- err = en.WriteInt(z.i)
- if err != nil {
- return
- }
- return
-}
-
-// Deprecated: MarshalMsg implements msgp.Marshaler
-func (z *bitmapContainerShortIterator) MarshalMsg(b []byte) (o []byte, err error) {
- o = msgp.Require(b, z.Msgsize())
- // map header, size 2
- // string "ptr"
- o = append(o, 0x82, 0xa3, 0x70, 0x74, 0x72)
- if z.ptr == nil {
- o = msgp.AppendNil(o)
- } else {
- // map header, size 2
- // string "cardinality"
- o = append(o, 0x82, 0xab, 0x63, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x74, 0x79)
- o = msgp.AppendInt(o, z.ptr.cardinality)
- // string "bitmap"
- o = append(o, 0xa6, 0x62, 0x69, 0x74, 0x6d, 0x61, 0x70)
- o = msgp.AppendArrayHeader(o, uint32(len(z.ptr.bitmap)))
- for zwht := range z.ptr.bitmap {
- o = msgp.AppendUint64(o, z.ptr.bitmap[zwht])
- }
- }
- // string "i"
- o = append(o, 0xa1, 0x69)
- o = msgp.AppendInt(o, z.i)
- return
-}
-
-// Deprecated: UnmarshalMsg implements msgp.Unmarshaler
-func (z *bitmapContainerShortIterator) UnmarshalMsg(bts []byte) (o []byte, err error) {
- var field []byte
- _ = field
- var zlqf uint32
- zlqf, bts, err = msgp.ReadMapHeaderBytes(bts)
- if err != nil {
- return
- }
- for zlqf > 0 {
- zlqf--
- field, bts, err = msgp.ReadMapKeyZC(bts)
- if err != nil {
- return
- }
- switch msgp.UnsafeString(field) {
- case "ptr":
- if msgp.IsNil(bts) {
- bts, err = msgp.ReadNilBytes(bts)
- if err != nil {
- return
- }
- z.ptr = nil
- } else {
- if z.ptr == nil {
- z.ptr = new(bitmapContainer)
- }
- var zdaf uint32
- zdaf, bts, err = msgp.ReadMapHeaderBytes(bts)
- if err != nil {
- return
- }
- for zdaf > 0 {
- zdaf--
- field, bts, err = msgp.ReadMapKeyZC(bts)
- if err != nil {
- return
- }
- switch msgp.UnsafeString(field) {
- case "cardinality":
- z.ptr.cardinality, bts, err = msgp.ReadIntBytes(bts)
- if err != nil {
- return
- }
- case "bitmap":
- var zpks uint32
- zpks, bts, err = msgp.ReadArrayHeaderBytes(bts)
- if err != nil {
- return
- }
- if cap(z.ptr.bitmap) >= int(zpks) {
- z.ptr.bitmap = (z.ptr.bitmap)[:zpks]
- } else {
- z.ptr.bitmap = make([]uint64, zpks)
- }
- for zwht := range z.ptr.bitmap {
- z.ptr.bitmap[zwht], bts, err = msgp.ReadUint64Bytes(bts)
- if err != nil {
- return
- }
- }
- default:
- bts, err = msgp.Skip(bts)
- if err != nil {
- return
- }
- }
- }
- }
- case "i":
- z.i, bts, err = msgp.ReadIntBytes(bts)
- if err != nil {
- return
- }
- default:
- bts, err = msgp.Skip(bts)
- if err != nil {
- return
- }
- }
- }
- o = bts
- return
-}
-
-// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (z *bitmapContainerShortIterator) Msgsize() (s int) {
- s = 1 + 4
- if z.ptr == nil {
- s += msgp.NilSize
- } else {
- s += 1 + 12 + msgp.IntSize + 7 + msgp.ArrayHeaderSize + (len(z.ptr.bitmap) * (msgp.Uint64Size))
- }
- s += 2 + msgp.IntSize
- return
-}
diff --git a/vendor/github.com/RoaringBitmap/roaring/clz.go b/vendor/github.com/RoaringBitmap/roaring/clz.go
index bcd80d3..ee0ebc6 100644
--- a/vendor/github.com/RoaringBitmap/roaring/clz.go
+++ b/vendor/github.com/RoaringBitmap/roaring/clz.go
@@ -1,4 +1,6 @@
+//go:build go1.9
// +build go1.9
+
// "go1.9", from Go version 1.9 onward
// See https://golang.org/pkg/go/build/#hdr-Build_Constraints
diff --git a/vendor/github.com/RoaringBitmap/roaring/clz_compat.go b/vendor/github.com/RoaringBitmap/roaring/clz_compat.go
index eeef4de..7ee16b4 100644
--- a/vendor/github.com/RoaringBitmap/roaring/clz_compat.go
+++ b/vendor/github.com/RoaringBitmap/roaring/clz_compat.go
@@ -1,3 +1,4 @@
+//go:build !go1.9
// +build !go1.9
package roaring
diff --git a/vendor/github.com/RoaringBitmap/roaring/ctz.go b/vendor/github.com/RoaringBitmap/roaring/ctz.go
index e399ddd..fbcfe91 100644
--- a/vendor/github.com/RoaringBitmap/roaring/ctz.go
+++ b/vendor/github.com/RoaringBitmap/roaring/ctz.go
@@ -1,4 +1,6 @@
+//go:build go1.9
// +build go1.9
+
// "go1.9", from Go version 1.9 onward
// See https://golang.org/pkg/go/build/#hdr-Build_Constraints
diff --git a/vendor/github.com/RoaringBitmap/roaring/ctz_compat.go b/vendor/github.com/RoaringBitmap/roaring/ctz_compat.go
index 80220e6..d01df82 100644
--- a/vendor/github.com/RoaringBitmap/roaring/ctz_compat.go
+++ b/vendor/github.com/RoaringBitmap/roaring/ctz_compat.go
@@ -1,3 +1,4 @@
+//go:build !go1.9
// +build !go1.9
package roaring
diff --git a/vendor/github.com/RoaringBitmap/roaring/fastaggregation.go b/vendor/github.com/RoaringBitmap/roaring/fastaggregation.go
index 762e500..7d0a92f 100644
--- a/vendor/github.com/RoaringBitmap/roaring/fastaggregation.go
+++ b/vendor/github.com/RoaringBitmap/roaring/fastaggregation.go
@@ -33,15 +33,6 @@ main:
s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
} else {
c1 := x1.highlowcontainer.getContainerAtIndex(pos1)
- switch t := c1.(type) {
- case *arrayContainer:
- c1 = t.toBitmapContainer()
- case *runContainer16:
- if !t.isFull() {
- c1 = t.toBitmapContainer()
- }
- }
-
answer.highlowcontainer.appendContainer(s1, c1.lazyOR(x2.highlowcontainer.getContainerAtIndex(pos2)), false)
pos1++
pos2++
@@ -89,18 +80,7 @@ main:
}
s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
} else {
- c1 := x1.highlowcontainer.getContainerAtIndex(pos1)
- switch t := c1.(type) {
- case *arrayContainer:
- c1 = t.toBitmapContainer()
- case *runContainer16:
- if !t.isFull() {
- c1 = t.toBitmapContainer()
- }
- case *bitmapContainer:
- c1 = x1.highlowcontainer.getWritableContainerAtIndex(pos1)
- }
-
+ c1 := x1.highlowcontainer.getWritableContainerAtIndex(pos1)
x1.highlowcontainer.containers[pos1] = c1.lazyIOR(x2.highlowcontainer.getContainerAtIndex(pos2))
x1.highlowcontainer.needCopyOnWrite[pos1] = false
pos1++
@@ -141,6 +121,10 @@ func (x1 *Bitmap) repairAfterLazy() {
// FastAnd computes the intersection between many bitmaps quickly
// Compared to the And function, it can take many bitmaps as input, thus saving the trouble
// of manually calling "And" many times.
+//
+// Performance hints: if you have very large and tiny bitmaps,
+// it may be beneficial performance-wise to put a tiny bitmap
+// in first position.
func FastAnd(bitmaps ...*Bitmap) *Bitmap {
if len(bitmaps) == 0 {
return NewBitmap()
@@ -213,3 +197,117 @@ func HeapXor(bitmaps ...*Bitmap) *Bitmap {
}
return heap.Pop(&pq).(*item).value
}
+
+// AndAny provides a result equivalent to x1.And(FastOr(bitmaps)).
+// It's optimized to minimize allocations. It also might be faster than separate calls.
+func (x1 *Bitmap) AndAny(bitmaps ...*Bitmap) {
+ if len(bitmaps) == 0 {
+ return
+ } else if len(bitmaps) == 1 {
+ x1.And(bitmaps[0])
+ return
+ }
+
+ type withPos struct {
+ bitmap *roaringArray
+ pos int
+ key uint16
+ }
+ filters := make([]withPos, 0, len(bitmaps))
+
+ for _, b := range bitmaps {
+ if b.highlowcontainer.size() > 0 {
+ filters = append(filters, withPos{
+ bitmap: &b.highlowcontainer,
+ pos: 0,
+ key: b.highlowcontainer.getKeyAtIndex(0),
+ })
+ }
+ }
+
+ basePos := 0
+ intersections := 0
+ keyContainers := make([]container, 0, len(filters))
+ var (
+ tmpArray *arrayContainer
+ tmpBitmap *bitmapContainer
+ minNextKey uint16
+ )
+
+ for basePos < x1.highlowcontainer.size() && len(filters) > 0 {
+ baseKey := x1.highlowcontainer.getKeyAtIndex(basePos)
+
+ // accumulate containers for current key, find next minimal key in filters
+ // and exclude filters that do not have related values anymore
+ i := 0
+ maxPossibleOr := 0
+ minNextKey = MaxUint16
+ for _, f := range filters {
+ if f.key < baseKey {
+ f.pos = f.bitmap.advanceUntil(baseKey, f.pos)
+ if f.pos == f.bitmap.size() {
+ continue
+ }
+ f.key = f.bitmap.getKeyAtIndex(f.pos)
+ }
+
+ if f.key == baseKey {
+ cont := f.bitmap.getContainerAtIndex(f.pos)
+ keyContainers = append(keyContainers, cont)
+ maxPossibleOr += cont.getCardinality()
+
+ f.pos++
+ if f.pos == f.bitmap.size() {
+ continue
+ }
+ f.key = f.bitmap.getKeyAtIndex(f.pos)
+ }
+
+ minNextKey = minOfUint16(minNextKey, f.key)
+ filters[i] = f
+ i++
+ }
+ filters = filters[:i]
+
+ if len(keyContainers) == 0 {
+ basePos = x1.highlowcontainer.advanceUntil(minNextKey, basePos)
+ continue
+ }
+
+ var ored container
+
+ if len(keyContainers) == 1 {
+ ored = keyContainers[0]
+ } else {
+ //TODO: special case for run containers?
+ if maxPossibleOr > arrayDefaultMaxSize {
+ if tmpBitmap == nil {
+ tmpBitmap = newBitmapContainer()
+ }
+ tmpBitmap.resetTo(keyContainers[0])
+ ored = tmpBitmap
+ } else {
+ if tmpArray == nil {
+ tmpArray = newArrayContainerCapacity(maxPossibleOr)
+ }
+ tmpArray.realloc(maxPossibleOr)
+ tmpArray.resetTo(keyContainers[0])
+ ored = tmpArray
+ }
+ for _, c := range keyContainers[1:] {
+ ored = ored.ior(c)
+ }
+ }
+
+ result := x1.highlowcontainer.getWritableContainerAtIndex(basePos).iand(ored)
+ if !result.isEmpty() {
+ x1.highlowcontainer.replaceKeyAndContainerAtIndex(intersections, baseKey, result, false)
+ intersections++
+ }
+
+ keyContainers = keyContainers[:0]
+ basePos = x1.highlowcontainer.advanceUntil(minNextKey, basePos)
+ }
+
+ x1.highlowcontainer.resize(intersections)
+}
diff --git a/vendor/github.com/RoaringBitmap/roaring/go.mod b/vendor/github.com/RoaringBitmap/roaring/go.mod
deleted file mode 100644
index 95406c9..0000000
--- a/vendor/github.com/RoaringBitmap/roaring/go.mod
+++ /dev/null
@@ -1,15 +0,0 @@
-module github.com/RoaringBitmap/roaring
-
-go 1.12
-
-require (
- github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2
- github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493 // indirect
- github.com/golang/snappy v0.0.1 // indirect
- github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae
- github.com/philhofer/fwd v1.0.0 // indirect
- github.com/smartystreets/goconvey v0.0.0-20190306220146-200a235640ff
- github.com/stretchr/testify v1.3.0
- github.com/tinylib/msgp v1.1.0
- github.com/willf/bitset v1.1.10
-)
diff --git a/vendor/github.com/RoaringBitmap/roaring/go.sum b/vendor/github.com/RoaringBitmap/roaring/go.sum
deleted file mode 100644
index 3dcbb91..0000000
--- a/vendor/github.com/RoaringBitmap/roaring/go.sum
+++ /dev/null
@@ -1,29 +0,0 @@
-github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2 h1:Ujru1hufTHVb++eG6OuNDKMxZnGIvF6o/u8q/8h2+I4=
-github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE=
-github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493 h1:OTanQnFt0bi5iLFSdbEVA/idR6Q2WhCm+deb7ir2CcM=
-github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24=
-github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
-github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
-github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
-github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
-github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
-github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
-github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae h1:VeRdUYdCw49yizlSbMEn2SZ+gT+3IUKx8BqxyQdz+BY=
-github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg=
-github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ=
-github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
-github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
-github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
-github.com/smartystreets/goconvey v0.0.0-20190306220146-200a235640ff h1:86HlEv0yBCry9syNuylzqznKXDK11p6D0DT596yNMys=
-github.com/smartystreets/goconvey v0.0.0-20190306220146-200a235640ff/go.mod h1:KSQcGKpxUMHk3nbYzs/tIBAM2iDooCn0BmttHOJEbLs=
-github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
-github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/tinylib/msgp v1.1.0 h1:9fQd+ICuRIu/ue4vxJZu6/LzxN0HwMds2nq/0cFvxHU=
-github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
-github.com/willf/bitset v1.1.10 h1:NotGKqX0KwQ72NUzqrjZq5ipPNDQex9lo3WpaS8L2sc=
-github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
diff --git a/vendor/github.com/RoaringBitmap/roaring/internal/byte_input.go b/vendor/github.com/RoaringBitmap/roaring/internal/byte_input.go
new file mode 100644
index 0000000..d5ebb91
--- /dev/null
+++ b/vendor/github.com/RoaringBitmap/roaring/internal/byte_input.go
@@ -0,0 +1,215 @@
+package internal
+
+import (
+ "encoding/binary"
+ "io"
+)
+
+// ByteInput typed interface around io.Reader or raw bytes
+type ByteInput interface {
+ // Next returns a slice containing the next n bytes from the buffer,
+ // advancing the buffer as if the bytes had been returned by Read.
+ Next(n int) ([]byte, error)
+ // NextReturnsSafeSlice returns true if Next() returns a safe slice as opposed
+ // to a slice that points to an underlying buffer possibly owned by another system.
+ // When NextReturnsSafeSlice returns false, the result from Next() should be copied
+ // before it is modified (i.e., it is immutable).
+ NextReturnsSafeSlice() bool
+ // ReadUInt32 reads uint32 with LittleEndian order
+ ReadUInt32() (uint32, error)
+ // ReadUInt16 reads uint16 with LittleEndian order
+ ReadUInt16() (uint16, error)
+ // GetReadBytes returns read bytes
+ GetReadBytes() int64
+ // SkipBytes skips exactly n bytes
+ SkipBytes(n int) error
+}
+
+// NewByteInputFromReader creates reader wrapper
+func NewByteInputFromReader(reader io.Reader) ByteInput {
+ return &ByteInputAdapter{
+ r: reader,
+ readBytes: 0,
+ }
+}
+
+// NewByteInput creates raw bytes wrapper
+func NewByteInput(buf []byte) ByteInput {
+ return &ByteBuffer{
+ buf: buf,
+ off: 0,
+ }
+}
+
+// ByteBuffer raw bytes wrapper
+type ByteBuffer struct {
+ buf []byte
+ off int
+}
+
+// NewByteBuffer creates a new ByteBuffer.
+func NewByteBuffer(buf []byte) *ByteBuffer {
+ return &ByteBuffer{
+ buf: buf,
+ }
+}
+
+var _ io.Reader = (*ByteBuffer)(nil)
+
+// Read implements io.Reader.
+func (b *ByteBuffer) Read(p []byte) (int, error) {
+ data, err := b.Next(len(p))
+ if err != nil {
+ return 0, err
+ }
+ copy(p, data)
+ return len(data), nil
+}
+
+// Next returns a slice containing the next n bytes from the reader
+// If there are fewer bytes than the given n, io.ErrUnexpectedEOF will be returned
+func (b *ByteBuffer) Next(n int) ([]byte, error) {
+ m := len(b.buf) - b.off
+
+ if n > m {
+ return nil, io.ErrUnexpectedEOF
+ }
+
+ data := b.buf[b.off : b.off+n]
+ b.off += n
+
+ return data, nil
+}
+
+// NextReturnsSafeSlice returns false since ByteBuffer might hold
+// an array owned by some other systems.
+func (b *ByteBuffer) NextReturnsSafeSlice() bool {
+ return false
+}
+
+// ReadUInt32 reads uint32 with LittleEndian order
+func (b *ByteBuffer) ReadUInt32() (uint32, error) {
+ if len(b.buf)-b.off < 4 {
+ return 0, io.ErrUnexpectedEOF
+ }
+
+ v := binary.LittleEndian.Uint32(b.buf[b.off:])
+ b.off += 4
+
+ return v, nil
+}
+
+// ReadUInt16 reads uint16 with LittleEndian order
+func (b *ByteBuffer) ReadUInt16() (uint16, error) {
+ if len(b.buf)-b.off < 2 {
+ return 0, io.ErrUnexpectedEOF
+ }
+
+ v := binary.LittleEndian.Uint16(b.buf[b.off:])
+ b.off += 2
+
+ return v, nil
+}
+
+// GetReadBytes returns read bytes
+func (b *ByteBuffer) GetReadBytes() int64 {
+ return int64(b.off)
+}
+
+// SkipBytes skips exactly n bytes
+func (b *ByteBuffer) SkipBytes(n int) error {
+ m := len(b.buf) - b.off
+
+ if n > m {
+ return io.ErrUnexpectedEOF
+ }
+
+ b.off += n
+
+ return nil
+}
+
+// Reset resets the given buffer with a new byte slice
+func (b *ByteBuffer) Reset(buf []byte) {
+ b.buf = buf
+ b.off = 0
+}
+
+// ByteInputAdapter reader wrapper
+type ByteInputAdapter struct {
+ r io.Reader
+ readBytes int
+ buf [4]byte
+}
+
+var _ io.Reader = (*ByteInputAdapter)(nil)
+
+// Read implements io.Reader.
+func (b *ByteInputAdapter) Read(buf []byte) (int, error) {
+ m, err := io.ReadAtLeast(b.r, buf, len(buf))
+ b.readBytes += m
+
+ if err != nil {
+ return 0, err
+ }
+
+ return m, nil
+}
+
+// Next returns a slice containing the next n bytes from the buffer,
+// advancing the buffer as if the bytes had been returned by Read.
+func (b *ByteInputAdapter) Next(n int) ([]byte, error) {
+ buf := make([]byte, n)
+ _, err := b.Read(buf)
+
+ if err != nil {
+ return nil, err
+ }
+ return buf, nil
+}
+
+// NextReturnsSafeSlice returns true since ByteInputAdapter always returns a slice
+// allocated with make([]byte, ...)
+func (b *ByteInputAdapter) NextReturnsSafeSlice() bool {
+ return true
+}
+
+// ReadUInt32 reads uint32 with LittleEndian order
+func (b *ByteInputAdapter) ReadUInt32() (uint32, error) {
+ buf := b.buf[:4]
+ _, err := b.Read(buf)
+ if err != nil {
+ return 0, err
+ }
+
+ return binary.LittleEndian.Uint32(buf), nil
+}
+
+// ReadUInt16 reads uint16 with LittleEndian order
+func (b *ByteInputAdapter) ReadUInt16() (uint16, error) {
+ buf := b.buf[:2]
+ _, err := b.Read(buf)
+ if err != nil {
+ return 0, err
+ }
+
+ return binary.LittleEndian.Uint16(buf), nil
+}
+
+// GetReadBytes returns read bytes
+func (b *ByteInputAdapter) GetReadBytes() int64 {
+ return int64(b.readBytes)
+}
+
+// SkipBytes skips exactly n bytes
+func (b *ByteInputAdapter) SkipBytes(n int) error {
+ _, err := b.Next(n)
+
+ return err
+}
+
+// Reset resets the given buffer with a new stream
+func (b *ByteInputAdapter) Reset(stream io.Reader) {
+ b.r = stream
+ b.readBytes = 0
+}
diff --git a/vendor/github.com/RoaringBitmap/roaring/internal/pools.go b/vendor/github.com/RoaringBitmap/roaring/internal/pools.go
new file mode 100644
index 0000000..d258356
--- /dev/null
+++ b/vendor/github.com/RoaringBitmap/roaring/internal/pools.go
@@ -0,0 +1,21 @@
+package internal
+
+import (
+ "sync"
+)
+
+var (
+ // ByteInputAdapterPool shared pool
+ ByteInputAdapterPool = sync.Pool{
+ New: func() interface{} {
+ return &ByteInputAdapter{}
+ },
+ }
+
+ // ByteBufferPool shared pool
+ ByteBufferPool = sync.Pool{
+ New: func() interface{} {
+ return &ByteBuffer{}
+ },
+ }
+)
diff --git a/vendor/github.com/RoaringBitmap/roaring/manyiterator.go b/vendor/github.com/RoaringBitmap/roaring/manyiterator.go
index b4f630a..eaa5b79 100644
--- a/vendor/github.com/RoaringBitmap/roaring/manyiterator.go
+++ b/vendor/github.com/RoaringBitmap/roaring/manyiterator.go
@@ -2,14 +2,10 @@ package roaring
type manyIterable interface {
nextMany(hs uint32, buf []uint32) int
+ nextMany64(hs uint64, buf []uint64) int
}
-type manyIterator struct {
- slice []uint16
- loc int
-}
-
-func (si *manyIterator) nextMany(hs uint32, buf []uint32) int {
+func (si *shortIterator) nextMany(hs uint32, buf []uint32) int {
n := 0
l := si.loc
s := si.slice
@@ -21,3 +17,16 @@ func (si *manyIterator) nextMany(hs uint32, buf []uint32) int {
si.loc = l
return n
}
+
+func (si *shortIterator) nextMany64(hs uint64, buf []uint64) int {
+ n := 0
+ l := si.loc
+ s := si.slice
+ for n < len(buf) && l < len(s) {
+ buf[n] = uint64(s[l]) | hs
+ l++
+ n++
+ }
+ si.loc = l
+ return n
+}
diff --git a/vendor/github.com/RoaringBitmap/roaring/parallel.go b/vendor/github.com/RoaringBitmap/roaring/parallel.go
index 2af1aed..9208e3e 100644
--- a/vendor/github.com/RoaringBitmap/roaring/parallel.go
+++ b/vendor/github.com/RoaringBitmap/roaring/parallel.go
@@ -166,7 +166,6 @@ func appenderRoutine(bitmapChan chan<- *Bitmap, resultChan <-chan keyedContainer
make([]container, 0, expectedKeys),
make([]bool, 0, expectedKeys),
false,
- nil,
},
}
for i := range keys {
@@ -286,14 +285,14 @@ func ParAnd(parallelism int, bitmaps ...*Bitmap) *Bitmap {
for input := range inputChan {
c := input.containers[0].and(input.containers[1])
for _, next := range input.containers[2:] {
- if c.getCardinality() == 0 {
+ if c.isEmpty() {
break
}
c = c.iand(next)
}
// Send a nil explicitly if the result of the intersection is an empty container
- if c.getCardinality() == 0 {
+ if c.isEmpty() {
c = nil
}
@@ -355,10 +354,10 @@ func ParOr(parallelism int, bitmaps ...*Bitmap) *Bitmap {
if lKey == MaxUint16 && hKey == 0 {
return New()
} else if len(bitmaps) == 1 {
- return bitmaps[0]
+ return bitmaps[0].Clone()
}
- keyRange := hKey - lKey + 1
+ keyRange := int(hKey) - int(lKey) + 1
if keyRange == 1 {
// revert to FastOr. Since the key range is 0
// no container-level aggregation parallelism is achievable
diff --git a/vendor/github.com/RoaringBitmap/roaring/popcnt.go b/vendor/github.com/RoaringBitmap/roaring/popcnt.go
index 9d99508..b4980aa 100644
--- a/vendor/github.com/RoaringBitmap/roaring/popcnt.go
+++ b/vendor/github.com/RoaringBitmap/roaring/popcnt.go
@@ -1,4 +1,6 @@
+//go:build go1.9
// +build go1.9
+
// "go1.9", from Go version 1.9 onward
// See https://golang.org/pkg/go/build/#hdr-Build_Constraints
diff --git a/vendor/github.com/RoaringBitmap/roaring/popcnt_asm.go b/vendor/github.com/RoaringBitmap/roaring/popcnt_asm.go
index 882d7f4..ba2dac9 100644
--- a/vendor/github.com/RoaringBitmap/roaring/popcnt_asm.go
+++ b/vendor/github.com/RoaringBitmap/roaring/popcnt_asm.go
@@ -1,3 +1,4 @@
+//go:build amd64 && !appengine && !go1.9
// +build amd64,!appengine,!go1.9
package roaring
diff --git a/vendor/github.com/RoaringBitmap/roaring/popcnt_compat.go b/vendor/github.com/RoaringBitmap/roaring/popcnt_compat.go
index 7ae82d4..5933e52 100644
--- a/vendor/github.com/RoaringBitmap/roaring/popcnt_compat.go
+++ b/vendor/github.com/RoaringBitmap/roaring/popcnt_compat.go
@@ -1,3 +1,4 @@
+//go:build !go1.9
// +build !go1.9
package roaring
diff --git a/vendor/github.com/RoaringBitmap/roaring/popcnt_generic.go b/vendor/github.com/RoaringBitmap/roaring/popcnt_generic.go
index edf2083..4ae6d5a 100644
--- a/vendor/github.com/RoaringBitmap/roaring/popcnt_generic.go
+++ b/vendor/github.com/RoaringBitmap/roaring/popcnt_generic.go
@@ -1,3 +1,4 @@
+//go:build !amd64 || appengine || go1.9
// +build !amd64 appengine go1.9
package roaring
diff --git a/vendor/github.com/RoaringBitmap/roaring/roaring.go b/vendor/github.com/RoaringBitmap/roaring/roaring.go
index 02daef9..a31cdbd 100644
--- a/vendor/github.com/RoaringBitmap/roaring/roaring.go
+++ b/vendor/github.com/RoaringBitmap/roaring/roaring.go
@@ -6,12 +6,14 @@
package roaring
import (
- "bufio"
"bytes"
"encoding/base64"
"fmt"
"io"
"strconv"
+
+ "github.com/RoaringBitmap/roaring/internal"
+ "github.com/bits-and-blooms/bitset"
)
// Bitmap represents a compressed bitmap where you can add integers.
@@ -52,23 +54,263 @@ func (rb *Bitmap) ToBytes() ([]byte, error) {
return rb.highlowcontainer.toBytes()
}
-// Deprecated: WriteToMsgpack writes a msgpack2/snappy-streaming compressed serialized
-// version of this bitmap to stream. The format is not
-// compatible with the WriteTo() format, and is
-// experimental: it may produce smaller on disk
-// footprint and/or be faster to read, depending
-// on your content. Currently only the Go roaring
-// implementation supports this format.
-func (rb *Bitmap) WriteToMsgpack(stream io.Writer) (int64, error) {
- return 0, rb.highlowcontainer.writeToMsgpack(stream)
+const wordSize = uint64(64)
+const log2WordSize = uint64(6)
+const capacity = ^uint64(0)
+const bitmapContainerSize = (1 << 16) / 64 // bitmap size in words
+
+// DenseSize returns the size of the bitmap when stored as a dense bitmap.
+func (rb *Bitmap) DenseSize() uint64 {
+ if rb.highlowcontainer.size() == 0 {
+ return 0
+ }
+
+ maximum := 1 + uint64(rb.Maximum())
+ if maximum > (capacity - wordSize + 1) {
+ return uint64(capacity >> log2WordSize)
+ }
+
+ return uint64((maximum + (wordSize - 1)) >> log2WordSize)
+}
+
+// ToDense returns a slice of uint64s representing the bitmap as a dense bitmap.
+// Useful to convert a roaring bitmap to a format that can be used by other libraries
+// like https://github.com/bits-and-blooms/bitset or https://github.com/kelindar/bitmap
+func (rb *Bitmap) ToDense() []uint64 {
+ sz := rb.DenseSize()
+ if sz == 0 {
+ return nil
+ }
+
+ bitmap := make([]uint64, sz)
+ rb.WriteDenseTo(bitmap)
+ return bitmap
+}
+
+// FromDense creates a bitmap from a slice of uint64s representing the bitmap as a dense bitmap.
+// Useful to convert bitmaps from libraries like https://github.com/bits-and-blooms/bitset or
+// https://github.com/kelindar/bitmap into roaring bitmaps fast and with convenience.
+//
+// This function will not create any run containers, only array and bitmap containers. It's up to
+// the caller to call RunOptimize if they want to further compress the runs of consecutive values.
+//
+// When doCopy is true, the bitmap is copied into a new slice for each bitmap container.
+// This is useful when the bitmap is going to be modified after this function returns or if it's
+// undesirable to hold references to large bitmaps which the GC would not be able to collect.
+// One copy can still happen even when doCopy is false if the bitmap length is not divisible
+// by bitmapContainerSize.
+//
+// See also FromBitSet.
+func FromDense(bitmap []uint64, doCopy bool) *Bitmap {
+ sz := (len(bitmap) + bitmapContainerSize - 1) / bitmapContainerSize // round up
+ rb := &Bitmap{
+ highlowcontainer: roaringArray{
+ containers: make([]container, 0, sz),
+ keys: make([]uint16, 0, sz),
+ needCopyOnWrite: make([]bool, 0, sz),
+ },
+ }
+ rb.FromDense(bitmap, doCopy)
+ return rb
+}
+
+// FromDense unmarshalls from a slice of uint64s representing the bitmap as a dense bitmap.
+// Useful to convert bitmaps from libraries like https://github.com/bits-and-blooms/bitset or
+// https://github.com/kelindar/bitmap into roaring bitmaps fast and with convenience.
+// Callers are responsible for ensuring that the bitmap is empty before calling this function.
+//
+// This function will not create any run containers, only array and bitmap containers. It is up to
+// the caller to call RunOptimize if they want to further compress the runs of consecutive values.
+//
+// When doCopy is true, the bitmap is copied into a new slice for each bitmap container.
+// This is useful when the bitmap is going to be modified after this function returns or if it's
+// undesirable to hold references to large bitmaps which the GC would not be able to collect.
+// One copy can still happen even when doCopy is false if the bitmap length is not divisible
+// by bitmapContainerSize.
+//
+// See FromBitSet.
+func (rb *Bitmap) FromDense(bitmap []uint64, doCopy bool) {
+ if len(bitmap) == 0 {
+ return
+ }
+
+ var k uint16
+ const size = bitmapContainerSize
+
+ for len(bitmap) > 0 {
+ hi := size
+ if len(bitmap) < size {
+ hi = len(bitmap)
+ }
+
+ words := bitmap[:hi]
+ count := int(popcntSlice(words))
+
+ switch {
+ case count > arrayDefaultMaxSize:
+ c := &bitmapContainer{cardinality: count, bitmap: words}
+ cow := true
+
+ if doCopy || len(words) < size {
+ c.bitmap = make([]uint64, size)
+ copy(c.bitmap, words)
+ cow = false
+ }
+
+ rb.highlowcontainer.appendContainer(k, c, cow)
+
+ case count > 0:
+ c := &arrayContainer{content: make([]uint16, count)}
+ var pos, base int
+ for _, w := range words {
+ for w != 0 {
+ t := w & -w
+ c.content[pos] = uint16(base + int(popcount(t-1)))
+ pos++
+ w ^= t
+ }
+ base += 64
+ }
+ rb.highlowcontainer.appendContainer(k, c, false)
+ }
+
+ bitmap = bitmap[hi:]
+ k++
+ }
+}
+
+// WriteDenseTo writes to a slice of uint64s representing the bitmap as a dense bitmap.
+// Callers are responsible for allocating enough space in the bitmap using DenseSize.
+// Useful to convert a roaring bitmap to a format that can be used by other libraries
+// like https://github.com/bits-and-blooms/bitset or https://github.com/kelindar/bitmap
+func (rb *Bitmap) WriteDenseTo(bitmap []uint64) {
+ for i, ct := range rb.highlowcontainer.containers {
+ hb := uint32(rb.highlowcontainer.keys[i]) << 16
+
+ switch c := ct.(type) {
+ case *arrayContainer:
+ for _, x := range c.content {
+ n := int(hb | uint32(x))
+ bitmap[n>>log2WordSize] |= uint64(1) << uint(x%64)
+ }
+
+ case *bitmapContainer:
+ copy(bitmap[int(hb)>>log2WordSize:], c.bitmap)
+
+ case *runContainer16:
+ for j := range c.iv {
+ start := uint32(c.iv[j].start)
+ end := start + uint32(c.iv[j].length) + 1
+ lo := int(hb|start) >> log2WordSize
+ hi := int(hb|(end-1)) >> log2WordSize
+
+ if lo == hi {
+ bitmap[lo] |= (^uint64(0) << uint(start%64)) &
+ (^uint64(0) >> (uint(-end) % 64))
+ continue
+ }
+
+ bitmap[lo] |= ^uint64(0) << uint(start%64)
+ for n := lo + 1; n < hi; n++ {
+ bitmap[n] = ^uint64(0)
+ }
+ bitmap[hi] |= ^uint64(0) >> (uint(-end) % 64)
+ }
+ default:
+ panic("unsupported container type")
+ }
+ }
+}
+
+// Checksum computes a hash (currently FNV-1a) for a bitmap that is suitable for
+// using bitmaps as elements in hash sets or as keys in hash maps, as well as
+// generally quicker comparisons.
+// The implementation is biased towards efficiency in little endian machines, so
+// expect some extra CPU cycles and memory to be used if your machine is big endian.
+// Likewise, do not use this to verify integrity unless you are certain you will load
+// the bitmap on a machine with the same endianess used to create it. (Thankfully
+// very few people use big endian machines these days.)
+func (rb *Bitmap) Checksum() uint64 {
+ const (
+ offset = 14695981039346656037
+ prime = 1099511628211
+ )
+
+ var bytes []byte
+
+ hash := uint64(offset)
+
+ bytes = uint16SliceAsByteSlice(rb.highlowcontainer.keys)
+
+ for _, b := range bytes {
+ hash ^= uint64(b)
+ hash *= prime
+ }
+
+ for _, c := range rb.highlowcontainer.containers {
+ // 0 separator
+ hash ^= 0
+ hash *= prime
+
+ switch c := c.(type) {
+ case *bitmapContainer:
+ bytes = uint64SliceAsByteSlice(c.bitmap)
+ case *arrayContainer:
+ bytes = uint16SliceAsByteSlice(c.content)
+ case *runContainer16:
+ bytes = interval16SliceAsByteSlice(c.iv)
+ default:
+ panic("invalid container type")
+ }
+
+ if len(bytes) == 0 {
+ panic("empty containers are not supported")
+ }
+
+ for _, b := range bytes {
+ hash ^= uint64(b)
+ hash *= prime
+ }
+ }
+
+ return hash
+}
+
+// FromUnsafeBytes reads a serialized version of this bitmap from the byte buffer without copy.
+// It is the caller's responsibility to ensure that the input data is not modified and remains valid for the entire lifetime of this bitmap.
+// This method avoids small allocations but holds references to the input data buffer. It is GC-friendly, but it may consume more memory eventually.
+// The containers in the resulting bitmap are immutable containers tied to the provided byte array and they rely on
+// copy-on-write which means that modifying them creates copies. Thus FromUnsafeBytes is more likely to be appropriate for read-only use cases,
+// when the resulting bitmap can be considered immutable.
+//
+// See also the FromBuffer function.
+// See https://github.com/RoaringBitmap/roaring/pull/395 for more details.
+func (rb *Bitmap) FromUnsafeBytes(data []byte, cookieHeader ...byte) (p int64, err error) {
+ stream := internal.NewByteBuffer(data)
+ return rb.ReadFrom(stream)
}
// ReadFrom reads a serialized version of this bitmap from stream.
// The format is compatible with other RoaringBitmap
// implementations (Java, C) and is documented here:
// https://github.com/RoaringBitmap/RoaringFormatSpec
-func (rb *Bitmap) ReadFrom(stream io.Reader) (int64, error) {
- return rb.highlowcontainer.readFrom(stream)
+// Since io.Reader is regarded as a stream and cannot be read twice.
+// So add cookieHeader to accept the 4-byte data that has been read in roaring64.ReadFrom.
+// It is not necessary to pass cookieHeader when call roaring.ReadFrom to read the roaring32 data directly.
+func (rb *Bitmap) ReadFrom(reader io.Reader, cookieHeader ...byte) (p int64, err error) {
+ stream, ok := reader.(internal.ByteInput)
+ if !ok {
+ byteInputAdapter := internal.ByteInputAdapterPool.Get().(*internal.ByteInputAdapter)
+ byteInputAdapter.Reset(reader)
+ stream = byteInputAdapter
+ }
+
+ p, err = rb.highlowcontainer.readFrom(stream, cookieHeader...)
+
+ if !ok {
+ internal.ByteInputAdapterPool.Put(stream.(*internal.ByteInputAdapter))
+ }
+ return
}
// FromBuffer creates a bitmap from its serialized version stored in buffer
@@ -87,8 +329,25 @@ func (rb *Bitmap) ReadFrom(stream io.Reader) (int64, error) {
// You should *not* change the copy-on-write status of the resulting
// bitmaps (SetCopyOnWrite).
//
-func (rb *Bitmap) FromBuffer(buf []byte) (int64, error) {
- return rb.highlowcontainer.fromBuffer(buf)
+// Thus FromBuffer is more likely to be appropriate for read-only use cases,
+// when the resulting bitmap can be considered immutable.
+//
+// If buf becomes unavailable, then a bitmap created with
+// FromBuffer would be effectively broken. Furthermore, any
+// bitmap derived from this bitmap (e.g., via Or, And) might
+// also be broken. Thus, before making buf unavailable, you should
+// call CloneCopyOnWriteContainers on all such bitmaps.
+//
+// See also the FromUnsafeBytes function which can have better performance
+// in some cases.
+func (rb *Bitmap) FromBuffer(buf []byte) (p int64, err error) {
+ stream := internal.ByteBufferPool.Get().(*internal.ByteBuffer)
+ stream.Reset(buf)
+
+ p, err = rb.highlowcontainer.readFrom(stream)
+ internal.ByteBufferPool.Put(stream)
+
+ return
}
// RunOptimize attempts to further compress the runs of consecutive values found in the bitmap
@@ -101,38 +360,16 @@ func (rb *Bitmap) HasRunCompression() bool {
return rb.highlowcontainer.hasRunCompression()
}
-// Deprecated: ReadFromMsgpack reads a msgpack2/snappy-streaming serialized
-// version of this bitmap from stream. The format is
-// expected is that written by the WriteToMsgpack()
-// call; see additional notes there.
-func (rb *Bitmap) ReadFromMsgpack(stream io.Reader) (int64, error) {
- return 0, rb.highlowcontainer.readFromMsgpack(stream)
-}
-
// MarshalBinary implements the encoding.BinaryMarshaler interface for the bitmap
+// (same as ToBytes)
func (rb *Bitmap) MarshalBinary() ([]byte, error) {
- var buf bytes.Buffer
- writer := bufio.NewWriter(&buf)
- _, err := rb.WriteTo(writer)
- if err != nil {
- return nil, err
- }
- err = writer.Flush()
- if err != nil {
- return nil, err
- }
- return buf.Bytes(), nil
+ return rb.ToBytes()
}
// UnmarshalBinary implements the encoding.BinaryUnmarshaler interface for the bitmap
func (rb *Bitmap) UnmarshalBinary(data []byte) error {
- var buf bytes.Buffer
- _, err := buf.Write(data)
- if err != nil {
- return err
- }
- reader := bufio.NewReader(&buf)
- _, err = rb.ReadFrom(reader)
+ r := bytes.NewReader(data)
+ _, err := rb.ReadFrom(r)
return err
}
@@ -152,6 +389,16 @@ func (rb *Bitmap) Clear() {
rb.highlowcontainer.clear()
}
+// ToBitSet copies the content of the RoaringBitmap into a bitset.BitSet instance
+func (rb *Bitmap) ToBitSet() *bitset.BitSet {
+ return bitset.From(rb.ToDense())
+}
+
+// FromBitSet creates a new RoaringBitmap from a bitset.BitSet instance
+func FromBitSet(bitset *bitset.BitSet) *Bitmap {
+ return FromDense(bitset.Bytes(), false)
+}
+
// ToArray creates a new slice containing all of the integers stored in the Bitmap in sorted order
func (rb *Bitmap) ToArray() []uint32 {
array := make([]uint32, rb.GetCardinality())
@@ -162,8 +409,7 @@ func (rb *Bitmap) ToArray() []uint32 {
hs := uint32(rb.highlowcontainer.getKeyAtIndex(pos)) << 16
c := rb.highlowcontainer.getContainerAtIndex(pos)
pos++
- c.fillLeastSignificant16bits(array, pos2, hs)
- pos2 += c.getCardinality()
+ pos2 = c.fillLeastSignificant16bits(array, pos2, hs)
}
return array
}
@@ -192,7 +438,7 @@ func BoundSerializedSizeInBytes(cardinality uint64, universeSize uint64) uint64
contnbr := (universeSize + uint64(65535)) / uint64(65536)
if contnbr > cardinality {
contnbr = cardinality
- // we can't have more containers than we have values
+ // we cannot have more containers than we have values
}
headermax := 8*contnbr + 4
if 4 > (contnbr+7)/8 {
@@ -215,11 +461,29 @@ type IntIterable interface {
Next() uint32
}
+// IntPeekable allows you to look at the next value without advancing and
+// advance as long as the next value is smaller than minval
+type IntPeekable interface {
+ IntIterable
+ // PeekNext peeks the next value without advancing the iterator
+ PeekNext() uint32
+ // AdvanceIfNeeded advances as long as the next value is smaller than minval
+ AdvanceIfNeeded(minval uint32)
+}
+
type intIterator struct {
pos int
hs uint32
- iter shortIterable
+ iter shortPeekable
highlowcontainer *roaringArray
+
+ // These embedded iterators per container type help reduce load in the GC.
+ // This way, instead of making up-to 64k allocations per full iteration
+ // we get a single allocation and simply reinitialize the appropriate
+ // iterator and point to it in the generic `iter` member on each key bound.
+ shortIter shortIterator
+ runIter runIterator16
+ bitmapIter bitmapContainerShortIterator
}
// HasNext returns true if there are more integers to iterate over
@@ -229,8 +493,19 @@ func (ii *intIterator) HasNext() bool {
func (ii *intIterator) init() {
if ii.highlowcontainer.size() > ii.pos {
- ii.iter = ii.highlowcontainer.getContainerAtIndex(ii.pos).getShortIterator()
ii.hs = uint32(ii.highlowcontainer.getKeyAtIndex(ii.pos)) << 16
+ c := ii.highlowcontainer.getContainerAtIndex(ii.pos)
+ switch t := c.(type) {
+ case *arrayContainer:
+ ii.shortIter = shortIterator{t.content, 0}
+ ii.iter = &ii.shortIter
+ case *runContainer16:
+ ii.runIter = runIterator16{rc: t, curIndex: 0, curPosInIndex: 0}
+ ii.iter = &ii.runIter
+ case *bitmapContainer:
+ ii.bitmapIter = bitmapContainerShortIterator{t, t.NextSetBit(0)}
+ ii.iter = &ii.bitmapIter
+ }
}
}
@@ -244,12 +519,40 @@ func (ii *intIterator) Next() uint32 {
return x
}
-func newIntIterator(a *Bitmap) *intIterator {
- p := new(intIterator)
- p.pos = 0
- p.highlowcontainer = &a.highlowcontainer
- p.init()
- return p
+// PeekNext peeks the next value without advancing the iterator
+func (ii *intIterator) PeekNext() uint32 {
+ return uint32(ii.iter.peekNext()&maxLowBit) | ii.hs
+}
+
+// AdvanceIfNeeded advances as long as the next value is smaller than minval
+func (ii *intIterator) AdvanceIfNeeded(minval uint32) {
+ to := minval & 0xffff0000
+
+ for ii.HasNext() && ii.hs < to {
+ ii.pos++
+ ii.init()
+ }
+
+ if ii.HasNext() && ii.hs == to {
+ ii.iter.advanceIfNeeded(lowbits(minval))
+
+ if !ii.iter.hasNext() {
+ ii.pos++
+ ii.init()
+ }
+ }
+}
+
+// IntIterator is meant to allow you to iterate through the values of a bitmap, see Initialize(a *Bitmap)
+type IntIterator = intIterator
+
+// Initialize configures the existing iterator so that it can iterate through the values of
+// the provided bitmap.
+// The iteration results are undefined if the bitmap is modified (e.g., with Add or Remove).
+func (ii *intIterator) Initialize(a *Bitmap) {
+ ii.pos = 0
+ ii.highlowcontainer = &a.highlowcontainer
+ ii.init()
}
type intReverseIterator struct {
@@ -257,6 +560,10 @@ type intReverseIterator struct {
hs uint32
iter shortIterable
highlowcontainer *roaringArray
+
+ shortIter reverseIterator
+ runIter runReverseIterator16
+ bitmapIter reverseBitmapContainerShortIterator
}
// HasNext returns true if there are more integers to iterate over
@@ -266,8 +573,30 @@ func (ii *intReverseIterator) HasNext() bool {
func (ii *intReverseIterator) init() {
if ii.pos >= 0 {
- ii.iter = ii.highlowcontainer.getContainerAtIndex(ii.pos).getReverseIterator()
ii.hs = uint32(ii.highlowcontainer.getKeyAtIndex(ii.pos)) << 16
+ c := ii.highlowcontainer.getContainerAtIndex(ii.pos)
+ switch t := c.(type) {
+ case *arrayContainer:
+ ii.shortIter = reverseIterator{t.content, len(t.content) - 1}
+ ii.iter = &ii.shortIter
+ case *runContainer16:
+ index := int(len(t.iv)) - 1
+ pos := uint16(0)
+
+ if index >= 0 {
+ pos = t.iv[index].length
+ }
+
+ ii.runIter = runReverseIterator16{rc: t, curIndex: index, curPosInIndex: pos}
+ ii.iter = &ii.runIter
+ case *bitmapContainer:
+ pos := -1
+ if t.cardinality > 0 {
+ pos = int(t.maximum())
+ }
+ ii.bitmapIter = reverseBitmapContainerShortIterator{t, pos}
+ ii.iter = &ii.bitmapIter
+ }
} else {
ii.iter = nil
}
@@ -283,18 +612,24 @@ func (ii *intReverseIterator) Next() uint32 {
return x
}
-func newIntReverseIterator(a *Bitmap) *intReverseIterator {
- p := new(intReverseIterator)
- p.highlowcontainer = &a.highlowcontainer
- p.pos = a.highlowcontainer.size() - 1
- p.init()
- return p
+// IntReverseIterator is meant to allow you to iterate through the values of a bitmap, see Initialize(a *Bitmap)
+type IntReverseIterator = intReverseIterator
+
+// Initialize configures the existing iterator so that it can iterate through the values of
+// the provided bitmap.
+// The iteration results are undefined if the bitmap is modified (e.g., with Add or Remove).
+func (ii *intReverseIterator) Initialize(a *Bitmap) {
+ ii.highlowcontainer = &a.highlowcontainer
+ ii.pos = a.highlowcontainer.size() - 1
+ ii.init()
}
// ManyIntIterable allows you to iterate over the values in a Bitmap
type ManyIntIterable interface {
- // pass in a buffer to fill up with values, returns how many values were returned
- NextMany([]uint32) int
+ // NextMany fills buf up with values, returns how many values were returned
+ NextMany(buf []uint32) int
+ // NextMany64 fills up buf with 64 bit values, uses hs as a mask (OR), returns how many values were returned
+ NextMany64(hs uint64, buf []uint64) int
}
type manyIntIterator struct {
@@ -302,12 +637,27 @@ type manyIntIterator struct {
hs uint32
iter manyIterable
highlowcontainer *roaringArray
+
+ shortIter shortIterator
+ runIter runIterator16
+ bitmapIter bitmapContainerManyIterator
}
func (ii *manyIntIterator) init() {
if ii.highlowcontainer.size() > ii.pos {
- ii.iter = ii.highlowcontainer.getContainerAtIndex(ii.pos).getManyIterator()
ii.hs = uint32(ii.highlowcontainer.getKeyAtIndex(ii.pos)) << 16
+ c := ii.highlowcontainer.getContainerAtIndex(ii.pos)
+ switch t := c.(type) {
+ case *arrayContainer:
+ ii.shortIter = shortIterator{t.content, 0}
+ ii.iter = &ii.shortIter
+ case *runContainer16:
+ ii.runIter = runIterator16{rc: t, curIndex: 0, curPosInIndex: 0}
+ ii.iter = &ii.runIter
+ case *bitmapContainer:
+ ii.bitmapIter = bitmapContainerManyIterator{t, -1, 0}
+ ii.iter = &ii.bitmapIter
+ }
} else {
ii.iter = nil
}
@@ -330,12 +680,35 @@ func (ii *manyIntIterator) NextMany(buf []uint32) int {
return n
}
-func newManyIntIterator(a *Bitmap) *manyIntIterator {
- p := new(manyIntIterator)
- p.pos = 0
- p.highlowcontainer = &a.highlowcontainer
- p.init()
- return p
+func (ii *manyIntIterator) NextMany64(hs64 uint64, buf []uint64) int {
+ n := 0
+ for n < len(buf) {
+ if ii.iter == nil {
+ break
+ }
+
+ hs := uint64(ii.hs) | hs64
+ moreN := ii.iter.nextMany64(hs, buf[n:])
+ n += moreN
+ if moreN == 0 {
+ ii.pos = ii.pos + 1
+ ii.init()
+ }
+ }
+
+ return n
+}
+
+// ManyIntIterator is meant to allow you to iterate through the values of a bitmap, see Initialize(a *Bitmap)
+type ManyIntIterator = manyIntIterator
+
+// Initialize configures the existing iterator so that it can iterate through the values of
+// the provided bitmap.
+// The iteration results are undefined if the bitmap is modified (e.g., with Add or Remove).
+func (ii *manyIntIterator) Initialize(a *Bitmap) {
+ ii.pos = 0
+ ii.highlowcontainer = &a.highlowcontainer
+ ii.init()
}
// String creates a string representation of the Bitmap
@@ -364,22 +737,60 @@ func (rb *Bitmap) String() string {
return buffer.String()
}
-// Iterator creates a new IntIterable to iterate over the integers contained in the bitmap, in sorted order;
+// Iterate iterates over the bitmap, calling the given callback with each value in the bitmap. If the callback returns
+// false, the iteration is halted.
+// The iteration results are undefined if the bitmap is modified (e.g., with Add or Remove).
+// There is no guarantee as to what order the values will be iterated.
+func (rb *Bitmap) Iterate(cb func(x uint32) bool) {
+ for i := 0; i < rb.highlowcontainer.size(); i++ {
+ hs := uint32(rb.highlowcontainer.getKeyAtIndex(i)) << 16
+ c := rb.highlowcontainer.getContainerAtIndex(i)
+
+ var shouldContinue bool
+ // This is hacky but it avoids allocations from invoking an interface method with a closure
+ switch t := c.(type) {
+ case *arrayContainer:
+ shouldContinue = t.iterate(func(x uint16) bool {
+ return cb(uint32(x) | hs)
+ })
+ case *runContainer16:
+ shouldContinue = t.iterate(func(x uint16) bool {
+ return cb(uint32(x) | hs)
+ })
+ case *bitmapContainer:
+ shouldContinue = t.iterate(func(x uint16) bool {
+ return cb(uint32(x) | hs)
+ })
+ }
+
+ if !shouldContinue {
+ break
+ }
+ }
+}
+
+// Iterator creates a new IntPeekable to iterate over the integers contained in the bitmap, in sorted order;
// the iterator becomes invalid if the bitmap is modified (e.g., with Add or Remove).
-func (rb *Bitmap) Iterator() IntIterable {
- return newIntIterator(rb)
+func (rb *Bitmap) Iterator() IntPeekable {
+ p := new(intIterator)
+ p.Initialize(rb)
+ return p
}
// ReverseIterator creates a new IntIterable to iterate over the integers contained in the bitmap, in sorted order;
// the iterator becomes invalid if the bitmap is modified (e.g., with Add or Remove).
func (rb *Bitmap) ReverseIterator() IntIterable {
- return newIntReverseIterator(rb)
+ p := new(intReverseIterator)
+ p.Initialize(rb)
+ return p
}
// ManyIterator creates a new ManyIntIterable to iterate over the integers contained in the bitmap, in sorted order;
// the iterator becomes invalid if the bitmap is modified (e.g., with Add or Remove).
func (rb *Bitmap) ManyIterator() ManyIntIterable {
- return newManyIntIterator(rb)
+ p := new(manyIntIterator)
+ p.Initialize(rb)
+ return p
}
// Clone creates a copy of the Bitmap
@@ -391,11 +802,17 @@ func (rb *Bitmap) Clone() *Bitmap {
// Minimum get the smallest value stored in this roaring bitmap, assumes that it is not empty
func (rb *Bitmap) Minimum() uint32 {
+ if len(rb.highlowcontainer.containers) == 0 {
+ panic("Empty bitmap")
+ }
return uint32(rb.highlowcontainer.containers[0].minimum()) | (uint32(rb.highlowcontainer.keys[0]) << 16)
}
// Maximum get the largest value stored in this roaring bitmap, assumes that it is not empty
func (rb *Bitmap) Maximum() uint32 {
+ if len(rb.highlowcontainer.containers) == 0 {
+ panic("Empty bitmap")
+ }
lastindex := len(rb.highlowcontainer.containers) - 1
return uint32(rb.highlowcontainer.containers[lastindex].maximum()) | (uint32(rb.highlowcontainer.keys[lastindex]) << 16)
}
@@ -423,41 +840,76 @@ func (rb *Bitmap) Equals(o interface{}) bool {
// AddOffset adds the value 'offset' to each and every value in a bitmap, generating a new bitmap in the process
func AddOffset(x *Bitmap, offset uint32) (answer *Bitmap) {
- containerOffset := highbits(offset)
- inOffset := lowbits(offset)
+ return AddOffset64(x, int64(offset))
+}
+
+// AddOffset64 adds the value 'offset' to each and every value in a bitmap, generating a new bitmap in the process
+// If offset + element is outside of the range [0,2^32), that the element will be dropped
+func AddOffset64(x *Bitmap, offset int64) (answer *Bitmap) {
+ // we need "offset" to be a long because we want to support values
+ // between -0xFFFFFFFF up to +-0xFFFFFFFF
+ var containerOffset64 int64
+
+ if offset < 0 {
+ containerOffset64 = (offset - (1 << 16) + 1) / (1 << 16)
+ } else {
+ containerOffset64 = offset >> 16
+ }
+
+ answer = New()
+
+ if containerOffset64 >= (1<<16) || containerOffset64 < -(1<<16) {
+ return answer
+ }
+
+ containerOffset := int32(containerOffset64)
+ inOffset := (uint16)(offset - containerOffset64*(1<<16))
+
if inOffset == 0 {
- answer = x.Clone()
- for pos := 0; pos < answer.highlowcontainer.size(); pos++ {
- key := answer.highlowcontainer.getKeyAtIndex(pos)
+ for pos := 0; pos < x.highlowcontainer.size(); pos++ {
+ key := int32(x.highlowcontainer.getKeyAtIndex(pos))
key += containerOffset
- answer.highlowcontainer.keys[pos] = key
+
+ if key >= 0 && key <= MaxUint16 {
+ c := x.highlowcontainer.getContainerAtIndex(pos).clone()
+ answer.highlowcontainer.appendContainer(uint16(key), c, false)
+ }
}
} else {
- answer = New()
for pos := 0; pos < x.highlowcontainer.size(); pos++ {
- key := x.highlowcontainer.getKeyAtIndex(pos)
+ key := int32(x.highlowcontainer.getKeyAtIndex(pos))
key += containerOffset
+
+ if key+1 < 0 || key > MaxUint16 {
+ continue
+ }
+
c := x.highlowcontainer.getContainerAtIndex(pos)
- offsetted := c.addOffset(inOffset)
- if offsetted[0].getCardinality() > 0 {
+ lo, hi := c.addOffset(inOffset)
+
+ if lo != nil && key >= 0 {
curSize := answer.highlowcontainer.size()
- lastkey := uint16(0)
+ lastkey := int32(0)
+
if curSize > 0 {
- lastkey = answer.highlowcontainer.getKeyAtIndex(curSize - 1)
+ lastkey = int32(answer.highlowcontainer.getKeyAtIndex(curSize - 1))
}
+
if curSize > 0 && lastkey == key {
prev := answer.highlowcontainer.getContainerAtIndex(curSize - 1)
- orrseult := prev.ior(offsetted[0])
- answer.highlowcontainer.setContainerAtIndex(curSize-1, orrseult)
+ orresult := prev.ior(lo)
+ answer.highlowcontainer.setContainerAtIndex(curSize-1, orresult)
} else {
- answer.highlowcontainer.appendContainer(key, offsetted[0], false)
+ answer.highlowcontainer.appendContainer(uint16(key), lo, false)
}
}
- if offsetted[1].getCardinality() > 0 {
- answer.highlowcontainer.appendContainer(key+1, offsetted[1], false)
+
+ if hi != nil && key+1 <= MaxUint16 {
+ answer.highlowcontainer.appendContainer(uint16(key+1), hi, false)
}
}
}
+
return answer
}
@@ -523,13 +975,13 @@ func (rb *Bitmap) Remove(x uint32) {
if i >= 0 {
c := rb.highlowcontainer.getWritableContainerAtIndex(i).iremoveReturnMinimized(lowbits(x))
rb.highlowcontainer.setContainerAtIndex(i, c)
- if rb.highlowcontainer.getContainerAtIndex(i).getCardinality() == 0 {
+ if rb.highlowcontainer.getContainerAtIndex(i).isEmpty() {
rb.highlowcontainer.removeAtIndex(i)
}
}
}
-// CheckedRemove removes the integer x from the bitmap and return true if the integer was effectively remove (and false if the integer was not present)
+// CheckedRemove removes the integer x from the bitmap and return true if the integer was effectively removed (and false if the integer was not present)
func (rb *Bitmap) CheckedRemove(x uint32) bool {
// TODO: add unit tests for this method
hb := highbits(x)
@@ -539,7 +991,7 @@ func (rb *Bitmap) CheckedRemove(x uint32) bool {
oldcard := C.getCardinality()
C = C.iremoveReturnMinimized(lowbits(x))
rb.highlowcontainer.setContainerAtIndex(i, C)
- if rb.highlowcontainer.getContainerAtIndex(i).getCardinality() == 0 {
+ if rb.highlowcontainer.getContainerAtIndex(i).isEmpty() {
rb.highlowcontainer.removeAtIndex(i)
return true
}
@@ -563,7 +1015,10 @@ func (rb *Bitmap) GetCardinality() uint64 {
return size
}
-// Rank returns the number of integers that are smaller or equal to x (Rank(infinity) would be GetCardinality())
+// Rank returns the number of integers that are smaller or equal to x (Rank(infinity) would be GetCardinality()).
+// If you pass the smallest value, you get the value 1. If you pass a value that is smaller than the smallest
+// value, you get 0. Note that this function differs in convention from the Select function since it
+// return 1 and not 0 on the smallest value.
func (rb *Bitmap) Rank(x uint32) uint64 {
size := uint64(0)
for i := 0; i < rb.highlowcontainer.size(); i++ {
@@ -580,23 +1035,22 @@ func (rb *Bitmap) Rank(x uint32) uint64 {
return size
}
-// Select returns the xth integer in the bitmap
+// Select returns the xth integer in the bitmap. If you pass 0, you get
+// the smallest element. Note that this function differs in convention from
+// the Rank function which returns 1 on the smallest value.
func (rb *Bitmap) Select(x uint32) (uint32, error) {
- if rb.GetCardinality() <= uint64(x) {
- return 0, fmt.Errorf("can't find %dth integer in a bitmap with only %d items", x, rb.GetCardinality())
- }
-
remaining := x
for i := 0; i < rb.highlowcontainer.size(); i++ {
c := rb.highlowcontainer.getContainerAtIndex(i)
- if remaining >= uint32(c.getCardinality()) {
- remaining -= uint32(c.getCardinality())
+ card := uint32(c.getCardinality())
+ if remaining >= card {
+ remaining -= card
} else {
key := rb.highlowcontainer.getKeyAtIndex(i)
return uint32(key)<<16 + uint32(c.selectInt(uint16(remaining))), nil
}
}
- return 0, fmt.Errorf("can't find %dth integer in a bitmap with only %d items", x, rb.GetCardinality())
+ return 0, fmt.Errorf("cannot find %dth integer in a bitmap with only %d items", x, rb.GetCardinality())
}
// And computes the intersection between two bitmaps and stores the result in the current bitmap
@@ -617,7 +1071,7 @@ main:
c1 := rb.highlowcontainer.getWritableContainerAtIndex(pos1)
c2 := x2.highlowcontainer.getContainerAtIndex(pos2)
diff := c1.iand(c2)
- if diff.getCardinality() > 0 {
+ if !diff.isEmpty() {
rb.highlowcontainer.replaceKeyAndContainerAtIndex(intersectionsize, s1, diff, false)
intersectionsize++
}
@@ -748,6 +1202,28 @@ main:
return answer
}
+// IntersectsWithInterval checks whether a bitmap 'rb' and an open interval '[x,y)' intersect.
+func (rb *Bitmap) IntersectsWithInterval(x, y uint64) bool {
+ if x >= y {
+ return false
+ }
+ if x > MaxUint32 {
+ return false
+ }
+
+ it := intIterator{}
+ it.Initialize(rb)
+ it.AdvanceIfNeeded(uint32(x))
+ if !it.HasNext() {
+ return false
+ }
+ if uint64(it.Next()) >= y {
+ return false
+ }
+
+ return true
+}
+
// Intersects checks whether two bitmap intersects, bitmaps are not modified
func (rb *Bitmap) Intersects(x2 *Bitmap) bool {
pos1 := 0
@@ -819,7 +1295,7 @@ func (rb *Bitmap) Xor(x2 *Bitmap) {
} else {
// TODO: couple be computed in-place for reduced memory usage
c := rb.highlowcontainer.getContainerAtIndex(pos1).xor(x2.highlowcontainer.getContainerAtIndex(pos2))
- if c.getCardinality() > 0 {
+ if !c.isEmpty() {
rb.highlowcontainer.setContainerAtIndex(pos1, c)
pos1++
} else {
@@ -865,7 +1341,7 @@ main:
}
s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
} else {
- rb.highlowcontainer.replaceKeyAndContainerAtIndex(pos1, s1, rb.highlowcontainer.getWritableContainerAtIndex(pos1).ior(x2.highlowcontainer.getContainerAtIndex(pos2)), false)
+ rb.highlowcontainer.replaceKeyAndContainerAtIndex(pos1, s1, rb.highlowcontainer.getUnionedWritableContainer(pos1, x2.highlowcontainer.getContainerAtIndex(pos2)), false)
pos1++
pos2++
if (pos1 == length1) || (pos2 == length2) {
@@ -899,7 +1375,7 @@ main:
c1 := rb.highlowcontainer.getWritableContainerAtIndex(pos1)
c2 := x2.highlowcontainer.getContainerAtIndex(pos2)
diff := c1.iandNot(c2)
- if diff.getCardinality() > 0 {
+ if !diff.isEmpty() {
rb.highlowcontainer.replaceKeyAndContainerAtIndex(intersectionsize, s1, diff, false)
intersectionsize++
}
@@ -1008,7 +1484,7 @@ main:
C := x1.highlowcontainer.getContainerAtIndex(pos1)
C = C.and(x2.highlowcontainer.getContainerAtIndex(pos2))
- if C.getCardinality() > 0 {
+ if !C.isEmpty() {
answer.highlowcontainer.appendContainer(s1, C, false)
}
pos1++
@@ -1055,7 +1531,7 @@ func Xor(x1, x2 *Bitmap) *Bitmap {
pos2++
} else {
c := x1.highlowcontainer.getContainerAtIndex(pos1).xor(x2.highlowcontainer.getContainerAtIndex(pos2))
- if c.getCardinality() > 0 {
+ if !c.isEmpty() {
answer.highlowcontainer.appendContainer(s1, c, false)
}
pos1++
@@ -1098,7 +1574,7 @@ main:
c1 := x1.highlowcontainer.getContainerAtIndex(pos1)
c2 := x2.highlowcontainer.getContainerAtIndex(pos2)
diff := c1.andNot(c2)
- if diff.getCardinality() > 0 {
+ if !diff.isEmpty() {
answer.highlowcontainer.appendContainer(s1, diff, false)
}
pos1++
@@ -1188,7 +1664,7 @@ func (rb *Bitmap) Flip(rangeStart, rangeEnd uint64) {
if i >= 0 {
c := rb.highlowcontainer.getWritableContainerAtIndex(i).inot(int(containerStart), int(containerLast)+1)
- if c.getCardinality() > 0 {
+ if !c.isEmpty() {
rb.highlowcontainer.setContainerAtIndex(i, c)
} else {
rb.highlowcontainer.removeAtIndex(i)
@@ -1269,7 +1745,7 @@ func (rb *Bitmap) RemoveRange(rangeStart, rangeEnd uint64) {
return
}
c := rb.highlowcontainer.getWritableContainerAtIndex(i).iremoveRange(int(lbStart), int(lbLast+1))
- if c.getCardinality() > 0 {
+ if !c.isEmpty() {
rb.highlowcontainer.setContainerAtIndex(i, c)
} else {
rb.highlowcontainer.removeAtIndex(i)
@@ -1282,7 +1758,7 @@ func (rb *Bitmap) RemoveRange(rangeStart, rangeEnd uint64) {
if ifirst >= 0 {
if lbStart != 0 {
c := rb.highlowcontainer.getWritableContainerAtIndex(ifirst).iremoveRange(int(lbStart), int(max+1))
- if c.getCardinality() > 0 {
+ if !c.isEmpty() {
rb.highlowcontainer.setContainerAtIndex(ifirst, c)
ifirst++
}
@@ -1293,7 +1769,7 @@ func (rb *Bitmap) RemoveRange(rangeStart, rangeEnd uint64) {
if ilast >= 0 {
if lbLast != max {
c := rb.highlowcontainer.getWritableContainerAtIndex(ilast).iremoveRange(int(0), int(lbLast+1))
- if c.getCardinality() > 0 {
+ if !c.isEmpty() {
rb.highlowcontainer.setContainerAtIndex(ilast, c)
} else {
ilast++
@@ -1349,7 +1825,7 @@ func Flip(bm *Bitmap, rangeStart, rangeEnd uint64) *Bitmap {
if i >= 0 {
c := bm.highlowcontainer.getContainerAtIndex(i).not(int(containerStart), int(containerLast)+1)
- if c.getCardinality() > 0 {
+ if !c.isEmpty() {
answer.highlowcontainer.insertNewKeyValueAt(-j-1, uint16(hb), c)
}
@@ -1378,6 +1854,21 @@ func (rb *Bitmap) GetCopyOnWrite() (val bool) {
return rb.highlowcontainer.copyOnWrite
}
+// CloneCopyOnWriteContainers clones all containers which have
+// needCopyOnWrite set to true.
+// This can be used to make sure it is safe to munmap a []byte
+// that the roaring array may still have a reference to, after
+// calling FromBuffer.
+// More generally this function is useful if you call FromBuffer
+// to construct a bitmap with a backing array buf
+// and then later discard the buf array. Note that you should call
+// CloneCopyOnWriteContainers on all bitmaps that were derived
+// from the 'FromBuffer' bitmap since they map have dependencies
+// on the buf array as well.
+func (rb *Bitmap) CloneCopyOnWriteContainers() {
+ rb.highlowcontainer.cloneCopyOnWriteContainers()
+}
+
// FlipInt calls Flip after casting the parameters (convenience method)
func FlipInt(bm *Bitmap, rangeStart, rangeEnd int) *Bitmap {
return Flip(bm, uint64(rangeStart), uint64(rangeEnd))
diff --git a/vendor/github.com/RoaringBitmap/roaring/roaringarray.go b/vendor/github.com/RoaringBitmap/roaring/roaringarray.go
index 3f77759..079195d 100644
--- a/vendor/github.com/RoaringBitmap/roaring/roaringarray.go
+++ b/vendor/github.com/RoaringBitmap/roaring/roaringarray.go
@@ -5,16 +5,15 @@ import (
"encoding/binary"
"fmt"
"io"
- "io/ioutil"
- snappy "github.com/glycerine/go-unsnap-stream"
- "github.com/tinylib/msgp/msgp"
+ "github.com/RoaringBitmap/roaring/internal"
)
-//go:generate msgp -unexported
-
type container interface {
- addOffset(uint16) []container
+ // addOffset returns the (low, high) parts of the shifted container.
+ // Whenever one of them would be empty, nil will be returned instead to
+ // avoid unnecessary allocations.
+ addOffset(uint16) (container, container)
clone() container
and(container) container
@@ -22,6 +21,7 @@ type container interface {
iand(container) container // i stands for inplace
andNot(container) container
iandNot(container) container // i stands for inplace
+ isEmpty() bool
getCardinality() int
// rank returns the number of integers that are
// smaller or equal to x. rank(infinity) would be getCardinality().
@@ -39,7 +39,8 @@ type container interface {
not(start, final int) container // range is [firstOfRange,lastOfRange)
inot(firstOfRange, endx int) container // i stands for inplace, range is [firstOfRange,endx)
xor(r container) container
- getShortIterator() shortIterable
+ getShortIterator() shortPeekable
+ iterate(cb func(x uint16) bool) bool
getReverseIterator() shortIterable
getManyIterator() manyIterable
contains(i uint16) bool
@@ -51,7 +52,7 @@ type container interface {
// any of the implementations.
equals(r container) bool
- fillLeastSignificant16bits(array []uint32, i int, mask uint32)
+ fillLeastSignificant16bits(array []uint32, i int, mask uint32) int
or(r container) container
orCardinality(r container) int
isFull() bool
@@ -64,7 +65,6 @@ type container interface {
iremoveRange(start, final int) container // i stands for inplace, range is [firstOfRange,lastOfRange)
selectInt(x uint16) int // selectInt returns the xth integer in the container
serializedSizeInBytes() int
- readFrom(io.Reader) (int, error)
writeTo(io.Writer) (int, error)
numberOfRuns() int
@@ -104,18 +104,6 @@ type roaringArray struct {
containers []container `msg:"-"` // don't try to serialize directly.
needCopyOnWrite []bool
copyOnWrite bool
-
- // conserz is used at serialization time
- // to serialize containers. Otherwise empty.
- conserz []containerSerz
-}
-
-// containerSerz facilitates serializing container (tricky to
-// serialize because it is an interface) by providing a
-// light wrapper with a type identifier.
-type containerSerz struct {
- t contype `msg:"t"` // type
- r msgp.Raw `msg:"r"` // Raw msgpack of the actual container type
}
func newRoaringArray() *roaringArray {
@@ -125,9 +113,10 @@ func newRoaringArray() *roaringArray {
// runOptimize compresses the element containers to minimize space consumed.
// Q: how does this interact with copyOnWrite and needCopyOnWrite?
// A: since we aren't changing the logical content, just the representation,
-// we don't bother to check the needCopyOnWrite bits. We replace
-// (possibly all) elements of ra.containers in-place with space
-// optimized versions.
+//
+// we don't bother to check the needCopyOnWrite bits. We replace
+// (possibly all) elements of ra.containers in-place with space
+// optimized versions.
func (ra *roaringArray) runOptimize() {
for i := range ra.containers {
ra.containers[i] = ra.containers[i].toEfficientContainer()
@@ -247,7 +236,6 @@ func (ra *roaringArray) resize(newsize int) {
func (ra *roaringArray) clear() {
ra.resize(0)
ra.copyOnWrite = false
- ra.conserz = nil
}
func (ra *roaringArray) clone() *roaringArray {
@@ -283,6 +271,18 @@ func (ra *roaringArray) clone() *roaringArray {
return &sa
}
+// clone all containers which have needCopyOnWrite set to true
+// This can be used to make sure it is safe to munmap a []byte
+// that the roaring array may still have a reference to.
+func (ra *roaringArray) cloneCopyOnWriteContainers() {
+ for i, needCopyOnWrite := range ra.needCopyOnWrite {
+ if needCopyOnWrite {
+ ra.containers[i] = ra.containers[i].clone()
+ ra.needCopyOnWrite[i] = false
+ }
+ }
+}
+
// unused function:
//func (ra *roaringArray) containsKey(x uint16) bool {
// return (ra.binarySearch(0, int64(len(ra.keys)), x) >= 0)
@@ -317,6 +317,17 @@ func (ra *roaringArray) getFastContainerAtIndex(i int, needsWriteable bool) cont
return c
}
+// getUnionedWritableContainer switches behavior for in-place Or
+// depending on whether the container requires a copy on write.
+// If it does using the non-inplace or() method leads to fewer allocations.
+func (ra *roaringArray) getUnionedWritableContainer(pos int, other container) container {
+ if ra.needCopyOnWrite[pos] {
+ return ra.getContainerAtIndex(pos).or(other)
+ }
+ return ra.getContainerAtIndex(pos).ior(other)
+
+}
+
func (ra *roaringArray) getWritableContainerAtIndex(i int) container {
if ra.needCopyOnWrite[i] {
ra.containers[i] = ra.containers[i].clone()
@@ -456,9 +467,7 @@ func (ra *roaringArray) serializedSizeInBytes() uint64 {
return answer
}
-//
// spec: https://github.com/RoaringBitmap/RoaringFormatSpec
-//
func (ra *roaringArray) writeTo(w io.Writer) (n int64, err error) {
hasRun := ra.hasRunCompression()
isRunSizeInBytes := 0
@@ -479,20 +488,15 @@ func (ra *roaringArray) writeTo(w io.Writer) (n int64, err error) {
nw += 2
binary.LittleEndian.PutUint16(buf[2:], uint16(len(ra.keys)-1))
nw += 2
-
- // compute isRun bitmap
- var ir []byte
-
- isRun := newBitmapContainer()
+ // compute isRun bitmap without temporary allocation
+ var runbitmapslice = buf[nw : nw+isRunSizeInBytes]
for i, c := range ra.containers {
switch c.(type) {
case *runContainer16:
- isRun.iadd(uint16(i))
+ runbitmapslice[i/8] |= 1 << (uint(i) % 8)
}
}
- // convert to little endian
- ir = isRun.asLittleEndianByteSlice()[:isRunSizeInBytes]
- nw += copy(buf[nw:], ir)
+ nw += isRunSizeInBytes
} else {
binary.LittleEndian.PutUint32(buf[0:], uint32(serialCookieNoRunContainer))
nw += 4
@@ -540,59 +544,69 @@ func (ra *roaringArray) writeTo(w io.Writer) (n int64, err error) {
return n, nil
}
-//
// spec: https://github.com/RoaringBitmap/RoaringFormatSpec
-//
func (ra *roaringArray) toBytes() ([]byte, error) {
var buf bytes.Buffer
_, err := ra.writeTo(&buf)
return buf.Bytes(), err
}
-func (ra *roaringArray) fromBuffer(buf []byte) (int64, error) {
- pos := 0
- if len(buf) < 8 {
- return 0, fmt.Errorf("buffer too small, expecting at least 8 bytes, was %d", len(buf))
+// Reads a serialized roaringArray from a byte slice.
+func (ra *roaringArray) readFrom(stream internal.ByteInput, cookieHeader ...byte) (int64, error) {
+ var cookie uint32
+ var err error
+ if len(cookieHeader) > 0 && len(cookieHeader) != 4 {
+ return int64(len(cookieHeader)), fmt.Errorf("error in roaringArray.readFrom: could not read initial cookie: incorrect size of cookie header")
}
+ if len(cookieHeader) == 4 {
+ cookie = binary.LittleEndian.Uint32(cookieHeader)
+ } else {
+ cookie, err = stream.ReadUInt32()
+ if err != nil {
+ return stream.GetReadBytes(), fmt.Errorf("error in roaringArray.readFrom: could not read initial cookie: %s", err)
+ }
+ }
+ // If NextReturnsSafeSlice is false, then willNeedCopyOnWrite should be true
+ willNeedCopyOnWrite := !stream.NextReturnsSafeSlice()
- cookie := binary.LittleEndian.Uint32(buf)
- pos += 4
- var size uint32 // number of containers
- haveRunContainers := false
+ var size uint32
var isRunBitmap []byte
- // cookie header
if cookie&0x0000FFFF == serialCookie {
- haveRunContainers = true
- size = uint32(uint16(cookie>>16) + 1) // number of containers
-
+ size = uint32(cookie>>16 + 1)
// create is-run-container bitmap
isRunBitmapSize := (int(size) + 7) / 8
- if pos+isRunBitmapSize > len(buf) {
- return 0, fmt.Errorf("malformed bitmap, is-run bitmap overruns buffer at %d", pos+isRunBitmapSize)
+ isRunBitmap, err = stream.Next(isRunBitmapSize)
+
+ if err != nil {
+ return stream.GetReadBytes(), fmt.Errorf("malformed bitmap, failed to read is-run bitmap, got: %s", err)
}
-
- isRunBitmap = buf[pos : pos+isRunBitmapSize]
- pos += isRunBitmapSize
} else if cookie == serialCookieNoRunContainer {
- size = binary.LittleEndian.Uint32(buf[pos:])
- pos += 4
+ size, err = stream.ReadUInt32()
+ if err != nil {
+ return stream.GetReadBytes(), fmt.Errorf("malformed bitmap, failed to read a bitmap size: %s", err)
+ }
} else {
- return 0, fmt.Errorf("error in roaringArray.readFrom: did not find expected serialCookie in header")
+ return stream.GetReadBytes(), fmt.Errorf("error in roaringArray.readFrom: did not find expected serialCookie in header")
}
- if size > (1 << 16) {
- return 0, fmt.Errorf("It is logically impossible to have more than (1<<16) containers.")
- }
- // descriptive header
- // keycard - is {key, cardinality} tuple slice
- if pos+2*2*int(size) > len(buf) {
- return 0, fmt.Errorf("malfomred bitmap, key-cardinality slice overruns buffer at %d", pos+2*2*int(size))
- }
- keycard := byteSliceAsUint16Slice(buf[pos : pos+2*2*int(size)])
- pos += 2 * 2 * int(size)
- if !haveRunContainers || size >= noOffsetThreshold {
- pos += 4 * int(size)
+ if size > (1 << 16) {
+ return stream.GetReadBytes(), fmt.Errorf("it is logically impossible to have more than (1<<16) containers")
+ }
+
+ // descriptive header
+ buf, err := stream.Next(2 * 2 * int(size))
+
+ if err != nil {
+ return stream.GetReadBytes(), fmt.Errorf("failed to read descriptive header: %s", err)
+ }
+
+ keycard := byteSliceAsUint16Slice(buf)
+
+ if isRunBitmap == nil || size >= noOffsetThreshold {
+ if err := stream.SkipBytes(int(size) * 4); err != nil {
+ return stream.GetReadBytes(), fmt.Errorf("failed to skip bytes: %s", err)
+ }
}
// Allocate slices upfront as number of containers is known
@@ -601,11 +615,13 @@ func (ra *roaringArray) fromBuffer(buf []byte) (int64, error) {
} else {
ra.containers = make([]container, size)
}
+
if cap(ra.keys) >= int(size) {
ra.keys = ra.keys[:size]
} else {
ra.keys = make([]uint16, size)
}
+
if cap(ra.needCopyOnWrite) >= int(size) {
ra.needCopyOnWrite = ra.needCopyOnWrite[:size]
} else {
@@ -613,129 +629,61 @@ func (ra *roaringArray) fromBuffer(buf []byte) (int64, error) {
}
for i := uint32(0); i < size; i++ {
- key := uint16(keycard[2*i])
+ key := keycard[2*i]
card := int(keycard[2*i+1]) + 1
ra.keys[i] = key
- ra.needCopyOnWrite[i] = true
+ ra.needCopyOnWrite[i] = willNeedCopyOnWrite
- if haveRunContainers && isRunBitmap[i/8]&(1<<(i%8)) != 0 {
+ if isRunBitmap != nil && isRunBitmap[i/8]&(1<<(i%8)) != 0 {
// run container
- nr := binary.LittleEndian.Uint16(buf[pos:])
- pos += 2
- if pos+int(nr)*4 > len(buf) {
- return 0, fmt.Errorf("malformed bitmap, a run container overruns buffer at %d:%d", pos, pos+int(nr)*4)
+ nr, err := stream.ReadUInt16()
+
+ if err != nil {
+ return 0, fmt.Errorf("failed to read runtime container size: %s", err)
}
+
+ buf, err := stream.Next(int(nr) * 4)
+
+ if err != nil {
+ return stream.GetReadBytes(), fmt.Errorf("failed to read runtime container content: %s", err)
+ }
+
nb := runContainer16{
- iv: byteSliceAsInterval16Slice(buf[pos : pos+int(nr)*4]),
- card: int64(card),
+ iv: byteSliceAsInterval16Slice(buf),
}
- pos += int(nr) * 4
+
ra.containers[i] = &nb
} else if card > arrayDefaultMaxSize {
// bitmap container
+ buf, err := stream.Next(arrayDefaultMaxSize * 2)
+
+ if err != nil {
+ return stream.GetReadBytes(), fmt.Errorf("failed to read bitmap container: %s", err)
+ }
+
nb := bitmapContainer{
cardinality: card,
- bitmap: byteSliceAsUint64Slice(buf[pos : pos+arrayDefaultMaxSize*2]),
+ bitmap: byteSliceAsUint64Slice(buf),
}
- pos += arrayDefaultMaxSize * 2
+
ra.containers[i] = &nb
} else {
// array container
- nb := arrayContainer{
- byteSliceAsUint16Slice(buf[pos : pos+card*2]),
+ buf, err := stream.Next(card * 2)
+
+ if err != nil {
+ return stream.GetReadBytes(), fmt.Errorf("failed to read array container: %s", err)
}
- pos += card * 2
+
+ nb := arrayContainer{
+ byteSliceAsUint16Slice(buf),
+ }
+
ra.containers[i] = &nb
}
}
- return int64(pos), nil
-}
-
-func (ra *roaringArray) readFrom(stream io.Reader) (int64, error) {
- pos := 0
- var cookie uint32
- err := binary.Read(stream, binary.LittleEndian, &cookie)
- if err != nil {
- return 0, fmt.Errorf("error in roaringArray.readFrom: could not read initial cookie: %s", err)
- }
- pos += 4
- var size uint32
- haveRunContainers := false
- var isRun *bitmapContainer
- if cookie&0x0000FFFF == serialCookie {
- haveRunContainers = true
- size = uint32(uint16(cookie>>16) + 1)
- bytesToRead := (int(size) + 7) / 8
- numwords := (bytesToRead + 7) / 8
- by := make([]byte, bytesToRead, numwords*8)
- nr, err := io.ReadFull(stream, by)
- if err != nil {
- return 8 + int64(nr), fmt.Errorf("error in readFrom: could not read the "+
- "runContainer bit flags of length %v bytes: %v", bytesToRead, err)
- }
- pos += bytesToRead
- by = by[:cap(by)]
- isRun = newBitmapContainer()
- for i := 0; i < numwords; i++ {
- isRun.bitmap[i] = binary.LittleEndian.Uint64(by)
- by = by[8:]
- }
- } else if cookie == serialCookieNoRunContainer {
- err = binary.Read(stream, binary.LittleEndian, &size)
- if err != nil {
- return 0, fmt.Errorf("error in roaringArray.readFrom: when reading size, got: %s", err)
- }
- pos += 4
- } else {
- return 0, fmt.Errorf("error in roaringArray.readFrom: did not find expected serialCookie in header")
- }
- if size > (1 << 16) {
- return 0, fmt.Errorf("It is logically impossible to have more than (1<<16) containers.")
- }
- // descriptive header
- keycard := make([]uint16, 2*size, 2*size)
- err = binary.Read(stream, binary.LittleEndian, keycard)
- if err != nil {
- return 0, err
- }
- pos += 2 * 2 * int(size)
- // offset header
- if !haveRunContainers || size >= noOffsetThreshold {
- io.CopyN(ioutil.Discard, stream, 4*int64(size)) // we never skip ahead so this data can be ignored
- pos += 4 * int(size)
- }
- for i := uint32(0); i < size; i++ {
- key := int(keycard[2*i])
- card := int(keycard[2*i+1]) + 1
- if haveRunContainers && isRun.contains(uint16(i)) {
- nb := newRunContainer16()
- nr, err := nb.readFrom(stream)
- if err != nil {
- return 0, err
- }
- pos += nr
- ra.appendContainer(uint16(key), nb, false)
- } else if card > arrayDefaultMaxSize {
- nb := newBitmapContainer()
- nr, err := nb.readFrom(stream)
- if err != nil {
- return 0, err
- }
- nb.cardinality = card
- pos += nr
- ra.appendContainer(keycard[2*i], nb, false)
- } else {
- nb := newArrayContainerSize(card)
- nr, err := nb.readFrom(stream)
- if err != nil {
- return 0, err
- }
- pos += nr
- ra.appendContainer(keycard[2*i], nb, false)
- }
- }
- return int64(pos), nil
+ return stream.GetReadBytes(), nil
}
func (ra *roaringArray) hasRunCompression() bool {
@@ -748,84 +696,6 @@ func (ra *roaringArray) hasRunCompression() bool {
return false
}
-func (ra *roaringArray) writeToMsgpack(stream io.Writer) error {
-
- ra.conserz = make([]containerSerz, len(ra.containers))
- for i, v := range ra.containers {
- switch cn := v.(type) {
- case *bitmapContainer:
- bts, err := cn.MarshalMsg(nil)
- if err != nil {
- return err
- }
- ra.conserz[i].t = bitmapContype
- ra.conserz[i].r = bts
- case *arrayContainer:
- bts, err := cn.MarshalMsg(nil)
- if err != nil {
- return err
- }
- ra.conserz[i].t = arrayContype
- ra.conserz[i].r = bts
- case *runContainer16:
- bts, err := cn.MarshalMsg(nil)
- if err != nil {
- return err
- }
- ra.conserz[i].t = run16Contype
- ra.conserz[i].r = bts
- default:
- panic(fmt.Errorf("Unrecognized container implementation: %T", cn))
- }
- }
- w := snappy.NewWriter(stream)
- err := msgp.Encode(w, ra)
- ra.conserz = nil
- return err
-}
-
-func (ra *roaringArray) readFromMsgpack(stream io.Reader) error {
- r := snappy.NewReader(stream)
- err := msgp.Decode(r, ra)
- if err != nil {
- return err
- }
-
- if len(ra.containers) != len(ra.keys) {
- ra.containers = make([]container, len(ra.keys))
- }
-
- for i, v := range ra.conserz {
- switch v.t {
- case bitmapContype:
- c := &bitmapContainer{}
- _, err = c.UnmarshalMsg(v.r)
- if err != nil {
- return err
- }
- ra.containers[i] = c
- case arrayContype:
- c := &arrayContainer{}
- _, err = c.UnmarshalMsg(v.r)
- if err != nil {
- return err
- }
- ra.containers[i] = c
- case run16Contype:
- c := &runContainer16{}
- _, err = c.UnmarshalMsg(v.r)
- if err != nil {
- return err
- }
- ra.containers[i] = c
- default:
- return fmt.Errorf("unrecognized contype serialization code: '%v'", v.t)
- }
- }
- ra.conserz = nil
- return nil
-}
-
func (ra *roaringArray) advanceUntil(min uint16, pos int) int {
lower := pos + 1
diff --git a/vendor/github.com/RoaringBitmap/roaring/roaringarray_gen.go b/vendor/github.com/RoaringBitmap/roaring/roaringarray_gen.go
deleted file mode 100644
index dcd7187..0000000
--- a/vendor/github.com/RoaringBitmap/roaring/roaringarray_gen.go
+++ /dev/null
@@ -1,529 +0,0 @@
-package roaring
-
-// NOTE: THIS FILE WAS PRODUCED BY THE
-// MSGP CODE GENERATION TOOL (github.com/tinylib/msgp)
-// DO NOT EDIT
-
-import (
- "github.com/tinylib/msgp/msgp"
-)
-
-// Deprecated: DecodeMsg implements msgp.Decodable
-func (z *containerSerz) DecodeMsg(dc *msgp.Reader) (err error) {
- var field []byte
- _ = field
- var zxvk uint32
- zxvk, err = dc.ReadMapHeader()
- if err != nil {
- return
- }
- for zxvk > 0 {
- zxvk--
- field, err = dc.ReadMapKeyPtr()
- if err != nil {
- return
- }
- switch msgp.UnsafeString(field) {
- case "t":
- {
- var zbzg uint8
- zbzg, err = dc.ReadUint8()
- z.t = contype(zbzg)
- }
- if err != nil {
- return
- }
- case "r":
- err = z.r.DecodeMsg(dc)
- if err != nil {
- return
- }
- default:
- err = dc.Skip()
- if err != nil {
- return
- }
- }
- }
- return
-}
-
-// Deprecated: EncodeMsg implements msgp.Encodable
-func (z *containerSerz) EncodeMsg(en *msgp.Writer) (err error) {
- // map header, size 2
- // write "t"
- err = en.Append(0x82, 0xa1, 0x74)
- if err != nil {
- return err
- }
- err = en.WriteUint8(uint8(z.t))
- if err != nil {
- return
- }
- // write "r"
- err = en.Append(0xa1, 0x72)
- if err != nil {
- return err
- }
- err = z.r.EncodeMsg(en)
- if err != nil {
- return
- }
- return
-}
-
-// Deprecated: MarshalMsg implements msgp.Marshaler
-func (z *containerSerz) MarshalMsg(b []byte) (o []byte, err error) {
- o = msgp.Require(b, z.Msgsize())
- // map header, size 2
- // string "t"
- o = append(o, 0x82, 0xa1, 0x74)
- o = msgp.AppendUint8(o, uint8(z.t))
- // string "r"
- o = append(o, 0xa1, 0x72)
- o, err = z.r.MarshalMsg(o)
- if err != nil {
- return
- }
- return
-}
-
-// Deprecated: UnmarshalMsg implements msgp.Unmarshaler
-func (z *containerSerz) UnmarshalMsg(bts []byte) (o []byte, err error) {
- var field []byte
- _ = field
- var zbai uint32
- zbai, bts, err = msgp.ReadMapHeaderBytes(bts)
- if err != nil {
- return
- }
- for zbai > 0 {
- zbai--
- field, bts, err = msgp.ReadMapKeyZC(bts)
- if err != nil {
- return
- }
- switch msgp.UnsafeString(field) {
- case "t":
- {
- var zcmr uint8
- zcmr, bts, err = msgp.ReadUint8Bytes(bts)
- z.t = contype(zcmr)
- }
- if err != nil {
- return
- }
- case "r":
- bts, err = z.r.UnmarshalMsg(bts)
- if err != nil {
- return
- }
- default:
- bts, err = msgp.Skip(bts)
- if err != nil {
- return
- }
- }
- }
- o = bts
- return
-}
-
-// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (z *containerSerz) Msgsize() (s int) {
- s = 1 + 2 + msgp.Uint8Size + 2 + z.r.Msgsize()
- return
-}
-
-// Deprecated: DecodeMsg implements msgp.Decodable
-func (z *contype) DecodeMsg(dc *msgp.Reader) (err error) {
- {
- var zajw uint8
- zajw, err = dc.ReadUint8()
- (*z) = contype(zajw)
- }
- if err != nil {
- return
- }
- return
-}
-
-// Deprecated: EncodeMsg implements msgp.Encodable
-func (z contype) EncodeMsg(en *msgp.Writer) (err error) {
- err = en.WriteUint8(uint8(z))
- if err != nil {
- return
- }
- return
-}
-
-// Deprecated: MarshalMsg implements msgp.Marshaler
-func (z contype) MarshalMsg(b []byte) (o []byte, err error) {
- o = msgp.Require(b, z.Msgsize())
- o = msgp.AppendUint8(o, uint8(z))
- return
-}
-
-// Deprecated: UnmarshalMsg implements msgp.Unmarshaler
-func (z *contype) UnmarshalMsg(bts []byte) (o []byte, err error) {
- {
- var zwht uint8
- zwht, bts, err = msgp.ReadUint8Bytes(bts)
- (*z) = contype(zwht)
- }
- if err != nil {
- return
- }
- o = bts
- return
-}
-
-// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (z contype) Msgsize() (s int) {
- s = msgp.Uint8Size
- return
-}
-
-// Deprecated: DecodeMsg implements msgp.Decodable
-func (z *roaringArray) DecodeMsg(dc *msgp.Reader) (err error) {
- var field []byte
- _ = field
- var zlqf uint32
- zlqf, err = dc.ReadMapHeader()
- if err != nil {
- return
- }
- for zlqf > 0 {
- zlqf--
- field, err = dc.ReadMapKeyPtr()
- if err != nil {
- return
- }
- switch msgp.UnsafeString(field) {
- case "keys":
- var zdaf uint32
- zdaf, err = dc.ReadArrayHeader()
- if err != nil {
- return
- }
- if cap(z.keys) >= int(zdaf) {
- z.keys = (z.keys)[:zdaf]
- } else {
- z.keys = make([]uint16, zdaf)
- }
- for zhct := range z.keys {
- z.keys[zhct], err = dc.ReadUint16()
- if err != nil {
- return
- }
- }
- case "needCopyOnWrite":
- var zpks uint32
- zpks, err = dc.ReadArrayHeader()
- if err != nil {
- return
- }
- if cap(z.needCopyOnWrite) >= int(zpks) {
- z.needCopyOnWrite = (z.needCopyOnWrite)[:zpks]
- } else {
- z.needCopyOnWrite = make([]bool, zpks)
- }
- for zcua := range z.needCopyOnWrite {
- z.needCopyOnWrite[zcua], err = dc.ReadBool()
- if err != nil {
- return
- }
- }
- case "copyOnWrite":
- z.copyOnWrite, err = dc.ReadBool()
- if err != nil {
- return
- }
- case "conserz":
- var zjfb uint32
- zjfb, err = dc.ReadArrayHeader()
- if err != nil {
- return
- }
- if cap(z.conserz) >= int(zjfb) {
- z.conserz = (z.conserz)[:zjfb]
- } else {
- z.conserz = make([]containerSerz, zjfb)
- }
- for zxhx := range z.conserz {
- var zcxo uint32
- zcxo, err = dc.ReadMapHeader()
- if err != nil {
- return
- }
- for zcxo > 0 {
- zcxo--
- field, err = dc.ReadMapKeyPtr()
- if err != nil {
- return
- }
- switch msgp.UnsafeString(field) {
- case "t":
- {
- var zeff uint8
- zeff, err = dc.ReadUint8()
- z.conserz[zxhx].t = contype(zeff)
- }
- if err != nil {
- return
- }
- case "r":
- err = z.conserz[zxhx].r.DecodeMsg(dc)
- if err != nil {
- return
- }
- default:
- err = dc.Skip()
- if err != nil {
- return
- }
- }
- }
- }
- default:
- err = dc.Skip()
- if err != nil {
- return
- }
- }
- }
- return
-}
-
-// Deprecated: EncodeMsg implements msgp.Encodable
-func (z *roaringArray) EncodeMsg(en *msgp.Writer) (err error) {
- // map header, size 4
- // write "keys"
- err = en.Append(0x84, 0xa4, 0x6b, 0x65, 0x79, 0x73)
- if err != nil {
- return err
- }
- err = en.WriteArrayHeader(uint32(len(z.keys)))
- if err != nil {
- return
- }
- for zhct := range z.keys {
- err = en.WriteUint16(z.keys[zhct])
- if err != nil {
- return
- }
- }
- // write "needCopyOnWrite"
- err = en.Append(0xaf, 0x6e, 0x65, 0x65, 0x64, 0x43, 0x6f, 0x70, 0x79, 0x4f, 0x6e, 0x57, 0x72, 0x69, 0x74, 0x65)
- if err != nil {
- return err
- }
- err = en.WriteArrayHeader(uint32(len(z.needCopyOnWrite)))
- if err != nil {
- return
- }
- for zcua := range z.needCopyOnWrite {
- err = en.WriteBool(z.needCopyOnWrite[zcua])
- if err != nil {
- return
- }
- }
- // write "copyOnWrite"
- err = en.Append(0xab, 0x63, 0x6f, 0x70, 0x79, 0x4f, 0x6e, 0x57, 0x72, 0x69, 0x74, 0x65)
- if err != nil {
- return err
- }
- err = en.WriteBool(z.copyOnWrite)
- if err != nil {
- return
- }
- // write "conserz"
- err = en.Append(0xa7, 0x63, 0x6f, 0x6e, 0x73, 0x65, 0x72, 0x7a)
- if err != nil {
- return err
- }
- err = en.WriteArrayHeader(uint32(len(z.conserz)))
- if err != nil {
- return
- }
- for zxhx := range z.conserz {
- // map header, size 2
- // write "t"
- err = en.Append(0x82, 0xa1, 0x74)
- if err != nil {
- return err
- }
- err = en.WriteUint8(uint8(z.conserz[zxhx].t))
- if err != nil {
- return
- }
- // write "r"
- err = en.Append(0xa1, 0x72)
- if err != nil {
- return err
- }
- err = z.conserz[zxhx].r.EncodeMsg(en)
- if err != nil {
- return
- }
- }
- return
-}
-
-// Deprecated: MarshalMsg implements msgp.Marshaler
-func (z *roaringArray) MarshalMsg(b []byte) (o []byte, err error) {
- o = msgp.Require(b, z.Msgsize())
- // map header, size 4
- // string "keys"
- o = append(o, 0x84, 0xa4, 0x6b, 0x65, 0x79, 0x73)
- o = msgp.AppendArrayHeader(o, uint32(len(z.keys)))
- for zhct := range z.keys {
- o = msgp.AppendUint16(o, z.keys[zhct])
- }
- // string "needCopyOnWrite"
- o = append(o, 0xaf, 0x6e, 0x65, 0x65, 0x64, 0x43, 0x6f, 0x70, 0x79, 0x4f, 0x6e, 0x57, 0x72, 0x69, 0x74, 0x65)
- o = msgp.AppendArrayHeader(o, uint32(len(z.needCopyOnWrite)))
- for zcua := range z.needCopyOnWrite {
- o = msgp.AppendBool(o, z.needCopyOnWrite[zcua])
- }
- // string "copyOnWrite"
- o = append(o, 0xab, 0x63, 0x6f, 0x70, 0x79, 0x4f, 0x6e, 0x57, 0x72, 0x69, 0x74, 0x65)
- o = msgp.AppendBool(o, z.copyOnWrite)
- // string "conserz"
- o = append(o, 0xa7, 0x63, 0x6f, 0x6e, 0x73, 0x65, 0x72, 0x7a)
- o = msgp.AppendArrayHeader(o, uint32(len(z.conserz)))
- for zxhx := range z.conserz {
- // map header, size 2
- // string "t"
- o = append(o, 0x82, 0xa1, 0x74)
- o = msgp.AppendUint8(o, uint8(z.conserz[zxhx].t))
- // string "r"
- o = append(o, 0xa1, 0x72)
- o, err = z.conserz[zxhx].r.MarshalMsg(o)
- if err != nil {
- return
- }
- }
- return
-}
-
-// Deprecated: UnmarshalMsg implements msgp.Unmarshaler
-func (z *roaringArray) UnmarshalMsg(bts []byte) (o []byte, err error) {
- var field []byte
- _ = field
- var zrsw uint32
- zrsw, bts, err = msgp.ReadMapHeaderBytes(bts)
- if err != nil {
- return
- }
- for zrsw > 0 {
- zrsw--
- field, bts, err = msgp.ReadMapKeyZC(bts)
- if err != nil {
- return
- }
- switch msgp.UnsafeString(field) {
- case "keys":
- var zxpk uint32
- zxpk, bts, err = msgp.ReadArrayHeaderBytes(bts)
- if err != nil {
- return
- }
- if cap(z.keys) >= int(zxpk) {
- z.keys = (z.keys)[:zxpk]
- } else {
- z.keys = make([]uint16, zxpk)
- }
- for zhct := range z.keys {
- z.keys[zhct], bts, err = msgp.ReadUint16Bytes(bts)
- if err != nil {
- return
- }
- }
- case "needCopyOnWrite":
- var zdnj uint32
- zdnj, bts, err = msgp.ReadArrayHeaderBytes(bts)
- if err != nil {
- return
- }
- if cap(z.needCopyOnWrite) >= int(zdnj) {
- z.needCopyOnWrite = (z.needCopyOnWrite)[:zdnj]
- } else {
- z.needCopyOnWrite = make([]bool, zdnj)
- }
- for zcua := range z.needCopyOnWrite {
- z.needCopyOnWrite[zcua], bts, err = msgp.ReadBoolBytes(bts)
- if err != nil {
- return
- }
- }
- case "copyOnWrite":
- z.copyOnWrite, bts, err = msgp.ReadBoolBytes(bts)
- if err != nil {
- return
- }
- case "conserz":
- var zobc uint32
- zobc, bts, err = msgp.ReadArrayHeaderBytes(bts)
- if err != nil {
- return
- }
- if cap(z.conserz) >= int(zobc) {
- z.conserz = (z.conserz)[:zobc]
- } else {
- z.conserz = make([]containerSerz, zobc)
- }
- for zxhx := range z.conserz {
- var zsnv uint32
- zsnv, bts, err = msgp.ReadMapHeaderBytes(bts)
- if err != nil {
- return
- }
- for zsnv > 0 {
- zsnv--
- field, bts, err = msgp.ReadMapKeyZC(bts)
- if err != nil {
- return
- }
- switch msgp.UnsafeString(field) {
- case "t":
- {
- var zkgt uint8
- zkgt, bts, err = msgp.ReadUint8Bytes(bts)
- z.conserz[zxhx].t = contype(zkgt)
- }
- if err != nil {
- return
- }
- case "r":
- bts, err = z.conserz[zxhx].r.UnmarshalMsg(bts)
- if err != nil {
- return
- }
- default:
- bts, err = msgp.Skip(bts)
- if err != nil {
- return
- }
- }
- }
- }
- default:
- bts, err = msgp.Skip(bts)
- if err != nil {
- return
- }
- }
- }
- o = bts
- return
-}
-
-// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (z *roaringArray) Msgsize() (s int) {
- s = 1 + 5 + msgp.ArrayHeaderSize + (len(z.keys) * (msgp.Uint16Size)) + 16 + msgp.ArrayHeaderSize + (len(z.needCopyOnWrite) * (msgp.BoolSize)) + 12 + msgp.BoolSize + 8 + msgp.ArrayHeaderSize
- for zxhx := range z.conserz {
- s += 1 + 2 + msgp.Uint8Size + 2 + z.conserz[zxhx].r.Msgsize()
- }
- return
-}
diff --git a/vendor/github.com/RoaringBitmap/roaring/runcontainer.go b/vendor/github.com/RoaringBitmap/roaring/runcontainer.go
index eed894b..7098ba2 100644
--- a/vendor/github.com/RoaringBitmap/roaring/runcontainer.go
+++ b/vendor/github.com/RoaringBitmap/roaring/runcontainer.go
@@ -44,16 +44,11 @@ import (
"unsafe"
)
-//go:generate msgp -unexported
-
// runContainer16 does run-length encoding of sets of
// uint16 integers.
type runContainer16 struct {
- iv []interval16
- card int64
-
- // avoid allocation during search
- myOpts searchOptions `msg:"-"`
+ // iv is a slice of sorted, non-overlapping, non-adjacent intervals.
+ iv []interval16
}
// interval16 is the internal to runContainer16
@@ -76,8 +71,8 @@ func newInterval16Range(start, last uint16) interval16 {
}
// runlen returns the count of integers in the interval.
-func (iv interval16) runlen() int64 {
- return int64(iv.length) + 1
+func (iv interval16) runlen() int {
+ return int(iv.length) + 1
}
func (iv interval16) last() uint16 {
@@ -120,8 +115,6 @@ func (p uint16Slice) Less(i, j int) bool { return p[i] < p[j] }
// Swap swaps elements i and j.
func (p uint16Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
-//msgp:ignore addHelper
-
// addHelper helps build a runContainer16.
type addHelper16 struct {
runstart uint16
@@ -201,7 +194,6 @@ func newRunContainer16FromVals(alreadySorted bool, vals ...uint16) *runContainer
ah.storeIval(ah.runstart, ah.runlen)
}
rc.iv = ah.m
- rc.card = int64(ah.actuallyAdded)
return rc
}
@@ -262,10 +254,8 @@ func newRunContainer16FromBitmapContainer(bc *bitmapContainer) *runContainer16 {
}
-//
// newRunContainer16FromArray populates a new
// runContainer16 from the contents of arr.
-//
func newRunContainer16FromArray(arr *arrayContainer) *runContainer16 {
// keep this in sync with newRunContainer16FromVals above
@@ -291,7 +281,6 @@ func newRunContainer16FromArray(arr *arrayContainer) *runContainer16 {
ah.storeIval(ah.runstart, ah.runlen)
}
rc.iv = ah.m
- rc.card = int64(ah.actuallyAdded)
return rc
}
@@ -308,7 +297,6 @@ func (rc *runContainer16) set(alreadySorted bool, vals ...uint16) {
rc2 := newRunContainer16FromVals(alreadySorted, vals...)
un := rc.union(rc2)
rc.iv = un.iv
- rc.card = 0
}
// canMerge returns true iff the intervals
@@ -316,10 +304,10 @@ func (rc *runContainer16) set(alreadySorted bool, vals ...uint16) {
// contiguous and so can be merged into
// a single interval.
func canMerge16(a, b interval16) bool {
- if int64(a.last())+1 < int64(b.start) {
+ if int(a.last())+1 < int(b.start) {
return false
}
- return int64(b.last())+1 >= int64(a.start)
+ return int(b.last())+1 >= int(a.start)
}
// haveOverlap differs from canMerge in that
@@ -328,10 +316,10 @@ func canMerge16(a, b interval16) bool {
// it would be the empty set, and we return
// false).
func haveOverlap16(a, b interval16) bool {
- if int64(a.last())+1 <= int64(b.start) {
+ if int(a.last())+1 <= int(b.start) {
return false
}
- return int64(b.last())+1 > int64(a.start)
+ return int(b.last())+1 > int(a.start)
}
// mergeInterval16s joins a and b into a
@@ -392,11 +380,11 @@ func (rc *runContainer16) union(b *runContainer16) *runContainer16 {
var m []interval16
- alim := int64(len(rc.iv))
- blim := int64(len(b.iv))
+ alim := int(len(rc.iv))
+ blim := int(len(b.iv))
- var na int64 // next from a
- var nb int64 // next from b
+ var na int // next from a
+ var nb int // next from b
// merged holds the current merge output, which might
// get additional merges before being appended to m.
@@ -416,12 +404,12 @@ func (rc *runContainer16) union(b *runContainer16) *runContainer16 {
mergedUpdated := false
if canMerge16(cura, merged) {
merged = mergeInterval16s(cura, merged)
- na = rc.indexOfIntervalAtOrAfter(int64(merged.last())+1, na+1)
+ na = rc.indexOfIntervalAtOrAfter(int(merged.last())+1, na+1)
mergedUpdated = true
}
if canMerge16(curb, merged) {
merged = mergeInterval16s(curb, merged)
- nb = b.indexOfIntervalAtOrAfter(int64(merged.last())+1, nb+1)
+ nb = b.indexOfIntervalAtOrAfter(int(merged.last())+1, nb+1)
mergedUpdated = true
}
if !mergedUpdated {
@@ -444,8 +432,8 @@ func (rc *runContainer16) union(b *runContainer16) *runContainer16 {
} else {
merged = mergeInterval16s(cura, curb)
mergedUsed = true
- na = rc.indexOfIntervalAtOrAfter(int64(merged.last())+1, na+1)
- nb = b.indexOfIntervalAtOrAfter(int64(merged.last())+1, nb+1)
+ na = rc.indexOfIntervalAtOrAfter(int(merged.last())+1, na+1)
+ nb = b.indexOfIntervalAtOrAfter(int(merged.last())+1, nb+1)
}
}
}
@@ -464,7 +452,7 @@ func (rc *runContainer16) union(b *runContainer16) *runContainer16 {
cura = rc.iv[na]
if canMerge16(cura, merged) {
merged = mergeInterval16s(cura, merged)
- na = rc.indexOfIntervalAtOrAfter(int64(merged.last())+1, na+1)
+ na = rc.indexOfIntervalAtOrAfter(int(merged.last())+1, na+1)
} else {
break aAdds
}
@@ -478,7 +466,7 @@ func (rc *runContainer16) union(b *runContainer16) *runContainer16 {
curb = b.iv[nb]
if canMerge16(curb, merged) {
merged = mergeInterval16s(curb, merged)
- nb = b.indexOfIntervalAtOrAfter(int64(merged.last())+1, nb+1)
+ nb = b.indexOfIntervalAtOrAfter(int(merged.last())+1, nb+1)
} else {
break bAdds
}
@@ -500,17 +488,17 @@ func (rc *runContainer16) union(b *runContainer16) *runContainer16 {
}
// unionCardinality returns the cardinality of the merger of two runContainer16s, the union of rc and b.
-func (rc *runContainer16) unionCardinality(b *runContainer16) uint64 {
+func (rc *runContainer16) unionCardinality(b *runContainer16) uint {
// rc is also known as 'a' here, but golint insisted we
// call it rc for consistency with the rest of the methods.
- answer := uint64(0)
+ answer := uint(0)
- alim := int64(len(rc.iv))
- blim := int64(len(b.iv))
+ alim := int(len(rc.iv))
+ blim := int(len(b.iv))
- var na int64 // next from a
- var nb int64 // next from b
+ var na int // next from a
+ var nb int // next from b
// merged holds the current merge output, which might
// get additional merges before being appended to m.
@@ -530,18 +518,18 @@ func (rc *runContainer16) unionCardinality(b *runContainer16) uint64 {
mergedUpdated := false
if canMerge16(cura, merged) {
merged = mergeInterval16s(cura, merged)
- na = rc.indexOfIntervalAtOrAfter(int64(merged.last())+1, na+1)
+ na = rc.indexOfIntervalAtOrAfter(int(merged.last())+1, na+1)
mergedUpdated = true
}
if canMerge16(curb, merged) {
merged = mergeInterval16s(curb, merged)
- nb = b.indexOfIntervalAtOrAfter(int64(merged.last())+1, nb+1)
+ nb = b.indexOfIntervalAtOrAfter(int(merged.last())+1, nb+1)
mergedUpdated = true
}
if !mergedUpdated {
// we know that merged is disjoint from cura and curb
//m = append(m, merged)
- answer += uint64(merged.last()) - uint64(merged.start) + 1
+ answer += uint(merged.last()) - uint(merged.start) + 1
mergedUsed = false
}
continue
@@ -550,19 +538,19 @@ func (rc *runContainer16) unionCardinality(b *runContainer16) uint64 {
// !mergedUsed
if !canMerge16(cura, curb) {
if cura.start < curb.start {
- answer += uint64(cura.last()) - uint64(cura.start) + 1
+ answer += uint(cura.last()) - uint(cura.start) + 1
//m = append(m, cura)
na++
} else {
- answer += uint64(curb.last()) - uint64(curb.start) + 1
+ answer += uint(curb.last()) - uint(curb.start) + 1
//m = append(m, curb)
nb++
}
} else {
merged = mergeInterval16s(cura, curb)
mergedUsed = true
- na = rc.indexOfIntervalAtOrAfter(int64(merged.last())+1, na+1)
- nb = b.indexOfIntervalAtOrAfter(int64(merged.last())+1, nb+1)
+ na = rc.indexOfIntervalAtOrAfter(int(merged.last())+1, na+1)
+ nb = b.indexOfIntervalAtOrAfter(int(merged.last())+1, nb+1)
}
}
}
@@ -581,7 +569,7 @@ func (rc *runContainer16) unionCardinality(b *runContainer16) uint64 {
cura = rc.iv[na]
if canMerge16(cura, merged) {
merged = mergeInterval16s(cura, merged)
- na = rc.indexOfIntervalAtOrAfter(int64(merged.last())+1, na+1)
+ na = rc.indexOfIntervalAtOrAfter(int(merged.last())+1, na+1)
} else {
break aAdds
}
@@ -595,7 +583,7 @@ func (rc *runContainer16) unionCardinality(b *runContainer16) uint64 {
curb = b.iv[nb]
if canMerge16(curb, merged) {
merged = mergeInterval16s(curb, merged)
- nb = b.indexOfIntervalAtOrAfter(int64(merged.last())+1, nb+1)
+ nb = b.indexOfIntervalAtOrAfter(int(merged.last())+1, nb+1)
} else {
break bAdds
}
@@ -604,23 +592,20 @@ func (rc *runContainer16) unionCardinality(b *runContainer16) uint64 {
}
//m = append(m, merged)
- answer += uint64(merged.last()) - uint64(merged.start) + 1
+ answer += uint(merged.last()) - uint(merged.start) + 1
}
for _, r := range rc.iv[na:] {
- answer += uint64(r.last()) - uint64(r.start) + 1
+ answer += uint(r.last()) - uint(r.start) + 1
}
for _, r := range b.iv[nb:] {
- answer += uint64(r.last()) - uint64(r.start) + 1
+ answer += uint(r.last()) - uint(r.start) + 1
}
return answer
}
// indexOfIntervalAtOrAfter is a helper for union.
-func (rc *runContainer16) indexOfIntervalAtOrAfter(key int64, startIndex int64) int64 {
- rc.myOpts.startIndex = startIndex
- rc.myOpts.endxIndex = 0
-
- w, already, _ := rc.search(key, &rc.myOpts)
+func (rc *runContainer16) indexOfIntervalAtOrAfter(key int, startIndex int) int {
+ w, already, _ := rc.searchRange(key, startIndex, 0)
if already {
return w
}
@@ -632,8 +617,8 @@ func (rc *runContainer16) indexOfIntervalAtOrAfter(key int64, startIndex int64)
func (rc *runContainer16) intersect(b *runContainer16) *runContainer16 {
a := rc
- numa := int64(len(a.iv))
- numb := int64(len(b.iv))
+ numa := int(len(a.iv))
+ numb := int(len(b.iv))
res := &runContainer16{}
if numa == 0 || numb == 0 {
return res
@@ -647,21 +632,21 @@ func (rc *runContainer16) intersect(b *runContainer16) *runContainer16 {
var output []interval16
- var acuri int64
- var bcuri int64
+ var acuri int
+ var bcuri int
- astart := int64(a.iv[acuri].start)
- bstart := int64(b.iv[bcuri].start)
+ astart := int(a.iv[acuri].start)
+ bstart := int(b.iv[bcuri].start)
var intersection interval16
- var leftoverstart int64
+ var leftoverstart int
var isOverlap, isLeftoverA, isLeftoverB bool
var done bool
toploop:
for acuri < numa && bcuri < numb {
isOverlap, isLeftoverA, isLeftoverB, leftoverstart, intersection =
- intersectWithLeftover16(astart, int64(a.iv[acuri].last()), bstart, int64(b.iv[bcuri].last()))
+ intersectWithLeftover16(astart, int(a.iv[acuri].last()), bstart, int(b.iv[bcuri].last()))
if !isOverlap {
switch {
@@ -670,17 +655,14 @@ toploop:
if done {
break toploop
}
- astart = int64(a.iv[acuri].start)
+ astart = int(a.iv[acuri].start)
case astart > bstart:
bcuri, done = b.findNextIntervalThatIntersectsStartingFrom(bcuri+1, astart)
if done {
break toploop
}
- bstart = int64(b.iv[bcuri].start)
-
- //default:
- // panic("impossible that astart == bstart, since !isOverlap")
+ bstart = int(b.iv[bcuri].start)
}
} else {
@@ -695,7 +677,7 @@ toploop:
if bcuri >= numb {
break toploop
}
- bstart = int64(b.iv[bcuri].start)
+ bstart = int(b.iv[bcuri].start)
case isLeftoverB:
// note that we change bstart without advancing bcuri,
// since we need to capture any 2ndary intersections with b.iv[bcuri]
@@ -704,27 +686,23 @@ toploop:
if acuri >= numa {
break toploop
}
- astart = int64(a.iv[acuri].start)
+ astart = int(a.iv[acuri].start)
default:
// neither had leftover, both completely consumed
- // optionally, assert for sanity:
- //if a.iv[acuri].endx != b.iv[bcuri].endx {
- // panic("huh? should only be possible that endx agree now!")
- //}
// advance to next a interval
acuri++
if acuri >= numa {
break toploop
}
- astart = int64(a.iv[acuri].start)
+ astart = int(a.iv[acuri].start)
// advance to next b interval
bcuri++
if bcuri >= numb {
break toploop
}
- bstart = int64(b.iv[bcuri].start)
+ bstart = int(b.iv[bcuri].start)
}
}
} // end for toploop
@@ -739,12 +717,12 @@ toploop:
// intersectCardinality returns the cardinality of the
// intersection of rc (also known as 'a') and b.
-func (rc *runContainer16) intersectCardinality(b *runContainer16) int64 {
- answer := int64(0)
+func (rc *runContainer16) intersectCardinality(b *runContainer16) int {
+ answer := int(0)
a := rc
- numa := int64(len(a.iv))
- numb := int64(len(b.iv))
+ numa := int(len(a.iv))
+ numb := int(len(b.iv))
if numa == 0 || numb == 0 {
return 0
}
@@ -755,14 +733,14 @@ func (rc *runContainer16) intersectCardinality(b *runContainer16) int64 {
}
}
- var acuri int64
- var bcuri int64
+ var acuri int
+ var bcuri int
- astart := int64(a.iv[acuri].start)
- bstart := int64(b.iv[bcuri].start)
+ astart := int(a.iv[acuri].start)
+ bstart := int(b.iv[bcuri].start)
var intersection interval16
- var leftoverstart int64
+ var leftoverstart int
var isOverlap, isLeftoverA, isLeftoverB bool
var done bool
pass := 0
@@ -771,7 +749,7 @@ toploop:
pass++
isOverlap, isLeftoverA, isLeftoverB, leftoverstart, intersection =
- intersectWithLeftover16(astart, int64(a.iv[acuri].last()), bstart, int64(b.iv[bcuri].last()))
+ intersectWithLeftover16(astart, int(a.iv[acuri].last()), bstart, int(b.iv[bcuri].last()))
if !isOverlap {
switch {
@@ -780,22 +758,19 @@ toploop:
if done {
break toploop
}
- astart = int64(a.iv[acuri].start)
+ astart = int(a.iv[acuri].start)
case astart > bstart:
bcuri, done = b.findNextIntervalThatIntersectsStartingFrom(bcuri+1, astart)
if done {
break toploop
}
- bstart = int64(b.iv[bcuri].start)
-
- //default:
- // panic("impossible that astart == bstart, since !isOverlap")
+ bstart = int(b.iv[bcuri].start)
}
} else {
// isOverlap
- answer += int64(intersection.last()) - int64(intersection.start) + 1
+ answer += int(intersection.last()) - int(intersection.start) + 1
switch {
case isLeftoverA:
// note that we change astart without advancing acuri,
@@ -805,7 +780,7 @@ toploop:
if bcuri >= numb {
break toploop
}
- bstart = int64(b.iv[bcuri].start)
+ bstart = int(b.iv[bcuri].start)
case isLeftoverB:
// note that we change bstart without advancing bcuri,
// since we need to capture any 2ndary intersections with b.iv[bcuri]
@@ -814,27 +789,23 @@ toploop:
if acuri >= numa {
break toploop
}
- astart = int64(a.iv[acuri].start)
+ astart = int(a.iv[acuri].start)
default:
// neither had leftover, both completely consumed
- // optionally, assert for sanity:
- //if a.iv[acuri].endx != b.iv[bcuri].endx {
- // panic("huh? should only be possible that endx agree now!")
- //}
// advance to next a interval
acuri++
if acuri >= numa {
break toploop
}
- astart = int64(a.iv[acuri].start)
+ astart = int(a.iv[acuri].start)
// advance to next b interval
bcuri++
if bcuri >= numb {
break toploop
}
- bstart = int64(b.iv[bcuri].start)
+ bstart = int(b.iv[bcuri].start)
}
}
} // end for toploop
@@ -844,7 +815,7 @@ toploop:
// get returns true iff key is in the container.
func (rc *runContainer16) contains(key uint16) bool {
- _, in, _ := rc.search(int64(key), nil)
+ _, in, _ := rc.search(int(key))
return in
}
@@ -853,22 +824,7 @@ func (rc *runContainer16) numIntervals() int {
return len(rc.iv)
}
-// searchOptions allows us to accelerate search with
-// prior knowledge of (mostly lower) bounds. This is used by Union
-// and Intersect.
-type searchOptions struct {
- // start here instead of at 0
- startIndex int64
-
- // upper bound instead of len(rc.iv);
- // endxIndex == 0 means ignore the bound and use
- // endxIndex == n ==len(rc.iv) which is also
- // naturally the default for search()
- // when opt = nil.
- endxIndex int64
-}
-
-// search returns alreadyPresent to indicate if the
+// searchRange returns alreadyPresent to indicate if the
// key is already in one of our interval16s.
//
// If key is alreadyPresent, then whichInterval16 tells
@@ -877,39 +833,30 @@ type searchOptions struct {
// If key is not already present, then whichInterval16 is
// set as follows:
//
-// a) whichInterval16 == len(rc.iv)-1 if key is beyond our
-// last interval16 in rc.iv;
+// a) whichInterval16 == len(rc.iv)-1 if key is beyond our
+// last interval16 in rc.iv;
//
-// b) whichInterval16 == -1 if key is before our first
-// interval16 in rc.iv;
+// b) whichInterval16 == -1 if key is before our first
+// interval16 in rc.iv;
//
-// c) whichInterval16 is set to the minimum index of rc.iv
-// which comes strictly before the key;
-// so rc.iv[whichInterval16].last < key,
-// and if whichInterval16+1 exists, then key < rc.iv[whichInterval16+1].start
-// (Note that whichInterval16+1 won't exist when
-// whichInterval16 is the last interval.)
+// c) whichInterval16 is set to the minimum index of rc.iv
+// which comes strictly before the key;
+// so rc.iv[whichInterval16].last < key,
+// and if whichInterval16+1 exists, then key < rc.iv[whichInterval16+1].start
+// (Note that whichInterval16+1 won't exist when
+// whichInterval16 is the last interval.)
//
// runContainer16.search always returns whichInterval16 < len(rc.iv).
//
-// If not nil, opts can be used to further restrict
-// the search space.
-//
-func (rc *runContainer16) search(key int64, opts *searchOptions) (whichInterval16 int64, alreadyPresent bool, numCompares int) {
- n := int64(len(rc.iv))
+// The search space is from startIndex to endxIndex. If endxIndex is set to zero, then there
+// no upper bound.
+func (rc *runContainer16) searchRange(key int, startIndex int, endxIndex int) (whichInterval16 int, alreadyPresent bool, numCompares int) {
+ n := int(len(rc.iv))
if n == 0 {
return -1, false, 0
}
-
- startIndex := int64(0)
- endxIndex := n
- if opts != nil {
- startIndex = opts.startIndex
-
- // let endxIndex == 0 mean no effect
- if opts.endxIndex > 0 {
- endxIndex = opts.endxIndex
- }
+ if endxIndex == 0 {
+ endxIndex = n
}
// sort.Search returns the smallest index i
@@ -927,7 +874,7 @@ func (rc *runContainer16) search(key int64, opts *searchOptions) (whichInterval1
h := i + (j-i)/2 // avoid overflow when computing h as the bisector
// i <= h < j
numCompares++
- if !(key < int64(rc.iv[h].start)) {
+ if !(key < int(rc.iv[h].start)) {
i = h + 1
} else {
j = h
@@ -947,7 +894,7 @@ func (rc *runContainer16) search(key int64, opts *searchOptions) (whichInterval1
if below == n {
// all falses => key is >= start of all interval16s
// ... so does it belong to the last interval16?
- if key < int64(rc.iv[n-1].last())+1 {
+ if key < int(rc.iv[n-1].last())+1 {
// yes, it belongs to the last interval16
alreadyPresent = true
return
@@ -968,7 +915,7 @@ func (rc *runContainer16) search(key int64, opts *searchOptions) (whichInterval1
// key is < rc.iv[below].start
// is key in below-1 interval16?
- if key >= int64(rc.iv[below-1].start) && key < int64(rc.iv[below-1].last())+1 {
+ if key >= int(rc.iv[below-1].start) && key < int(rc.iv[below-1].last())+1 {
// yes, it is. key is in below-1 interval16.
alreadyPresent = true
return
@@ -979,28 +926,54 @@ func (rc *runContainer16) search(key int64, opts *searchOptions) (whichInterval1
return
}
-// cardinality returns the count of the integers stored in the
-// runContainer16.
-func (rc *runContainer16) cardinality() int64 {
- if len(rc.iv) == 0 {
- rc.card = 0
- return 0
- }
- if rc.card > 0 {
- return rc.card // already cached
- }
+// search returns alreadyPresent to indicate if the
+// key is already in one of our interval16s.
+//
+// If key is alreadyPresent, then whichInterval16 tells
+// you where.
+//
+// If key is not already present, then whichInterval16 is
+// set as follows:
+//
+// a) whichInterval16 == len(rc.iv)-1 if key is beyond our
+// last interval16 in rc.iv;
+//
+// b) whichInterval16 == -1 if key is before our first
+// interval16 in rc.iv;
+//
+// c) whichInterval16 is set to the minimum index of rc.iv
+// which comes strictly before the key;
+// so rc.iv[whichInterval16].last < key,
+// and if whichInterval16+1 exists, then key < rc.iv[whichInterval16+1].start
+// (Note that whichInterval16+1 won't exist when
+// whichInterval16 is the last interval.)
+//
+// runContainer16.search always returns whichInterval16 < len(rc.iv).
+func (rc *runContainer16) search(key int) (whichInterval16 int, alreadyPresent bool, numCompares int) {
+ return rc.searchRange(key, 0, 0)
+}
+
+// getCardinality returns the count of the integers stored in the
+// runContainer16. The running complexity depends on the size
+// of the container.
+func (rc *runContainer16) getCardinality() int {
// have to compute it
- var n int64
+ n := 0
for _, p := range rc.iv {
n += p.runlen()
}
- rc.card = n // cache it
return n
}
+// isEmpty returns true if the container is empty.
+// It runs in constant time.
+func (rc *runContainer16) isEmpty() bool {
+ return len(rc.iv) == 0
+}
+
// AsSlice decompresses the contents into a []uint16 slice.
func (rc *runContainer16) AsSlice() []uint16 {
- s := make([]uint16, rc.cardinality())
+ s := make([]uint16, rc.getCardinality())
j := 0
for _, p := range rc.iv {
for i := p.start; i <= p.last(); i++ {
@@ -1018,7 +991,6 @@ func newRunContainer16() *runContainer16 {
// newRunContainer16CopyIv creates a run container, initializing
// with a copy of the supplied iv slice.
-//
func newRunContainer16CopyIv(iv []interval16) *runContainer16 {
rc := &runContainer16{
iv: make([]interval16, len(iv)),
@@ -1035,7 +1007,6 @@ func (rc *runContainer16) Clone() *runContainer16 {
// newRunContainer16TakeOwnership returns a new runContainer16
// backed by the provided iv slice, which we will
// assume exclusive control over from now on.
-//
func newRunContainer16TakeOwnership(iv []interval16) *runContainer16 {
rc := &runContainer16{
iv: iv,
@@ -1070,19 +1041,15 @@ func (rc *runContainer16) Add(k uint16) (wasNew bool) {
// but note that some unit tests use this method to build up test
// runcontainers without calling runOptimize
- k64 := int64(k)
+ k64 := int(k)
- index, present, _ := rc.search(k64, nil)
+ index, present, _ := rc.search(k64)
if present {
return // already there
}
wasNew = true
- // increment card if it is cached already
- if rc.card > 0 {
- rc.card++
- }
- n := int64(len(rc.iv))
+ n := int(len(rc.iv))
if index == -1 {
// we may need to extend the first run
if n > 0 {
@@ -1099,7 +1066,7 @@ func (rc *runContainer16) Add(k uint16) (wasNew bool) {
// are we off the end? handle both index == n and index == n-1:
if index >= n-1 {
- if int64(rc.iv[n-1].last())+1 == k64 {
+ if int(rc.iv[n-1].last())+1 == k64 {
rc.iv[n-1].length++
return
}
@@ -1118,7 +1085,7 @@ func (rc *runContainer16) Add(k uint16) (wasNew bool) {
right := index + 1
// are we fusing left and right by adding k?
- if int64(rc.iv[left].last())+1 == k64 && int64(rc.iv[right].start) == k64+1 {
+ if int(rc.iv[left].last())+1 == k64 && int(rc.iv[right].start) == k64+1 {
// fuse into left
rc.iv[left].length = rc.iv[right].last() - rc.iv[left].start
// remove redundant right
@@ -1127,14 +1094,14 @@ func (rc *runContainer16) Add(k uint16) (wasNew bool) {
}
// are we an addition to left?
- if int64(rc.iv[left].last())+1 == k64 {
+ if int(rc.iv[left].last())+1 == k64 {
// yes
rc.iv[left].length++
return
}
// are we an addition to right?
- if int64(rc.iv[right].start) == k64+1 {
+ if int(rc.iv[right].start) == k64+1 {
// yes
rc.iv[right].start = k
rc.iv[right].length++
@@ -1147,238 +1114,237 @@ func (rc *runContainer16) Add(k uint16) (wasNew bool) {
return
}
-//msgp:ignore runIterator
-
-// runIterator16 advice: you must call Next() at least once
-// before calling Cur(); and you should call HasNext()
-// before calling Next() to insure there are contents.
+// runIterator16 advice: you must call hasNext()
+// before calling next()/peekNext() to insure there are contents.
type runIterator16 struct {
rc *runContainer16
- curIndex int64
+ curIndex int
curPosInIndex uint16
- curSeq int64
}
// newRunIterator16 returns a new empty run container.
func (rc *runContainer16) newRunIterator16() *runIterator16 {
- return &runIterator16{rc: rc, curIndex: -1}
+ return &runIterator16{rc: rc, curIndex: 0, curPosInIndex: 0}
}
-// HasNext returns false if calling Next will panic. It
+func (rc *runContainer16) iterate(cb func(x uint16) bool) bool {
+ iterator := runIterator16{rc, 0, 0}
+
+ for iterator.hasNext() {
+ if !cb(iterator.next()) {
+ return false
+ }
+ }
+
+ return true
+}
+
+// hasNext returns false if calling next will panic. It
// returns true when there is at least one more value
// available in the iteration sequence.
func (ri *runIterator16) hasNext() bool {
- if len(ri.rc.iv) == 0 {
- return false
- }
- if ri.curIndex == -1 {
- return true
- }
- return ri.curSeq+1 < ri.rc.cardinality()
+ return int(len(ri.rc.iv)) > ri.curIndex+1 ||
+ (int(len(ri.rc.iv)) == ri.curIndex+1 && ri.rc.iv[ri.curIndex].length >= ri.curPosInIndex)
}
-// cur returns the current value pointed to by the iterator.
-func (ri *runIterator16) cur() uint16 {
+// next returns the next value in the iteration sequence.
+func (ri *runIterator16) next() uint16 {
+ next := ri.rc.iv[ri.curIndex].start + ri.curPosInIndex
+
+ if ri.curPosInIndex == ri.rc.iv[ri.curIndex].length {
+ ri.curPosInIndex = 0
+ ri.curIndex++
+ } else {
+ ri.curPosInIndex++
+ }
+
+ return next
+}
+
+// peekNext returns the next value in the iteration sequence without advancing the iterator
+func (ri *runIterator16) peekNext() uint16 {
return ri.rc.iv[ri.curIndex].start + ri.curPosInIndex
}
-// Next returns the next value in the iteration sequence.
-func (ri *runIterator16) next() uint16 {
- if !ri.hasNext() {
- panic("no Next available")
+// advanceIfNeeded advances as long as the next value is smaller than minval
+func (ri *runIterator16) advanceIfNeeded(minval uint16) {
+ if !ri.hasNext() || ri.peekNext() >= minval {
+ return
}
- if ri.curIndex >= int64(len(ri.rc.iv)) {
- panic("runIterator.Next() going beyond what is available")
- }
- if ri.curIndex == -1 {
- // first time is special
- ri.curIndex = 0
+
+ // interval cannot be -1 because of minval > peekNext
+ interval, isPresent, _ := ri.rc.searchRange(int(minval), ri.curIndex, int(len(ri.rc.iv)))
+
+ // if the minval is present, set the curPosIndex at the right position
+ if isPresent {
+ ri.curIndex = interval
+ ri.curPosInIndex = minval - ri.rc.iv[ri.curIndex].start
} else {
- ri.curPosInIndex++
- if int64(ri.rc.iv[ri.curIndex].start)+int64(ri.curPosInIndex) == int64(ri.rc.iv[ri.curIndex].last())+1 {
- ri.curPosInIndex = 0
- ri.curIndex++
- }
- ri.curSeq++
+ // otherwise interval is set to to the minimum index of rc.iv
+ // which comes strictly before the key, that's why we set the next interval
+ ri.curIndex = interval + 1
+ ri.curPosInIndex = 0
}
- return ri.cur()
}
-// remove removes the element that the iterator
-// is on from the run container. You can use
-// Cur if you want to double check what is about
-// to be deleted.
-func (ri *runIterator16) remove() uint16 {
- n := ri.rc.cardinality()
- if n == 0 {
- panic("runIterator.Remove called on empty runContainer16")
- }
- cur := ri.cur()
-
- ri.rc.deleteAt(&ri.curIndex, &ri.curPosInIndex, &ri.curSeq)
- return cur
-}
-
-// runReverseIterator16 advice: you must call next() at least once
-// before calling cur(); and you should call hasNext()
+// runReverseIterator16 advice: you must call hasNext()
// before calling next() to insure there are contents.
type runReverseIterator16 struct {
rc *runContainer16
- curIndex int64 // index into rc.iv
+ curIndex int // index into rc.iv
curPosInIndex uint16 // offset in rc.iv[curIndex]
- curSeq int64 // 0->cardinality, performance optimization in hasNext()
}
// newRunReverseIterator16 returns a new empty run iterator.
func (rc *runContainer16) newRunReverseIterator16() *runReverseIterator16 {
- return &runReverseIterator16{rc: rc, curIndex: -2}
+ index := int(len(rc.iv)) - 1
+ pos := uint16(0)
+
+ if index >= 0 {
+ pos = rc.iv[index].length
+ }
+
+ return &runReverseIterator16{
+ rc: rc,
+ curIndex: index,
+ curPosInIndex: pos,
+ }
}
// hasNext returns false if calling next will panic. It
// returns true when there is at least one more value
// available in the iteration sequence.
func (ri *runReverseIterator16) hasNext() bool {
- if len(ri.rc.iv) == 0 {
- return false
- }
- if ri.curIndex == -2 {
- return true
- }
- return ri.rc.cardinality()-ri.curSeq > 1
-}
-
-// cur returns the current value pointed to by the iterator.
-func (ri *runReverseIterator16) cur() uint16 {
- return ri.rc.iv[ri.curIndex].start + ri.curPosInIndex
+ return ri.curIndex > 0 || ri.curIndex == 0 && ri.curPosInIndex >= 0
}
// next returns the next value in the iteration sequence.
func (ri *runReverseIterator16) next() uint16 {
- if !ri.hasNext() {
- panic("no next available")
- }
- if ri.curIndex == -1 {
- panic("runReverseIterator.next() going beyond what is available")
- }
- if ri.curIndex == -2 {
- // first time is special
- ri.curIndex = int64(len(ri.rc.iv)) - 1
- ri.curPosInIndex = ri.rc.iv[ri.curIndex].length
+ next := ri.rc.iv[ri.curIndex].start + ri.curPosInIndex
+
+ if ri.curPosInIndex > 0 {
+ ri.curPosInIndex--
} else {
- if ri.curPosInIndex > 0 {
- ri.curPosInIndex--
- } else {
- ri.curIndex--
+ ri.curIndex--
+
+ if ri.curIndex >= 0 {
ri.curPosInIndex = ri.rc.iv[ri.curIndex].length
}
- ri.curSeq++
}
- return ri.cur()
+
+ return next
}
-// remove removes the element that the iterator
-// is on from the run container. You can use
-// cur if you want to double check what is about
-// to be deleted.
-func (ri *runReverseIterator16) remove() uint16 {
- n := ri.rc.cardinality()
- if n == 0 {
- panic("runReverseIterator.Remove called on empty runContainer16")
- }
- cur := ri.cur()
-
- ri.rc.deleteAt(&ri.curIndex, &ri.curPosInIndex, &ri.curSeq)
- return cur
-}
-
-type manyRunIterator16 struct {
- rc *runContainer16
- curIndex int64
- curPosInIndex uint16
- curSeq int64
-}
-
-func (rc *runContainer16) newManyRunIterator16() *manyRunIterator16 {
- return &manyRunIterator16{rc: rc, curIndex: -1}
-}
-
-func (ri *manyRunIterator16) hasNext() bool {
- if len(ri.rc.iv) == 0 {
- return false
- }
- if ri.curIndex == -1 {
- return true
- }
- return ri.curSeq+1 < ri.rc.cardinality()
+func (rc *runContainer16) newManyRunIterator16() *runIterator16 {
+ return rc.newRunIterator16()
}
// hs are the high bits to include to avoid needing to reiterate over the buffer in NextMany
-func (ri *manyRunIterator16) nextMany(hs uint32, buf []uint32) int {
+func (ri *runIterator16) nextMany(hs uint32, buf []uint32) int {
n := 0
+
if !ri.hasNext() {
return n
}
+
// start and end are inclusive
for n < len(buf) {
- if ri.curIndex == -1 || int(ri.rc.iv[ri.curIndex].length-ri.curPosInIndex) <= 0 {
+ moreVals := 0
+
+ if ri.rc.iv[ri.curIndex].length >= ri.curPosInIndex {
+ // add as many as you can from this seq
+ moreVals = minOfInt(int(ri.rc.iv[ri.curIndex].length-ri.curPosInIndex)+1, len(buf)-n)
+ base := uint32(ri.rc.iv[ri.curIndex].start+ri.curPosInIndex) | hs
+
+ // allows BCE
+ buf2 := buf[n : n+moreVals]
+ for i := range buf2 {
+ buf2[i] = base + uint32(i)
+ }
+
+ // update values
+ n += moreVals
+ }
+
+ if moreVals+int(ri.curPosInIndex) > int(ri.rc.iv[ri.curIndex].length) {
ri.curPosInIndex = 0
ri.curIndex++
- if ri.curIndex == int64(len(ri.rc.iv)) {
+
+ if ri.curIndex == int(len(ri.rc.iv)) {
break
}
- buf[n] = uint32(ri.rc.iv[ri.curIndex].start) | hs
- if ri.curIndex != 0 {
- ri.curSeq++
- }
- n++
- // not strictly necessarily due to len(buf)-n min check, but saves some work
- continue
+ } else {
+ ri.curPosInIndex += uint16(moreVals) //moreVals always fits in uint16
}
- // add as many as you can from this seq
- moreVals := minOfInt(int(ri.rc.iv[ri.curIndex].length-ri.curPosInIndex), len(buf)-n)
-
- base := uint32(ri.rc.iv[ri.curIndex].start+ri.curPosInIndex+1) | hs
-
- // allows BCE
- buf2 := buf[n : n+moreVals]
- for i := range buf2 {
- buf2[i] = base + uint32(i)
- }
-
- // update values
- ri.curPosInIndex += uint16(moreVals) //moreVals always fits in uint16
- ri.curSeq += int64(moreVals)
- n += moreVals
}
+
+ return n
+}
+
+func (ri *runIterator16) nextMany64(hs uint64, buf []uint64) int {
+ n := 0
+
+ if !ri.hasNext() {
+ return n
+ }
+
+ // start and end are inclusive
+ for n < len(buf) {
+ moreVals := 0
+
+ if ri.rc.iv[ri.curIndex].length >= ri.curPosInIndex {
+ // add as many as you can from this seq
+ moreVals = minOfInt(int(ri.rc.iv[ri.curIndex].length-ri.curPosInIndex)+1, len(buf)-n)
+ base := uint64(ri.rc.iv[ri.curIndex].start+ri.curPosInIndex) | hs
+
+ // allows BCE
+ buf2 := buf[n : n+moreVals]
+ for i := range buf2 {
+ buf2[i] = base + uint64(i)
+ }
+
+ // update values
+ n += moreVals
+ }
+
+ if moreVals+int(ri.curPosInIndex) > int(ri.rc.iv[ri.curIndex].length) {
+ ri.curPosInIndex = 0
+ ri.curIndex++
+
+ if ri.curIndex == int(len(ri.rc.iv)) {
+ break
+ }
+ } else {
+ ri.curPosInIndex += uint16(moreVals) //moreVals always fits in uint16
+ }
+ }
+
return n
}
// remove removes key from the container.
func (rc *runContainer16) removeKey(key uint16) (wasPresent bool) {
- var index int64
- var curSeq int64
- index, wasPresent, _ = rc.search(int64(key), nil)
+ var index int
+ index, wasPresent, _ = rc.search(int(key))
if !wasPresent {
return // already removed, nothing to do.
}
pos := key - rc.iv[index].start
- rc.deleteAt(&index, &pos, &curSeq)
+ rc.deleteAt(&index, &pos)
return
}
// internal helper functions
-func (rc *runContainer16) deleteAt(curIndex *int64, curPosInIndex *uint16, curSeq *int64) {
- rc.card--
- *curSeq--
+func (rc *runContainer16) deleteAt(curIndex *int, curPosInIndex *uint16) {
ci := *curIndex
pos := *curPosInIndex
// are we first, last, or in the middle of our interval16?
switch {
case pos == 0:
- if int64(rc.iv[ci].length) == 0 {
+ if int(rc.iv[ci].length) == 0 {
// our interval disappears
rc.iv = append(rc.iv[:ci], rc.iv[ci+1:]...)
// curIndex stays the same, since the delete did
@@ -1399,8 +1365,8 @@ func (rc *runContainer16) deleteAt(curIndex *int64, curPosInIndex *uint16, curSe
// split into two, adding an interval16
new0 := newInterval16Range(rc.iv[ci].start, rc.iv[ci].start+*curPosInIndex-1)
- new1start := int64(rc.iv[ci].start+*curPosInIndex) + 1
- if new1start > int64(MaxUint16) {
+ new1start := int(rc.iv[ci].start+*curPosInIndex) + 1
+ if new1start > int(MaxUint16) {
panic("overflow?!?!")
}
new1 := newInterval16Range(uint16(new1start), rc.iv[ci].last())
@@ -1413,14 +1379,14 @@ func (rc *runContainer16) deleteAt(curIndex *int64, curPosInIndex *uint16, curSe
}
-func have4Overlap16(astart, alast, bstart, blast int64) bool {
+func have4Overlap16(astart, alast, bstart, blast int) bool {
if alast+1 <= bstart {
return false
}
return blast+1 > astart
}
-func intersectWithLeftover16(astart, alast, bstart, blast int64) (isOverlap, isLeftoverA, isLeftoverB bool, leftoverstart int64, intersection interval16) {
+func intersectWithLeftover16(astart, alast, bstart, blast int) (isOverlap, isLeftoverA, isLeftoverB bool, leftoverstart int, intersection interval16) {
if !have4Overlap16(astart, alast, bstart, blast) {
return
}
@@ -1450,17 +1416,13 @@ func intersectWithLeftover16(astart, alast, bstart, blast int64) (isOverlap, isL
return
}
-func (rc *runContainer16) findNextIntervalThatIntersectsStartingFrom(startIndex int64, key int64) (index int64, done bool) {
-
- rc.myOpts.startIndex = startIndex
- rc.myOpts.endxIndex = 0
-
- w, _, _ := rc.search(key, &rc.myOpts)
+func (rc *runContainer16) findNextIntervalThatIntersectsStartingFrom(startIndex int, key int) (index int, done bool) {
+ w, _, _ := rc.searchRange(key, startIndex, 0)
// rc.search always returns w < len(rc.iv)
if w < startIndex {
// not found and comes before lower bound startIndex,
// so just use the lower bound.
- if startIndex == int64(len(rc.iv)) {
+ if startIndex == int(len(rc.iv)) {
// also this bump up means that we are done
return startIndex, true
}
@@ -1478,25 +1440,6 @@ func sliceToString16(m []interval16) string {
return s
}
-// selectInt16 returns the j-th value in the container.
-// We panic of j is out of bounds.
-func (rc *runContainer16) selectInt16(j uint16) int {
- n := rc.cardinality()
- if int64(j) > n {
- panic(fmt.Sprintf("Cannot select %v since Cardinality is %v", j, n))
- }
-
- var offset int64
- for k := range rc.iv {
- nextOffset := offset + rc.iv[k].runlen() + 1
- if nextOffset > int64(j) {
- return int(int64(rc.iv[k].start) + (int64(j) - offset))
- }
- offset = nextOffset
- }
- panic(fmt.Sprintf("Cannot select %v since Cardinality is %v", j, n))
-}
-
// helper for invert
func (rc *runContainer16) invertlastInterval(origin uint16, lastIdx int) []interval16 {
cur := rc.iv[lastIdx]
@@ -1528,7 +1471,7 @@ func (rc *runContainer16) invert() *runContainer16 {
case 1:
return &runContainer16{iv: rc.invertlastInterval(0, 0)}
}
- var invstart int64
+ var invstart int
ult := ni - 1
for i, cur := range rc.iv {
if i == ult {
@@ -1547,7 +1490,7 @@ func (rc *runContainer16) invert() *runContainer16 {
if cur.start > 0 {
m = append(m, newInterval16Range(uint16(invstart), cur.start-1))
}
- invstart = int64(cur.last() + 1)
+ invstart = int(cur.last() + 1)
}
return &runContainer16{iv: m}
}
@@ -1560,7 +1503,7 @@ func (iv interval16) isSuperSetOf(b interval16) bool {
return iv.start <= b.start && b.last() <= iv.last()
}
-func (iv interval16) subtractInterval(del interval16) (left []interval16, delcount int64) {
+func (iv interval16) subtractInterval(del interval16) (left []interval16, delcount int) {
isect, isEmpty := intersectInterval16s(iv, del)
if isEmpty {
@@ -1585,7 +1528,7 @@ func (iv interval16) subtractInterval(del interval16) (left []interval16, delcou
func (rc *runContainer16) isubtract(del interval16) {
origiv := make([]interval16, len(rc.iv))
copy(origiv, rc.iv)
- n := int64(len(rc.iv))
+ n := int(len(rc.iv))
if n == 0 {
return // already done.
}
@@ -1596,9 +1539,8 @@ func (rc *runContainer16) isubtract(del interval16) {
}
// INVAR there is some intersection between rc and del
- istart, startAlready, _ := rc.search(int64(del.start), nil)
- ilast, lastAlready, _ := rc.search(int64(del.last()), nil)
- rc.card = -1
+ istart, startAlready, _ := rc.search(int(del.start))
+ ilast, lastAlready, _ := rc.search(int(del.last()))
if istart == -1 {
if ilast == n-1 && !lastAlready {
rc.iv = nil
@@ -1613,8 +1555,8 @@ func (rc *runContainer16) isubtract(del interval16) {
// would overwrite values in iv b/c res0 can have len 2. so
// write to origiv instead.
lost := 1 + ilast - istart
- changeSize := int64(len(res0)) - lost
- newSize := int64(len(rc.iv)) + changeSize
+ changeSize := int(len(res0)) - lost
+ newSize := int(len(rc.iv)) + changeSize
// rc.iv = append(pre, caboose...)
// return
@@ -1622,19 +1564,19 @@ func (rc *runContainer16) isubtract(del interval16) {
if ilast != istart {
res1, _ := rc.iv[ilast].subtractInterval(del)
res0 = append(res0, res1...)
- changeSize = int64(len(res0)) - lost
- newSize = int64(len(rc.iv)) + changeSize
+ changeSize = int(len(res0)) - lost
+ newSize = int(len(rc.iv)) + changeSize
}
switch {
case changeSize < 0:
// shrink
- copy(rc.iv[istart+int64(len(res0)):], rc.iv[ilast+1:])
- copy(rc.iv[istart:istart+int64(len(res0))], res0)
+ copy(rc.iv[istart+int(len(res0)):], rc.iv[ilast+1:])
+ copy(rc.iv[istart:istart+int(len(res0))], res0)
rc.iv = rc.iv[:newSize]
return
case changeSize == 0:
// stay the same
- copy(rc.iv[istart:istart+int64(len(res0))], res0)
+ copy(rc.iv[istart:istart+int(len(res0))], res0)
return
default:
// changeSize > 0 is only possible when ilast == istart.
@@ -1691,7 +1633,7 @@ func (rc *runContainer16) isubtract(del interval16) {
// INVAR: ilast < n-1
lost := ilast - istart
changeSize := -lost
- newSize := int64(len(rc.iv)) + changeSize
+ newSize := int(len(rc.iv)) + changeSize
if changeSize != 0 {
copy(rc.iv[ilast+1+changeSize:], rc.iv[ilast+1:])
}
@@ -1708,8 +1650,8 @@ func (rc *runContainer16) isubtract(del interval16) {
rc.iv[istart] = res0[0]
}
lost := 1 + (ilast - istart)
- changeSize := int64(len(res0)) - lost
- newSize := int64(len(rc.iv)) + changeSize
+ changeSize := int(len(res0)) - lost
+ newSize := int(len(rc.iv)) + changeSize
if changeSize != 0 {
copy(rc.iv[ilast+1+changeSize:], rc.iv[ilast+1:])
}
@@ -1720,8 +1662,8 @@ func (rc *runContainer16) isubtract(del interval16) {
// we can only shrink or stay the same size
res1, _ := rc.iv[ilast].subtractInterval(del)
lost := ilast - istart
- changeSize := int64(len(res1)) - lost
- newSize := int64(len(rc.iv)) + changeSize
+ changeSize := int(len(res1)) - lost
+ newSize := int(len(rc.iv)) + changeSize
if changeSize != 0 {
// move the tail first to make room for res1
copy(rc.iv[ilast+1+changeSize:], rc.iv[ilast+1:])
@@ -1925,8 +1867,6 @@ func (rc *runContainer16) iand(a container) container {
}
func (rc *runContainer16) inplaceIntersect(rc2 *runContainer16) container {
- // TODO: optimize by doing less allocation, possibly?
- // sect will be new
sect := rc.intersect(rc2)
*rc = *sect
return rc
@@ -1980,20 +1920,21 @@ func (rc *runContainer16) andNot(a container) container {
panic("unsupported container type")
}
-func (rc *runContainer16) fillLeastSignificant16bits(x []uint32, i int, mask uint32) {
- k := 0
- var val int64
+func (rc *runContainer16) fillLeastSignificant16bits(x []uint32, i int, mask uint32) int {
+ k := i
+ var val int
for _, p := range rc.iv {
n := p.runlen()
- for j := int64(0); j < n; j++ {
- val = int64(p.start) + j
- x[k+i] = uint32(val) | mask
+ for j := int(0); j < n; j++ {
+ val = int(p.start) + j
+ x[k] = uint32(val) | mask
k++
}
}
+ return k
}
-func (rc *runContainer16) getShortIterator() shortIterable {
+func (rc *runContainer16) getShortIterator() shortPeekable {
return rc.newRunIterator16()
}
@@ -2009,8 +1950,11 @@ func (rc *runContainer16) getManyIterator() manyIterable {
// is still abe to express 2^16 because it is an int not an uint16.
func (rc *runContainer16) iaddRange(firstOfRange, endx int) container {
- if firstOfRange >= endx {
- panic(fmt.Sprintf("invalid %v = endx >= firstOfRange", endx))
+ if firstOfRange > endx {
+ panic(fmt.Sprintf("invalid %v = endx > firstOfRange", endx))
+ }
+ if firstOfRange == endx {
+ return rc
}
addme := newRunContainer16TakeOwnership([]interval16{
{
@@ -2024,10 +1968,13 @@ func (rc *runContainer16) iaddRange(firstOfRange, endx int) container {
// remove the values in the range [firstOfRange,endx)
func (rc *runContainer16) iremoveRange(firstOfRange, endx int) container {
- if firstOfRange >= endx {
+ if firstOfRange > endx {
panic(fmt.Sprintf("request to iremove empty set [%v, %v),"+
" nothing to do.", firstOfRange, endx))
- //return rc
+ }
+ // empty removal
+ if firstOfRange == endx {
+ return rc
}
x := newInterval16Range(uint16(firstOfRange), uint16(endx-1))
rc.isubtract(x)
@@ -2036,8 +1983,8 @@ func (rc *runContainer16) iremoveRange(firstOfRange, endx int) container {
// not flip the values in the range [firstOfRange,endx)
func (rc *runContainer16) not(firstOfRange, endx int) container {
- if firstOfRange >= endx {
- panic(fmt.Sprintf("invalid %v = endx >= firstOfRange = %v", endx, firstOfRange))
+ if firstOfRange > endx {
+ panic(fmt.Sprintf("invalid %v = endx > firstOfRange = %v", endx, firstOfRange))
}
return rc.Not(firstOfRange, endx)
@@ -2054,11 +2001,10 @@ func (rc *runContainer16) not(firstOfRange, endx int) container {
// Current routine is correct but
// makes 2 more passes through the arrays than should be
// strictly necessary. Measure both ways though--this may not matter.
-//
func (rc *runContainer16) Not(firstOfRange, endx int) *runContainer16 {
- if firstOfRange >= endx {
- panic(fmt.Sprintf("invalid %v = endx >= firstOfRange == %v", endx, firstOfRange))
+ if firstOfRange > endx {
+ panic(fmt.Sprintf("invalid %v = endx > firstOfRange == %v", endx, firstOfRange))
}
if firstOfRange >= endx {
@@ -2196,9 +2142,21 @@ func (rc *runContainer16) orBitmapContainerCardinality(bc *bitmapContainer) int
// orArray finds the union of rc and ac.
func (rc *runContainer16) orArray(ac *arrayContainer) container {
- bc1 := newBitmapContainerFromRun(rc)
- bc2 := ac.toBitmapContainer()
- return bc1.orBitmap(bc2)
+ if ac.isEmpty() {
+ return rc.clone()
+ }
+ if rc.isEmpty() {
+ return ac.clone()
+ }
+ intervals, cardMinusOne := runArrayUnionToRuns(rc, ac)
+ result := newRunContainer16TakeOwnership(intervals)
+ if len(intervals) >= 2048 && cardMinusOne >= arrayDefaultMaxSize {
+ return newBitmapContainerFromRun(result)
+ }
+ if len(intervals)*2 > 1+int(cardMinusOne) {
+ return result.toArrayContainer()
+ }
+ return result
}
// orArray finds the union of rc and ac.
@@ -2223,8 +2181,8 @@ func (rc *runContainer16) ior(a container) container {
func (rc *runContainer16) inplaceUnion(rc2 *runContainer16) container {
for _, p := range rc2.iv {
- last := int64(p.last())
- for i := int64(p.start); i <= last; i++ {
+ last := int(p.last())
+ for i := int(p.start); i <= last; i++ {
rc.Add(uint16(i))
}
}
@@ -2241,13 +2199,88 @@ func (rc *runContainer16) iorBitmapContainer(bc *bitmapContainer) container {
}
func (rc *runContainer16) iorArray(ac *arrayContainer) container {
- it := ac.getShortIterator()
- for it.hasNext() {
- rc.Add(it.next())
+ if rc.isEmpty() {
+ return ac.clone()
+ }
+ if ac.isEmpty() {
+ return rc
+ }
+ var cardMinusOne uint16
+ //TODO: perform the union algorithm in-place using rc.iv
+ // this can be done with methods like the in-place array container union
+ // but maybe lazily moving the remaining elements back.
+ rc.iv, cardMinusOne = runArrayUnionToRuns(rc, ac)
+ if len(rc.iv) >= 2048 && cardMinusOne >= arrayDefaultMaxSize {
+ return newBitmapContainerFromRun(rc)
+ }
+ if len(rc.iv)*2 > 1+int(cardMinusOne) {
+ return rc.toArrayContainer()
}
return rc
}
+func runArrayUnionToRuns(rc *runContainer16, ac *arrayContainer) ([]interval16, uint16) {
+ pos1 := 0
+ pos2 := 0
+ length1 := len(ac.content)
+ length2 := len(rc.iv)
+ target := make([]interval16, 0, len(rc.iv))
+ // have to find the first range
+ // options are
+ // 1. from array container
+ // 2. from run container
+ var previousInterval interval16
+ var cardMinusOne uint16
+ if ac.content[0] < rc.iv[0].start {
+ previousInterval.start = ac.content[0]
+ previousInterval.length = 0
+ pos1++
+ } else {
+ previousInterval.start = rc.iv[0].start
+ previousInterval.length = rc.iv[0].length
+ pos2++
+ }
+
+ for pos1 < length1 || pos2 < length2 {
+ if pos1 < length1 {
+ s1 := ac.content[pos1]
+ if s1 <= previousInterval.start+previousInterval.length {
+ pos1++
+ continue
+ }
+ if previousInterval.last() < MaxUint16 && previousInterval.last()+1 == s1 {
+ previousInterval.length++
+ pos1++
+ continue
+ }
+ }
+ if pos2 < length2 {
+ range2 := rc.iv[pos2]
+ if range2.start <= previousInterval.last() || range2.start > 0 && range2.start-1 == previousInterval.last() {
+ pos2++
+ if previousInterval.last() < range2.last() {
+ previousInterval.length = range2.last() - previousInterval.start
+ }
+ continue
+ }
+ }
+ cardMinusOne += previousInterval.length + 1
+ target = append(target, previousInterval)
+ if pos2 == length2 || pos1 < length1 && ac.content[pos1] < rc.iv[pos2].start {
+ previousInterval.start = ac.content[pos1]
+ previousInterval.length = 0
+ pos1++
+ } else {
+ previousInterval = rc.iv[pos2]
+ pos2++
+ }
+ }
+ cardMinusOne += previousInterval.length
+ target = append(target, previousInterval)
+
+ return target, cardMinusOne
+}
+
// lazyIOR is described (not yet implemented) in
// this nice note from @lemire on
// https://github.com/RoaringBitmap/roaring/pull/70#issuecomment-263613737
@@ -2290,7 +2323,6 @@ func (rc *runContainer16) iorArray(ac *arrayContainer) container {
// the backing array, and then you write
// the answer at the beginning. What this
// trick does is minimize memory allocations.
-//
func (rc *runContainer16) lazyIOR(a container) container {
// not lazy at the moment
return rc.ior(a)
@@ -2303,9 +2335,9 @@ func (rc *runContainer16) lazyOR(a container) container {
}
func (rc *runContainer16) intersects(a container) bool {
- // TODO: optimize by doing inplace/less allocation, possibly?
+ // TODO: optimize by doing inplace/less allocation
isect := rc.and(a)
- return isect.getCardinality() > 0
+ return !isect.isEmpty()
}
func (rc *runContainer16) xor(a container) container {
@@ -2334,44 +2366,51 @@ func (rc *runContainer16) iandNot(a container) container {
// flip the values in the range [firstOfRange,endx)
func (rc *runContainer16) inot(firstOfRange, endx int) container {
- if firstOfRange >= endx {
- panic(fmt.Sprintf("invalid %v = endx >= firstOfRange = %v", endx, firstOfRange))
+ if firstOfRange > endx {
+ panic(fmt.Sprintf("invalid %v = endx > firstOfRange = %v", endx, firstOfRange))
+ }
+ if firstOfRange > endx {
+ return rc
}
// TODO: minimize copies, do it all inplace; not() makes a copy.
rc = rc.Not(firstOfRange, endx)
return rc
}
-func (rc *runContainer16) getCardinality() int {
- return int(rc.cardinality())
-}
-
func (rc *runContainer16) rank(x uint16) int {
- n := int64(len(rc.iv))
- xx := int64(x)
- w, already, _ := rc.search(xx, nil)
+ n := int(len(rc.iv))
+ xx := int(x)
+ w, already, _ := rc.search(xx)
if w < 0 {
return 0
}
if !already && w == n-1 {
return rc.getCardinality()
}
- var rnk int64
+ var rnk int
if !already {
- for i := int64(0); i <= w; i++ {
+ for i := int(0); i <= w; i++ {
rnk += rc.iv[i].runlen()
}
return int(rnk)
}
- for i := int64(0); i < w; i++ {
+ for i := int(0); i < w; i++ {
rnk += rc.iv[i].runlen()
}
- rnk += int64(x-rc.iv[w].start) + 1
+ rnk += int(x-rc.iv[w].start) + 1
return int(rnk)
}
func (rc *runContainer16) selectInt(x uint16) int {
- return rc.selectInt16(x)
+ var offset int
+ for k := range rc.iv {
+ nextOffset := offset + rc.iv[k].runlen()
+ if nextOffset > int(x) {
+ return int(int(rc.iv[k].start) + (int(x) - offset))
+ }
+ offset = nextOffset
+ }
+ panic("cannot select x")
}
func (rc *runContainer16) andNotRunContainer16(b *runContainer16) container {
@@ -2449,11 +2488,9 @@ func (rc *runContainer16) xorBitmap(bc *bitmapContainer) container {
// convert to bitmap or array *if needed*
func (rc *runContainer16) toEfficientContainer() container {
-
- // runContainer16SerializedSizeInBytes(numRuns)
sizeAsRunContainer := rc.getSizeInBytes()
sizeAsBitmapContainer := bitmapContainerSizeInBytes()
- card := int(rc.cardinality())
+ card := rc.getCardinality()
sizeAsArrayContainer := arrayContainerSizeInBytes(card)
if sizeAsRunContainer <= minOfInt(sizeAsBitmapContainer, sizeAsArrayContainer) {
return rc
@@ -2538,9 +2575,27 @@ func (rc *runContainer16) serializedSizeInBytes() int {
return 2 + len(rc.iv)*4
}
-func (rc *runContainer16) addOffset(x uint16) []container {
- low := newRunContainer16()
- high := newRunContainer16()
+func (rc *runContainer16) addOffset(x uint16) (container, container) {
+ var low, high *runContainer16
+
+ if len(rc.iv) == 0 {
+ return nil, nil
+ }
+
+ first := uint32(rc.iv[0].start) + uint32(x)
+ if highbits(first) == 0 {
+ // Some elements will fall into low part, allocate a container.
+ // Checking the first one is enough because they are ordered.
+ low = newRunContainer16()
+ }
+ last := uint32(rc.iv[len(rc.iv)-1].start)
+ last += uint32(rc.iv[len(rc.iv)-1].length)
+ last += uint32(x)
+ if highbits(last) > 0 {
+ // Some elements will fall into high part, allocate a container.
+ // Checking the last one is enough because they are ordered.
+ high = newRunContainer16()
+ }
for _, iv := range rc.iv {
val := int(iv.start) + int(x)
@@ -2556,5 +2611,14 @@ func (rc *runContainer16) addOffset(x uint16) []container {
high.iv = append(high.iv, interval16{uint16(val & 0xffff), iv.length})
}
}
- return []container{low, high}
+
+ // Ensure proper nil interface.
+ if low == nil {
+ return nil, high
+ }
+ if high == nil {
+ return low, nil
+ }
+
+ return low, high
}
diff --git a/vendor/github.com/RoaringBitmap/roaring/runcontainer_gen.go b/vendor/github.com/RoaringBitmap/roaring/runcontainer_gen.go
deleted file mode 100644
index cb91052..0000000
--- a/vendor/github.com/RoaringBitmap/roaring/runcontainer_gen.go
+++ /dev/null
@@ -1,1126 +0,0 @@
-package roaring
-
-// NOTE: THIS FILE WAS PRODUCED BY THE
-// MSGP CODE GENERATION TOOL (github.com/tinylib/msgp)
-// DO NOT EDIT
-
-import "github.com/tinylib/msgp/msgp"
-
-// Deprecated: DecodeMsg implements msgp.Decodable
-func (z *addHelper16) DecodeMsg(dc *msgp.Reader) (err error) {
- var field []byte
- _ = field
- var zbai uint32
- zbai, err = dc.ReadMapHeader()
- if err != nil {
- return
- }
- for zbai > 0 {
- zbai--
- field, err = dc.ReadMapKeyPtr()
- if err != nil {
- return
- }
- switch msgp.UnsafeString(field) {
- case "runstart":
- z.runstart, err = dc.ReadUint16()
- if err != nil {
- return
- }
- case "runlen":
- z.runlen, err = dc.ReadUint16()
- if err != nil {
- return
- }
- case "actuallyAdded":
- z.actuallyAdded, err = dc.ReadUint16()
- if err != nil {
- return
- }
- case "m":
- var zcmr uint32
- zcmr, err = dc.ReadArrayHeader()
- if err != nil {
- return
- }
- if cap(z.m) >= int(zcmr) {
- z.m = (z.m)[:zcmr]
- } else {
- z.m = make([]interval16, zcmr)
- }
- for zxvk := range z.m {
- var zajw uint32
- zajw, err = dc.ReadMapHeader()
- if err != nil {
- return
- }
- for zajw > 0 {
- zajw--
- field, err = dc.ReadMapKeyPtr()
- if err != nil {
- return
- }
- switch msgp.UnsafeString(field) {
- case "start":
- z.m[zxvk].start, err = dc.ReadUint16()
- if err != nil {
- return
- }
- case "last":
- z.m[zxvk].length, err = dc.ReadUint16()
- z.m[zxvk].length -= z.m[zxvk].start
- if err != nil {
- return
- }
- default:
- err = dc.Skip()
- if err != nil {
- return
- }
- }
- }
- }
- case "rc":
- if dc.IsNil() {
- err = dc.ReadNil()
- if err != nil {
- return
- }
- z.rc = nil
- } else {
- if z.rc == nil {
- z.rc = new(runContainer16)
- }
- var zwht uint32
- zwht, err = dc.ReadMapHeader()
- if err != nil {
- return
- }
- for zwht > 0 {
- zwht--
- field, err = dc.ReadMapKeyPtr()
- if err != nil {
- return
- }
- switch msgp.UnsafeString(field) {
- case "iv":
- var zhct uint32
- zhct, err = dc.ReadArrayHeader()
- if err != nil {
- return
- }
- if cap(z.rc.iv) >= int(zhct) {
- z.rc.iv = (z.rc.iv)[:zhct]
- } else {
- z.rc.iv = make([]interval16, zhct)
- }
- for zbzg := range z.rc.iv {
- var zcua uint32
- zcua, err = dc.ReadMapHeader()
- if err != nil {
- return
- }
- for zcua > 0 {
- zcua--
- field, err = dc.ReadMapKeyPtr()
- if err != nil {
- return
- }
- switch msgp.UnsafeString(field) {
- case "start":
- z.rc.iv[zbzg].start, err = dc.ReadUint16()
- if err != nil {
- return
- }
- case "last":
- z.rc.iv[zbzg].length, err = dc.ReadUint16()
- z.rc.iv[zbzg].length -= z.rc.iv[zbzg].start
- if err != nil {
- return
- }
- default:
- err = dc.Skip()
- if err != nil {
- return
- }
- }
- }
- }
- case "card":
- z.rc.card, err = dc.ReadInt64()
- if err != nil {
- return
- }
- default:
- err = dc.Skip()
- if err != nil {
- return
- }
- }
- }
- }
- default:
- err = dc.Skip()
- if err != nil {
- return
- }
- }
- }
- return
-}
-
-// Deprecated: EncodeMsg implements msgp.Encodable
-func (z *addHelper16) EncodeMsg(en *msgp.Writer) (err error) {
- // map header, size 5
- // write "runstart"
- err = en.Append(0x85, 0xa8, 0x72, 0x75, 0x6e, 0x73, 0x74, 0x61, 0x72, 0x74)
- if err != nil {
- return err
- }
- err = en.WriteUint16(z.runstart)
- if err != nil {
- return
- }
- // write "runlen"
- err = en.Append(0xa6, 0x72, 0x75, 0x6e, 0x6c, 0x65, 0x6e)
- if err != nil {
- return err
- }
- err = en.WriteUint16(z.runlen)
- if err != nil {
- return
- }
- // write "actuallyAdded"
- err = en.Append(0xad, 0x61, 0x63, 0x74, 0x75, 0x61, 0x6c, 0x6c, 0x79, 0x41, 0x64, 0x64, 0x65, 0x64)
- if err != nil {
- return err
- }
- err = en.WriteUint16(z.actuallyAdded)
- if err != nil {
- return
- }
- // write "m"
- err = en.Append(0xa1, 0x6d)
- if err != nil {
- return err
- }
- err = en.WriteArrayHeader(uint32(len(z.m)))
- if err != nil {
- return
- }
- for zxvk := range z.m {
- // map header, size 2
- // write "start"
- err = en.Append(0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74)
- if err != nil {
- return err
- }
- err = en.WriteUint16(z.m[zxvk].start)
- if err != nil {
- return
- }
- // write "last"
- err = en.Append(0xa4, 0x6c, 0x61, 0x73, 0x74)
- if err != nil {
- return err
- }
- err = en.WriteUint16(z.m[zxvk].last())
- if err != nil {
- return
- }
- }
- // write "rc"
- err = en.Append(0xa2, 0x72, 0x63)
- if err != nil {
- return err
- }
- if z.rc == nil {
- err = en.WriteNil()
- if err != nil {
- return
- }
- } else {
- // map header, size 2
- // write "iv"
- err = en.Append(0x82, 0xa2, 0x69, 0x76)
- if err != nil {
- return err
- }
- err = en.WriteArrayHeader(uint32(len(z.rc.iv)))
- if err != nil {
- return
- }
- for zbzg := range z.rc.iv {
- // map header, size 2
- // write "start"
- err = en.Append(0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74)
- if err != nil {
- return err
- }
- err = en.WriteUint16(z.rc.iv[zbzg].start)
- if err != nil {
- return
- }
- // write "last"
- err = en.Append(0xa4, 0x6c, 0x61, 0x73, 0x74)
- if err != nil {
- return err
- }
- err = en.WriteUint16(z.rc.iv[zbzg].last())
- if err != nil {
- return
- }
- }
- // write "card"
- err = en.Append(0xa4, 0x63, 0x61, 0x72, 0x64)
- if err != nil {
- return err
- }
- err = en.WriteInt64(z.rc.card)
- if err != nil {
- return
- }
- }
- return
-}
-
-// Deprecated: MarshalMsg implements msgp.Marshaler
-func (z *addHelper16) MarshalMsg(b []byte) (o []byte, err error) {
- o = msgp.Require(b, z.Msgsize())
- // map header, size 5
- // string "runstart"
- o = append(o, 0x85, 0xa8, 0x72, 0x75, 0x6e, 0x73, 0x74, 0x61, 0x72, 0x74)
- o = msgp.AppendUint16(o, z.runstart)
- // string "runlen"
- o = append(o, 0xa6, 0x72, 0x75, 0x6e, 0x6c, 0x65, 0x6e)
- o = msgp.AppendUint16(o, z.runlen)
- // string "actuallyAdded"
- o = append(o, 0xad, 0x61, 0x63, 0x74, 0x75, 0x61, 0x6c, 0x6c, 0x79, 0x41, 0x64, 0x64, 0x65, 0x64)
- o = msgp.AppendUint16(o, z.actuallyAdded)
- // string "m"
- o = append(o, 0xa1, 0x6d)
- o = msgp.AppendArrayHeader(o, uint32(len(z.m)))
- for zxvk := range z.m {
- // map header, size 2
- // string "start"
- o = append(o, 0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74)
- o = msgp.AppendUint16(o, z.m[zxvk].start)
- // string "last"
- o = append(o, 0xa4, 0x6c, 0x61, 0x73, 0x74)
- o = msgp.AppendUint16(o, z.m[zxvk].last())
- }
- // string "rc"
- o = append(o, 0xa2, 0x72, 0x63)
- if z.rc == nil {
- o = msgp.AppendNil(o)
- } else {
- // map header, size 2
- // string "iv"
- o = append(o, 0x82, 0xa2, 0x69, 0x76)
- o = msgp.AppendArrayHeader(o, uint32(len(z.rc.iv)))
- for zbzg := range z.rc.iv {
- // map header, size 2
- // string "start"
- o = append(o, 0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74)
- o = msgp.AppendUint16(o, z.rc.iv[zbzg].start)
- // string "last"
- o = append(o, 0xa4, 0x6c, 0x61, 0x73, 0x74)
- o = msgp.AppendUint16(o, z.rc.iv[zbzg].last())
- }
- // string "card"
- o = append(o, 0xa4, 0x63, 0x61, 0x72, 0x64)
- o = msgp.AppendInt64(o, z.rc.card)
- }
- return
-}
-
-// Deprecated: UnmarshalMsg implements msgp.Unmarshaler
-func (z *addHelper16) UnmarshalMsg(bts []byte) (o []byte, err error) {
- var field []byte
- _ = field
- var zxhx uint32
- zxhx, bts, err = msgp.ReadMapHeaderBytes(bts)
- if err != nil {
- return
- }
- for zxhx > 0 {
- zxhx--
- field, bts, err = msgp.ReadMapKeyZC(bts)
- if err != nil {
- return
- }
- switch msgp.UnsafeString(field) {
- case "runstart":
- z.runstart, bts, err = msgp.ReadUint16Bytes(bts)
- if err != nil {
- return
- }
- case "runlen":
- z.runlen, bts, err = msgp.ReadUint16Bytes(bts)
- if err != nil {
- return
- }
- case "actuallyAdded":
- z.actuallyAdded, bts, err = msgp.ReadUint16Bytes(bts)
- if err != nil {
- return
- }
- case "m":
- var zlqf uint32
- zlqf, bts, err = msgp.ReadArrayHeaderBytes(bts)
- if err != nil {
- return
- }
- if cap(z.m) >= int(zlqf) {
- z.m = (z.m)[:zlqf]
- } else {
- z.m = make([]interval16, zlqf)
- }
- for zxvk := range z.m {
- var zdaf uint32
- zdaf, bts, err = msgp.ReadMapHeaderBytes(bts)
- if err != nil {
- return
- }
- for zdaf > 0 {
- zdaf--
- field, bts, err = msgp.ReadMapKeyZC(bts)
- if err != nil {
- return
- }
- switch msgp.UnsafeString(field) {
- case "start":
- z.m[zxvk].start, bts, err = msgp.ReadUint16Bytes(bts)
- if err != nil {
- return
- }
- case "last":
- z.m[zxvk].length, bts, err = msgp.ReadUint16Bytes(bts)
- z.m[zxvk].length -= z.m[zxvk].start
- if err != nil {
- return
- }
- default:
- bts, err = msgp.Skip(bts)
- if err != nil {
- return
- }
- }
- }
- }
- case "rc":
- if msgp.IsNil(bts) {
- bts, err = msgp.ReadNilBytes(bts)
- if err != nil {
- return
- }
- z.rc = nil
- } else {
- if z.rc == nil {
- z.rc = new(runContainer16)
- }
- var zpks uint32
- zpks, bts, err = msgp.ReadMapHeaderBytes(bts)
- if err != nil {
- return
- }
- for zpks > 0 {
- zpks--
- field, bts, err = msgp.ReadMapKeyZC(bts)
- if err != nil {
- return
- }
- switch msgp.UnsafeString(field) {
- case "iv":
- var zjfb uint32
- zjfb, bts, err = msgp.ReadArrayHeaderBytes(bts)
- if err != nil {
- return
- }
- if cap(z.rc.iv) >= int(zjfb) {
- z.rc.iv = (z.rc.iv)[:zjfb]
- } else {
- z.rc.iv = make([]interval16, zjfb)
- }
- for zbzg := range z.rc.iv {
- var zcxo uint32
- zcxo, bts, err = msgp.ReadMapHeaderBytes(bts)
- if err != nil {
- return
- }
- for zcxo > 0 {
- zcxo--
- field, bts, err = msgp.ReadMapKeyZC(bts)
- if err != nil {
- return
- }
- switch msgp.UnsafeString(field) {
- case "start":
- z.rc.iv[zbzg].start, bts, err = msgp.ReadUint16Bytes(bts)
- if err != nil {
- return
- }
- case "last":
- z.rc.iv[zbzg].length, bts, err = msgp.ReadUint16Bytes(bts)
- z.rc.iv[zbzg].length -= z.rc.iv[zbzg].start
- if err != nil {
- return
- }
- default:
- bts, err = msgp.Skip(bts)
- if err != nil {
- return
- }
- }
- }
- }
- case "card":
- z.rc.card, bts, err = msgp.ReadInt64Bytes(bts)
- if err != nil {
- return
- }
- default:
- bts, err = msgp.Skip(bts)
- if err != nil {
- return
- }
- }
- }
- }
- default:
- bts, err = msgp.Skip(bts)
- if err != nil {
- return
- }
- }
- }
- o = bts
- return
-}
-
-// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (z *addHelper16) Msgsize() (s int) {
- s = 1 + 9 + msgp.Uint16Size + 7 + msgp.Uint16Size + 14 + msgp.Uint16Size + 2 + msgp.ArrayHeaderSize + (len(z.m) * (12 + msgp.Uint16Size + msgp.Uint16Size)) + 3
- if z.rc == nil {
- s += msgp.NilSize
- } else {
- s += 1 + 3 + msgp.ArrayHeaderSize + (len(z.rc.iv) * (12 + msgp.Uint16Size + msgp.Uint16Size)) + 5 + msgp.Int64Size
- }
- return
-}
-
-// Deprecated: DecodeMsg implements msgp.Decodable
-func (z *interval16) DecodeMsg(dc *msgp.Reader) (err error) {
- var field []byte
- _ = field
- var zeff uint32
- zeff, err = dc.ReadMapHeader()
- if err != nil {
- return
- }
- for zeff > 0 {
- zeff--
- field, err = dc.ReadMapKeyPtr()
- if err != nil {
- return
- }
- switch msgp.UnsafeString(field) {
- case "start":
- z.start, err = dc.ReadUint16()
- if err != nil {
- return
- }
- case "last":
- z.length, err = dc.ReadUint16()
- z.length = -z.start
- if err != nil {
- return
- }
- default:
- err = dc.Skip()
- if err != nil {
- return
- }
- }
- }
- return
-}
-
-// Deprecated: EncodeMsg implements msgp.Encodable
-func (z interval16) EncodeMsg(en *msgp.Writer) (err error) {
- // map header, size 2
- // write "start"
- err = en.Append(0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74)
- if err != nil {
- return err
- }
- err = en.WriteUint16(z.start)
- if err != nil {
- return
- }
- // write "last"
- err = en.Append(0xa4, 0x6c, 0x61, 0x73, 0x74)
- if err != nil {
- return err
- }
- err = en.WriteUint16(z.last())
- if err != nil {
- return
- }
- return
-}
-
-// Deprecated: MarshalMsg implements msgp.Marshaler
-func (z interval16) MarshalMsg(b []byte) (o []byte, err error) {
- o = msgp.Require(b, z.Msgsize())
- // map header, size 2
- // string "start"
- o = append(o, 0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74)
- o = msgp.AppendUint16(o, z.start)
- // string "last"
- o = append(o, 0xa4, 0x6c, 0x61, 0x73, 0x74)
- o = msgp.AppendUint16(o, z.last())
- return
-}
-
-// Deprecated: UnmarshalMsg implements msgp.Unmarshaler
-func (z *interval16) UnmarshalMsg(bts []byte) (o []byte, err error) {
- var field []byte
- _ = field
- var zrsw uint32
- zrsw, bts, err = msgp.ReadMapHeaderBytes(bts)
- if err != nil {
- return
- }
- for zrsw > 0 {
- zrsw--
- field, bts, err = msgp.ReadMapKeyZC(bts)
- if err != nil {
- return
- }
- switch msgp.UnsafeString(field) {
- case "start":
- z.start, bts, err = msgp.ReadUint16Bytes(bts)
- if err != nil {
- return
- }
- case "last":
- z.length, bts, err = msgp.ReadUint16Bytes(bts)
- z.length -= z.start
- if err != nil {
- return
- }
- default:
- bts, err = msgp.Skip(bts)
- if err != nil {
- return
- }
- }
- }
- o = bts
- return
-}
-
-// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (z interval16) Msgsize() (s int) {
- s = 1 + 6 + msgp.Uint16Size + 5 + msgp.Uint16Size
- return
-}
-
-// Deprecated: DecodeMsg implements msgp.Decodable
-func (z *runContainer16) DecodeMsg(dc *msgp.Reader) (err error) {
- var field []byte
- _ = field
- var zdnj uint32
- zdnj, err = dc.ReadMapHeader()
- if err != nil {
- return
- }
- for zdnj > 0 {
- zdnj--
- field, err = dc.ReadMapKeyPtr()
- if err != nil {
- return
- }
- switch msgp.UnsafeString(field) {
- case "iv":
- var zobc uint32
- zobc, err = dc.ReadArrayHeader()
- if err != nil {
- return
- }
- if cap(z.iv) >= int(zobc) {
- z.iv = (z.iv)[:zobc]
- } else {
- z.iv = make([]interval16, zobc)
- }
- for zxpk := range z.iv {
- var zsnv uint32
- zsnv, err = dc.ReadMapHeader()
- if err != nil {
- return
- }
- for zsnv > 0 {
- zsnv--
- field, err = dc.ReadMapKeyPtr()
- if err != nil {
- return
- }
- switch msgp.UnsafeString(field) {
- case "start":
- z.iv[zxpk].start, err = dc.ReadUint16()
- if err != nil {
- return
- }
- case "last":
- z.iv[zxpk].length, err = dc.ReadUint16()
- z.iv[zxpk].length -= z.iv[zxpk].start
- if err != nil {
- return
- }
- default:
- err = dc.Skip()
- if err != nil {
- return
- }
- }
- }
- }
- case "card":
- z.card, err = dc.ReadInt64()
- if err != nil {
- return
- }
- default:
- err = dc.Skip()
- if err != nil {
- return
- }
- }
- }
- return
-}
-
-// Deprecated: EncodeMsg implements msgp.Encodable
-func (z *runContainer16) EncodeMsg(en *msgp.Writer) (err error) {
- // map header, size 2
- // write "iv"
- err = en.Append(0x82, 0xa2, 0x69, 0x76)
- if err != nil {
- return err
- }
- err = en.WriteArrayHeader(uint32(len(z.iv)))
- if err != nil {
- return
- }
- for zxpk := range z.iv {
- // map header, size 2
- // write "start"
- err = en.Append(0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74)
- if err != nil {
- return err
- }
- err = en.WriteUint16(z.iv[zxpk].start)
- if err != nil {
- return
- }
- // write "last"
- err = en.Append(0xa4, 0x6c, 0x61, 0x73, 0x74)
- if err != nil {
- return err
- }
- err = en.WriteUint16(z.iv[zxpk].last())
- if err != nil {
- return
- }
- }
- // write "card"
- err = en.Append(0xa4, 0x63, 0x61, 0x72, 0x64)
- if err != nil {
- return err
- }
- err = en.WriteInt64(z.card)
- if err != nil {
- return
- }
- return
-}
-
-// Deprecated: MarshalMsg implements msgp.Marshaler
-func (z *runContainer16) MarshalMsg(b []byte) (o []byte, err error) {
- o = msgp.Require(b, z.Msgsize())
- // map header, size 2
- // string "iv"
- o = append(o, 0x82, 0xa2, 0x69, 0x76)
- o = msgp.AppendArrayHeader(o, uint32(len(z.iv)))
- for zxpk := range z.iv {
- // map header, size 2
- // string "start"
- o = append(o, 0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74)
- o = msgp.AppendUint16(o, z.iv[zxpk].start)
- // string "last"
- o = append(o, 0xa4, 0x6c, 0x61, 0x73, 0x74)
- o = msgp.AppendUint16(o, z.iv[zxpk].last())
- }
- // string "card"
- o = append(o, 0xa4, 0x63, 0x61, 0x72, 0x64)
- o = msgp.AppendInt64(o, z.card)
- return
-}
-
-// Deprecated: UnmarshalMsg implements msgp.Unmarshaler
-func (z *runContainer16) UnmarshalMsg(bts []byte) (o []byte, err error) {
- var field []byte
- _ = field
- var zkgt uint32
- zkgt, bts, err = msgp.ReadMapHeaderBytes(bts)
- if err != nil {
- return
- }
- for zkgt > 0 {
- zkgt--
- field, bts, err = msgp.ReadMapKeyZC(bts)
- if err != nil {
- return
- }
- switch msgp.UnsafeString(field) {
- case "iv":
- var zema uint32
- zema, bts, err = msgp.ReadArrayHeaderBytes(bts)
- if err != nil {
- return
- }
- if cap(z.iv) >= int(zema) {
- z.iv = (z.iv)[:zema]
- } else {
- z.iv = make([]interval16, zema)
- }
- for zxpk := range z.iv {
- var zpez uint32
- zpez, bts, err = msgp.ReadMapHeaderBytes(bts)
- if err != nil {
- return
- }
- for zpez > 0 {
- zpez--
- field, bts, err = msgp.ReadMapKeyZC(bts)
- if err != nil {
- return
- }
- switch msgp.UnsafeString(field) {
- case "start":
- z.iv[zxpk].start, bts, err = msgp.ReadUint16Bytes(bts)
- if err != nil {
- return
- }
- case "last":
- z.iv[zxpk].length, bts, err = msgp.ReadUint16Bytes(bts)
- z.iv[zxpk].length -= z.iv[zxpk].start
- if err != nil {
- return
- }
- default:
- bts, err = msgp.Skip(bts)
- if err != nil {
- return
- }
- }
- }
- }
- case "card":
- z.card, bts, err = msgp.ReadInt64Bytes(bts)
- if err != nil {
- return
- }
- default:
- bts, err = msgp.Skip(bts)
- if err != nil {
- return
- }
- }
- }
- o = bts
- return
-}
-
-// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (z *runContainer16) Msgsize() (s int) {
- s = 1 + 3 + msgp.ArrayHeaderSize + (len(z.iv) * (12 + msgp.Uint16Size + msgp.Uint16Size)) + 5 + msgp.Int64Size
- return
-}
-
-// Deprecated: DecodeMsg implements msgp.Decodable
-func (z *runIterator16) DecodeMsg(dc *msgp.Reader) (err error) {
- var field []byte
- _ = field
- var zqke uint32
- zqke, err = dc.ReadMapHeader()
- if err != nil {
- return
- }
- for zqke > 0 {
- zqke--
- field, err = dc.ReadMapKeyPtr()
- if err != nil {
- return
- }
- switch msgp.UnsafeString(field) {
- case "rc":
- if dc.IsNil() {
- err = dc.ReadNil()
- if err != nil {
- return
- }
- z.rc = nil
- } else {
- if z.rc == nil {
- z.rc = new(runContainer16)
- }
- err = z.rc.DecodeMsg(dc)
- if err != nil {
- return
- }
- }
- case "curIndex":
- z.curIndex, err = dc.ReadInt64()
- if err != nil {
- return
- }
- case "curPosInIndex":
- z.curPosInIndex, err = dc.ReadUint16()
- if err != nil {
- return
- }
- case "curSeq":
- z.curSeq, err = dc.ReadInt64()
- if err != nil {
- return
- }
- default:
- err = dc.Skip()
- if err != nil {
- return
- }
- }
- }
- return
-}
-
-// Deprecated: EncodeMsg implements msgp.Encodable
-func (z *runIterator16) EncodeMsg(en *msgp.Writer) (err error) {
- // map header, size 4
- // write "rc"
- err = en.Append(0x84, 0xa2, 0x72, 0x63)
- if err != nil {
- return err
- }
- if z.rc == nil {
- err = en.WriteNil()
- if err != nil {
- return
- }
- } else {
- err = z.rc.EncodeMsg(en)
- if err != nil {
- return
- }
- }
- // write "curIndex"
- err = en.Append(0xa8, 0x63, 0x75, 0x72, 0x49, 0x6e, 0x64, 0x65, 0x78)
- if err != nil {
- return err
- }
- err = en.WriteInt64(z.curIndex)
- if err != nil {
- return
- }
- // write "curPosInIndex"
- err = en.Append(0xad, 0x63, 0x75, 0x72, 0x50, 0x6f, 0x73, 0x49, 0x6e, 0x49, 0x6e, 0x64, 0x65, 0x78)
- if err != nil {
- return err
- }
- err = en.WriteUint16(z.curPosInIndex)
- if err != nil {
- return
- }
- // write "curSeq"
- err = en.Append(0xa6, 0x63, 0x75, 0x72, 0x53, 0x65, 0x71)
- if err != nil {
- return err
- }
- err = en.WriteInt64(z.curSeq)
- if err != nil {
- return
- }
- return
-}
-
-// Deprecated: MarshalMsg implements msgp.Marshaler
-func (z *runIterator16) MarshalMsg(b []byte) (o []byte, err error) {
- o = msgp.Require(b, z.Msgsize())
- // map header, size 4
- // string "rc"
- o = append(o, 0x84, 0xa2, 0x72, 0x63)
- if z.rc == nil {
- o = msgp.AppendNil(o)
- } else {
- o, err = z.rc.MarshalMsg(o)
- if err != nil {
- return
- }
- }
- // string "curIndex"
- o = append(o, 0xa8, 0x63, 0x75, 0x72, 0x49, 0x6e, 0x64, 0x65, 0x78)
- o = msgp.AppendInt64(o, z.curIndex)
- // string "curPosInIndex"
- o = append(o, 0xad, 0x63, 0x75, 0x72, 0x50, 0x6f, 0x73, 0x49, 0x6e, 0x49, 0x6e, 0x64, 0x65, 0x78)
- o = msgp.AppendUint16(o, z.curPosInIndex)
- // string "curSeq"
- o = append(o, 0xa6, 0x63, 0x75, 0x72, 0x53, 0x65, 0x71)
- o = msgp.AppendInt64(o, z.curSeq)
- return
-}
-
-// Deprecated: UnmarshalMsg implements msgp.Unmarshaler
-func (z *runIterator16) UnmarshalMsg(bts []byte) (o []byte, err error) {
- var field []byte
- _ = field
- var zqyh uint32
- zqyh, bts, err = msgp.ReadMapHeaderBytes(bts)
- if err != nil {
- return
- }
- for zqyh > 0 {
- zqyh--
- field, bts, err = msgp.ReadMapKeyZC(bts)
- if err != nil {
- return
- }
- switch msgp.UnsafeString(field) {
- case "rc":
- if msgp.IsNil(bts) {
- bts, err = msgp.ReadNilBytes(bts)
- if err != nil {
- return
- }
- z.rc = nil
- } else {
- if z.rc == nil {
- z.rc = new(runContainer16)
- }
- bts, err = z.rc.UnmarshalMsg(bts)
- if err != nil {
- return
- }
- }
- case "curIndex":
- z.curIndex, bts, err = msgp.ReadInt64Bytes(bts)
- if err != nil {
- return
- }
- case "curPosInIndex":
- z.curPosInIndex, bts, err = msgp.ReadUint16Bytes(bts)
- if err != nil {
- return
- }
- case "curSeq":
- z.curSeq, bts, err = msgp.ReadInt64Bytes(bts)
- if err != nil {
- return
- }
- default:
- bts, err = msgp.Skip(bts)
- if err != nil {
- return
- }
- }
- }
- o = bts
- return
-}
-
-// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (z *runIterator16) Msgsize() (s int) {
- s = 1 + 3
- if z.rc == nil {
- s += msgp.NilSize
- } else {
- s += z.rc.Msgsize()
- }
- s += 9 + msgp.Int64Size + 14 + msgp.Uint16Size + 7 + msgp.Int64Size
- return
-}
-
-// Deprecated: DecodeMsg implements msgp.Decodable
-func (z *uint16Slice) DecodeMsg(dc *msgp.Reader) (err error) {
- var zjpj uint32
- zjpj, err = dc.ReadArrayHeader()
- if err != nil {
- return
- }
- if cap((*z)) >= int(zjpj) {
- (*z) = (*z)[:zjpj]
- } else {
- (*z) = make(uint16Slice, zjpj)
- }
- for zywj := range *z {
- (*z)[zywj], err = dc.ReadUint16()
- if err != nil {
- return
- }
- }
- return
-}
-
-// Deprecated: EncodeMsg implements msgp.Encodable
-func (z uint16Slice) EncodeMsg(en *msgp.Writer) (err error) {
- err = en.WriteArrayHeader(uint32(len(z)))
- if err != nil {
- return
- }
- for zzpf := range z {
- err = en.WriteUint16(z[zzpf])
- if err != nil {
- return
- }
- }
- return
-}
-
-// Deprecated: MarshalMsg implements msgp.Marshaler
-func (z uint16Slice) MarshalMsg(b []byte) (o []byte, err error) {
- o = msgp.Require(b, z.Msgsize())
- o = msgp.AppendArrayHeader(o, uint32(len(z)))
- for zzpf := range z {
- o = msgp.AppendUint16(o, z[zzpf])
- }
- return
-}
-
-// Deprecated: UnmarshalMsg implements msgp.Unmarshaler
-func (z *uint16Slice) UnmarshalMsg(bts []byte) (o []byte, err error) {
- var zgmo uint32
- zgmo, bts, err = msgp.ReadArrayHeaderBytes(bts)
- if err != nil {
- return
- }
- if cap((*z)) >= int(zgmo) {
- (*z) = (*z)[:zgmo]
- } else {
- (*z) = make(uint16Slice, zgmo)
- }
- for zrfe := range *z {
- (*z)[zrfe], bts, err = msgp.ReadUint16Bytes(bts)
- if err != nil {
- return
- }
- }
- o = bts
- return
-}
-
-// Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (z uint16Slice) Msgsize() (s int) {
- s = msgp.ArrayHeaderSize + (len(z) * (msgp.Uint16Size))
- return
-}
diff --git a/vendor/github.com/RoaringBitmap/roaring/serialization.go b/vendor/github.com/RoaringBitmap/roaring/serialization.go
index 98fef3c..dbfecc8 100644
--- a/vendor/github.com/RoaringBitmap/roaring/serialization.go
+++ b/vendor/github.com/RoaringBitmap/roaring/serialization.go
@@ -2,16 +2,11 @@ package roaring
import (
"encoding/binary"
- "errors"
- "fmt"
"io"
-
- "github.com/tinylib/msgp/msgp"
)
// writeTo for runContainer16 follows this
// spec: https://github.com/RoaringBitmap/RoaringFormatSpec
-//
func (b *runContainer16) writeTo(stream io.Writer) (int, error) {
buf := make([]byte, 2+4*len(b.iv))
binary.LittleEndian.PutUint16(buf[0:], uint16(len(b.iv)))
@@ -21,50 +16,3 @@ func (b *runContainer16) writeTo(stream io.Writer) (int, error) {
}
return stream.Write(buf)
}
-
-func (b *runContainer16) writeToMsgpack(stream io.Writer) (int, error) {
- bts, err := b.MarshalMsg(nil)
- if err != nil {
- return 0, err
- }
- return stream.Write(bts)
-}
-
-func (b *runContainer16) readFromMsgpack(stream io.Reader) (int, error) {
- err := msgp.Decode(stream, b)
- return 0, err
-}
-
-var errCorruptedStream = errors.New("insufficient/odd number of stored bytes, corrupted stream detected")
-
-func (b *runContainer16) readFrom(stream io.Reader) (int, error) {
- b.iv = b.iv[:0]
- b.card = 0
- var numRuns uint16
- err := binary.Read(stream, binary.LittleEndian, &numRuns)
- if err != nil {
- return 0, err
- }
- nr := int(numRuns)
- encRun := make([]uint16, 2*nr)
- by := make([]byte, 4*nr)
- err = binary.Read(stream, binary.LittleEndian, &by)
- if err != nil {
- return 0, err
- }
- for i := range encRun {
- if len(by) < 2 {
- return 0, errCorruptedStream
- }
- encRun[i] = binary.LittleEndian.Uint16(by)
- by = by[2:]
- }
- for i := 0; i < nr; i++ {
- if i > 0 && b.iv[i-1].last() >= encRun[i*2] {
- return 0, fmt.Errorf("error: stored runContainer had runs that were not in sorted order!! (b.iv[i-1=%v].last = %v >= encRun[i=%v] = %v)", i-1, b.iv[i-1].last(), i, encRun[i*2])
- }
- b.iv = append(b.iv, interval16{start: encRun[i*2], length: encRun[i*2+1]})
- b.card += int64(encRun[i*2+1]) + 1
- }
- return 0, err
-}
diff --git a/vendor/github.com/RoaringBitmap/roaring/serialization_generic.go b/vendor/github.com/RoaringBitmap/roaring/serialization_generic.go
index f4805ca..7e1f180 100644
--- a/vendor/github.com/RoaringBitmap/roaring/serialization_generic.go
+++ b/vendor/github.com/RoaringBitmap/roaring/serialization_generic.go
@@ -1,4 +1,5 @@
-// +build !amd64,!386 appengine
+//go:build (!amd64 && !386 && !arm && !arm64 && !ppc64le && !mipsle && !mips64le && !mips64p32le && !wasm) || appengine
+// +build !amd64,!386,!arm,!arm64,!ppc64le,!mipsle,!mips64le,!mips64p32le,!wasm appengine
package roaring
@@ -74,6 +75,27 @@ func uint64SliceAsByteSlice(slice []uint64) []byte {
return by
}
+func uint16SliceAsByteSlice(slice []uint16) []byte {
+ by := make([]byte, len(slice)*2)
+
+ for i, v := range slice {
+ binary.LittleEndian.PutUint16(by[i*2:], v)
+ }
+
+ return by
+}
+
+func interval16SliceAsByteSlice(slice []interval16) []byte {
+ by := make([]byte, len(slice)*4)
+
+ for i, v := range slice {
+ binary.LittleEndian.PutUint16(by[i*2:], v.start)
+ binary.LittleEndian.PutUint16(by[i*2+2:], v.length)
+ }
+
+ return by
+}
+
func byteSliceAsUint16Slice(slice []byte) []uint16 {
if len(slice)%2 != 0 {
panic("Slice size should be divisible by 2")
diff --git a/vendor/github.com/RoaringBitmap/roaring/serialization_littleendian.go b/vendor/github.com/RoaringBitmap/roaring/serialization_littleendian.go
index ba5b753..6e3a5d5 100644
--- a/vendor/github.com/RoaringBitmap/roaring/serialization_littleendian.go
+++ b/vendor/github.com/RoaringBitmap/roaring/serialization_littleendian.go
@@ -1,11 +1,14 @@
-// +build 386 amd64,!appengine
+//go:build (386 && !appengine) || (amd64 && !appengine) || (arm && !appengine) || (arm64 && !appengine) || (ppc64le && !appengine) || (mipsle && !appengine) || (mips64le && !appengine) || (mips64p32le && !appengine) || (wasm && !appengine)
+// +build 386,!appengine amd64,!appengine arm,!appengine arm64,!appengine ppc64le,!appengine mipsle,!appengine mips64le,!appengine mips64p32le,!appengine wasm,!appengine
package roaring
import (
+ "encoding/binary"
"errors"
"io"
"reflect"
+ "runtime"
"unsafe"
)
@@ -22,22 +25,6 @@ func (bc *bitmapContainer) writeTo(stream io.Writer) (int, error) {
return stream.Write(buf)
}
-// readFrom reads an arrayContainer from stream.
-// PRE-REQUISITE: you must size the arrayContainer correctly (allocate b.content)
-// *before* you call readFrom. We can't guess the size in the stream
-// by this point.
-func (ac *arrayContainer) readFrom(stream io.Reader) (int, error) {
- buf := uint16SliceAsByteSlice(ac.content)
- return io.ReadFull(stream, buf)
-}
-
-func (bc *bitmapContainer) readFrom(stream io.Reader) (int, error) {
- buf := uint64SliceAsByteSlice(bc.bitmap)
- n, err := io.ReadFull(stream, buf)
- bc.computeCardinality()
- return n, err
-}
-
func uint64SliceAsByteSlice(slice []uint64) []byte {
// make a new slice header
header := *(*reflect.SliceHeader)(unsafe.Pointer(&slice))
@@ -46,8 +33,12 @@ func uint64SliceAsByteSlice(slice []uint64) []byte {
header.Len *= 8
header.Cap *= 8
+ // instantiate result and use KeepAlive so data isn't unmapped.
+ result := *(*[]byte)(unsafe.Pointer(&header))
+ runtime.KeepAlive(&slice)
+
// return it
- return *(*[]byte)(unsafe.Pointer(&header))
+ return result
}
func uint16SliceAsByteSlice(slice []uint16) []byte {
@@ -58,8 +49,28 @@ func uint16SliceAsByteSlice(slice []uint16) []byte {
header.Len *= 2
header.Cap *= 2
+ // instantiate result and use KeepAlive so data isn't unmapped.
+ result := *(*[]byte)(unsafe.Pointer(&header))
+ runtime.KeepAlive(&slice)
+
// return it
- return *(*[]byte)(unsafe.Pointer(&header))
+ return result
+}
+
+func interval16SliceAsByteSlice(slice []interval16) []byte {
+ // make a new slice header
+ header := *(*reflect.SliceHeader)(unsafe.Pointer(&slice))
+
+ // update its capacity and length
+ header.Len *= 4
+ header.Cap *= 4
+
+ // instantiate result and use KeepAlive so data isn't unmapped.
+ result := *(*[]byte)(unsafe.Pointer(&header))
+ runtime.KeepAlive(&slice)
+
+ // return it
+ return result
}
func (bc *bitmapContainer) asLittleEndianByteSlice() []byte {
@@ -68,50 +79,584 @@ func (bc *bitmapContainer) asLittleEndianByteSlice() []byte {
// Deserialization code follows
-func byteSliceAsUint16Slice(slice []byte) []uint16 {
+// //
+// These methods (byteSliceAsUint16Slice,...) do not make copies,
+// they are pointer-based (unsafe). The caller is responsible to
+// ensure that the input slice does not get garbage collected, deleted
+// or modified while you hold the returned slince.
+// //
+func byteSliceAsUint16Slice(slice []byte) (result []uint16) { // here we create a new slice holder
if len(slice)%2 != 0 {
panic("Slice size should be divisible by 2")
}
+ // reference: https://go101.org/article/unsafe.html
// make a new slice header
- header := *(*reflect.SliceHeader)(unsafe.Pointer(&slice))
+ bHeader := (*reflect.SliceHeader)(unsafe.Pointer(&slice))
+ rHeader := (*reflect.SliceHeader)(unsafe.Pointer(&result))
- // update its capacity and length
- header.Len /= 2
- header.Cap /= 2
+ // transfer the data from the given slice to a new variable (our result)
+ rHeader.Data = bHeader.Data
+ rHeader.Len = bHeader.Len / 2
+ rHeader.Cap = bHeader.Cap / 2
- // return it
- return *(*[]uint16)(unsafe.Pointer(&header))
+ // instantiate result and use KeepAlive so data isn't unmapped.
+ runtime.KeepAlive(&slice) // it is still crucial, GC can free it)
+
+ // return result
+ return
}
-func byteSliceAsUint64Slice(slice []byte) []uint64 {
+func byteSliceAsUint64Slice(slice []byte) (result []uint64) {
if len(slice)%8 != 0 {
panic("Slice size should be divisible by 8")
}
+ // reference: https://go101.org/article/unsafe.html
// make a new slice header
- header := *(*reflect.SliceHeader)(unsafe.Pointer(&slice))
+ bHeader := (*reflect.SliceHeader)(unsafe.Pointer(&slice))
+ rHeader := (*reflect.SliceHeader)(unsafe.Pointer(&result))
- // update its capacity and length
- header.Len /= 8
- header.Cap /= 8
+ // transfer the data from the given slice to a new variable (our result)
+ rHeader.Data = bHeader.Data
+ rHeader.Len = bHeader.Len / 8
+ rHeader.Cap = bHeader.Cap / 8
- // return it
- return *(*[]uint64)(unsafe.Pointer(&header))
+ // instantiate result and use KeepAlive so data isn't unmapped.
+ runtime.KeepAlive(&slice) // it is still crucial, GC can free it)
+
+ // return result
+ return
}
-func byteSliceAsInterval16Slice(slice []byte) []interval16 {
+func byteSliceAsInterval16Slice(slice []byte) (result []interval16) {
if len(slice)%4 != 0 {
panic("Slice size should be divisible by 4")
}
+ // reference: https://go101.org/article/unsafe.html
// make a new slice header
- header := *(*reflect.SliceHeader)(unsafe.Pointer(&slice))
+ bHeader := (*reflect.SliceHeader)(unsafe.Pointer(&slice))
+ rHeader := (*reflect.SliceHeader)(unsafe.Pointer(&result))
- // update its capacity and length
- header.Len /= 4
- header.Cap /= 4
+ // transfer the data from the given slice to a new variable (our result)
+ rHeader.Data = bHeader.Data
+ rHeader.Len = bHeader.Len / 4
+ rHeader.Cap = bHeader.Cap / 4
- // return it
- return *(*[]interval16)(unsafe.Pointer(&header))
+ // instantiate result and use KeepAlive so data isn't unmapped.
+ runtime.KeepAlive(&slice) // it is still crucial, GC can free it)
+
+ // return result
+ return
+}
+
+func byteSliceAsContainerSlice(slice []byte) (result []container) {
+ var c container
+ containerSize := int(unsafe.Sizeof(c))
+
+ if len(slice)%containerSize != 0 {
+ panic("Slice size should be divisible by unsafe.Sizeof(container)")
+ }
+ // reference: https://go101.org/article/unsafe.html
+
+ // make a new slice header
+ bHeader := (*reflect.SliceHeader)(unsafe.Pointer(&slice))
+ rHeader := (*reflect.SliceHeader)(unsafe.Pointer(&result))
+
+ // transfer the data from the given slice to a new variable (our result)
+ rHeader.Data = bHeader.Data
+ rHeader.Len = bHeader.Len / containerSize
+ rHeader.Cap = bHeader.Cap / containerSize
+
+ // instantiate result and use KeepAlive so data isn't unmapped.
+ runtime.KeepAlive(&slice) // it is still crucial, GC can free it)
+
+ // return result
+ return
+}
+
+func byteSliceAsBitsetSlice(slice []byte) (result []bitmapContainer) {
+ bitsetSize := int(unsafe.Sizeof(bitmapContainer{}))
+ if len(slice)%bitsetSize != 0 {
+ panic("Slice size should be divisible by unsafe.Sizeof(bitmapContainer)")
+ }
+ // reference: https://go101.org/article/unsafe.html
+
+ // make a new slice header
+ bHeader := (*reflect.SliceHeader)(unsafe.Pointer(&slice))
+ rHeader := (*reflect.SliceHeader)(unsafe.Pointer(&result))
+
+ // transfer the data from the given slice to a new variable (our result)
+ rHeader.Data = bHeader.Data
+ rHeader.Len = bHeader.Len / bitsetSize
+ rHeader.Cap = bHeader.Cap / bitsetSize
+
+ // instantiate result and use KeepAlive so data isn't unmapped.
+ runtime.KeepAlive(&slice) // it is still crucial, GC can free it)
+
+ // return result
+ return
+}
+
+func byteSliceAsArraySlice(slice []byte) (result []arrayContainer) {
+ arraySize := int(unsafe.Sizeof(arrayContainer{}))
+ if len(slice)%arraySize != 0 {
+ panic("Slice size should be divisible by unsafe.Sizeof(arrayContainer)")
+ }
+ // reference: https://go101.org/article/unsafe.html
+
+ // make a new slice header
+ bHeader := (*reflect.SliceHeader)(unsafe.Pointer(&slice))
+ rHeader := (*reflect.SliceHeader)(unsafe.Pointer(&result))
+
+ // transfer the data from the given slice to a new variable (our result)
+ rHeader.Data = bHeader.Data
+ rHeader.Len = bHeader.Len / arraySize
+ rHeader.Cap = bHeader.Cap / arraySize
+
+ // instantiate result and use KeepAlive so data isn't unmapped.
+ runtime.KeepAlive(&slice) // it is still crucial, GC can free it)
+
+ // return result
+ return
+}
+
+func byteSliceAsRun16Slice(slice []byte) (result []runContainer16) {
+ run16Size := int(unsafe.Sizeof(runContainer16{}))
+ if len(slice)%run16Size != 0 {
+ panic("Slice size should be divisible by unsafe.Sizeof(runContainer16)")
+ }
+ // reference: https://go101.org/article/unsafe.html
+
+ // make a new slice header
+ bHeader := (*reflect.SliceHeader)(unsafe.Pointer(&slice))
+ rHeader := (*reflect.SliceHeader)(unsafe.Pointer(&result))
+
+ // transfer the data from the given slice to a new variable (our result)
+ rHeader.Data = bHeader.Data
+ rHeader.Len = bHeader.Len / run16Size
+ rHeader.Cap = bHeader.Cap / run16Size
+
+ // instantiate result and use KeepAlive so data isn't unmapped.
+ runtime.KeepAlive(&slice) // it is still crucial, GC can free it)
+
+ // return result
+ return
+}
+
+func byteSliceAsBoolSlice(slice []byte) (result []bool) {
+ boolSize := int(unsafe.Sizeof(true))
+ if len(slice)%boolSize != 0 {
+ panic("Slice size should be divisible by unsafe.Sizeof(bool)")
+ }
+ // reference: https://go101.org/article/unsafe.html
+
+ // make a new slice header
+ bHeader := (*reflect.SliceHeader)(unsafe.Pointer(&slice))
+ rHeader := (*reflect.SliceHeader)(unsafe.Pointer(&result))
+
+ // transfer the data from the given slice to a new variable (our result)
+ rHeader.Data = bHeader.Data
+ rHeader.Len = bHeader.Len / boolSize
+ rHeader.Cap = bHeader.Cap / boolSize
+
+ // instantiate result and use KeepAlive so data isn't unmapped.
+ runtime.KeepAlive(&slice) // it is still crucial, GC can free it)
+
+ // return result
+ return
+}
+
+// FrozenView creates a static view of a serialized bitmap stored in buf.
+// It uses CRoaring's frozen bitmap format.
+//
+// The format specification is available here:
+// https://github.com/RoaringBitmap/CRoaring/blob/2c867e9f9c9e2a3a7032791f94c4c7ae3013f6e0/src/roaring.c#L2756-L2783
+//
+// The provided byte array (buf) is expected to be a constant.
+// The function makes the best effort attempt not to copy data.
+// Only little endian is supported. The function will err if it detects a big
+// endian serialized file.
+// You should take care not to modify buff as it will likely result in
+// unexpected program behavior.
+// If said buffer comes from a memory map, it's advisable to give it read
+// only permissions, either at creation or by calling Mprotect from the
+// golang.org/x/sys/unix package.
+//
+// Resulting bitmaps are effectively immutable in the following sense:
+// a copy-on-write marker is used so that when you modify the resulting
+// bitmap, copies of selected data (containers) are made.
+// You should *not* change the copy-on-write status of the resulting
+// bitmaps (SetCopyOnWrite).
+//
+// If buf becomes unavailable, then a bitmap created with
+// FromBuffer would be effectively broken. Furthermore, any
+// bitmap derived from this bitmap (e.g., via Or, And) might
+// also be broken. Thus, before making buf unavailable, you should
+// call CloneCopyOnWriteContainers on all such bitmaps.
+func (rb *Bitmap) FrozenView(buf []byte) error {
+ return rb.highlowcontainer.frozenView(buf)
+}
+
+/* Verbatim specification from CRoaring.
+ *
+ * FROZEN SERIALIZATION FORMAT DESCRIPTION
+ *
+ * -- (beginning must be aligned by 32 bytes) --
+ *
uint64_t[BITSET_CONTAINER_SIZE_IN_WORDS * num_bitset_containers]
+ * rle16_t[total number of rle elements in all run containers]
+ * uint16_t[total number of array elements in all array containers]
+ * uint16_t[num_containers]
+ * uint16_t[num_containers]
+ * uint8_t[num_containers]
+ * uint32_t
+ *
+ * is a 4-byte value which is a bit union of frozenCookie (15 bits)
+ * and the number of containers (17 bits).
+ *
+ * stores number of elements for every container.
+ * Its meaning depends on container type.
+ * For array and bitset containers, this value is the container cardinality minus one.
+ * For run container, it is the number of rle_t elements (n_runs).
+ *
+ * ,, are flat arrays of elements of
+ * all containers of respective type.
+ *
+ * <*_data> and are kept close together because they are not accessed
+ * during deserilization. This may reduce IO in case of large mmaped bitmaps.
+ * All members have their native alignments during deserilization except