diff --git a/vendor/git.giftfish.de/ston1th/jwt/v3/LICENSE b/vendor/git.giftfish.de/ston1th/jwt/v3/LICENSE index 115e96e..74cee2b 100644 --- a/vendor/git.giftfish.de/ston1th/jwt/v3/LICENSE +++ b/vendor/git.giftfish.de/ston1th/jwt/v3/LICENSE @@ -1,4 +1,4 @@ -Copyright (C) 2018 Marius Schellenberger +Copyright (C) 2025 Marius Schellenberger All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/vendor/git.giftfish.de/ston1th/jwt/v3/blacklist.go b/vendor/git.giftfish.de/ston1th/jwt/v3/blacklist.go index b60a928..180c5fd 100644 --- a/vendor/git.giftfish.de/ston1th/jwt/v3/blacklist.go +++ b/vendor/git.giftfish.de/ston1th/jwt/v3/blacklist.go @@ -1,4 +1,4 @@ -// Copyright (C) 2018 Marius Schellenberger +// Copyright (C) 2025 Marius Schellenberger package jwt diff --git a/vendor/git.giftfish.de/ston1th/jwt/v3/claims.go b/vendor/git.giftfish.de/ston1th/jwt/v3/claims.go index 10cbcfe..a5f4364 100644 --- a/vendor/git.giftfish.de/ston1th/jwt/v3/claims.go +++ b/vendor/git.giftfish.de/ston1th/jwt/v3/claims.go @@ -1,4 +1,4 @@ -// Copyright (C) 2018 Marius Schellenberger +// Copyright (C) 2025 Marius Schellenberger package jwt @@ -30,37 +30,28 @@ func (c Claims) GetBool(key string) (b bool) { // GetInt returns an int from the claims map func (c Claims) GetInt(key string) (i int) { - if f, ok := c.getFloat64(key); ok { - return int(f) - } - if v, ok := c.Get(key); ok && v != nil { - i, _ = v.(int) - } + i = int(c.GetInt64(key)) return } // GetInt64 returns an int64 from the claims map func (c Claims) GetInt64(key string) (i int64) { - if f, ok := c.getFloat64(key); ok { - return int64(f) - } if v, ok := c.Get(key); ok && v != nil { - i, _ = v.(int64) + switch val := v.(type) { + case int64: + i = val + case float64: + i = int64(val) + } } return } // getFloat64 returns a float64 and ok from the claims map -func (c Claims) getFloat64(key string) (f float64, fok bool) { - if v, ok := c.Get(key); ok && v != nil { - f, fok = v.(float64) - } - return -} - -// GetFloat64 returns a float64 from the claims map func (c Claims) GetFloat64(key string) (f float64) { - f, _ = c.getFloat64(key) + if v, ok := c.Get(key); ok && v != nil { + f, _ = v.(float64) + } return } diff --git a/vendor/git.giftfish.de/ston1th/jwt/v3/encoding.go b/vendor/git.giftfish.de/ston1th/jwt/v3/encoding.go index 73b81cf..3693f5f 100644 --- a/vendor/git.giftfish.de/ston1th/jwt/v3/encoding.go +++ b/vendor/git.giftfish.de/ston1th/jwt/v3/encoding.go @@ -1,4 +1,4 @@ -// Copyright (C) 2018 Marius Schellenberger +// Copyright (C) 2025 Marius Schellenberger package jwt diff --git a/vendor/git.giftfish.de/ston1th/jwt/v3/hash.go b/vendor/git.giftfish.de/ston1th/jwt/v3/hash.go index 2c224ab..e47c0cc 100644 --- a/vendor/git.giftfish.de/ston1th/jwt/v3/hash.go +++ b/vendor/git.giftfish.de/ston1th/jwt/v3/hash.go @@ -1,4 +1,4 @@ -// Copyright (C) 2018 Marius Schellenberger +// Copyright (C) 2025 Marius Schellenberger package jwt @@ -14,19 +14,25 @@ const ( HS512Name = "HS512" ) -// ParseHash returns the Hash type equal to the input string -func ParseHash(alg string) Hash { +// NewHash returns the Hash type equal to the input string +func NewHash(alg string) Hash { switch alg { case HS256Name: - return NewHS256() + return hs256 case HS384Name: - return NewHS384() + return hs384 case HS512Name: - return NewHS512() + return hs512 } return nil } +var ( + hs256 = HS256{} + hs384 = HS384{} + hs512 = HS512{} +) + // Hash is the hashsum interface for signing the jwt type Hash interface { Hash() hash.Hash @@ -36,11 +42,6 @@ type Hash interface { // HS256 implements the Hash interface with SHA256 type HS256 struct{} -// NewHS256 returns a new HS256 instance -func NewHS256() Hash { - return HS256{} -} - // Alg returns the algorithm name "HS256" func (HS256) Alg() string { return HS256Name @@ -54,11 +55,6 @@ func (HS256) Hash() hash.Hash { // HS384 implements the Hash interface with SHA384 type HS384 struct{} -// NewHS384 returns a new HS384 instance -func NewHS384() Hash { - return HS384{} -} - // Alg returns the algorithm name "HS384" func (HS384) Alg() string { return HS384Name @@ -72,11 +68,6 @@ func (HS384) Hash() hash.Hash { // HS512 implements the Hash interface with SHA512 type HS512 struct{} -// NewHS512 returns a new HS512 instance -func NewHS512() Hash { - return HS512{} -} - // Alg returns the algorithm name "HS512" func (HS512) Alg() string { return HS512Name diff --git a/vendor/git.giftfish.de/ston1th/jwt/v3/jwt.go b/vendor/git.giftfish.de/ston1th/jwt/v3/jwt.go index 44f82ec..f9ce6af 100644 --- a/vendor/git.giftfish.de/ston1th/jwt/v3/jwt.go +++ b/vendor/git.giftfish.de/ston1th/jwt/v3/jwt.go @@ -1,4 +1,4 @@ -// Copyright (C) 2018 Marius Schellenberger +// Copyright (C) 2025 Marius Schellenberger // Package jwt provides a easy to use JSON Web Token and blacklisting library package jwt @@ -31,6 +31,8 @@ const ( ExpClaim = "exp" // NbfClaim is the nbf claim name NbfClaim = "nbf" + // NonceClaim + NonceClaim = "nonce" // TokenSeparator is the tokens separator char TokenSeparator = "." ) @@ -57,14 +59,39 @@ var ( ErrInvalidKeySize = errors.New(jwtErr + "invalid secret key size") ) +type JWTOption func(*JWT) + +func WithExpiry(expiry time.Duration) JWTOption { + return func(jwt *JWT) { + jwt.expiry = expiry + } +} + +func WithBlacklist(blacklist Blacklist) JWTOption { + return func(jwt *JWT) { + jwt.blacklist = blacklist + } +} +func WithSecret(secret io.Reader) JWTOption { + return func(jwt *JWT) { + jwt.secretReader = secret + } +} +func WithNonce() JWTOption { + return func(jwt *JWT) { + jwt.nonce = true + } +} + // JWT represents the JSON Web Token signing and blacklisting infrastructure type JWT struct { - key []byte - expiry time.Duration - - blacklist Blacklist - stopOnce sync.Once - done chan struct{} + secretReader io.Reader + key []byte + expiry time.Duration + blacklist Blacklist + nonce bool + stopOnce sync.Once + done chan struct{} } // New returns a new JWT object with the given expiry timeout. @@ -73,14 +100,18 @@ type JWT struct { // If secret is nil the DefaultSecretReader is used. // If blacklisting is enabled, the JWT object leaks a goroutine to garbage-collect expired blacklisted tokens. // Call the Stop() method to exit the goroutine. -func New(expiry time.Duration, blacklist Blacklist, secret io.Reader) (*JWT, error) { - if expiry <= 0 { - expiry = DefaultExpiry +func New(options ...JWTOption) (*JWT, error) { + jwt := &JWT{} + for _, option := range options { + option(jwt) } - if secret == nil { - secret = DefaultSecretReader + if jwt.expiry <= 0 { + jwt.expiry = DefaultExpiry } - secret = io.LimitReader(secret, KeySize) + if jwt.secretReader == nil { + jwt.secretReader = DefaultSecretReader + } + secret := io.LimitReader(jwt.secretReader, KeySize) key := make([]byte, KeySize) i, err := secret.Read(key) if err != nil { @@ -89,12 +120,8 @@ func New(expiry time.Duration, blacklist Blacklist, secret io.Reader) (*JWT, err if i < KeySize { return nil, ErrInvalidKeySize } - jwt := &JWT{ - key: key, - expiry: expiry, - blacklist: blacklist, - } - if blacklist != nil { + jwt.key = key + if jwt.blacklist != nil { jwt.done = make(chan struct{}) go jwt.clean() } @@ -128,7 +155,7 @@ func (jwt *JWT) Invalidate(t *Token) error { if t.Header.GetString(TypClaim) != Typ { return ErrNoJWT } - h := ParseHash(t.Header.GetString(AlgClaim)) + h := NewHash(t.Header.GetString(AlgClaim)) if h == nil { return ErrUnsupportedAlg } @@ -190,7 +217,7 @@ func (jwt *JWT) Sign(t *Token) (err error) { if t.Header.GetString(TypClaim) != Typ { return ErrNoJWT } - h := ParseHash(t.Header.GetString(AlgClaim)) + h := NewHash(t.Header.GetString(AlgClaim)) if h == nil { return ErrUnsupportedAlg } @@ -201,6 +228,9 @@ func (jwt *JWT) Sign(t *Token) (err error) { if _, ok := t.Claims.Get(NbfClaim); !ok { t.Claims.Set(NbfClaim, NewNbf(now)) } + if jwt.nonce { + t.Claims.Set(NonceClaim, enc.EncodeToString(key16())) + } head, err := json.Marshal(t.Header) if err != nil { return @@ -230,7 +260,7 @@ func (jwt *JWT) Verify(t *Token) error { if t.Header.GetString(TypClaim) != Typ { return ErrNoJWT } - h := ParseHash(t.Header.GetString(AlgClaim)) + h := NewHash(t.Header.GetString(AlgClaim)) if h == nil { return ErrUnsupportedAlg } diff --git a/vendor/git.giftfish.de/ston1th/jwt/v3/rand.go b/vendor/git.giftfish.de/ston1th/jwt/v3/rand.go new file mode 100644 index 0000000..bd1b191 --- /dev/null +++ b/vendor/git.giftfish.de/ston1th/jwt/v3/rand.go @@ -0,0 +1,11 @@ +// Copyright (C) 2025 Marius Schellenberger + +package jwt + +import "crypto/rand" + +func key16() []byte { + b := make([]byte, 16) + rand.Read(b) + return b +} diff --git a/vendor/git.giftfish.de/ston1th/jwt/v3/time.go b/vendor/git.giftfish.de/ston1th/jwt/v3/time.go index 71b10ca..77e151f 100644 --- a/vendor/git.giftfish.de/ston1th/jwt/v3/time.go +++ b/vendor/git.giftfish.de/ston1th/jwt/v3/time.go @@ -1,3 +1,5 @@ +// Copyright (C) 2025 Marius Schellenberger + package jwt import "time" diff --git a/vendor/git.giftfish.de/ston1th/jwt/v3/token.go b/vendor/git.giftfish.de/ston1th/jwt/v3/token.go index a1f1dd1..772415c 100644 --- a/vendor/git.giftfish.de/ston1th/jwt/v3/token.go +++ b/vendor/git.giftfish.de/ston1th/jwt/v3/token.go @@ -1,4 +1,4 @@ -// Copyright (C) 2018 Marius Schellenberger +// Copyright (C) 2025 Marius Schellenberger package jwt @@ -45,7 +45,7 @@ func NewToken(claims Claims, hash Hash) *Token { claims = make(Claims) } if hash == nil { - hash = NewHS256() + hash = NewHash(HS256Name) } return &Token{ Header: Claims{ diff --git a/vendor/github.com/RoaringBitmap/roaring/.drone.yml b/vendor/github.com/RoaringBitmap/roaring/.drone.yml index 698cd0e..7936bfe 100644 --- a/vendor/github.com/RoaringBitmap/roaring/.drone.yml +++ b/vendor/github.com/RoaringBitmap/roaring/.drone.yml @@ -11,7 +11,6 @@ steps: commands: - go get -t - go test - - go test -race -run TestConcurrent* - go build -tags appengine - go test -tags appengine - GOARCH=386 go build diff --git a/vendor/github.com/RoaringBitmap/roaring/Makefile b/vendor/github.com/RoaringBitmap/roaring/Makefile deleted file mode 100644 index 0a4f9f0..0000000 --- a/vendor/github.com/RoaringBitmap/roaring/Makefile +++ /dev/null @@ -1,107 +0,0 @@ -.PHONY: help all test format fmtcheck vet lint qa deps clean nuke ser fetch-real-roaring-datasets - - - - - - - - -# Display general help about this command -help: - @echo "" - @echo "The following commands are available:" - @echo "" - @echo " make qa : Run all the tests" - @echo " make test : Run the unit tests" - @echo "" - @echo " make format : Format the source code" - @echo " make fmtcheck : Check if the source code has been formatted" - @echo " make vet : Check for suspicious constructs" - @echo " make lint : Check for style errors" - @echo "" - @echo " make deps : Get the dependencies" - @echo " make clean : Remove any build artifact" - @echo " make nuke : Deletes any intermediate file" - @echo "" - @echo " make fuzz-smat : Fuzzy testing with smat" - @echo " make fuzz-stream : Fuzzy testing with stream deserialization" - @echo " make fuzz-buffer : Fuzzy testing with buffer deserialization" - @echo "" - -# Alias for help target -all: help -test: - go test - go test -race -run TestConcurrent* -# Format the source code -format: - @find ./ -type f -name "*.go" -exec gofmt -w {} \; - -# Check if the source code has been formatted -fmtcheck: - @mkdir -p target - @find ./ -type f -name "*.go" -exec gofmt -d {} \; | tee target/format.diff - @test ! -s target/format.diff || { echo "ERROR: the source code has not been formatted - please use 'make format' or 'gofmt'"; exit 1; } - -# Check for syntax errors -vet: - GOPATH=$(GOPATH) go vet ./... - -# Check for style errors -lint: - GOPATH=$(GOPATH) PATH=$(GOPATH)/bin:$(PATH) golint ./... - - - - - -# Alias to run all quality-assurance checks -qa: fmtcheck test vet lint - -# --- INSTALL --- - -# Get the dependencies -deps: - GOPATH=$(GOPATH) go get github.com/stretchr/testify - GOPATH=$(GOPATH) go get github.com/bits-and-blooms/bitset - GOPATH=$(GOPATH) go get github.com/golang/lint/golint - GOPATH=$(GOPATH) go get github.com/mschoch/smat - GOPATH=$(GOPATH) go get github.com/dvyukov/go-fuzz/go-fuzz - GOPATH=$(GOPATH) go get github.com/dvyukov/go-fuzz/go-fuzz-build - GOPATH=$(GOPATH) go get github.com/glycerine/go-unsnap-stream - GOPATH=$(GOPATH) go get github.com/philhofer/fwd - GOPATH=$(GOPATH) go get github.com/jtolds/gls - -fuzz-smat: - go test -tags=gofuzz -run=TestGenerateSmatCorpus - go-fuzz-build -func FuzzSmat github.com/RoaringBitmap/roaring - go-fuzz -bin=./roaring-fuzz.zip -workdir=workdir/ -timeout=200 - - -fuzz-stream: - go-fuzz-build -func FuzzSerializationStream github.com/RoaringBitmap/roaring - go-fuzz -bin=./roaring-fuzz.zip -workdir=workdir/ -timeout=200 - - -fuzz-buffer: - go-fuzz-build -func FuzzSerializationBuffer github.com/RoaringBitmap/roaring - go-fuzz -bin=./roaring-fuzz.zip -workdir=workdir/ -timeout=200 - -# Remove any build artifact -clean: - GOPATH=$(GOPATH) go clean ./... - -# Deletes any intermediate file -nuke: - rm -rf ./target - GOPATH=$(GOPATH) go clean -i ./... - -cover: - go test -coverprofile=coverage.out - go tool cover -html=coverage.out - -fetch-real-roaring-datasets: - # pull github.com/RoaringBitmap/real-roaring-datasets -> testdata/real-roaring-datasets - git submodule init - git submodule update diff --git a/vendor/github.com/RoaringBitmap/roaring/README.md b/vendor/github.com/RoaringBitmap/roaring/README.md index 753b806..f6705df 100644 --- a/vendor/github.com/RoaringBitmap/roaring/README.md +++ b/vendor/github.com/RoaringBitmap/roaring/README.md @@ -1,5 +1,7 @@ -roaring [![GoDoc](https://godoc.org/github.com/RoaringBitmap/roaring/roaring64?status.svg)](https://godoc.org/github.com/RoaringBitmap/roaring/roaring64) [![Go Report Card](https://goreportcard.com/badge/RoaringBitmap/roaring)](https://goreportcard.com/report/github.com/RoaringBitmap/roaring) -[![Build Status](https://cloud.drone.io/api/badges/RoaringBitmap/roaring/status.svg)](https://cloud.drone.io/RoaringBitmap/roaring) +# roaring + +[![GoDoc](https://godoc.org/github.com/RoaringBitmap/roaring?status.svg)](https://godoc.org/github.com/RoaringBitmap/roaring) [![Go Report Card](https://goreportcard.com/badge/RoaringBitmap/roaring)](https://goreportcard.com/report/github.com/RoaringBitmap/roaring) + ![Go-CI](https://github.com/RoaringBitmap/roaring/workflows/Go-CI/badge.svg) ![Go-ARM-CI](https://github.com/RoaringBitmap/roaring/workflows/Go-ARM-CI/badge.svg) ![Go-Windows-CI](https://github.com/RoaringBitmap/roaring/workflows/Go-Windows-CI/badge.svg) @@ -8,7 +10,7 @@ roaring [![GoDoc](https://godoc.org/github.com/RoaringBitmap/roaring/roaring64?s This is a go version of the Roaring bitmap data structure. Roaring bitmaps are used by several major systems such as [Apache Lucene][lucene] and derivative systems such as [Solr][solr] and -[Elasticsearch][elasticsearch], [Apache Druid (Incubating)][druid], [LinkedIn Pinot][pinot], [Netflix Atlas][atlas], [Apache Spark][spark], [OpenSearchServer][opensearchserver], [anacrolix/torrent][anacrolix/torrent], [Whoosh][whoosh], [Pilosa][pilosa], [Microsoft Visual Studio Team Services (VSTS)][vsts], and eBay's [Apache Kylin][kylin]. The YouTube SQL Engine, [Google Procella](https://research.google/pubs/pub48388/), uses Roaring bitmaps for indexing. +[Elasticsearch][elasticsearch], [Apache Druid (Incubating)][druid], [LinkedIn Pinot][pinot], [Netflix Atlas][atlas], [Apache Spark][spark], [OpenSearchServer][opensearchserver], [anacrolix/torrent][anacrolix/torrent], [Whoosh][whoosh], [Redpanda](https://github.com/redpanda-data/redpanda), [Pilosa][pilosa], [Microsoft Visual Studio Team Services (VSTS)][vsts], and eBay's [Apache Kylin][kylin]. The YouTube SQL Engine, [Google Procella](https://research.google/pubs/pub48388/), uses Roaring bitmaps for indexing. [lucene]: https://lucene.apache.org/ [solr]: https://lucene.apache.org/solr/ @@ -31,17 +33,17 @@ Roaring bitmaps are found to work well in many important applications: The ``roaring`` Go library is used by * [anacrolix/torrent] -* [runv](https://github.com/hyperhq/runv) * [InfluxDB](https://www.influxdata.com) * [Pilosa](https://www.pilosa.com/) * [Bleve](http://www.blevesearch.com) +* [Weaviate](https://github.com/weaviate/weaviate) * [lindb](https://github.com/lindb/lindb) * [Elasticell](https://github.com/deepfabric/elasticell) * [SourceGraph](https://github.com/sourcegraph/sourcegraph) * [M3](https://github.com/m3db/m3) * [trident](https://github.com/NetApp/trident) * [Husky](https://www.datadoghq.com/blog/engineering/introducing-husky/) - +* [FrostDB](https://github.com/polarsignals/frostdb) This library is used in production in several systems, it is part of the [Awesome Go collection](https://awesome-go.com). @@ -99,7 +101,7 @@ whether you like it or not. That can become very wasteful. This being said, there are definitively cases where attempting to use compressed bitmaps is wasteful. For example, if you have a small universe size. E.g., your bitmaps represent sets of integers -from [0,n) where n is small (e.g., n=64 or n=128). If you are able to uncompressed BitSet and +from [0,n) where n is small (e.g., n=64 or n=128). If you can use uncompressed BitSet and it does not blow up your memory usage, then compressed bitmaps are probably not useful to you. In fact, if you do not need compression, then a BitSet offers remarkable speed. @@ -134,7 +136,7 @@ There is a big problem with these formats however that can hurt you badly in som Roaring solves this problem. It works in the following manner. It divides the data into chunks of 216 integers (e.g., [0, 216), [216, 2 x 216), ...). Within a chunk, it can use an uncompressed bitmap, a simple list of integers, -or a list of runs. Whatever format it uses, they all allow you to check for the present of any one value quickly +or a list of runs. Whatever format it uses, they all allow you to check for the presence of any one value quickly (e.g., with a binary search). The net result is that Roaring can compute many operations much faster than run-length-encoded formats like WAH, EWAH, Concise... Maybe surprisingly, Roaring also generally offers better compression ratios. diff --git a/vendor/github.com/RoaringBitmap/roaring/arraycontainer.go b/vendor/github.com/RoaringBitmap/roaring/arraycontainer.go index 9541fd5..80fa676 100644 --- a/vendor/github.com/RoaringBitmap/roaring/arraycontainer.go +++ b/vendor/github.com/RoaringBitmap/roaring/arraycontainer.go @@ -17,8 +17,17 @@ func (ac *arrayContainer) String() string { } func (ac *arrayContainer) fillLeastSignificant16bits(x []uint32, i int, mask uint32) int { + if i < 0 { + panic("negative index") + } + if len(ac.content) == 0 { + return i + } + _ = x[len(ac.content)-1+i] + _ = ac.content[len(ac.content)-1] for k := 0; k < len(ac.content); k++ { - x[k+i] = uint32(ac.content[k]) | mask + x[k+i] = + uint32(ac.content[k]) | mask } return i + len(ac.content) } @@ -655,10 +664,54 @@ func (ac *arrayContainer) iandNot(a container) container { } func (ac *arrayContainer) iandNotRun16(rc *runContainer16) container { - rcb := rc.toBitmapContainer() - acb := ac.toBitmapContainer() - acb.iandNotBitmapSurely(rcb) - *ac = *(acb.toArrayContainer()) + // Fast path: if either the array container or the run container is empty, the result is the array. + if ac.isEmpty() || rc.isEmpty() { + // Empty + return ac + } + // Fast path: if the run container is full, the result is empty. + if rc.isFull() { + ac.content = ac.content[:0] + return ac + } + current_run := 0 + // All values in [start_run, end_end] are part of the run + start_run := rc.iv[current_run].start + end_end := start_run + rc.iv[current_run].length + // We are going to read values in the array at index i, and we are + // going to write them at index pos. So we do in-place processing. + // We always have that pos <= i by construction. So we can either + // overwrite a value just read, or a value that was previous read. + pos := 0 + i := 0 + for ; i < len(ac.content); i++ { + if ac.content[i] < start_run { + // the value in the array appears before the run [start_run, end_end] + ac.content[pos] = ac.content[i] + pos++ + } else if ac.content[i] <= end_end { + // nothing to do, the value is in the array but also in the run. + } else { + // We have the value in the array after the run. We cannot tell + // whether we need to keep it or not. So let us move to another run. + if current_run+1 < len(rc.iv) { + current_run++ + start_run = rc.iv[current_run].start + end_end = start_run + rc.iv[current_run].length + i-- // retry with the same i + } else { + // We have exhausted the number of runs. We can keep the rest of the values + // from i to len(ac.content) - 1 inclusively. + break // We are done, the rest of the array will be kept + } + } + } + for ; i < len(ac.content); i++ { + ac.content[pos] = ac.content[i] + pos++ + } + // We 'shink' the slice. + ac.content = ac.content[:pos] return ac } diff --git a/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer.go b/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer.go index 71029f4..bf08bfc 100644 --- a/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer.go +++ b/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer.go @@ -888,13 +888,67 @@ func (bc *bitmapContainer) iandNot(a container) container { } func (bc *bitmapContainer) iandNotArray(ac *arrayContainer) container { - acb := ac.toBitmapContainer() - return bc.iandNotBitmapSurely(acb) + if ac.isEmpty() || bc.isEmpty() { + // Nothing to do. + return bc + } + + // Word by word, we remove the elements in ac from bc. The approach is to build + // a mask of the elements to remove, and then apply it to the bitmap. + wordIdx := uint16(0) + mask := uint64(0) + for i, v := range ac.content { + if v/64 != wordIdx { + // Flush the current word. + if i != 0 { + // We're removing bits that are set in the mask and in the current word. + // To figure out the cardinality change, we count the number of bits that + // are set in the mask and in the current word. + mask &= bc.bitmap[wordIdx] + bc.bitmap[wordIdx] &= ^mask + bc.cardinality -= int(popcount(mask)) + } + + wordIdx = v / 64 + mask = 0 + } + mask |= 1 << (v % 64) + } + + // Flush the last word. + mask &= bc.bitmap[wordIdx] + bc.bitmap[wordIdx] &= ^mask + bc.cardinality -= int(popcount(mask)) + + if bc.getCardinality() <= arrayDefaultMaxSize { + return bc.toArrayContainer() + } + return bc } func (bc *bitmapContainer) iandNotRun16(rc *runContainer16) container { - rcb := rc.toBitmapContainer() - return bc.iandNotBitmapSurely(rcb) + if rc.isEmpty() || bc.isEmpty() { + // Nothing to do. + return bc + } + + wordRangeStart := rc.iv[0].start / 64 + wordRangeEnd := (rc.iv[len(rc.iv)-1].last()) / 64 // inclusive + + cardinalityChange := popcntSlice(bc.bitmap[wordRangeStart : wordRangeEnd+1]) // before cardinality - after cardinality (for word range) + + for _, iv := range rc.iv { + resetBitmapRange(bc.bitmap, int(iv.start), int(iv.last())+1) + } + + cardinalityChange -= popcntSlice(bc.bitmap[wordRangeStart : wordRangeEnd+1]) + + bc.cardinality -= int(cardinalityChange) + + if bc.getCardinality() <= arrayDefaultMaxSize { + return bc.toArrayContainer() + } + return bc } func (bc *bitmapContainer) andNotArray(value2 *arrayContainer) container { @@ -1062,7 +1116,6 @@ func (bc *bitmapContainer) PrevSetBit(i int) int { // reference the java implementation // https://github.com/RoaringBitmap/RoaringBitmap/blob/master/src/main/java/org/roaringbitmap/BitmapContainer.java#L875-L892 -// func (bc *bitmapContainer) numberOfRuns() int { if bc.cardinality == 0 { return 0 diff --git a/vendor/github.com/RoaringBitmap/roaring/internal/byte_input.go b/vendor/github.com/RoaringBitmap/roaring/internal/byte_input.go index 3e5490a..d5ebb91 100644 --- a/vendor/github.com/RoaringBitmap/roaring/internal/byte_input.go +++ b/vendor/github.com/RoaringBitmap/roaring/internal/byte_input.go @@ -10,6 +10,11 @@ type ByteInput interface { // Next returns a slice containing the next n bytes from the buffer, // advancing the buffer as if the bytes had been returned by Read. Next(n int) ([]byte, error) + // NextReturnsSafeSlice returns true if Next() returns a safe slice as opposed + // to a slice that points to an underlying buffer possibly owned by another system. + // When NextReturnsSafeSlice returns false, the result from Next() should be copied + // before it is modified (i.e., it is immutable). + NextReturnsSafeSlice() bool // ReadUInt32 reads uint32 with LittleEndian order ReadUInt32() (uint32, error) // ReadUInt16 reads uint16 with LittleEndian order @@ -42,6 +47,25 @@ type ByteBuffer struct { off int } +// NewByteBuffer creates a new ByteBuffer. +func NewByteBuffer(buf []byte) *ByteBuffer { + return &ByteBuffer{ + buf: buf, + } +} + +var _ io.Reader = (*ByteBuffer)(nil) + +// Read implements io.Reader. +func (b *ByteBuffer) Read(p []byte) (int, error) { + data, err := b.Next(len(p)) + if err != nil { + return 0, err + } + copy(p, data) + return len(data), nil +} + // Next returns a slice containing the next n bytes from the reader // If there are fewer bytes than the given n, io.ErrUnexpectedEOF will be returned func (b *ByteBuffer) Next(n int) ([]byte, error) { @@ -57,6 +81,12 @@ func (b *ByteBuffer) Next(n int) ([]byte, error) { return data, nil } +// NextReturnsSafeSlice returns false since ByteBuffer might hold +// an array owned by some other systems. +func (b *ByteBuffer) NextReturnsSafeSlice() bool { + return false +} + // ReadUInt32 reads uint32 with LittleEndian order func (b *ByteBuffer) ReadUInt32() (uint32, error) { if len(b.buf)-b.off < 4 { @@ -109,26 +139,45 @@ func (b *ByteBuffer) Reset(buf []byte) { type ByteInputAdapter struct { r io.Reader readBytes int + buf [4]byte +} + +var _ io.Reader = (*ByteInputAdapter)(nil) + +// Read implements io.Reader. +func (b *ByteInputAdapter) Read(buf []byte) (int, error) { + m, err := io.ReadAtLeast(b.r, buf, len(buf)) + b.readBytes += m + + if err != nil { + return 0, err + } + + return m, nil } // Next returns a slice containing the next n bytes from the buffer, // advancing the buffer as if the bytes had been returned by Read. func (b *ByteInputAdapter) Next(n int) ([]byte, error) { buf := make([]byte, n) - m, err := io.ReadAtLeast(b.r, buf, n) - b.readBytes += m + _, err := b.Read(buf) if err != nil { return nil, err } - return buf, nil } +// NextReturnsSafeSlice returns true since ByteInputAdapter always returns a slice +// allocated with make([]byte, ...) +func (b *ByteInputAdapter) NextReturnsSafeSlice() bool { + return true +} + // ReadUInt32 reads uint32 with LittleEndian order func (b *ByteInputAdapter) ReadUInt32() (uint32, error) { - buf, err := b.Next(4) - + buf := b.buf[:4] + _, err := b.Read(buf) if err != nil { return 0, err } @@ -138,8 +187,8 @@ func (b *ByteInputAdapter) ReadUInt32() (uint32, error) { // ReadUInt16 reads uint16 with LittleEndian order func (b *ByteInputAdapter) ReadUInt16() (uint16, error) { - buf, err := b.Next(2) - + buf := b.buf[:2] + _, err := b.Read(buf) if err != nil { return 0, err } diff --git a/vendor/github.com/RoaringBitmap/roaring/roaring.go b/vendor/github.com/RoaringBitmap/roaring/roaring.go index 7220da2..a31cdbd 100644 --- a/vendor/github.com/RoaringBitmap/roaring/roaring.go +++ b/vendor/github.com/RoaringBitmap/roaring/roaring.go @@ -13,6 +13,7 @@ import ( "strconv" "github.com/RoaringBitmap/roaring/internal" + "github.com/bits-and-blooms/bitset" ) // Bitmap represents a compressed bitmap where you can add integers. @@ -53,17 +54,186 @@ func (rb *Bitmap) ToBytes() ([]byte, error) { return rb.highlowcontainer.toBytes() } +const wordSize = uint64(64) +const log2WordSize = uint64(6) +const capacity = ^uint64(0) +const bitmapContainerSize = (1 << 16) / 64 // bitmap size in words + +// DenseSize returns the size of the bitmap when stored as a dense bitmap. +func (rb *Bitmap) DenseSize() uint64 { + if rb.highlowcontainer.size() == 0 { + return 0 + } + + maximum := 1 + uint64(rb.Maximum()) + if maximum > (capacity - wordSize + 1) { + return uint64(capacity >> log2WordSize) + } + + return uint64((maximum + (wordSize - 1)) >> log2WordSize) +} + +// ToDense returns a slice of uint64s representing the bitmap as a dense bitmap. +// Useful to convert a roaring bitmap to a format that can be used by other libraries +// like https://github.com/bits-and-blooms/bitset or https://github.com/kelindar/bitmap +func (rb *Bitmap) ToDense() []uint64 { + sz := rb.DenseSize() + if sz == 0 { + return nil + } + + bitmap := make([]uint64, sz) + rb.WriteDenseTo(bitmap) + return bitmap +} + +// FromDense creates a bitmap from a slice of uint64s representing the bitmap as a dense bitmap. +// Useful to convert bitmaps from libraries like https://github.com/bits-and-blooms/bitset or +// https://github.com/kelindar/bitmap into roaring bitmaps fast and with convenience. +// +// This function will not create any run containers, only array and bitmap containers. It's up to +// the caller to call RunOptimize if they want to further compress the runs of consecutive values. +// +// When doCopy is true, the bitmap is copied into a new slice for each bitmap container. +// This is useful when the bitmap is going to be modified after this function returns or if it's +// undesirable to hold references to large bitmaps which the GC would not be able to collect. +// One copy can still happen even when doCopy is false if the bitmap length is not divisible +// by bitmapContainerSize. +// +// See also FromBitSet. +func FromDense(bitmap []uint64, doCopy bool) *Bitmap { + sz := (len(bitmap) + bitmapContainerSize - 1) / bitmapContainerSize // round up + rb := &Bitmap{ + highlowcontainer: roaringArray{ + containers: make([]container, 0, sz), + keys: make([]uint16, 0, sz), + needCopyOnWrite: make([]bool, 0, sz), + }, + } + rb.FromDense(bitmap, doCopy) + return rb +} + +// FromDense unmarshalls from a slice of uint64s representing the bitmap as a dense bitmap. +// Useful to convert bitmaps from libraries like https://github.com/bits-and-blooms/bitset or +// https://github.com/kelindar/bitmap into roaring bitmaps fast and with convenience. +// Callers are responsible for ensuring that the bitmap is empty before calling this function. +// +// This function will not create any run containers, only array and bitmap containers. It is up to +// the caller to call RunOptimize if they want to further compress the runs of consecutive values. +// +// When doCopy is true, the bitmap is copied into a new slice for each bitmap container. +// This is useful when the bitmap is going to be modified after this function returns or if it's +// undesirable to hold references to large bitmaps which the GC would not be able to collect. +// One copy can still happen even when doCopy is false if the bitmap length is not divisible +// by bitmapContainerSize. +// +// See FromBitSet. +func (rb *Bitmap) FromDense(bitmap []uint64, doCopy bool) { + if len(bitmap) == 0 { + return + } + + var k uint16 + const size = bitmapContainerSize + + for len(bitmap) > 0 { + hi := size + if len(bitmap) < size { + hi = len(bitmap) + } + + words := bitmap[:hi] + count := int(popcntSlice(words)) + + switch { + case count > arrayDefaultMaxSize: + c := &bitmapContainer{cardinality: count, bitmap: words} + cow := true + + if doCopy || len(words) < size { + c.bitmap = make([]uint64, size) + copy(c.bitmap, words) + cow = false + } + + rb.highlowcontainer.appendContainer(k, c, cow) + + case count > 0: + c := &arrayContainer{content: make([]uint16, count)} + var pos, base int + for _, w := range words { + for w != 0 { + t := w & -w + c.content[pos] = uint16(base + int(popcount(t-1))) + pos++ + w ^= t + } + base += 64 + } + rb.highlowcontainer.appendContainer(k, c, false) + } + + bitmap = bitmap[hi:] + k++ + } +} + +// WriteDenseTo writes to a slice of uint64s representing the bitmap as a dense bitmap. +// Callers are responsible for allocating enough space in the bitmap using DenseSize. +// Useful to convert a roaring bitmap to a format that can be used by other libraries +// like https://github.com/bits-and-blooms/bitset or https://github.com/kelindar/bitmap +func (rb *Bitmap) WriteDenseTo(bitmap []uint64) { + for i, ct := range rb.highlowcontainer.containers { + hb := uint32(rb.highlowcontainer.keys[i]) << 16 + + switch c := ct.(type) { + case *arrayContainer: + for _, x := range c.content { + n := int(hb | uint32(x)) + bitmap[n>>log2WordSize] |= uint64(1) << uint(x%64) + } + + case *bitmapContainer: + copy(bitmap[int(hb)>>log2WordSize:], c.bitmap) + + case *runContainer16: + for j := range c.iv { + start := uint32(c.iv[j].start) + end := start + uint32(c.iv[j].length) + 1 + lo := int(hb|start) >> log2WordSize + hi := int(hb|(end-1)) >> log2WordSize + + if lo == hi { + bitmap[lo] |= (^uint64(0) << uint(start%64)) & + (^uint64(0) >> (uint(-end) % 64)) + continue + } + + bitmap[lo] |= ^uint64(0) << uint(start%64) + for n := lo + 1; n < hi; n++ { + bitmap[n] = ^uint64(0) + } + bitmap[hi] |= ^uint64(0) >> (uint(-end) % 64) + } + default: + panic("unsupported container type") + } + } +} + // Checksum computes a hash (currently FNV-1a) for a bitmap that is suitable for // using bitmaps as elements in hash sets or as keys in hash maps, as well as // generally quicker comparisons. // The implementation is biased towards efficiency in little endian machines, so // expect some extra CPU cycles and memory to be used if your machine is big endian. -// Likewise, don't use this to verify integrity unless you're certain you'll load -// the bitmap on a machine with the same endianess used to create it. +// Likewise, do not use this to verify integrity unless you are certain you will load +// the bitmap on a machine with the same endianess used to create it. (Thankfully +// very few people use big endian machines these days.) func (rb *Bitmap) Checksum() uint64 { const ( offset = 14695981039346656037 - prime = 1099511628211 + prime = 1099511628211 ) var bytes []byte @@ -106,6 +276,20 @@ func (rb *Bitmap) Checksum() uint64 { return hash } +// FromUnsafeBytes reads a serialized version of this bitmap from the byte buffer without copy. +// It is the caller's responsibility to ensure that the input data is not modified and remains valid for the entire lifetime of this bitmap. +// This method avoids small allocations but holds references to the input data buffer. It is GC-friendly, but it may consume more memory eventually. +// The containers in the resulting bitmap are immutable containers tied to the provided byte array and they rely on +// copy-on-write which means that modifying them creates copies. Thus FromUnsafeBytes is more likely to be appropriate for read-only use cases, +// when the resulting bitmap can be considered immutable. +// +// See also the FromBuffer function. +// See https://github.com/RoaringBitmap/roaring/pull/395 for more details. +func (rb *Bitmap) FromUnsafeBytes(data []byte, cookieHeader ...byte) (p int64, err error) { + stream := internal.NewByteBuffer(data) + return rb.ReadFrom(stream) +} + // ReadFrom reads a serialized version of this bitmap from stream. // The format is compatible with other RoaringBitmap // implementations (Java, C) and is documented here: @@ -114,12 +298,18 @@ func (rb *Bitmap) Checksum() uint64 { // So add cookieHeader to accept the 4-byte data that has been read in roaring64.ReadFrom. // It is not necessary to pass cookieHeader when call roaring.ReadFrom to read the roaring32 data directly. func (rb *Bitmap) ReadFrom(reader io.Reader, cookieHeader ...byte) (p int64, err error) { - stream := internal.ByteInputAdapterPool.Get().(*internal.ByteInputAdapter) - stream.Reset(reader) + stream, ok := reader.(internal.ByteInput) + if !ok { + byteInputAdapter := internal.ByteInputAdapterPool.Get().(*internal.ByteInputAdapter) + byteInputAdapter.Reset(reader) + stream = byteInputAdapter + } p, err = rb.highlowcontainer.readFrom(stream, cookieHeader...) - internal.ByteInputAdapterPool.Put(stream) + if !ok { + internal.ByteInputAdapterPool.Put(stream.(*internal.ByteInputAdapter)) + } return } @@ -139,12 +329,17 @@ func (rb *Bitmap) ReadFrom(reader io.Reader, cookieHeader ...byte) (p int64, err // You should *not* change the copy-on-write status of the resulting // bitmaps (SetCopyOnWrite). // +// Thus FromBuffer is more likely to be appropriate for read-only use cases, +// when the resulting bitmap can be considered immutable. +// // If buf becomes unavailable, then a bitmap created with // FromBuffer would be effectively broken. Furthermore, any // bitmap derived from this bitmap (e.g., via Or, And) might // also be broken. Thus, before making buf unavailable, you should // call CloneCopyOnWriteContainers on all such bitmaps. // +// See also the FromUnsafeBytes function which can have better performance +// in some cases. func (rb *Bitmap) FromBuffer(buf []byte) (p int64, err error) { stream := internal.ByteBufferPool.Get().(*internal.ByteBuffer) stream.Reset(buf) @@ -194,6 +389,16 @@ func (rb *Bitmap) Clear() { rb.highlowcontainer.clear() } +// ToBitSet copies the content of the RoaringBitmap into a bitset.BitSet instance +func (rb *Bitmap) ToBitSet() *bitset.BitSet { + return bitset.From(rb.ToDense()) +} + +// FromBitSet creates a new RoaringBitmap from a bitset.BitSet instance +func FromBitSet(bitset *bitset.BitSet) *Bitmap { + return FromDense(bitset.Bytes(), false) +} + // ToArray creates a new slice containing all of the integers stored in the Bitmap in sorted order func (rb *Bitmap) ToArray() []uint32 { array := make([]uint32, rb.GetCardinality()) @@ -233,7 +438,7 @@ func BoundSerializedSizeInBytes(cardinality uint64, universeSize uint64) uint64 contnbr := (universeSize + uint64(65535)) / uint64(65536) if contnbr > cardinality { contnbr = cardinality - // we can't have more containers than we have values + // we cannot have more containers than we have values } headermax := 8*contnbr + 4 if 4 > (contnbr+7)/8 { @@ -276,9 +481,9 @@ type intIterator struct { // This way, instead of making up-to 64k allocations per full iteration // we get a single allocation and simply reinitialize the appropriate // iterator and point to it in the generic `iter` member on each key bound. - shortIter shortIterator - runIter runIterator16 - bitmapIter bitmapContainerShortIterator + shortIter shortIterator + runIter runIterator16 + bitmapIter bitmapContainerShortIterator } // HasNext returns true if there are more integers to iterate over @@ -341,14 +546,13 @@ func (ii *intIterator) AdvanceIfNeeded(minval uint32) { // IntIterator is meant to allow you to iterate through the values of a bitmap, see Initialize(a *Bitmap) type IntIterator = intIterator - // Initialize configures the existing iterator so that it can iterate through the values of // the provided bitmap. // The iteration results are undefined if the bitmap is modified (e.g., with Add or Remove). -func (p *intIterator) Initialize(a *Bitmap) { - p.pos = 0 - p.highlowcontainer = &a.highlowcontainer - p.init() +func (ii *intIterator) Initialize(a *Bitmap) { + ii.pos = 0 + ii.highlowcontainer = &a.highlowcontainer + ii.init() } type intReverseIterator struct { @@ -357,9 +561,9 @@ type intReverseIterator struct { iter shortIterable highlowcontainer *roaringArray - shortIter reverseIterator - runIter runReverseIterator16 - bitmapIter reverseBitmapContainerShortIterator + shortIter reverseIterator + runIter runReverseIterator16 + bitmapIter reverseBitmapContainerShortIterator } // HasNext returns true if there are more integers to iterate over @@ -414,10 +618,10 @@ type IntReverseIterator = intReverseIterator // Initialize configures the existing iterator so that it can iterate through the values of // the provided bitmap. // The iteration results are undefined if the bitmap is modified (e.g., with Add or Remove). -func (p *intReverseIterator) Initialize(a *Bitmap) { - p.highlowcontainer = &a.highlowcontainer - p.pos = a.highlowcontainer.size() - 1 - p.init() +func (ii *intReverseIterator) Initialize(a *Bitmap) { + ii.highlowcontainer = &a.highlowcontainer + ii.pos = a.highlowcontainer.size() - 1 + ii.init() } // ManyIntIterable allows you to iterate over the values in a Bitmap @@ -434,9 +638,9 @@ type manyIntIterator struct { iter manyIterable highlowcontainer *roaringArray - shortIter shortIterator - runIter runIterator16 - bitmapIter bitmapContainerManyIterator + shortIter shortIterator + runIter runIterator16 + bitmapIter bitmapContainerManyIterator } func (ii *manyIntIterator) init() { @@ -495,17 +699,16 @@ func (ii *manyIntIterator) NextMany64(hs64 uint64, buf []uint64) int { return n } - // ManyIntIterator is meant to allow you to iterate through the values of a bitmap, see Initialize(a *Bitmap) type ManyIntIterator = manyIntIterator // Initialize configures the existing iterator so that it can iterate through the values of // the provided bitmap. // The iteration results are undefined if the bitmap is modified (e.g., with Add or Remove). -func (p *manyIntIterator) Initialize(a *Bitmap) { - p.pos = 0 - p.highlowcontainer = &a.highlowcontainer - p.init() +func (ii *manyIntIterator) Initialize(a *Bitmap) { + ii.pos = 0 + ii.highlowcontainer = &a.highlowcontainer + ii.init() } // String creates a string representation of the Bitmap @@ -569,7 +772,7 @@ func (rb *Bitmap) Iterate(cb func(x uint32) bool) { // Iterator creates a new IntPeekable to iterate over the integers contained in the bitmap, in sorted order; // the iterator becomes invalid if the bitmap is modified (e.g., with Add or Remove). func (rb *Bitmap) Iterator() IntPeekable { - p := new(intIterator) + p := new(intIterator) p.Initialize(rb) return p } @@ -847,7 +1050,7 @@ func (rb *Bitmap) Select(x uint32) (uint32, error) { return uint32(key)<<16 + uint32(c.selectInt(uint16(remaining))), nil } } - return 0, fmt.Errorf("can't find %dth integer in a bitmap with only %d items", x, rb.GetCardinality()) + return 0, fmt.Errorf("cannot find %dth integer in a bitmap with only %d items", x, rb.GetCardinality()) } // And computes the intersection between two bitmaps and stores the result in the current bitmap diff --git a/vendor/github.com/RoaringBitmap/roaring/roaringarray.go b/vendor/github.com/RoaringBitmap/roaring/roaringarray.go index eeb3d31..079195d 100644 --- a/vendor/github.com/RoaringBitmap/roaring/roaringarray.go +++ b/vendor/github.com/RoaringBitmap/roaring/roaringarray.go @@ -4,8 +4,9 @@ import ( "bytes" "encoding/binary" "fmt" - "github.com/RoaringBitmap/roaring/internal" "io" + + "github.com/RoaringBitmap/roaring/internal" ) type container interface { @@ -112,9 +113,10 @@ func newRoaringArray() *roaringArray { // runOptimize compresses the element containers to minimize space consumed. // Q: how does this interact with copyOnWrite and needCopyOnWrite? // A: since we aren't changing the logical content, just the representation, -// we don't bother to check the needCopyOnWrite bits. We replace -// (possibly all) elements of ra.containers in-place with space -// optimized versions. +// +// we don't bother to check the needCopyOnWrite bits. We replace +// (possibly all) elements of ra.containers in-place with space +// optimized versions. func (ra *roaringArray) runOptimize() { for i := range ra.containers { ra.containers[i] = ra.containers[i].toEfficientContainer() @@ -465,9 +467,7 @@ func (ra *roaringArray) serializedSizeInBytes() uint64 { return answer } -// // spec: https://github.com/RoaringBitmap/RoaringFormatSpec -// func (ra *roaringArray) writeTo(w io.Writer) (n int64, err error) { hasRun := ra.hasRunCompression() isRunSizeInBytes := 0 @@ -544,15 +544,14 @@ func (ra *roaringArray) writeTo(w io.Writer) (n int64, err error) { return n, nil } -// // spec: https://github.com/RoaringBitmap/RoaringFormatSpec -// func (ra *roaringArray) toBytes() ([]byte, error) { var buf bytes.Buffer _, err := ra.writeTo(&buf) return buf.Bytes(), err } +// Reads a serialized roaringArray from a byte slice. func (ra *roaringArray) readFrom(stream internal.ByteInput, cookieHeader ...byte) (int64, error) { var cookie uint32 var err error @@ -567,6 +566,8 @@ func (ra *roaringArray) readFrom(stream internal.ByteInput, cookieHeader ...byte return stream.GetReadBytes(), fmt.Errorf("error in roaringArray.readFrom: could not read initial cookie: %s", err) } } + // If NextReturnsSafeSlice is false, then willNeedCopyOnWrite should be true + willNeedCopyOnWrite := !stream.NextReturnsSafeSlice() var size uint32 var isRunBitmap []byte @@ -631,7 +632,7 @@ func (ra *roaringArray) readFrom(stream internal.ByteInput, cookieHeader ...byte key := keycard[2*i] card := int(keycard[2*i+1]) + 1 ra.keys[i] = key - ra.needCopyOnWrite[i] = true + ra.needCopyOnWrite[i] = willNeedCopyOnWrite if isRunBitmap != nil && isRunBitmap[i/8]&(1<<(i%8)) != 0 { // run container diff --git a/vendor/github.com/RoaringBitmap/roaring/runcontainer.go b/vendor/github.com/RoaringBitmap/roaring/runcontainer.go index 4ce48a2..7098ba2 100644 --- a/vendor/github.com/RoaringBitmap/roaring/runcontainer.go +++ b/vendor/github.com/RoaringBitmap/roaring/runcontainer.go @@ -47,6 +47,7 @@ import ( // runContainer16 does run-length encoding of sets of // uint16 integers. type runContainer16 struct { + // iv is a slice of sorted, non-overlapping, non-adjacent intervals. iv []interval16 } @@ -253,10 +254,8 @@ func newRunContainer16FromBitmapContainer(bc *bitmapContainer) *runContainer16 { } -// // newRunContainer16FromArray populates a new // runContainer16 from the contents of arr. -// func newRunContainer16FromArray(arr *arrayContainer) *runContainer16 { // keep this in sync with newRunContainer16FromVals above @@ -834,24 +833,23 @@ func (rc *runContainer16) numIntervals() int { // If key is not already present, then whichInterval16 is // set as follows: // -// a) whichInterval16 == len(rc.iv)-1 if key is beyond our -// last interval16 in rc.iv; +// a) whichInterval16 == len(rc.iv)-1 if key is beyond our +// last interval16 in rc.iv; // -// b) whichInterval16 == -1 if key is before our first -// interval16 in rc.iv; +// b) whichInterval16 == -1 if key is before our first +// interval16 in rc.iv; // -// c) whichInterval16 is set to the minimum index of rc.iv -// which comes strictly before the key; -// so rc.iv[whichInterval16].last < key, -// and if whichInterval16+1 exists, then key < rc.iv[whichInterval16+1].start -// (Note that whichInterval16+1 won't exist when -// whichInterval16 is the last interval.) +// c) whichInterval16 is set to the minimum index of rc.iv +// which comes strictly before the key; +// so rc.iv[whichInterval16].last < key, +// and if whichInterval16+1 exists, then key < rc.iv[whichInterval16+1].start +// (Note that whichInterval16+1 won't exist when +// whichInterval16 is the last interval.) // // runContainer16.search always returns whichInterval16 < len(rc.iv). // // The search space is from startIndex to endxIndex. If endxIndex is set to zero, then there // no upper bound. -// func (rc *runContainer16) searchRange(key int, startIndex int, endxIndex int) (whichInterval16 int, alreadyPresent bool, numCompares int) { n := int(len(rc.iv)) if n == 0 { @@ -937,21 +935,20 @@ func (rc *runContainer16) searchRange(key int, startIndex int, endxIndex int) (w // If key is not already present, then whichInterval16 is // set as follows: // -// a) whichInterval16 == len(rc.iv)-1 if key is beyond our -// last interval16 in rc.iv; +// a) whichInterval16 == len(rc.iv)-1 if key is beyond our +// last interval16 in rc.iv; // -// b) whichInterval16 == -1 if key is before our first -// interval16 in rc.iv; +// b) whichInterval16 == -1 if key is before our first +// interval16 in rc.iv; // -// c) whichInterval16 is set to the minimum index of rc.iv -// which comes strictly before the key; -// so rc.iv[whichInterval16].last < key, -// and if whichInterval16+1 exists, then key < rc.iv[whichInterval16+1].start -// (Note that whichInterval16+1 won't exist when -// whichInterval16 is the last interval.) +// c) whichInterval16 is set to the minimum index of rc.iv +// which comes strictly before the key; +// so rc.iv[whichInterval16].last < key, +// and if whichInterval16+1 exists, then key < rc.iv[whichInterval16+1].start +// (Note that whichInterval16+1 won't exist when +// whichInterval16 is the last interval.) // // runContainer16.search always returns whichInterval16 < len(rc.iv). -// func (rc *runContainer16) search(key int) (whichInterval16 int, alreadyPresent bool, numCompares int) { return rc.searchRange(key, 0, 0) } @@ -994,7 +991,6 @@ func newRunContainer16() *runContainer16 { // newRunContainer16CopyIv creates a run container, initializing // with a copy of the supplied iv slice. -// func newRunContainer16CopyIv(iv []interval16) *runContainer16 { rc := &runContainer16{ iv: make([]interval16, len(iv)), @@ -1011,7 +1007,6 @@ func (rc *runContainer16) Clone() *runContainer16 { // newRunContainer16TakeOwnership returns a new runContainer16 // backed by the provided iv slice, which we will // assume exclusive control over from now on. -// func newRunContainer16TakeOwnership(iv []interval16) *runContainer16 { rc := &runContainer16{ iv: iv, @@ -2006,7 +2001,6 @@ func (rc *runContainer16) not(firstOfRange, endx int) container { // Current routine is correct but // makes 2 more passes through the arrays than should be // strictly necessary. Measure both ways though--this may not matter. -// func (rc *runContainer16) Not(firstOfRange, endx int) *runContainer16 { if firstOfRange > endx { @@ -2329,7 +2323,6 @@ func runArrayUnionToRuns(rc *runContainer16, ac *arrayContainer) ([]interval16, // the backing array, and then you write // the answer at the beginning. What this // trick does is minimize memory allocations. -// func (rc *runContainer16) lazyIOR(a container) container { // not lazy at the moment return rc.ior(a) diff --git a/vendor/github.com/RoaringBitmap/roaring/serialization.go b/vendor/github.com/RoaringBitmap/roaring/serialization.go index 70e3bbc..dbfecc8 100644 --- a/vendor/github.com/RoaringBitmap/roaring/serialization.go +++ b/vendor/github.com/RoaringBitmap/roaring/serialization.go @@ -7,7 +7,6 @@ import ( // writeTo for runContainer16 follows this // spec: https://github.com/RoaringBitmap/RoaringFormatSpec -// func (b *runContainer16) writeTo(stream io.Writer) (int, error) { buf := make([]byte, 2+4*len(b.iv)) binary.LittleEndian.PutUint16(buf[0:], uint16(len(b.iv))) diff --git a/vendor/github.com/RoaringBitmap/roaring/serialization_littleendian.go b/vendor/github.com/RoaringBitmap/roaring/serialization_littleendian.go index 2e4ea59..6e3a5d5 100644 --- a/vendor/github.com/RoaringBitmap/roaring/serialization_littleendian.go +++ b/vendor/github.com/RoaringBitmap/roaring/serialization_littleendian.go @@ -79,12 +79,12 @@ func (bc *bitmapContainer) asLittleEndianByteSlice() []byte { // Deserialization code follows -//// +// // // These methods (byteSliceAsUint16Slice,...) do not make copies, // they are pointer-based (unsafe). The caller is responsible to // ensure that the input slice does not get garbage collected, deleted // or modified while you hold the returned slince. -//// +// // func byteSliceAsUint16Slice(slice []byte) (result []uint16) { // here we create a new slice holder if len(slice)%2 != 0 { panic("Slice size should be divisible by 2") @@ -295,7 +295,6 @@ func byteSliceAsBoolSlice(slice []byte) (result []bool) { // bitmap derived from this bitmap (e.g., via Or, And) might // also be broken. Thus, before making buf unavailable, you should // call CloneCopyOnWriteContainers on all such bitmaps. -// func (rb *Bitmap) FrozenView(buf []byte) error { return rb.highlowcontainer.frozenView(buf) } @@ -313,7 +312,7 @@ func (rb *Bitmap) FrozenView(buf []byte) error { * uint8_t[num_containers] *
uint32_t * - *
is a 4-byte value which is a bit union of FROZEN_COOKIE (15 bits) + *
is a 4-byte value which is a bit union of frozenCookie (15 bits) * and the number of containers (17 bits). * * stores number of elements for every container. @@ -329,43 +328,50 @@ func (rb *Bitmap) FrozenView(buf []byte) error { * All members have their native alignments during deserilization except
, * which is not guaranteed to be aligned by 4 bytes. */ -const FROZEN_COOKIE = 13766 +const frozenCookie = 13766 var ( - FrozenBitmapInvalidCookie = errors.New("header does not contain the FROZEN_COOKIE") - FrozenBitmapBigEndian = errors.New("loading big endian frozen bitmaps is not supported") - FrozenBitmapIncomplete = errors.New("input buffer too small to contain a frozen bitmap") - FrozenBitmapOverpopulated = errors.New("too many containers") - FrozenBitmapUnexpectedData = errors.New("spurious data in input") - FrozenBitmapInvalidTypecode = errors.New("unrecognized typecode") - FrozenBitmapBufferTooSmall = errors.New("buffer too small") + // ErrFrozenBitmapInvalidCookie is returned when the header does not contain the frozenCookie. + ErrFrozenBitmapInvalidCookie = errors.New("header does not contain the frozenCookie") + // ErrFrozenBitmapBigEndian is returned when the header is big endian. + ErrFrozenBitmapBigEndian = errors.New("loading big endian frozen bitmaps is not supported") + // ErrFrozenBitmapIncomplete is returned when the buffer is too small to contain a frozen bitmap. + ErrFrozenBitmapIncomplete = errors.New("input buffer too small to contain a frozen bitmap") + // ErrFrozenBitmapOverpopulated is returned when the number of containers is too large. + ErrFrozenBitmapOverpopulated = errors.New("too many containers") + // ErrFrozenBitmapUnexpectedData is returned when the buffer contains unexpected data. + ErrFrozenBitmapUnexpectedData = errors.New("spurious data in input") + // ErrFrozenBitmapInvalidTypecode is returned when the typecode is invalid. + ErrFrozenBitmapInvalidTypecode = errors.New("unrecognized typecode") + // ErrFrozenBitmapBufferTooSmall is returned when the buffer is too small. + ErrFrozenBitmapBufferTooSmall = errors.New("buffer too small") ) func (ra *roaringArray) frozenView(buf []byte) error { if len(buf) < 4 { - return FrozenBitmapIncomplete + return ErrFrozenBitmapIncomplete } headerBE := binary.BigEndian.Uint32(buf[len(buf)-4:]) - if headerBE&0x7fff == FROZEN_COOKIE { - return FrozenBitmapBigEndian + if headerBE&0x7fff == frozenCookie { + return ErrFrozenBitmapBigEndian } header := binary.LittleEndian.Uint32(buf[len(buf)-4:]) buf = buf[:len(buf)-4] - if header&0x7fff != FROZEN_COOKIE { - return FrozenBitmapInvalidCookie + if header&0x7fff != frozenCookie { + return ErrFrozenBitmapInvalidCookie } nCont := int(header >> 15) if nCont > (1 << 16) { - return FrozenBitmapOverpopulated + return ErrFrozenBitmapOverpopulated } // 1 byte per type, 2 bytes per key, 2 bytes per count. if len(buf) < 5*nCont { - return FrozenBitmapIncomplete + return ErrFrozenBitmapIncomplete } types := buf[len(buf)-nCont:] @@ -390,12 +396,12 @@ func (ra *roaringArray) frozenView(buf []byte) error { nRun++ nRunEl += int(counts[i]) default: - return FrozenBitmapInvalidTypecode + return ErrFrozenBitmapInvalidTypecode } } if len(buf) < (1<<13)*nBitmap+4*nRunEl+2*nArrayEl { - return FrozenBitmapIncomplete + return ErrFrozenBitmapIncomplete } bitsetsArena := byteSliceAsUint64Slice(buf[:(1<<13)*nBitmap]) @@ -408,15 +414,15 @@ func (ra *roaringArray) frozenView(buf []byte) error { buf = buf[2*nArrayEl:] if len(buf) != 0 { - return FrozenBitmapUnexpectedData + return ErrFrozenBitmapUnexpectedData } var c container - containersSz := int(unsafe.Sizeof(c))*nCont - bitsetsSz := int(unsafe.Sizeof(bitmapContainer{}))*nBitmap - arraysSz := int(unsafe.Sizeof(arrayContainer{}))*nArray - runsSz := int(unsafe.Sizeof(runContainer16{}))*nRun - needCOWSz := int(unsafe.Sizeof(true))*nCont + containersSz := int(unsafe.Sizeof(c)) * nCont + bitsetsSz := int(unsafe.Sizeof(bitmapContainer{})) * nBitmap + arraysSz := int(unsafe.Sizeof(arrayContainer{})) * nArray + runsSz := int(unsafe.Sizeof(runContainer16{})) * nRun + needCOWSz := int(unsafe.Sizeof(true)) * nCont bitmapArenaSz := containersSz + bitsetsSz + arraysSz + runsSz + needCOWSz bitmapArena := make([]byte, bitmapArenaSz) @@ -475,9 +481,10 @@ func (ra *roaringArray) frozenView(buf []byte) error { return nil } -func (bm *Bitmap) GetFrozenSizeInBytes() uint64 { +// GetFrozenSizeInBytes returns the size in bytes of the frozen bitmap. +func (rb *Bitmap) GetFrozenSizeInBytes() uint64 { nBits, nArrayEl, nRunEl := uint64(0), uint64(0), uint64(0) - for _, c := range bm.highlowcontainer.containers { + for _, c := range rb.highlowcontainer.containers { switch v := c.(type) { case *bitmapContainer: nBits++ @@ -487,19 +494,21 @@ func (bm *Bitmap) GetFrozenSizeInBytes() uint64 { nRunEl += uint64(len(v.iv)) } } - return 4 + 5*uint64(len(bm.highlowcontainer.containers)) + + return 4 + 5*uint64(len(rb.highlowcontainer.containers)) + (nBits << 13) + 2*nArrayEl + 4*nRunEl } -func (bm *Bitmap) Freeze() ([]byte, error) { - sz := bm.GetFrozenSizeInBytes() +// Freeze serializes the bitmap in the CRoaring's frozen format. +func (rb *Bitmap) Freeze() ([]byte, error) { + sz := rb.GetFrozenSizeInBytes() buf := make([]byte, sz) - _, err := bm.FreezeTo(buf) + _, err := rb.FreezeTo(buf) return buf, err } -func (bm *Bitmap) FreezeTo(buf []byte) (int, error) { - containers := bm.highlowcontainer.containers +// FreezeTo serializes the bitmap in the CRoaring's frozen format. +func (rb *Bitmap) FreezeTo(buf []byte) (int, error) { + containers := rb.highlowcontainer.containers nCont := len(containers) nBits, nArrayEl, nRunEl := 0, 0, 0 @@ -516,7 +525,7 @@ func (bm *Bitmap) FreezeTo(buf []byte) (int, error) { serialSize := 4 + 5*nCont + (1<<13)*nBits + 4*nRunEl + 2*nArrayEl if len(buf) < serialSize { - return 0, FrozenBitmapBufferTooSmall + return 0, ErrFrozenBitmapBufferTooSmall } bitsArena := byteSliceAsUint64Slice(buf[:(1<<13)*nBits]) @@ -537,10 +546,10 @@ func (bm *Bitmap) FreezeTo(buf []byte) (int, error) { types := buf[:nCont] buf = buf[nCont:] - header := uint32(FROZEN_COOKIE | (nCont << 15)) + header := uint32(frozenCookie | (nCont << 15)) binary.LittleEndian.PutUint32(buf[:4], header) - copy(keys, bm.highlowcontainer.keys[:]) + copy(keys, rb.highlowcontainer.keys[:]) for i, c := range containers { switch v := c.(type) { @@ -567,11 +576,12 @@ func (bm *Bitmap) FreezeTo(buf []byte) (int, error) { return serialSize, nil } -func (bm *Bitmap) WriteFrozenTo(wr io.Writer) (int, error) { +// WriteFrozenTo serializes the bitmap in the CRoaring's frozen format. +func (rb *Bitmap) WriteFrozenTo(wr io.Writer) (int, error) { // FIXME: this is a naive version that iterates 4 times through the // containers and allocates 3*len(containers) bytes; it's quite likely // it can be done more efficiently. - containers := bm.highlowcontainer.containers + containers := rb.highlowcontainer.containers written := 0 for _, c := range containers { @@ -610,7 +620,7 @@ func (bm *Bitmap) WriteFrozenTo(wr io.Writer) (int, error) { } } - n, err := wr.Write(uint16SliceAsByteSlice(bm.highlowcontainer.keys)) + n, err := wr.Write(uint16SliceAsByteSlice(rb.highlowcontainer.keys)) written += n if err != nil { return written, err @@ -642,7 +652,7 @@ func (bm *Bitmap) WriteFrozenTo(wr io.Writer) (int, error) { return written, err } - header := uint32(FROZEN_COOKIE | (len(containers) << 15)) + header := uint32(frozenCookie | (len(containers) << 15)) if err := binary.Write(wr, binary.LittleEndian, header); err != nil { return written, err } diff --git a/vendor/github.com/bits-and-blooms/bitset/README.md b/vendor/github.com/bits-and-blooms/bitset/README.md index 97e8307..599982f 100644 --- a/vendor/github.com/bits-and-blooms/bitset/README.md +++ b/vendor/github.com/bits-and-blooms/bitset/README.md @@ -7,6 +7,15 @@ [![PkgGoDev](https://pkg.go.dev/badge/github.com/bits-and-blooms/bitset?tab=doc)](https://pkg.go.dev/github.com/bits-and-blooms/bitset?tab=doc) +This library is part of the [awesome go collection](https://github.com/avelino/awesome-go). It is used in production by several important systems: + +* [beego](https://github.com/beego/beego) +* [CubeFS](https://github.com/cubefs/cubefs) +* [Amazon EKS Distro](https://github.com/aws/eks-distro) +* [sourcegraph](https://github.com/sourcegraph/sourcegraph-public-snapshot) +* [torrent](https://github.com/anacrolix/torrent) + + ## Description Package bitset implements bitsets, a mapping between non-negative integers and boolean values. @@ -16,7 +25,7 @@ It provides methods for setting, clearing, flipping, and testing individual inte But it also provides set intersection, union, difference, complement, and symmetric operations, as well as tests to check whether any, all, or no bits are set, and querying a bitset's current length and number of positive bits. -BitSets are expanded to the size of the largest set bit; the memory allocation is approximately Max bits, where Max is the largest set bit. BitSets are never shrunk. On creation, a hint can be given for the number of bits that will be used. +BitSets are expanded to the size of the largest set bit; the memory allocation is approximately Max bits, where Max is the largest set bit. BitSets are never shrunk automatically, but `Shrink` and `Compact` methods are available. On creation, a hint can be given for the number of bits that will be used. Many of the methods, including Set, Clear, and Flip, return a BitSet pointer, which allows for chaining. @@ -60,19 +69,83 @@ func main() { } ``` -As an alternative to BitSets, one should check out the 'big' package, which provides a (less set-theoretical) view of bitsets. +If you have Go 1.23 or better, you can iterate over the set bits like so: + +```go +for i := range b.EachSet() {} +``` + + Package documentation is at: https://pkg.go.dev/github.com/bits-and-blooms/bitset?tab=doc +## Serialization + + +You may serialize a bitset safely and portably to a stream +of bytes as follows: +```Go + const length = 9585 + const oneEvery = 97 + bs := bitset.New(length) + // Add some bits + for i := uint(0); i < length; i += oneEvery { + bs = bs.Set(i) + } + + var buf bytes.Buffer + n, err := bs.WriteTo(&buf) + if err != nil { + // failure + } + // Here n == buf.Len() +``` +You can later deserialize the result as follows: + +```Go + // Read back from buf + bs = bitset.New() + n, err = bs.ReadFrom(&buf) + if err != nil { + // error + } + // n is the number of bytes read +``` + +The `ReadFrom` function attempts to read the data into the existing +BitSet instance, to minimize memory allocations. + + +*Performance tip*: +When reading and writing to a file or a network connection, you may get better performance by +wrapping your streams with `bufio` instances. + +E.g., +```Go + f, err := os.Create("myfile") + w := bufio.NewWriter(f) +``` +```Go + f, err := os.Open("myfile") + r := bufio.NewReader(f) +``` + ## Memory Usage -The memory usage of a bitset using N bits is at least N/8 bytes. The number of bits in a bitset is at least as large as one plus the greatest bit index you have accessed. Thus it is possible to run out of memory while using a bitset. If you have lots of bits, you might prefer compressed bitsets, like the [Roaring bitmaps](http://roaringbitmap.org) and its [Go implementation](https://github.com/RoaringBitmap/roaring). +The memory usage of a bitset using `N` bits is at least `N/8` bytes. The number of bits in a bitset is at least as large as one plus the greatest bit index you have accessed. Thus it is possible to run out of memory while using a bitset. If you have lots of bits, you might prefer compressed bitsets, like the [Roaring bitmaps](https://roaringbitmap.org) and its [Go implementation](https://github.com/RoaringBitmap/roaring). -## Implementation Note +The `roaring` library allows you to go back and forth between compressed Roaring bitmaps and the conventional bitset instances: +```Go + mybitset := roaringbitmap.ToBitSet() + newroaringbitmap := roaring.FromBitSet(mybitset) +``` -Go 1.9 introduced a native `math/bits` library. We provide backward compatibility to Go 1.7, which might be removed. -It is possible that a later version will match the `math/bits` return signature for counts (which is `int`, rather than our library's `unit64`). If so, the version will be bumped. +### Goroutine safety + +In general, it's not safe to access the same BitSet using different goroutines--they are unsynchronized for performance. + +Should you want to access a BitSet from more than one goroutine, you should provide synchronization. Typically this is done by using channels to pass the *BitSet around (in Go style; so there is only ever one owner), or by using `sync.Mutex` to serialize operations on BitSets. ## Installation @@ -91,3 +164,13 @@ Before committing the code, please check if it passes tests, has adequate covera go test go test -cover ``` + +## Stars + + +[![Star History Chart](https://api.star-history.com/svg?repos=bits-and-blooms/bitset&type=Date)](https://www.star-history.com/#bits-and-blooms/bitset&Date) + +## Further reading + +

Mastering Programming: From Testing to Performance in Go

+
diff --git a/vendor/github.com/bits-and-blooms/bitset/SECURITY.md b/vendor/github.com/bits-and-blooms/bitset/SECURITY.md new file mode 100644 index 0000000..f888420 --- /dev/null +++ b/vendor/github.com/bits-and-blooms/bitset/SECURITY.md @@ -0,0 +1,5 @@ +# Security Policy + +## Reporting a Vulnerability + +You can report privately a vulnerability by email at daniel@lemire.me (current maintainer). diff --git a/vendor/github.com/bits-and-blooms/bitset/bitset.go b/vendor/github.com/bits-and-blooms/bitset/bitset.go index 5751b68..c8d8ddb 100644 --- a/vendor/github.com/bits-and-blooms/bitset/bitset.go +++ b/vendor/github.com/bits-and-blooms/bitset/bitset.go @@ -37,7 +37,6 @@ which provides a (less set-theoretical) view of bitsets. package bitset import ( - "bufio" "bytes" "encoding/base64" "encoding/binary" @@ -45,14 +44,21 @@ import ( "errors" "fmt" "io" + "math/bits" "strconv" ) // the wordSize of a bit set -const wordSize = uint(64) +const wordSize = 64 + +// the wordSize of a bit set in bytes +const wordBytes = wordSize / 8 + +// wordMask is wordSize-1, used for bit indexing in a word +const wordMask = wordSize - 1 // log2WordSize is lg(wordSize) -const log2WordSize = uint(6) +const log2WordSize = 6 // allBits has every bit set const allBits uint64 = 0xffffffffffffffff @@ -66,9 +72,16 @@ var base64Encoding = base64.URLEncoding // Base64StdEncoding Marshal/Unmarshal BitSet with base64.StdEncoding(Default: base64.URLEncoding) func Base64StdEncoding() { base64Encoding = base64.StdEncoding } -// LittleEndian Marshal/Unmarshal Binary as Little Endian(Default: binary.BigEndian) +// LittleEndian sets Marshal/Unmarshal Binary as Little Endian (Default: binary.BigEndian) func LittleEndian() { binaryOrder = binary.LittleEndian } +// BigEndian sets Marshal/Unmarshal Binary as Big Endian (Default: binary.BigEndian) +func BigEndian() { binaryOrder = binary.BigEndian } + +// BinaryOrder returns the current binary order, see also LittleEndian() +// and BigEndian() to change the order. +func BinaryOrder() binary.ByteOrder { return binaryOrder } + // A BitSet is a set of bits. The zero value of a BitSet is an empty set of length 0. type BitSet struct { length uint @@ -92,41 +105,63 @@ func (b *BitSet) SetBitsetFrom(buf []uint64) { b.set = buf } -// From is a constructor used to create a BitSet from an array of integers +// From is a constructor used to create a BitSet from an array of words func From(buf []uint64) *BitSet { return FromWithLength(uint(len(buf))*64, buf) } -// FromWithLength constructs from an array of integers and length. -func FromWithLength(len uint, set []uint64) *BitSet { - return &BitSet{len, set} +// FromWithLength constructs from an array of words and length in bits. +// This function is for advanced users, most users should prefer +// the From function. +// As a user of FromWithLength, you are responsible for ensuring +// that the length is correct: your slice should have length at +// least (length+63)/64 in 64-bit words. +func FromWithLength(length uint, set []uint64) *BitSet { + if len(set) < wordsNeeded(length) { + panic("BitSet.FromWithLength: slice is too short") + } + return &BitSet{length, set} } -// Bytes returns the bitset as array of integers +// Bytes returns the bitset as array of 64-bit words, giving direct access to the internal representation. +// It is not a copy, so changes to the returned slice will affect the bitset. +// It is meant for advanced users. +// +// Deprecated: Bytes is deprecated. Use [BitSet.Words] instead. func (b *BitSet) Bytes() []uint64 { return b.set } +// Words returns the bitset as array of 64-bit words, giving direct access to the internal representation. +// It is not a copy, so changes to the returned slice will affect the bitset. +// It is meant for advanced users. +func (b *BitSet) Words() []uint64 { + return b.set +} + // wordsNeeded calculates the number of words needed for i bits func wordsNeeded(i uint) int { - if i > (Cap() - wordSize + 1) { + if i > (Cap() - wordMask) { return int(Cap() >> log2WordSize) } - return int((i + (wordSize - 1)) >> log2WordSize) + return int((i + wordMask) >> log2WordSize) } // wordsNeededUnbound calculates the number of words needed for i bits, possibly exceeding the capacity. -// This function is useful if you know that the capacity cannot be exceeded (e.g., you have an existing bitmap). +// This function is useful if you know that the capacity cannot be exceeded (e.g., you have an existing BitSet). func wordsNeededUnbound(i uint) int { - return int((i + (wordSize - 1)) >> log2WordSize) + return (int(i) + wordMask) >> log2WordSize } // wordsIndex calculates the index of words in a `uint64` func wordsIndex(i uint) uint { - return i & (wordSize - 1) + return i & wordMask } -// New creates a new BitSet with a hint that length bits will be required +// New creates a new BitSet with a hint that length bits will be required. +// The memory usage is at least length/8 bytes. +// In case of allocation failure, the function will return a BitSet with zero +// capacity. func New(length uint) (bset *BitSet) { defer func() { if r := recover(); r != nil { @@ -145,13 +180,30 @@ func New(length uint) (bset *BitSet) { return bset } +// MustNew creates a new BitSet with the given length bits. +// It panics if length exceeds the possible capacity or by a lack of memory. +func MustNew(length uint) (bset *BitSet) { + if length >= Cap() { + panic("You are exceeding the capacity") + } + + return &BitSet{ + length, + make([]uint64, wordsNeeded(length)), // may panic on lack of memory + } +} + // Cap returns the total possible capacity, or number of bits +// that can be stored in the BitSet theoretically. Under 32-bit system, +// it is 4294967295 and under 64-bit system, it is 18446744073709551615. +// Note that this is further limited by the maximum allocation size in Go, +// and your available memory, as any Go data structure. func Cap() uint { return ^uint(0) } // Len returns the number of bits in the BitSet. -// Note the difference to method Count, see example. +// Note that it differ from Count function. func (b *BitSet) Len() uint { return b.length } @@ -182,12 +234,32 @@ func (b *BitSet) Test(i uint) bool { return b.set[i>>log2WordSize]&(1<> log2WordSize) + subWordIndex := wordsIndex(i) + + // The word that the index falls within, shifted so the index is at bit 0 + var firstWord, secondWord uint64 + if firstWordIndex < len(b.set) { + firstWord = b.set[firstWordIndex] >> subWordIndex + } + + // The next word, masked to only include the necessary bits and shifted to cover the + // top of the word + if (firstWordIndex + 1) < len(b.set) { + secondWord = b.set[firstWordIndex+1] << uint64(wordSize-subWordIndex) + } + + return firstWord | secondWord +} + // Set bit i to 1, the capacity of the bitset is automatically // increased accordingly. -// If i>= Cap(), this function will panic. // Warning: using a very large value for 'i' // may lead to a memory shortage and a panic: the caller is responsible // for providing sensible parameters in line with their memory capacity. +// The memory usage is at least slightly over i/8 bytes. func (b *BitSet) Set(i uint) *BitSet { if i >= b.length { // if we need more bits, make 'em b.extendSet(i) @@ -196,7 +268,7 @@ func (b *BitSet) Set(i uint) *BitSet { return b } -// Clear bit i to 0 +// Clear bit i to 0. This never cause a memory allocation. It is always safe. func (b *BitSet) Clear(i uint) *BitSet { if i >= b.length { return b @@ -206,7 +278,6 @@ func (b *BitSet) Clear(i uint) *BitSet { } // SetTo sets bit i to value. -// If i>= Cap(), this function will panic. // Warning: using a very large value for 'i' // may lead to a memory shortage and a panic: the caller is responsible // for providing sensible parameters in line with their memory capacity. @@ -218,7 +289,6 @@ func (b *BitSet) SetTo(i uint, value bool) *BitSet { } // Flip bit at i. -// If i>= Cap(), this function will panic. // Warning: using a very large value for 'i' // may lead to a memory shortage and a panic: the caller is responsible // for providing sensible parameters in line with their memory capacity. @@ -231,7 +301,6 @@ func (b *BitSet) Flip(i uint) *BitSet { } // FlipRange bit in [start, end). -// If end>= Cap(), this function will panic. // Warning: using a very large value for 'end' // may lead to a memory shortage and a panic: the caller is responsible // for providing sensible parameters in line with their memory capacity. @@ -239,18 +308,54 @@ func (b *BitSet) FlipRange(start, end uint) *BitSet { if start >= end { return b } + if end-1 >= b.length { // if we need more bits, make 'em b.extendSet(end - 1) } - var startWord uint = start >> log2WordSize - var endWord uint = end >> log2WordSize + + startWord := int(start >> log2WordSize) + endWord := int(end >> log2WordSize) + + // b.set[startWord] ^= ^(^uint64(0) << wordsIndex(start)) + // e.g: + // start = 71, + // startWord = 1 + // wordsIndex(start) = 71 % 64 = 7 + // (^uint64(0) << 7) = 0b111111....11110000000 + // + // mask = ^(^uint64(0) << 7) = 0b000000....00001111111 + // + // flips the first 7 bits in b.set[1] and + // in the range loop, the b.set[1] gets again flipped + // so the two expressions flip results in a flip + // in b.set[1] from [7,63] + // + // handle startWord special, get's reflipped in range loop b.set[startWord] ^= ^(^uint64(0) << wordsIndex(start)) - for i := startWord; i < endWord; i++ { - b.set[i] = ^b.set[i] + + for idx := range b.set[startWord:endWord] { + b.set[startWord+idx] = ^b.set[startWord+idx] } - if end&(wordSize-1) != 0 { - b.set[endWord] ^= ^uint64(0) >> wordsIndex(-end) + + // handle endWord special + // e.g. + // end = 135 + // endWord = 2 + // + // wordsIndex(-7) = 57 + // see the golang spec: + // "For unsigned integer values, the operations +, -, *, and << are computed + // modulo 2n, where n is the bit width of the unsigned integer's type." + // + // mask = ^uint64(0) >> 57 = 0b00000....0001111111 + // + // flips in b.set[2] from [0,7] + // + // is end at word boundary? + if idx := wordsIndex(-end); idx != 0 { + b.set[endWord] ^= ^uint64(0) >> wordsIndex(idx) } + return b } @@ -268,6 +373,7 @@ func (b *BitSet) FlipRange(start, end uint) *BitSet { // memory usage until the GC runs. Normally this should not be a problem, but if you // have an extremely large BitSet its important to understand that the old BitSet will // remain in memory until the GC frees it. +// If you are memory constrained, this function may cause a panic. func (b *BitSet) Shrink(lastbitindex uint) *BitSet { length := lastbitindex + 1 idx := wordsNeeded(length) @@ -287,6 +393,11 @@ func (b *BitSet) Shrink(lastbitindex uint) *BitSet { // Compact shrinks BitSet to so that we preserve all set bits, while minimizing // memory usage. Compact calls Shrink. +// A new slice is allocated to store the new bits, so you may see an increase in +// memory usage until the GC runs. Normally this should not be a problem, but if you +// have an extremely large BitSet its important to understand that the old BitSet will +// remain in memory until the GC frees it. +// If you are memory constrained, this function may cause a panic. func (b *BitSet) Compact() *BitSet { idx := len(b.set) - 1 for ; idx >= 0 && b.set[idx] == 0; idx-- { @@ -346,7 +457,8 @@ func (b *BitSet) InsertAt(idx uint) *BitSet { return b } -// String creates a string representation of the Bitmap +// String creates a string representation of the BitSet. It is only intended for +// human-readable output and not for serialization. func (b *BitSet) String() string { // follows code from https://github.com/RoaringBitmap/roaring var buffer bytes.Buffer @@ -408,6 +520,50 @@ func (b *BitSet) DeleteAt(i uint) *BitSet { return b } +// AppendTo appends all set bits to buf and returns the (maybe extended) buf. +// In case of allocation failure, the function will panic. +// +// See also [BitSet.AsSlice] and [BitSet.NextSetMany]. +func (b *BitSet) AppendTo(buf []uint) []uint { + // In theory, we could overflow uint, but in practice, we will not. + for idx, word := range b.set { + for word != 0 { + // In theory idx<= len(b.set) { return 0, false } - w := b.set[x] - w = w >> wordsIndex(i) - if w != 0 { - return i + trailingZeroes64(w), true - } - x = x + 1 - for x < len(b.set) { - if b.set[x] != 0 { - return uint(x)*wordSize + trailingZeroes64(b.set[x]), true - } - x = x + 1 + // process first (partial) word + word := b.set[x] >> wordsIndex(i) + if word != 0 { + return i + uint(bits.TrailingZeros64(word)), true } + + // process the following full words until next bit is set + // x < len(b.set), no out-of-bounds panic in following slice expression + x++ + for idx, word := range b.set[x:] { + if word != 0 { + return uint((x+idx)<> log2WordSize) if x >= len(b.set) || capacity == 0 { - return 0, myanswer[:0] + return 0, result[:0] } - skip := wordsIndex(i) - word := b.set[x] >> skip - myanswer = myanswer[:capacity] - size := int(0) + + // process first (partial) word + word := b.set[x] >> wordsIndex(i) + + size := 0 for word != 0 { - r := trailingZeroes64(word) - t := word & ((^word) + 1) - myanswer[size] = r + i + result[size] = i + uint(bits.TrailingZeros64(word)) + size++ if size == capacity { - goto End + return result[size-1], result[:size] } - word = word ^ t + + // clear the rightmost set bit + word &= word - 1 } + + // process the following full words + // x < len(b.set), no out-of-bounds panic in following slice expression x++ for idx, word := range b.set[x:] { for word != 0 { - r := trailingZeroes64(word) - t := word & ((^word) + 1) - myanswer[size] = r + (uint(x+idx) << 6) + result[size] = uint((x+idx)< 0 { - return myanswer[size-1], myanswer[:size] + return result[size-1], result[:size] } - return 0, myanswer[:0] + return 0, result[:0] } // NextClear returns the next clear bit from the specified index, @@ -506,25 +676,89 @@ func (b *BitSet) NextClear(i uint) (uint, bool) { if x >= len(b.set) { return 0, false } - w := b.set[x] - w = w >> wordsIndex(i) - wA := allBits >> wordsIndex(i) - index := i + trailingZeroes64(^w) - if w != wA && index < b.length { + + // process first (maybe partial) word + word := b.set[x] + word = word >> wordsIndex(i) + wordAll := allBits >> wordsIndex(i) + + index := i + uint(bits.TrailingZeros64(^word)) + if word != wordAll && index < b.length { return index, true } + + // process the following full words until next bit is cleared + // x < len(b.set), no out-of-bounds panic in following slice expression x++ - for x < len(b.set) { - index = uint(x)*wordSize + trailingZeroes64(^b.set[x]) - if b.set[x] != allBits && index < b.length { - return index, true + for idx, word := range b.set[x:] { + if word != allBits { + index = uint((x+idx)*wordSize + bits.TrailingZeros64(^word)) + if index < b.length { + return index, true + } + } + } + + return 0, false +} + +// PreviousSet returns the previous set bit from the specified index, +// including possibly the current index +// along with an error code (true = valid, false = no bit found i.e. all bits are clear) +func (b *BitSet) PreviousSet(i uint) (uint, bool) { + x := int(i >> log2WordSize) + if x >= len(b.set) { + return 0, false + } + word := b.set[x] + + // Clear the bits above the index + word = word & ((1 << (wordsIndex(i) + 1)) - 1) + if word != 0 { + return uint(x<= 0; x-- { + word = b.set[x] + if word != 0 { + return uint(x<> log2WordSize) + if x >= len(b.set) { + return 0, false + } + word := b.set[x] + + // Flip all bits and find the highest one bit + word = ^word + + // Clear the bits above the index + word = word & ((1 << (wordsIndex(i) + 1)) - 1) + + if word != 0 { + return uint(x<= 0; x-- { + word = b.set[x] + word = ^word + if word != 0 { + return uint(x< int(b.wordCount()) { - l = int(b.wordCount()) + l := compare.wordCount() + if l > b.wordCount() { + l = b.wordCount() } for i := 0; i < l; i++ { result.set[i] = b.set[i] &^ compare.set[i] @@ -643,16 +896,18 @@ func (b *BitSet) Difference(compare *BitSet) (result *BitSet) { return } -// DifferenceCardinality computes the cardinality of the differnce +// DifferenceCardinality computes the cardinality of the difference func (b *BitSet) DifferenceCardinality(compare *BitSet) uint { panicIfNull(b) panicIfNull(compare) - l := int(compare.wordCount()) - if l > int(b.wordCount()) { - l = int(b.wordCount()) + l := compare.wordCount() + if l > b.wordCount() { + l = b.wordCount() } cnt := uint64(0) - cnt += popcntMaskSlice(b.set[:l], compare.set[:l]) + if l > 0 { + cnt += popcntMaskSlice(b.set[:l], compare.set[:l]) + } cnt += popcntSlice(b.set[l:]) return uint(cnt) } @@ -662,12 +917,19 @@ func (b *BitSet) DifferenceCardinality(compare *BitSet) uint { func (b *BitSet) InPlaceDifference(compare *BitSet) { panicIfNull(b) panicIfNull(compare) - l := int(compare.wordCount()) - if l > int(b.wordCount()) { - l = int(b.wordCount()) + l := compare.wordCount() + if l > b.wordCount() { + l = b.wordCount() } + if l <= 0 { + return + } + // bounds check elimination + data, cmpData := b.set, compare.set + _ = data[l-1] + _ = cmpData[l-1] for i := 0; i < l; i++ { - b.set[i] &^= compare.set[i] + data[i] &^= cmpData[i] } } @@ -684,6 +946,7 @@ func sortByLength(a *BitSet, b *BitSet) (ap *BitSet, bp *BitSet) { // Intersection of base set and other set // This is the BitSet equivalent of & (and) +// In case of allocation failure, the function will return an empty BitSet. func (b *BitSet) Intersection(compare *BitSet) (result *BitSet) { panicIfNull(b) panicIfNull(compare) @@ -695,10 +958,13 @@ func (b *BitSet) Intersection(compare *BitSet) (result *BitSet) { return } -// IntersectionCardinality computes the cardinality of the union +// IntersectionCardinality computes the cardinality of the intersection func (b *BitSet) IntersectionCardinality(compare *BitSet) uint { panicIfNull(b) panicIfNull(compare) + if b.length == 0 || compare.length == 0 { + return 0 + } b, compare = sortByLength(b, compare) cnt := popcntAndSlice(b.set, compare.set) return uint(cnt) @@ -710,15 +976,24 @@ func (b *BitSet) IntersectionCardinality(compare *BitSet) uint { func (b *BitSet) InPlaceIntersection(compare *BitSet) { panicIfNull(b) panicIfNull(compare) - l := int(compare.wordCount()) - if l > int(b.wordCount()) { - l = int(b.wordCount()) + l := compare.wordCount() + if l > b.wordCount() { + l = b.wordCount() } - for i := 0; i < l; i++ { - b.set[i] &= compare.set[i] + if l > 0 { + // bounds check elimination + data, cmpData := b.set, compare.set + _ = data[l-1] + _ = cmpData[l-1] + + for i := 0; i < l; i++ { + data[i] &= cmpData[i] + } } - for i := l; i < len(b.set); i++ { - b.set[i] = 0 + if l >= 0 { + for i := l; i < len(b.set); i++ { + b.set[i] = 0 + } } if compare.length > 0 { if compare.length-1 >= b.length { @@ -746,7 +1021,10 @@ func (b *BitSet) UnionCardinality(compare *BitSet) uint { panicIfNull(b) panicIfNull(compare) b, compare = sortByLength(b, compare) - cnt := popcntOrSlice(b.set, compare.set) + cnt := uint64(0) + if len(b.set) > 0 { + cnt += popcntOrSlice(b.set, compare.set) + } if len(compare.set) > len(b.set) { cnt += popcntSlice(compare.set[len(b.set):]) } @@ -758,15 +1036,22 @@ func (b *BitSet) UnionCardinality(compare *BitSet) uint { func (b *BitSet) InPlaceUnion(compare *BitSet) { panicIfNull(b) panicIfNull(compare) - l := int(compare.wordCount()) - if l > int(b.wordCount()) { - l = int(b.wordCount()) + l := compare.wordCount() + if l > b.wordCount() { + l = b.wordCount() } if compare.length > 0 && compare.length-1 >= b.length { b.extendSet(compare.length - 1) } - for i := 0; i < l; i++ { - b.set[i] |= compare.set[i] + if l > 0 { + // bounds check elimination + data, cmpData := b.set, compare.set + _ = data[l-1] + _ = cmpData[l-1] + + for i := 0; i < l; i++ { + data[i] |= cmpData[i] + } } if len(compare.set) > l { for i := l; i < len(compare.set); i++ { @@ -794,7 +1079,10 @@ func (b *BitSet) SymmetricDifferenceCardinality(compare *BitSet) uint { panicIfNull(b) panicIfNull(compare) b, compare = sortByLength(b, compare) - cnt := popcntXorSlice(b.set, compare.set) + cnt := uint64(0) + if len(b.set) > 0 { + cnt += popcntXorSlice(b.set, compare.set) + } if len(compare.set) > len(b.set) { cnt += popcntSlice(compare.set[len(b.set):]) } @@ -806,15 +1094,21 @@ func (b *BitSet) SymmetricDifferenceCardinality(compare *BitSet) uint { func (b *BitSet) InPlaceSymmetricDifference(compare *BitSet) { panicIfNull(b) panicIfNull(compare) - l := int(compare.wordCount()) - if l > int(b.wordCount()) { - l = int(b.wordCount()) + l := compare.wordCount() + if l > b.wordCount() { + l = b.wordCount() } if compare.length > 0 && compare.length-1 >= b.length { b.extendSet(compare.length - 1) } - for i := 0; i < l; i++ { - b.set[i] ^= compare.set[i] + if l > 0 { + // bounds check elimination + data, cmpData := b.set, compare.set + _ = data[l-1] + _ = cmpData[l-1] + for i := 0; i < l; i++ { + data[i] ^= cmpData[i] + } } if len(compare.set) > l { for i := l; i < len(compare.set); i++ { @@ -836,6 +1130,7 @@ func (b *BitSet) cleanLastWord() { } // Complement computes the (local) complement of a bitset (up to length bits) +// In case of allocation failure, the function will return an empty BitSet. func (b *BitSet) Complement() (result *BitSet) { panicIfNull(b) result = New(b.length) @@ -875,12 +1170,16 @@ func (b *BitSet) Any() bool { // IsSuperSet returns true if this is a superset of the other set func (b *BitSet) IsSuperSet(other *BitSet) bool { - for i, e := other.NextSet(0); e; i, e = other.NextSet(i + 1) { - if !b.Test(i) { + l := other.wordCount() + if b.wordCount() < l { + l = b.wordCount() + } + for i, word := range other.set[:l] { + if b.set[i]&word != word { return false } } - return true + return popcntSlice(other.set[l:]) == 0 } // IsStrictSuperSet returns true if this is a strict superset of the other set @@ -888,7 +1187,9 @@ func (b *BitSet) IsStrictSuperSet(other *BitSet) bool { return b.Count() > other.Count() && b.IsSuperSet(other) } -// DumpAsBits dumps a bit set as a string of bits +// DumpAsBits dumps a bit set as a string of bits. Following the usual convention in Go, +// the least significant bits are printed last (index 0 is at the end of the string). +// This is useful for debugging and testing. It is not suitable for serialization. func (b *BitSet) DumpAsBits() string { if b.set == nil { return "." @@ -901,44 +1202,110 @@ func (b *BitSet) DumpAsBits() string { return buffer.String() } -// BinaryStorageSize returns the binary storage requirements +// BinaryStorageSize returns the binary storage requirements (see WriteTo) in bytes. func (b *BitSet) BinaryStorageSize() int { - nWords := b.wordCount() - return binary.Size(uint64(0)) + binary.Size(b.set[:nWords]) + return wordBytes + wordBytes*b.wordCount() } -// WriteTo writes a BitSet to a stream -func (b *BitSet) WriteTo(stream io.Writer) (int64, error) { - length := uint64(b.length) - - // Write length - err := binary.Write(stream, binaryOrder, length) - if err != nil { - return 0, err - } - - // Write set - // current implementation of bufio.Writer is more memory efficient than - // binary.Write for large set - writer := bufio.NewWriter(stream) - var item = make([]byte, binary.Size(uint64(0))) // for serializing one uint64 - nWords := b.wordCount() - for i := range b.set[:nWords] { - binaryOrder.PutUint64(item, b.set[i]) - if nn, err := writer.Write(item); err != nil { - return int64(i*binary.Size(uint64(0)) + nn), err +func readUint64Array(reader io.Reader, data []uint64) error { + length := len(data) + bufferSize := 128 + buffer := make([]byte, bufferSize*wordBytes) + for i := 0; i < length; i += bufferSize { + end := i + bufferSize + if end > length { + end = length + buffer = buffer[:wordBytes*(end-i)] + } + chunk := data[i:end] + if _, err := io.ReadFull(reader, buffer); err != nil { + return err + } + for i := range chunk { + chunk[i] = uint64(binaryOrder.Uint64(buffer[8*i:])) } } + return nil +} - err = writer.Flush() - return int64(b.BinaryStorageSize()), err +func writeUint64Array(writer io.Writer, data []uint64) error { + bufferSize := 128 + buffer := make([]byte, bufferSize*wordBytes) + for i := 0; i < len(data); i += bufferSize { + end := i + bufferSize + if end > len(data) { + end = len(data) + buffer = buffer[:wordBytes*(end-i)] + } + chunk := data[i:end] + for i, x := range chunk { + binaryOrder.PutUint64(buffer[8*i:], x) + } + _, err := writer.Write(buffer) + if err != nil { + return err + } + } + return nil +} + +// WriteTo writes a BitSet to a stream. The format is: +// 1. uint64 length +// 2. []uint64 set +// The length is the number of bits in the BitSet. +// +// The set is a slice of uint64s containing between length and length + 63 bits. +// It is interpreted as a big-endian array of uint64s by default (see BinaryOrder()) +// meaning that the first 8 bits are stored at byte index 7, the next 8 bits are stored +// at byte index 6... the bits 64 to 71 are stored at byte index 8, etc. +// If you change the binary order, you need to do so for both reading and writing. +// We recommend using the default binary order. +// +// Upon success, the number of bytes written is returned. +// +// Performance: if this function is used to write to a disk or network +// connection, it might be beneficial to wrap the stream in a bufio.Writer. +// E.g., +// +// f, err := os.Create("myfile") +// w := bufio.NewWriter(f) +func (b *BitSet) WriteTo(stream io.Writer) (int64, error) { + length := uint64(b.length) + // Write length + err := binary.Write(stream, binaryOrder, &length) + if err != nil { + // Upon failure, we do not guarantee that we + // return the number of bytes written. + return int64(0), err + } + err = writeUint64Array(stream, b.set[:b.wordCount()]) + if err != nil { + // Upon failure, we do not guarantee that we + // return the number of bytes written. + return int64(wordBytes), err + } + return int64(b.BinaryStorageSize()), nil } // ReadFrom reads a BitSet from a stream written using WriteTo +// The format is: +// 1. uint64 length +// 2. []uint64 set +// See WriteTo for details. +// Upon success, the number of bytes read is returned. +// If the current BitSet is not large enough to hold the data, +// it is extended. In case of error, the BitSet is either +// left unchanged or made empty if the error occurs too late +// to preserve the content. +// +// Performance: if this function is used to read from a disk or network +// connection, it might be beneficial to wrap the stream in a bufio.Reader. +// E.g., +// +// f, err := os.Open("myfile") +// r := bufio.NewReader(f) func (b *BitSet) ReadFrom(stream io.Reader) (int64, error) { var length uint64 - - // Read length first err := binary.Read(stream, binaryOrder, &length) if err != nil { if err == io.EOF { @@ -946,30 +1313,37 @@ func (b *BitSet) ReadFrom(stream io.Reader) (int64, error) { } return 0, err } - newset := New(uint(length)) + newlength := uint(length) - if uint64(newset.length) != length { + if uint64(newlength) != length { return 0, errors.New("unmarshalling error: type mismatch") } - - var item [8]byte - nWords := wordsNeeded(uint(length)) - reader := bufio.NewReader(io.LimitReader(stream, 8*int64(nWords))) - for i := 0; i < nWords; i++ { - if _, err := io.ReadFull(reader, item[:]); err != nil { - if err == io.EOF { - err = io.ErrUnexpectedEOF - } - return 0, err - } - newset.set[i] = binaryOrder.Uint64(item[:]) + nWords := wordsNeeded(uint(newlength)) + if cap(b.set) >= nWords { + b.set = b.set[:nWords] + } else { + b.set = make([]uint64, nWords) + } + + b.length = newlength + + err = readUint64Array(stream, b.set) + if err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + // We do not want to leave the BitSet partially filled as + // it is error prone. + b.set = b.set[:0] + b.length = 0 + return 0, err } - *b = *newset return int64(b.BinaryStorageSize()), nil } // MarshalBinary encodes a BitSet into a binary form and returns the result. +// Please see WriteTo for details. func (b *BitSet) MarshalBinary() ([]byte, error) { var buf bytes.Buffer _, err := b.WriteTo(&buf) @@ -981,6 +1355,7 @@ func (b *BitSet) MarshalBinary() ([]byte, error) { } // UnmarshalBinary decodes the binary form generated by MarshalBinary. +// Please see WriteTo for details. func (b *BitSet) UnmarshalBinary(data []byte) error { buf := bytes.NewReader(data) _, err := b.ReadFrom(buf) @@ -988,7 +1363,7 @@ func (b *BitSet) UnmarshalBinary(data []byte) error { } // MarshalJSON marshals a BitSet as a JSON structure -func (b *BitSet) MarshalJSON() ([]byte, error) { +func (b BitSet) MarshalJSON() ([]byte, error) { buffer := bytes.NewBuffer(make([]byte, 0, b.BinaryStorageSize())) _, err := b.WriteTo(buffer) if err != nil { @@ -1017,3 +1392,376 @@ func (b *BitSet) UnmarshalJSON(data []byte) error { _, err = b.ReadFrom(bytes.NewReader(buf)) return err } + +// Rank returns the number of set bits up to and including the index +// that are set in the bitset. +// See https://en.wikipedia.org/wiki/Ranking#Ranking_in_statistics +func (b *BitSet) Rank(index uint) (rank uint) { + index++ // Rank is up to and including + + // needed more than once + length := len(b.set) + + // TODO: built-in min requires go1.21 or later + // idx := min(int(index>>6), len(b.set)) + idx := int(index >> 6) + if idx > length { + idx = length + } + + // sum up the popcounts until idx ... + // TODO: cannot range over idx (...): requires go1.22 or later + // for j := range idx { + for j := 0; j < idx; j++ { + if w := b.set[j]; w != 0 { + rank += uint(bits.OnesCount64(w)) + } + } + + // ... plus partial word at idx, + // make Rank inlineable and faster in the end + // don't test index&63 != 0, just add, less branching + if idx < length { + rank += uint(bits.OnesCount64(b.set[idx] << (64 - index&63))) + } + + return +} + +// Select returns the index of the jth set bit, where j is the argument. +// The caller is responsible to ensure that 0 <= j < Count(): when j is +// out of range, the function returns the length of the bitset (b.length). +// +// Note that this function differs in convention from the Rank function which +// returns 1 when ranking the smallest value. We follow the conventional +// textbook definition of Select and Rank. +func (b *BitSet) Select(index uint) uint { + leftover := index + for idx, word := range b.set { + w := uint(bits.OnesCount64(word)) + if w > leftover { + return uint(idx)*64 + select64(word, leftover) + } + leftover -= w + } + return b.length +} + +// top detects the top bit set +func (b *BitSet) top() (uint, bool) { + for idx := len(b.set) - 1; idx >= 0; idx-- { + if word := b.set[idx]; word != 0 { + return uint(idx<= b.length { + b.length = top + bits + 1 + } + + pad, idx := top%wordSize, top>>log2WordSize + shift, pages := bits%wordSize, bits>>log2WordSize + if bits%wordSize == 0 { // happy case: just add pages + copy(dst[pages:nsize], b.set) + } else { + if pad+shift >= wordSize { + dst[idx+pages+1] = b.set[idx] >> (wordSize - shift) + } + + for i := int(idx); i >= 0; i-- { + if i > 0 { + dst[i+int(pages)] = (b.set[i] << shift) | (b.set[i-1] >> (wordSize - shift)) + } else { + dst[i+int(pages)] = b.set[i] << shift + } + } + } + + // zeroing extra pages + for i := 0; i < int(pages); i++ { + dst[i] = 0 + } + + b.set = dst +} + +// ShiftRight shifts the bitset like >> operation would do. +func (b *BitSet) ShiftRight(bits uint) { + panicIfNull(b) + + if bits == 0 { + return + } + + top, ok := b.top() + if !ok { + return + } + + if bits > top { + b.set = make([]uint64, wordsNeeded(b.length)) + return + } + + pad, idx := top%wordSize, top>>log2WordSize + shift, pages := bits%wordSize, bits>>log2WordSize + if bits%wordSize == 0 { // happy case: just clear pages + b.set = b.set[pages:] + b.length -= pages * wordSize + } else { + for i := 0; i <= int(idx-pages); i++ { + if i < int(idx-pages) { + b.set[i] = (b.set[i+int(pages)] >> shift) | (b.set[i+int(pages)+1] << (wordSize - shift)) + } else { + b.set[i] = b.set[i+int(pages)] >> shift + } + } + + if pad < shift { + b.set[int(idx-pages)] = 0 + } + } + + for i := int(idx-pages) + 1; i <= int(idx); i++ { + b.set[i] = 0 + } +} + +// OnesBetween returns the number of set bits in the range [from, to). +// The range is inclusive of 'from' and exclusive of 'to'. +// Returns 0 if from >= to. +func (b *BitSet) OnesBetween(from, to uint) uint { + panicIfNull(b) + + if from >= to { + return 0 + } + + // Calculate indices and masks for the starting and ending words + startWord := from >> log2WordSize // Divide by wordSize + endWord := to >> log2WordSize + startOffset := from & wordMask // Mod wordSize + endOffset := to & wordMask + + // Case 1: Bits lie within a single word + if startWord == endWord { + // Create mask for bits between from and to + mask := uint64((1<= startOffset + count = uint(bits.OnesCount64(b.set[startWord] & startMask)) + + // 2b: Count all bits in complete words between start and end + if endWord > startWord+1 { + count += uint(popcntSlice(b.set[startWord+1 : endWord])) + } + + // 2c: Count bits in last word (from start of word to endOffset) + if endOffset > 0 { + endMask := uint64(1<> log2WordSize + bitOffset := outPos & wordMask + + // Write extracted bits, handling word boundary crossing + dst.set[wordIdx] |= extracted << bitOffset + if bitOffset+bitsExtracted > wordSize { + dst.set[wordIdx+1] = extracted >> (wordSize - bitOffset) + } + + outPos += bitsExtracted + } +} + +// Deposit creates a new BitSet and deposits bits according to a mask. +// See DepositTo for details. +func (b *BitSet) Deposit(mask *BitSet) *BitSet { + dst := New(mask.length) + b.DepositTo(mask, dst) + return dst +} + +// DepositTo spreads bits from a compacted form in the BitSet into positions +// specified by mask in dst. This is the inverse operation of Extract. +// +// For example, if mask has bits set at positions 1,4,5, then DepositTo will +// take consecutive bits 0,1,2 from the source BitSet and place them into +// positions 1,4,5 in the destination BitSet. +func (b *BitSet) DepositTo(mask *BitSet, dst *BitSet) { + panicIfNull(b) + panicIfNull(mask) + panicIfNull(dst) + + if len(dst.set) == 0 || len(mask.set) == 0 || len(b.set) == 0 { + return + } + + inPos := uint(0) + length := len(mask.set) + if len(dst.set) < length { + length = len(dst.set) + } + + // Process each word + for i := 0; i < length; i++ { + if mask.set[i] == 0 { + continue // Skip words with no bits to deposit + } + + // Calculate source word index + wordIdx := inPos >> log2WordSize + if wordIdx >= uint(len(b.set)) { + break // No more source bits available + } + + // Get source bits, handling word boundary crossing + sourceBits := b.set[wordIdx] + bitOffset := inPos & wordMask + if wordIdx+1 < uint(len(b.set)) && bitOffset != 0 { + // Combine bits from current and next word + sourceBits = (sourceBits >> bitOffset) | + (b.set[wordIdx+1] << (wordSize - bitOffset)) + } else { + sourceBits >>= bitOffset + } + + // Deposit bits according to mask + dst.set[i] = (dst.set[i] &^ mask.set[i]) | pdep(sourceBits, mask.set[i]) + inPos += uint(bits.OnesCount64(mask.set[i])) + } +} + +//go:generate go run cmd/pextgen/main.go -pkg=bitset + +func pext(w, m uint64) (result uint64) { + var outPos uint + + // Process byte by byte + for i := 0; i < 8; i++ { + shift := i << 3 // i * 8 using bit shift + b := uint8(w >> shift) + mask := uint8(m >> shift) + + extracted := pextLUT[b][mask] + bits := popLUT[mask] + + result |= uint64(extracted) << outPos + outPos += uint(bits) + } + + return result +} + +func pdep(w, m uint64) (result uint64) { + var inPos uint + + // Process byte by byte + for i := 0; i < 8; i++ { + shift := i << 3 // i * 8 using bit shift + mask := uint8(m >> shift) + bits := popLUT[mask] + + // Get the bits we'll deposit from the source + b := uint8(w >> inPos) + + // Deposit them according to the mask for this byte + deposited := pdepLUT[b][mask] + + // Add to result + result |= uint64(deposited) << shift + inPos += uint(bits) + } + + return result +} diff --git a/vendor/github.com/bits-and-blooms/bitset/bitset_iter.go b/vendor/github.com/bits-and-blooms/bitset/bitset_iter.go new file mode 100644 index 0000000..79bf8a0 --- /dev/null +++ b/vendor/github.com/bits-and-blooms/bitset/bitset_iter.go @@ -0,0 +1,23 @@ +//go:build go1.23 +// +build go1.23 + +package bitset + +import ( + "iter" + "math/bits" +) + +func (b *BitSet) EachSet() iter.Seq[uint] { + return func(yield func(uint) bool) { + for wordIndex, word := range b.set { + idx := 0 + for trail := bits.TrailingZeros64(word); trail != 64; trail = bits.TrailingZeros64(word >> idx) { + if !yield(uint(wordIndex<> 1) & 0x5555555555555555 - x = (x>>2)&0x3333333333333333 + x&0x3333333333333333 - x += x >> 4 - x &= 0x0f0f0f0f0f0f0f0f - x *= 0x0101010101010101 - return x >> 56 -} +import "math/bits" -func popcntSliceGo(s []uint64) uint64 { - cnt := uint64(0) +func popcntSlice(s []uint64) (cnt uint64) { for _, x := range s { - cnt += popcount(x) + cnt += uint64(bits.OnesCount64(x)) } - return cnt + return } -func popcntMaskSliceGo(s, m []uint64) uint64 { - cnt := uint64(0) +func popcntMaskSlice(s, m []uint64) (cnt uint64) { + // The next line is to help the bounds checker, it matters! + _ = m[len(s)-1] // BCE for i := range s { - cnt += popcount(s[i] &^ m[i]) + cnt += uint64(bits.OnesCount64(s[i] &^ m[i])) } - return cnt + return } -func popcntAndSliceGo(s, m []uint64) uint64 { - cnt := uint64(0) +// popcntAndSlice computes the population count of the AND of two slices. +// It assumes that len(m) >= len(s) > 0. +func popcntAndSlice(s, m []uint64) (cnt uint64) { + // The next line is to help the bounds checker, it matters! + _ = m[len(s)-1] // BCE for i := range s { - cnt += popcount(s[i] & m[i]) + cnt += uint64(bits.OnesCount64(s[i] & m[i])) } - return cnt + return } -func popcntOrSliceGo(s, m []uint64) uint64 { - cnt := uint64(0) +// popcntOrSlice computes the population count of the OR of two slices. +// It assumes that len(m) >= len(s) > 0. +func popcntOrSlice(s, m []uint64) (cnt uint64) { + // The next line is to help the bounds checker, it matters! + _ = m[len(s)-1] // BCE for i := range s { - cnt += popcount(s[i] | m[i]) + cnt += uint64(bits.OnesCount64(s[i] | m[i])) } - return cnt + return } -func popcntXorSliceGo(s, m []uint64) uint64 { - cnt := uint64(0) +// popcntXorSlice computes the population count of the XOR of two slices. +// It assumes that len(m) >= len(s) > 0. +func popcntXorSlice(s, m []uint64) (cnt uint64) { + // The next line is to help the bounds checker, it matters! + _ = m[len(s)-1] // BCE for i := range s { - cnt += popcount(s[i] ^ m[i]) + cnt += uint64(bits.OnesCount64(s[i] ^ m[i])) } - return cnt + return } diff --git a/vendor/github.com/bits-and-blooms/bitset/popcnt_19.go b/vendor/github.com/bits-and-blooms/bitset/popcnt_19.go deleted file mode 100644 index fc8ff4f..0000000 --- a/vendor/github.com/bits-and-blooms/bitset/popcnt_19.go +++ /dev/null @@ -1,45 +0,0 @@ -// +build go1.9 - -package bitset - -import "math/bits" - -func popcntSlice(s []uint64) uint64 { - var cnt int - for _, x := range s { - cnt += bits.OnesCount64(x) - } - return uint64(cnt) -} - -func popcntMaskSlice(s, m []uint64) uint64 { - var cnt int - for i := range s { - cnt += bits.OnesCount64(s[i] &^ m[i]) - } - return uint64(cnt) -} - -func popcntAndSlice(s, m []uint64) uint64 { - var cnt int - for i := range s { - cnt += bits.OnesCount64(s[i] & m[i]) - } - return uint64(cnt) -} - -func popcntOrSlice(s, m []uint64) uint64 { - var cnt int - for i := range s { - cnt += bits.OnesCount64(s[i] | m[i]) - } - return uint64(cnt) -} - -func popcntXorSlice(s, m []uint64) uint64 { - var cnt int - for i := range s { - cnt += bits.OnesCount64(s[i] ^ m[i]) - } - return uint64(cnt) -} diff --git a/vendor/github.com/bits-and-blooms/bitset/popcnt_amd64.go b/vendor/github.com/bits-and-blooms/bitset/popcnt_amd64.go deleted file mode 100644 index 4cf64f2..0000000 --- a/vendor/github.com/bits-and-blooms/bitset/popcnt_amd64.go +++ /dev/null @@ -1,68 +0,0 @@ -// +build !go1.9 -// +build amd64,!appengine - -package bitset - -// *** the following functions are defined in popcnt_amd64.s - -//go:noescape - -func hasAsm() bool - -// useAsm is a flag used to select the GO or ASM implementation of the popcnt function -var useAsm = hasAsm() - -//go:noescape - -func popcntSliceAsm(s []uint64) uint64 - -//go:noescape - -func popcntMaskSliceAsm(s, m []uint64) uint64 - -//go:noescape - -func popcntAndSliceAsm(s, m []uint64) uint64 - -//go:noescape - -func popcntOrSliceAsm(s, m []uint64) uint64 - -//go:noescape - -func popcntXorSliceAsm(s, m []uint64) uint64 - -func popcntSlice(s []uint64) uint64 { - if useAsm { - return popcntSliceAsm(s) - } - return popcntSliceGo(s) -} - -func popcntMaskSlice(s, m []uint64) uint64 { - if useAsm { - return popcntMaskSliceAsm(s, m) - } - return popcntMaskSliceGo(s, m) -} - -func popcntAndSlice(s, m []uint64) uint64 { - if useAsm { - return popcntAndSliceAsm(s, m) - } - return popcntAndSliceGo(s, m) -} - -func popcntOrSlice(s, m []uint64) uint64 { - if useAsm { - return popcntOrSliceAsm(s, m) - } - return popcntOrSliceGo(s, m) -} - -func popcntXorSlice(s, m []uint64) uint64 { - if useAsm { - return popcntXorSliceAsm(s, m) - } - return popcntXorSliceGo(s, m) -} diff --git a/vendor/github.com/bits-and-blooms/bitset/popcnt_amd64.s b/vendor/github.com/bits-and-blooms/bitset/popcnt_amd64.s deleted file mode 100644 index 666c0dc..0000000 --- a/vendor/github.com/bits-and-blooms/bitset/popcnt_amd64.s +++ /dev/null @@ -1,104 +0,0 @@ -// +build !go1.9 -// +build amd64,!appengine - -TEXT ·hasAsm(SB),4,$0-1 -MOVQ $1, AX -CPUID -SHRQ $23, CX -ANDQ $1, CX -MOVB CX, ret+0(FP) -RET - -#define POPCNTQ_DX_DX BYTE $0xf3; BYTE $0x48; BYTE $0x0f; BYTE $0xb8; BYTE $0xd2 - -TEXT ·popcntSliceAsm(SB),4,$0-32 -XORQ AX, AX -MOVQ s+0(FP), SI -MOVQ s_len+8(FP), CX -TESTQ CX, CX -JZ popcntSliceEnd -popcntSliceLoop: -BYTE $0xf3; BYTE $0x48; BYTE $0x0f; BYTE $0xb8; BYTE $0x16 // POPCNTQ (SI), DX -ADDQ DX, AX -ADDQ $8, SI -LOOP popcntSliceLoop -popcntSliceEnd: -MOVQ AX, ret+24(FP) -RET - -TEXT ·popcntMaskSliceAsm(SB),4,$0-56 -XORQ AX, AX -MOVQ s+0(FP), SI -MOVQ s_len+8(FP), CX -TESTQ CX, CX -JZ popcntMaskSliceEnd -MOVQ m+24(FP), DI -popcntMaskSliceLoop: -MOVQ (DI), DX -NOTQ DX -ANDQ (SI), DX -POPCNTQ_DX_DX -ADDQ DX, AX -ADDQ $8, SI -ADDQ $8, DI -LOOP popcntMaskSliceLoop -popcntMaskSliceEnd: -MOVQ AX, ret+48(FP) -RET - -TEXT ·popcntAndSliceAsm(SB),4,$0-56 -XORQ AX, AX -MOVQ s+0(FP), SI -MOVQ s_len+8(FP), CX -TESTQ CX, CX -JZ popcntAndSliceEnd -MOVQ m+24(FP), DI -popcntAndSliceLoop: -MOVQ (DI), DX -ANDQ (SI), DX -POPCNTQ_DX_DX -ADDQ DX, AX -ADDQ $8, SI -ADDQ $8, DI -LOOP popcntAndSliceLoop -popcntAndSliceEnd: -MOVQ AX, ret+48(FP) -RET - -TEXT ·popcntOrSliceAsm(SB),4,$0-56 -XORQ AX, AX -MOVQ s+0(FP), SI -MOVQ s_len+8(FP), CX -TESTQ CX, CX -JZ popcntOrSliceEnd -MOVQ m+24(FP), DI -popcntOrSliceLoop: -MOVQ (DI), DX -ORQ (SI), DX -POPCNTQ_DX_DX -ADDQ DX, AX -ADDQ $8, SI -ADDQ $8, DI -LOOP popcntOrSliceLoop -popcntOrSliceEnd: -MOVQ AX, ret+48(FP) -RET - -TEXT ·popcntXorSliceAsm(SB),4,$0-56 -XORQ AX, AX -MOVQ s+0(FP), SI -MOVQ s_len+8(FP), CX -TESTQ CX, CX -JZ popcntXorSliceEnd -MOVQ m+24(FP), DI -popcntXorSliceLoop: -MOVQ (DI), DX -XORQ (SI), DX -POPCNTQ_DX_DX -ADDQ DX, AX -ADDQ $8, SI -ADDQ $8, DI -LOOP popcntXorSliceLoop -popcntXorSliceEnd: -MOVQ AX, ret+48(FP) -RET diff --git a/vendor/github.com/bits-and-blooms/bitset/popcnt_generic.go b/vendor/github.com/bits-and-blooms/bitset/popcnt_generic.go deleted file mode 100644 index 21e0ff7..0000000 --- a/vendor/github.com/bits-and-blooms/bitset/popcnt_generic.go +++ /dev/null @@ -1,24 +0,0 @@ -// +build !go1.9 -// +build !amd64 appengine - -package bitset - -func popcntSlice(s []uint64) uint64 { - return popcntSliceGo(s) -} - -func popcntMaskSlice(s, m []uint64) uint64 { - return popcntMaskSliceGo(s, m) -} - -func popcntAndSlice(s, m []uint64) uint64 { - return popcntAndSliceGo(s, m) -} - -func popcntOrSlice(s, m []uint64) uint64 { - return popcntOrSliceGo(s, m) -} - -func popcntXorSlice(s, m []uint64) uint64 { - return popcntXorSliceGo(s, m) -} diff --git a/vendor/github.com/bits-and-blooms/bitset/select.go b/vendor/github.com/bits-and-blooms/bitset/select.go new file mode 100644 index 0000000..a43c6bd --- /dev/null +++ b/vendor/github.com/bits-and-blooms/bitset/select.go @@ -0,0 +1,47 @@ +package bitset + +import "math/bits" + +func select64(w uint64, j uint) uint { + seen := 0 + // Divide 64bit + part := w & 0xFFFFFFFF + n := uint(bits.OnesCount64(part)) + if n <= j { + part = w >> 32 + seen += 32 + j -= n + } + ww := part + + // Divide 32bit + part = ww & 0xFFFF + + n = uint(bits.OnesCount64(part)) + if n <= j { + part = ww >> 16 + seen += 16 + j -= n + } + ww = part + + // Divide 16bit + part = ww & 0xFF + n = uint(bits.OnesCount64(part)) + if n <= j { + part = ww >> 8 + seen += 8 + j -= n + } + ww = part + + // Lookup in final byte + counter := 0 + for ; counter < 8; counter++ { + j -= uint((ww >> counter) & 1) + if j+1 == 0 { + break + } + } + return uint(seen + counter) +} diff --git a/vendor/github.com/bits-and-blooms/bitset/trailing_zeros_18.go b/vendor/github.com/bits-and-blooms/bitset/trailing_zeros_18.go deleted file mode 100644 index c52b61b..0000000 --- a/vendor/github.com/bits-and-blooms/bitset/trailing_zeros_18.go +++ /dev/null @@ -1,14 +0,0 @@ -// +build !go1.9 - -package bitset - -var deBruijn = [...]byte{ - 0, 1, 56, 2, 57, 49, 28, 3, 61, 58, 42, 50, 38, 29, 17, 4, - 62, 47, 59, 36, 45, 43, 51, 22, 53, 39, 33, 30, 24, 18, 12, 5, - 63, 55, 48, 27, 60, 41, 37, 16, 46, 35, 44, 21, 52, 32, 23, 11, - 54, 26, 40, 15, 34, 20, 31, 10, 25, 14, 19, 9, 13, 8, 7, 6, -} - -func trailingZeroes64(v uint64) uint { - return uint(deBruijn[((v&-v)*0x03f79d71b4ca8b09)>>58]) -} diff --git a/vendor/github.com/bits-and-blooms/bitset/trailing_zeros_19.go b/vendor/github.com/bits-and-blooms/bitset/trailing_zeros_19.go deleted file mode 100644 index 36a988e..0000000 --- a/vendor/github.com/bits-and-blooms/bitset/trailing_zeros_19.go +++ /dev/null @@ -1,9 +0,0 @@ -// +build go1.9 - -package bitset - -import "math/bits" - -func trailingZeroes64(v uint64) uint { - return uint(bits.TrailingZeros64(v)) -} diff --git a/vendor/github.com/boombuler/barcode/barcode.go b/vendor/github.com/boombuler/barcode/barcode.go index 25f4a69..9d23881 100644 --- a/vendor/github.com/boombuler/barcode/barcode.go +++ b/vendor/github.com/boombuler/barcode/barcode.go @@ -1,6 +1,8 @@ package barcode -import "image" +import ( + "image" +) const ( TypeAztec = "Aztec" @@ -40,3 +42,7 @@ type BarcodeIntCS interface { Barcode CheckSum() int } + +type BarcodeColor interface { + ColorScheme() ColorScheme +} diff --git a/vendor/github.com/boombuler/barcode/color_scheme.go b/vendor/github.com/boombuler/barcode/color_scheme.go new file mode 100644 index 0000000..e8ccc7a --- /dev/null +++ b/vendor/github.com/boombuler/barcode/color_scheme.go @@ -0,0 +1,39 @@ +package barcode + +import "image/color" + +// ColorScheme defines a structure for color schemes used in barcode rendering. +// It includes the color model, background color, and foreground color. +type ColorScheme struct { + Model color.Model // Color model to be used (e.g., grayscale, RGB, RGBA) + Background color.Color // Color of the background + Foreground color.Color // Color of the foreground (e.g., bars in a barcode) +} + +// ColorScheme8 represents a color scheme with 8-bit grayscale colors. +var ColorScheme8 = ColorScheme{ + Model: color.GrayModel, + Background: color.Gray{Y: 255}, + Foreground: color.Gray{Y: 0}, +} + +// ColorScheme16 represents a color scheme with 16-bit grayscale colors. +var ColorScheme16 = ColorScheme{ + Model: color.Gray16Model, + Background: color.White, + Foreground: color.Black, +} + +// ColorScheme24 represents a color scheme with 24-bit RGB colors. +var ColorScheme24 = ColorScheme{ + Model: color.RGBAModel, + Background: color.RGBA{255, 255, 255, 255}, + Foreground: color.RGBA{0, 0, 0, 255}, +} + +// ColorScheme32 represents a color scheme with 32-bit RGBA colors, which is similar to ColorScheme24 but typically includes alpha for transparency. +var ColorScheme32 = ColorScheme{ + Model: color.RGBAModel, + Background: color.RGBA{255, 255, 255, 255}, + Foreground: color.RGBA{0, 0, 0, 255}, +} diff --git a/vendor/github.com/boombuler/barcode/qr/encoder.go b/vendor/github.com/boombuler/barcode/qr/encoder.go index 2c6ab21..81ceb88 100644 --- a/vendor/github.com/boombuler/barcode/qr/encoder.go +++ b/vendor/github.com/boombuler/barcode/qr/encoder.go @@ -54,8 +54,8 @@ func (e Encoding) String() string { return "" } -// Encode returns a QR barcode with the given content, error correction level and uses the given encoding -func Encode(content string, level ErrorCorrectionLevel, mode Encoding) (barcode.Barcode, error) { +// Encode returns a QR barcode with the given content and color scheme, error correction level and uses the given encoding +func EncodeWithColor(content string, level ErrorCorrectionLevel, mode Encoding, color barcode.ColorScheme) (barcode.Barcode, error) { bits, vi, err := mode.getEncoder()(content, level) if err != nil { return nil, err @@ -63,19 +63,23 @@ func Encode(content string, level ErrorCorrectionLevel, mode Encoding) (barcode. blocks := splitToBlocks(bits.IterateBytes(), vi) data := blocks.interleave(vi) - result := render(data, vi) + result := render(data, vi, color) result.content = content return result, nil } -func render(data []byte, vi *versionInfo) *qrcode { +func Encode(content string, level ErrorCorrectionLevel, mode Encoding) (barcode.Barcode, error) { + return EncodeWithColor(content, level, mode, barcode.ColorScheme16) +} + +func render(data []byte, vi *versionInfo, color barcode.ColorScheme) *qrcode { dim := vi.modulWidth() results := make([]*qrcode, 8) for i := 0; i < 8; i++ { - results[i] = newBarcode(dim) + results[i] = newBarCodeWithColor(dim, color) } - occupied := newBarcode(dim) + occupied := newBarCodeWithColor(dim, color) setAll := func(x int, y int, val bool) { occupied.Set(x, y, true) diff --git a/vendor/github.com/boombuler/barcode/qr/qrcode.go b/vendor/github.com/boombuler/barcode/qr/qrcode.go index 1360760..2fb44ab 100644 --- a/vendor/github.com/boombuler/barcode/qr/qrcode.go +++ b/vendor/github.com/boombuler/barcode/qr/qrcode.go @@ -13,6 +13,7 @@ type qrcode struct { dimension int data *utils.BitList content string + color barcode.ColorScheme } func (qr *qrcode) Content() string { @@ -24,7 +25,11 @@ func (qr *qrcode) Metadata() barcode.Metadata { } func (qr *qrcode) ColorModel() color.Model { - return color.Gray16Model + return qr.color.Model +} + +func (c *qrcode) ColorScheme() barcode.ColorScheme { + return c.color } func (qr *qrcode) Bounds() image.Rectangle { @@ -33,9 +38,9 @@ func (qr *qrcode) Bounds() image.Rectangle { func (qr *qrcode) At(x, y int) color.Color { if qr.Get(x, y) { - return color.Black + return qr.color.Foreground } - return color.White + return qr.color.Background } func (qr *qrcode) Get(x, y int) bool { @@ -158,9 +163,14 @@ func (qr *qrcode) calcPenaltyRule4() uint { return uint(math.Min(floor, ceil) * 10) } -func newBarcode(dim int) *qrcode { +func newBarCodeWithColor(dim int, color barcode.ColorScheme) *qrcode { res := new(qrcode) res.dimension = dim res.data = utils.NewBitList(dim * dim) + res.color = color return res } + +func newBarcode(dim int) *qrcode { + return newBarCodeWithColor(dim, barcode.ColorScheme16) +} diff --git a/vendor/github.com/boombuler/barcode/scaledbarcode.go b/vendor/github.com/boombuler/barcode/scaledbarcode.go index 152b180..491cc3f 100644 --- a/vendor/github.com/boombuler/barcode/scaledbarcode.go +++ b/vendor/github.com/boombuler/barcode/scaledbarcode.go @@ -49,11 +49,22 @@ func (bc *intCSscaledBC) CheckSum() int { // Scale returns a resized barcode with the given width and height. func Scale(bc Barcode, width, height int) (Barcode, error) { + var fill color.Color + if v, ok := bc.(BarcodeColor); ok { + fill = v.ColorScheme().Background + } else { + fill = color.White + } + return ScaleWithFill(bc, width, height, fill) +} + +// Scale returns a resized barcode with the given width, height and fill color. +func ScaleWithFill(bc Barcode, width, height int, fill color.Color) (Barcode, error) { switch bc.Metadata().Dimensions { case 1: - return scale1DCode(bc, width, height) + return scale1DCode(bc, width, height, fill) case 2: - return scale2DCode(bc, width, height) + return scale2DCode(bc, width, height, fill) } return nil, errors.New("unsupported barcode format") @@ -72,7 +83,7 @@ func newScaledBC(wrapped Barcode, wrapperFunc wrapFunc, rect image.Rectangle) Ba return result } -func scale2DCode(bc Barcode, width, height int) (Barcode, error) { +func scale2DCode(bc Barcode, width, height int, fill color.Color) (Barcode, error) { orgBounds := bc.Bounds() orgWidth := orgBounds.Max.X - orgBounds.Min.X orgHeight := orgBounds.Max.Y - orgBounds.Min.Y @@ -87,12 +98,12 @@ func scale2DCode(bc Barcode, width, height int) (Barcode, error) { wrap := func(x, y int) color.Color { if x < offsetX || y < offsetY { - return color.White + return fill } x = (x - offsetX) / factor y = (y - offsetY) / factor if x >= orgWidth || y >= orgHeight { - return color.White + return fill } return bc.At(x, y) } @@ -104,7 +115,7 @@ func scale2DCode(bc Barcode, width, height int) (Barcode, error) { ), nil } -func scale1DCode(bc Barcode, width, height int) (Barcode, error) { +func scale1DCode(bc Barcode, width, height int, fill color.Color) (Barcode, error) { orgBounds := bc.Bounds() orgWidth := orgBounds.Max.X - orgBounds.Min.X factor := int(float64(width) / float64(orgWidth)) @@ -116,12 +127,12 @@ func scale1DCode(bc Barcode, width, height int) (Barcode, error) { wrap := func(x, y int) color.Color { if x < offsetX { - return color.White + return fill } x = (x - offsetX) / factor if x >= orgWidth { - return color.White + return fill } return bc.At(x, 0) } diff --git a/vendor/github.com/boombuler/barcode/utils/base1dcode.go b/vendor/github.com/boombuler/barcode/utils/base1dcode.go index a335c0c..1b77e4f 100644 --- a/vendor/github.com/boombuler/barcode/utils/base1dcode.go +++ b/vendor/github.com/boombuler/barcode/utils/base1dcode.go @@ -12,6 +12,7 @@ type base1DCode struct { *BitList kind string content string + color barcode.ColorScheme } type base1DCodeIntCS struct { @@ -28,7 +29,11 @@ func (c *base1DCode) Metadata() barcode.Metadata { } func (c *base1DCode) ColorModel() color.Model { - return color.Gray16Model + return c.color.Model +} + +func (c *base1DCode) ColorScheme() barcode.ColorScheme { + return c.color } func (c *base1DCode) Bounds() image.Rectangle { @@ -37,9 +42,9 @@ func (c *base1DCode) Bounds() image.Rectangle { func (c *base1DCode) At(x, y int) color.Color { if c.GetBit(x) { - return color.Black + return c.color.Foreground } - return color.White + return c.color.Background } func (c *base1DCodeIntCS) CheckSum() int { @@ -48,10 +53,20 @@ func (c *base1DCodeIntCS) CheckSum() int { // New1DCodeIntCheckSum creates a new 1D barcode where the bars are represented by the bits in the bars BitList func New1DCodeIntCheckSum(codeKind, content string, bars *BitList, checksum int) barcode.BarcodeIntCS { - return &base1DCodeIntCS{base1DCode{bars, codeKind, content}, checksum} + return &base1DCodeIntCS{base1DCode{bars, codeKind, content, barcode.ColorScheme16}, checksum} +} + +// New1DCodeIntCheckSum creates a new 1D barcode where the bars are represented by the bits in the bars BitList +func New1DCodeIntCheckSumWithColor(codeKind, content string, bars *BitList, checksum int, color barcode.ColorScheme) barcode.BarcodeIntCS { + return &base1DCodeIntCS{base1DCode{bars, codeKind, content, color}, checksum} } // New1DCode creates a new 1D barcode where the bars are represented by the bits in the bars BitList func New1DCode(codeKind, content string, bars *BitList) barcode.Barcode { - return &base1DCode{bars, codeKind, content} + return &base1DCode{bars, codeKind, content, barcode.ColorScheme16} +} + +// New1DCode creates a new 1D barcode where the bars are represented by the bits in the bars BitList +func New1DCodeWithColor(codeKind, content string, bars *BitList, color barcode.ColorScheme) barcode.Barcode { + return &base1DCode{bars, codeKind, content, color} } diff --git a/vendor/github.com/cpuguy83/go-md2man/v2/md2man/debug.go b/vendor/github.com/cpuguy83/go-md2man/v2/md2man/debug.go new file mode 100644 index 0000000..0ec4b12 --- /dev/null +++ b/vendor/github.com/cpuguy83/go-md2man/v2/md2man/debug.go @@ -0,0 +1,62 @@ +package md2man + +import ( + "fmt" + "io" + "os" + "strings" + + "github.com/russross/blackfriday/v2" +) + +func fmtListFlags(flags blackfriday.ListType) string { + knownFlags := []struct { + name string + flag blackfriday.ListType + }{ + {"ListTypeOrdered", blackfriday.ListTypeOrdered}, + {"ListTypeDefinition", blackfriday.ListTypeDefinition}, + {"ListTypeTerm", blackfriday.ListTypeTerm}, + {"ListItemContainsBlock", blackfriday.ListItemContainsBlock}, + {"ListItemBeginningOfList", blackfriday.ListItemBeginningOfList}, + {"ListItemEndOfList", blackfriday.ListItemEndOfList}, + } + + var f []string + for _, kf := range knownFlags { + if flags&kf.flag != 0 { + f = append(f, kf.name) + flags &^= kf.flag + } + } + if flags != 0 { + f = append(f, fmt.Sprintf("Unknown(%#x)", flags)) + } + return strings.Join(f, "|") +} + +type debugDecorator struct { + blackfriday.Renderer +} + +func depth(node *blackfriday.Node) int { + d := 0 + for n := node.Parent; n != nil; n = n.Parent { + d++ + } + return d +} + +func (d *debugDecorator) RenderNode(w io.Writer, node *blackfriday.Node, entering bool) blackfriday.WalkStatus { + fmt.Fprintf(os.Stderr, "%s%s %v %v\n", + strings.Repeat(" ", depth(node)), + map[bool]string{true: "+", false: "-"}[entering], + node, + fmtListFlags(node.ListFlags)) + var b strings.Builder + status := d.Renderer.RenderNode(io.MultiWriter(&b, w), node, entering) + if b.Len() > 0 { + fmt.Fprintf(os.Stderr, ">> %q\n", b.String()) + } + return status +} diff --git a/vendor/github.com/cpuguy83/go-md2man/v2/md2man/md2man.go b/vendor/github.com/cpuguy83/go-md2man/v2/md2man/md2man.go index b480056..5673f5c 100644 --- a/vendor/github.com/cpuguy83/go-md2man/v2/md2man/md2man.go +++ b/vendor/github.com/cpuguy83/go-md2man/v2/md2man/md2man.go @@ -1,14 +1,24 @@ +// Package md2man aims in converting markdown into roff (man pages). package md2man import ( + "os" + "strconv" + "github.com/russross/blackfriday/v2" ) // Render converts a markdown document into a roff formatted document. func Render(doc []byte) []byte { renderer := NewRoffRenderer() + var r blackfriday.Renderer = renderer + if v, _ := strconv.ParseBool(os.Getenv("MD2MAN_DEBUG")); v { + r = &debugDecorator{Renderer: r} + } return blackfriday.Run(doc, - []blackfriday.Option{blackfriday.WithRenderer(renderer), - blackfriday.WithExtensions(renderer.GetExtensions())}...) + []blackfriday.Option{ + blackfriday.WithRenderer(r), + blackfriday.WithExtensions(renderer.GetExtensions()), + }...) } diff --git a/vendor/github.com/cpuguy83/go-md2man/v2/md2man/roff.go b/vendor/github.com/cpuguy83/go-md2man/v2/md2man/roff.go index be2b343..4f1070f 100644 --- a/vendor/github.com/cpuguy83/go-md2man/v2/md2man/roff.go +++ b/vendor/github.com/cpuguy83/go-md2man/v2/md2man/roff.go @@ -1,6 +1,8 @@ package md2man import ( + "bufio" + "bytes" "fmt" "io" "os" @@ -12,68 +14,72 @@ import ( // roffRenderer implements the blackfriday.Renderer interface for creating // roff format (manpages) from markdown text type roffRenderer struct { - extensions blackfriday.Extensions listCounters []int firstHeader bool - firstDD bool listDepth int } const ( - titleHeader = ".TH " - topLevelHeader = "\n\n.SH " - secondLevelHdr = "\n.SH " - otherHeader = "\n.SS " - crTag = "\n" - emphTag = "\\fI" - emphCloseTag = "\\fP" - strongTag = "\\fB" - strongCloseTag = "\\fP" - breakTag = "\n.br\n" - paraTag = "\n.PP\n" - hruleTag = "\n.ti 0\n\\l'\\n(.lu'\n" - linkTag = "\n\\[la]" - linkCloseTag = "\\[ra]" - codespanTag = "\\fB\\fC" - codespanCloseTag = "\\fR" - codeTag = "\n.PP\n.RS\n\n.nf\n" - codeCloseTag = "\n.fi\n.RE\n" - quoteTag = "\n.PP\n.RS\n" - quoteCloseTag = "\n.RE\n" - listTag = "\n.RS\n" - listCloseTag = "\n.RE\n" - dtTag = "\n.TP\n" - dd2Tag = "\n" - tableStart = "\n.TS\nallbox;\n" - tableEnd = ".TE\n" - tableCellStart = "T{\n" - tableCellEnd = "\nT}\n" + titleHeader = ".TH " + topLevelHeader = "\n\n.SH " + secondLevelHdr = "\n.SH " + otherHeader = "\n.SS " + crTag = "\n" + emphTag = "\\fI" + emphCloseTag = "\\fP" + strongTag = "\\fB" + strongCloseTag = "\\fP" + breakTag = "\n.br\n" + paraTag = "\n.PP\n" + hruleTag = "\n.ti 0\n\\l'\\n(.lu'\n" + linkTag = "\n\\[la]" + linkCloseTag = "\\[ra]" + codespanTag = "\\fB" + codespanCloseTag = "\\fR" + codeTag = "\n.EX\n" + codeCloseTag = ".EE\n" // Do not prepend a newline character since code blocks, by definition, include a newline already (or at least as how blackfriday gives us on). + quoteTag = "\n.PP\n.RS\n" + quoteCloseTag = "\n.RE\n" + listTag = "\n.RS\n" + listCloseTag = ".RE\n" + dtTag = "\n.TP\n" + dd2Tag = "\n" + tableStart = "\n.TS\nallbox;\n" + tableEnd = ".TE\n" + tableCellStart = "T{\n" + tableCellEnd = "\nT}" + tablePreprocessor = `'\" t` ) // NewRoffRenderer creates a new blackfriday Renderer for generating roff documents // from markdown -func NewRoffRenderer() *roffRenderer { // nolint: golint - var extensions blackfriday.Extensions - - extensions |= blackfriday.NoIntraEmphasis - extensions |= blackfriday.Tables - extensions |= blackfriday.FencedCode - extensions |= blackfriday.SpaceHeadings - extensions |= blackfriday.Footnotes - extensions |= blackfriday.Titleblock - extensions |= blackfriday.DefinitionLists - return &roffRenderer{ - extensions: extensions, - } +func NewRoffRenderer() *roffRenderer { + return &roffRenderer{} } // GetExtensions returns the list of extensions used by this renderer implementation -func (r *roffRenderer) GetExtensions() blackfriday.Extensions { - return r.extensions +func (*roffRenderer) GetExtensions() blackfriday.Extensions { + return blackfriday.NoIntraEmphasis | + blackfriday.Tables | + blackfriday.FencedCode | + blackfriday.SpaceHeadings | + blackfriday.Footnotes | + blackfriday.Titleblock | + blackfriday.DefinitionLists } // RenderHeader handles outputting the header at document start func (r *roffRenderer) RenderHeader(w io.Writer, ast *blackfriday.Node) { + // We need to walk the tree to check if there are any tables. + // If there are, we need to enable the roff table preprocessor. + ast.Walk(func(node *blackfriday.Node, entering bool) blackfriday.WalkStatus { + if node.Type == blackfriday.Table { + out(w, tablePreprocessor+"\n") + return blackfriday.Terminate + } + return blackfriday.GoToNext + }) + // disable hyphenation out(w, ".nh\n") } @@ -86,12 +92,27 @@ func (r *roffRenderer) RenderFooter(w io.Writer, ast *blackfriday.Node) { // RenderNode is called for each node in a markdown document; based on the node // type the equivalent roff output is sent to the writer func (r *roffRenderer) RenderNode(w io.Writer, node *blackfriday.Node, entering bool) blackfriday.WalkStatus { - - var walkAction = blackfriday.GoToNext + walkAction := blackfriday.GoToNext switch node.Type { case blackfriday.Text: - escapeSpecialChars(w, node.Literal) + // Special case: format the NAME section as required for proper whatis parsing. + // Refer to the lexgrog(1) and groff_man(7) manual pages for details. + if node.Parent != nil && + node.Parent.Type == blackfriday.Paragraph && + node.Parent.Prev != nil && + node.Parent.Prev.Type == blackfriday.Heading && + node.Parent.Prev.FirstChild != nil && + bytes.EqualFold(node.Parent.Prev.FirstChild.Literal, []byte("NAME")) { + before, after, found := bytesCut(node.Literal, []byte(" - ")) + escapeSpecialChars(w, before) + if found { + out(w, ` \- `) + escapeSpecialChars(w, after) + } + } else { + escapeSpecialChars(w, node.Literal) + } case blackfriday.Softbreak: out(w, crTag) case blackfriday.Hardbreak: @@ -109,9 +130,16 @@ func (r *roffRenderer) RenderNode(w io.Writer, node *blackfriday.Node, entering out(w, strongCloseTag) } case blackfriday.Link: - if !entering { - out(w, linkTag+string(node.LinkData.Destination)+linkCloseTag) + // Don't render the link text for automatic links, because this + // will only duplicate the URL in the roff output. + // See https://daringfireball.net/projects/markdown/syntax#autolink + if !bytes.Equal(node.LinkData.Destination, node.FirstChild.Literal) { + out(w, string(node.FirstChild.Literal)) } + // Hyphens in a link must be escaped to avoid word-wrap in the rendered man page. + escapedLink := strings.ReplaceAll(string(node.LinkData.Destination), "-", "\\-") + out(w, linkTag+escapedLink+linkCloseTag) + walkAction = blackfriday.SkipChildren case blackfriday.Image: // ignore images walkAction = blackfriday.SkipChildren @@ -122,14 +150,25 @@ func (r *roffRenderer) RenderNode(w io.Writer, node *blackfriday.Node, entering case blackfriday.Document: break case blackfriday.Paragraph: - // roff .PP markers break lists - if r.listDepth > 0 { - return blackfriday.GoToNext - } if entering { - out(w, paraTag) + if r.listDepth > 0 { + // roff .PP markers break lists + if node.Prev != nil { // continued paragraph + if node.Prev.Type == blackfriday.List && node.Prev.ListFlags&blackfriday.ListTypeDefinition == 0 { + out(w, ".IP\n") + } else { + out(w, crTag) + } + } + } else if node.Prev != nil && node.Prev.Type == blackfriday.Heading { + out(w, crTag) + } else { + out(w, paraTag) + } } else { - out(w, crTag) + if node.Next == nil || node.Next.Type != blackfriday.List { + out(w, crTag) + } } case blackfriday.BlockQuote: if entering { @@ -160,6 +199,11 @@ func (r *roffRenderer) RenderNode(w io.Writer, node *blackfriday.Node, entering r.handleTableCell(w, node, entering) case blackfriday.HTMLSpan: // ignore other HTML tags + case blackfriday.HTMLBlock: + if bytes.HasPrefix(node.Literal, []byte("