forgejo/vendor/github.com/RoaringBitmap/roaring/roaring.go

1346 lines
39 KiB
Go
Raw Normal View History

2018-05-19 15:49:46 +03:00
// Package roaring is an implementation of Roaring Bitmaps in Go.
// They provide fast compressed bitmap data structures (also called bitset).
// They are ideally suited to represent sets of integers over
// relatively small ranges.
// See http://roaringbitmap.org for details.
package roaring
import (
"bufio"
"bytes"
"encoding/base64"
"fmt"
"io"
"strconv"
)
// Bitmap represents a compressed bitmap where you can add integers.
type Bitmap struct {
highlowcontainer roaringArray
}
// ToBase64 serializes a bitmap as Base64
func (rb *Bitmap) ToBase64() (string, error) {
buf := new(bytes.Buffer)
_, err := rb.WriteTo(buf)
return base64.StdEncoding.EncodeToString(buf.Bytes()), err
}
// FromBase64 deserializes a bitmap from Base64
func (rb *Bitmap) FromBase64(str string) (int64, error) {
data, err := base64.StdEncoding.DecodeString(str)
if err != nil {
return 0, err
}
buf := bytes.NewBuffer(data)
return rb.ReadFrom(buf)
}
// WriteTo writes a serialized version of this bitmap to stream.
// The format is compatible with other RoaringBitmap
// implementations (Java, C) and is documented here:
// https://github.com/RoaringBitmap/RoaringFormatSpec
func (rb *Bitmap) WriteTo(stream io.Writer) (int64, error) {
return rb.highlowcontainer.writeTo(stream)
}
// ToBytes returns an array of bytes corresponding to what is written
// when calling WriteTo
func (rb *Bitmap) ToBytes() ([]byte, error) {
return rb.highlowcontainer.toBytes()
}
// WriteToMsgpack writes a msgpack2/snappy-streaming compressed serialized
// version of this bitmap to stream. The format is not
// compatible with the WriteTo() format, and is
// experimental: it may produce smaller on disk
// footprint and/or be faster to read, depending
// on your content. Currently only the Go roaring
// implementation supports this format.
func (rb *Bitmap) WriteToMsgpack(stream io.Writer) (int64, error) {
return 0, rb.highlowcontainer.writeToMsgpack(stream)
}
// ReadFrom reads a serialized version of this bitmap from stream.
// The format is compatible with other RoaringBitmap
// implementations (Java, C) and is documented here:
// https://github.com/RoaringBitmap/RoaringFormatSpec
func (rb *Bitmap) ReadFrom(stream io.Reader) (int64, error) {
return rb.highlowcontainer.readFrom(stream)
}
// FromBuffer creates a bitmap from its serialized version stored in buffer
//
// The format specification is available here:
// https://github.com/RoaringBitmap/RoaringFormatSpec
//
// The provided byte array (buf) is expected to be a constant.
// The function makes the best effort attempt not to copy data.
// You should take care not to modify buff as it will
// likely result in unexpected program behavior.
//
// Resulting bitmaps are effectively immutable in the following sense:
// a copy-on-write marker is used so that when you modify the resulting
// bitmap, copies of selected data (containers) are made.
// You should *not* change the copy-on-write status of the resulting
// bitmaps (SetCopyOnWrite).
//
func (rb *Bitmap) FromBuffer(buf []byte) (int64, error) {
return rb.highlowcontainer.fromBuffer(buf)
}
// RunOptimize attempts to further compress the runs of consecutive values found in the bitmap
func (rb *Bitmap) RunOptimize() {
rb.highlowcontainer.runOptimize()
}
// HasRunCompression returns true if the bitmap benefits from run compression
func (rb *Bitmap) HasRunCompression() bool {
return rb.highlowcontainer.hasRunCompression()
}
// ReadFromMsgpack reads a msgpack2/snappy-streaming serialized
// version of this bitmap from stream. The format is
// expected is that written by the WriteToMsgpack()
// call; see additional notes there.
func (rb *Bitmap) ReadFromMsgpack(stream io.Reader) (int64, error) {
return 0, rb.highlowcontainer.readFromMsgpack(stream)
}
// MarshalBinary implements the encoding.BinaryMarshaler interface for the bitmap
func (rb *Bitmap) MarshalBinary() ([]byte, error) {
var buf bytes.Buffer
writer := bufio.NewWriter(&buf)
_, err := rb.WriteTo(writer)
if err != nil {
return nil, err
}
err = writer.Flush()
if err != nil {
return nil, err
}
return buf.Bytes(), nil
}
// UnmarshalBinary implements the encoding.BinaryUnmarshaler interface for the bitmap
func (rb *Bitmap) UnmarshalBinary(data []byte) error {
var buf bytes.Buffer
_, err := buf.Write(data)
if err != nil {
return err
}
reader := bufio.NewReader(&buf)
_, err = rb.ReadFrom(reader)
return err
}
// NewBitmap creates a new empty Bitmap (see also New)
func NewBitmap() *Bitmap {
return &Bitmap{}
}
// New creates a new empty Bitmap (same as NewBitmap)
func New() *Bitmap {
return &Bitmap{}
}
// Clear resets the Bitmap to be logically empty, but may retain
// some memory allocations that may speed up future operations
func (rb *Bitmap) Clear() {
rb.highlowcontainer.clear()
}
// ToArray creates a new slice containing all of the integers stored in the Bitmap in sorted order
func (rb *Bitmap) ToArray() []uint32 {
array := make([]uint32, rb.GetCardinality())
pos := 0
pos2 := 0
for pos < rb.highlowcontainer.size() {
hs := uint32(rb.highlowcontainer.getKeyAtIndex(pos)) << 16
c := rb.highlowcontainer.getContainerAtIndex(pos)
pos++
c.fillLeastSignificant16bits(array, pos2, hs)
pos2 += c.getCardinality()
}
return array
}
// GetSizeInBytes estimates the memory usage of the Bitmap. Note that this
// might differ slightly from the amount of bytes required for persistent storage
func (rb *Bitmap) GetSizeInBytes() uint64 {
size := uint64(8)
for _, c := range rb.highlowcontainer.containers {
size += uint64(2) + uint64(c.getSizeInBytes())
}
return size
}
// GetSerializedSizeInBytes computes the serialized size in bytes
// of the Bitmap. It should correspond to the
// number of bytes written when invoking WriteTo. You can expect
// that this function is much cheaper computationally than WriteTo.
func (rb *Bitmap) GetSerializedSizeInBytes() uint64 {
return rb.highlowcontainer.serializedSizeInBytes()
}
// BoundSerializedSizeInBytes returns an upper bound on the serialized size in bytes
// assuming that one wants to store "cardinality" integers in [0, universe_size)
func BoundSerializedSizeInBytes(cardinality uint64, universeSize uint64) uint64 {
contnbr := (universeSize + uint64(65535)) / uint64(65536)
if contnbr > cardinality {
contnbr = cardinality
// we can't have more containers than we have values
}
headermax := 8*contnbr + 4
if 4 > (contnbr+7)/8 {
headermax += 4
} else {
headermax += (contnbr + 7) / 8
}
valsarray := uint64(arrayContainerSizeInBytes(int(cardinality)))
valsbitmap := contnbr * uint64(bitmapContainerSizeInBytes())
valsbest := valsarray
if valsbest > valsbitmap {
valsbest = valsbitmap
}
return valsbest + headermax
}
// IntIterable allows you to iterate over the values in a Bitmap
type IntIterable interface {
HasNext() bool
Next() uint32
}
type intIterator struct {
pos int
hs uint32
iter shortIterable
highlowcontainer *roaringArray
}
// HasNext returns true if there are more integers to iterate over
func (ii *intIterator) HasNext() bool {
return ii.pos < ii.highlowcontainer.size()
}
func (ii *intIterator) init() {
if ii.highlowcontainer.size() > ii.pos {
ii.iter = ii.highlowcontainer.getContainerAtIndex(ii.pos).getShortIterator()
ii.hs = uint32(ii.highlowcontainer.getKeyAtIndex(ii.pos)) << 16
}
}
// Next returns the next integer
func (ii *intIterator) Next() uint32 {
x := uint32(ii.iter.next()) | ii.hs
if !ii.iter.hasNext() {
ii.pos = ii.pos + 1
ii.init()
}
return x
}
func newIntIterator(a *Bitmap) *intIterator {
p := new(intIterator)
p.pos = 0
p.highlowcontainer = &a.highlowcontainer
p.init()
return p
}
// ManyIntIterable allows you to iterate over the values in a Bitmap
type ManyIntIterable interface {
// pass in a buffer to fill up with values, returns how many values were returned
NextMany([]uint32) int
}
type manyIntIterator struct {
pos int
hs uint32
iter manyIterable
highlowcontainer *roaringArray
}
func (ii *manyIntIterator) init() {
if ii.highlowcontainer.size() > ii.pos {
ii.iter = ii.highlowcontainer.getContainerAtIndex(ii.pos).getManyIterator()
ii.hs = uint32(ii.highlowcontainer.getKeyAtIndex(ii.pos)) << 16
} else {
ii.iter = nil
}
}
func (ii *manyIntIterator) NextMany(buf []uint32) int {
n := 0
for n < len(buf) {
if ii.iter == nil {
break
}
moreN := ii.iter.nextMany(ii.hs, buf[n:])
n += moreN
if moreN == 0 {
ii.pos = ii.pos + 1
ii.init()
}
}
return n
}
func newManyIntIterator(a *Bitmap) *manyIntIterator {
p := new(manyIntIterator)
p.pos = 0
p.highlowcontainer = &a.highlowcontainer
p.init()
return p
}
// String creates a string representation of the Bitmap
func (rb *Bitmap) String() string {
// inspired by https://github.com/fzandona/goroar/
var buffer bytes.Buffer
start := []byte("{")
buffer.Write(start)
i := rb.Iterator()
counter := 0
if i.HasNext() {
counter = counter + 1
buffer.WriteString(strconv.FormatInt(int64(i.Next()), 10))
}
for i.HasNext() {
buffer.WriteString(",")
counter = counter + 1
// to avoid exhausting the memory
if counter > 0x40000 {
buffer.WriteString("...")
break
}
buffer.WriteString(strconv.FormatInt(int64(i.Next()), 10))
}
buffer.WriteString("}")
return buffer.String()
}
// Iterator creates a new IntIterable to iterate over the integers contained in the bitmap, in sorted order
func (rb *Bitmap) Iterator() IntIterable {
return newIntIterator(rb)
}
// Iterator creates a new ManyIntIterable to iterate over the integers contained in the bitmap, in sorted order
func (rb *Bitmap) ManyIterator() ManyIntIterable {
return newManyIntIterator(rb)
}
// Clone creates a copy of the Bitmap
func (rb *Bitmap) Clone() *Bitmap {
ptr := new(Bitmap)
ptr.highlowcontainer = *rb.highlowcontainer.clone()
return ptr
}
// Minimum get the smallest value stored in this roaring bitmap, assumes that it is not empty
func (rb *Bitmap) Minimum() uint32 {
return uint32(rb.highlowcontainer.containers[0].minimum()) | (uint32(rb.highlowcontainer.keys[0]) << 16)
}
// Maximum get the largest value stored in this roaring bitmap, assumes that it is not empty
func (rb *Bitmap) Maximum() uint32 {
lastindex := len(rb.highlowcontainer.containers) - 1
return uint32(rb.highlowcontainer.containers[lastindex].maximum()) | (uint32(rb.highlowcontainer.keys[lastindex]) << 16)
}
// Contains returns true if the integer is contained in the bitmap
func (rb *Bitmap) Contains(x uint32) bool {
hb := highbits(x)
c := rb.highlowcontainer.getContainer(hb)
return c != nil && c.contains(lowbits(x))
}
// ContainsInt returns true if the integer is contained in the bitmap (this is a convenience method, the parameter is casted to uint32 and Contains is called)
func (rb *Bitmap) ContainsInt(x int) bool {
return rb.Contains(uint32(x))
}
// Equals returns true if the two bitmaps contain the same integers
func (rb *Bitmap) Equals(o interface{}) bool {
srb, ok := o.(*Bitmap)
if ok {
return srb.highlowcontainer.equals(rb.highlowcontainer)
}
return false
}
// Add the integer x to the bitmap
func (rb *Bitmap) Add(x uint32) {
hb := highbits(x)
ra := &rb.highlowcontainer
i := ra.getIndex(hb)
if i >= 0 {
var c container
c = ra.getWritableContainerAtIndex(i).iaddReturnMinimized(lowbits(x))
rb.highlowcontainer.setContainerAtIndex(i, c)
} else {
newac := newArrayContainer()
rb.highlowcontainer.insertNewKeyValueAt(-i-1, hb, newac.iaddReturnMinimized(lowbits(x)))
}
}
// add the integer x to the bitmap, return the container and its index
func (rb *Bitmap) addwithptr(x uint32) (int, container) {
hb := highbits(x)
ra := &rb.highlowcontainer
i := ra.getIndex(hb)
var c container
if i >= 0 {
c = ra.getWritableContainerAtIndex(i).iaddReturnMinimized(lowbits(x))
rb.highlowcontainer.setContainerAtIndex(i, c)
return i, c
}
newac := newArrayContainer()
c = newac.iaddReturnMinimized(lowbits(x))
rb.highlowcontainer.insertNewKeyValueAt(-i-1, hb, c)
return -i - 1, c
}
// CheckedAdd adds the integer x to the bitmap and return true if it was added (false if the integer was already present)
func (rb *Bitmap) CheckedAdd(x uint32) bool {
// TODO: add unit tests for this method
hb := highbits(x)
i := rb.highlowcontainer.getIndex(hb)
if i >= 0 {
C := rb.highlowcontainer.getWritableContainerAtIndex(i)
oldcard := C.getCardinality()
C = C.iaddReturnMinimized(lowbits(x))
rb.highlowcontainer.setContainerAtIndex(i, C)
return C.getCardinality() > oldcard
}
newac := newArrayContainer()
rb.highlowcontainer.insertNewKeyValueAt(-i-1, hb, newac.iaddReturnMinimized(lowbits(x)))
return true
}
// AddInt adds the integer x to the bitmap (convenience method: the parameter is casted to uint32 and we call Add)
func (rb *Bitmap) AddInt(x int) {
rb.Add(uint32(x))
}
// Remove the integer x from the bitmap
func (rb *Bitmap) Remove(x uint32) {
hb := highbits(x)
i := rb.highlowcontainer.getIndex(hb)
if i >= 0 {
c := rb.highlowcontainer.getWritableContainerAtIndex(i).iremoveReturnMinimized(lowbits(x))
rb.highlowcontainer.setContainerAtIndex(i, c)
if rb.highlowcontainer.getContainerAtIndex(i).getCardinality() == 0 {
rb.highlowcontainer.removeAtIndex(i)
}
}
}
// CheckedRemove removes the integer x from the bitmap and return true if the integer was effectively remove (and false if the integer was not present)
func (rb *Bitmap) CheckedRemove(x uint32) bool {
// TODO: add unit tests for this method
hb := highbits(x)
i := rb.highlowcontainer.getIndex(hb)
if i >= 0 {
C := rb.highlowcontainer.getWritableContainerAtIndex(i)
oldcard := C.getCardinality()
C = C.iremoveReturnMinimized(lowbits(x))
rb.highlowcontainer.setContainerAtIndex(i, C)
if rb.highlowcontainer.getContainerAtIndex(i).getCardinality() == 0 {
rb.highlowcontainer.removeAtIndex(i)
return true
}
return C.getCardinality() < oldcard
}
return false
}
// IsEmpty returns true if the Bitmap is empty (it is faster than doing (GetCardinality() == 0))
func (rb *Bitmap) IsEmpty() bool {
return rb.highlowcontainer.size() == 0
}
// GetCardinality returns the number of integers contained in the bitmap
func (rb *Bitmap) GetCardinality() uint64 {
size := uint64(0)
for _, c := range rb.highlowcontainer.containers {
size += uint64(c.getCardinality())
}
return size
}
// Rank returns the number of integers that are smaller or equal to x (Rank(infinity) would be GetCardinality())
func (rb *Bitmap) Rank(x uint32) uint64 {
size := uint64(0)
for i := 0; i < rb.highlowcontainer.size(); i++ {
key := rb.highlowcontainer.getKeyAtIndex(i)
if key > highbits(x) {
return size
}
if key < highbits(x) {
size += uint64(rb.highlowcontainer.getContainerAtIndex(i).getCardinality())
} else {
return size + uint64(rb.highlowcontainer.getContainerAtIndex(i).rank(lowbits(x)))
}
}
return size
}
// Select returns the xth integer in the bitmap
func (rb *Bitmap) Select(x uint32) (uint32, error) {
if rb.GetCardinality() <= uint64(x) {
return 0, fmt.Errorf("can't find %dth integer in a bitmap with only %d items", x, rb.GetCardinality())
}
remaining := x
for i := 0; i < rb.highlowcontainer.size(); i++ {
c := rb.highlowcontainer.getContainerAtIndex(i)
if remaining >= uint32(c.getCardinality()) {
remaining -= uint32(c.getCardinality())
} else {
key := rb.highlowcontainer.getKeyAtIndex(i)
return uint32(key)<<16 + uint32(c.selectInt(uint16(remaining))), nil
}
}
return 0, fmt.Errorf("can't find %dth integer in a bitmap with only %d items", x, rb.GetCardinality())
}
// And computes the intersection between two bitmaps and stores the result in the current bitmap
func (rb *Bitmap) And(x2 *Bitmap) {
pos1 := 0
pos2 := 0
intersectionsize := 0
length1 := rb.highlowcontainer.size()
length2 := x2.highlowcontainer.size()
main:
for {
if pos1 < length1 && pos2 < length2 {
s1 := rb.highlowcontainer.getKeyAtIndex(pos1)
s2 := x2.highlowcontainer.getKeyAtIndex(pos2)
for {
if s1 == s2 {
c1 := rb.highlowcontainer.getWritableContainerAtIndex(pos1)
c2 := x2.highlowcontainer.getContainerAtIndex(pos2)
diff := c1.iand(c2)
if diff.getCardinality() > 0 {
rb.highlowcontainer.replaceKeyAndContainerAtIndex(intersectionsize, s1, diff, false)
intersectionsize++
}
pos1++
pos2++
if (pos1 == length1) || (pos2 == length2) {
break main
}
s1 = rb.highlowcontainer.getKeyAtIndex(pos1)
s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
} else if s1 < s2 {
pos1 = rb.highlowcontainer.advanceUntil(s2, pos1)
if pos1 == length1 {
break main
}
s1 = rb.highlowcontainer.getKeyAtIndex(pos1)
} else { //s1 > s2
pos2 = x2.highlowcontainer.advanceUntil(s1, pos2)
if pos2 == length2 {
break main
}
s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
}
}
} else {
break
}
}
rb.highlowcontainer.resize(intersectionsize)
}
// OrCardinality returns the cardinality of the union between two bitmaps, bitmaps are not modified
func (rb *Bitmap) OrCardinality(x2 *Bitmap) uint64 {
pos1 := 0
pos2 := 0
length1 := rb.highlowcontainer.size()
length2 := x2.highlowcontainer.size()
answer := uint64(0)
main:
for {
if (pos1 < length1) && (pos2 < length2) {
s1 := rb.highlowcontainer.getKeyAtIndex(pos1)
s2 := x2.highlowcontainer.getKeyAtIndex(pos2)
for {
if s1 < s2 {
answer += uint64(rb.highlowcontainer.getContainerAtIndex(pos1).getCardinality())
pos1++
if pos1 == length1 {
break main
}
s1 = rb.highlowcontainer.getKeyAtIndex(pos1)
} else if s1 > s2 {
answer += uint64(x2.highlowcontainer.getContainerAtIndex(pos2).getCardinality())
pos2++
if pos2 == length2 {
break main
}
s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
} else {
// TODO: could be faster if we did not have to materialize the container
answer += uint64(rb.highlowcontainer.getContainerAtIndex(pos1).or(x2.highlowcontainer.getContainerAtIndex(pos2)).getCardinality())
pos1++
pos2++
if (pos1 == length1) || (pos2 == length2) {
break main
}
s1 = rb.highlowcontainer.getKeyAtIndex(pos1)
s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
}
}
} else {
break
}
}
for ; pos1 < length1; pos1++ {
answer += uint64(rb.highlowcontainer.getContainerAtIndex(pos1).getCardinality())
}
for ; pos2 < length2; pos2++ {
answer += uint64(x2.highlowcontainer.getContainerAtIndex(pos2).getCardinality())
}
return answer
}
// AndCardinality returns the cardinality of the intersection between two bitmaps, bitmaps are not modified
func (rb *Bitmap) AndCardinality(x2 *Bitmap) uint64 {
pos1 := 0
pos2 := 0
answer := uint64(0)
length1 := rb.highlowcontainer.size()
length2 := x2.highlowcontainer.size()
main:
for {
if pos1 < length1 && pos2 < length2 {
s1 := rb.highlowcontainer.getKeyAtIndex(pos1)
s2 := x2.highlowcontainer.getKeyAtIndex(pos2)
for {
if s1 == s2 {
c1 := rb.highlowcontainer.getContainerAtIndex(pos1)
c2 := x2.highlowcontainer.getContainerAtIndex(pos2)
answer += uint64(c1.andCardinality(c2))
pos1++
pos2++
if (pos1 == length1) || (pos2 == length2) {
break main
}
s1 = rb.highlowcontainer.getKeyAtIndex(pos1)
s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
} else if s1 < s2 {
pos1 = rb.highlowcontainer.advanceUntil(s2, pos1)
if pos1 == length1 {
break main
}
s1 = rb.highlowcontainer.getKeyAtIndex(pos1)
} else { //s1 > s2
pos2 = x2.highlowcontainer.advanceUntil(s1, pos2)
if pos2 == length2 {
break main
}
s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
}
}
} else {
break
}
}
return answer
}
// Intersects checks whether two bitmap intersects, bitmaps are not modified
func (rb *Bitmap) Intersects(x2 *Bitmap) bool {
pos1 := 0
pos2 := 0
length1 := rb.highlowcontainer.size()
length2 := x2.highlowcontainer.size()
main:
for {
if pos1 < length1 && pos2 < length2 {
s1 := rb.highlowcontainer.getKeyAtIndex(pos1)
s2 := x2.highlowcontainer.getKeyAtIndex(pos2)
for {
if s1 == s2 {
c1 := rb.highlowcontainer.getContainerAtIndex(pos1)
c2 := x2.highlowcontainer.getContainerAtIndex(pos2)
if c1.intersects(c2) {
return true
}
pos1++
pos2++
if (pos1 == length1) || (pos2 == length2) {
break main
}
s1 = rb.highlowcontainer.getKeyAtIndex(pos1)
s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
} else if s1 < s2 {
pos1 = rb.highlowcontainer.advanceUntil(s2, pos1)
if pos1 == length1 {
break main
}
s1 = rb.highlowcontainer.getKeyAtIndex(pos1)
} else { //s1 > s2
pos2 = x2.highlowcontainer.advanceUntil(s1, pos2)
if pos2 == length2 {
break main
}
s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
}
}
} else {
break
}
}
return false
}
// Xor computes the symmetric difference between two bitmaps and stores the result in the current bitmap
func (rb *Bitmap) Xor(x2 *Bitmap) {
pos1 := 0
pos2 := 0
length1 := rb.highlowcontainer.size()
length2 := x2.highlowcontainer.size()
for {
if (pos1 < length1) && (pos2 < length2) {
s1 := rb.highlowcontainer.getKeyAtIndex(pos1)
s2 := x2.highlowcontainer.getKeyAtIndex(pos2)
if s1 < s2 {
pos1 = rb.highlowcontainer.advanceUntil(s2, pos1)
if pos1 == length1 {
break
}
} else if s1 > s2 {
c := x2.highlowcontainer.getWritableContainerAtIndex(pos2)
rb.highlowcontainer.insertNewKeyValueAt(pos1, x2.highlowcontainer.getKeyAtIndex(pos2), c)
length1++
pos1++
pos2++
} else {
// TODO: couple be computed in-place for reduced memory usage
c := rb.highlowcontainer.getContainerAtIndex(pos1).xor(x2.highlowcontainer.getContainerAtIndex(pos2))
if c.getCardinality() > 0 {
rb.highlowcontainer.setContainerAtIndex(pos1, c)
pos1++
} else {
rb.highlowcontainer.removeAtIndex(pos1)
length1--
}
pos2++
}
} else {
break
}
}
if pos1 == length1 {
rb.highlowcontainer.appendCopyMany(x2.highlowcontainer, pos2, length2)
}
}
// Or computes the union between two bitmaps and stores the result in the current bitmap
func (rb *Bitmap) Or(x2 *Bitmap) {
pos1 := 0
pos2 := 0
length1 := rb.highlowcontainer.size()
length2 := x2.highlowcontainer.size()
main:
for (pos1 < length1) && (pos2 < length2) {
s1 := rb.highlowcontainer.getKeyAtIndex(pos1)
s2 := x2.highlowcontainer.getKeyAtIndex(pos2)
for {
if s1 < s2 {
pos1++
if pos1 == length1 {
break main
}
s1 = rb.highlowcontainer.getKeyAtIndex(pos1)
} else if s1 > s2 {
rb.highlowcontainer.insertNewKeyValueAt(pos1, s2, x2.highlowcontainer.getContainerAtIndex(pos2).clone())
pos1++
length1++
pos2++
if pos2 == length2 {
break main
}
s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
} else {
rb.highlowcontainer.replaceKeyAndContainerAtIndex(pos1, s1, rb.highlowcontainer.getWritableContainerAtIndex(pos1).ior(x2.highlowcontainer.getContainerAtIndex(pos2)), false)
pos1++
pos2++
if (pos1 == length1) || (pos2 == length2) {
break main
}
s1 = rb.highlowcontainer.getKeyAtIndex(pos1)
s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
}
}
}
if pos1 == length1 {
rb.highlowcontainer.appendCopyMany(x2.highlowcontainer, pos2, length2)
}
}
/*func (rb *Bitmap) Or(x2 *Bitmap) {
results := Or(rb, x2) // Todo: could be computed in-place for reduced memory usage
rb.highlowcontainer = results.highlowcontainer
}*/
// AndNot computes the difference between two bitmaps and stores the result in the current bitmap
func (rb *Bitmap) AndNot(x2 *Bitmap) {
pos1 := 0
pos2 := 0
intersectionsize := 0
length1 := rb.highlowcontainer.size()
length2 := x2.highlowcontainer.size()
main:
for {
if pos1 < length1 && pos2 < length2 {
s1 := rb.highlowcontainer.getKeyAtIndex(pos1)
s2 := x2.highlowcontainer.getKeyAtIndex(pos2)
for {
if s1 == s2 {
c1 := rb.highlowcontainer.getWritableContainerAtIndex(pos1)
c2 := x2.highlowcontainer.getContainerAtIndex(pos2)
diff := c1.iandNot(c2)
if diff.getCardinality() > 0 {
rb.highlowcontainer.replaceKeyAndContainerAtIndex(intersectionsize, s1, diff, false)
intersectionsize++
}
pos1++
pos2++
if (pos1 == length1) || (pos2 == length2) {
break main
}
s1 = rb.highlowcontainer.getKeyAtIndex(pos1)
s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
} else if s1 < s2 {
c1 := rb.highlowcontainer.getContainerAtIndex(pos1)
mustCopyOnWrite := rb.highlowcontainer.needsCopyOnWrite(pos1)
rb.highlowcontainer.replaceKeyAndContainerAtIndex(intersectionsize, s1, c1, mustCopyOnWrite)
intersectionsize++
pos1++
if pos1 == length1 {
break main
}
s1 = rb.highlowcontainer.getKeyAtIndex(pos1)
} else { //s1 > s2
pos2 = x2.highlowcontainer.advanceUntil(s1, pos2)
if pos2 == length2 {
break main
}
s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
}
}
} else {
break
}
}
// TODO:implement as a copy
for pos1 < length1 {
c1 := rb.highlowcontainer.getContainerAtIndex(pos1)
s1 := rb.highlowcontainer.getKeyAtIndex(pos1)
mustCopyOnWrite := rb.highlowcontainer.needsCopyOnWrite(pos1)
rb.highlowcontainer.replaceKeyAndContainerAtIndex(intersectionsize, s1, c1, mustCopyOnWrite)
intersectionsize++
pos1++
}
rb.highlowcontainer.resize(intersectionsize)
}
// Or computes the union between two bitmaps and returns the result
func Or(x1, x2 *Bitmap) *Bitmap {
answer := NewBitmap()
pos1 := 0
pos2 := 0
length1 := x1.highlowcontainer.size()
length2 := x2.highlowcontainer.size()
main:
for (pos1 < length1) && (pos2 < length2) {
s1 := x1.highlowcontainer.getKeyAtIndex(pos1)
s2 := x2.highlowcontainer.getKeyAtIndex(pos2)
for {
if s1 < s2 {
answer.highlowcontainer.appendCopy(x1.highlowcontainer, pos1)
pos1++
if pos1 == length1 {
break main
}
s1 = x1.highlowcontainer.getKeyAtIndex(pos1)
} else if s1 > s2 {
answer.highlowcontainer.appendCopy(x2.highlowcontainer, pos2)
pos2++
if pos2 == length2 {
break main
}
s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
} else {
answer.highlowcontainer.appendContainer(s1, x1.highlowcontainer.getContainerAtIndex(pos1).or(x2.highlowcontainer.getContainerAtIndex(pos2)), false)
pos1++
pos2++
if (pos1 == length1) || (pos2 == length2) {
break main
}
s1 = x1.highlowcontainer.getKeyAtIndex(pos1)
s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
}
}
}
if pos1 == length1 {
answer.highlowcontainer.appendCopyMany(x2.highlowcontainer, pos2, length2)
} else if pos2 == length2 {
answer.highlowcontainer.appendCopyMany(x1.highlowcontainer, pos1, length1)
}
return answer
}
// And computes the intersection between two bitmaps and returns the result
func And(x1, x2 *Bitmap) *Bitmap {
answer := NewBitmap()
pos1 := 0
pos2 := 0
length1 := x1.highlowcontainer.size()
length2 := x2.highlowcontainer.size()
main:
for pos1 < length1 && pos2 < length2 {
s1 := x1.highlowcontainer.getKeyAtIndex(pos1)
s2 := x2.highlowcontainer.getKeyAtIndex(pos2)
for {
if s1 == s2 {
C := x1.highlowcontainer.getContainerAtIndex(pos1)
C = C.and(x2.highlowcontainer.getContainerAtIndex(pos2))
if C.getCardinality() > 0 {
answer.highlowcontainer.appendContainer(s1, C, false)
}
pos1++
pos2++
if (pos1 == length1) || (pos2 == length2) {
break main
}
s1 = x1.highlowcontainer.getKeyAtIndex(pos1)
s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
} else if s1 < s2 {
pos1 = x1.highlowcontainer.advanceUntil(s2, pos1)
if pos1 == length1 {
break main
}
s1 = x1.highlowcontainer.getKeyAtIndex(pos1)
} else { // s1 > s2
pos2 = x2.highlowcontainer.advanceUntil(s1, pos2)
if pos2 == length2 {
break main
}
s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
}
}
}
return answer
}
// Xor computes the symmetric difference between two bitmaps and returns the result
func Xor(x1, x2 *Bitmap) *Bitmap {
answer := NewBitmap()
pos1 := 0
pos2 := 0
length1 := x1.highlowcontainer.size()
length2 := x2.highlowcontainer.size()
for {
if (pos1 < length1) && (pos2 < length2) {
s1 := x1.highlowcontainer.getKeyAtIndex(pos1)
s2 := x2.highlowcontainer.getKeyAtIndex(pos2)
if s1 < s2 {
answer.highlowcontainer.appendCopy(x1.highlowcontainer, pos1)
pos1++
} else if s1 > s2 {
answer.highlowcontainer.appendCopy(x2.highlowcontainer, pos2)
pos2++
} else {
c := x1.highlowcontainer.getContainerAtIndex(pos1).xor(x2.highlowcontainer.getContainerAtIndex(pos2))
if c.getCardinality() > 0 {
answer.highlowcontainer.appendContainer(s1, c, false)
}
pos1++
pos2++
}
} else {
break
}
}
if pos1 == length1 {
answer.highlowcontainer.appendCopyMany(x2.highlowcontainer, pos2, length2)
} else if pos2 == length2 {
answer.highlowcontainer.appendCopyMany(x1.highlowcontainer, pos1, length1)
}
return answer
}
// AndNot computes the difference between two bitmaps and returns the result
func AndNot(x1, x2 *Bitmap) *Bitmap {
answer := NewBitmap()
pos1 := 0
pos2 := 0
length1 := x1.highlowcontainer.size()
length2 := x2.highlowcontainer.size()
main:
for {
if pos1 < length1 && pos2 < length2 {
s1 := x1.highlowcontainer.getKeyAtIndex(pos1)
s2 := x2.highlowcontainer.getKeyAtIndex(pos2)
for {
if s1 < s2 {
answer.highlowcontainer.appendCopy(x1.highlowcontainer, pos1)
pos1++
if pos1 == length1 {
break main
}
s1 = x1.highlowcontainer.getKeyAtIndex(pos1)
} else if s1 == s2 {
c1 := x1.highlowcontainer.getContainerAtIndex(pos1)
c2 := x2.highlowcontainer.getContainerAtIndex(pos2)
diff := c1.andNot(c2)
if diff.getCardinality() > 0 {
answer.highlowcontainer.appendContainer(s1, diff, false)
}
pos1++
pos2++
if (pos1 == length1) || (pos2 == length2) {
break main
}
s1 = x1.highlowcontainer.getKeyAtIndex(pos1)
s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
} else { //s1 > s2
pos2 = x2.highlowcontainer.advanceUntil(s1, pos2)
if pos2 == length2 {
break main
}
s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
}
}
} else {
break
}
}
if pos2 == length2 {
answer.highlowcontainer.appendCopyMany(x1.highlowcontainer, pos1, length1)
}
return answer
}
// AddMany add all of the values in dat
func (rb *Bitmap) AddMany(dat []uint32) {
if len(dat) == 0 {
return
}
prev := dat[0]
idx, c := rb.addwithptr(prev)
for _, i := range dat[1:] {
if highbits(prev) == highbits(i) {
c = c.iaddReturnMinimized(lowbits(i))
rb.highlowcontainer.setContainerAtIndex(idx, c)
} else {
idx, c = rb.addwithptr(i)
}
prev = i
}
}
// BitmapOf generates a new bitmap filled with the specified integers
func BitmapOf(dat ...uint32) *Bitmap {
ans := NewBitmap()
ans.AddMany(dat)
return ans
}
// Flip negates the bits in the given range (i.e., [rangeStart,rangeEnd)), any integer present in this range and in the bitmap is removed,
// and any integer present in the range and not in the bitmap is added.
// The function uses 64-bit parameters even though a Bitmap stores 32-bit values because it is allowed and meaningful to use [0,uint64(0x100000000)) as a range
// while uint64(0x100000000) cannot be represented as a 32-bit value.
func (rb *Bitmap) Flip(rangeStart, rangeEnd uint64) {
if rangeEnd > MaxUint32+1 {
panic("rangeEnd > MaxUint32+1")
}
if rangeStart > MaxUint32+1 {
panic("rangeStart > MaxUint32+1")
}
if rangeStart >= rangeEnd {
return
}
hbStart := highbits(uint32(rangeStart))
lbStart := lowbits(uint32(rangeStart))
hbLast := highbits(uint32(rangeEnd - 1))
lbLast := lowbits(uint32(rangeEnd - 1))
var max uint32 = maxLowBit
for hb := hbStart; hb <= hbLast; hb++ {
var containerStart uint32
if hb == hbStart {
containerStart = uint32(lbStart)
}
containerLast := max
if hb == hbLast {
containerLast = uint32(lbLast)
}
i := rb.highlowcontainer.getIndex(hb)
if i >= 0 {
c := rb.highlowcontainer.getWritableContainerAtIndex(i).inot(int(containerStart), int(containerLast)+1)
if c.getCardinality() > 0 {
rb.highlowcontainer.setContainerAtIndex(i, c)
} else {
rb.highlowcontainer.removeAtIndex(i)
}
} else { // *think* the range of ones must never be
// empty.
rb.highlowcontainer.insertNewKeyValueAt(-i-1, hb, rangeOfOnes(int(containerStart), int(containerLast)))
}
}
}
// FlipInt calls Flip after casting the parameters (convenience method)
func (rb *Bitmap) FlipInt(rangeStart, rangeEnd int) {
rb.Flip(uint64(rangeStart), uint64(rangeEnd))
}
// AddRange adds the integers in [rangeStart, rangeEnd) to the bitmap.
// The function uses 64-bit parameters even though a Bitmap stores 32-bit values because it is allowed and meaningful to use [0,uint64(0x100000000)) as a range
// while uint64(0x100000000) cannot be represented as a 32-bit value.
func (rb *Bitmap) AddRange(rangeStart, rangeEnd uint64) {
if rangeStart >= rangeEnd {
return
}
if rangeEnd-1 > MaxUint32 {
panic("rangeEnd-1 > MaxUint32")
}
hbStart := uint32(highbits(uint32(rangeStart)))
lbStart := uint32(lowbits(uint32(rangeStart)))
hbLast := uint32(highbits(uint32(rangeEnd - 1)))
lbLast := uint32(lowbits(uint32(rangeEnd - 1)))
var max uint32 = maxLowBit
for hb := uint16(hbStart); hb <= uint16(hbLast); hb++ {
containerStart := uint32(0)
if hb == uint16(hbStart) {
containerStart = lbStart
}
containerLast := max
if hb == uint16(hbLast) {
containerLast = lbLast
}
i := rb.highlowcontainer.getIndex(hb)
if i >= 0 {
c := rb.highlowcontainer.getWritableContainerAtIndex(i).iaddRange(int(containerStart), int(containerLast)+1)
rb.highlowcontainer.setContainerAtIndex(i, c)
} else { // *think* the range of ones must never be
// empty.
rb.highlowcontainer.insertNewKeyValueAt(-i-1, hb, rangeOfOnes(int(containerStart), int(containerLast)))
}
}
}
// RemoveRange removes the integers in [rangeStart, rangeEnd) from the bitmap.
// The function uses 64-bit parameters even though a Bitmap stores 32-bit values because it is allowed and meaningful to use [0,uint64(0x100000000)) as a range
// while uint64(0x100000000) cannot be represented as a 32-bit value.
func (rb *Bitmap) RemoveRange(rangeStart, rangeEnd uint64) {
if rangeStart >= rangeEnd {
return
}
if rangeEnd-1 > MaxUint32 {
// logically, we should assume that the user wants to
// remove all values from rangeStart to infinity
// see https://github.com/RoaringBitmap/roaring/issues/141
rangeEnd = uint64(0x100000000)
}
hbStart := uint32(highbits(uint32(rangeStart)))
lbStart := uint32(lowbits(uint32(rangeStart)))
hbLast := uint32(highbits(uint32(rangeEnd - 1)))
lbLast := uint32(lowbits(uint32(rangeEnd - 1)))
var max uint32 = maxLowBit
if hbStart == hbLast {
i := rb.highlowcontainer.getIndex(uint16(hbStart))
if i < 0 {
return
}
c := rb.highlowcontainer.getWritableContainerAtIndex(i).iremoveRange(int(lbStart), int(lbLast+1))
if c.getCardinality() > 0 {
rb.highlowcontainer.setContainerAtIndex(i, c)
} else {
rb.highlowcontainer.removeAtIndex(i)
}
return
}
ifirst := rb.highlowcontainer.getIndex(uint16(hbStart))
ilast := rb.highlowcontainer.getIndex(uint16(hbLast))
if ifirst >= 0 {
if lbStart != 0 {
c := rb.highlowcontainer.getWritableContainerAtIndex(ifirst).iremoveRange(int(lbStart), int(max+1))
if c.getCardinality() > 0 {
rb.highlowcontainer.setContainerAtIndex(ifirst, c)
ifirst++
}
}
} else {
ifirst = -ifirst - 1
}
if ilast >= 0 {
if lbLast != max {
c := rb.highlowcontainer.getWritableContainerAtIndex(ilast).iremoveRange(int(0), int(lbLast+1))
if c.getCardinality() > 0 {
rb.highlowcontainer.setContainerAtIndex(ilast, c)
} else {
ilast++
}
} else {
ilast++
}
} else {
ilast = -ilast - 1
}
rb.highlowcontainer.removeIndexRange(ifirst, ilast)
}
// Flip negates the bits in the given range (i.e., [rangeStart,rangeEnd)), any integer present in this range and in the bitmap is removed,
// and any integer present in the range and not in the bitmap is added, a new bitmap is returned leaving
// the current bitmap unchanged.
// The function uses 64-bit parameters even though a Bitmap stores 32-bit values because it is allowed and meaningful to use [0,uint64(0x100000000)) as a range
// while uint64(0x100000000) cannot be represented as a 32-bit value.
func Flip(bm *Bitmap, rangeStart, rangeEnd uint64) *Bitmap {
if rangeStart >= rangeEnd {
return bm.Clone()
}
if rangeStart > MaxUint32 {
panic("rangeStart > MaxUint32")
}
if rangeEnd-1 > MaxUint32 {
panic("rangeEnd-1 > MaxUint32")
}
answer := NewBitmap()
hbStart := highbits(uint32(rangeStart))
lbStart := lowbits(uint32(rangeStart))
hbLast := highbits(uint32(rangeEnd - 1))
lbLast := lowbits(uint32(rangeEnd - 1))
// copy the containers before the active area
answer.highlowcontainer.appendCopiesUntil(bm.highlowcontainer, hbStart)
var max uint32 = maxLowBit
for hb := hbStart; hb <= hbLast; hb++ {
var containerStart uint32
if hb == hbStart {
containerStart = uint32(lbStart)
}
containerLast := max
if hb == hbLast {
containerLast = uint32(lbLast)
}
i := bm.highlowcontainer.getIndex(hb)
j := answer.highlowcontainer.getIndex(hb)
if i >= 0 {
c := bm.highlowcontainer.getContainerAtIndex(i).not(int(containerStart), int(containerLast)+1)
if c.getCardinality() > 0 {
answer.highlowcontainer.insertNewKeyValueAt(-j-1, hb, c)
}
} else { // *think* the range of ones must never be
// empty.
answer.highlowcontainer.insertNewKeyValueAt(-j-1, hb,
rangeOfOnes(int(containerStart), int(containerLast)))
}
}
// copy the containers after the active area.
answer.highlowcontainer.appendCopiesAfter(bm.highlowcontainer, hbLast)
return answer
}
// SetCopyOnWrite sets this bitmap to use copy-on-write so that copies are fast and memory conscious
// if the parameter is true, otherwise we leave the default where hard copies are made
// (copy-on-write requires extra care in a threaded context).
// Calling SetCopyOnWrite(true) on a bitmap created with FromBuffer is unsafe.
func (rb *Bitmap) SetCopyOnWrite(val bool) {
rb.highlowcontainer.copyOnWrite = val
}
// GetCopyOnWrite gets this bitmap's copy-on-write property
func (rb *Bitmap) GetCopyOnWrite() (val bool) {
return rb.highlowcontainer.copyOnWrite
}
// FlipInt calls Flip after casting the parameters (convenience method)
func FlipInt(bm *Bitmap, rangeStart, rangeEnd int) *Bitmap {
return Flip(bm, uint64(rangeStart), uint64(rangeEnd))
}
// Statistics provides details on the container types in use.
type Statistics struct {
Cardinality uint64
Containers uint64
ArrayContainers uint64
ArrayContainerBytes uint64
ArrayContainerValues uint64
BitmapContainers uint64
BitmapContainerBytes uint64
BitmapContainerValues uint64
RunContainers uint64
RunContainerBytes uint64
RunContainerValues uint64
}
// Stats returns details on container type usage in a Statistics struct.
func (rb *Bitmap) Stats() Statistics {
stats := Statistics{}
stats.Containers = uint64(len(rb.highlowcontainer.containers))
for _, c := range rb.highlowcontainer.containers {
stats.Cardinality += uint64(c.getCardinality())
switch c.(type) {
case *arrayContainer:
stats.ArrayContainers++
stats.ArrayContainerBytes += uint64(c.getSizeInBytes())
stats.ArrayContainerValues += uint64(c.getCardinality())
case *bitmapContainer:
stats.BitmapContainers++
stats.BitmapContainerBytes += uint64(c.getSizeInBytes())
stats.BitmapContainerValues += uint64(c.getCardinality())
case *runContainer16:
stats.RunContainers++
stats.RunContainerBytes += uint64(c.getSizeInBytes())
stats.RunContainerValues += uint64(c.getCardinality())
}
}
return stats
}