Skip to content

Commit

Permalink
Merging upstream xorfilter.h changes, bumping version
Browse files Browse the repository at this point in the history
  • Loading branch information
@mpope9
mpope9 committed Nov 20, 2021
1 parent 83ecaa4 commit e227c07
Show file tree
Hide file tree
Showing 2 changed files with 38 additions and 43 deletions.
79 changes: 37 additions & 42 deletions c_src/xorfilter.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@

#ifndef XOR_MAX_ITERATIONS
#define XOR_MAX_ITERATIONS 100 // probabillity of success should always be > 0.5 so 100 iterations is highly unlikely
#endif
#endif

/**
* We assume that you have a large set of 64-bit integers
Expand Down Expand Up @@ -136,10 +136,9 @@ static inline bool xor8_allocate(uint32_t size, xor8_t *filter) {
}

// allocate enough capacity for a set containing up to 'size' elements
// caller is responsible to call xor8_free(filter)
// caller is responsible to call xor16_free(filter)
static inline bool xor16_allocate(uint32_t size, xor16_t *filter) {
size_t capacity = 32 + 1.23 * size;
filter->blockLength = capacity / 3;
capacity = capacity / 3 * 3;
filter->fingerprints = (uint16_t *)malloc(capacity * sizeof(uint16_t));
if (filter->fingerprints != NULL) {
Expand Down Expand Up @@ -212,18 +211,6 @@ struct xor_h0h1h2_s {

typedef struct xor_h0h1h2_s xor_h0h1h2_t;

static inline xor_h0h1h2_t xor8_get_just_h0_h1_h2(uint64_t hash,
const xor8_t *filter) {
xor_h0h1h2_t answer;
uint32_t r0 = (uint32_t)hash;
uint32_t r1 = (uint32_t)xor_rotl64(hash, 21);
uint32_t r2 = (uint32_t)xor_rotl64(hash, 42);

answer.h0 = xor_reduce(r0, filter->blockLength);
answer.h1 = xor_reduce(r1, filter->blockLength);
answer.h2 = xor_reduce(r2, filter->blockLength);
return answer;
}
static inline uint32_t xor8_get_h0(uint64_t hash, const xor8_t *filter) {
uint32_t r0 = (uint32_t)hash;
return xor_reduce(r0, filter->blockLength);
Expand Down Expand Up @@ -345,7 +332,7 @@ static inline void xor_make_buffer_current(xor_setbuffer_t *buffer,
}
*Qsize = qsize;
buffer->counts[slot] = 0;
}
}
}


Expand Down Expand Up @@ -458,6 +445,7 @@ static inline uint32_t xor_flushone_decrement_buffer(xor_setbuffer_t *buffer,
// a return value of false is provided.
//
bool xor8_buffered_populate(const uint64_t *keys, uint32_t size, xor8_t *filter) {
if(size == 0) { return false; }
uint64_t rng_counter = 1;
filter->seed = xor_rng_splitmix64(&rng_counter);
size_t arrayLength = filter->blockLength * 3; // size of the backing array
Expand All @@ -476,14 +464,9 @@ bool xor8_buffered_populate(const uint64_t *keys, uint32_t size, xor8_t *filter)
xor_xorset_t *sets =
(xor_xorset_t *)malloc(arrayLength * sizeof(xor_xorset_t));
xor_xorset_t *sets0 = sets;
xor_xorset_t *sets1 = sets + blockLength;
xor_xorset_t *sets2 = sets + 2 * blockLength;

xor_keyindex_t *Q =
(xor_keyindex_t *)malloc(arrayLength * sizeof(xor_keyindex_t));
xor_keyindex_t *Q0 = Q;
xor_keyindex_t *Q1 = Q + blockLength;
xor_keyindex_t *Q2 = Q + 2 * blockLength;

xor_keyindex_t *stack =
(xor_keyindex_t *)malloc(size * sizeof(xor_keyindex_t));
Expand All @@ -497,6 +480,12 @@ bool xor8_buffered_populate(const uint64_t *keys, uint32_t size, xor8_t *filter)
free(stack);
return false;
}
xor_xorset_t *sets1 = sets + blockLength;
xor_xorset_t *sets2 = sets + 2 * blockLength;
xor_keyindex_t *Q0 = Q;
xor_keyindex_t *Q1 = Q + blockLength;
xor_keyindex_t *Q2 = Q + 2 * blockLength;

int iterations = 0;

while (true) {
Expand Down Expand Up @@ -532,7 +521,7 @@ bool xor8_buffered_populate(const uint64_t *keys, uint32_t size, xor8_t *filter)
Q0[Q0size].index = i;
Q0[Q0size].hash = sets0[i].xormask;
Q0size++;
}
}
}

for (size_t i = 0; i < filter->blockLength; i++) {
Expand Down Expand Up @@ -668,22 +657,17 @@ bool xor8_buffered_populate(const uint64_t *keys, uint32_t size, xor8_t *filter)
// a return value of false is provided.
//
bool xor8_populate(const uint64_t *keys, uint32_t size, xor8_t *filter) {
if(size == 0) { return false; }
uint64_t rng_counter = 1;
filter->seed = xor_rng_splitmix64(&rng_counter);
size_t arrayLength = filter->blockLength * 3; // size of the backing array
size_t blockLength = filter->blockLength;

xor_xorset_t *sets =
(xor_xorset_t *)malloc(arrayLength * sizeof(xor_xorset_t));
xor_xorset_t *sets0 = sets;
xor_xorset_t *sets1 = sets + blockLength;
xor_xorset_t *sets2 = sets + 2 * blockLength;

xor_keyindex_t *Q =
(xor_keyindex_t *)malloc(arrayLength * sizeof(xor_keyindex_t));
xor_keyindex_t *Q0 = Q;
xor_keyindex_t *Q1 = Q + blockLength;
xor_keyindex_t *Q2 = Q + 2 * blockLength;

xor_keyindex_t *stack =
(xor_keyindex_t *)malloc(size * sizeof(xor_keyindex_t));
Expand All @@ -694,6 +678,13 @@ bool xor8_populate(const uint64_t *keys, uint32_t size, xor8_t *filter) {
free(stack);
return false;
}
xor_xorset_t *sets0 = sets;
xor_xorset_t *sets1 = sets + blockLength;
xor_xorset_t *sets2 = sets + 2 * blockLength;
xor_keyindex_t *Q0 = Q;
xor_keyindex_t *Q1 = Q + blockLength;
xor_keyindex_t *Q2 = Q + 2 * blockLength;

int iterations = 0;

while (true) {
Expand Down Expand Up @@ -874,12 +865,13 @@ bool xor8_populate(const uint64_t *keys, uint32_t size, xor8_t *filter) {
// a return value of false is provided.
//
bool xor16_buffered_populate(const uint64_t *keys, uint32_t size, xor16_t *filter) {
if(size == 0) { return false; }
uint64_t rng_counter = 1;
filter->seed = xor_rng_splitmix64(&rng_counter);
size_t arrayLength = filter->blockLength * 3; // size of the backing array
xor_setbuffer_t buffer0, buffer1, buffer2;
size_t blockLength = filter->blockLength;
bool ok0 = xor_init_buffer(&buffer0, blockLength);
bool ok0 = xor_init_buffer(&buffer0, blockLength);
bool ok1 = xor_init_buffer(&buffer1, blockLength);
bool ok2 = xor_init_buffer(&buffer2, blockLength);
if (!ok0 || !ok1 || !ok2) {
Expand All @@ -891,15 +883,9 @@ bool xor16_buffered_populate(const uint64_t *keys, uint32_t size, xor16_t *filte

xor_xorset_t *sets =
(xor_xorset_t *)malloc(arrayLength * sizeof(xor_xorset_t));
xor_xorset_t *sets0 = sets;
xor_xorset_t *sets1 = sets + blockLength;
xor_xorset_t *sets2 = sets + 2 * blockLength;

xor_keyindex_t *Q =
(xor_keyindex_t *)malloc(arrayLength * sizeof(xor_keyindex_t));
xor_keyindex_t *Q0 = Q;
xor_keyindex_t *Q1 = Q + blockLength;
xor_keyindex_t *Q2 = Q + 2 * blockLength;

xor_keyindex_t *stack =
(xor_keyindex_t *)malloc(size * sizeof(xor_keyindex_t));
Expand All @@ -913,6 +899,13 @@ bool xor16_buffered_populate(const uint64_t *keys, uint32_t size, xor16_t *filte
free(stack);
return false;
}
xor_xorset_t *sets0 = sets;
xor_xorset_t *sets1 = sets + blockLength;
xor_xorset_t *sets2 = sets + 2 * blockLength;
xor_keyindex_t *Q0 = Q;
xor_keyindex_t *Q1 = Q + blockLength;
xor_keyindex_t *Q2 = Q + 2 * blockLength;

int iterations = 0;

while (true) {
Expand Down Expand Up @@ -1087,22 +1080,17 @@ bool xor16_buffered_populate(const uint64_t *keys, uint32_t size, xor16_t *filte
// a return value of false is provided.
//
bool xor16_populate(const uint64_t *keys, uint32_t size, xor16_t *filter) {
if(size == 0) { return false; }
uint64_t rng_counter = 1;
filter->seed = xor_rng_splitmix64(&rng_counter);
size_t arrayLength = filter->blockLength * 3; // size of the backing array
size_t blockLength = filter->blockLength;

xor_xorset_t *sets =
(xor_xorset_t *)malloc(arrayLength * sizeof(xor_xorset_t));
xor_xorset_t *sets0 = sets;
xor_xorset_t *sets1 = sets + blockLength;
xor_xorset_t *sets2 = sets + 2 * blockLength;

xor_keyindex_t *Q =
(xor_keyindex_t *)malloc(arrayLength * sizeof(xor_keyindex_t));
xor_keyindex_t *Q0 = Q;
xor_keyindex_t *Q1 = Q + blockLength;
xor_keyindex_t *Q2 = Q + 2 * blockLength;

xor_keyindex_t *stack =
(xor_keyindex_t *)malloc(size * sizeof(xor_keyindex_t));
Expand All @@ -1113,6 +1101,14 @@ bool xor16_populate(const uint64_t *keys, uint32_t size, xor16_t *filter) {
free(stack);
return false;
}
xor_xorset_t *sets0 = sets;
xor_xorset_t *sets1 = sets + blockLength;
xor_xorset_t *sets2 = sets + 2 * blockLength;

xor_keyindex_t *Q0 = Q;
xor_keyindex_t *Q1 = Q + blockLength;
xor_keyindex_t *Q2 = Q + 2 * blockLength;

int iterations = 0;

while (true) {
Expand Down Expand Up @@ -1284,4 +1280,3 @@ bool xor16_populate(const uint64_t *keys, uint32_t size, xor16_t *filter) {


#endif

2 changes: 1 addition & 1 deletion src/exor_filter.app.src
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{application, exor_filter,
[{description, "Nif wrapper for xor_filters."},
{vsn, "0.8.0"},
{vsn, "0.8.1"},
{registered, []},
{applications,
[kernel,
Expand Down

0 comments on commit e227c07

Please sign in to comment.