diff --git a/README.md b/README.md index 91af6ee..aaa84fd 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,40 @@ # hashset This PostgreSQL extension implements hashset, a data structure (type) -providing a collection of unique, not null integer items with fast lookup. +providing a collection of unique integer items with fast lookup. +It provides several functions for working with these sets, including operations +like addition, containment check, conversion to array, union, intersection, +difference, equality check, and cardinality calculation. + +`NULL` values are also allowed in the hash set, and are considered as a unique +element. When multiple `NULL` values are present in the input, they are treated +as a single `NULL`. + +## Table of Contents +1. [Version](#version) +2. [Data Types](#data-types) + - [int4hashset](#int4hashset) +3. [Functions](#functions) + - [int4hashset](#int4hashset-1) + - [hashset_add](#hashset_add) + - [hashset_contains](#hashset_contains) + - [hashset_to_array](#hashset_to_array) + - [hashset_to_sorted_array](#hashset_to_sorted_array) + - [hashset_cardinality](#hashset_cardinality) + - [hashset_capacity](#hashset_capacity) + - [hashset_max_collisions](#hashset_max_collisions) + - [hashset_union](#hashset_union) + - [hashset_intersection](#hashset_intersection) + - [hashset_difference](#hashset_difference) + - [hashset_symmetric_difference](#hashset_symmetric_difference) +4. [Aggregation Functions](#aggregation-functions) +5. [Operators](#operators) +6. [Hashset Hash Operators](#hashset-hash-operators) +7. [Hashset Btree Operators](#hashset-btree-operators) +8. [Limitations](#limitations) +9. [Installation](#installation) +10. [License](#license) ## Version @@ -14,80 +46,235 @@ with possible breaking changes, we are not providing any migration scripts until we reach our first release. -## Usage - -After installing the extension, you can use the `int4hashset` data type and -associated functions within your PostgreSQL queries. - -To demonstrate the usage, let's consider a hypothetical table `users` which has -a `user_id` and a `user_likes` of type `int4hashset`. - -Firstly, let's create the table: - -```sql -CREATE TABLE users( - user_id int PRIMARY KEY, - user_likes int4hashset DEFAULT int4hashset() -); -``` -In the above statement, the `int4hashset()` initializes an empty hashset -with zero capacity. The hashset will automatically resize itself when more -elements are added. - -Now, we can perform operations on this table. Here are some examples: - -```sql --- Insert a new user with id 1. The user_likes will automatically be initialized --- as an empty hashset -INSERT INTO users (user_id) VALUES (1); - --- Add elements (likes) for a user -UPDATE users SET user_likes = hashset_add(user_likes, 101) WHERE user_id = 1; -UPDATE users SET user_likes = hashset_add(user_likes, 202) WHERE user_id = 1; - --- Check if a user likes a particular item -SELECT hashset_contains(user_likes, 101) FROM users WHERE user_id = 1; -- true - --- Count the number of likes a user has -SELECT hashset_cardinality(user_likes) FROM users WHERE user_id = 1; -- 2 -``` - -You can also use the aggregate functions to perform operations on multiple rows. - - ## Data types -- **int4hashset**: This data type represents a set of integers. Internally, it uses -a combination of a bitmap and a value array to store the elements in a set. It's -a variable-length type. +### int4hashset + +This data type represents a set of integers. Internally, it uses a combination +of a bitmap and a value array to store the elements in a set. It's a +variable-length type. ## Functions -- `int4hashset([capacity int, load_factor float4, growth_factor float4, hashfn_id int4]) -> int4hashset`: - Initialize an empty int4hashset with optional parameters. - - `capacity` specifies the initial capacity, which is zero by default. - - `load_factor` represents the threshold for resizing the hashset and defaults to 0.75. - - `growth_factor` is the multiplier for resizing and defaults to 2.0. - - `hashfn_id` represents the hash function used. - - 1=Jenkins/lookup3 (default) - - 2=MurmurHash32 - - 3=Naive hash function -- `hashset_add(int4hashset, int) -> int4hashset`: Adds an integer to an int4hashset. -- `hashset_contains(int4hashset, int) -> boolean`: Checks if an int4hashset contains a given integer. -- `hashset_union(int4hashset, int4hashset) -> int4hashset`: Merges two int4hashsets into a new int4hashset. -- `hashset_to_array(int4hashset) -> int[]`: Converts an int4hashset to an array of integers. -- `hashset_cardinality(int4hashset) -> bigint`: Returns the number of elements in an int4hashset. -- `hashset_capacity(int4hashset) -> bigint`: Returns the current capacity of an int4hashset. -- `hashset_max_collisions(int4hashset) -> bigint`: Returns the maximum number of collisions that have occurred for a single element -- `hashset_intersection(int4hashset, int4hashset) -> int4hashset`: Returns a new int4hashset that is the intersection of the two input sets. -- `hashset_difference(int4hashset, int4hashset) -> int4hashset`: Returns a new int4hashset that contains the elements present in the first set but not in the second set. -- `hashset_symmetric_difference(int4hashset, int4hashset) -> int4hashset`: Returns a new int4hashset containing elements that are in either of the input sets, but not in their intersection. +### int4hashset() + +`int4hashset([capacity int, load_factor float4, growth_factor float4, hashfn_id int4]) -> int4hashset` + +Initialize an empty int4hashset with optional parameters. + - `capacity` specifies the initial capacity, which is zero by default. + - `load_factor` represents the threshold for resizing the hashset and defaults to 0.75. + - `growth_factor` is the multiplier for resizing and defaults to 2.0. + - `hashfn_id` represents the hash function used. + - 1=Jenkins/lookup3 (default) + - 2=MurmurHash32 + - 3=Naive hash function + + +### hashset_add() + +`hashset_add(int4hashset, int) -> int4hashset` + +Adds an integer to an int4hashset. + +```sql +SELECT hashset_add(NULL, 1); -- {1} +SELECT hashset_add('{NULL}', 1); -- {1,NULL} +SELECT hashset_add('{1}', NULL); -- {1,NULL} +SELECT hashset_add('{1}', 1); -- {1} +SELECT hashset_add('{1}', 2); -- {1,2} +``` + + +### hashset_contains() + +`hashset_contains(int4hashset, int) -> boolean` + +Checks if an int4hashset contains a given integer. + +```sql +SELECT hashset_contains('{1}', 1); -- TRUE +SELECT hashset_contains('{1}', 2); -- FALSE +``` + +If the *cardinality* of the hashset is zero (0), it is known that it doesn't +contain any value, not even an Unknown value represented as `NULL`, so even in +that case it returns `FALSE`. + +```sql +SELECT hashset_contains('{}', 1); -- FALSE +SELECT hashset_contains('{}', NULL); -- FALSE +``` + +If the hashset is `NULL`, then the result is `NULL`. + +```sql +SELECT hashset_contains(NULL, NULL); -- NULL +SELECT hashset_contains(NULL, 1); -- NULL +``` + + +### hashset_to_array() + +`hashset_to_array(int4hashset) -> int[]` + +Converts an int4hashset to an array of unsorted integers. + +```sql +SELECT hashset_to_array('{2,1,3}'); -- {3,2,1} +``` + + +### hashset_to_sorted_array() + +`hashset_to_sorted_array(int4hashset) -> int[]` + +Converts an int4hashset to an array of sorted integers. + +```sql +SELECT hashset_to_sorted_array('{2,1,3}'); -- {1,2,3} +``` + +If the hashset contains a `NULL` element, it follows the same behavior as the +`ORDER BY` clause in SQL: the `NULL` element is positioned at the end of the +sorted array. + +```sql +SELECT hashset_to_sorted_array('{2,1,NULL,3}'); -- {1,2,3,NULL} +``` + + +### hashset_cardinality() + +`hashset_cardinality(int4hashset) -> bigint` + +Returns the number of elements in an int4hashset. + +```sql +SELECT hashset_cardinality(NULL); -- NULL +SELECT hashset_cardinality('{}'); -- 0 +SELECT hashset_cardinality('{1}'); -- 1 +SELECT hashset_cardinality('{1,1}'); -- 1 +SELECT hashset_cardinality('{NULL,NULL}'); -- 1 +SELECT hashset_cardinality('{1,NULL}'); -- 2 +SELECT hashset_cardinality('{1,2,3}'); -- 3 +``` + + +### hashset_capacity() + +`hashset_capacity(int4hashset) -> bigint` + +Returns the current capacity of an int4hashset. + + +### hashset_max_collisions() + +`hashset_max_collisions(int4hashset) -> bigint` + +Returns the maximum number of collisions that have occurred for a single element + + +### hashset_union() + +`hashset_union(int4hashset, int4hashset) -> int4hashset` + +Merges two int4hashsets into a new int4hashset. + +```sql +SELECT hashset_union('{1,2}', '{2,3}'); -- '{1,2,3} +``` + +If any of the operands are `NULL`, the result is `NULL`. + +```sql +SELECT hashset_union('{1}', NULL); -- NULL +SELECT hashset_union(NULL, '{1}'); -- NULL +``` + + +### hashset_intersection() + +`hashset_intersection(int4hashset, int4hashset) -> int4hashset` + +Returns a new int4hashset that is the intersection of the two input sets. + +```sql +SELECT hashset_intersection('{1,2}', '{2,3}'); -- {2} +SELECT hashset_intersection('{1,2,NULL}', '{2,3,NULL}'); -- {2,NULL} +``` + +If any of the operands are `NULL`, the result is `NULL`. + +```sql +SELECT hashset_intersection('{1,2}', NULL); -- NULL +SELECT hashset_intersection(NULL, '{2,3}'); -- NULL +``` + + +### hashset_difference() + +`hashset_difference(int4hashset, int4hashset) -> int4hashset` + +Returns a new int4hashset that contains the elements present in the first set +but not in the second set. + +```sql +SELECT hashset_difference('{1,2}', '{2,3}'); -- {1} +SELECT hashset_difference('{1,2,NULL}', '{2,3,NULL}'); -- {1} +SELECT hashset_difference('{1,2,NULL}', '{2,3}'); -- {1,NULL} +``` + +If any of the operands are `NULL`, the result is `NULL`. + +```sql +SELECT hashset_difference('{1,2}', NULL); -- NULL +SELECT hashset_difference(NULL, '{2,3}'); -- NULL +``` + + +### hashset_symmetric_difference() + +`hashset_symmetric_difference(int4hashset, int4hashset) -> int4hashset` + +Returns a new int4hashset containing elements that are in either of the input sets, but not in their intersection. + +```sql +SELECT hashset_symmetric_difference('{1,2}', '{2,3}'); -- {1,3} +SELECT hashset_symmetric_difference('{1,2,NULL}', '{2,3,NULL}'); -- {1,3} +SELECT hashset_symmetric_difference('{1,2,NULL}', '{2,3}'); -- {1,3,NULL} +``` + +If any of the operands are `NULL`, the result is `NULL`. + +```sql +SELECT hashset_symmetric_difference('{1,2}', NULL); -- NULL +SELECT hashset_symmetric_difference(NULL, '{2,3}'); -- NULL +``` + ## Aggregation Functions -- `hashset_agg(int) -> int4hashset`: Aggregate integers into a hashset. -- `hashset_agg(int4hashset) -> int4hashset`: Aggregate hashsets into a hashset. +### hashset_agg(int4) + +`hashset_agg(int4) -> int4hashset` + +Aggregate integers into a hashset. + +```sql +SELECT hashset_agg(some_int4_column) FROM some_table; +``` + + +### hashset_agg(int4hashset) + +`hashset_agg(int4hashset) -> int4hashset` + +Aggregate hashsets into a hashset. + +```sql +SELECT hashset_agg(some_int4hashset_column) FROM some_table; +``` ## Operators diff --git a/benchmark/.gitignore b/benchmark/.gitignore new file mode 100644 index 0000000..f3c4e1c --- /dev/null +++ b/benchmark/.gitignore @@ -0,0 +1,2 @@ +soc-pokec-relationships.txt.gz +soc-pokec-relationships.txt diff --git a/benchmark/friends_of_friends.sh b/benchmark/friends_of_friends.sh new file mode 100755 index 0000000..d4570c0 --- /dev/null +++ b/benchmark/friends_of_friends.sh @@ -0,0 +1,10 @@ +#!/bin/sh +if [ ! -f "soc-pokec-relationships.txt" ]; then + wget https://snap.stanford.edu/data/soc-pokec-relationships.txt.gz + gunzip soc-pokec-relationships.txt.gz +fi + +psql -X -c "CREATE TABLE edges (from_node INT, to_node INT);" +psql -X -c "\COPY edges FROM soc-pokec-relationships.txt;" +psql -X -c "ALTER TABLE edges ADD PRIMARY KEY (from_node, to_node);" +psql -X -f friends_of_friends.sql diff --git a/benchmark/friends_of_friends.sql b/benchmark/friends_of_friends.sql new file mode 100644 index 0000000..ecc8f0e --- /dev/null +++ b/benchmark/friends_of_friends.sql @@ -0,0 +1,72 @@ +CREATE EXTENSION IF NOT EXISTS hashset; + +\timing on + +CREATE OR REPLACE VIEW vfriends_of_friends_array_agg_distinct AS +WITH RECURSIVE friends_of_friends AS ( + SELECT + ARRAY[5867::bigint] AS current, + 0 AS depth + UNION ALL + SELECT + new_current, + friends_of_friends.depth + 1 + FROM + friends_of_friends + CROSS JOIN LATERAL ( + SELECT + array_agg(DISTINCT edges.to_node) AS new_current + FROM + edges + WHERE + from_node = ANY(friends_of_friends.current) + ) q + WHERE + friends_of_friends.depth < 3 +) +SELECT + COALESCE(array_length(current, 1), 0) AS count_friends_at_depth_3 +FROM + friends_of_friends +WHERE + depth = 3; + +CREATE OR REPLACE VIEW vfriends_of_friends_hashset_agg AS +WITH RECURSIVE friends_of_friends AS +( + SELECT + '{5867}'::int4hashset AS current, + 0 AS depth + UNION ALL + SELECT + new_current, + friends_of_friends.depth + 1 + FROM + friends_of_friends + CROSS JOIN LATERAL + ( + SELECT + hashset_agg(edges.to_node) AS new_current + FROM + edges + WHERE + from_node = ANY(hashset_to_array(friends_of_friends.current)) + ) q + WHERE + friends_of_friends.depth < 3 +) +SELECT + depth, + hashset_cardinality(current) +FROM + friends_of_friends +WHERE + depth = 3; + +SELECT * FROM vfriends_of_friends_array_agg_distinct; +SELECT * FROM vfriends_of_friends_array_agg_distinct; +SELECT * FROM vfriends_of_friends_array_agg_distinct; + +SELECT * FROM vfriends_of_friends_hashset_agg; +SELECT * FROM vfriends_of_friends_hashset_agg; +SELECT * FROM vfriends_of_friends_hashset_agg; diff --git a/hashset-api.c b/hashset-api.c index a4beef4..2b92363 100644 --- a/hashset-api.c +++ b/hashset-api.c @@ -1,8 +1,6 @@ #include "hashset.h" -#include #include -#include #include #include #include @@ -188,15 +186,11 @@ int4hashset_in(PG_FUNCTION_ARGS) Datum int4hashset_out(PG_FUNCTION_ARGS) { - int4hashset_t *set = PG_GETARG_INT4HASHSET(0); - char *bitmap; - int32 *values; - int i; - StringInfoData str; - - /* Calculate the pointer to the bitmap and values array */ - bitmap = set->data; - values = (int32 *) (set->data + CEIL_DIV(set->capacity, 8)); + int i; + int4hashset_t *set = PG_GETARG_INT4HASHSET(0); + char *bitmap = HASHSET_GET_BITMAP(set); + int32 *values = HASHSET_GET_VALUES(set); + StringInfoData str; /* Initialize the StringInfo buffer */ initStringInfo(&str); @@ -412,8 +406,8 @@ int4hashset_union(PG_FUNCTION_ARGS) int i; int4hashset_t *seta = PG_GETARG_INT4HASHSET_COPY(0); int4hashset_t *setb = PG_GETARG_INT4HASHSET(1); - char *bitmap = setb->data; - int32 *values = (int32 *) (bitmap + CEIL_DIV(setb->capacity, 8)); + char *bitmap = HASHSET_GET_BITMAP(setb); + int32 *values = HASHSET_GET_VALUES(setb); for (i = 0; i < setb->capacity; i++) { @@ -593,14 +587,9 @@ int4hashset_agg_add_set(PG_FUNCTION_ARGS) { int i; - char *bitmap; - int32 *values; - int4hashset_t *value; - - value = PG_GETARG_INT4HASHSET(1); - - bitmap = value->data; - values = (int32 *) (value->data + CEIL_DIV(value->capacity, 8)); + int4hashset_t *value = PG_GETARG_INT4HASHSET(1); + char *bitmap = HASHSET_GET_BITMAP(value); + int32 *values = HASHSET_GET_VALUES(value); for (i = 0; i < value->capacity; i++) { @@ -666,8 +655,8 @@ int4hashset_agg_combine(PG_FUNCTION_ARGS) src = (int4hashset_t *) PG_GETARG_POINTER(1); dst = (int4hashset_t *) PG_GETARG_POINTER(0); - bitmap = src->data; - values = (int32 *) (src->data + CEIL_DIV(src->capacity, 8)); + bitmap = HASHSET_GET_BITMAP(src); + values = HASHSET_GET_VALUES(src); for (i = 0; i < src->capacity; i++) { @@ -687,22 +676,18 @@ int4hashset_to_array(PG_FUNCTION_ARGS) { int i, idx; - int4hashset_t *set; + int4hashset_t *set = PG_GETARG_INT4HASHSET(0); int32 *values; int nvalues; char *sbitmap; int32 *svalues; - set = PG_GETARG_INT4HASHSET(0); - /* if hashset is empty and does not contain null, return an empty array */ - if(set->nelements == 0 && !set->null_element) { - Datum d = PointerGetDatum(construct_empty_array(INT4OID)); - PG_RETURN_ARRAYTYPE_P(d); - } + if(set->nelements == 0 && !set->null_element) + PG_RETURN_ARRAYTYPE_P(construct_empty_array(INT4OID)); - sbitmap = set->data; - svalues = (int32 *) (set->data + CEIL_DIV(set->capacity, 8)); + sbitmap = HASHSET_GET_BITMAP(set); + svalues = HASHSET_GET_VALUES(set); /* number of values to store in the array */ nvalues = set->nelements; @@ -733,10 +718,8 @@ int4hashset_to_sorted_array(PG_FUNCTION_ARGS) set = PG_GETARG_INT4HASHSET(0); /* if hashset is empty and does not contain null, return an empty array */ - if(set->nelements == 0 && !set->null_element) { - Datum d = PointerGetDatum(construct_empty_array(INT4OID)); - PG_RETURN_ARRAYTYPE_P(d); - } + if(set->nelements == 0 && !set->null_element) + PG_RETURN_ARRAYTYPE_P(construct_empty_array(INT4OID)); /* extract the sorted elements from the hashset */ values = int4hashset_extract_sorted_elements(set); @@ -762,8 +745,8 @@ int4hashset_eq(PG_FUNCTION_ARGS) if (a->nelements != b->nelements) PG_RETURN_BOOL(false); - bitmap_a = a->data; - values_a = (int32 *)(a->data + CEIL_DIV(a->capacity, 8)); + bitmap_a = HASHSET_GET_BITMAP(a); + values_a = HASHSET_GET_VALUES(a); /* * Check if every element in a is also in b @@ -940,8 +923,9 @@ int4hashset_intersection(PG_FUNCTION_ARGS) int i; int4hashset_t *seta = PG_GETARG_INT4HASHSET(0); int4hashset_t *setb = PG_GETARG_INT4HASHSET(1); - char *bitmap = setb->data; - int32 *values = (int32 *)(bitmap + CEIL_DIV(setb->capacity, 8)); + char *bitmap = HASHSET_GET_BITMAP(setb); + int32 *values = HASHSET_GET_VALUES(setb); + int4hashset_t *intersection; intersection = int4hashset_allocate( @@ -976,8 +960,8 @@ int4hashset_difference(PG_FUNCTION_ARGS) int4hashset_t *seta = PG_GETARG_INT4HASHSET(0); int4hashset_t *setb = PG_GETARG_INT4HASHSET(1); int4hashset_t *difference; - char *bitmap = seta->data; - int32 *values = (int32 *)(bitmap + CEIL_DIV(seta->capacity, 8)); + char *bitmap = HASHSET_GET_BITMAP(seta); + int32 *values = HASHSET_GET_VALUES(seta); difference = int4hashset_allocate( seta->capacity, @@ -1011,10 +995,10 @@ int4hashset_symmetric_difference(PG_FUNCTION_ARGS) int4hashset_t *seta = PG_GETARG_INT4HASHSET(0); int4hashset_t *setb = PG_GETARG_INT4HASHSET(1); int4hashset_t *result; - char *bitmapa = seta->data; - char *bitmapb = setb->data; - int32 *valuesa = (int32 *) (bitmapa + CEIL_DIV(seta->capacity, 8)); - int32 *valuesb = (int32 *) (bitmapb + CEIL_DIV(setb->capacity, 8)); + char *bitmapa = HASHSET_GET_BITMAP(seta); + char *bitmapb = HASHSET_GET_BITMAP(setb); + int32 *valuesa = HASHSET_GET_VALUES(seta); + int32 *valuesb = HASHSET_GET_VALUES(setb); result = int4hashset_allocate( seta->nelements + setb->nelements, diff --git a/hashset.c b/hashset.c index 91907ab..65ab25f 100644 --- a/hashset.c +++ b/hashset.c @@ -58,8 +58,8 @@ int4hashset_resize(int4hashset_t * set) { int i; int4hashset_t *new; - char *bitmap; - int32 *values; + char *bitmap = HASHSET_GET_BITMAP(set); + int32 *values = HASHSET_GET_VALUES(set); int new_capacity; new_capacity = (int)(set->capacity * set->growth_factor); @@ -80,10 +80,6 @@ int4hashset_resize(int4hashset_t * set) set->hashfn_id ); - /* Calculate the pointer to the bitmap and values array */ - bitmap = set->data; - values = (int32 *) (set->data + CEIL_DIV(set->capacity, 8)); - for (i = 0; i < set->capacity; i++) { int byte = (i / 8); @@ -131,8 +127,8 @@ int4hashset_add_element(int4hashset_t *set, int32 value) position = hash % set->capacity; - bitmap = set->data; - values = (int32 *) (set->data + CEIL_DIV(set->capacity, 8)); + bitmap = HASHSET_GET_BITMAP(set); + values = HASHSET_GET_VALUES(set); while (true) { @@ -178,8 +174,8 @@ int4hashset_contains_element(int4hashset_t *set, int32 value) int bit; uint32 hash; uint32 position; - char *bitmap; - int32 *values; + char *bitmap = HASHSET_GET_BITMAP(set); + int32 *values = HASHSET_GET_VALUES(set); int num_probes = 0; /* Counter for the number of probes */ if (set->hashfn_id == JENKINS_LOOKUP3_HASHFN_ID) @@ -203,9 +199,6 @@ int4hashset_contains_element(int4hashset_t *set, int32 value) position = hash % set->capacity; - bitmap = set->data; - values = (int32 *) (set->data + CEIL_DIV(set->capacity, 8)); - while (true) { byte = (position / 8); @@ -233,15 +226,10 @@ int4hashset_contains_element(int4hashset_t *set, int32 value) int32 * int4hashset_extract_sorted_elements(int4hashset_t *set) { - /* Allocate memory for the elements array */ - int32 *elements = palloc(set->nelements * sizeof(int32)); - - /* Access the data array */ - char *bitmap = set->data; - int32 *values = (int32 *)(set->data + CEIL_DIV(set->capacity, 8)); - - /* Counter for the number of extracted elements */ - int32 nextracted = 0; + int32 *elements = palloc(set->nelements * sizeof(int32)); + char *bitmap = HASHSET_GET_BITMAP(set); + int32 *values = HASHSET_GET_VALUES(set); + int32 nextracted = 0; /* Iterate through all elements */ for (int32 i = 0; i < set->capacity; i++) diff --git a/hashset.h b/hashset.h index 86f5d1b..3631e22 100644 --- a/hashset.h +++ b/hashset.h @@ -12,6 +12,8 @@ #include "common/hashfn.h" #define CEIL_DIV(a, b) (((a) + (b) - 1) / (b)) +#define HASHSET_GET_BITMAP(set) ((set)->data) +#define HASHSET_GET_VALUES(set) ((int32 *) ((set)->data + CEIL_DIV((set)->capacity, 8))) #define HASHSET_STEP 13 #define JENKINS_LOOKUP3_HASHFN_ID 1 #define MURMURHASH32_HASHFN_ID 2