diff --git a/duckdb/.gitignore b/duckdb/.gitignore new file mode 100644 index 00000000..5691c660 --- /dev/null +++ b/duckdb/.gitignore @@ -0,0 +1,2 @@ +*.duckdb* +ldbc_parquets/ diff --git a/duckdb/README.md b/duckdb/README.md new file mode 100644 index 00000000..d6e30a65 --- /dev/null +++ b/duckdb/README.md @@ -0,0 +1,111 @@ +# LDBC SNB BI Umbra implementation + +[Umbra](https://umbra-db.com/) implementation of the [LDBC Social Network Benchmark's BI workload](https://github.com/ldbc/ldbc_snb_docs). + +:warning: Note that while this implementation uses containerized execution for Umbra, it does not work on MacOS (neither on x86, nor on Apple Silicon). + +## Getting the container + +The Umbra container is currently available upon request. + +## Generating the data set + +The Umbra implementation expects the data to be in `composite-merged-fk` CSV layout, with headers and without quoted fields. +To generate data that confirms this requirement, run Datagen without any layout or formatting arguments (`--explode-*` or `--format-options`). + +In Datagen's directory (`ldbc_snb_datagen_spark`), issue the following commands. We assume that the Datagen project is built and the `${PLATFORM_VERSION}`, `${DATAGEN_VERSION}` environment variables are set correctly. + +```bash +export SF=desired_scale_factor +export LDBC_SNB_DATAGEN_MAX_MEM=available_memory +export LDBC_SNB_DATAGEN_JAR=$(sbt -batch -error 'print assembly / assemblyOutputPath') +``` + +```bash +rm -rf out-sf${SF}/ +tools/run.py \ + --cores $(nproc) \ + --memory ${LDBC_SNB_DATAGEN_MAX_MEM} \ + -- \ + --format csv \ + --scale-factor ${SF} \ + --mode bi \ + --output-dir out-sf${SF} +``` + +## Loading the data set + +Note that unlike Postgres, Umbra does not support directly loading from compressed files (`.csv.gz`). + +1. Set the `${UMBRA_CSV_DIR}` environment variable to point to the data set. + + * To use a locally generated data set, set the `${LDBC_SNB_DATAGEN_DIR}` and `${SF}` environment variables and run: + + ```bash + export UMBRA_CSV_DIR=${LDBC_SNB_DATAGEN_DIR}/out-sf${SF}/graphs/csv/bi/composite-merged-fk/ + ``` + + Or, simply run: + + ```bash + . scripts/use-datagen-data-set.sh + ``` + + * To download and use the sample data set, run: + + ```bash + wget -q https://ldbcouncil.org/ldbc_snb_datagen_spark/social-network-sf0.003-bi-composite-merged-fk.zip + unzip -q social-network-sf0.003-bi-composite-merged-fk.zip + export UMBRA_CSV_DIR=`pwd`/social-network-sf0.003-bi-composite-merged-fk/graphs/csv/bi/composite-merged-fk/ + ``` + + Or, simply run: + + ``` + scripts/get-sample-data-set.sh + . scripts/use-sample-data-set.sh + ``` + +1. The data set should consist of uncompressed CSVs. If you retrieved a compressed data set (`.csv.gz` files), set the `${UMBRA_CSV_DIR}` environment variable and uncompress the files (note that doing so deletes the original compressed files): + + ```bash + scripts/decompress-data-set.sh + ``` + +1. To start the DBMS, create a database and load the data, run: + + ```bash + scripts/load-in-one-step.sh + ``` + +1. The substitution parameters should be generated using the [`paramgen`](../paramgen). + +## Queries + +To run the queries, issue: + +```bash +scripts/queries.sh +``` + +For a test run, use: + +```bash +scripts/queries.sh ${SF} --test +``` + +## Benchmark + +To run the queries and the batches alternately, as specified by the benchmark, run: + +```bash +scripts/benchmark.sh +``` + +## Running queries interactively + +To connect to the database through the SQL console, use: + +```bash +scripts/connect.sh +``` diff --git a/duckdb/benchmark.py b/duckdb/benchmark.py new file mode 100644 index 00000000..b5313cd2 --- /dev/null +++ b/duckdb/benchmark.py @@ -0,0 +1,223 @@ +import csv +import datetime +from dateutil.relativedelta import relativedelta +import os +import psycopg2 +import time +from queries import run_script, run_queries, run_precomputations, load_mht, load_plm, load_post +from pathlib import Path +from itertools import cycle +import argparse + + +def execute(cur, query): + cur.execute(query) + + +query_variants = ["1", "2a", "2b", "3", "4", "5", "6", "7", "8a", "8b", "9", "10a", "10b", "11", "12", "13", "14a", "14b", "15a", "15b", "16a", "16b", "17", "18", "19a", "19b", "20a", "20b"] + +def run_batch_updates(pg_con, data_dir, batch_date, batch_type, timings_file): + batch_id = batch_date.strftime('%Y-%m-%d') + batch_dir = f"batch_id={batch_id}" + print(f"#################### {batch_dir} ####################") + + start = time.time() + + print("## Inserts") + for entity in insert_entities: + batch_path = f"{data_dir}/inserts/dynamic/{entity}/{batch_dir}" + if not os.path.exists(batch_path): + continue + + for csv_file in [f for f in os.listdir(batch_path) if f.endswith(".csv")]: + csv_path = f"{batch_path}/{csv_file}" + print(f"- {csv_path}") + execute(cur, "BEGIN BULK WRITE;") + execute(cur, f"COPY {entity} FROM '{dbs_data_dir}/inserts/dynamic/{entity}/{batch_dir}/{csv_file}' (DELIMITER '|', HEADER, NULL '', FORMAT text)") + execute(cur, "COMMIT;") + if entity == "Person_knows_Person": + execute(cur, "BEGIN BULK WRITE;") + execute(cur, f"COPY {entity} (creationDate, Person2id, Person1id) FROM '{dbs_data_dir}/inserts/dynamic/{entity}/{batch_dir}/{csv_file}' (DELIMITER '|', HEADER, NULL '', FORMAT text)") + execute(cur, "COMMIT;") + + for entity in ["Comment_hasTag_Tag", "Post_hasTag_Tag"]: + batch_path = f"{data_dir}/inserts/dynamic/{entity}/{batch_dir}" + if not os.path.exists(batch_path): + continue + for csv_file in [f for f in os.listdir(batch_path) if f.endswith(".csv")]: + csvpath = f"{dbs_data_dir}/inserts/dynamic/{entity}/{batch_dir}/{csv_file}" + load_mht(cur, csvpath) + + for entity in ["Person_likes_Comment", "Person_likes_Post"]: + batch_path = f"{data_dir}/inserts/dynamic/{entity}/{batch_dir}" + if not os.path.exists(batch_path): + continue + for csv_file in [f for f in os.listdir(batch_path) if f.endswith(".csv")]: + csvpath = f"{dbs_data_dir}/inserts/dynamic/{entity}/{batch_dir}/{csv_file}" + load_plm(cur, csvpath) + + for entity in ["Post"]: + batch_path = f"{data_dir}/inserts/dynamic/{entity}/{batch_dir}" + if not os.path.exists(batch_path): + continue + for csv_file in [f for f in os.listdir(batch_path) if f.endswith(".csv")]: + csvpath = f"{dbs_data_dir}/inserts/dynamic/{entity}/{batch_dir}/{csv_file}" + load_post(cur, csvpath) + + + print("## Deletes") + # Deletes are implemented using a SQL script which use auxiliary tables. + # Entities to be deleted are first put into {entity}_Delete_candidate tables. + # These are cleaned up before running the delete script. + for entity in delete_entities: + execute(cur, "BEGIN BULK WRITE;") + execute(cur, f"DELETE FROM {entity}_Delete_candidates") + execute(cur, "COMMIT;") + + batch_path = f"{data_dir}/deletes/dynamic/{entity}/{batch_dir}" + if not os.path.exists(batch_path): + continue + + for csv_file in [f for f in os.listdir(batch_path) if f.endswith(".csv")]: + csv_path = f"{batch_path}/{csv_file}" + print(f"- {csv_path}") + execute(cur, "BEGIN BULK WRITE;") + execute(cur, f"COPY {entity}_Delete_candidates FROM '{dbs_data_dir}/deletes/dynamic/{entity}/{batch_dir}/{csv_file}' (DELIMITER '|', HEADER, NULL '', FORMAT text)") + execute(cur, "COMMIT;") + + print("Maintain materialized views . . .") + run_script(pg_con, cur, "dml/maintain-views.sql") + print("Done.") + print() + + print("Apply deletes . . .") + # Invoke delete script which makes use of the {entity}_Delete_candidates tables + run_script(pg_con, cur, "dml/apply-deletes.sql") + print("Done.") + print() + + print("Apply precomp . . .") + run_precomputations(query_variants, pg_con, cur, batch_id, batch_type, sf, timings_file) + print("Done.") + print() + + end = time.time() + duration = end - start + timings_file.write(f"Umbra|{sf}|{batch_id}|{batch_type}|writes||{duration}\n") + + +parser = argparse.ArgumentParser() +parser.add_argument('--scale_factor', type=str, help='Scale factor', required=True) +parser.add_argument('--test', action='store_true', help='Test execution: 1 query/batch', required=False) +parser.add_argument('--validate', action='store_true', help='Validation mode', required=False) +parser.add_argument('--pgtuning', action='store_true', help='Paramgen tuning execution: 100 queries/batch', required=False) +parser.add_argument('--local', action='store_true', help='Local run (outside of a container)', required=False) +parser.add_argument('--data_dir', type=str, help='Directory with the initial_snapshot, insert, and delete directories', required=True) +parser.add_argument('--param_dir', type=str, help='Directory with the initial_snapshot, insert, and delete directories') +parser.add_argument('--queries', action='store_true', help='Only run queries', required=False) +args = parser.parse_args() +sf = args.scale_factor +test = args.test +pgtuning = args.pgtuning +local = args.local +data_dir = args.data_dir +queries_only = args.queries +validate = args.validate +if args.param_dir is not None: + param_dir = args.param_dir +else: + param_dir = f'../parameters/parameters-sf{sf}' + + +if local: + dbs_data_dir = data_dir +else: + dbs_data_dir = '/data' + +parameter_csvs = {} +for query_variant in query_variants: + # wrap parameters into infinite loop iterator + parameter_csvs[query_variant] = cycle(csv.DictReader(open(f'{param_dir}/bi-{query_variant}.csv'), delimiter='|')) + +print(f"- Input data directory, ${{UMBRA_CSV_DIR}}: {data_dir}") + +insert_nodes = ["Forum", "Person", "Comment"] +insert_edges = ["Forum_hasMember_Person", "Forum_hasTag_Tag", "Person_hasInterest_Tag", "Person_knows_Person", "Person_studyAt_University", "Person_workAt_Company"] +insert_entities = insert_nodes + insert_edges + +# set the order of deletions to reflect the dependencies between node labels (:Comment)-[:REPLY_OF]->(:Post)<-[:CONTAINER_OF]-(:Forum)-[:HAS_MODERATOR]->(:Person) +delete_nodes = ["Comment", "Post", "Forum", "Person"] +delete_edges = ["Forum_hasMember_Person", "Person_knows_Person", "Person_likes_Comment", "Person_likes_Post"] +delete_entities = delete_nodes + delete_edges + +output = Path(f"output/output-sf{sf}") +output.mkdir(parents=True, exist_ok=True) +open(f"output/output-sf{sf}/results.csv", "w").close() +open(f"output/output-sf{sf}/timings.csv", "w").close() + +timings_file = open(f"output/output-sf{sf}/timings.csv", "a") +timings_file.write(f"tool|sf|day|batch_type|q|parameters|time\n") +results_file = open(f"output/output-sf{sf}/results.csv", "a") + +pg_con = psycopg2.connect(host="localhost", user="postgres", password="mysecretpassword", port=8000) +pg_con.autocommit = True +cur = pg_con.cursor() + +network_start_date = datetime.date(2012, 11, 29) +network_end_date = datetime.date(2013, 1, 1) +test_end_date = datetime.date(2012, 12, 2) +batch_size = relativedelta(days=1) +batch_date = network_start_date + +benchmark_start = time.time() + +run_script(pg_con, cur, f"ddl/schema-delete-candidates.sql") + +if queries_only: + run_queries(query_variants, parameter_csvs, pg_con, sf, test, pgtuning, batch_date, "power", timings_file, results_file) +else: + # Run alternating write-read blocks. + # The first write-read block is the power batch, while the rest are the throughput batches. + current_batch = 1 + while batch_date < network_end_date and \ + (not test or batch_date < test_end_date) and \ + (not validate or batch_date == network_start_date): + if current_batch == 1: + batch_type = "power" + else: + batch_type = "throughput" + print() + print(f"----------------> Batch date: {batch_date}, batch type: {batch_type} <---------------") + + if current_batch == 2: + start = time.time() + + run_batch_updates(pg_con, data_dir, batch_date, batch_type, timings_file) + reads_time = run_queries(query_variants, parameter_csvs, pg_con, sf, test, pgtuning, batch_date, batch_type, timings_file, results_file) + timings_file.write(f"Umbra|{sf}|{batch_date}|{batch_type}|reads||{reads_time:.6f}\n") + + # checking if 1 hour (and a bit) has elapsed for the throughput batches + if current_batch >= 2: + end = time.time() + duration = end - start + if duration > 3605: + print("""Throughput batches finished successfully. Termination criteria met: + - At least 1 throughput batch was executed + - The total execution time of the throughput batch(es) was at least 1h""") + break + + current_batch = current_batch + 1 + batch_date = batch_date + batch_size + +cur.close() +pg_con.close() + +benchmark_end = time.time() +benchmark_duration = benchmark_end - benchmark_start +benchmark_file = open(f"output/output-sf{sf}/benchmark.csv", "w") +benchmark_file.write(f"time\n") +benchmark_file.write(f"{benchmark_duration:.6f}\n") +benchmark_file.close() + +timings_file.close() +results_file.close() diff --git a/duckdb/ddl/drop-tables.sql b/duckdb/ddl/drop-tables.sql new file mode 100644 index 00000000..d512d394 --- /dev/null +++ b/duckdb/ddl/drop-tables.sql @@ -0,0 +1,35 @@ +DROP VIEW IF EXISTS Comment_View; +DROP VIEW IF EXISTS Post_View; + +DROP TABLE IF EXISTS Organisation; +DROP TABLE IF EXISTS Place; +DROP TABLE IF EXISTS Tag; +DROP TABLE IF EXISTS TagClass; +DROP TABLE IF EXISTS Company; +DROP TABLE IF EXISTS University; +DROP TABLE IF EXISTS Continent; +DROP TABLE IF EXISTS Country; +DROP TABLE IF EXISTS City; +DROP TABLE IF EXISTS Comment; +DROP TABLE IF EXISTS Forum; +DROP TABLE IF EXISTS Post; +DROP TABLE IF EXISTS Person; +DROP TABLE IF EXISTS Comment_hasTag_Tag; +DROP TABLE IF EXISTS Post_hasTag_Tag; +DROP TABLE IF EXISTS Forum_hasMember_Person; +DROP TABLE IF EXISTS Forum_hasTag_Tag; +DROP TABLE IF EXISTS Person_hasInterest_Tag; +DROP TABLE IF EXISTS Person_likes_Comment; +DROP TABLE IF EXISTS Person_likes_Post; +DROP TABLE IF EXISTS Person_studyAt_University; +DROP TABLE IF EXISTS Person_workAt_Company; +DROP TABLE IF EXISTS Person_knows_Person; + +DROP TABLE IF EXISTS Message; +DROP TABLE IF EXISTS Message_hasTag_Tag; +DROP TABLE IF EXISTS Person_likes_Message; + +DROP TABLE IF EXISTS Top100PopularForumsQ04; +DROP TABLE IF EXISTS PopularityScoreQ06; +DROP TABLE IF EXISTS PathQ19; +DROP TABLE IF EXISTS PathQ20; diff --git a/duckdb/ddl/output-env.sql b/duckdb/ddl/output-env.sql new file mode 100644 index 00000000..75c07fb6 --- /dev/null +++ b/duckdb/ddl/output-env.sql @@ -0,0 +1,3 @@ + +show debug.buffersize; +show debug.usedirectio; diff --git a/duckdb/ddl/schema-composite-merged-fk.sql b/duckdb/ddl/schema-composite-merged-fk.sql new file mode 100644 index 00000000..fe50cc1b --- /dev/null +++ b/duckdb/ddl/schema-composite-merged-fk.sql @@ -0,0 +1,220 @@ +-- static tables + +CREATE TABLE Organisation ( + id bigint PRIMARY KEY, + type text NOT NULL, + name text NOT NULL, + url text NOT NULL, + LocationPlaceId bigint NOT NULL +); + +CREATE TABLE Place ( + id bigint PRIMARY KEY, + name text NOT NULL, + url text NOT NULL, + type text NOT NULL, + PartOfPlaceId bigint -- null for continents +); + +CREATE TABLE Tag ( + id bigint PRIMARY KEY, + name text NOT NULL, + url text NOT NULL, + TypeTagClassId bigint NOT NULL +); + +CREATE TABLE TagClass ( + id bigint PRIMARY KEY, + name text NOT NULL, + url text NOT NULL, + SubclassOfTagClassId bigint -- null for the root TagClass (Thing) +); + +-- static tables / separate table per individual subtype + +CREATE TABLE Country ( + id bigint primary key, + name text not null, + url text not null, + PartOfContinentId bigint +); + +CREATE TABLE City ( + id bigint primary key, + name text not null, + url text not null, + PartOfCountryId bigint +); + +CREATE TABLE Company ( + id bigint primary key, + name text not null, + url text not null, + LocationPlaceId bigint not null +); + +CREATE TABLE University ( + id bigint primary key, + name text not null, + url text not null, + LocationPlaceId bigint not null +); + +-- dynamic tables + +CREATE TABLE Comment ( + creationDate timestamp with time zone NOT NULL, + id bigint NOT NULL, --PRIMARY KEY, + locationIP text NOT NULL, + browserUsed text NOT NULL, + content text NOT NULL, + length int NOT NULL, + CreatorPersonId bigint NOT NULL, + LocationCountryId bigint NOT NULL, + ParentPostId bigint, + ParentCommentId bigint +); + +CREATE TABLE Forum ( + creationDate timestamp with time zone NOT NULL, + id bigint PRIMARY KEY, + title text NOT NULL, + ModeratorPersonId bigint -- can be null as its cardinality is 0..1 +); + +CREATE TABLE Post ( + creationDate timestamp with time zone NOT NULL, + id bigint NOT NULL, --PRIMARY KEY, + imageFile text, + locationIP text NOT NULL, + browserUsed text NOT NULL, + language text, + content text, + length int NOT NULL, + CreatorPersonId bigint NOT NULL, + ContainerForumId bigint NOT NULL, + LocationCountryId bigint NOT NULL +); + +CREATE TABLE Person ( + creationDate timestamp with time zone NOT NULL, + id bigint PRIMARY KEY, + firstName text NOT NULL, + lastName text NOT NULL, + gender text NOT NULL, + birthday date NOT NULL, + locationIP text NOT NULL, + browserUsed text NOT NULL, + LocationCityId bigint NOT NULL, + speaks text NOT NULL, + email text NOT NULL +); + + +-- edges +CREATE TABLE Comment_hasTag_Tag ( + creationDate timestamp with time zone NOT NULL, + CommentId bigint NOT NULL, + TagId bigint NOT NULL +); + +CREATE TABLE Post_hasTag_Tag ( + creationDate timestamp with time zone NOT NULL, + PostId bigint NOT NULL, + TagId bigint NOT NULL +); + +CREATE TABLE Forum_hasMember_Person ( + creationDate timestamp with time zone NOT NULL, + ForumId bigint NOT NULL, + PersonId bigint NOT NULL +); + +CREATE TABLE Forum_hasTag_Tag ( + creationDate timestamp with time zone NOT NULL, + ForumId bigint NOT NULL, + TagId bigint NOT NULL +); + +CREATE TABLE Person_hasInterest_Tag ( + creationDate timestamp with time zone NOT NULL, + PersonId bigint NOT NULL, + TagId bigint NOT NULL +); + +CREATE TABLE Person_likes_Comment ( + creationDate timestamp with time zone NOT NULL, + PersonId bigint NOT NULL, + CommentId bigint NOT NULL +); + +CREATE TABLE Person_likes_Post ( + creationDate timestamp with time zone NOT NULL, + PersonId bigint NOT NULL, + PostId bigint NOT NULL +); + +CREATE TABLE Person_studyAt_University ( + creationDate timestamp with time zone NOT NULL, + PersonId bigint NOT NULL, + UniversityId bigint NOT NULL, + classYear int NOT NULL +); + +CREATE TABLE Person_workAt_Company ( + creationDate timestamp with time zone NOT NULL, + PersonId bigint NOT NULL, + CompanyId bigint NOT NULL, + workFrom int NOT NULL +); + +CREATE TABLE Person_knows_Person ( + creationDate timestamp with time zone NOT NULL, + Person1id bigint NOT NULL, + Person2id bigint NOT NULL, + PRIMARY KEY (Person1id, Person2id) +); + + +-- materialized views + +-- A recursive materialized view containing the root Post of each Message (for Posts, themselves, for Comments, traversing up the Message thread to the root Post of the tree) +CREATE TABLE Message ( + creationDate timestamp with time zone not null, + MessageId bigint primary key, + RootPostId bigint not null, + RootPostLanguage text, + content text, + imageFile text, + locationIP text not null, + browserUsed text not null, + length int not null, + CreatorPersonId bigint not null, + ContainerForumId bigint, + LocationCountryId bigint not null, + ParentMessageId bigint +); + +CREATE TABLE Person_likes_Message ( + creationDate timestamp with time zone NOT NULL, + PersonId bigint NOT NULL, + MessageId bigint NOT NULL +); + +CREATE TABLE Message_hasTag_Tag ( + creationDate timestamp with time zone NOT NULL, + MessageId bigint NOT NULL, + TagId bigint NOT NULL +); + +-- views + +CREATE VIEW Comment_View AS + SELECT creationDate, MessageId AS id, locationIP, browserUsed, content, length, CreatorPersonId, LocationCountryId, ParentMessageId + FROM Message + WHERE ParentMessageId IS NOT NULL; + +CREATE VIEW Post_View AS + SELECT creationDate, MessageId AS id, imageFile, locationIP, browserUsed, RootPostLanguage, content, length, CreatorPersonId, ContainerForumId, LocationCountryId + From Message + WHERE ParentMessageId IS NULL; diff --git a/duckdb/ddl/schema-delete-candidates.sql b/duckdb/ddl/schema-delete-candidates.sql new file mode 100644 index 00000000..ad4a4966 --- /dev/null +++ b/duckdb/ddl/schema-delete-candidates.sql @@ -0,0 +1,29 @@ +DROP TABLE IF EXISTS Person_Delete_candidates; +DROP TABLE IF EXISTS Forum_Delete_candidates; +DROP TABLE IF EXISTS Comment_Delete_candidates; +DROP TABLE IF EXISTS Post_Delete_candidates; +DROP TABLE IF EXISTS Person_likes_Comment_Delete_candidates; +DROP TABLE IF EXISTS Person_likes_Post_Delete_candidates; +DROP TABLE IF EXISTS Forum_hasMember_Person_Delete_candidates; +DROP TABLE IF EXISTS Person_knows_Person_Delete_candidates; + +CREATE TABLE Person_Delete_candidates (deletionDate timestamp with time zone not null, id bigint not null) WITH (storage = paged); +CREATE TABLE Forum_Delete_candidates (deletionDate timestamp with time zone not null, id bigint not null) WITH (storage = paged); +CREATE TABLE Comment_Delete_candidates (deletionDate timestamp with time zone not null, id bigint not null) WITH (storage = paged); +CREATE TABLE Post_Delete_candidates (deletionDate timestamp with time zone not null, id bigint not null) WITH (storage = paged); +CREATE TABLE Person_likes_Comment_Delete_candidates (deletionDate timestamp with time zone not null, src bigint not null, trg bigint not null) WITH (storage = paged); +CREATE TABLE Person_likes_Post_Delete_candidates (deletionDate timestamp with time zone not null, src bigint not null, trg bigint not null) WITH (storage = paged); +CREATE TABLE Forum_hasMember_Person_Delete_candidates(deletionDate timestamp with time zone not null, src bigint not null, trg bigint not null) WITH (storage = paged); +CREATE TABLE Person_knows_Person_Delete_candidates (deletionDate timestamp with time zone not null, src bigint not null, trg bigint not null) WITH (storage = paged); + +-- If DELETE USING matches multiple times on a single row (in the table-under-deletion), +-- Umbra throws an error: 'more than one row returned by a subquery used as an expression' +-- To prevent this, we use '..._unique' tables which do not contain timestamps (which are +-- not required for performing the delete) and are filtered to distinct id values. +DROP TABLE IF EXISTS Comment_Delete_candidates_unique; +DROP TABLE IF EXISTS Post_Delete_candidates_unique; +DROP TABLE IF EXISTS Forum_Delete_candidates_unique; + +CREATE TABLE Comment_Delete_candidates_unique(id bigint not null) WITH (storage = paged); +CREATE TABLE Post_Delete_candidates_unique(id bigint not null) WITH (storage = paged); +CREATE TABLE Forum_Delete_candidates_unique(deletionDate timestamp with time zone not null, id bigint not null) WITH (storage = paged); diff --git a/duckdb/dml/apply-deletes.sql b/duckdb/dml/apply-deletes.sql new file mode 100644 index 00000000..3bbb8235 --- /dev/null +++ b/duckdb/dml/apply-deletes.sql @@ -0,0 +1,235 @@ +---------------------------------------------------------------------------------------------------- +--------------------------------------- APPLY DELETES ---------------------------------------------- +---------------------------------------------------------------------------------------------------- + +DELETE FROM Comment_Delete_candidates_unique; +DELETE FROM Post_Delete_candidates_unique; +DELETE FROM Forum_Delete_candidates_unique; + +---------------------------------------------------------------------------------------------------- +-- DEL1: Remove Person, its personal Forums, and its Message (sub)threads -------------------------- +---------------------------------------------------------------------------------------------------- +DELETE FROM Person +USING Person_Delete_candidates +WHERE Person_Delete_candidates.id = Person.id +; + +DELETE FROM Person_likes_Message +USING Person_Delete_candidates +WHERE Person_Delete_candidates.id = Person_likes_Message.PersonId +; + +DELETE FROM Person_workAt_Company +USING Person_Delete_candidates +WHERE Person_Delete_candidates.id = Person_workAt_Company.PersonId +; + +DELETE FROM Person_studyAt_University +USING Person_Delete_candidates +WHERE Person_Delete_candidates.id = Person_studyAt_University.PersonId +; + +-- treat KNOWS edges as undirected +DELETE FROM Person_knows_Person +USING Person_Delete_candidates +WHERE Person_Delete_candidates.id = Person_knows_Person.Person1Id +; +DELETE FROM Person_knows_Person +USING Person_Delete_candidates +WHERE Person_Delete_candidates.id = Person_knows_Person.Person2Id +; + +DELETE FROM Person_hasInterest_Tag +USING Person_Delete_candidates +WHERE Person_Delete_candidates.id = Person_hasInterest_Tag.PersonId +; + +DELETE FROM Forum_hasMember_Person +USING Person_Delete_candidates +WHERE Person_Delete_candidates.id = Forum_hasMember_Person.PersonId +; + +UPDATE Forum +SET ModeratorPersonId = NULL +WHERE Forum.title LIKE 'Group %' + AND ModeratorPersonId IN (SELECT id FROM Person_Delete_candidates) +; + +-- offload cascading Forum deletes to DEL4 +INSERT INTO Forum_Delete_candidates +SELECT Person_Delete_candidates.deletionDate AS deletionDate, Forum.id AS id +FROM Person_Delete_candidates +JOIN Forum + ON Forum.ModeratorPersonId = Person_Delete_candidates.id +WHERE Forum.title LIKE 'Album %' + OR Forum.title LIKE 'Wall %' +; + +-- offload cascading Post deletes to DEL6 +INSERT INTO Post_Delete_candidates +SELECT Person_Delete_candidates.deletionDate AS deletionDate, Post_View.id AS id +FROM Person_Delete_candidates +JOIN Post_View + ON Post_View.CreatorPersonId = Person_Delete_candidates.id +; + +-- offload cascading Comment deletes to DEL7 +INSERT INTO Comment_Delete_candidates +SELECT Person_Delete_candidates.deletionDate AS deletionDate, Comment_View.id AS id +FROM Person_Delete_candidates +JOIN Comment_View + ON Comment_View.CreatorPersonId = Person_Delete_candidates.id +; + +---------------------------------------------------------------------------------------------------- +-- DEL2: Remove Post like -------------------------------------------------------------------------- +---------------------------------------------------------------------------------------------------- +DELETE FROM Person_likes_Message +USING Person_likes_Post_Delete_candidates +WHERE Person_likes_Post_Delete_candidates.src = Person_likes_Message.PersonId + AND Person_likes_Post_Delete_candidates.trg = Person_likes_Message.MessageId +; + +---------------------------------------------------------------------------------------------------- +-- DEL3: Remove Comment like ----------------------------------------------------------------------- +---------------------------------------------------------------------------------------------------- +DELETE FROM Person_likes_Message +USING Person_likes_Comment_Delete_candidates +WHERE Person_likes_Comment_Delete_candidates.src = Person_likes_Message.PersonId + AND Person_likes_Comment_Delete_candidates.trg = Person_likes_Message.MessageId +; + +---------------------------------------------------------------------------------------------------- +-- DEL4: Remove Forum and its content -------------------------------------------------------------- +---------------------------------------------------------------------------------------------------- +INSERT INTO Forum_Delete_candidates_unique + SELECT min(deletionDate), id + FROM Forum_Delete_candidates + GROUP BY id +; + +DELETE FROM Forum +USING Forum_Delete_candidates_unique +WHERE Forum.id = Forum_Delete_candidates_unique.id +; + +DELETE FROM Forum_hasMember_Person +USING Forum_Delete_candidates_unique +WHERE Forum_Delete_candidates_unique.id = Forum_hasMember_Person.ForumId +; + +-- offload cascading Post deletes to DEL6 +INSERT INTO Post_Delete_candidates +SELECT Forum_Delete_candidates_unique.deletionDate AS deletionDate, Post_View.id AS id +FROM Post_View +JOIN Forum_Delete_candidates_unique + ON Forum_Delete_candidates_unique.id = Post_View.ContainerForumId +; + +---------------------------------------------------------------------------------------------------- +-- DEL5: Remove Forum membership ------------------------------------------------------------------- +---------------------------------------------------------------------------------------------------- +DELETE FROM Forum_hasMember_Person +USING Forum_hasMember_Person_Delete_candidates +WHERE Forum_hasMember_Person_Delete_candidates.src = Forum_hasMember_Person.ForumId + AND Forum_hasMember_Person_Delete_candidates.trg = Forum_hasMember_Person.PersonId +; + +---------------------------------------------------------------------------------------------------- +-- DEL6: Remove Post thread ------------------------------------------------------------------------ +---------------------------------------------------------------------------------------------------- +INSERT INTO Post_Delete_candidates_unique + SELECT DISTINCT id + FROM Post_Delete_candidates; + +DELETE FROM Message +USING Post_Delete_candidates_unique -- starting from the delete candidate post +WHERE Post_Delete_candidates_unique.id = Message.MessageId +; + +DELETE FROM Person_likes_Message +USING Post_Delete_candidates_unique +WHERE Post_Delete_candidates_unique.id = Person_likes_Message.MessageId +; + +DELETE FROM Post_hasTag_Tag +USING Post_Delete_candidates_unique +WHERE Post_Delete_candidates_unique.id = Post_hasTag_Tag.PostId +; + +-- offload cascading Comment deletes to DEL7 +INSERT INTO Comment_Delete_candidates +SELECT Post_Delete_candidates.deletionDate AS deletionDate, Comment_View.id AS id +FROM Comment_View +JOIN Post_Delete_candidates + ON Post_Delete_candidates.id = Comment_View.ParentMessageId +; + +---------------------------------------------------------------------------------------------------- +-- DEL7: Remove Comment subthread ------------------------------------------------------------------ +---------------------------------------------------------------------------------------------------- +INSERT INTO Comment_Delete_candidates_unique + SELECT DISTINCT id + FROM Comment_Delete_candidates; + +DELETE FROM Message +USING ( + WITH RECURSIVE MessagesToDelete AS ( + SELECT id + FROM Comment_Delete_candidates_unique -- starting from the delete candidate comments + UNION + SELECT ChildComment.MessageId AS id + FROM MessagesToDelete + JOIN Message ChildComment + ON ChildComment.ParentMessageId = MessagesToDelete.id + ) + SELECT id + FROM MessagesToDelete + ) sub +WHERE sub.id = Message.MessageId +; + +DELETE FROM Person_likes_Message +USING ( + WITH RECURSIVE MessagesToDelete AS ( + SELECT id + FROM Comment_Delete_candidates_unique -- starting from the delete candidate comments + UNION ALL + SELECT ChildComment.MessageId AS id + FROM MessagesToDelete + JOIN Message ChildComment + ON ChildComment.ParentMessageId = MessagesToDelete.id + ) + SELECT id + FROM MessagesToDelete + ) sub +WHERE sub.id = Person_likes_Message.MessageId +; + +DELETE FROM Comment_hasTag_Tag +USING ( + WITH RECURSIVE MessagesToDelete AS ( + SELECT id + FROM Comment_Delete_candidates_unique -- starting from the delete candidate comments + UNION ALL + SELECT ChildComment.MessageId AS id + FROM MessagesToDelete + JOIN Message ChildComment + ON ChildComment.ParentMessageId = MessagesToDelete.id + ) + SELECT id + FROM MessagesToDelete + ) sub +WHERE sub.id = Comment_hasTag_Tag.CommentId +; + +---------------------------------------------------------------------------------------------------- +-- DEL8: Remove friendship ------------------------------------------------------------------------- +---------------------------------------------------------------------------------------------------- +DELETE FROM Person_knows_Person +USING Person_knows_Person_Delete_candidates +WHERE +least(Person_knows_Person.Person1Id, Person_knows_Person.Person2Id) = least(Person_knows_Person_Delete_candidates.src, Person_knows_Person_Delete_candidates.trg) +AND +greatest(Person_knows_Person.Person1Id, Person_knows_Person.Person2Id) = greatest(Person_knows_Person_Delete_candidates.src, Person_knows_Person_Delete_candidates.trg) +; diff --git a/duckdb/dml/create-static-materialized-views.sql b/duckdb/dml/create-static-materialized-views.sql new file mode 100644 index 00000000..ee15f376 --- /dev/null +++ b/duckdb/dml/create-static-materialized-views.sql @@ -0,0 +1,23 @@ +INSERT INTO Country + SELECT id, name, url, PartOfPlaceId AS PartOfContinentId + FROM Place + WHERE type = 'Country' +; + +INSERT INTO City + SELECT id, name, url, PartOfPlaceId AS PartOfCountryId + FROM Place + WHERE type = 'City' +; + +INSERT INTO Company + SELECT id, name, url, LocationPlaceId AS LocatedInCountryId + FROM Organisation + WHERE type = 'Company' +; + +INSERT INTO University + SELECT id, name, url, LocationPlaceId AS LocatedInCityId + FROM Organisation + WHERE type = 'University' +; diff --git a/duckdb/dml/maintain-views.sql b/duckdb/dml/maintain-views.sql new file mode 100644 index 00000000..245e9626 --- /dev/null +++ b/duckdb/dml/maintain-views.sql @@ -0,0 +1,68 @@ +-- maintain materialized views + +-- Comments attaching to existing Message trees +INSERT INTO Message + WITH RECURSIVE Message_CTE(MessageId, RootPostId, RootPostLanguage, ContainerForumId, ParentMessageId) AS ( + -- first half of the union: Comments attaching directly to the existing tree + SELECT + Comment.id AS MessageId, + Message.RootPostId AS RootPostId, + Message.RootPostLanguage AS RootPostLanguage, + Message.ContainerForumId AS ContainerForumId, + coalesce(Comment.ParentPostId, Comment.ParentCommentId) AS ParentMessageId + FROM Comment + JOIN Message + ON Message.MessageId = coalesce(Comment.ParentPostId, Comment.ParentCommentId) + UNION ALL + -- second half of the union: Comments attaching newly inserted Comments + SELECT + Comment.id AS MessageId, + Message_CTE.RootPostId AS RootPostId, + Message_CTE.RootPostLanguage AS RootPostLanguage, + Message_CTE.ContainerForumId AS ContainerForumId, + Comment.ParentCommentId AS ParentMessageId + FROM Comment + JOIN Message_CTE + -- FORCEORDER is an optimizer hint, + -- see https://github.com/ldbc/ldbc_snb_bi/issues/163#issuecomment-1621316001 + ON FORCEORDER(Comment.ParentCommentId = Message_CTE.MessageId) + ) + SELECT + Comment.creationDate AS creationDate, + Comment.id AS MessageId, + Message_CTE.RootPostId AS RootPostId, + Message_CTE.RootPostLanguage AS RootPostLanguage, + Comment.content AS content, + NULL::text AS imageFile, + Comment.locationIP AS locationIP, + Comment.browserUsed AS browserUsed, + Comment.length AS length, + Comment.CreatorPersonId AS CreatorPersonId, + Message_CTE.ContainerForumId AS ContainerForumId, + Comment.LocationCountryId AS LocationCityId, + coalesce(Comment.ParentPostId, Comment.ParentCommentId) AS ParentMessageId + FROM Message_CTE + JOIN Comment + -- FORCEORDER is an optimizer hint, + -- see https://github.com/ldbc/ldbc_snb_bi/issues/163#issuecomment-1621316001 + ON FORCEORDER(Message_CTE.MessageId = Comment.id) +; + +DROP TABLE IF EXISTS Comment; + +---------------------------------------------------------------------------------------------------- +------------------------------------------- CLEANUP ------------------------------------------------ +---------------------------------------------------------------------------------------------------- + +CREATE TABLE Comment ( + creationDate timestamp with time zone NOT NULL, + id bigint NOT NULL, --PRIMARY KEY, + locationIP text NOT NULL, + browserUsed text NOT NULL, + content text NOT NULL, + length int NOT NULL, + CreatorPersonId bigint NOT NULL, + LocationCountryId bigint NOT NULL, + ParentPostId bigint, + ParentCommentId bigint +) WITH (storage = paged); diff --git a/duckdb/dml/precomp/bi-19.sql b/duckdb/dml/precomp/bi-19.sql new file mode 100644 index 00000000..7ff02d82 --- /dev/null +++ b/duckdb/dml/precomp/bi-19.sql @@ -0,0 +1,22 @@ + +DROP TABLE IF EXISTS PathQ19; +CREATE TABLE PathQ19 ( + src bigint not null, + dst bigint not null, + w double precision not null +) with (storage = paged); +INSERT INTO PathQ19(src, dst, w) +WITH +weights(src, dst, w) AS ( + SELECT + person1id AS src, + person2id AS dst, + greatest(round(40 - sqrt(count(*)))::bigint, 1) AS w + FROM (SELECT person1id, person2id FROM Person_knows_person WHERE person1id < person2id) pp, Message m1, Message m2 + WHERE pp.person1id = least(m1.creatorpersonid, m2.creatorpersonid) and pp.person2id = greatest(m1.creatorpersonid, m2.creatorpersonid) and m1.parentmessageid = m2.messageid and m1.creatorpersonid <> m2.creatorpersonid + GROUP BY src, dst +) +SELECT src, dst, w FROM weights +UNION ALL +SELECT dst, src, w FROM weights; +ALTER TABLE PathQ19 ADD PRIMARY KEY (src, dst); diff --git a/duckdb/dml/precomp/bi-20.sql b/duckdb/dml/precomp/bi-20.sql new file mode 100644 index 00000000..a9254d71 --- /dev/null +++ b/duckdb/dml/precomp/bi-20.sql @@ -0,0 +1,13 @@ + +DROP TABLE IF EXISTS PathQ20; +CREATE TABLE PathQ20 ( + src bigint not null, + dst bigint not null, + w int not null +) with (storage = paged); +INSERT INTO PathQ20(src, dst, w) +select p1.personid, p2.personid, min(abs(p1.classYear - p2.classYear)) + 1 +from Person_knows_person pp, Person_studyAt_University p1, Person_studyAt_University p2 +where pp.person1id = p1.personid and pp.person2id = p2.personid and p1.universityid = p2.universityid +group by p1.personid, p2.personid; +ALTER TABLE PathQ20 ADD PRIMARY KEY (src, dst); diff --git a/duckdb/dml/precomp/bi-4.sql b/duckdb/dml/precomp/bi-4.sql new file mode 100644 index 00000000..a5e6f2db --- /dev/null +++ b/duckdb/dml/precomp/bi-4.sql @@ -0,0 +1,22 @@ + +DROP TABLE IF EXISTS Top100PopularForumsQ04; +CREATE TABLE Top100PopularForumsQ04( + id bigint not null, + creationDate timestamp with time zone NOT NULL, + maxNumberOfMembers bigint not null +) with (storage = paged); +INSERT INTO Top100PopularForumsQ04(id, creationDate, maxNumberOfMembers) +SELECT T.id, Forum.creationdate, T.maxNumberOfMembers +FROM (SELECT ForumId AS id, max(numberOfMembers) AS maxNumberOfMembers +FROM ( +SELECT Forum_hasMember_Person.ForumId AS ForumId, count(Person.id) AS numberOfMembers, City.PartOfCountryId AS CountryId + FROM Forum_hasMember_Person + JOIN Person + ON Person.id = Forum_hasMember_Person.PersonId + JOIN City + ON City.id = Person.LocationCityId + GROUP BY City.PartOfCountryId, Forum_hasMember_Person.ForumId +) ForumMembershipPerCountry +GROUP BY ForumId) T, Forum +WHERE T.id = Forum.id; +ALTER TABLE Top100PopularForumsQ04 ADD PRIMARY KEY (id); diff --git a/duckdb/dml/precomp/bi-6.sql b/duckdb/dml/precomp/bi-6.sql new file mode 100644 index 00000000..b18c662c --- /dev/null +++ b/duckdb/dml/precomp/bi-6.sql @@ -0,0 +1,15 @@ + +DROP TABLE IF EXISTS PopularityScoreQ06; +CREATE TABLE PopularityScoreQ06 ( + person2id bigint not null, + popularityScore bigint not null +) with (storage = paged); +INSERT INTO PopularityScoreQ06(person2id, popularityScore) +SELECT + message2.CreatorPersonId AS person2id, + count(*) AS popularityScore +FROM Message message2 +JOIN Person_likes_Message like2 + ON like2.MessageId = message2.MessageId +GROUP BY message2.CreatorPersonId; +ALTER TABLE PopularityScoreQ06 ADD PRIMARY KEY (person2id); diff --git a/duckdb/load.py b/duckdb/load.py new file mode 100644 index 00000000..3781c3c6 --- /dev/null +++ b/duckdb/load.py @@ -0,0 +1,171 @@ +import duckdb +import psycopg2 +import sys +import os +import re +import time +import argparse +from queries import run_script + + +parser = argparse.ArgumentParser() +parser.add_argument("--data_dir", type=str, help="Directory with the initial_snapshot, insert, and delete directories", required=True) +args = parser.parse_args() +data_dir = args.data_dir + +con = duckdb.connect("ldbc.duckdb") + +run_script(con, "ddl/drop-tables.sql") +run_script(con, "ddl/schema-composite-merged-fk.sql") +#run_script(pg_con, con, "ddl/schema-delete-candidates.sql") + + +print("Load initial snapshot") + +# initial snapshot +static_path = f"{data_dir}/initial_snapshot/static" +dynamic_path = f"{data_dir}/initial_snapshot/dynamic" +static_entities = ["Organisation", "Place", "Tag", "TagClass"] +dynamic_entities = ["Comment", "Post", "Forum", "Forum_hasMember_Person", "Forum_hasTag_Tag", "Person", "Person_hasInterest_Tag", "Person_knows_Person", "Person_studyAt_University", "Person_workAt_Company", "Comment_hasTag_Tag", "Post_hasTag_Tag", "Person_likes_Comment", "Person_likes_Post"] + +print("## Static entities") +for entity in static_entities: + print(f"- {entity}") + for csv_file in [f for f in os.listdir(f"{static_path}/{entity}") if f.startswith("part-") and f.endswith(".csv.gz")]: + print(f" - {csv_file}") + con.execute(f"COPY {entity} FROM '{static_path}/{entity}/{csv_file}' (DELIMITER '|', HEADER, NULL '', FORMAT csv)") +print("Loaded static entities.") + +print("## Dynamic entities") +for entity in dynamic_entities: + print(f"- {entity}") + for csv_file in [f for f in os.listdir(f"{dynamic_path}/{entity}") if f.startswith("part-") and f.endswith(".csv.gz")]: + print(f" - {csv_file}") + con.execute(f"COPY {entity} FROM '{dynamic_path}/{entity}/{csv_file}' (DELIMITER '|', HEADER, NULL '', FORMAT csv)") + if entity == "Person_knows_Person": + con.execute(f"COPY {entity} (creationDate, Person2id, Person1id) FROM '{dynamic_path}/{entity}/{csv_file}' (DELIMITER '|', HEADER, NULL '', FORMAT csv)") + +print("Create materialized views") + +# insert posts to message +con.execute(""" + INSERT INTO Message + SELECT + creationDate, + id AS MessageId, + id AS RootPostId, + language AS RootPostLanguage, + content, + imageFile, + locationIP, + browserUsed, + length, + CreatorPersonId, + ContainerForumId, + LocationCountryId, + NULL::bigint AS ParentMessageId + FROM Post + """ +) + +# insert comments to message +con.execute(""" + INSERT INTO Message + WITH RECURSIVE Message_CTE(MessageId, RootPostId, RootPostLanguage, ContainerForumId, ParentMessageId) AS ( + -- first half of the union: Comments attaching directly to the existing tree + SELECT + Comment.id AS MessageId, + Message.RootPostId AS RootPostId, + Message.RootPostLanguage AS RootPostLanguage, + Message.ContainerForumId AS ContainerForumId, + coalesce(Comment.ParentPostId, Comment.ParentCommentId) AS ParentMessageId + FROM Comment + JOIN Message + ON Message.MessageId = coalesce(Comment.ParentPostId, Comment.ParentCommentId) + UNION ALL + -- second half of the union: Comments attaching newly inserted Comments + SELECT + Comment.id AS MessageId, + Message_CTE.RootPostId AS RootPostId, + Message_CTE.RootPostLanguage AS RootPostLanguage, + Message_CTE.ContainerForumId AS ContainerForumId, + Comment.ParentCommentId AS ParentMessageId + FROM Comment + JOIN Message_CTE + ON Comment.ParentCommentId = Message_CTE.MessageId + ) + SELECT + Comment.creationDate AS creationDate, + Comment.id AS MessageId, + Message_CTE.RootPostId AS RootPostId, + Message_CTE.RootPostLanguage AS RootPostLanguage, + Comment.content AS content, + NULL::text AS imageFile, + Comment.locationIP AS locationIP, + Comment.browserUsed AS browserUsed, + Comment.length AS length, + Comment.CreatorPersonId AS CreatorPersonId, + Message_CTE.ContainerForumId AS ContainerForumId, + Comment.LocationCountryId AS LocationCityId, + coalesce(Comment.ParentPostId, Comment.ParentCommentId) AS ParentMessageId + FROM Message_CTE + JOIN Comment + ON Message_CTE.MessageId = Comment.id; + """ +) + +con.execute(""" + INSERT INTO Message_hasTag_Tag + SELECT creationDate, PostId, TagId + FROM Post_hasTag_Tag; + """) + +con.execute(""" + INSERT INTO Message_hasTag_Tag + SELECT creationDate, CommentId, TagId + FROM Comment_hasTag_Tag; + """) + +con.execute(""" + INSERT INTO Person_likes_Message + SELECT creationDate, PersonId, PostId + FROM Person_likes_Post; + """) + +con.execute(""" + INSERT INTO Person_likes_Message + SELECT creationDate, PersonId, CommentId + FROM Person_likes_Comment; + """) + +con.execute(""" + INSERT INTO Country + SELECT id, name, url, PartOfPlaceId AS PartOfContinentId + FROM Place + WHERE type = 'Country' + ; + """) + +con.execute(""" + INSERT INTO City + SELECT id, name, url, PartOfPlaceId AS PartOfCountryId + FROM Place + WHERE type = 'City' + ; + """) + +con.execute(""" + INSERT INTO Company + SELECT id, name, url, LocationPlaceId AS LocatedInCountryId + FROM Organisation + WHERE type = 'Company' + ; + """) + +con.execute(""" + INSERT INTO University + SELECT id, name, url, LocationPlaceId AS LocatedInCityId + FROM Organisation + WHERE type = 'University' + ; + """) diff --git a/duckdb/output/.gitignore b/duckdb/output/.gitignore new file mode 100644 index 00000000..d6b7ef32 --- /dev/null +++ b/duckdb/output/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/duckdb/queries.py b/duckdb/queries.py new file mode 100644 index 00000000..59081bbd --- /dev/null +++ b/duckdb/queries.py @@ -0,0 +1,24 @@ +import datetime +import json +import os +import re +import time + +def run_script(con, filename): + with open(filename, "r") as f: + queries_file = f.read() + # strip comments + queries_file = re.sub(r"\n--.*", "", queries_file) + queries = queries_file.split(";") + for query in queries: + if query == "" or query.isspace(): + continue + + sql_statement = re.findall(r"^((CREATE|INSERT|DROP|DELETE|SELECT|COPY|UPDATE|ALTER) [A-Za-z0-9_ ]*)", query, re.MULTILINE) + is_update = True if re.match(r"^((CREATE|INSERT|DROP|DELETE|COPY|UPDATE|ALTER) [A-Za-z0-9_ ]*)", sql_statement[0][0], re.MULTILINE) else False + + print(f"{sql_statement[0][0].strip()} ...") + con.execute(query) + start = time.time() + end = time.time() + duration = end - start diff --git a/duckdb/queries/bi-1.sql b/duckdb/queries/bi-1.sql new file mode 100644 index 00000000..6046f636 --- /dev/null +++ b/duckdb/queries/bi-1.sql @@ -0,0 +1,33 @@ +/* Q1. Posting summary +\set datetime '\'2011-12-01T00:00:00.000+00:00\''::timestamp + */ +WITH + message_count AS ( + SELECT 0.0 + count(*) AS cnt + FROM Message + WHERE creationDate < :datetime +) +, message_prep AS ( + SELECT extract(year from creationDate) AS messageYear + , ParentMessageId IS NOT NULL AS isComment + , CASE + WHEN length < 40 THEN 0 -- short + WHEN length < 80 THEN 1 -- one liner + WHEN length < 160 THEN 2 -- tweet + ELSE 3 -- long + END AS lengthCategory + , length + FROM Message + WHERE creationDate < :datetime + AND content IS NOT NULL +) +SELECT messageYear, isComment, lengthCategory + , count(*) AS messageCount + , avg(length::bigint) AS averageMessageLength + , sum(length::bigint) AS sumMessageLength + , count(*) / mc.cnt AS percentageOfMessages + FROM message_prep + , message_count mc + GROUP BY messageYear, isComment, lengthCategory, mc.cnt + ORDER BY messageYear DESC, isComment ASC, lengthCategory ASC +; diff --git a/duckdb/queries/bi-10.sql b/duckdb/queries/bi-10.sql new file mode 100644 index 00000000..4d3680e7 --- /dev/null +++ b/duckdb/queries/bi-10.sql @@ -0,0 +1,77 @@ +/* Q10. Experts in social circle +\set personId 30786325588624 +\set country '\'China\'' +\set tagClass '\'MusicalArtist\'' +\set minPathDistance 3 -- fixed value +\set maxPathDistance 4 -- fixed value + */ +WITH friends AS + (SELECT Person2Id + FROM Person_knows_Person + WHERE Person1Id = :personId + ) + , friends_of_friends AS + (SELECT knowsB.Person2Id AS Person2Id + FROM friends + JOIN Person_knows_Person knowsB + ON friends.Person2Id = knowsB.Person1Id + ) + , friends_and_friends_of_friends AS + (SELECT Person2Id + FROM friends + UNION -- using plain UNION to eliminate duplicates + SELECT Person2Id + FROM friends_of_friends + ) + , friends_between_3_and_4_hops AS ( + -- people reachable through 1..4 hops + (SELECT DISTINCT knowsD.Person2Id AS Person2Id + FROM friends_and_friends_of_friends ffoaf + JOIN Person_knows_Person knowsC + ON knowsC.Person1Id = ffoaf.Person2Id + JOIN Person_knows_Person knowsD + ON knowsD.Person1Id = knowsC.Person2Id + ) + -- removing people reachable through 1..2 hops, yielding the ones reachable through 3..4 hops + EXCEPT + (SELECT Person2Id + FROM friends_and_friends_of_friends + ) + ) + , friend_list AS ( + SELECT f.person2Id AS friendId + FROM friends_between_3_and_4_hops f + JOIN Person tf -- the friend's person record + ON tf.id = f.person2Id + JOIN City + ON City.id = tf.LocationCityId + JOIN Country + ON Country.id = City.PartOfCountryId + AND Country.name = :country + ) + , messages_of_tagclass_by_friends AS ( + SELECT DISTINCT f.friendId + , Message.MessageId AS messageid + FROM friend_list f + JOIN Message + ON Message.CreatorPersonId = f.friendId + JOIN Message_hasTag_Tag + ON Message_hasTag_Tag.MessageId = Message.MessageId + JOIN Tag + ON Tag.id = Message_hasTag_Tag.TagId + JOIN TagClass + ON TagClass.id = Tag.TypeTagClassId + WHERE TagClass.name = :tagClass + ) +SELECT m.friendId AS "person.id" + , Tag.name AS "tag.name" + , count(*) AS messageCount + FROM messages_of_tagclass_by_friends m + JOIN Message_hasTag_Tag + ON Message_hasTag_Tag.MessageId = m.MessageId + JOIN Tag + ON Tag.id = Message_hasTag_Tag.TagId + GROUP BY m.friendId, Tag.name + ORDER BY messageCount DESC, Tag.name, m.friendId + LIMIT 100 +; diff --git a/duckdb/queries/bi-11.sql b/duckdb/queries/bi-11.sql new file mode 100644 index 00000000..9d2e2c98 --- /dev/null +++ b/duckdb/queries/bi-11.sql @@ -0,0 +1,34 @@ +/* Q11. Friend triangles +\set country '\'China\'' +\set startDate '\'2010-06-01\''::timestamp +\set endDate '\'2010-07-01\''::timestamp + */ +WITH Persons_of_country_w_friends AS ( + SELECT Person.id AS PersonId + , Person_knows_Person.Person2Id AS FriendId + , Person_knows_Person.creationDate AS creationDate + FROM Person + JOIN City + ON City.id = Person.LocationCityId + JOIN Country + ON Country.id = City.PartOfCountryId + AND Country.name = :country + JOIN Person_knows_Person + ON Person_knows_Person.Person1Id = Person.id +) +SELECT count(*) + FROM Persons_of_country_w_friends p1 + JOIN Persons_of_country_w_friends p2 + ON p1.FriendId = p2.PersonId + JOIN Persons_of_country_w_friends p3 + ON p2.FriendId = p3.PersonId + AND p3.FriendId = p1.PersonId + WHERE true + -- filter: unique triangles only + AND p1.PersonId < p2.PersonId + AND p2.PersonId < p3.PersonId + -- filter: only edges created after :startDate and before :endDate + AND :startDate <= p1.creationDate AND p1.creationDate <= :endDate + AND :startDate <= p2.creationDate AND p2.creationDate <= :endDate + AND :startDate <= p3.creationDate AND p3.creationDate <= :endDate +; diff --git a/duckdb/queries/bi-12.sql b/duckdb/queries/bi-12.sql new file mode 100644 index 00000000..4c5ee2f5 --- /dev/null +++ b/duckdb/queries/bi-12.sql @@ -0,0 +1,26 @@ +/* Q12. How many persons have a given number of messages +\set startDate '\'2010-07-22\''::timestamp +\set lengthThreshold '20' +\set languages '\'{"ar", "hu"}\''::varchar[] + */ +WITH person_w_posts AS ( + SELECT Person.id, count(Message.MessageId) as messageCount + FROM Person + LEFT JOIN Message + ON Person.id = Message.CreatorPersonId + AND Message.content IS NOT NULL + AND Message.length < :lengthThreshold + AND Message.creationDate > :startDate + AND Message.RootPostLanguage IN :languages + GROUP BY Person.id +) +, message_count_distribution AS ( + SELECT pp.messageCount, count(*) as personCount + FROM person_w_posts pp + GROUP BY pp.messageCount + ORDER BY personCount DESC, messageCount DESC +) +SELECT * + FROM message_count_distribution +ORDER BY personCount DESC, messageCount DESC +; diff --git a/duckdb/queries/bi-13.sql b/duckdb/queries/bi-13.sql new file mode 100644 index 00000000..d2cee68c --- /dev/null +++ b/duckdb/queries/bi-13.sql @@ -0,0 +1,36 @@ +/* Q13. Zombies in a country +\set country '\'France\'' +\set endDate '\'2013-01-01\''::timestamp + */ +WITH Zombies AS ( + SELECT Person.id AS zombieid + FROM Country + JOIN City + ON City.PartOfCountryId = Country.id + JOIN Person + ON Person.LocationCityId = City.id + LEFT JOIN Message + ON Person.id = Message.CreatorPersonId + AND Message.creationDate BETWEEN Person.creationDate AND :endDate -- the lower bound is an optmization to prune messages + WHERE Country.name = :country + AND Person.creationDate < :endDate + GROUP BY Person.id, Person.creationDate + -- average of [0, 1) messages per month is equivalent with having less messages than the month span between person creationDate and parameter :endDate + HAVING count(Message.MessageId) < 12*extract(YEAR FROM :endDate) + extract(MONTH FROM :endDate) + - (12*extract(YEAR FROM Person.creationDate) + extract(MONTH FROM Person.creationDate)) + + 1 +) +SELECT Z.zombieid AS "zombie.id" + , coalesce(t.zombieLikeCount, 0) AS zombieLikeCount + , coalesce(t.totalLikeCount, 0) AS totalLikeCount + , CASE WHEN t.totalLikeCount > 0 THEN t.zombieLikeCount::float/t.totalLikeCount ELSE 0 END AS zombieScore + FROM Zombies Z LEFT JOIN ( + SELECT Z.zombieid, count(*) as totalLikeCount, sum(case when exists (SELECT 1 FROM Zombies ZL WHERE ZL.zombieid = p.id) then 1 else 0 end) AS zombieLikeCount + FROM Person p, Person_likes_Message plm, Message m, Zombies Z + WHERE Z.zombieid = m.CreatorPersonId AND p.creationDate < :endDate + AND p.id = plm.PersonId AND m.MessageId = plm.MessageId + GROUP BY Z.zombieid + ) t ON (Z.zombieid = t.zombieid) + ORDER BY zombieScore DESC, Z.zombieid + LIMIT 100 +; diff --git a/duckdb/queries/bi-14.sql b/duckdb/queries/bi-14.sql new file mode 100644 index 00000000..b81edfb3 --- /dev/null +++ b/duckdb/queries/bi-14.sql @@ -0,0 +1,62 @@ +/* Q14. International dialog +\set country1 '\'Chile\'' +\set country2 '\'Argentina\'' + */ +WITH PersonPairCandidates AS ( + SELECT Person1.id AS Person1Id + , Person2.id AS Person2Id + , City1.id AS cityId + , City1.name AS cityName + FROM Country Country1 + JOIN City City1 + ON City1.PartOfCountryId = Country1.id + JOIN Person Person1 + ON Person1.LocationCityId = City1.id + JOIN Person_knows_Person + ON Person_knows_Person.Person1Id = Person1.id + JOIN Person Person2 + ON Person2.id = Person_knows_Person.Person2Id + JOIN City City2 + ON Person2.LocationCityId = City2.id + JOIN Country Country2 + ON Country2.id = City2.PartOfCountryId + WHERE Country1.name = :country1 + AND Country2.name = :country2 +) +, PPC(Person1Id, Person2Id, Flipped) AS ( + SELECT Person1Id AS Person1Id, Person2Id AS Person2Id, false AS Flipped FROM PersonPairCandidates + UNION ALL + SELECT Person2Id AS Person1Id, Person1Id AS Person2Id, true As Flipped FROM PersonPairCandidates +) +, pair_scores AS ( + SELECT CASE WHEN Flipped THEN Person2Id ELSE Person1Id END AS Person1Id, + CASE WHEN Flipped THEN Person1Id ELSE Person2Id END AS Person2Id, + ( + CASE WHEN EXISTS (SELECT 1 FROM Message m, Message r WHERE m.MessageId = r.ParentMessageId AND Person1Id = r.CreatorPersonId AND Person2Id = m.CreatorPersonId AND EXISTS (SELECT 1 FROM PPC x WHERE x.Person1Id = r.CreatorPersonId)) THEN (CASE WHEN Flipped THEN 1 ELSE 4 END) ELSE 0 END + + CASE WHEN EXISTS (SELECT 1 FROM Message m, Person_likes_Message l WHERE Person2Id = m.CreatorPersonId AND m.MessageId = l.MessageId AND l.PersonId = Person1Id AND EXISTS (SELECT 1 FROM PPC x WHERE x.Person1Id = l.PersonId)) THEN (CASE WHEN Flipped THEN 1 ELSE 10 END) ELSE 0 END + ) as score + FROM PPC +) +, pair_scoresX AS ( + SELECT Person1Id, Person2Id, sum(score) as score + FROM pair_scores + GROUP BY Person1Id, Person2Id +) +, score_ranks AS ( + SELECT DISTINCT ON (cityId) + PersonPairCandidates.Person1Id, PersonPairCandidates.Person2Id, cityId, cityName + , s.score AS score + FROM PersonPairCandidates + LEFT JOIN pair_scoresX s + ON s.Person1Id = PersonPairCandidates.Person1Id + AND s.person2Id = PersonPairCandidates.Person2Id + ORDER BY cityId, s.score DESC, PersonPairCandidates.Person1Id, PersonPairCandidates.Person2Id +) +SELECT score_ranks.Person1Id AS "person1.id" + , score_ranks.Person2Id AS "person2.id" + , score_ranks.cityName AS "city1.name" + , score_ranks.score + FROM score_ranks + ORDER BY score_ranks.score DESC, score_ranks.Person1Id, score_ranks.Person2Id + LIMIT 100 +; diff --git a/duckdb/queries/bi-15.sql b/duckdb/queries/bi-15.sql new file mode 100644 index 00000000..79245ffc --- /dev/null +++ b/duckdb/queries/bi-15.sql @@ -0,0 +1,105 @@ +/* Q15. Trusted connection paths through forums created in a given timeframe +\set person1Id 21990232564808 +\set person2Id 26388279076936 +\set startDate '\'2010-11-01\''::timestamp +\set endDate '\'2010-12-01\''::timestamp + */ +with recursive +srcs(f) as (select :person1Id), +dsts(t) as (select :person2Id), +myForums(id) as ( + select id from Forum f where f.creationDate between :startDate and :endDate +), +mm as ( + select least(msg.CreatorPersonId, reply.CreatorPersonId) as src, greatest(msg.CreatorPersonId, reply.CreatorPersonId) as dst, sum(case when msg.ParentMessageId is null then 10 else 5 end) as w + from Person_knows_Person pp, Message msg, Message reply + where true + and pp.person1id = msg.CreatorPersonId + and pp.person2id = reply.CreatorPersonId + and reply.ParentMessageId = msg.MessageId + and exists (select * from myForums f where f.id = msg.containerforumid) + and exists (select * from myForums f where f.id = reply.containerforumid) + group by src, dst +), +path(src, dst, w) as ( + select pp.person1id, pp.person2id, 10::double precision / (coalesce(w, 0) + 10) + from Person_knows_Person pp left join mm on least(pp.person1id, pp.person2id) = mm.src and greatest(pp.person1id, pp.person2id) = mm.dst +), +-- bidirectional bfs for nonexistant paths +pexists(src, dir) as ( + ( + select f, true from srcs + union all + select t, false from dsts + ) + union + ( + with + ss(src, dir) as (select src, dir from pexists), + ns(src, dir) as (select p.dst, ss.dir from ss, path p where ss.src = p.src), + bb(src, dir) as (select src, dir from ns union all select src, dir from ss), + found as ( + select 1 + from bb b1, bb b2 + where b1.dir and (not b2.dir) and b1.src = b2.src + ) + select src, dir + from ns + where not exists (select 1 from found) + union all + select -1, true + where exists (select 1 from found) + ) +), +pathfound(c) as ( + select 1 + from pexists + where src = -1 and dir +), +shorts(dir, gsrc, dst, w, dead, iter) as ( + ( + select false, f, f, 0::double precision, false, 0 from srcs where exists (select 1 from pathfound) + union all + select true, t, t, 0::double precision, false, 0 from dsts where exists (select 1 from pathfound) + ) + union all + ( + with + ss as (select * from shorts), + toExplore as (select * from ss where dead = false order by w limit 1000), + -- assumes graph is undirected + newPoints(dir, gsrc, dst, w, dead) as ( + select e.dir, e.gsrc as gsrc, p.dst as dst, e.w + p.w as w, false as dead + from path p join toExplore e on (e.dst = p.src) + union all + select dir, gsrc, dst, w, dead or exists (select * from toExplore e where e.dir = o.dir and e.gsrc = o.gsrc and e.dst = o.dst) from ss o + ), + fullTable as ( + select distinct on(dir, gsrc, dst) dir, gsrc, dst, w, dead + from newPoints + order by dir, gsrc, dst, w, dead desc + ), + found as ( + select min(l.w + r.w) as w + from fullTable l, fullTable r + where l.dir = false and r.dir = true and l.dst = r.dst + ) + select dir, + gsrc, + dst, + w, + dead or (coalesce(t.w > (select f.w/2 from found f), false)), + e.iter + 1 as iter + from fullTable t, (select iter from toExplore limit 1) e + ) +), +ss(dir, gsrc, dst, w, iter) as ( + select dir, gsrc, dst, w, iter from shorts where iter = (select max(iter) from shorts) +), +results(f, t, w) as ( + select l.gsrc, r.gsrc, min(l.w + r.w) + from ss l, ss r + where l.dir = false and r.dir = true and l.dst = r.dst + group by l.gsrc, r.gsrc +) +select coalesce(min(w), -1) from results; diff --git a/duckdb/queries/bi-16.sql b/duckdb/queries/bi-16.sql new file mode 100644 index 00000000..bc0532ed --- /dev/null +++ b/duckdb/queries/bi-16.sql @@ -0,0 +1,68 @@ +/* Q16. Fake news detection +\set tagA '\'Augustine_of_Hippo\'' +\set dateA '\'2011-10-10\''::timestamp +\set tagB '\'Manuel_Noriega\'' +\set dateB '\'2012-06-02\''::timestamp +\set maxKnowsLimit '5' + */ +WITH + subgraphA AS ( + SELECT DISTINCT Person.id AS PersonId, Message.MessageId AS MessageId + FROM Person + JOIN Message + ON Message.CreatorPersonId = Person.id + AND Message.creationDate::date = :dateA + JOIN Message_hasTag_Tag + ON Message_hasTag_Tag.MessageId = Message.MessageId + JOIN Tag + ON Tag.id = Message_hasTag_Tag.TagId + AND Tag.name = :tagA + ), + personA AS ( + SELECT + subgraphA1.PersonId, + count(DISTINCT subgraphA1.MessageId) AS cm, + count(DISTINCT Person_knows_Person.Person2Id) AS cp2 + FROM subgraphA subgraphA1 + LEFT JOIN Person_knows_Person + ON Person_knows_Person.Person1Id = subgraphA1.PersonId + AND Person_knows_Person.Person2Id IN (SELECT PersonId FROM subgraphA) + GROUP BY subgraphA1.PersonId + HAVING count(DISTINCT Person_knows_Person.Person2Id) <= :maxKnowsLimit + ORDER BY subgraphA1.PersonId ASC + ), + subgraphB AS ( + SELECT DISTINCT Person.id AS PersonId, Message.MessageId AS MessageId + FROM Person + JOIN Message + ON Message.CreatorPersonId = Person.id + AND Message.creationDate::date = :dateB + JOIN Message_hasTag_Tag + ON Message_hasTag_Tag.MessageId = Message.MessageId + JOIN Tag + ON Tag.id = Message_hasTag_Tag.TagId + AND Tag.name = :tagB + ), + personB AS ( + SELECT + subgraphB1.PersonId, + count(DISTINCT subgraphB1.MessageId) AS cm, + count(DISTINCT Person_knows_Person.Person2Id) AS cp2 + FROM subgraphB subgraphB1 + LEFT JOIN Person_knows_Person + ON Person_knows_Person.Person1Id = subgraphB1.PersonId + AND Person_knows_Person.Person2Id IN (SELECT PersonId FROM subgraphB) + GROUP BY subgraphB1.PersonId + HAVING count(DISTINCT Person_knows_Person.Person2Id) <= :maxKnowsLimit + ORDER BY subgraphB1.PersonId ASC + ) +SELECT + personA.PersonId AS PersonId, + personA.cm AS messageCountA, + personB.cm AS messageCountB +FROM personA +JOIN personB + ON personB.PersonId = personA.PersonId +ORDER BY personA.cm + personB.cm DESC, PersonId ASC +LIMIT 20 +; diff --git a/duckdb/queries/bi-17.sql b/duckdb/queries/bi-17.sql new file mode 100644 index 00000000..12a179a7 --- /dev/null +++ b/duckdb/queries/bi-17.sql @@ -0,0 +1,39 @@ +/* Q17. Information propagation analysis +\set tag '\'Slavoj_Žižek\'' +\set delta '4' + */ +WITH MyMessage as ( + SELECT * + FROM Message +-- (tag)<-[:HAS_TAG]-(message) + WHERE MessageId in (SELECT MessageId FROM Message_hasTag_Tag WHERE TagId IN (SELECT id FROM Tag WHERE Tag.name = 'Slavoj_Žižek')) +) +-- (message1)-[:HAS_CREATOR]->(person1) +SELECT Message1.CreatorPersonId AS "person1.id", count(DISTINCT Message2.MessageId) AS messageCount +FROM MyMessage Message1 +-- (message2 }) +JOIN MyMessage Message2 + ON (Message1.creationDate + '4 hour'::interval) < Message2.creationDate +JOIN MyMessage Comment + ON Comment.ParentMessageId = Message2.MessageId +-- (forum1)-[:Has_MEMBER]->(person2) +JOIN Forum_hasMember_Person Forum_hasMember_Person2 + ON Forum_hasMember_Person2.ForumId = Message1.ContainerForumId -- forum1 + AND Forum_hasMember_Person2.PersonId = Comment.CreatorPersonId -- person2 +-- (forum1)-[:Has_MEMBER]->(person3) +JOIN Forum_hasMember_Person Forum_hasMember_Person3 + ON Forum_hasMember_Person3.ForumId = Message1.ContainerForumId -- forum1 + AND Forum_hasMember_Person3.PersonId = Message2.CreatorPersonId -- person3 +WHERE Message1.ContainerForumId <> Message2.ContainerForumId + -- person2 <> person3 + AND Forum_hasMember_Person2.PersonId <> Forum_hasMember_Person3.PersonId + -- NOT (forum2)-[:HAS_MEMBER]->(person1) + AND NOT EXISTS (SELECT 1 + FROM Forum_hasMember_Person Forum_hasMember_Person1 + WHERE Forum_hasMember_Person1.ForumId = Message2.ContainerForumId -- forum2 + AND Forum_hasMember_Person1.PersonId = Message1.CreatorPersonId -- person1 + ) +GROUP BY Message1.CreatorPersonId +ORDER BY messageCount DESC, Message1.CreatorPersonId ASC +LIMIT 10 +; diff --git a/duckdb/queries/bi-18.sql b/duckdb/queries/bi-18.sql new file mode 100644 index 00000000..6e7fde3e --- /dev/null +++ b/duckdb/queries/bi-18.sql @@ -0,0 +1,29 @@ +/* Q18. Friend recommendation +\set tag '\'Frank_Sinatra\'' + */ +WITH +PersonWithInterest AS ( +SELECT pt.PersonId as PersonId +FROM Person_hasInterest_Tag pt, Tag t +WHERE t.name = :tag and pt.TagId = t.id +), +FriendsOfInterested AS ( +SELECT k.Person1Id AS InterestedId, k.Person2Id AS FriendId +FROM PersonWithInterest p, Person_knows_Person k +WHERE p.PersonId = k.Person1Id +) +SELECT k1.InterestedId AS "person1.id", k2.InterestedId AS "person2.id", count(k1.FriendId) AS mutualFriendCount +FROM FriendsOfInterested k1 +JOIN FriendsOfInterested k2 + ON k1.FriendId = k2.FriendId -- pattern: mutualFriend + -- negative edge +WHERE k1.InterestedId != k2.InterestedId + AND NOT EXISTS (SELECT 1 + FROM Person_knows_Person k3 + WHERE k3.Person1Id = k2.InterestedId -- pattern: person2 + AND k3.Person2Id = k1.InterestedId -- pattern: person1 + ) +GROUP BY k1.InterestedId, k2.InterestedId +ORDER BY mutualFriendCount DESC, k1.InterestedId ASC, k2.InterestedId ASC +LIMIT 20 +; diff --git a/duckdb/queries/bi-19.sql b/duckdb/queries/bi-19.sql new file mode 100644 index 00000000..12dd2eed --- /dev/null +++ b/duckdb/queries/bi-19.sql @@ -0,0 +1,54 @@ +/* Q19. Interaction path between cities +\set city1Id 608 +\set city2Id 1148 + */ +with recursive +srcs(f) as (select id from Person where locationcityid = :city1Id), +dsts(t) as (select id from Person where locationcityid = :city2Id), +shorts(dir, gsrc, dst, w, dead, iter) as ( + ( + select false, f, f, 0::double precision, false, 0 from srcs + union all + select true, t, t, 0::double precision, false, 0 from dsts + ) + union all + ( + with + ss as (select * from shorts), + toExplore as (select * from ss where dead = false order by w limit 1000), + -- assumes graph is undirected + newPoints(dir, gsrc, dst, w, dead) as ( + select e.dir, e.gsrc as gsrc, p.dst as dst, e.w + p.w as w, false as dead + from PathQ19 p join toExplore e on (e.dst = p.src) + union all + select dir, gsrc, dst, w, dead or exists (select * from toExplore e where e.dir = o.dir and e.gsrc = o.gsrc and e.dst = o.dst) from ss o + ), + fullTable as ( + select distinct on(dir, gsrc, dst) dir, gsrc, dst, w, dead + from newPoints + order by dir, gsrc, dst, w, dead desc + ), + found as ( + select min(l.w + r.w) as w + from fullTable l, fullTable r + where l.dir = false and r.dir = true and l.dst = r.dst + ) + select dir, + gsrc, + dst, + w, + dead or (coalesce(t.w > (select f.w/2 from found f), false)), + e.iter + 1 as iter + from fullTable t, (select iter from toExplore limit 1) e + ) +), +ss(dir, gsrc, dst, w, iter) as ( + select dir, gsrc, dst, w, iter from shorts where iter = (select max(iter) from shorts) +), +results(f, t, w) as ( + select l.gsrc, r.gsrc, min(l.w + r.w) + from ss l, ss r + where l.dir = false and r.dir = true and l.dst = r.dst + group by l.gsrc, r.gsrc +) +select * from results where w = (select min(w) from results) order by f, t; diff --git a/duckdb/queries/bi-2.sql b/duckdb/queries/bi-2.sql new file mode 100644 index 00000000..52b2b89c --- /dev/null +++ b/duckdb/queries/bi-2.sql @@ -0,0 +1,33 @@ +/* Q2. Tag evolution +\set date '\'2012-06-01\''::timestamp +\set tagClass '\'MusicalArtist\'' + */ +WITH +MyTag AS ( +SELECT Tag.id AS id, Tag.name AS name + FROM TagClass + JOIN Tag + ON Tag.TypeTagClassId = TagClass.id + WHERE TagClass.name = :tagClass +), +detail AS ( +SELECT t.id as TagId + , count(CASE WHEN Message.creationDate < :date + INTERVAL '100 days' THEN Message.MessageId ELSE NULL END) AS countMonth1 + , count(CASE WHEN Message.creationDate >= :date + INTERVAL '100 days' THEN Message.MessageId ELSE NULL END) AS countMonth2 + FROM MyTag t + JOIN Message_hasTag_Tag + ON Message_hasTag_tag.TagId = t.id + JOIN Message + ON Message.MessageId = Message_hasTag_tag.MessageId + AND Message.creationDate >= :date + AND Message.creationDate < :date + INTERVAL '200 days' + GROUP BY t.id +) +SELECT t.name AS "tag.name" + , coalesce(countMonth1, 0) + , coalesce(countMonth2, 0) + , abs(coalesce(countMonth1, 0)-coalesce(countMonth2, 0)) AS diff + FROM MyTag t LEFT JOIN detail ON t.id = detail.TagId + ORDER BY diff desc, t.name + LIMIT 100 +; diff --git a/duckdb/queries/bi-20.sql b/duckdb/queries/bi-20.sql new file mode 100644 index 00000000..af2d2842 --- /dev/null +++ b/duckdb/queries/bi-20.sql @@ -0,0 +1,73 @@ +/* Q20. Recruitment +\set person2Id 32985348834889 +\set company 'Express_Air' + */ +with recursive +srcs(f) as (select :person2Id), +dsts(t) as ( + select personid + from Person_workat_company pwc, Company c + where pwc.companyid = c.id and c.name=:company +), +-- Try to find any path with a faster two way BFS +anyPath(pos) as ( + select f from srcs + union + ( + with + ss as (select pos from anyPath) + select dst + from ss, PathQ20 + where pos = src and not exists (select 1 from ss, dsts where ss.pos = dsts.t) + ) +), +pathExists as ( + select 1 where exists (select 1 from anyPath ss, dsts where ss.pos = dsts.t) +), +shorts(dir, gsrc, dst, w, dead, iter) as ( + ( + select false, f, f, 0, false, 0 from srcs where exists (select 1 from pathExists) + union all + select true, t, t, 0, false, 0 from dsts where exists (select 1 from pathExists) + ) + union all + ( + with + ss as (select * from shorts), + toExplore as (select * from ss where dead = false order by w limit 1000), + -- assumes graph is undirected + newPoints(dir, gsrc, dst, w, dead) as ( + select e.dir, e.gsrc as gsrc, p.dst as dst, e.w + p.w as w, false as dead + from PathQ20 p join toExplore e on (e.dst = p.src) + union all + select dir, gsrc, dst, w, dead or exists (select * from toExplore e where e.dir = o.dir and e.gsrc = o.gsrc and e.dst = o.dst) from ss o + ), + fullTable as ( + select distinct on(dir, gsrc, dst) dir, gsrc, dst, w, dead + from newPoints + order by dir, gsrc, dst, w, dead desc + ), + found as ( + select min(l.w + r.w) as w + from fullTable l, fullTable r + where l.dir = false and r.dir = true and l.dst = r.dst + ) + select dir, + gsrc, + dst, + w, + dead or (coalesce(t.w > (select f.w/2 from found f), false)), + e.iter + 1 as iter + from fullTable t, (select iter from toExplore limit 1) e + ) +), +ss(dir, gsrc, dst, w, iter) as ( + select dir, gsrc, dst, w, iter from shorts where iter = (select max(iter) from shorts) +), +results(f, t, w) as ( + select l.gsrc, r.gsrc, min(l.w + r.w) + from ss l, ss r + where l.dir = false and r.dir = true and l.dst = r.dst + group by l.gsrc, r.gsrc +) +select t, w from results where w = (select min(w) from results) order by t limit 20; diff --git a/duckdb/queries/bi-3.sql b/duckdb/queries/bi-3.sql new file mode 100644 index 00000000..ea4ba36a --- /dev/null +++ b/duckdb/queries/bi-3.sql @@ -0,0 +1,31 @@ +/* Q3. Popular topics in a country +\set tagClass '\'MusicalArtist\'' +\set country '\'Burma\'' + */ +SELECT Forum.id AS "forum.id" + , Forum.title AS "forum.title" + , Forum.creationDate AS "forum.creationDate" + , Forum.ModeratorPersonId AS "person.id" + , count(Message.MessageId) AS messageCount +FROM Message +JOIN Forum + ON Forum.id = Message.ContainerForumId +JOIN Person AS ModeratorPerson + ON ModeratorPerson.id = Forum.ModeratorPersonId +JOIN City + ON City.id = ModeratorPerson.LocationCityId +JOIN Country + ON Country.id = City.PartOfCountryId + AND Country.name = :country +WHERE EXISTS ( + SELECT 1 + FROM TagClass + JOIN Tag + ON Tag.TypeTagClassId = TagClass.id + JOIN Message_hasTag_Tag + ON Message_hasTag_Tag.TagId = Tag.id + WHERE Message.MessageId = Message_hasTag_Tag.MessageId AND TagClass.name = :tagClass) +GROUP BY Forum.id, Forum.title, Forum.creationDate, Forum.ModeratorPersonId +ORDER BY messageCount DESC, Forum.id +LIMIT 20 +; diff --git a/duckdb/queries/bi-4.sql b/duckdb/queries/bi-4.sql new file mode 100644 index 00000000..6c26603f --- /dev/null +++ b/duckdb/queries/bi-4.sql @@ -0,0 +1,31 @@ +/* Q4. Top message creators by country +\set date '\'2012-09-01\''::timestamp + */ +WITH Top100_Popular_Forums AS ( + SELECT id, creationDate, maxNumberOfMembers + FROM Top100PopularForumsQ04 + WHERE creationDate > :date + ORDER BY maxNumberOfMembers DESC, id + LIMIT 100 +) +SELECT au.id AS "person.id" + , au.firstName AS "person.firstName" + , au.lastName AS "person.lastName" + , au.creationDate + -- a single person might be member of more than 1 of the top100 forums, so their messages should be DISTINCT counted + , count(Message.MessageId) AS messageCount + FROM + Person au + LEFT JOIN Message + ON au.id = Message.CreatorPersonId + AND Message.ContainerForumId IN (SELECT id FROM Top100_Popular_Forums) + WHERE EXISTS (SELECT 1 + FROM Top100_Popular_Forums + INNER JOIN Forum_hasMember_Person + ON Forum_hasMember_Person.ForumId = Top100_Popular_Forums.id + WHERE Forum_hasMember_Person.PersonId = au.id + ) +GROUP BY au.id, au.firstName, au.lastName, au.creationDate +ORDER BY messageCount DESC, au.id +LIMIT 100 +; diff --git a/duckdb/queries/bi-5.sql b/duckdb/queries/bi-5.sql new file mode 100644 index 00000000..40c8f716 --- /dev/null +++ b/duckdb/queries/bi-5.sql @@ -0,0 +1,27 @@ +/* Q5. Most active posters in a given topic +\set tag '\'Abbas_I_of_Persia\'' + */ +WITH detail AS ( +SELECT Message.CreatorPersonId AS CreatorPersonId + , sum(coalesce(Cs.c, 0)) AS replyCount + , sum(coalesce(Plm.c, 0)) AS likeCount + , count(Message.MessageId) AS messageCount + FROM Tag + JOIN Message_hasTag_Tag + ON Message_hasTag_Tag.TagId = Tag.id + JOIN Message + ON Message.MessageId = Message_hasTag_Tag.MessageId + LEFT JOIN (SELECT ParentMessageId, count(*) FROM Message c WHERE ParentMessageId IS NOT NULL GROUP BY ParentMessageId) Cs(id, c) ON Cs.id = Message.MessageId + LEFT JOIN (SELECT MessageId, count(*) FROM Person_likes_Message GROUP BY MessageId) Plm(id, c) ON Plm.id = Message.MessageId + WHERE Tag.name = :tag + GROUP BY Message.CreatorPersonId +) +SELECT CreatorPersonId AS "person.id" + , replyCount + , likeCount + , messageCount + , 1*messageCount + 2*replyCount + 10*likeCount AS score + FROM detail + ORDER BY score DESC, CreatorPersonId + LIMIT 100 +; diff --git a/duckdb/queries/bi-6.sql b/duckdb/queries/bi-6.sql new file mode 100644 index 00000000..8f648547 --- /dev/null +++ b/duckdb/queries/bi-6.sql @@ -0,0 +1,26 @@ +/* Q6. Most authoritative users on a given topic +\set tag '\'Arnold_Schwarzenegger\'' + */ +WITH poster_w_liker AS ( + SELECT DISTINCT + message1.CreatorPersonId AS person1id, + like2.PersonId AS person2id + FROM Tag + JOIN Message_hasTag_Tag + ON Message_hasTag_Tag.TagId = Tag.id + JOIN Message message1 + ON message1.MessageId = Message_hasTag_Tag.MessageId + LEFT JOIN Person_likes_Message like2 + ON like2.MessageId = message1.MessageId + -- we don't need the Person itself as its ID is in the like + WHERE Tag.name = :tag + ) +SELECT pl.person1id AS "person1.id", + sum(coalesce(ps.popularityScore, 0)) AS authorityScore +FROM poster_w_liker pl +LEFT JOIN PopularityScoreQ06 ps + ON ps.person2id = pl.person2id +GROUP BY pl.person1id +ORDER BY authorityScore DESC, pl.person1id ASC +LIMIT 100 +; diff --git a/duckdb/queries/bi-7.sql b/duckdb/queries/bi-7.sql new file mode 100644 index 00000000..5399f81f --- /dev/null +++ b/duckdb/queries/bi-7.sql @@ -0,0 +1,28 @@ +/* Q7. Related topics +\set tag '\'Enrique_Iglesias\'' + */ +WITH MyMessage AS ( + SELECT m.MessageId + FROM Message_hasTag_Tag m, Tag + WHERE Tag.name = :tag and m.TagId = Tag.Id +) +SELECT RelatedTag.name AS "relatedTag.name" + , count(*) AS count + FROM MyMessage ParentMessage_HasTag_Tag + -- as an optimization, we don't need message here as its ID is in ParentMessage_HasTag_Tag + -- so proceed to the comment directly + INNER JOIN Message Comment + ON ParentMessage_HasTag_Tag.MessageId = Comment.ParentMessageId + -- comment's tag + LEFT JOIN Message_hasTag_Tag ct + ON Comment.MessageId = ct.MessageId + INNER JOIN Tag RelatedTag + ON RelatedTag.id = ct.TagId + WHERE TRUE + -- comment doesn't have the given tag + AND Comment.MessageId NOT IN (SELECT MessageId FROM MyMessage) + AND Comment.ParentMessageId IS NOT NULL + GROUP BY RelatedTag.Name + ORDER BY count DESC, RelatedTag.name + LIMIT 100 +; diff --git a/duckdb/queries/bi-8.sql b/duckdb/queries/bi-8.sql new file mode 100644 index 00000000..3ec5b36e --- /dev/null +++ b/duckdb/queries/bi-8.sql @@ -0,0 +1,49 @@ +/* Q8. Central person for a tag +\set tag '\'Che_Guevara\'' +\set startDate '\'2011-07-20\''::timestamp +\set endDate '\'2011-07-25\''::timestamp + */ +WITH Person_interested_in_Tag AS ( + SELECT Person.id AS PersonId + FROM Person + JOIN Person_hasInterest_Tag + ON Person_hasInterest_Tag.PersonId = Person.id + JOIN Tag + ON Tag.id = Person_hasInterest_Tag.TagId + AND Tag.name = :tag +) + , Person_Message_score AS ( + SELECT Person.id AS PersonId + , count(*) AS message_score + FROM Tag + JOIN Message_hasTag_Tag + ON Message_hasTag_Tag.TagId = Tag.id + JOIN Message + ON Message_hasTag_Tag.MessageId = Message.MessageId + AND :startDate < Message.creationDate + JOIN Person + ON Person.id = Message.CreatorPersonId + WHERE Tag.name = :tag + AND Message.creationDate < :endDate + GROUP BY Person.id +) + , Person_score AS ( + SELECT coalesce(Person_interested_in_Tag.PersonId, pms.PersonId) AS PersonId + , CASE WHEN Person_interested_in_Tag.PersonId IS NULL then 0 ELSE 100 END -- scored from interest in the given tag + + coalesce(pms.message_score, 0) AS score + FROM Person_interested_in_Tag + FULL JOIN Person_Message_score pms + ON Person_interested_in_Tag.PersonId = pms.PersonId +) +SELECT p.PersonId AS "person.id" + , p.score AS score + , coalesce(sum(f.score), 0) AS friendsScore + FROM Person_score p + LEFT JOIN Person_knows_Person + ON Person_knows_Person.Person1Id = p.PersonId + LEFT JOIN Person_score f -- the friend + ON f.PersonId = Person_knows_Person.Person2Id + GROUP BY p.PersonId, p.score + ORDER BY p.score + coalesce(sum(f.score), 0) DESC, p.PersonId + LIMIT 100 +; diff --git a/duckdb/queries/bi-9.sql b/duckdb/queries/bi-9.sql new file mode 100644 index 00000000..e910c954 --- /dev/null +++ b/duckdb/queries/bi-9.sql @@ -0,0 +1,21 @@ +/* Q9. Top thread initiators +\set startDate '\'2011-10-01\''::timestamp +\set endDate '\'2011-10-15\''::timestamp + */ +WITH +MPP AS (SELECT RootPostId, count(*) as MessageCount FROM Message WHERE Message.creationDate BETWEEN :startDate AND :endDate GROUP BY RootPostId) +SELECT Person.id AS "person.id" + , Person.firstName AS "person.firstName" + , Person.lastName AS "person.lastName" + , count(Post.id) AS threadCount + , sum(MPP.MessageCount) AS messageCount + FROM Person + JOIN Post_View Post + ON Person.id = Post.CreatorPersonId + JOIN MPP + ON Post.id = MPP.RootPostId + WHERE Post.creationDate BETWEEN :startDate AND :endDate + GROUP BY Person.id, Person.firstName, Person.lastName + ORDER BY messageCount DESC, Person.id + LIMIT 100 +; diff --git a/duckdb/scratch/.gitignore b/duckdb/scratch/.gitignore new file mode 100644 index 00000000..d6b7ef32 --- /dev/null +++ b/duckdb/scratch/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/duckdb/scripts/backup-database.sh b/duckdb/scripts/backup-database.sh new file mode 100755 index 00000000..7f65188c --- /dev/null +++ b/duckdb/scripts/backup-database.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd .. + +. scripts/vars.sh + +scripts/stop.sh +docker run \ + --volume=${UMBRA_DATABASE_DIR}:/var/db/:z \ + --volume=${UMBRA_BACKUP_DIR}:/var/backup/:z \ + ${UMBRA_DOCKER_IMAGE} \ + bash -c "rm -rf /var/backup/* && cp -r /var/db/* /var/backup/" +scripts/start.sh diff --git a/duckdb/scripts/benchmark-local.sh b/duckdb/scripts/benchmark-local.sh new file mode 100755 index 00000000..9ef2462e --- /dev/null +++ b/duckdb/scripts/benchmark-local.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +scripts/benchmark.sh $@ --local diff --git a/duckdb/scripts/benchmark.sh b/duckdb/scripts/benchmark.sh new file mode 100755 index 00000000..4154af61 --- /dev/null +++ b/duckdb/scripts/benchmark.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd .. + +. scripts/vars.sh + +echo "===============================================================================" +echo "Running Umbra benchmark" +echo "-------------------------------------------------------------------------------" +echo "SF: ${SF}" +echo "UMBRA_DATABASE_DIR: ${UMBRA_DATABASE_DIR}" +echo "UMBRA_LOG_DIR: ${UMBRA_LOG_DIR}" +echo "UMBRA_DDL_DIR: ${UMBRA_DDL_DIR}" +echo "UMBRA_CONTAINER_NAME: ${UMBRA_CONTAINER_NAME}" +echo "UMBRA_DOCKER_IMAGE: ${UMBRA_DOCKER_IMAGE}" +echo "UMBRA_CSV_DIR: ${UMBRA_CSV_DIR}" +echo "===============================================================================" + +python3 benchmark.py --scale_factor ${SF} --data_dir ${UMBRA_CSV_DIR} $@ diff --git a/duckdb/scripts/connect.sh b/duckdb/scripts/connect.sh new file mode 100755 index 00000000..d8a674d8 --- /dev/null +++ b/duckdb/scripts/connect.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd .. + +. scripts/vars.sh + +export PGPASSWORD=mysecretpassword + +psql -h localhost -U postgres -p 8000 diff --git a/duckdb/scripts/create-db.sh b/duckdb/scripts/create-db.sh new file mode 100755 index 00000000..04a50930 --- /dev/null +++ b/duckdb/scripts/create-db.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd .. + +echo -n "Cleaning up . . ." +# ensure database and log dirs exists and are empty +mkdir -p ${UMBRA_DATABASE_DIR}/ +mkdir -p ${UMBRA_LOG_DIR}/ +docker run \ + --volume=${UMBRA_DATABASE_DIR}:/var/db/:z \ + ${UMBRA_DOCKER_IMAGE} \ + rm -rf "/var/db/*" "/var/log/*" +echo " Cleanup done." + +echo -n "Creating database . . ." +docker run \ + --volume=${UMBRA_DATABASE_DIR}:/var/db/:z \ + --volume=${UMBRA_DDL_DIR}:/ddl/:z \ + ${UMBRA_DOCKER_IMAGE} \ + umbra_sql \ + --createdb \ + /var/db/ldbc.db \ + /ddl/create-role.sql \ + /ddl/schema-composite-merged-fk.sql +echo " Database created." diff --git a/duckdb/scripts/decompress-data-set.sh b/duckdb/scripts/decompress-data-set.sh new file mode 100755 index 00000000..68571f95 --- /dev/null +++ b/duckdb/scripts/decompress-data-set.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd .. + +find ${UMBRA_CSV_DIR} -name "*.csv.gz" -print0 | parallel -q0 gunzip diff --git a/duckdb/scripts/docker-load.sh b/duckdb/scripts/docker-load.sh new file mode 100755 index 00000000..31dec458 --- /dev/null +++ b/duckdb/scripts/docker-load.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd .. + +. scripts/vars.sh + +curl -s ${UMBRA_URL_PREFIX}${UMBRA_VERSION}.tar.gz | docker load diff --git a/duckdb/scripts/force-stop.sh b/duckdb/scripts/force-stop.sh new file mode 100755 index 00000000..0cc2de6e --- /dev/null +++ b/duckdb/scripts/force-stop.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd .. + +. scripts/vars.sh + +echo -n "Force stop and remove Umbra container . . ." +docker rm -f ${UMBRA_CONTAINER_NAME} +echo " Removed." diff --git a/duckdb/scripts/get-max-memory-consumption.sh b/duckdb/scripts/get-max-memory-consumption.sh new file mode 100755 index 00000000..b8ec2a38 --- /dev/null +++ b/duckdb/scripts/get-max-memory-consumption.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd .. + +# drop head, select column of %memused and sort to select the top-1 value +tail -n +3 output/memory.log | cut -c45-52 | sort -nr | head -n 1 diff --git a/duckdb/scripts/get-sample-data-set.sh b/duckdb/scripts/get-sample-data-set.sh new file mode 100755 index 00000000..39cbd71b --- /dev/null +++ b/duckdb/scripts/get-sample-data-set.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd .. + +export SAMPLE_DATA_SET_NAME=social-network-sf0.003-bi-composite-merged-fk-postgres-compressed +rm -f ${SAMPLE_DATA_SET_NAME}.zip +wget -q https://ldbcouncil.org/ldbc_snb_datagen_spark/${SAMPLE_DATA_SET_NAME}.zip +rm -rf ${SAMPLE_DATA_SET_NAME}/ +unzip -q ${SAMPLE_DATA_SET_NAME}.zip + +. scripts/use-sample-data-set.sh +scripts/decompress-data-set.sh diff --git a/duckdb/scripts/install-dependencies.sh b/duckdb/scripts/install-dependencies.sh new file mode 100755 index 00000000..5405a947 --- /dev/null +++ b/duckdb/scripts/install-dependencies.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +if [[ ! -z $(which yum) ]]; then + sudo yum install -y python3-pip postgresql-devel postgresql sysstat parallel +elif [[ ! -z $(which apt-get) ]]; then + sudo apt-get update + sudo apt-get install -y python3-pip libpq-dev postgresql-client sysstat parallel +else + echo "Operating system not supported, please install the dependencies manually" +fi + +pip3 install --user --progress-bar off psycopg2-binary python-dateutil diff --git a/duckdb/scripts/load-in-one-step.sh b/duckdb/scripts/load-in-one-step.sh new file mode 100755 index 00000000..b2f81be9 --- /dev/null +++ b/duckdb/scripts/load-in-one-step.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd .. + +. scripts/vars.sh + +echo "===============================================================================" +echo "Loading the Umbra database" +echo "-------------------------------------------------------------------------------" +echo "SF: ${SF}" +echo "UMBRA_BACKUP_DIR: ${UMBRA_BACKUP_DIR}" +echo "UMBRA_DATABASE_DIR: ${UMBRA_DATABASE_DIR}" +echo "UMBRA_LOG_DIR: ${UMBRA_LOG_DIR}" +echo "UMBRA_DDL_DIR: ${UMBRA_DDL_DIR}" +echo "UMBRA_CONTAINER_NAME: ${UMBRA_CONTAINER_NAME}" +echo "UMBRA_DOCKER_IMAGE: ${UMBRA_DOCKER_IMAGE}" +echo "UMBRA_CSV_DIR: ${UMBRA_CSV_DIR}" +echo "UMBRA_DOCKER_BUFFERSIZE_ENV_VAR: ${UMBRA_DOCKER_BUFFERSIZE_ENV_VAR}" +echo "===============================================================================" + +if [ ! -d "${UMBRA_CSV_DIR}" ]; then + echo "Umbra directory does not exist. \${UMBRA_CSV_DIR} is set to: ${UMBRA_CSV_DIR}" + exit 1 +fi + +if [ "$(uname)" == "Darwin" ]; then + DATE_COMMAND=gdate +else + DATE_COMMAND=date +fi + +scripts/stop.sh +scripts/decompress-data-set.sh + +start_time=$(${DATE_COMMAND} +%s.%3N) + +scripts/create-db.sh +scripts/start.sh +scripts/load.sh + +end_time=$(${DATE_COMMAND} +%s.%3N) + +mkdir -p output/output-sf${SF} +elapsed=$(python3 -c "import argparse; parser = argparse.ArgumentParser(); parser.add_argument('--start_time', type=float); parser.add_argument('--end_time', type=float); args = parser.parse_args(); elapsed = args.end_time - args.start_time; print(f'{elapsed:.3f}')" --start_time $start_time --end_time $end_time) +echo -e "time\n${elapsed}" > output/output-sf${SF}/load.csv diff --git a/duckdb/scripts/load-local.sh b/duckdb/scripts/load-local.sh new file mode 100755 index 00000000..139f3f1b --- /dev/null +++ b/duckdb/scripts/load-local.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd .. + +. scripts/vars.sh + +rm -rf ${UMBRA_DATABASE_DIR}/* + +python3 load.py ${UMBRA_CSV_DIR} --local diff --git a/duckdb/scripts/load.sh b/duckdb/scripts/load.sh new file mode 100755 index 00000000..7761b090 --- /dev/null +++ b/duckdb/scripts/load.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd .. + +. scripts/vars.sh + +python3 load.py --data_dir ${UMBRA_CSV_DIR} diff --git a/duckdb/scripts/logs.sh b/duckdb/scripts/logs.sh new file mode 100755 index 00000000..0816e146 --- /dev/null +++ b/duckdb/scripts/logs.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd .. + +. scripts/vars.sh + +# to follow the logs (a'la "tail -f") use: "logs.sh -f" +docker logs $@ ${UMBRA_CONTAINER_NAME} diff --git a/duckdb/scripts/memory.sh b/duckdb/scripts/memory.sh new file mode 100755 index 00000000..7e5d309a --- /dev/null +++ b/duckdb/scripts/memory.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd .. + +. scripts/vars.sh + +sar -r 1 | tee output/memory.log diff --git a/duckdb/scripts/queries.sh b/duckdb/scripts/queries.sh new file mode 100755 index 00000000..7c7489dc --- /dev/null +++ b/duckdb/scripts/queries.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd .. + +. scripts/vars.sh + +python3 benchmark.py --queries --scale_factor ${SF} --data_dir ${UMBRA_CSV_DIR} $@ diff --git a/duckdb/scripts/restart.sh b/duckdb/scripts/restart.sh new file mode 100755 index 00000000..378bf575 --- /dev/null +++ b/duckdb/scripts/restart.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd .. + +. scripts/vars.sh + +scripts/stop.sh +scripts/start.sh diff --git a/duckdb/scripts/restore-database.sh b/duckdb/scripts/restore-database.sh new file mode 100755 index 00000000..b69c3f2e --- /dev/null +++ b/duckdb/scripts/restore-database.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd .. + +. scripts/vars.sh + +scripts/stop.sh +docker run \ + --volume=${UMBRA_DATABASE_DIR}:/var/db/:z \ + --volume=${UMBRA_BACKUP_DIR}:/var/backup/:z \ + ${UMBRA_DOCKER_IMAGE} \ + bash -c "rm -rf /var/db/* && cp -r /var/backup/* /var/db/" +scripts/start.sh diff --git a/duckdb/scripts/run-benchmark.sh b/duckdb/scripts/run-benchmark.sh new file mode 100755 index 00000000..76a0661c --- /dev/null +++ b/duckdb/scripts/run-benchmark.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd .. + +mkdir -p output/output-sf${SF} +scripts/load-in-one-step.sh |& tee output/output-sf${SF}/load.log +scripts/benchmark.sh |& tee output/output-sf${SF}/benchmark.log +scripts/stop.sh diff --git a/duckdb/scripts/start.sh b/duckdb/scripts/start.sh new file mode 100755 index 00000000..f51cf5dd --- /dev/null +++ b/duckdb/scripts/start.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd .. + +. scripts/vars.sh + +python3 -c 'import psycopg2' || (echo "psycopg2 Python package is missing or broken" && exit 1) + +echo -n "Outputting Umbra parameters" +docker run \ + --volume=${UMBRA_DATABASE_DIR}:/var/db/:z \ + --volume=${UMBRA_DDL_DIR}:/ddl/:z \ + --env USEDIRECTIO=1 \ + ${UMBRA_DOCKER_BUFFERSIZE_ENV_VAR} \ + ${UMBRA_DOCKER_IMAGE} \ + umbra_sql \ + "" \ + /ddl/output-env.sql + +echo -n "Starting the database . " +docker run \ + --name ${UMBRA_CONTAINER_NAME} \ + --detach \ + --volume=${UMBRA_CSV_DIR}:/data/:z \ + --volume=${UMBRA_DATABASE_DIR}:/var/db/:z \ + --volume=${UMBRA_DDL_DIR}:/ddl/:z \ + --volume=${UMBRA_LOG_DIR}:/var/log/:z \ + --publish=8000:5432 \ + --env USEDIRECTIO=1 \ + ${UMBRA_DOCKER_BUFFERSIZE_ENV_VAR} \ + ${UMBRA_DOCKER_IMAGE} \ + umbra_server \ + --address 0.0.0.0 \ + /var/db/ldbc.db \ + >/dev/null + +until python3 scripts/test-db-connection.py > /dev/null 2>&1; do + echo -n ". " + sleep 1 +done +echo "Database started." diff --git a/duckdb/scripts/stop.sh b/duckdb/scripts/stop.sh new file mode 100755 index 00000000..6455c076 --- /dev/null +++ b/duckdb/scripts/stop.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd .. + +. scripts/vars.sh + +echo -n "Stopping Umbra container . . ." +(docker ps -a --format {{.Names}} | grep --quiet --word-regexp ${UMBRA_CONTAINER_NAME}) && docker stop --time 1200 ${UMBRA_CONTAINER_NAME} >/dev/null +echo " Stopped." + +echo -n "Removing Umbra container . . ." +(docker ps -a --format {{.Names}} | grep --quiet --word-regexp ${UMBRA_CONTAINER_NAME}) && docker rm ${UMBRA_CONTAINER_NAME} >/dev/null +echo " Removed." diff --git a/duckdb/scripts/test-db-connection.py b/duckdb/scripts/test-db-connection.py new file mode 100644 index 00000000..cca7e5be --- /dev/null +++ b/duckdb/scripts/test-db-connection.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python3 + +import os +import psycopg2 + +pg_con = psycopg2.connect(host="localhost", user="postgres", password="mysecretpassword", port=8000) +pg_con.close() diff --git a/duckdb/scripts/use-datagen-data-set.sh b/duckdb/scripts/use-datagen-data-set.sh new file mode 100644 index 00000000..b6e3b2ac --- /dev/null +++ b/duckdb/scripts/use-datagen-data-set.sh @@ -0,0 +1 @@ +export UMBRA_CSV_DIR=${LDBC_SNB_DATAGEN_DIR}/out-sf${SF}/graphs/csv/bi/composite-merged-fk/ diff --git a/duckdb/scripts/use-sample-data-set.sh b/duckdb/scripts/use-sample-data-set.sh new file mode 100644 index 00000000..027d6031 --- /dev/null +++ b/duckdb/scripts/use-sample-data-set.sh @@ -0,0 +1,2 @@ +export SF=0.003 +export UMBRA_CSV_DIR=`pwd`/social-network-sf${SF}-bi-composite-merged-fk-postgres-compressed/graphs/csv/bi/composite-merged-fk/ diff --git a/duckdb/scripts/vars.sh b/duckdb/scripts/vars.sh new file mode 100644 index 00000000..bf0bd870 --- /dev/null +++ b/duckdb/scripts/vars.sh @@ -0,0 +1,22 @@ +pushd . > /dev/null + +cd "$( cd "$( dirname "${BASH_SOURCE[0]:-${(%):-%x}}" )" >/dev/null 2>&1 && pwd )" +cd .. + +export UMBRA_BACKUP_DIR=`pwd`/scratch/backup/ +export UMBRA_DATABASE_DIR=`pwd`/scratch/db/ +export UMBRA_LOG_DIR=`pwd`/scratch/log/ +export UMBRA_DDL_DIR=`pwd`/ddl/ +export UMBRA_CONTAINER_NAME=snb-bi-umbra +export UMBRA_VERSION=cbad59200 +export UMBRA_DOCKER_IMAGE=umbra-release:${UMBRA_VERSION} + +if [ -z "${UMBRA_BUFFERSIZE+x}" ]; then + export UMBRA_DOCKER_BUFFERSIZE_ENV_VAR= +else + export UMBRA_DOCKER_BUFFERSIZE_ENV_VAR="--env BUFFERSIZE=${UMBRA_BUFFERSIZE}" +fi + +cd scripts + +popd > /dev/null diff --git a/duckdb/views.sql b/duckdb/views.sql new file mode 100644 index 00000000..f6961df5 --- /dev/null +++ b/duckdb/views.sql @@ -0,0 +1,34 @@ +.sh rm -rf ldbc/ + +attach 'ldbc.duckdb' as ldbc; +use ldbc; +export database 'ldbc_parquets' (format parquet); + +use memory; +detach ldbc; + +create view city as from read_parquet('ldbc_parquets/city.parquet'); +create view comment_hastag_tag as from read_parquet('ldbc_parquets/comment_hastag_tag.parquet'); +create view comment as from read_parquet('ldbc_parquets/comment.parquet'); +create view company as from read_parquet('ldbc_parquets/company.parquet'); +create view country as from read_parquet('ldbc_parquets/country.parquet'); +create view forum_hasmember_person as from read_parquet('ldbc_parquets/forum_hasmember_person.parquet'); +create view forum_hastag_tag as from read_parquet('ldbc_parquets/forum_hastag_tag.parquet'); +create view forum as from read_parquet('ldbc_parquets/forum.parquet'); +create view message_hastag_tag as from read_parquet('ldbc_parquets/message_hastag_tag.parquet'); +create view message as from read_parquet('ldbc_parquets/message.parquet'); +create view organisation as from read_parquet('ldbc_parquets/organisation.parquet'); +create view person_hasinterest_tag as from read_parquet('ldbc_parquets/person_hasinterest_tag.parquet'); +create view person_knows_person as from read_parquet('ldbc_parquets/person_knows_person.parquet'); +create view person_likes_comment as from read_parquet('ldbc_parquets/person_likes_comment.parquet'); +create view person_likes_message as from read_parquet('ldbc_parquets/person_likes_message.parquet'); +create view person_likes_post as from read_parquet('ldbc_parquets/person_likes_post.parquet'); +create view person_studyat_university as from read_parquet('ldbc_parquets/person_studyat_university.parquet'); +create view person_workat_company as from read_parquet('ldbc_parquets/person_workat_company.parquet'); +create view person as from read_parquet('ldbc_parquets/person.parquet'); +create view place as from read_parquet('ldbc_parquets/place.parquet'); +create view post_hastag_tag as from read_parquet('ldbc_parquets/post_hastag_tag.parquet'); +create view post as from read_parquet('ldbc_parquets/post.parquet'); +create view tag as from read_parquet('ldbc_parquets/tag.parquet'); +create view tagclass as from read_parquet('ldbc_parquets/tagclass.parquet'); +create view university as from read_parquet('ldbc_parquets/university.parquet');