From 4bf969969fc7b9cd3e7aef599b55a5a20723338e Mon Sep 17 00:00:00 2001 From: Stephen Epps Date: Mon, 16 Jun 2025 16:41:03 -0700 Subject: [PATCH 1/4] update duckdb query examples, adjust transportation example with note --- docs/getting-data/duckdb.mdx | 3 ++- src/queries/duckdb/buildings_detroit.sql | 14 +++++++---- .../duckdb/county_level_geometries.sql | 23 +++++++++--------- src/queries/duckdb/mountain_peaks.sql | 23 ++++++++++-------- src/queries/duckdb/new_york_pizza.sql | 21 ++++++++-------- src/queries/duckdb/paris_roads.sql | 24 ++++++++++--------- src/queries/duckdb/write_geoparquet.sql | 5 +++- 7 files changed, 64 insertions(+), 49 deletions(-) diff --git a/docs/getting-data/duckdb.mdx b/docs/getting-data/duckdb.mdx index 01492f0ae..44a9a9799 100644 --- a/docs/getting-data/duckdb.mdx +++ b/docs/getting-data/duckdb.mdx @@ -46,9 +46,10 @@ Access Overture buildings data hosted on Azure to download named buildings aroun -Download roads in Paris around the _Arc de Triomphe_ and save as GeoJSON. +Download all roads intersecting a given bounding box around the Arc de Triomphe in Paris and save as GeoJSON. +_Note: This will result in a GeoJSON file that contains all road segments that intersect the bounding box, not just road segments contained within the bounding box._ diff --git a/src/queries/duckdb/buildings_detroit.sql b/src/queries/duckdb/buildings_detroit.sql index 40e78f58d..9e60ed755 100644 --- a/src/queries/duckdb/buildings_detroit.sql +++ b/src/queries/duckdb/buildings_detroit.sql @@ -1,4 +1,6 @@ -LOAD spatial; --noqa +INSTALL spatial; +LOAD spatial; -- noqa +SET s3_region='us-west-2'; COPY( SELECT @@ -6,9 +8,11 @@ COPY( names.primary as primary_name, height, geometry - FROM read_parquet('az://overturemapswestus2.blob.core.windows.net/release/__OVERTURE_RELEASE/theme=buildings/type=building/*', filename=true, hive_partitioning=1) - WHERE names.primary IS NOT NULL - AND bbox.xmin BETWEEN -84.36 AND -82.42 - AND bbox.ymin BETWEEN 41.71 AND 43.33 + FROM + read_parquet('az://overturemapswestus2.blob.core.windows.net/release/__OVERTURE_RELEASE/theme=buildings/type=building/*', filename=true, hive_partitioning=1) + WHERE + names.primary IS NOT NULL + AND bbox.xmin BETWEEN -84.36 AND -82.42 + AND bbox.ymin BETWEEN 41.71 AND 43.33 LIMIT 100 ) TO 'detroit_buildings.geojsonseq' WITH (FORMAT GDAL, DRIVER 'GeoJSONSeq'); diff --git a/src/queries/duckdb/county_level_geometries.sql b/src/queries/duckdb/county_level_geometries.sql index f57a45a64..c3abfe064 100644 --- a/src/queries/duckdb/county_level_geometries.sql +++ b/src/queries/duckdb/county_level_geometries.sql @@ -1,16 +1,17 @@ +INSTALL spatial; LOAD spatial; -- noqa SET s3_region='us-west-2'; COPY( - SELECT - id, - division_id, - names.primary, - geometry -- DuckDB v.1.1.0 will autoload this as a `geometry` type - FROM - read_parquet('s3://overturemaps-us-west-2/release/__OVERTURE_RELEASE/theme=divisions/type=division_area/*', hive_partitioning=1) - WHERE - subtype = 'county' - AND country = 'US' - AND region = 'US-PA' + SELECT + id, + division_id, + names.primary, + geometry -- DuckDB v.1.1.0 will autoload this as a `geometry` type + FROM + read_parquet('s3://overturemaps-us-west-2/release/__OVERTURE_RELEASE/theme=divisions/type=division_area/*', hive_partitioning=1) + WHERE + subtype = 'county' + AND country = 'US' + AND region = 'US-PA' ) TO 'pennsylvania_counties.gpkg' WITH (FORMAT GDAL, DRIVER 'GPKG'); diff --git a/src/queries/duckdb/mountain_peaks.sql b/src/queries/duckdb/mountain_peaks.sql index ea7a6464f..3943c01ef 100644 --- a/src/queries/duckdb/mountain_peaks.sql +++ b/src/queries/duckdb/mountain_peaks.sql @@ -1,16 +1,19 @@ +INSTALL spatial; LOAD spatial; -- noqa - SET s3_region='us-west-2'; COPY( - SELECT - id, - names.primary as name, - CAST(elevation * 3.28084 AS INT) AS elevation_ft, - geometry -- DuckDB v.1.1.0 will autoload this as a `geometry` type - FROM read_parquet('s3://overturemaps-us-west-2/release/__OVERTURE_RELEASE/theme=base/type=land/*', filename=true, hive_partitioning=1) - WHERE subtype = 'physical' AND class IN ('peak','volcano') AND elevation IS NOT NULL + SELECT + id, + names.primary as name, + CAST(elevation * 3.28084 AS INT) AS elevation_ft, + geometry -- DuckDB v.1.1.0 will autoload this as a `geometry` type + FROM + read_parquet('s3://overturemaps-us-west-2/release/__OVERTURE_RELEASE/theme=base/type=land/*', filename=true, hive_partitioning=1) + WHERE + subtype = 'physical' + AND class IN ('peak','volcano') + AND elevation IS NOT NULL AND bbox.xmin BETWEEN -124.71 AND -116.47 AND bbox.ymin BETWEEN 41.99 AND 46.30 -) TO 'oregon_peaks.geojson' -WITH (FORMAT GDAL, DRIVER 'GeoJSON'); +) TO 'oregon_peaks.geojson' WITH (FORMAT GDAL, DRIVER 'GeoJSON'); diff --git a/src/queries/duckdb/new_york_pizza.sql b/src/queries/duckdb/new_york_pizza.sql index 8c391d6ea..98f57609c 100644 --- a/src/queries/duckdb/new_york_pizza.sql +++ b/src/queries/duckdb/new_york_pizza.sql @@ -1,17 +1,18 @@ +INSTALL spatial; LOAD spatial; -- noqa - SET s3_region='us-west-2'; COPY( -- COPY TO saves the results to disk. - SELECT - id, - names.primary as name, - confidence AS confidence, - CAST(socials AS JSON) as socials, -- Ensure each attribute can be serialized to JSON - geometry -- DuckDB understands this to be a geometry type - FROM read_parquet('s3://overturemaps-us-west-2/release/__OVERTURE_RELEASE/theme=places/type=place/*', filename=true, hive_partitioning=1) - WHERE categories.primary = 'pizza_restaurant' + SELECT + id, + names.primary as name, + confidence AS confidence, + CAST(socials AS JSON) as socials, -- Ensure each attribute can be serialized to JSON + geometry -- DuckDB understands this to be a geometry type + FROM + read_parquet('s3://overturemaps-us-west-2/release/__OVERTURE_RELEASE/theme=places/type=place/*', filename=true, hive_partitioning=1) + WHERE + categories.primary = 'pizza_restaurant' AND bbox.xmin BETWEEN -75 AND -73 -- Only use the bbox min values AND bbox.ymin BETWEEN 40 AND 41 -- because they are point geometries. - ) TO 'nyc_pizza.geojson' WITH (FORMAT GDAL, DRIVER 'GeoJSON'); diff --git a/src/queries/duckdb/paris_roads.sql b/src/queries/duckdb/paris_roads.sql index 956b94afd..5b5f7b1c1 100644 --- a/src/queries/duckdb/paris_roads.sql +++ b/src/queries/duckdb/paris_roads.sql @@ -1,16 +1,18 @@ +INSTALL spatial; LOAD spatial; -- noqa - SET s3_region='us-west-2'; COPY( - SELECT - id, - names.primary as name, - class, - geometry -- DuckDB v.1.1.0 will autoload this as a `geometry` type - FROM read_parquet('s3://overturemaps-us-west-2/release/__OVERTURE_RELEASE/theme=transportation/type=segment/*', filename=true, hive_partitioning=1) - WHERE bbox.xmin > 2.276 - AND bbox.ymin > 48.865 - AND bbox.xmax < 2.314 - AND bbox.ymax < 48.882 + SELECT + id, + names.primary as name, + class, + geometry -- DuckDB v.1.1.0 will autoload this as a `geometry` type + FROM + read_parquet('s3://overturemaps-us-west-2/release/__OVERTURE_RELEASE/theme=transportation/type=segment/*', filename=true, hive_partitioning=1) + WHERE + bbox.xmin < 2.314 + AND bbox.ymin < 48.882 + AND bbox.xmax > 2.276 + AND bbox.ymax > 48.865 ) TO 'paris_roads.geojson' WITH (FORMAT GDAL, DRIVER 'GeoJSON'); diff --git a/src/queries/duckdb/write_geoparquet.sql b/src/queries/duckdb/write_geoparquet.sql index 60345c42d..155ac6354 100644 --- a/src/queries/duckdb/write_geoparquet.sql +++ b/src/queries/duckdb/write_geoparquet.sql @@ -1,8 +1,11 @@ +INSTALL spatial; LOAD spatial; -- noqa +SET s3_region='us-west-2'; COPY( SELECT * - FROM read_parquet('s3://overturemaps-us-west-2/release/__OVERTURE_RELEASE/theme=places/type=place/*') + FROM + read_parquet('s3://overturemaps-us-west-2/release/__OVERTURE_RELEASE/theme=places/type=place/*') LIMIT 100000 ) TO 'places.parquet'; From 1d6ae04df7b4bf3ee8ba102f2dc4bc718520ed99 Mon Sep 17 00:00:00 2001 From: Stephen Epps Date: Mon, 16 Jun 2025 17:29:36 -0700 Subject: [PATCH 2/4] adjust duckdb queries --- src/queries/duckdb/buildings_detroit.sql | 2 +- src/queries/duckdb/county_level_geometries.sql | 2 +- src/queries/duckdb/mountain_peaks.sql | 2 +- src/queries/duckdb/new_york_pizza.sql | 2 +- src/queries/duckdb/paris_roads.sql | 2 +- src/queries/duckdb/write_geoparquet.sql | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/queries/duckdb/buildings_detroit.sql b/src/queries/duckdb/buildings_detroit.sql index 9e60ed755..23d495876 100644 --- a/src/queries/duckdb/buildings_detroit.sql +++ b/src/queries/duckdb/buildings_detroit.sql @@ -1,5 +1,5 @@ -INSTALL spatial; LOAD spatial; -- noqa + SET s3_region='us-west-2'; COPY( diff --git a/src/queries/duckdb/county_level_geometries.sql b/src/queries/duckdb/county_level_geometries.sql index c3abfe064..23cb15dfa 100644 --- a/src/queries/duckdb/county_level_geometries.sql +++ b/src/queries/duckdb/county_level_geometries.sql @@ -1,5 +1,5 @@ -INSTALL spatial; LOAD spatial; -- noqa + SET s3_region='us-west-2'; COPY( diff --git a/src/queries/duckdb/mountain_peaks.sql b/src/queries/duckdb/mountain_peaks.sql index 3943c01ef..3fa4658ca 100644 --- a/src/queries/duckdb/mountain_peaks.sql +++ b/src/queries/duckdb/mountain_peaks.sql @@ -1,5 +1,5 @@ -INSTALL spatial; LOAD spatial; -- noqa + SET s3_region='us-west-2'; COPY( diff --git a/src/queries/duckdb/new_york_pizza.sql b/src/queries/duckdb/new_york_pizza.sql index 98f57609c..0bc4c0102 100644 --- a/src/queries/duckdb/new_york_pizza.sql +++ b/src/queries/duckdb/new_york_pizza.sql @@ -1,5 +1,5 @@ -INSTALL spatial; LOAD spatial; -- noqa + SET s3_region='us-west-2'; COPY( -- COPY TO saves the results to disk. diff --git a/src/queries/duckdb/paris_roads.sql b/src/queries/duckdb/paris_roads.sql index 5b5f7b1c1..6c949a350 100644 --- a/src/queries/duckdb/paris_roads.sql +++ b/src/queries/duckdb/paris_roads.sql @@ -1,5 +1,5 @@ -INSTALL spatial; LOAD spatial; -- noqa + SET s3_region='us-west-2'; COPY( diff --git a/src/queries/duckdb/write_geoparquet.sql b/src/queries/duckdb/write_geoparquet.sql index 155ac6354..3262e97a7 100644 --- a/src/queries/duckdb/write_geoparquet.sql +++ b/src/queries/duckdb/write_geoparquet.sql @@ -1,5 +1,5 @@ -INSTALL spatial; LOAD spatial; -- noqa + SET s3_region='us-west-2'; COPY( From bd4691cc69a8485d0aec5f777521c5b760b181e7 Mon Sep 17 00:00:00 2001 From: Stephen Epps Date: Tue, 17 Jun 2025 13:39:43 -0700 Subject: [PATCH 3/4] update paris transportation duckdb example, add queries --- docs/getting-data/duckdb.mdx | 8 ++++++-- ...ris_roads.sql => paris_roads_contained.sql} | 0 .../duckdb/paris_roads_intersecting.sql | 18 ++++++++++++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) rename src/queries/duckdb/{paris_roads.sql => paris_roads_contained.sql} (100%) create mode 100644 src/queries/duckdb/paris_roads_intersecting.sql diff --git a/docs/getting-data/duckdb.mdx b/docs/getting-data/duckdb.mdx index 44a9a9799..97dd732e1 100644 --- a/docs/getting-data/duckdb.mdx +++ b/docs/getting-data/duckdb.mdx @@ -9,7 +9,8 @@ import Peaks from '!!raw-loader!@site/src/queries/duckdb/mountain_peaks.sql'; import DetroitBuildings from '!!raw-loader!@site/src/queries/duckdb/buildings_detroit.sql'; import CountyGeometries from '!!raw-loader!@site/src/queries/duckdb/county_level_geometries.sql'; import NewYorkPizza from '!!raw-loader!@site/src/queries/duckdb/new_york_pizza.sql'; -import ParisRoads from '!!raw-loader!@site/src/queries/duckdb/paris_roads.sql'; +import ParisRoadsContained from '!!raw-loader!@site/src/queries/duckdb/paris_roads_contained.sql'; +import ParisRoadsIntersecting from '!!raw-loader!@site/src/queries/duckdb/paris_roads_intersecting.sql'; import WriteGeoparquet from '!!raw-loader!@site/src/queries/duckdb/write_geoparquet.sql'; import BuildingsInDivision from '!!raw-loader!@site/src/queries/duckdb/buildings_in_division_area.sql'; @@ -48,8 +49,11 @@ Access Overture buildings data hosted on Azure to download named buildings aroun Download all roads intersecting a given bounding box around the Arc de Triomphe in Paris and save as GeoJSON. - + _Note: This will result in a GeoJSON file that contains all road segments that intersect the bounding box, not just road segments contained within the bounding box._ + +To download only roads contained within that bounding box around the Arc de Triomphe in Paris, use the following: + diff --git a/src/queries/duckdb/paris_roads.sql b/src/queries/duckdb/paris_roads_contained.sql similarity index 100% rename from src/queries/duckdb/paris_roads.sql rename to src/queries/duckdb/paris_roads_contained.sql diff --git a/src/queries/duckdb/paris_roads_intersecting.sql b/src/queries/duckdb/paris_roads_intersecting.sql new file mode 100644 index 000000000..6c949a350 --- /dev/null +++ b/src/queries/duckdb/paris_roads_intersecting.sql @@ -0,0 +1,18 @@ +LOAD spatial; -- noqa + +SET s3_region='us-west-2'; + +COPY( + SELECT + id, + names.primary as name, + class, + geometry -- DuckDB v.1.1.0 will autoload this as a `geometry` type + FROM + read_parquet('s3://overturemaps-us-west-2/release/__OVERTURE_RELEASE/theme=transportation/type=segment/*', filename=true, hive_partitioning=1) + WHERE + bbox.xmin < 2.314 + AND bbox.ymin < 48.882 + AND bbox.xmax > 2.276 + AND bbox.ymax > 48.865 +) TO 'paris_roads.geojson' WITH (FORMAT GDAL, DRIVER 'GeoJSON'); From 6c25a8b3ab8026a389e9a0c02bb1ce80b2ad5751 Mon Sep 17 00:00:00 2001 From: Stephen Epps Date: Wed, 18 Jun 2025 08:02:06 -0700 Subject: [PATCH 4/4] update paris road queries --- src/queries/duckdb/paris_roads_contained.sql | 10 +++++----- src/queries/duckdb/paris_roads_intersecting.sql | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/queries/duckdb/paris_roads_contained.sql b/src/queries/duckdb/paris_roads_contained.sql index 6c949a350..6edaef72e 100644 --- a/src/queries/duckdb/paris_roads_contained.sql +++ b/src/queries/duckdb/paris_roads_contained.sql @@ -11,8 +11,8 @@ COPY( FROM read_parquet('s3://overturemaps-us-west-2/release/__OVERTURE_RELEASE/theme=transportation/type=segment/*', filename=true, hive_partitioning=1) WHERE - bbox.xmin < 2.314 - AND bbox.ymin < 48.882 - AND bbox.xmax > 2.276 - AND bbox.ymax > 48.865 -) TO 'paris_roads.geojson' WITH (FORMAT GDAL, DRIVER 'GeoJSON'); + bbox.xmin > 2.276 + AND bbox.ymin > 48.865 + AND bbox.xmax < 2.314 + AND bbox.ymax < 48.882 +) TO 'paris_roads_contained.geojson' WITH (FORMAT GDAL, DRIVER 'GeoJSON'); diff --git a/src/queries/duckdb/paris_roads_intersecting.sql b/src/queries/duckdb/paris_roads_intersecting.sql index 6c949a350..18b307bb3 100644 --- a/src/queries/duckdb/paris_roads_intersecting.sql +++ b/src/queries/duckdb/paris_roads_intersecting.sql @@ -15,4 +15,4 @@ COPY( AND bbox.ymin < 48.882 AND bbox.xmax > 2.276 AND bbox.ymax > 48.865 -) TO 'paris_roads.geojson' WITH (FORMAT GDAL, DRIVER 'GeoJSON'); +) TO 'paris_roads_intersecting.geojson' WITH (FORMAT GDAL, DRIVER 'GeoJSON');