From 72757f7600e87062b54fb6fff03df277de6fe275 Mon Sep 17 00:00:00 2001 From: dgonzdev Date: Sat, 14 Feb 2026 08:20:00 -0500 Subject: [PATCH 1/4] remove kiba from models --- .../bicycle_counter.rb | 38 ------------------- .../nyc_permitted_event_information.rb | 38 ------------------- ..._permitted_event_information_historical.rb | 38 ------------------- 3 files changed, 114 deletions(-) diff --git a/app/models/department_of_transportation/bicycle_counter.rb b/app/models/department_of_transportation/bicycle_counter.rb index 2e3b5e8..1debf08 100755 --- a/app/models/department_of_transportation/bicycle_counter.rb +++ b/app/models/department_of_transportation/bicycle_counter.rb @@ -193,43 +193,5 @@ def self.import_soda3_csv end end private_class_method :import_soda3_csv - - def self.run_import(api_version: '2', content_type: 'json') - if api_version == '2' && content_type == 'json' - run_import_soda2 - end - - if api_version == '2' && content_type == 'csv' - run_import_soda2_csv - end - - if api_version == '3' && content_type == 'json' - run_import_soda3 - end - - if api_version == '3' && content_type == 'csv' - run_import_soda3_csv - end - end - - def self.run_import_soda2 - Etl::Runners::PrimaryDbImports::BicycleCounters::Soda2IntoPrimaryDb.run - end - private_class_method :run_import_soda2 - - def self.run_import_soda2_csv - Etl::Runners::PrimaryDbImports::BicycleCounters::Soda2CsvIntoPrimaryDb.run - end - private_class_method :run_import_soda2_csv - - def self.run_import_soda3 - Etl::Runners::PrimaryDbImports::BicycleCounters::Soda3IntoPrimaryDb.run - end - private_class_method :run_import_soda3 - - def self.run_import_soda3_csv - Etl::Runners::PrimaryDbImports::BicycleCounters::Soda3CsvIntoPrimaryDb.run - end - private_class_method :run_import_soda3_csv end end diff --git a/app/models/office_of_citywide_event_coordination_and_management/nyc_permitted_event_information.rb b/app/models/office_of_citywide_event_coordination_and_management/nyc_permitted_event_information.rb index 90089c5..4624f0d 100644 --- a/app/models/office_of_citywide_event_coordination_and_management/nyc_permitted_event_information.rb +++ b/app/models/office_of_citywide_event_coordination_and_management/nyc_permitted_event_information.rb @@ -233,43 +233,5 @@ def self.import_soda3_csv end end private_class_method :import_soda3_csv - - def self.run_import(api_version: '2', content_type: 'json') - if api_version == '2' && content_type == 'json' - run_import_soda2 - end - - if api_version == '2' && content_type == 'csv' - run_import_soda2_csv - end - - if api_version == '3' && content_type == 'json' - run_import_soda3 - end - - if api_version == '3' && content_type == 'csv' - run_import_soda3_csv - end - end - - def self.run_import_soda2 - Etl::Runners::PrimaryDbImports::NycPermittedEventInformation::Soda2IntoPrimaryDb.run - end - private_class_method :run_import_soda2 - - def self.run_import_soda2_csv - Etl::Runners::PrimaryDbImports::NycPermittedEventInformation::Soda2CsvIntoPrimaryDb.run - end - private_class_method :run_import_soda2_csv - - def self.run_import_soda3 - Etl::Runners::PrimaryDbImports::NycPermittedEventInformation::Soda3IntoPrimaryDb.run - end - private_class_method :run_import_soda3 - - def self.run_import_soda3_csv - Etl::Runners::PrimaryDbImports::NycPermittedEventInformation::Soda3CsvIntoPrimaryDb.run - end - private_class_method :run_import_soda3_csv end end diff --git a/app/models/office_of_citywide_event_coordination_and_management/nyc_permitted_event_information_historical.rb b/app/models/office_of_citywide_event_coordination_and_management/nyc_permitted_event_information_historical.rb index 78b3996..a97c64e 100644 --- a/app/models/office_of_citywide_event_coordination_and_management/nyc_permitted_event_information_historical.rb +++ b/app/models/office_of_citywide_event_coordination_and_management/nyc_permitted_event_information_historical.rb @@ -233,43 +233,5 @@ def self.import_soda3_csv end end private_class_method :import_soda3_csv - - def self.run_import(api_version: '2', content_type: 'json') - if api_version == '2' && content_type == 'json' - run_import_soda2 - end - - if api_version == '2' && content_type == 'csv' - run_import_soda2_csv - end - - if api_version == '3' && content_type == 'json' - run_import_soda3 - end - - if api_version == '3' && content_type == 'csv' - run_import_soda3_csv - end - end - - def self.run_import_soda2 - Etl::Runners::PrimaryDbImports::NycPermittedEventInformationHistorical::Soda2IntoPrimaryDb.run - end - private_class_method :run_import_soda2 - - def self.run_import_soda2_csv - Etl::Runners::PrimaryDbImports::NycPermittedEventInformationHistorical::Soda2CsvIntoPrimaryDb.run - end - private_class_method :run_import_soda2_csv - - def self.run_import_soda3 - Etl::Runners::PrimaryDbImports::NycPermittedEventInformationHistorical::Soda3IntoPrimaryDb.run - end - private_class_method :run_import_soda3 - - def self.run_import_soda3_csv - Etl::Runners::PrimaryDbImports::NycPermittedEventInformationHistorical::Soda3CsvIntoPrimaryDb.run - end - private_class_method :run_import_soda3_csv end end \ No newline at end of file From a8f3af20db1478a3416b40903c69e3d9a04807a8 Mon Sep 17 00:00:00 2001 From: dgonzdev Date: Sat, 14 Feb 2026 08:25:41 -0500 Subject: [PATCH 2/4] update local setup --- docs/LOCAL_SETUP.md | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/docs/LOCAL_SETUP.md b/docs/LOCAL_SETUP.md index 9d960fb..6b300eb 100644 --- a/docs/LOCAL_SETUP.md +++ b/docs/LOCAL_SETUP.md @@ -103,16 +103,6 @@ DepartmentOfTransportation::BicycleCounter.import See [nyc open data](YOUR_NYC_OPEN_DATA.md) for more information. -## Import Data into the Primary DB with Kiba - -Import data into the primary database with Kiba. - -``` -DepartmentOfTransportation::BicycleCounter.run_import -``` - -See [nyc open data](YOUR_NYC_OPEN_DATA.md) for more information. - ## Run ETL Workflows Perform transformations on open data and store the results in the analytics database. From c9eb56abbe2ef55e277e40bcd1ee3b5ea1a608e0 Mon Sep 17 00:00:00 2001 From: dgonzdev Date: Sat, 14 Feb 2026 08:29:16 -0500 Subject: [PATCH 3/4] update your nyc open data --- docs/YOUR_NYC_OPEN_DATA.md | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/docs/YOUR_NYC_OPEN_DATA.md b/docs/YOUR_NYC_OPEN_DATA.md index 185b0e6..a3eef5f 100644 --- a/docs/YOUR_NYC_OPEN_DATA.md +++ b/docs/YOUR_NYC_OPEN_DATA.md @@ -35,35 +35,6 @@ DepartmentOfTransportation::BicycleCounter.import(api_version: '3', content_type DepartmentOfTransportation::BicycleCounter.import(api_version: '3', content_type: 'csv') ``` -### run_import - -The run_import method provides the same functionality as the import method. - -It was implemented to demonstrate how to use the Kiba library to define/run ETL workflows. - -Note: We are writing to the primary database here but in most cases Kiba will be used to write to the analytics database. - -#### SODA2 - -``` -DepartmentOfTransportation::BicycleCounter.run_import -DepartmentOfTransportation::BicycleCounter.run_import(api_version: '2', content_type: 'json') -``` - -``` -DepartmentOfTransportation::BicycleCounter.run_import(api_version: '2', content_type: 'csv') -``` - -#### SODA3 - -``` -DepartmentOfTransportation::BicycleCounter.run_import(api_version: '3', content_type: 'json') -``` - -``` -DepartmentOfTransportation::BicycleCounter.run_import(api_version: '3', content_type: 'csv') -``` - ## Dataset Properties The models contain the following class methods to describe a dataset's properties. From 7a8d7c2e5da4e270ed610180b116def5659ffa25 Mon Sep 17 00:00:00 2001 From: dgonzdev Date: Sat, 14 Feb 2026 08:37:23 -0500 Subject: [PATCH 4/4] update etl --- docs/ETL.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/ETL.md b/docs/ETL.md index a8485b1..42fd49c 100644 --- a/docs/ETL.md +++ b/docs/ETL.md @@ -85,4 +85,10 @@ They live in ``` lib/etl/runners/ +``` + +The runners that that pull data from the NYC Open Data Portal and store an exact copy in the primary database live in + +``` +lib/etl/runners/primary_db_imports/ ``` \ No newline at end of file