diff --git a/app/models/department_of_transportation/bicycle_counter.rb b/app/models/department_of_transportation/bicycle_counter.rb index 2e3b5e8..1debf08 100755 --- a/app/models/department_of_transportation/bicycle_counter.rb +++ b/app/models/department_of_transportation/bicycle_counter.rb @@ -193,43 +193,5 @@ def self.import_soda3_csv end end private_class_method :import_soda3_csv - - def self.run_import(api_version: '2', content_type: 'json') - if api_version == '2' && content_type == 'json' - run_import_soda2 - end - - if api_version == '2' && content_type == 'csv' - run_import_soda2_csv - end - - if api_version == '3' && content_type == 'json' - run_import_soda3 - end - - if api_version == '3' && content_type == 'csv' - run_import_soda3_csv - end - end - - def self.run_import_soda2 - Etl::Runners::PrimaryDbImports::BicycleCounters::Soda2IntoPrimaryDb.run - end - private_class_method :run_import_soda2 - - def self.run_import_soda2_csv - Etl::Runners::PrimaryDbImports::BicycleCounters::Soda2CsvIntoPrimaryDb.run - end - private_class_method :run_import_soda2_csv - - def self.run_import_soda3 - Etl::Runners::PrimaryDbImports::BicycleCounters::Soda3IntoPrimaryDb.run - end - private_class_method :run_import_soda3 - - def self.run_import_soda3_csv - Etl::Runners::PrimaryDbImports::BicycleCounters::Soda3CsvIntoPrimaryDb.run - end - private_class_method :run_import_soda3_csv end end diff --git a/app/models/office_of_citywide_event_coordination_and_management/nyc_permitted_event_information.rb b/app/models/office_of_citywide_event_coordination_and_management/nyc_permitted_event_information.rb index 90089c5..4624f0d 100644 --- a/app/models/office_of_citywide_event_coordination_and_management/nyc_permitted_event_information.rb +++ b/app/models/office_of_citywide_event_coordination_and_management/nyc_permitted_event_information.rb @@ -233,43 +233,5 @@ def self.import_soda3_csv end end private_class_method :import_soda3_csv - - def self.run_import(api_version: '2', content_type: 'json') - if api_version == '2' && content_type == 'json' - run_import_soda2 - end - - if api_version == '2' && content_type == 'csv' - run_import_soda2_csv - end - - if api_version == '3' && content_type == 'json' - run_import_soda3 - end - - if api_version == '3' && content_type == 'csv' - run_import_soda3_csv - end - end - - def self.run_import_soda2 - Etl::Runners::PrimaryDbImports::NycPermittedEventInformation::Soda2IntoPrimaryDb.run - end - private_class_method :run_import_soda2 - - def self.run_import_soda2_csv - Etl::Runners::PrimaryDbImports::NycPermittedEventInformation::Soda2CsvIntoPrimaryDb.run - end - private_class_method :run_import_soda2_csv - - def self.run_import_soda3 - Etl::Runners::PrimaryDbImports::NycPermittedEventInformation::Soda3IntoPrimaryDb.run - end - private_class_method :run_import_soda3 - - def self.run_import_soda3_csv - Etl::Runners::PrimaryDbImports::NycPermittedEventInformation::Soda3CsvIntoPrimaryDb.run - end - private_class_method :run_import_soda3_csv end end diff --git a/app/models/office_of_citywide_event_coordination_and_management/nyc_permitted_event_information_historical.rb b/app/models/office_of_citywide_event_coordination_and_management/nyc_permitted_event_information_historical.rb index 78b3996..a97c64e 100644 --- a/app/models/office_of_citywide_event_coordination_and_management/nyc_permitted_event_information_historical.rb +++ b/app/models/office_of_citywide_event_coordination_and_management/nyc_permitted_event_information_historical.rb @@ -233,43 +233,5 @@ def self.import_soda3_csv end end private_class_method :import_soda3_csv - - def self.run_import(api_version: '2', content_type: 'json') - if api_version == '2' && content_type == 'json' - run_import_soda2 - end - - if api_version == '2' && content_type == 'csv' - run_import_soda2_csv - end - - if api_version == '3' && content_type == 'json' - run_import_soda3 - end - - if api_version == '3' && content_type == 'csv' - run_import_soda3_csv - end - end - - def self.run_import_soda2 - Etl::Runners::PrimaryDbImports::NycPermittedEventInformationHistorical::Soda2IntoPrimaryDb.run - end - private_class_method :run_import_soda2 - - def self.run_import_soda2_csv - Etl::Runners::PrimaryDbImports::NycPermittedEventInformationHistorical::Soda2CsvIntoPrimaryDb.run - end - private_class_method :run_import_soda2_csv - - def self.run_import_soda3 - Etl::Runners::PrimaryDbImports::NycPermittedEventInformationHistorical::Soda3IntoPrimaryDb.run - end - private_class_method :run_import_soda3 - - def self.run_import_soda3_csv - Etl::Runners::PrimaryDbImports::NycPermittedEventInformationHistorical::Soda3CsvIntoPrimaryDb.run - end - private_class_method :run_import_soda3_csv end end \ No newline at end of file diff --git a/docs/ETL.md b/docs/ETL.md index a8485b1..42fd49c 100644 --- a/docs/ETL.md +++ b/docs/ETL.md @@ -85,4 +85,10 @@ They live in ``` lib/etl/runners/ +``` + +The runners that that pull data from the NYC Open Data Portal and store an exact copy in the primary database live in + +``` +lib/etl/runners/primary_db_imports/ ``` \ No newline at end of file diff --git a/docs/LOCAL_SETUP.md b/docs/LOCAL_SETUP.md index 9d960fb..6b300eb 100644 --- a/docs/LOCAL_SETUP.md +++ b/docs/LOCAL_SETUP.md @@ -103,16 +103,6 @@ DepartmentOfTransportation::BicycleCounter.import See [nyc open data](YOUR_NYC_OPEN_DATA.md) for more information. -## Import Data into the Primary DB with Kiba - -Import data into the primary database with Kiba. - -``` -DepartmentOfTransportation::BicycleCounter.run_import -``` - -See [nyc open data](YOUR_NYC_OPEN_DATA.md) for more information. - ## Run ETL Workflows Perform transformations on open data and store the results in the analytics database. diff --git a/docs/YOUR_NYC_OPEN_DATA.md b/docs/YOUR_NYC_OPEN_DATA.md index 185b0e6..a3eef5f 100644 --- a/docs/YOUR_NYC_OPEN_DATA.md +++ b/docs/YOUR_NYC_OPEN_DATA.md @@ -35,35 +35,6 @@ DepartmentOfTransportation::BicycleCounter.import(api_version: '3', content_type DepartmentOfTransportation::BicycleCounter.import(api_version: '3', content_type: 'csv') ``` -### run_import - -The run_import method provides the same functionality as the import method. - -It was implemented to demonstrate how to use the Kiba library to define/run ETL workflows. - -Note: We are writing to the primary database here but in most cases Kiba will be used to write to the analytics database. - -#### SODA2 - -``` -DepartmentOfTransportation::BicycleCounter.run_import -DepartmentOfTransportation::BicycleCounter.run_import(api_version: '2', content_type: 'json') -``` - -``` -DepartmentOfTransportation::BicycleCounter.run_import(api_version: '2', content_type: 'csv') -``` - -#### SODA3 - -``` -DepartmentOfTransportation::BicycleCounter.run_import(api_version: '3', content_type: 'json') -``` - -``` -DepartmentOfTransportation::BicycleCounter.run_import(api_version: '3', content_type: 'csv') -``` - ## Dataset Properties The models contain the following class methods to describe a dataset's properties.