From 0e76064406e84e69474ef7052c2c0518564705d3 Mon Sep 17 00:00:00 2001 From: Dave Paul Date: Wed, 29 Aug 2018 11:44:04 -0400 Subject: [PATCH 01/24] readme --- README.md | 25 +------------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/README.md b/README.md index 7db80e4..52c1596 100644 --- a/README.md +++ b/README.md @@ -1,24 +1 @@ -# README - -This README would normally document whatever steps are necessary to get the -application up and running. - -Things you may want to cover: - -* Ruby version - -* System dependencies - -* Configuration - -* Database creation - -* Database initialization - -* How to run the test suite - -* Services (job queues, cache servers, search engines, etc.) - -* Deployment instructions - -* ... +$ foreman start From 1a757630652090477883bad17d636fcd47effc14 Mon Sep 17 00:00:00 2001 From: Dave Paul Date: Thu, 6 Sep 2018 10:58:34 -0400 Subject: [PATCH 02/24] comments --- features/platforms/refresh.feature | 1 + 1 file changed, 1 insertion(+) diff --git a/features/platforms/refresh.feature b/features/platforms/refresh.feature index 6d0ff6f..7669c4d 100644 --- a/features/platforms/refresh.feature +++ b/features/platforms/refresh.feature @@ -16,6 +16,7 @@ Feature: Refresh Platform Scenario: Refresh a platform that already has some episodes Given this html for multiple entries """ +
From f1345687a195889ee19b802f5e16b0c70cb02390 Mon Sep 17 00:00:00 2001 From: Dave Paul Date: Thu, 6 Sep 2018 11:32:08 -0400 Subject: [PATCH 03/24] steps for remembering the page num --- features/platforms/refresh.feature | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/features/platforms/refresh.feature b/features/platforms/refresh.feature index 7669c4d..4f05133 100644 --- a/features/platforms/refresh.feature +++ b/features/platforms/refresh.feature @@ -375,3 +375,9 @@ Feature: Refresh Platform |Ted Draws - 1997 Special 13.02.18 Radio Episode|13.02.18| |Obongjayar 13.02.18 Radio Episode|13.02.18| |Nabihah Iqbal 13.02.18 Radio Episode|13.02.18| + + @http @nts @pagination + Scenario: Remember the last page captured in refresh + Given the platform's lastPage property is '2' + When I refresh the Platform + Then the platform's lastPage property is incremented \ No newline at end of file From a72d25b4e1b59d87b6481f2c9d69a966174d8205 Mon Sep 17 00:00:00 2001 From: Dave Paul Date: Thu, 6 Sep 2018 11:36:13 -0400 Subject: [PATCH 04/24] no method yet --- features/platforms/refresh.feature | 6 +++--- features/platforms/step_definitions/refresh_steps.rb | 6 ++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/features/platforms/refresh.feature b/features/platforms/refresh.feature index 4f05133..d7d1ba0 100644 --- a/features/platforms/refresh.feature +++ b/features/platforms/refresh.feature @@ -378,6 +378,6 @@ Feature: Refresh Platform @http @nts @pagination Scenario: Remember the last page captured in refresh - Given the platform's lastPage property is '2' - When I refresh the Platform - Then the platform's lastPage property is incremented \ No newline at end of file + Given the platform has a lastPage + When I refresh the Platform + Then the platform's lastPage is incremented \ No newline at end of file diff --git a/features/platforms/step_definitions/refresh_steps.rb b/features/platforms/step_definitions/refresh_steps.rb index 3326df4..22f4143 100644 --- a/features/platforms/step_definitions/refresh_steps.rb +++ b/features/platforms/step_definitions/refresh_steps.rb @@ -27,4 +27,10 @@ expect(slice).to match_array(uniques.hashes) end +Given("the platform has a lastPage") do + expect(@platform.lastPage).to_be(0) +end + +Then("the platform's lastPage is incremented") do +end From 91454be2fb6e00ddf2b661432b365643dccc5217 Mon Sep 17 00:00:00 2001 From: Dave Paul Date: Thu, 6 Sep 2018 11:41:58 -0400 Subject: [PATCH 05/24] counter exists but is not incremented --- db/migrate/20180906153751_add_last_page_to_platform.rb | 5 +++++ db/schema.rb | 3 ++- features/platforms/refresh.feature | 4 ++-- features/platforms/step_definitions/refresh_steps.rb | 8 +++++--- 4 files changed, 14 insertions(+), 6 deletions(-) create mode 100644 db/migrate/20180906153751_add_last_page_to_platform.rb diff --git a/db/migrate/20180906153751_add_last_page_to_platform.rb b/db/migrate/20180906153751_add_last_page_to_platform.rb new file mode 100644 index 0000000..5e6c03e --- /dev/null +++ b/db/migrate/20180906153751_add_last_page_to_platform.rb @@ -0,0 +1,5 @@ +class AddLastPageToPlatform < ActiveRecord::Migration[5.1] + def change + add_column :platforms, :last_page, :integer + end +end diff --git a/db/schema.rb b/db/schema.rb index eeff194..90a5876 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20180330142948) do +ActiveRecord::Schema.define(version: 20180906153751) do create_table "episodes", force: :cascade do |t| t.string "name" @@ -36,6 +36,7 @@ t.boolean "use_relative_images" t.string "default_image" t.text "post_processing_rules" + t.integer "last_page" end end diff --git a/features/platforms/refresh.feature b/features/platforms/refresh.feature index d7d1ba0..dbc28fd 100644 --- a/features/platforms/refresh.feature +++ b/features/platforms/refresh.feature @@ -378,6 +378,6 @@ Feature: Refresh Platform @http @nts @pagination Scenario: Remember the last page captured in refresh - Given the platform has a lastPage + Given the platform has a last_page When I refresh the Platform - Then the platform's lastPage is incremented \ No newline at end of file + Then the platform's last_page is incremented \ No newline at end of file diff --git a/features/platforms/step_definitions/refresh_steps.rb b/features/platforms/step_definitions/refresh_steps.rb index 22f4143..20dc064 100644 --- a/features/platforms/step_definitions/refresh_steps.rb +++ b/features/platforms/step_definitions/refresh_steps.rb @@ -27,10 +27,12 @@ expect(slice).to match_array(uniques.hashes) end -Given("the platform has a lastPage") do - expect(@platform.lastPage).to_be(0) +Given("the platform has a last_page") do + @platform.last_page = 1 end -Then("the platform's lastPage is incremented") do + +Then("the platform's last_page is incremented") do + expect(@platform.last_page).to eq(2) end From 6d4d21341c4307befc487042e3de90f3e228f55f Mon Sep 17 00:00:00 2001 From: Dave Paul Date: Thu, 6 Sep 2018 11:54:26 -0400 Subject: [PATCH 06/24] passing tests --- app/models/platform.rb | 10 ++++++++++ features/platforms/refresh.feature | 4 ++-- features/platforms/step_definitions/refresh_steps.rb | 8 ++++---- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/app/models/platform.rb b/app/models/platform.rb index 0002a07..3c0c982 100644 --- a/app/models/platform.rb +++ b/app/models/platform.rb @@ -180,9 +180,19 @@ def refresh(opts={}) def ready(result) puts "platform#ready" if result[:success] + increment_last_page! return create_episodes_from_html result[:body] else raise "#{result[:code]} error\n #{result[:message]} " end end + + def increment_last_page! + if last_page.nil? + self.last_page = 1 + else + self.last_page += 1 + end + + end end diff --git a/features/platforms/refresh.feature b/features/platforms/refresh.feature index dbc28fd..f96669b 100644 --- a/features/platforms/refresh.feature +++ b/features/platforms/refresh.feature @@ -378,6 +378,6 @@ Feature: Refresh Platform @http @nts @pagination Scenario: Remember the last page captured in refresh - Given the platform has a last_page + Given the platform's last_page is 2 When I refresh the Platform - Then the platform's last_page is incremented \ No newline at end of file + Then the platform's last_page is incremented to 3 \ No newline at end of file diff --git a/features/platforms/step_definitions/refresh_steps.rb b/features/platforms/step_definitions/refresh_steps.rb index 20dc064..cdf6583 100644 --- a/features/platforms/step_definitions/refresh_steps.rb +++ b/features/platforms/step_definitions/refresh_steps.rb @@ -27,12 +27,12 @@ expect(slice).to match_array(uniques.hashes) end -Given("the platform has a last_page") do - @platform.last_page = 1 +Given("the platform's last_page is {int}") do |p| + @platform.last_page = p end -Then("the platform's last_page is incremented") do - expect(@platform.last_page).to eq(2) +Then("the platform's last_page is incremented to {int}") do |p| + expect(@platform.last_page).to eq(p) end From ecff2caa47f5cfe06f8ba125723ea721f7636ce8 Mon Sep 17 00:00:00 2001 From: Dave Paul Date: Thu, 6 Sep 2018 11:57:16 -0400 Subject: [PATCH 07/24] dont allow null `last_page` at startup --- app/models/platform.rb | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/app/models/platform.rb b/app/models/platform.rb index 3c0c982..02980f0 100644 --- a/app/models/platform.rb +++ b/app/models/platform.rb @@ -24,6 +24,7 @@ def initialize(opts={}) end after_find do + initialize_last_page init_client end @@ -187,12 +188,13 @@ def ready(result) end end - def increment_last_page! - if last_page.nil? - self.last_page = 1 - else - self.last_page += 1 + def initialize_last_page + if self.last_page.nil? + self.last_page = 0 end - + end + + def increment_last_page! + self.last_page += 1 end end From 48f86019668252262f39b7830887105af7238dfd Mon Sep 17 00:00:00 2001 From: Dave Paul Date: Thu, 6 Sep 2018 12:57:40 -0400 Subject: [PATCH 08/24] edits --- app/controllers/platforms_controller.rb | 1 + app/javascripts/components/PlatformDetails.js | 4 +++- app/models/platform.rb | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/app/controllers/platforms_controller.rb b/app/controllers/platforms_controller.rb index f572a65..4c83248 100644 --- a/app/controllers/platforms_controller.rb +++ b/app/controllers/platforms_controller.rb @@ -47,6 +47,7 @@ def delete_episodes begin platform_from_nickname_param @platform.episodes.delete_all + @platform.update(last_page: 0) render json: { :success => :true, :platform => platform_with_episodes diff --git a/app/javascripts/components/PlatformDetails.js b/app/javascripts/components/PlatformDetails.js index aebadba..7e1a83a 100644 --- a/app/javascripts/components/PlatformDetails.js +++ b/app/javascripts/components/PlatformDetails.js @@ -19,6 +19,7 @@ class PlatformDetails extends Component { pagination, refreshPlatform, episodes, + last_page, currentPage } = this.props @@ -53,7 +54,8 @@ class PlatformDetails extends Component { Edit Platform

- Episodes{' '}{this.refreshButton()}{' '}{this.deleteEpisodesButton()} + Episodes{' '}{this.refreshButton()}{' '}{this.deleteEpisodesButton()}
+ Last page fetched: {last_page}

{this.episodeGrid()} diff --git a/app/models/platform.rb b/app/models/platform.rb index 02980f0..1a05653 100644 --- a/app/models/platform.rb +++ b/app/models/platform.rb @@ -196,5 +196,6 @@ def initialize_last_page def increment_last_page! self.last_page += 1 + save end end From 18193caba17d5b29c3ab83e5241048c501641f78 Mon Sep 17 00:00:00 2001 From: Dave Paul Date: Thu, 6 Sep 2018 17:01:10 -0400 Subject: [PATCH 09/24] readme --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 52c1596..63fed49 100644 --- a/README.md +++ b/README.md @@ -1 +1,7 @@ +asbo: +an aggregator of a few London-based internet radio stations - using Nokogiri to scrape the html where needed, caching the resulting objects in a local db + +backend is in rails, front-end is in react/redux + +to start the backend api and the client, use the following command: $ foreman start From ca44a0d35cf1e09dad9a8bbbd2b98b29d4f9013b Mon Sep 17 00:00:00 2001 From: Dave Paul Date: Tue, 11 Sep 2018 17:00:04 -0400 Subject: [PATCH 10/24] comments and pagiantion tweaks --- app/javascripts/redux/pagination.js | 18 +++++++++++++++--- app/javascripts/redux/platforms.js | 5 ++++- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/app/javascripts/redux/pagination.js b/app/javascripts/redux/pagination.js index 18ec46c..36a2ed6 100644 --- a/app/javascripts/redux/pagination.js +++ b/app/javascripts/redux/pagination.js @@ -1,6 +1,6 @@ import {combineReducers} from 'redux' import {createAction,createActions,handleActions} from 'redux-actions' -import {refreshPlatform} from './platforms' +import {refreshPlatform, LOAD_PLATFORM} from './platforms' // constants export const SET_PAGE = 'SET_PAGE'; // export const SET_PAGE_AND_REFRESH_PLATFORM = 'SET_PAGE_AND_REFRESH_PLATFORM' @@ -25,8 +25,20 @@ export const setPage = createAction(SET_PAGE) // reducers -export const currentPage = (currentPage = 1, action = {}) => { - return action.type == SET_PAGE ? action.payload.page : currentPage +export const currentPage = (currentPage=0, action = {}) => { + switch (action.type){ + + case `${LOAD_PLATFORM}_FULFILLED`: + const {platform} = action.payload + const last_page = platform.last_page + return last_page !== undefined ? last_page : currentPage // sync counter with db + + case SET_PAGE: + return action.payload.page + + default: + return currentPage + } } export const pages = (pages={}, action={}) => { diff --git a/app/javascripts/redux/platforms.js b/app/javascripts/redux/platforms.js index a969cb2..1aa1e0d 100644 --- a/app/javascripts/redux/platforms.js +++ b/app/javascripts/redux/platforms.js @@ -103,12 +103,15 @@ export const refreshPlatform = function({nickname}) { type: REFRESH_PLATFORM, payload: API.refreshPlatform({nickname,filter,page}) .then (json => { + // handle with middleware or chaining? + console.log(`|platforms| on refresh, set last_page to ${json.last_page +1}`) return { platform: json } }) - }) + } } + } export const destroyPlatform = function({nickname}){ From a2bbb2bb72452518a39dd69e05e91e34644d1c0d Mon Sep 17 00:00:00 2001 From: Dave Paul Date: Fri, 16 Nov 2018 15:53:35 -0500 Subject: [PATCH 11/24] misc test issues fixed --- app/javascripts/redux/platforms.js | 2 +- app/javascripts/tests/episodes.test.js | 5 +++-- app/javascripts/tests/pagination.test.js | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/app/javascripts/redux/platforms.js b/app/javascripts/redux/platforms.js index 1aa1e0d..909adf1 100644 --- a/app/javascripts/redux/platforms.js +++ b/app/javascripts/redux/platforms.js @@ -109,7 +109,7 @@ export const refreshPlatform = function({nickname}) { platform: json } }) - } + }) } } diff --git a/app/javascripts/tests/episodes.test.js b/app/javascripts/tests/episodes.test.js index 28d523b..7050e7d 100644 --- a/app/javascripts/tests/episodes.test.js +++ b/app/javascripts/tests/episodes.test.js @@ -82,17 +82,18 @@ describe('Episodes', () => { describe('episodes#show', () => { it('dispatches an action to get the episode entry', async () => { const store = mockStore({}) - const opts = {'id':'1960'} + const opts = {'id':1960} await store.dispatch(e.loadEpisode(opts)) .then(() => { expectActions(store, [ "LOAD_EPISODE_PENDING", + "FETCH_EMBED_PENDING", "LOAD_EPISODE_FULFILLED" ]); const state = resultingState(store, reducer) expect(state.episode).toBeTruthy() - // expect(state.episode.id).toEqual(opts.id) + expect(state.episode.id).toEqual(opts.id) }) }) }) diff --git a/app/javascripts/tests/pagination.test.js b/app/javascripts/tests/pagination.test.js index c7299ca..2f9925f 100644 --- a/app/javascripts/tests/pagination.test.js +++ b/app/javascripts/tests/pagination.test.js @@ -11,8 +11,8 @@ import { describe('Pagination', () => { describe('currentPage', () => { - it('is set to 1 at init', () => { - expect(reducer().currentPage).toBe(1) + it('is set to 0 at init', () => { + expect(reducer().currentPage).toBe(0) }) it('can be changed by setPage action', () => { const store = mockStore() From dd3f9c4d00e3935d82907ee6c7e35d7bd99d14b2 Mon Sep 17 00:00:00 2001 From: Dave Paul Date: Fri, 16 Nov 2018 16:48:08 -0500 Subject: [PATCH 12/24] increment last_page aftet fetch in mock api --- app/javascripts/__mocks__/api.js | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/app/javascripts/__mocks__/api.js b/app/javascripts/__mocks__/api.js index 289c4a0..6f997ac 100644 --- a/app/javascripts/__mocks__/api.js +++ b/app/javascripts/__mocks__/api.js @@ -14,8 +14,9 @@ const MockAPI = class { } getPlatform({nickname}){ - // console.log(`mockAPI.fetch /platforms/${nickname}`) - const platform = platforms.find(p => p.nickname == nickname) + console.log(`mockAPI.fetch /platforms/${nickname}`) + let platform = platforms.find(p => p.nickname == nickname) + return new Promise((resolve,reject) => { if(!!platform) { setTimeout(resolve, 0, { @@ -38,7 +39,7 @@ const MockAPI = class { // in the real api, filter is passed to backend so repsonse contains a sub-set of total items.. // in the mock api, the filter is just post-processing in the callback refreshPlatform({nickname,filter,page}){ - console.log(`mockAPI.fetch /platforms/${nickname}/refresh ${JSON.stringify({filter,page})}`); + // console.log(`mockAPI.fetch /platforms/${nickname}/refresh ${JSON.stringify({filter,page})}`); const platform = platforms.find(p => p.nickname == nickname) return new Promise((resolve,reject) => { if(!platform) { @@ -51,7 +52,9 @@ const MockAPI = class { //console.log(totalEps.map(ep => ep.name)) const matches = totalEps.filter(e => this.isMatchingEpisode(e, filter)) setTimeout(resolve, 0, { - ...platform, episodes: matches + ...platform, + last_page: (platform.last_page + 1), + episodes: matches }) } }) From 2b74c1747eda41aecf5dcd9e8a2a53d1c3245a2a Mon Sep 17 00:00:00 2001 From: Dave Paul Date: Fri, 16 Nov 2018 16:48:51 -0500 Subject: [PATCH 13/24] added last_page to mock platforms --- app/javascripts/__mocks__/platforms.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/app/javascripts/__mocks__/platforms.js b/app/javascripts/__mocks__/platforms.js index e876f6e..09ad2af 100644 --- a/app/javascripts/__mocks__/platforms.js +++ b/app/javascripts/__mocks__/platforms.js @@ -27,7 +27,8 @@ export default [{ "100x100", "800x800" ] - }] + }], + "last_page" : 0 }, { "id": 2, @@ -51,7 +52,8 @@ export default [{ "nickname": "rinse", "use_relative_images": false, "default_image": "https://etc.levels.io/rinse-fm/logo.png", - "post_processing_rules": null + "post_processing_rules": null, + "last_page" : 3 }, { "id": 3, "name": "radar radio", @@ -75,5 +77,6 @@ export default [{ "nickname": "radar", "use_relative_images": true, "default_image": null, - "post_processing_rules": null + "post_processing_rules": null, + "last_page" : 2 }] \ No newline at end of file From deb61b0dce79ef75571bab97db7f2eff8833ebec Mon Sep 17 00:00:00 2001 From: Dave Paul Date: Fri, 16 Nov 2018 16:49:30 -0500 Subject: [PATCH 14/24] last_page tests --- app/javascripts/tests/platforms.test.js | 52 +++++++++++++++++++------ 1 file changed, 40 insertions(+), 12 deletions(-) diff --git a/app/javascripts/tests/platforms.test.js b/app/javascripts/tests/platforms.test.js index 3fb9846..73be9a9 100644 --- a/app/javascripts/tests/platforms.test.js +++ b/app/javascripts/tests/platforms.test.js @@ -68,19 +68,20 @@ describe('Platforms', () => { }) }) + const refreshPlatform = ({nickname}) => { + const store = mockStore({}) + return store.dispatch(p.refreshPlatform({nickname})) + .then(() => { + expectActions(store, [ + `${p.REFRESH_PLATFORM}_PENDING`, + `${p.REFRESH_PLATFORM}_FULFILLED`, + ]) + const state = resultingState(store, combinedRootReducer) + return {state, store: mockStore(state)} + }) + } + describe('platforms#refresh', () => { - const refreshPlatform = ({nickname}) => { - const store = mockStore({}) - return store.dispatch(p.refreshPlatform({nickname})) - .then(() => { - expectActions(store, [ - `${p.REFRESH_PLATFORM}_PENDING`, - `${p.REFRESH_PLATFORM}_FULFILLED`, - ]) - const state = resultingState(store, combinedRootReducer) - return {state, store: mockStore(state)} - }) - } const nickname = 'rinse' const opts = {nickname} @@ -159,6 +160,33 @@ describe('Platforms', () => { }) }) + + describe('lastPage', () => { + it('is incremented when after api returns a page of episodes', (done) => { + const opts = {'nickname':'rinse'} + + let lastPageBefore, + lastPageAfter, + refreshedState + + // 1. get a platform + getPlatform(opts).then((state1) => { + lastPageBefore = state1.platforms.platform.last_page + }) + .then(() => { + // 2. get a page of episodes + return refreshPlatform(opts).then((xhr) => { + refreshedState = xhr.state + }) + }).then(() => { + // 3. compare last_page before and after + lastPageAfter = refreshedState.platforms.platform.last_page + expect(lastPageAfter).toBeGreaterThan(lastPageBefore) + return done() + }) + }) + }) + describe('platforms#delete_episodes', () => { // see episodes tests }) From 5842a16f43ddc3a663855d87ce5924145a90a1ba Mon Sep 17 00:00:00 2001 From: Dave Paul Date: Tue, 8 Oct 2019 14:38:57 -0400 Subject: [PATCH 15/24] bumped implicit year to 2019 --- features/episodes/dates.feature | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/features/episodes/dates.feature b/features/episodes/dates.feature index 02c899c..bde33e0 100644 --- a/features/episodes/dates.feature +++ b/features/episodes/dates.feature @@ -14,8 +14,8 @@ Feature: Handle Dates @date @ordinal @radar Scenario: Parse dates for Radar Given this date '23rd Feb' - And this implicit year '2018' + And this implicit year '2019' And this date format '%e %b' When I set the date_str for the episode - Then episode.date.to_s will be '2018-02-23' + Then episode.date.to_s will be '2019-02-23' \ No newline at end of file From f3772c71dcf08fee18e804c40da97d41ac48a2b9 Mon Sep 17 00:00:00 2001 From: Dave Paul Date: Tue, 8 Oct 2019 14:46:07 -0400 Subject: [PATCH 16/24] updated souncloud embed test to expect https in url --- features/episodes/media_embed.feature | 2 +- features/episodes/step_definitions/media_steps.rb | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/features/episodes/media_embed.feature b/features/episodes/media_embed.feature index f20ee29..c753e62 100644 --- a/features/episodes/media_embed.feature +++ b/features/episodes/media_embed.feature @@ -13,7 +13,7 @@ Scenario: fetch a soundcloud embed When I post the attributes to the media endpoint Then I will get this embed back """ - + """ @media @soundcloud @embed @error diff --git a/features/episodes/step_definitions/media_steps.rb b/features/episodes/step_definitions/media_steps.rb index fa10c05..bee2777 100644 --- a/features/episodes/step_definitions/media_steps.rb +++ b/features/episodes/step_definitions/media_steps.rb @@ -10,12 +10,14 @@ @media = { media: JSON.parse(attrs) } + puts @media end When("I post the attributes to the media endpoint") do post "/media", @media @response = last_response.body @json = JSON.parse(@response) + puts @json end Then("I will get this embed back") do |html| From 626917f51c476fad2bccb5c78573bc2668c7b2f6 Mon Sep 17 00:00:00 2001 From: Dave Paul Date: Fri, 18 Oct 2019 14:02:41 -0400 Subject: [PATCH 17/24] had to update attr-map in nts --- db/platforms.rb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/db/platforms.rb b/db/platforms.rb index 5f40e33..3f9b332 100644 --- a/db/platforms.rb +++ b/db/platforms.rb @@ -9,12 +9,12 @@ def self.sample_attributes has_details: true, use_relative_images: false, attr_map: { - "item" => ".nts-grid-item", - "name" => ".nts-grid-item__img img[alt]", - "image" => ".nts-grid-item__img img[src]", - "media" => ".nts-grid-item__img__play-btn[data-src]", - "details" => ".nts-grid-item__img__play-btn[data-permalink]", - "date_str" => ".nts-grid-item__subtitle__left" + "item" => ".nts-grid-v2-item", + "name" => ".nts-grid-v2-item__header__title", + "image" => ".nts-grid-picture__img[src]", + "media" => ".nts-grid-picture__overlay.mixcloud-btn[data-src]", + "date_str" => ".nts-grid-v2-item__header span:first-child", + "details" => ".nts-grid-picture__overlay.mixcloud-btn[data-permalink]" }, pagination: { "route" => "/page/:page", From ae64db7d21082e806a8bea3b4cac368f4c7b050f Mon Sep 17 00:00:00 2001 From: Dave Paul Date: Fri, 18 Oct 2019 14:04:53 -0400 Subject: [PATCH 18/24] added empty tag --- features/platforms/refresh.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/features/platforms/refresh.feature b/features/platforms/refresh.feature index f96669b..d6a169a 100644 --- a/features/platforms/refresh.feature +++ b/features/platforms/refresh.feature @@ -7,7 +7,7 @@ Feature: Refresh Platform Background: Given the platform nickname=nts - @http + @http @empty Scenario: Refresh a platform without any episodes When I refresh the Platform Then the episodes list should contain some episodes From 018258faa087cb80a5c5dbfe4aecacb18fa885db Mon Sep 17 00:00:00 2001 From: Dave Paul Date: Fri, 18 Oct 2019 14:26:32 -0400 Subject: [PATCH 19/24] fixed by updating stub html in feature --- features/episodes/image_paths.feature | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/features/episodes/image_paths.feature b/features/episodes/image_paths.feature index 6691e65..41cf925 100644 --- a/features/episodes/image_paths.feature +++ b/features/episodes/image_paths.feature @@ -9,11 +9,17 @@ I want to prefix relative paths with the platform's base path And the platform has relative_image_paths='false' And html for an episode """ -

Grandmixxer 22.02.18 Radio Episode
Grandmixxer 22.02.18 Radio Episode
Grandmixxer

22.02.18

LDN

Grime
TRACKLIST
+
+
+
Mija MGMT w/ San Cha +
+
+
""" When I parse the html - Then the platform will contain an episode with this image 'https://media.ntslive.co.uk/resize/800x800/8daa0f85-a541-47a1-88cf-26c18ebf342f_1444643834.jpg' + Then the platform will contain an episode with this image 'https://media2.ntslive.co.uk/resize/800x800/58dab62a-db5d-4bc9-9c2c-9708a6b846c8_1571097600.jpeg' @episode @images Scenario: image is absolute From e5acbf95a210d87baca6d3e6857c85b9578a0aac Mon Sep 17 00:00:00 2001 From: Dave Paul Date: Fri, 18 Oct 2019 14:29:46 -0400 Subject: [PATCH 20/24] fixed mixcloud test by refeshing html snippet --- features/episodes/media_type.feature | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/features/episodes/media_type.feature b/features/episodes/media_type.feature index 5871cc0..6d2a314 100644 --- a/features/episodes/media_type.feature +++ b/features/episodes/media_type.feature @@ -8,12 +8,13 @@ Scenario: nts x mixcloud Given the platform nickname=nts And html for an episode """ -
Grandmixxer 22.02.18 Radio Episode
Grandmixxer 22.02.18 Radio Episode
Grandmixxer

22.02.18

LDN

Grime
TRACKLIST
+ """ When I parse the html Then the platform will contain an episode with this media property |type|url| - |mixcloud|https://www.mixcloud.com/NTSRadio/grandmixxer-22nd-february-2018/| + |mixcloud|https://www.mixcloud.com/NTSRadio/hit-a-lick-14th-october-2019/| @media @soundcloud From 2a46c98dfcc3f3f2550cd6a6ddcef8ead8909547 Mon Sep 17 00:00:00 2001 From: Dave Paul Date: Fri, 18 Oct 2019 15:31:01 -0400 Subject: [PATCH 21/24] refreshed html --- features/episodes/post_processing.feature | 6 +++--- features/episodes/step_definitions/image_steps.rb | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/features/episodes/post_processing.feature b/features/episodes/post_processing.feature index af3f967..c2e388f 100644 --- a/features/episodes/post_processing.feature +++ b/features/episodes/post_processing.feature @@ -8,9 +8,9 @@ I want to perform post-processing on the episode after it is created Given the platform nickname=nts And html for an episode """ -
Whits & Giggles w/ Ideal Black Female 29.03.18 Radio Episode
Whits & Giggles w/ Ideal Black Female 29.03.18 Radio Episode
Whits & Giggles w/ Ideal Black Female

29.03.18

LA

 -
+ """ When I parse the html - Then the platform will contain an episode with this image 'https://media2.ntslive.co.uk/resize/800x800/f0cf5a5b-ea8b-4123-ae58-2b72662c6ac8_1522281600.jpeg' + Then the platform will contain an episode with this image 'https://media2.ntslive.co.uk/resize/800x800/2fa12bb5-b8a3-4984-9d79-d5f102ae3005_1571097600.png' diff --git a/features/episodes/step_definitions/image_steps.rb b/features/episodes/step_definitions/image_steps.rb index 4aff23b..2850894 100644 --- a/features/episodes/step_definitions/image_steps.rb +++ b/features/episodes/step_definitions/image_steps.rb @@ -10,6 +10,7 @@ Then("the platform will contain an episode with this image {string}") do |src| @ep = @platform.episodes.find do |ep| + puts ep.image ep.image == src end expect(@ep).to be_truthy From 40b045a2ff0175ea58dea16aa39c03870a763cae Mon Sep 17 00:00:00 2001 From: Dave Paul Date: Fri, 18 Oct 2019 16:09:39 -0400 Subject: [PATCH 22/24] updated fixtures and tests --- db/platforms.rb | 44 +++++++------------ features/platforms/attributes.feature | 10 ++--- .../step_definitions/attributes_steps.rb | 6 +++ 3 files changed, 27 insertions(+), 33 deletions(-) diff --git a/db/platforms.rb b/db/platforms.rb index 3f9b332..5fba331 100644 --- a/db/platforms.rb +++ b/db/platforms.rb @@ -10,7 +10,7 @@ def self.sample_attributes use_relative_images: false, attr_map: { "item" => ".nts-grid-v2-item", - "name" => ".nts-grid-v2-item__header__title", + "name" => ".nts-grid-picture__img[alt]", "image" => ".nts-grid-picture__img[src]", "media" => ".nts-grid-picture__overlay.mixcloud-btn[data-src]", "date_str" => ".nts-grid-v2-item__header span:first-child", @@ -69,33 +69,21 @@ def self.sample_attributes def self.sample_html_entries <<-END -
-
Murlo 31.01.18 Radio Episode -
-
-
-
-
-
Murlo 31.01.18 Radio Episode
- -
-
Murlo
-
-

31.01.18

-

LDN

-
-
- -
-
Dancehall Garage Grime Soca
-
- TRACKLIST
-
-
+ _BREAK_
diff --git a/features/platforms/attributes.feature b/features/platforms/attributes.feature index 33819a6..ab6f55b 100644 --- a/features/platforms/attributes.feature +++ b/features/platforms/attributes.feature @@ -3,17 +3,17 @@ Feature: Store scraping rules in platform I want to abstract the rules into an attribute map And share the implementation across different platforms - @attr + @attr @nts @platform Scenario: Parse NTS.live When I call create_episodes_from_html on platform 1 Then I should get an episode with these attributes """ { "platform_id" : 1, - "name":"Murlo 31.01.18 Radio Episode", - "media":{"type":"mixcloud", "url":"https://www.mixcloud.com/NTSRadio/murlo-31st-february-2018/"}, - "image":"https://media.ntslive.co.uk/resize/800x800/24463c4a-6d6d-48d7-9026-62c9af3b996f_1456963200.jpeg", - "date": "2018-01-31" + "name":"Murlo 18.07.18 Radio Episode", + "media":{"type":"mixcloud", "url":"https://www.mixcloud.com/NTSRadio/murlo-18th-july-2018/", "path":"/NTSRadio/murlo-18th-july-2018/"}, + "image":"https://media.ntslive.co.uk/resize/400x400/24463c4a-6d6d-48d7-9026-62c9af3b996f_1456963200.jpeg", + "date": "2018-07-18" } """ diff --git a/features/platforms/step_definitions/attributes_steps.rb b/features/platforms/step_definitions/attributes_steps.rb index 2795739..21f5878 100644 --- a/features/platforms/step_definitions/attributes_steps.rb +++ b/features/platforms/step_definitions/attributes_steps.rb @@ -17,8 +17,14 @@ @platform = @platform || @platforms.find {|p|p.id == attrs['platform_id']} @episode = @platform.episodes.first # iterate over attributes so we can specify date format for comparison + # puts @episode.attributes @ep_attrs = @episode.attributes.merge({ "date" => @episode.attributes['date'].to_s }) + # @episode.attributes.each do |pair| + # key = pair.first + # val = pair.last + # puts "#{key} => #{val} == #{@attrs[key]} ?" + # end expect(@ep_attrs).to include(@attrs) end From d66b47b4041811f590dd9d7b7851abec66c250ce Mon Sep 17 00:00:00 2001 From: Dave Paul Date: Fri, 18 Oct 2019 16:09:54 -0400 Subject: [PATCH 23/24] expose date format in form --- app/javascripts/components/PlatformForm.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/javascripts/components/PlatformForm.js b/app/javascripts/components/PlatformForm.js index 7a189cb..01fe50f 100644 --- a/app/javascripts/components/PlatformForm.js +++ b/app/javascripts/components/PlatformForm.js @@ -11,7 +11,7 @@ class PlatformForm extends Component { render() { const {initialValues, handleSubmit, pristine, reset, submitting, loading, isNew } = this.props const {nickname} = initialValues || {} - + console.log(initialValues) return (
@@ -29,6 +29,7 @@ class PlatformForm extends Component { parent='pagination' attrs='param,url,itemsPerPage' /> +